From noreply at buildbot.pypy.org Tue Jan 1 03:36:45 2013 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 1 Jan 2013 03:36:45 +0100 (CET) Subject: [pypy-commit] pypy default: update LICENSE Message-ID: <20130101023645.691031C03A5@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r59646:215ea1454865 Date: 2013-01-01 04:36 +0200 http://bitbucket.org/pypy/pypy/changeset/215ea1454865/ Log: update LICENSE diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -28,7 +28,7 @@ DEALINGS IN THE SOFTWARE. -PyPy Copyright holders 2003-2012 +PyPy Copyright holders 2003-2013 ----------------------------------- Except when otherwise stated (look for LICENSE files or information at From noreply at buildbot.pypy.org Tue Jan 1 03:41:40 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Tue, 1 Jan 2013 03:41:40 +0100 (CET) Subject: [pypy-commit] pypy default: Fix another place with 2012 Message-ID: <20130101024140.BB0E41C01E4@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: Changeset: r59647:29b8abf6816b Date: 2012-12-31 18:41 -0800 http://bitbucket.org/pypy/pypy/changeset/29b8abf6816b/ Log: Fix another place with 2012 diff --git a/pypy/module/sys/app.py b/pypy/module/sys/app.py --- a/pypy/module/sys/app.py +++ b/pypy/module/sys/app.py @@ -66,11 +66,11 @@ return None copyright_str = """ -Copyright 2003-2012 PyPy development team. +Copyright 2003-2013 PyPy development team. All Rights Reserved. For further information, see -Portions Copyright (c) 2001-2012 Python Software Foundation. +Portions Copyright (c) 2001-2013 Python Software Foundation. All Rights Reserved. Portions Copyright (c) 2000 BeOpen.com. From noreply at buildbot.pypy.org Tue Jan 1 11:01:53 2013 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 1 Jan 2013 11:01:53 +0100 (CET) Subject: [pypy-commit] cffi default: Skip test_zintegration if we can't find 'virtualenv'. Message-ID: <20130101100153.6D7241C00E2@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1106:3643242e05cd Date: 2013-01-01 10:34 +0100 http://bitbucket.org/cffi/cffi/changeset/3643242e05cd/ Log: Skip test_zintegration if we can't find 'virtualenv'. diff --git a/testing/test_zintegration.py b/testing/test_zintegration.py --- a/testing/test_zintegration.py +++ b/testing/test_zintegration.py @@ -5,7 +5,11 @@ def create_venv(name): tmpdir = udir.join(name) - subprocess.check_call(['virtualenv', '-p', sys.executable, str(tmpdir)]) + try: + subprocess.check_call(['virtualenv', '-p', sys.executable, + str(tmpdir)]) + except OSError, e: + py.test.skip("Cannot execute virtualenv: %s" % (e,)) site_packages = None for dirpath, dirnames, filenames in os.walk(str(tmpdir)): From noreply at buildbot.pypy.org Tue Jan 1 11:01:54 2013 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 1 Jan 2013 11:01:54 +0100 (CET) Subject: [pypy-commit] cffi default: Try harder to run the tests even if PYTHONPATH was not set and cffi not Message-ID: <20130101100154.6E8671C00E2@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1107:9730bd3cf1da Date: 2013-01-01 10:55 +0100 http://bitbucket.org/cffi/cffi/changeset/9730bd3cf1da/ Log: Try harder to run the tests even if PYTHONPATH was not set and cffi not installed. Tested with a "python setup.py build_ext -i" as well as with a fresh "python setup.py install". diff --git a/c/test_c.py b/c/test_c.py --- a/c/test_c.py +++ b/c/test_c.py @@ -1,4 +1,8 @@ import py +def _setup_path(): + import os, sys + sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) +_setup_path() from _cffi_backend import * from _cffi_backend import _testfunc, _get_types diff --git a/testing/callback_in_thread.py b/testing/callback_in_thread.py --- a/testing/callback_in_thread.py +++ b/testing/callback_in_thread.py @@ -1,4 +1,5 @@ -import time +import sys, time +sys.path.insert(0, sys.argv[1]) from cffi import FFI def _run_callback_in_thread(): diff --git a/testing/test_verify.py b/testing/test_verify.py --- a/testing/test_verify.py +++ b/testing/test_verify.py @@ -1276,9 +1276,10 @@ def test_callback_in_thread(): if sys.platform == 'win32': py.test.skip("pthread only") - import os, subprocess + import os, subprocess, imp arg = os.path.join(os.path.dirname(__file__), 'callback_in_thread.py') - g = subprocess.Popen([sys.executable, arg]) + g = subprocess.Popen([sys.executable, arg, + os.path.dirname(imp.find_module('cffi')[1])]) result = g.wait() assert result == 0 diff --git a/testing/test_zintegration.py b/testing/test_zintegration.py --- a/testing/test_zintegration.py +++ b/testing/test_zintegration.py @@ -17,10 +17,10 @@ site_packages = dirpath break if site_packages: - for module in ('pycparser', 'ply'): - os.symlink(imp.find_module(module)[1], - os.path.join(site_packages, module)) - + for module in ('cffi', '_cffi_backend', 'pycparser', 'ply'): + target = imp.find_module(module)[1] + os.symlink(target, os.path.join(site_packages, + os.path.basename(target))) return tmpdir SNIPPET_DIR = py.path.local(__file__).join('..', 'snippets') From noreply at buildbot.pypy.org Tue Jan 1 11:01:55 2013 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 1 Jan 2013 11:01:55 +0100 (CET) Subject: [pypy-commit] cffi default: Try harder to clean up the __pycache__. Message-ID: <20130101100155.6D89E1C00E2@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1108:85ca8a6f8a66 Date: 2013-01-01 11:01 +0100 http://bitbucket.org/cffi/cffi/changeset/85ca8a6f8a66/ Log: Try harder to clean up the __pycache__. diff --git a/testing/test_zintegration.py b/testing/test_zintegration.py --- a/testing/test_zintegration.py +++ b/testing/test_zintegration.py @@ -26,10 +26,13 @@ SNIPPET_DIR = py.path.local(__file__).join('..', 'snippets') def really_run_setup_and_program(dirname, venv_dir, python_snippet): - shutil.rmtree(str(SNIPPET_DIR.join(dirname, 'build')), - ignore_errors=True) - shutil.rmtree(str(SNIPPET_DIR.join(dirname, '__pycache__')), - ignore_errors=True) + def remove(dir): + dir = str(SNIPPET_DIR.join(dirname, dir)) + shutil.rmtree(dir, ignore_errors=True) + remove('build') + remove('__pycache__') + for basedir in os.listdir(str(SNIPPET_DIR.join(dirname))): + remove(os.path.join(basedir, '__pycache__')) olddir = os.getcwd() python_f = udir.join('x.py') python_f.write(py.code.Source(python_snippet)) From noreply at buildbot.pypy.org Tue Jan 1 14:13:14 2013 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 1 Jan 2013 14:13:14 +0100 (CET) Subject: [pypy-commit] cffi default: issue #46: don't pass -Werror on recent Macs Message-ID: <20130101131314.BB7611C0046@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1109:2af1ce343291 Date: 2013-01-01 14:13 +0100 http://bitbucket.org/cffi/cffi/changeset/2af1ce343291/ Log: issue #46: don't pass -Werror on recent Macs diff --git a/testing/test_verify.py b/testing/test_verify.py --- a/testing/test_verify.py +++ b/testing/test_verify.py @@ -1,13 +1,20 @@ import py -import sys, math, weakref +import sys, os, math, weakref from cffi import FFI, VerificationError, VerificationMissing, model from testing.support import * -if sys.platform != 'win32': +if sys.platform == 'win32': + pass # no obvious -Werror equivalent on MSVC +elif (sys.platform == 'darwin' and + map(int, os.uname()[2].split('.')) >= [11, 0, 0]): + pass # recent MacOSX come with clang by default, and passing some + # flags from the interpreter (-mno-fused-madd) generates a + # warning --- which is interpreted as an error with -Werror +else: + # assume a standard GCC class FFI(FFI): def verify(self, *args, **kwds): - # XXX a GCC-only way to say "crash upon warnings too" return super(FFI, self).verify( *args, extra_compile_args=['-Werror'], **kwds) From noreply at buildbot.pypy.org Wed Jan 2 11:12:15 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 2 Jan 2013 11:12:15 +0100 (CET) Subject: [pypy-commit] pypy callback-jit: a failing test Message-ID: <20130102101215.CB0381C0238@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: callback-jit Changeset: r59648:0969449422fb Date: 2013-01-02 12:11 +0200 http://bitbucket.org/pypy/pypy/changeset/0969449422fb/ Log: a failing test diff --git a/pypy/jit/metainterp/test/test_call.py b/pypy/jit/metainterp/test/test_call.py new file mode 100644 --- /dev/null +++ b/pypy/jit/metainterp/test/test_call.py @@ -0,0 +1,27 @@ + +from pypy.jit.metainterp.test.support import LLJitMixin +from pypy.rlib import jit + +class TestCall(LLJitMixin): + def test_indirect_call(self): + @jit.dont_look_inside + def f1(x): + return x + 1 + + @jit.dont_look_inside + def f2(x): + return x + 2 + + @jit.dont_look_inside + def choice(i): + if i: + return f1 + return f2 + + def f(i): + func = choice(i) + return func(i) + + res = self.interp_operations(f, [3]) + assert res == f(3) + From noreply at buildbot.pypy.org Wed Jan 2 11:13:51 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 2 Jan 2013 11:13:51 +0100 (CET) Subject: [pypy-commit] pypy callback-jit: fix the test Message-ID: <20130102101351.C2E1D1C0238@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: callback-jit Changeset: r59649:198e1894f7d9 Date: 2013-01-02 12:13 +0200 http://bitbucket.org/pypy/pypy/changeset/198e1894f7d9/ Log: fix the test diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py --- a/pypy/jit/metainterp/pyjitpl.py +++ b/pypy/jit/metainterp/pyjitpl.py @@ -1376,16 +1376,15 @@ """The 'residual_call' operation is emitted in two cases: when we have to generate a residual CALL operation, but also to handle an indirect_call that may need to be inlined.""" - assert isinstance(funcbox, Const) - sd = self.metainterp.staticdata - key = sd.cpu.ts.getaddr_for_box(funcbox) - jitcode = sd.bytecode_for_address(key) - if jitcode is not None: - # we should follow calls to this graph - return self.metainterp.perform_call(jitcode, argboxes) - else: - # but we should not follow calls to that graph - return self.do_residual_call(funcbox, argboxes, calldescr) + if isinstance(funcbox, Const): + sd = self.metainterp.staticdata + key = sd.cpu.ts.getaddr_for_box(funcbox) + jitcode = sd.bytecode_for_address(key) + if jitcode is not None: + # we should follow calls to this graph + return self.metainterp.perform_call(jitcode, argboxes) + # but we should not follow calls to that graph + return self.do_residual_call(funcbox, argboxes, calldescr) # ____________________________________________________________ diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py --- a/pypy/jit/metainterp/test/test_ajit.py +++ b/pypy/jit/metainterp/test/test_ajit.py @@ -3980,4 +3980,4 @@ return 3 self.interp_operations(f, []) - + diff --git a/pypy/jit/metainterp/test/test_call.py b/pypy/jit/metainterp/test/test_call.py --- a/pypy/jit/metainterp/test/test_call.py +++ b/pypy/jit/metainterp/test/test_call.py @@ -24,4 +24,4 @@ res = self.interp_operations(f, [3]) assert res == f(3) - + From noreply at buildbot.pypy.org Wed Jan 2 15:06:18 2013 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Wed, 2 Jan 2013 15:06:18 +0100 (CET) Subject: [pypy-commit] cffi default: integration test: always pass --distribute to virtualenv, addresses #49 Message-ID: <20130102140618.82FFA1C1143@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: Changeset: r1110:15697595208a Date: 2013-01-02 15:06 +0100 http://bitbucket.org/cffi/cffi/changeset/15697595208a/ Log: integration test: always pass --distribute to virtualenv, addresses #49 diff --git a/testing/test_zintegration.py b/testing/test_zintegration.py --- a/testing/test_zintegration.py +++ b/testing/test_zintegration.py @@ -6,7 +6,8 @@ def create_venv(name): tmpdir = udir.join(name) try: - subprocess.check_call(['virtualenv', '-p', sys.executable, + subprocess.check_call(['virtualenv', '--distribute', + '-p', sys.executable, str(tmpdir)]) except OSError, e: py.test.skip("Cannot execute virtualenv: %s" % (e,)) From noreply at buildbot.pypy.org Wed Jan 2 17:47:40 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 2 Jan 2013 17:47:40 +0100 (CET) Subject: [pypy-commit] pypy callback-jit: (SimonSapin) fix the link Message-ID: <20130102164740.74BAF1C071B@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: callback-jit Changeset: r59650:276816e6a599 Date: 2013-01-02 18:47 +0200 http://bitbucket.org/pypy/pypy/changeset/276816e6a599/ Log: (SimonSapin) fix the link diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst --- a/pypy/doc/index.rst +++ b/pypy/doc/index.rst @@ -67,7 +67,7 @@ .. _Python: http://docs.python.org/index.html .. _`more...`: architecture.html#mission-statement .. _`PyPy blog`: http://morepypy.blogspot.com/ -.. _`development bug/feature tracker`: https://codespeak.net/issue/pypy-dev/ +.. _`development bug/feature tracker`: https://bugs.pypy.org .. _here: http://tismerysoft.de/pypy/irc-logs/pypy .. _`Mercurial commit mailing list`: http://python.org/mailman/listinfo/pypy-commit .. _`development mailing list`: http://python.org/mailman/listinfo/pypy-dev From noreply at buildbot.pypy.org Wed Jan 2 17:48:17 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 2 Jan 2013 17:48:17 +0100 (CET) Subject: [pypy-commit] pypy default: (SimonSapin) fix the link Message-ID: <20130102164817.0D78B1C071B@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r59651:d229a2884bc1 Date: 2013-01-02 18:47 +0200 http://bitbucket.org/pypy/pypy/changeset/d229a2884bc1/ Log: (SimonSapin) fix the link diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst --- a/pypy/doc/index.rst +++ b/pypy/doc/index.rst @@ -67,7 +67,7 @@ .. _Python: http://docs.python.org/index.html .. _`more...`: architecture.html#mission-statement .. _`PyPy blog`: http://morepypy.blogspot.com/ -.. _`development bug/feature tracker`: https://codespeak.net/issue/pypy-dev/ +.. _`development bug/feature tracker`: https://bugs.pypy.org .. _here: http://tismerysoft.de/pypy/irc-logs/pypy .. _`Mercurial commit mailing list`: http://python.org/mailman/listinfo/pypy-commit .. _`development mailing list`: http://python.org/mailman/listinfo/pypy-dev From noreply at buildbot.pypy.org Wed Jan 2 19:59:54 2013 From: noreply at buildbot.pypy.org (antocuni) Date: Wed, 2 Jan 2013 19:59:54 +0100 (CET) Subject: [pypy-commit] pypy virtual-raw-mallocs: refactor virtuals_cache and virtuals_int_cache: put them in a separate class because it is easier to handle and to make it rpython Message-ID: <20130102185954.9CD591C0107@cobra.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: virtual-raw-mallocs Changeset: r59652:10697b95df08 Date: 2013-01-02 19:58 +0100 http://bitbucket.org/pypy/pypy/changeset/10697b95df08/ Log: refactor virtuals_cache and virtuals_int_cache: put them in a separate class because it is easier to handle and to make it rpython diff --git a/pypy/jit/metainterp/blackhole.py b/pypy/jit/metainterp/blackhole.py --- a/pypy/jit/metainterp/blackhole.py +++ b/pypy/jit/metainterp/blackhole.py @@ -1546,7 +1546,7 @@ return blackholeinterp, lle def resume_in_blackhole(metainterp_sd, jitdriver_sd, resumedescr, - all_virtuals=None): + all_virtuals = None): from pypy.jit.metainterp.resume import blackhole_from_resumedata #debug_start('jit-blackhole') blackholeinterp = blackhole_from_resumedata( diff --git a/pypy/jit/metainterp/resume.py b/pypy/jit/metainterp/resume.py --- a/pypy/jit/metainterp/resume.py +++ b/pypy/jit/metainterp/resume.py @@ -494,7 +494,7 @@ @specialize.argtype(1) def allocate(self, decoder, index): struct = decoder.allocate_with_vtable(self.known_class) - decoder.virtuals_cache[index] = struct + decoder.virtuals_cache.set_ptr(index, struct) return self.setfields(decoder, struct) def debug_prints(self): @@ -510,7 +510,7 @@ @specialize.argtype(1) def allocate(self, decoder, index): struct = decoder.allocate_struct(self.typedescr) - decoder.virtuals_cache[index] = struct + decoder.virtuals_cache.set_ptr(index, struct) return self.setfields(decoder, struct) def debug_prints(self): @@ -527,7 +527,7 @@ length = len(self.fieldnums) arraydescr = self.arraydescr array = decoder.allocate_array(length, arraydescr) - decoder.virtuals_cache[index] = array + decoder.virtuals_cache.set_ptr(index, array) # NB. the check for the kind of array elements is moved out of the loop if arraydescr.is_array_of_pointers(): for i in range(length): @@ -561,7 +561,7 @@ def allocate_int(self, decoder, index): length = len(self.fieldnums) buffer = decoder.allocate_raw_buffer(self.size) - decoder.virtuals_int_cache[index] = buffer + decoder.virtuals_cache.set_int(index, buffer) for i in range(len(self.offsets)): offset = self.offsets[i] descr = self.descrs[i] @@ -587,7 +587,7 @@ @specialize.argtype(1) def allocate(self, decoder, index): array = decoder.allocate_array(len(self.fielddescrs), self.arraydescr) - decoder.virtuals_cache[index] = array + decoder.virtuals_cache.set_ptr(index, array) p = 0 for i in range(len(self.fielddescrs)): for j in range(len(self.fielddescrs[i])): @@ -604,7 +604,7 @@ def allocate(self, decoder, index): length = len(self.fieldnums) string = decoder.allocate_string(length) - decoder.virtuals_cache[index] = string + decoder.virtuals_cache.set_ptr(index, string) for i in range(length): charnum = self.fieldnums[i] if not tagged_eq(charnum, UNINITIALIZED): @@ -626,7 +626,7 @@ # efficient. Not sure we care. left, right = self.fieldnums string = decoder.concat_strings(left, right) - decoder.virtuals_cache[index] = string + decoder.virtuals_cache.set_ptr(index, string) return string def debug_prints(self): @@ -642,7 +642,7 @@ def allocate(self, decoder, index): largerstr, start, length = self.fieldnums string = decoder.slice_string(largerstr, start, length) - decoder.virtuals_cache[index] = string + decoder.virtuals_cache.set_ptr(index, string) return string def debug_prints(self): @@ -659,7 +659,7 @@ def allocate(self, decoder, index): length = len(self.fieldnums) string = decoder.allocate_unicode(length) - decoder.virtuals_cache[index] = string + decoder.virtuals_cache.set_ptr(index, string) for i in range(length): charnum = self.fieldnums[i] if not tagged_eq(charnum, UNINITIALIZED): @@ -681,7 +681,7 @@ # efficient. Not sure we care. left, right = self.fieldnums string = decoder.concat_unicodes(left, right) - decoder.virtuals_cache[index] = string + decoder.virtuals_cache.set_ptr(index, string) return string def debug_prints(self): @@ -698,7 +698,7 @@ def allocate(self, decoder, index): largerstr, start, length = self.fieldnums string = decoder.slice_unicode(largerstr, start, length) - decoder.virtuals_cache[index] = string + decoder.virtuals_cache.set_ptr(index, string) return string def debug_prints(self): @@ -708,6 +708,33 @@ # ____________________________________________________________ +class AbstractVirtualCache(object): + pass + +def get_VirtualCache_class(suffix): + # we need to create two copy of this class, because virtuals_*_cache will + # be lists of different types (one for ResumeDataDirectReader and one for + # ResumeDataBoxReader) + class VirtualCache(AbstractVirtualCache): + def __init__(self, virtuals_ptr_cache, virtuals_int_cache): + self.virtuals_ptr_cache = virtuals_ptr_cache + self.virtuals_int_cache = virtuals_int_cache + + def get_ptr(self, i): + return self.virtuals_ptr_cache[i] + + def get_int(self, i): + return self.virtuals_int_cache[i] + + def set_ptr(self, i, v): + self.virtuals_ptr_cache[i] = v + + def set_int(self, i, v): + self.virtuals_int_cache[i] = v + + VirtualCache.__name__ += suffix + return VirtualCache + class AbstractResumeDataReader(object): """A base mixin containing the logic to reconstruct virtuals out of guard failure. There are two implementations of this mixin: @@ -718,10 +745,10 @@ _mixin_ = True rd_virtuals = None virtuals_cache = None - virtuals_int_cache = None - virtual_default = None + virtual_ref_default = None virtual_int_default = None + def _init(self, cpu, storage): self.cpu = cpu self.cur_numb = storage.rd_numb @@ -731,24 +758,24 @@ self._prepare_virtuals(storage.rd_virtuals) self._prepare_pendingfields(storage.rd_pendingfields) - def getvirtual(self, index): + def getvirtual_ref(self, index): # Returns the index'th virtual, building it lazily if needed. # Note that this may be called recursively; that's why the # allocate() methods must fill in the cache as soon as they # have the object, before they fill its fields. assert self.virtuals_cache is not None - v = self.virtuals_cache[index] + v = self.virtuals_cache.get_ptr(index) if not v: v = self.rd_virtuals[index].allocate(self, index) - ll_assert(v == self.virtuals_cache[index], "resume.py: bad cache") + ll_assert(v == self.virtuals_cache.get_ptr(index), "resume.py: bad cache") return v def getvirtual_int(self, index): - assert self.virtuals_int_cache is not None - v = self.virtuals_int_cache[index] + assert self.virtuals_cache is not None + v = self.virtuals_cache.get_int(index) if not v: v = self.rd_virtuals[index].allocate_int(self, index) - ll_assert(v == self.virtuals_int_cache[index], "resume.py: bad cache") + ll_assert(v == self.virtuals_cache.get_int(index), "resume.py: bad cache") return v def force_all_virtuals(self): @@ -758,12 +785,12 @@ rd_virtual = rd_virtuals[i] if rd_virtual is not None: if rd_virtual.kind == REF: - self.getvirtual(i) + self.getvirtual_ref(i) elif rd_virtual.kind == INT: self.getvirtual_int(i) else: assert False - return self.virtuals_cache, self.virtuals_int_cache + return self.virtuals_cache def _prepare_virtuals(self, virtuals): if virtuals: @@ -772,8 +799,8 @@ # for REFs and one for INTs: but for each index, we are using # either one or the other, so we should think of a way to # "compact" them - self.virtuals_cache = [self.virtual_default] * len(virtuals) - self.virtuals_int_cache = [self.virtual_int_default] * len(virtuals) + self.virtuals_cache = self.VirtualCache([self.virtual_ref_default] * len(virtuals), + [self.virtual_int_default] * len(virtuals)) def _prepare_pendingfields(self, pendingfields): if pendingfields: @@ -846,6 +873,7 @@ class ResumeDataBoxReader(AbstractResumeDataReader): unique_id = lambda: None + VirtualCache = get_VirtualCache_class('BoxReader') def __init__(self, storage, metainterp): self._init(metainterp.cpu, storage) @@ -1030,7 +1058,7 @@ if kind == INT: box = self.getvirtual_int(num) else: - box = self.getvirtual(num) + box = self.getvirtual_ref(num) elif tag == TAGINT: box = ConstInt(num) else: @@ -1126,9 +1154,10 @@ class ResumeDataDirectReader(AbstractResumeDataReader): unique_id = lambda: None - virtual_default = lltype.nullptr(llmemory.GCREF.TO) + virtual_ref_default = lltype.nullptr(llmemory.GCREF.TO) virtual_int_default = 0 resume_after_guard_not_forced = 0 + VirtualCache = get_VirtualCache_class('DirectReader') # 0: not a GUARD_NOT_FORCED # 1: in handle_async_forcing # 2: resuming from the GUARD_NOT_FORCED @@ -1142,7 +1171,7 @@ # special case for resuming after a GUARD_NOT_FORCED: we already # have the virtuals self.resume_after_guard_not_forced = 2 - self.virtuals_cache, self.virtuals_int_cache = all_virtuals + self.virtuals_cache = all_virtuals # self.rd_virtuals can remain None, because virtuals_cache is # already filled @@ -1339,7 +1368,7 @@ return self.cpu.ts.NULLREF return self.consts[num].getref_base() elif tag == TAGVIRTUAL: - return self.getvirtual(num) + return self.getvirtual_ref(num) else: assert tag == TAGBOX if num < 0: diff --git a/pypy/jit/metainterp/test/test_resume.py b/pypy/jit/metainterp/test/test_resume.py --- a/pypy/jit/metainterp/test/test_resume.py +++ b/pypy/jit/metainterp/test/test_resume.py @@ -228,9 +228,10 @@ def test_prepare_virtuals(): class FakeVinfo(object): + kind = REF def allocate(self, decoder, index): s = "allocated" - decoder.virtuals_cache[index] = s + decoder.virtuals_cache.set_ptr(index, s) return s class FakeStorage(object): rd_virtuals = [FakeVinfo(), None] @@ -241,11 +242,14 @@ _already_allocated_resume_virtuals = None cpu = None reader = ResumeDataDirectReader(MyMetaInterp(None), FakeStorage()) - assert reader.force_all_virtuals() == ["allocated", reader.virtual_default] + cache = reader.force_all_virtuals() + assert cache.virtuals_ptr_cache == ["allocated", reader.virtual_ref_default] # ____________________________________________________________ class FakeResumeDataReader(AbstractResumeDataReader): + VirtualCache = get_VirtualCache_class('Fake') + def allocate_with_vtable(self, known_class): return FakeBuiltObject(vtable=known_class) def allocate_struct(self, typedescr): @@ -300,7 +304,8 @@ info.fieldnums = [tag(456, TAGINT)] reader = FakeResumeDataReader() reader._prepare_virtuals([info]) - assert reader.force_all_virtuals() == [ + cache = reader.force_all_virtuals() + assert cache.virtuals_ptr_cache == [ FakeBuiltObject(vtable=123, fielddescr1=tag(456, TAGINT))] def test_vstructinfo(): @@ -308,7 +313,8 @@ info.fieldnums = [tag(456, TAGINT)] reader = FakeResumeDataReader() reader._prepare_virtuals([info]) - assert reader.force_all_virtuals() == [ + cache = reader.force_all_virtuals() + assert cache.virtuals_ptr_cache == [ FakeBuiltObject(typedescr=124, fielddescr1=tag(456, TAGINT))] def test_varrayinfo(): @@ -317,7 +323,7 @@ info.fieldnums = [tag(456, TAGINT)] reader = FakeResumeDataReader() reader._prepare_virtuals([info]) - assert reader.force_all_virtuals() == [ + assert reader.force_all_virtuals().virtuals_ptr_cache == [ FakeBuiltObject(arraydescr=arraydescr, items=[tag(456, TAGINT)])] def test_vstrplaininfo(): @@ -325,7 +331,7 @@ info.fieldnums = [tag(60, TAGINT)] reader = FakeResumeDataReader() reader._prepare_virtuals([info]) - assert reader.force_all_virtuals() == [ + assert reader.force_all_virtuals().virtuals_ptr_cache == [ FakeBuiltObject(string=[60])] def test_vstrconcatinfo(): @@ -333,7 +339,7 @@ info.fieldnums = [tag(10, TAGBOX), tag(20, TAGBOX)] reader = FakeResumeDataReader() reader._prepare_virtuals([info]) - assert reader.force_all_virtuals() == [ + assert reader.force_all_virtuals().virtuals_ptr_cache == [ FakeBuiltObject(strconcat=info.fieldnums)] def test_vstrsliceinfo(): @@ -341,7 +347,7 @@ info.fieldnums = [tag(10, TAGBOX), tag(20, TAGBOX), tag(30, TAGBOX)] reader = FakeResumeDataReader() reader._prepare_virtuals([info]) - assert reader.force_all_virtuals() == [ + assert reader.force_all_virtuals().virtuals_ptr_cache == [ FakeBuiltObject(strslice=info.fieldnums)] def test_vuniplaininfo(): @@ -349,7 +355,7 @@ info.fieldnums = [tag(60, TAGINT)] reader = FakeResumeDataReader() reader._prepare_virtuals([info]) - assert reader.force_all_virtuals() == [ + assert reader.force_all_virtuals().virtuals_ptr_cache == [ FakeBuiltObject(unistring=[60])] def test_vuniconcatinfo(): @@ -357,7 +363,7 @@ info.fieldnums = [tag(10, TAGBOX), tag(20, TAGBOX)] reader = FakeResumeDataReader() reader._prepare_virtuals([info]) - assert reader.force_all_virtuals() == [ + assert reader.force_all_virtuals().virtuals_ptr_cache == [ FakeBuiltObject(uniconcat=info.fieldnums)] def test_vunisliceinfo(): @@ -365,7 +371,7 @@ info.fieldnums = [tag(10, TAGBOX), tag(20, TAGBOX), tag(30, TAGBOX)] reader = FakeResumeDataReader() reader._prepare_virtuals([info]) - assert reader.force_all_virtuals() == [ + assert reader.force_all_virtuals().virtuals_ptr_cache == [ FakeBuiltObject(unislice=info.fieldnums)] # ____________________________________________________________ @@ -1116,7 +1122,7 @@ metainterp = MyMetaInterp() reader = ResumeDataFakeReader(storage, newboxes, metainterp) - assert len(reader.virtuals_cache) == 2 + assert len(reader.virtuals_cache.virtuals_ptr_cache) == 2 b2t = reader.decode_ref(modifier._gettagged(b2s)) b4t = reader.decode_ref(modifier._gettagged(b4s)) trace = metainterp.trace @@ -1184,7 +1190,7 @@ # resume metainterp = MyMetaInterp() reader = ResumeDataFakeReader(storage, newboxes, metainterp) - assert len(reader.virtuals_cache) == 1 + assert len(reader.virtuals_cache.virtuals_ptr_cache) == 1 b2t = reader.decode_ref(tag(0, TAGVIRTUAL)) trace = metainterp.trace expected = [ @@ -1230,7 +1236,7 @@ NULL = ConstPtr.value metainterp = MyMetaInterp() reader = ResumeDataFakeReader(storage, newboxes, metainterp) - assert len(reader.virtuals_cache) == 1 + assert len(reader.virtuals_cache.virtuals_ptr_cache) == 1 b2t = reader.decode_ref(tag(0, TAGVIRTUAL)) trace = metainterp.trace From noreply at buildbot.pypy.org Wed Jan 2 20:36:34 2013 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 2 Jan 2013 20:36:34 +0100 (CET) Subject: [pypy-commit] pypy numpypy-real-as-view: create a branch to make real, imag views into ndarrays Message-ID: <20130102193634.6EA5D1C1143@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: numpypy-real-as-view Changeset: r59653:c3c307061151 Date: 2012-12-30 07:37 +0200 http://bitbucket.org/pypy/pypy/changeset/c3c307061151/ Log: create a branch to make real, imag views into ndarrays From noreply at buildbot.pypy.org Wed Jan 2 20:36:35 2013 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 2 Jan 2013 20:36:35 +0100 (CET) Subject: [pypy-commit] pypy numpypy-real-as-view: add tests Message-ID: <20130102193635.A52601C1143@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: numpypy-real-as-view Changeset: r59654:afd84911fb34 Date: 2012-12-30 07:55 +0200 http://bitbucket.org/pypy/pypy/changeset/afd84911fb34/ Log: add tests diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1169,8 +1169,8 @@ assert (d == [[1, 0, 0], [0, 1, 0], [0, 0, 1]]).all() def test_eye(self): - from _numpypy import eye, array - from _numpypy import int32, float64, dtype + from _numpypy import eye + from _numpypy import int32, dtype a = eye(0) assert len(a) == 0 assert a.dtype == dtype('float64') @@ -1345,6 +1345,25 @@ assert b[0] == 3 assert b[1] == 2 + def test_realimag_views(self): + from _numpypy import arange, array + a = arange(15) + b = a.real + b[5]=50 + assert a[5] == 50 + b = a.imag + assert b[7] == 0 + raises(RuntimeError, 'b[7] = -2') + a = array(['abc','def'],dtype='S3') + b = a.real + assert (b==a).all() + b[1] = 'xyz' + assert a[1] == 'xyz' + a=array([1+1j, 2-3j]) + assert a.real[1] == 2 + a.real[1] = -20 + assert a[1].real == -20 + def test_tolist_scalar(self): from _numpypy import int32, bool_ x = int32(23) @@ -2273,7 +2292,7 @@ assert a[0]['x'] == 'a' def test_stringarray(self): - from _numpypy import array, flexible + from _numpypy import array a = array(['abc'],'S3') assert str(a.dtype) == '|S3' a = array(['abc']) @@ -2293,7 +2312,6 @@ def test_flexible_repr(self): # import overrides str(), repr() for array - from numpypy.core import arrayprint from _numpypy import array a = array(['abc'],'S3') s = repr(a) From noreply at buildbot.pypy.org Wed Jan 2 22:39:43 2013 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 2 Jan 2013 22:39:43 +0100 (CET) Subject: [pypy-commit] pypy numpypy-real-as-view: implement real, imag for all except complex Message-ID: <20130102213943.45F0D1C071B@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: numpypy-real-as-view Changeset: r59655:db688ce74db3 Date: 2012-12-30 10:01 +0200 http://bitbucket.org/pypy/pypy/changeset/db688ce74db3/ Log: implement real, imag for all except complex diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py --- a/pypy/module/micronumpy/arrayimpl/concrete.py +++ b/pypy/module/micronumpy/arrayimpl/concrete.py @@ -220,6 +220,27 @@ new_shape, self) else: return None + + def get_real(self): + if self.dtype.is_complex_type(): + raise NotImplementedError('waiting for astype()') + return SliceArray(self.start, self.get_strides(), + self.get_backstrides(), self.get_shape(), self) + + def get_imag(self): + if self.dtype.is_complex_type(): + raise NotImplementedError('waiting for astype()') + if self.dtype.is_flexible_type(): + # numpy returns self for self.imag + return SliceArray(self.start, self.get_strides(), + self.get_backstrides(), self.get_shape(), self) + + strides, backstrides = support.calc_strides(self.get_shape(), self.dtype, + self.order) + impl = NonWritableArray(self.get_shape(), self.dtype, self.order, strides, + backstrides) + impl.fill(self.dtype.box(0)) + return impl # -------------------- applevel get/setitem ----------------------- @@ -424,6 +445,12 @@ self.order) return SliceArray(0, strides, backstrides, new_shape, self) +class NonWritableArray(ConcreteArray): + def descr_setitem(self, space, w_index, w_value): + raise OperationError(space.w_RuntimeError, space.wrap( + "array is not writable")) + + class SliceArray(BaseConcreteArray): def __init__(self, start, strides, backstrides, shape, parent, dtype=None): self.strides = strides diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -235,6 +235,24 @@ def descr_copy(self, space): return W_NDimArray(self.implementation.copy()) + def descr_get_real(self, space): + return W_NDimArray(self.implementation.get_real()) + + def descr_get_imag(self, space): + ret = self.implementation.get_imag() + if ret: + return W_NDimArray(ret) + raise OperationError(space.w_NotImplementedError, + space.wrap('imag not implemented for this dtype')) + + def descr_set_real(self, space, w_new_val): + # copy (broadcast) values into self + self.implementation.descr_set_real(space, w_new_val) + + def descr_set_imag(self, space, w_new_val): + # if possible, copy (broadcast) values into self + self.implementation.descr_set_imag(space, w_new_val) + def descr_reshape(self, space, args_w): """reshape(...) a.reshape(shape) @@ -387,8 +405,6 @@ descr_neg = _unaryop_impl("negative") descr_abs = _unaryop_impl("absolute") descr_invert = _unaryop_impl("invert") - descr_get_real = _unaryop_impl("real") - descr_get_imag = _unaryop_impl("imag") def descr_nonzero(self, space): if self.get_size() > 1: @@ -635,9 +651,10 @@ swapaxes = interp2app(W_NDimArray.descr_swapaxes), flat = GetSetProperty(W_NDimArray.descr_get_flatiter), item = interp2app(W_NDimArray.descr_item), - real = GetSetProperty(W_NDimArray.descr_get_real), - imag = GetSetProperty(W_NDimArray.descr_get_imag), - + real = GetSetProperty(W_NDimArray.descr_get_real, + W_NDimArray.descr_set_real), + imag = GetSetProperty(W_NDimArray.descr_get_imag, + W_NDimArray.descr_set_imag), __array_interface__ = GetSetProperty(W_NDimArray.descr_array_iface), ) diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1356,7 +1356,8 @@ raises(RuntimeError, 'b[7] = -2') a = array(['abc','def'],dtype='S3') b = a.real - assert (b==a).all() + assert a[0] == b[0] + assert a[1] == b[1] b[1] = 'xyz' assert a[1] == 'xyz' a=array([1+1j, 2-3j]) From noreply at buildbot.pypy.org Wed Jan 2 22:42:54 2013 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 2 Jan 2013 22:42:54 +0100 (CET) Subject: [pypy-commit] cffi default: issue #50: a test file for unicode literals, testing roughly all methods Message-ID: <20130102214254.1CEC71C1146@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1111:c3bb227619ff Date: 2013-01-02 09:15 +0100 http://bitbucket.org/cffi/cffi/changeset/c3bb227619ff/ Log: issue #50: a test file for unicode literals, testing roughly all methods of FFI. diff --git a/testing/test_unicode_literals.py b/testing/test_unicode_literals.py new file mode 100644 --- /dev/null +++ b/testing/test_unicode_literals.py @@ -0,0 +1,72 @@ +# +# ---------------------------------------------- +# WARNING, ALL LITERALS IN THIS FILE ARE UNICODE +# ---------------------------------------------- +# +from __future__ import unicode_literals +# +# +# +import sys, math +from cffi import FFI + + +def test_cast(): + ffi = FFI() + assert int(ffi.cast("int", 3.14)) == 3 # unicode literal + +def test_new(): + ffi = FFI() + assert ffi.new("int[]", [3, 4, 5])[2] == 5 # unicode literal + +def test_typeof(): + ffi = FFI() + tp = ffi.typeof("int[51]") # unicode literal + assert tp.length == 51 + +def test_sizeof(): + ffi = FFI() + assert ffi.sizeof("int[51]") == 51 * 4 # unicode literal + +def test_alignof(): + ffi = FFI() + assert ffi.alignof("int[51]") == 4 # unicode literal + +def test_getctype(): + ffi = FFI() + assert ffi.getctype("int**") == "int * *" # unicode literal + assert type(ffi.getctype("int**")) is str + +def test_cdef(): + ffi = FFI() + ffi.cdef("typedef int foo_t[50];") # unicode literal + +def test_offsetof(): + ffi = FFI() + ffi.cdef("typedef struct { int x, y; } foo_t;") + assert ffi.offsetof("foo_t", "y") == 4 # unicode literal + +def test_enum(): + ffi = FFI() + ffi.cdef("enum foo_e { AA, BB, CC };") # unicode literal + x = ffi.cast("enum foo_e", "BB") + assert int(ffi.cast("int", x)) == 1 + +def test_dlopen(): + ffi = FFI() + ffi.cdef("double sin(double x);") + m = ffi.dlopen("m") # unicode literal + x = m.sin(1.23) + assert x == math.sin(1.23) + +def test_verify(): + ffi = FFI() + ffi.cdef("double test_verify_1(double x);") # unicode literal + lib = ffi.verify("double test_verify_1(double x) { return x * 42.0; }") + assert lib.test_verify_1(-1.5) == -63.0 + +def test_callback(): + ffi = FFI() + cb = ffi.callback("int(int)", # unicode literal + lambda x: x + 42) + assert cb(5) == 47 From noreply at buildbot.pypy.org Wed Jan 2 22:42:55 2013 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 2 Jan 2013 22:42:55 +0100 (CET) Subject: [pypy-commit] cffi default: issue #50: fix, at least as much as I could find tests for. Message-ID: <20130102214255.3582B1C1146@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1112:76bef4539c42 Date: 2013-01-02 22:35 +0100 http://bitbucket.org/cffi/cffi/changeset/76bef4539c42/ Log: issue #50: fix, at least as much as I could find tests for. diff --git a/cffi/api.py b/cffi/api.py --- a/cffi/api.py +++ b/cffi/api.py @@ -83,6 +83,10 @@ then be accessed via either 'ffi.dlopen()' or 'ffi.verify()'. The types can be used in 'ffi.new()' and other functions. """ + if not isinstance(csource, str): # unicode, on Python 2 + if not isinstance(csource, basestring): + raise TypeError("cdef() argument must be a string") + csource = csource.encode('ascii') self._parser.parse(csource, override=override) self._cdefsources.append(csource) if override: @@ -96,7 +100,7 @@ linked to a particular library, just like C headers; in the library we only look for the actual (untyped) symbols. """ - assert isinstance(name, str) or name is None + assert isinstance(name, basestring) or name is None lib, function_cache = _make_ffi_library(self, name, flags) self._function_caches.append(function_cache) self._libraries.append(lib) @@ -109,11 +113,14 @@ if consider_function_as_funcptr and not cfaf: raise KeyError except KeyError: + key = cdecl + if not isinstance(cdecl, str): # unicode, on Python 2 + cdecl = cdecl.encode('ascii') cfaf = consider_function_as_funcptr type = self._parser.parse_type(cdecl, consider_function_as_funcptr=cfaf) btype = self._get_cached_btype(type) - self._parsed_types[cdecl] = btype, cfaf + self._parsed_types[key] = btype, cfaf return btype def typeof(self, cdecl): @@ -121,7 +128,7 @@ corresponding Python type: '>. It can also be used on 'cdata' instance to get its C type. """ - if isinstance(cdecl, str): + if isinstance(cdecl, basestring): return self._typeof(cdecl) else: return self._backend.typeof(cdecl) @@ -130,7 +137,7 @@ """Return the size in bytes of the argument. It can be a string naming a C type, or a 'cdata' instance. """ - if isinstance(cdecl, str): + if isinstance(cdecl, basestring): BType = self._typeof(cdecl) return self._backend.sizeof(BType) else: @@ -140,7 +147,7 @@ """Return the natural alignment size in bytes of the C type given as a string. """ - if isinstance(cdecl, str): + if isinstance(cdecl, basestring): cdecl = self._typeof(cdecl) return self._backend.alignof(cdecl) @@ -148,7 +155,7 @@ """Return the offset of the named field inside the given structure, which must be given as a C type name. """ - if isinstance(cdecl, str): + if isinstance(cdecl, basestring): cdecl = self._typeof(cdecl) return self._backend.typeoffsetof(cdecl, fieldname)[1] @@ -175,7 +182,7 @@ about that when copying the pointer to the memory somewhere else, e.g. into another structure. """ - if isinstance(cdecl, str): + if isinstance(cdecl, basestring): cdecl = self._typeof(cdecl) return self._backend.newp(cdecl, init) @@ -184,7 +191,7 @@ type initialized with the given 'source'. The source is casted between integers or pointers of any type. """ - if isinstance(cdecl, str): + if isinstance(cdecl, basestring): cdecl = self._typeof(cdecl) return self._backend.cast(cdecl, source) @@ -232,7 +239,7 @@ raise TypeError("the 'python_callable' argument " "is not callable") return self._backend.callback(cdecl, python_callable, error) - if isinstance(cdecl, str): + if isinstance(cdecl, basestring): cdecl = self._typeof(cdecl, consider_function_as_funcptr=True) if python_callable is None: return callback_decorator_wrap # decorator mode @@ -245,7 +252,7 @@ extra text to append (or insert for more complicated C types), like a variable name, or '*' to get actually the C type 'pointer-to-cdecl'. """ - if isinstance(cdecl, str): + if isinstance(cdecl, basestring): cdecl = self._typeof(cdecl) replace_with = replace_with.strip() if (replace_with.startswith('*') @@ -381,6 +388,11 @@ property.__set__(self, value) # if libname is not None: - FFILibrary.__name__ = 'FFILibrary_%s' % libname + try: + if not isinstance(libname, str): # unicode, on Python 2 + libname = libname.encode('utf-8') + FFILibrary.__name__ = 'FFILibrary_%s' % libname + except UnicodeError: + pass library = FFILibrary() return library, library.__dict__ From noreply at buildbot.pypy.org Wed Jan 2 22:42:56 2013 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 2 Jan 2013 22:42:56 +0100 (CET) Subject: [pypy-commit] cffi default: merge heads Message-ID: <20130102214256.5A7271C1146@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1113:ef0728d36e2b Date: 2013-01-02 22:42 +0100 http://bitbucket.org/cffi/cffi/changeset/ef0728d36e2b/ Log: merge heads diff --git a/testing/test_zintegration.py b/testing/test_zintegration.py --- a/testing/test_zintegration.py +++ b/testing/test_zintegration.py @@ -6,7 +6,8 @@ def create_venv(name): tmpdir = udir.join(name) try: - subprocess.check_call(['virtualenv', '-p', sys.executable, + subprocess.check_call(['virtualenv', '--distribute', + '-p', sys.executable, str(tmpdir)]) except OSError, e: py.test.skip("Cannot execute virtualenv: %s" % (e,)) From noreply at buildbot.pypy.org Wed Jan 2 23:43:09 2013 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 2 Jan 2013 23:43:09 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: pass tests using cpython by adding '>g', ' Author: mattip Branch: missing-ndarray-attributes Changeset: r59656:a130f2718318 Date: 2012-12-30 11:04 +0200 http://bitbucket.org/pypy/pypy/changeset/a130f2718318/ Log: pass tests using cpython by adding '>g','>= PERTURB_SHIFT + + @staticmethod + def _make_index(n): + 'New sequence of indices using the smallest possible datatype' + #if n <= 2**7: return array.array('b', [FREE]) * n # signed char + #if n <= 2**15: return array.array('h', [FREE]) * n # signed short + #if n <= 2**31: return array.array('l', [FREE]) * n # signed long + return [FREE] * n # python integers + + def _resize(self, n): + '''Reindex the existing hash/key/value entries. + Entries do not get moved, they only get new indices. + No calls are made to hash() or __eq__(). + + ''' + new_size = 8 + while new_size < n: + new_size <<= 1 + n = new_size + self.indices = self._make_index(n) + PERTURB_SHIFT = 5 + for index, hashvalue in enumerate(self.hashlist): + if hashvalue < 0: + perturb = -hashvalue + else: + perturb = hashvalue + i = hashvalue & (n-1) + while True: + if self.indices[i] == FREE: + break + i = 5 * i + perturb + 1 + i = i & (n-1) + perturb >>= PERTURB_SHIFT + self.indices[i] = index + self.filled = self.used + + def clear(self): + self.indices = self._make_index(8) + self.hashlist = [] + self.keylist = [] + self.valuelist = [] + self.used = 0 + self.filled = 0 # used + dummies + + def __init__(self): + self.clear() + + def __getitem__(self, key): + hashvalue = hash(key) + index, i = self._lookup(key, hashvalue) + if index < 0: + raise KeyError(key) + return self.valuelist[index] + + def __setitem__(self, key, value): + hashvalue = key # hash + index, i = self._lookup(key, hashvalue) + if index < 0: + self.indices[i] = self.used + self.hashlist.append(hashvalue) + self.keylist.append(key) + self.valuelist.append(value) + self.used += 1 + if index == FREE: + self.filled += 1 + if self.filled * 3 > len(self.indices) * 2: + self._resize(4 * self.__len__()) + else: + self.valuelist[index] = value + + def __delitem__(self, key): + hashvalue = hash(key) + index, i = self._lookup(key, hashvalue) + if index < 0: + raise KeyError(key) + self.indices[i] = DUMMY + self.used -= 1 + # If needed, swap with the lastmost entry to avoid leaving a "hole" + if index != self.used: + lasthash = self.hashlist[-1] + lastkey = self.keylist[-1] + lastvalue = self.valuelist[-1] + lastindex, j = self._lookup(lastkey, lasthash) + assert lastindex >= 0 and i != j + self.indices[j] = index + self.hashlist[index] = lasthash + self.keylist[index] = lastkey + self.valuelist[index] = lastvalue + # Remove the lastmost entry + self.hashlist.pop() + self.keylist.pop() + self.valuelist.pop() + + def __len__(self): + return self.used + + def __iter__(self): + return iter(self.keylist) + + def iterkeys(self): + return iter(self.keylist) + + def keys(self): + return list(self.keylist) + + def itervalues(self): + return iter(self.valuelist) + + def values(self): + return list(self.valuelist) + + def iteritems(self): + return itertools.izip(self.keylist, self.valuelist) + + def items(self): + return zip(self.keylist, self.valuelist) + + def __contains__(self, key): + index, i = self._lookup(key, hash(key)) + return index >= 0 + + def get(self, key, default=None): + 'D.get(k[,d]) -> D[k] if k in D, else d. d defaults to None.' + index, i = self._lookup(key, hash(key)) + return self.valuelist[index] if index >= 0 else default + + def popitem(self): + ''' D.popitem() -> (k, v), remove and return some (key, value) pair as a + 2-tuple; but raise KeyError if D is empty. + ''' + try: + key = self.keylist[-1] + value = self.valuelist[-1] + except IndexError: + raise KeyError( 'popitem(): dictionary is empty') + del self[key] + return key, value + + def __repr__(self): + return 'Dict(%r)' % self.items() + + def show_structure(self): + 'Diagnostic method. Not part of the API.' + print '=' * 50 + print self + print 'Indices:', self.indices + for i, row in enumerate(zip(self.hashlist, self.keylist, self.valuelist)): + print i, row + print '-' * 50 + + +if __name__ == '__main__': + import sys + def f(): + if len(sys.argv) > 1: + d = {} + else: + d = Dict() + class A(object): + pass + for i in range(10000000): + d[i] = A() + f() diff --git a/pypy/translator/goal/targetnopstandalone.py b/pypy/translator/goal/targetnopstandalone.py --- a/pypy/translator/goal/targetnopstandalone.py +++ b/pypy/translator/goal/targetnopstandalone.py @@ -7,13 +7,24 @@ actually implementing argv of the executable. """ -def debug(msg): - print "debug:", msg - # __________ Entry point __________ +class A(object): + def __init__(self, a, b, c): + self.a = a + self.b = b + self.c = c + def entry_point(argv): - debug("hello world") + if len(argv) > 2: + d = {} + for i in range(int(argv[1])): + d[i] = A(i, i, i) + else: + from pypy.rlib import dict + d = dict.Dict() + for i in range(int(argv[1])): + d.__setitem__(i, A(i, i, i)) return 0 # _____ Define and setup target ___ From noreply at buildbot.pypy.org Thu Jan 3 10:20:01 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 3 Jan 2013 10:20:01 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments: try to use int-based dicts using lltype Message-ID: <20130103092001.850511C0046@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments Changeset: r59659:95d768d08ca8 Date: 2013-01-03 11:19 +0200 http://bitbucket.org/pypy/pypy/changeset/95d768d08ca8/ Log: try to use int-based dicts using lltype diff --git a/pypy/rlib/dict.py b/pypy/rlib/dict.py --- a/pypy/rlib/dict.py +++ b/pypy/rlib/dict.py @@ -1,11 +1,18 @@ from pypy.rlib.objectmodel import compute_identity_hash +from pypy.rlib.jit_hooks import _cast_to_gcref +from pypy.rpython.lltypesystem import lltype, llmemory # Placeholder constants FREE = -1 DUMMY = -2 +TP = lltype.GcArray(lltype.Struct('dictentry', + (lltype.Signed, 'key'), + (llmemory.GCREF, 'value'))) +MAIN_TP = lltype.GcArray(lltype.Signed) + class Dict(object): 'Space efficient dictionary with fast iteration and cheap resizes.' @@ -26,21 +33,19 @@ return (FREE, i) if freeslot == -1 else (DUMMY, freeslot) elif index == DUMMY: freeslot = i - elif (self.keylist[index] is key or - self.hashlist[index] == hashvalue - and self.keylist[index] == key): - return (index, i) + elif self.values[index].key == key: + return (index, i) i = 5 * i + perturb + 1 i = i & (n-1) perturb >>= PERTURB_SHIFT + _lookup._always_inline_ = True - @staticmethod - def _make_index(n): + def _make_index(self, n): 'New sequence of indices using the smallest possible datatype' #if n <= 2**7: return array.array('b', [FREE]) * n # signed char #if n <= 2**15: return array.array('h', [FREE]) * n # signed short #if n <= 2**31: return array.array('l', [FREE]) * n # signed long - return [FREE] * n # python integers + return lltype.malloc(MAIN_TP, n) def _resize(self, n): '''Reindex the existing hash/key/value entries. @@ -68,12 +73,15 @@ perturb >>= PERTURB_SHIFT self.indices[i] = index self.filled = self.used + old_values = self.values + self.values = lltype.malloc(TP, new_size * 2 / 3) + for i in range(self.used): + self.values[i].key = old_values[i].key + self.values[i].value = old_values[i].value def clear(self): self.indices = self._make_index(8) - self.hashlist = [] - self.keylist = [] - self.valuelist = [] + self.values = lltype.malloc(TP, 8 * 3 / 2) self.used = 0 self.filled = 0 # used + dummies @@ -92,9 +100,8 @@ index, i = self._lookup(key, hashvalue) if index < 0: self.indices[i] = self.used - self.hashlist.append(hashvalue) - self.keylist.append(key) - self.valuelist.append(value) + self.values[self.used].key = key + self.values[self.used].value = _cast_to_gcref(value) self.used += 1 if index == FREE: self.filled += 1 From noreply at buildbot.pypy.org Thu Jan 3 10:23:14 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 3 Jan 2013 10:23:14 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments: oops Message-ID: <20130103092314.8C9D21C078F@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments Changeset: r59660:676cd7534a84 Date: 2013-01-03 11:22 +0200 http://bitbucket.org/pypy/pypy/changeset/676cd7534a84/ Log: oops diff --git a/pypy/rlib/dict.py b/pypy/rlib/dict.py --- a/pypy/rlib/dict.py +++ b/pypy/rlib/dict.py @@ -9,8 +9,8 @@ DUMMY = -2 TP = lltype.GcArray(lltype.Struct('dictentry', - (lltype.Signed, 'key'), - (llmemory.GCREF, 'value'))) + ('key', lltype.Signed), + ('value', llmemory.GCREF))) MAIN_TP = lltype.GcArray(lltype.Signed) class Dict(object): From noreply at buildbot.pypy.org Thu Jan 3 10:25:02 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 3 Jan 2013 10:25:02 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments: another oops Message-ID: <20130103092502.0E5481C078F@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments Changeset: r59661:fc93c6306383 Date: 2013-01-03 11:24 +0200 http://bitbucket.org/pypy/pypy/changeset/fc93c6306383/ Log: another oops diff --git a/pypy/rlib/dict.py b/pypy/rlib/dict.py --- a/pypy/rlib/dict.py +++ b/pypy/rlib/dict.py @@ -148,9 +148,6 @@ def itervalues(self): return iter(self.valuelist) - def values(self): - return list(self.valuelist) - def iteritems(self): return itertools.izip(self.keylist, self.valuelist) From noreply at buildbot.pypy.org Thu Jan 3 10:26:48 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 3 Jan 2013 10:26:48 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments: small fixes Message-ID: <20130103092648.17B6D1C078F@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments Changeset: r59662:73c379658d78 Date: 2013-01-03 11:26 +0200 http://bitbucket.org/pypy/pypy/changeset/73c379658d78/ Log: small fixes diff --git a/pypy/rlib/dict.py b/pypy/rlib/dict.py --- a/pypy/rlib/dict.py +++ b/pypy/rlib/dict.py @@ -59,7 +59,8 @@ n = new_size self.indices = self._make_index(n) PERTURB_SHIFT = 5 - for index, hashvalue in enumerate(self.hashlist): + for index in range(self.used): + hashvalue = self.values[index].key if hashvalue < 0: perturb = -hashvalue else: @@ -108,7 +109,7 @@ if self.filled * 3 > len(self.indices) * 2: self._resize(4 * self.__len__()) else: - self.valuelist[index] = value + self.values[index].value = value def __delitem__(self, key): hashvalue = hash(key) From noreply at buildbot.pypy.org Thu Jan 3 10:27:30 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 3 Jan 2013 10:27:30 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments: missing cast Message-ID: <20130103092730.67CB21C078F@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments Changeset: r59663:caddef001cdc Date: 2013-01-03 11:26 +0200 http://bitbucket.org/pypy/pypy/changeset/caddef001cdc/ Log: missing cast diff --git a/pypy/rlib/dict.py b/pypy/rlib/dict.py --- a/pypy/rlib/dict.py +++ b/pypy/rlib/dict.py @@ -109,7 +109,7 @@ if self.filled * 3 > len(self.indices) * 2: self._resize(4 * self.__len__()) else: - self.values[index].value = value + self.values[index].value = _cast_to_gcref(value) def __delitem__(self, key): hashvalue = hash(key) From noreply at buildbot.pypy.org Thu Jan 3 10:34:24 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 3 Jan 2013 10:34:24 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments: fixes Message-ID: <20130103093424.A89BF1C0CA4@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments Changeset: r59664:49ea63e22428 Date: 2013-01-03 11:33 +0200 http://bitbucket.org/pypy/pypy/changeset/49ea63e22428/ Log: fixes diff --git a/pypy/rlib/dict.py b/pypy/rlib/dict.py --- a/pypy/rlib/dict.py +++ b/pypy/rlib/dict.py @@ -1,7 +1,7 @@ -from pypy.rlib.objectmodel import compute_identity_hash from pypy.rlib.jit_hooks import _cast_to_gcref -from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.lltypesystem import lltype, llmemory, rclass +from pypy.rpython.annlowlevel import cast_base_ptr_to_instance # Placeholder constants @@ -45,7 +45,10 @@ #if n <= 2**7: return array.array('b', [FREE]) * n # signed char #if n <= 2**15: return array.array('h', [FREE]) * n # signed short #if n <= 2**31: return array.array('l', [FREE]) * n # signed long - return lltype.malloc(MAIN_TP, n) + v = lltype.malloc(MAIN_TP, n) + for i in range(n): + v[i] = FREE + return v def _resize(self, n): '''Reindex the existing hash/key/value entries. @@ -75,7 +78,7 @@ self.indices[i] = index self.filled = self.used old_values = self.values - self.values = lltype.malloc(TP, new_size * 2 / 3) + self.values = lltype.malloc(TP, new_size * 2 / 3 + 1) for i in range(self.used): self.values[i].key = old_values[i].key self.values[i].value = old_values[i].value @@ -89,12 +92,14 @@ def __init__(self): self.clear() - def __getitem__(self, key): - hashvalue = hash(key) - index, i = self._lookup(key, hashvalue) + def __getitem__(self, CLS, key): + index, i = self._lookup(key, key) if index < 0: raise KeyError(key) - return self.valuelist[index] + llref = self.values[index].value + ptr = lltype.cast_opaque_ptr(rclass.OBJECTPTR, llref) + return cast_base_ptr_to_instance(CLS, ptr) + def __setitem__(self, key, value): hashvalue = key # hash @@ -198,6 +203,6 @@ d = Dict() class A(object): pass - for i in range(10000000): + for i in range(100000): d[i] = A() f() From noreply at buildbot.pypy.org Thu Jan 3 10:42:49 2013 From: noreply at buildbot.pypy.org (bivab) Date: Thu, 3 Jan 2013 10:42:49 +0100 (CET) Subject: [pypy-commit] pypy default: fix import Message-ID: <20130103094249.DC8031C0ACA@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: Changeset: r59665:c838e03a8079 Date: 2013-01-03 10:44 +0100 http://bitbucket.org/pypy/pypy/changeset/c838e03a8079/ Log: fix import diff --git a/pypy/jit/backend/arm/test/test_loop_unroll.py b/pypy/jit/backend/arm/test/test_loop_unroll.py --- a/pypy/jit/backend/arm/test/test_loop_unroll.py +++ b/pypy/jit/backend/arm/test/test_loop_unroll.py @@ -1,8 +1,8 @@ import py -from pypy.jit.backend.x86.test.test_basic import Jit386Mixin +from pypy.jit.backend.arm.test.support import JitARMMixin from pypy.jit.metainterp.test import test_loop_unroll -class TestLoopSpec(Jit386Mixin, test_loop_unroll.LoopUnrollTest): +class TestLoopSpec(JitARMMixin, test_loop_unroll.LoopUnrollTest): # for the individual tests see # ====> ../../../metainterp/test/test_loop.py pass From noreply at buildbot.pypy.org Thu Jan 3 10:43:34 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 3 Jan 2013 10:43:34 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments: change the resie strategy to more resemble rdict Message-ID: <20130103094334.D87F61C0ACA@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments Changeset: r59666:9158464f10b2 Date: 2013-01-03 11:43 +0200 http://bitbucket.org/pypy/pypy/changeset/9158464f10b2/ Log: change the resie strategy to more resemble rdict diff --git a/pypy/rlib/dict.py b/pypy/rlib/dict.py --- a/pypy/rlib/dict.py +++ b/pypy/rlib/dict.py @@ -112,7 +112,10 @@ if index == FREE: self.filled += 1 if self.filled * 3 > len(self.indices) * 2: - self._resize(4 * self.__len__()) + if self.used < 50000: + self._resize(4 * self.__len__()) + else: + self._resize(2 * self.__len__()) else: self.values[index].value = _cast_to_gcref(value) From noreply at buildbot.pypy.org Thu Jan 3 10:47:53 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 3 Jan 2013 10:47:53 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments: lower the amount of young objects Message-ID: <20130103094753.2AA8E1C0ACA@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments Changeset: r59667:a6b8812c0300 Date: 2013-01-03 11:47 +0200 http://bitbucket.org/pypy/pypy/changeset/a6b8812c0300/ Log: lower the amount of young objects diff --git a/pypy/translator/goal/targetnopstandalone.py b/pypy/translator/goal/targetnopstandalone.py --- a/pypy/translator/goal/targetnopstandalone.py +++ b/pypy/translator/goal/targetnopstandalone.py @@ -18,13 +18,19 @@ def entry_point(argv): if len(argv) > 2: d = {} + a = None for i in range(int(argv[1])): - d[i] = A(i, i, i) + if i % 100 == 0: + a = A(i, i, i) + d[i] = a else: from pypy.rlib import dict d = dict.Dict() + a = None for i in range(int(argv[1])): - d.__setitem__(i, A(i, i, i)) + if i % 100 == 0: + a = A(i, i, i) + d.__setitem__(i, a) return 0 # _____ Define and setup target ___ From noreply at buildbot.pypy.org Thu Jan 3 12:03:42 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 3 Jan 2013 12:03:42 +0100 (CET) Subject: [pypy-commit] pypy remove-globals-in-jit: finish fixing optimizeopt tests Message-ID: <20130103110342.066961C0CA4@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: remove-globals-in-jit Changeset: r59668:222ce19769eb Date: 2013-01-03 13:03 +0200 http://bitbucket.org/pypy/pypy/changeset/222ce19769eb/ Log: finish fixing optimizeopt tests diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -216,40 +216,6 @@ """ self.optimize_loop(ops, expected) - def test_constfold_all(self): - from pypy.jit.backend.llgraph.llimpl import TYPES # xxx fish - from pypy.jit.metainterp.executor import execute_nonspec - from pypy.jit.metainterp.history import BoxInt - import random - for opnum in range(rop.INT_ADD, rop.SAME_AS+1): - try: - op = opname[opnum] - except KeyError: - continue - if 'FLOAT' in op: - continue - argtypes, restype = TYPES[op.lower()] - args = [] - for argtype in argtypes: - assert argtype in ('int', 'bool') - args.append(random.randrange(1, 20)) - assert restype in ('int', 'bool') - ops = """ - [] - i1 = %s(%s) - escape(i1) - jump() - """ % (op.lower(), ', '.join(map(str, args))) - argboxes = [BoxInt(a) for a in args] - expected_value = execute_nonspec(self.cpu, None, opnum, - argboxes).getint() - expected = """ - [] - escape(%d) - jump() - """ % expected_value - self.optimize_loop(ops, expected) - # ---------- def test_remove_guard_class_1(self): @@ -658,8 +624,8 @@ escape(i3) p1 = new_with_vtable(ConstClass(node_vtable)) p1sub = new_with_vtable(ConstClass(node_vtable2)) + setfield_gc(p1sub, i1, descr=valuedescr) setfield_gc(p1, i1, descr=valuedescr) - setfield_gc(p1sub, i1, descr=valuedescr) setfield_gc(p1, p1sub, descr=nextdescr) jump(i1, p1, p2) """ @@ -994,10 +960,10 @@ ops = """ [f0, f1, f2, f3] p0 = new_array(2, descr=complexarraydescr) + setinteriorfield_gc(p0, 0, f1, descr=compleximagdescr) setinteriorfield_gc(p0, 0, f0, descr=complexrealdescr) - setinteriorfield_gc(p0, 0, f1, descr=compleximagdescr) + setinteriorfield_gc(p0, 1, f3, descr=compleximagdescr) setinteriorfield_gc(p0, 1, f2, descr=complexrealdescr) - setinteriorfield_gc(p0, 1, f3, descr=compleximagdescr) f4 = getinteriorfield_gc(p0, 0, descr=complexrealdescr) f5 = getinteriorfield_gc(p0, 1, descr=complexrealdescr) f6 = float_mul(f4, f5) @@ -1032,8 +998,8 @@ [f0, f1] f2 = float_mul(f0, f1) p0 = new_array(1, descr=complexarraydescr) + setinteriorfield_gc(p0, 0, f1, descr=compleximagdescr) setinteriorfield_gc(p0, 0, f0, descr=complexrealdescr) - setinteriorfield_gc(p0, 0, f1, descr=compleximagdescr) i0 = escape(f2, p0) finish(i0) """ diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -1,21 +1,14 @@ import py from pypy.rlib.objectmodel import instantiate from pypy.jit.metainterp.optimizeopt.test.test_util import ( - LLtypeMixin, BaseTest, Storage, _sortboxes, convert_old_style_to_targets) -import pypy.jit.metainterp.optimizeopt.optimizer as optimizeopt -import pypy.jit.metainterp.optimizeopt.virtualize as virtualize -from pypy.jit.metainterp.optimizeopt import ALL_OPTS_DICT, build_opt_chain + LLtypeMixin, BaseTest, convert_old_style_to_targets) +from pypy.jit.metainterp.optimizeopt import build_opt_chain from pypy.jit.metainterp.optimize import InvalidLoop from pypy.jit.metainterp.history import AbstractDescr, ConstInt, BoxInt -from pypy.jit.metainterp.history import TreeLoop, JitCellToken, TargetToken -from pypy.jit.metainterp.jitprof import EmptyProfiler -from pypy.jit.metainterp import executor, compile, resume, history -from pypy.jit.metainterp.resoperation import rop, opname, ResOperation -from pypy.jit.tool.oparser import pure_parse -from pypy.jit.metainterp.optimizeopt.util import args_dict +from pypy.jit.metainterp.history import TreeLoop +from pypy.jit.metainterp import compile, resume +from pypy.jit.metainterp.resoperation import rop, opname, opargnum from pypy.jit.metainterp.optimizeopt.test.test_optimizebasic import FakeMetaInterpStaticData -from pypy.config.pypyoption import get_pypy_config -from pypy.jit.metainterp.optimizeopt.unroll import Inliner def test_build_opt_chain(): def check(chain, expected_names): @@ -185,9 +178,7 @@ self.optimize_loop(ops, expected) def test_constfold_all(self): - from pypy.jit.backend.llgraph.llimpl import TYPES # xxx fish from pypy.jit.metainterp.executor import execute_nonspec - from pypy.jit.metainterp.history import BoxInt import random for opnum in range(rop.INT_ADD, rop.SAME_AS+1): try: @@ -196,12 +187,9 @@ continue if 'FLOAT' in op: continue - argtypes, restype = TYPES[op.lower()] args = [] - for argtype in argtypes: - assert argtype in ('int', 'bool') + for _ in range(opargnum[opnum]): args.append(random.randrange(1, 20)) - assert restype in ('int', 'bool') ops = """ [] i1 = %s(%s) @@ -836,8 +824,8 @@ escape(i3) p4 = new_with_vtable(ConstClass(node_vtable)) p1sub = new_with_vtable(ConstClass(node_vtable2)) + setfield_gc(p1sub, i1, descr=valuedescr) setfield_gc(p4, i1, descr=valuedescr) - setfield_gc(p1sub, i1, descr=valuedescr) setfield_gc(p4, p1sub, descr=nextdescr) jump(i1, p4) """ @@ -1168,13 +1156,13 @@ i2 = int_sub(i1, 1) i3 = int_add(i0, i1) i4 = same_as(i2) # This same_as should be killed by backend - jump(i3, i2, i1) + jump(i3, i1, i2) """ expected = """ [i0, i1, i1bis] - i2 = int_sub(i1, 1) - i3 = int_add(i0, i1) - jump(i3, i2, i1) + i2 = int_sub(i1bis, 1) + i3 = int_add(i0, i1bis) + jump(i3, i1bis, i2) """ self.optimize_loop(ops, expected, preamble) @@ -1226,8 +1214,8 @@ i1 = int_add(i0, 1) p1 = new_with_vtable(ConstClass(node_vtable2)) p2 = new_with_vtable(ConstClass(node_vtable2)) + setfield_gc(p2, p1, descr=nextdescr) setfield_gc(p2, i1, descr=valuedescr) - setfield_gc(p2, p1, descr=nextdescr) setfield_gc(p1, p2, descr=nextdescr) setfield_gc(p0, p1, descr=nextdescr) jump(p1) @@ -1239,8 +1227,8 @@ i1 = int_add(i0, 1) p1 = new_with_vtable(ConstClass(node_vtable2)) p2 = new_with_vtable(ConstClass(node_vtable2)) + setfield_gc(p2, p1, descr=nextdescr) setfield_gc(p2, i1, descr=valuedescr) - setfield_gc(p2, p1, descr=nextdescr) setfield_gc(p1, p2, descr=nextdescr) setfield_gc(p0, p1, descr=nextdescr) jump(p1) diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py --- a/pypy/jit/metainterp/resoperation.py +++ b/pypy/jit/metainterp/resoperation.py @@ -539,6 +539,7 @@ pass opclasses = [] # mapping numbers to the concrete ResOp class +opargnum = [] # mapping numbers to number or args (or -1) opname = {} # mapping numbers to the original names, for debugging oparity = [] # mapping numbers to the arity of the operation or -1 opwithdescr = [] # mapping numbers to a flag "takes a descr" @@ -567,6 +568,7 @@ else: cls = None opclasses.append(cls) + opargnum.append(arity) oparity.append(arity) opwithdescr.append(withdescr) opboolresult.append(boolresult) From noreply at buildbot.pypy.org Thu Jan 3 12:51:46 2013 From: noreply at buildbot.pypy.org (bivab) Date: Thu, 3 Jan 2013 12:51:46 +0100 (CET) Subject: [pypy-commit] pypy remove-globals-in-jit: move methods to llmodel Message-ID: <20130103115146.4562E1C078F@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: remove-globals-in-jit Changeset: r59669:5214949fc6a9 Date: 2012-12-20 15:41 +0100 http://bitbucket.org/pypy/pypy/changeset/5214949fc6a9/ Log: move methods to llmodel diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py --- a/pypy/jit/backend/llsupport/llmodel.py +++ b/pypy/jit/backend/llsupport/llmodel.py @@ -193,6 +193,12 @@ # ------------------- helpers and descriptions -------------------- + def gc_set_extra_threshold(self): + llop.gc_set_extra_threshold(lltype.Void, self.deadframe_size_max) + + def gc_clear_extra_threshold(self): + llop.gc_set_extra_threshold(lltype.Void, 0) + @staticmethod def _cast_int_to_gcref(x): # dangerous! only use if you are sure no collection could occur diff --git a/pypy/jit/backend/x86/runner.py b/pypy/jit/backend/x86/runner.py --- a/pypy/jit/backend/x86/runner.py +++ b/pypy/jit/backend/x86/runner.py @@ -1,6 +1,5 @@ import py from pypy.rpython.lltypesystem import lltype, llmemory, rffi -from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.llinterp import LLInterpreter from pypy.rlib.objectmodel import we_are_translated, specialize from pypy.rlib.jit_hooks import LOOP_RUN_CONTAINER @@ -58,12 +57,6 @@ else: return 1000 - def gc_set_extra_threshold(self): - llop.gc_set_extra_threshold(lltype.Void, self.deadframe_size_max) - - def gc_clear_extra_threshold(self): - llop.gc_set_extra_threshold(lltype.Void, 0) - def setup(self): self.assembler = Assembler386(self, self.translate_support_code) From noreply at buildbot.pypy.org Thu Jan 3 12:51:47 2013 From: noreply at buildbot.pypy.org (bivab) Date: Thu, 3 Jan 2013 12:51:47 +0100 (CET) Subject: [pypy-commit] pypy remove-globals-in-jit: update test Message-ID: <20130103115147.6FF7D1C078F@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: remove-globals-in-jit Changeset: r59670:0c732f14fcb4 Date: 2012-12-20 15:42 +0100 http://bitbucket.org/pypy/pypy/changeset/0c732f14fcb4/ Log: update test diff --git a/pypy/jit/backend/arm/test/test_runner.py b/pypy/jit/backend/arm/test/test_runner.py --- a/pypy/jit/backend/arm/test/test_runner.py +++ b/pypy/jit/backend/arm/test/test_runner.py @@ -60,8 +60,8 @@ ] cpu.compile_loop(inp, operations, looptoken) args = [i for i in range(1, 15)] - self.cpu.execute_token(looptoken, *args) - output = [self.cpu.get_latest_value_int(i - 1) for i in range(1, 15)] + deadframe = self.cpu.execute_token(looptoken, *args) + output = [self.cpu.get_latest_value_int(deadframe, i - 1) for i in range(1, 15)] expected = [3, 7, 11, 15, 19, 23, 27, 3, 7, 11, 15, 19, 23, 27] assert output == expected From noreply at buildbot.pypy.org Thu Jan 3 12:51:48 2013 From: noreply at buildbot.pypy.org (bivab) Date: Thu, 3 Jan 2013 12:51:48 +0100 (CET) Subject: [pypy-commit] pypy remove-globals-in-jit: (wip) porting branch changes to ARM, some tests pass again Message-ID: <20130103115148.A06DF1C078F@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: remove-globals-in-jit Changeset: r59671:d2595b151ff3 Date: 2012-12-20 15:44 +0100 http://bitbucket.org/pypy/pypy/changeset/d2595b151ff3/ Log: (wip) porting branch changes to ARM, some tests pass again diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py --- a/pypy/jit/backend/arm/assembler.py +++ b/pypy/jit/backend/arm/assembler.py @@ -30,7 +30,7 @@ from pypy.rlib.objectmodel import compute_unique_id # XXX Move to llsupport -from pypy.jit.backend.x86.support import values_array, memcpy_fn +from pypy.jit.backend.x86.support import memcpy_fn DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', ('i', lltype.Signed), ('type', lltype.Char), # 'b'ridge, 'l'abel or @@ -44,14 +44,8 @@ debug = True - def __init__(self, cpu, failargs_limit=1000): + def __init__(self, cpu, translate_support_code=False): self.cpu = cpu - self.fail_boxes_int = values_array(lltype.Signed, failargs_limit) - self.fail_boxes_float = values_array(longlong.FLOATSTORAGE, - failargs_limit) - self.fail_boxes_ptr = values_array(llmemory.GCREF, failargs_limit) - self.fail_boxes_count = 0 - self.fail_force_index = 0 self.setup_failure_recovery() self.mc = None self.memcpy_addr = 0 @@ -68,6 +62,7 @@ self._debug = False self.loop_run_counters = [] self.debug_counter_descr = cpu.fielddescrof(DEBUG_COUNTER, 'i') + self.force_token_to_dead_frame = {} # XXX temporary hack def set_debug(self, v): r = self._debug @@ -123,9 +118,6 @@ self._build_release_gil(gc_ll_descr.gcrootmap) self.memcpy_addr = self.cpu.cast_ptr_to_int(memcpy_fn) self._exit_code_addr = self._gen_exit_path() - self._leave_jitted_hook_save_exc = \ - self._gen_leave_jitted_hook_code(True) - self._leave_jitted_hook = self._gen_leave_jitted_hook_code(False) if not self._debug: # if self._debug is already set it means that someone called # set_debug by hand before initializing the assembler. Leave it @@ -239,17 +231,12 @@ return # not supported (for tests, or non-translated) # mc = ARMv7Builder() - # call on_leave_jitted_save_exc() - if self.cpu.supports_floats: - floats = r.caller_vfp_resp - else: - floats = [] - with saved_registers(mc, r.caller_resp + [r.lr], floats): - addr = self.cpu.get_on_leave_jitted_int(save_exception=True, - default_to_memoryerror=True) - mc.BL(addr) - mc.gen_load_int(r.ip.value, self.cpu.propagate_exception_v) - mc.MOV_rr(r.r0.value, r.ip.value) + # + # Call the helper, which will return a dead frame object with + # the correct exception set, or MemoryError by default + # XXX make sure we return the correct value here + addr = rffi.cast(lltype.Signed, self.cpu.get_propagate_exception()) + mc.BL(addr) self.gen_func_epilog(mc=mc) self.propagate_exception_path = mc.materialize(self.cpu.asmmemmgr, []) @@ -282,12 +269,12 @@ # restore registers and return # We check for c.EQ here, meaning all bits zero in this case mc.POP([reg.value for reg in r.argument_regs] + [r.pc.value], cond=c.EQ) - # call on_leave_jitted_save_exc() - addr = self.cpu.get_on_leave_jitted_int(save_exception=True) + # + # Call the helper, which will return a dead frame object with + # the correct exception set, or MemoryError by default + addr = rffi.cast(lltype.Signed, self.cpu.get_propagate_exception()) mc.BL(addr) # - mc.gen_load_int(r.r0.value, self.cpu.propagate_exception_v) - # # footer -- note the ADD, which skips the return address of this # function, and will instead return to the caller's caller. Note # also that we completely ignore the saved arguments, because we @@ -347,17 +334,18 @@ vfp_registers = rffi.cast(rffi.LONGLONGP, stack_pointer) registers = rffi.ptradd(vfp_registers, len(r.all_vfp_regs)) registers = rffi.cast(rffi.LONGP, registers) - return self.decode_registers_and_descr(mem_loc, frame_pointer, + return self.decode_registers_and_descr(self.cpu, mem_loc, frame_pointer, registers, vfp_registers) self.failure_recovery_func = failure_recovery_func recovery_func_sign = lltype.Ptr(lltype.FuncType([lltype.Signed] * 3, - lltype.Signed)) + llmemory.GCREF)) - @rgc.no_collect - def decode_registers_and_descr(self, mem_loc, frame_pointer, + @staticmethod + def decode_registers_and_descr(cpu, mem_loc, frame_pointer, registers, vfp_registers): + # XXX rewrite """Decode locations encoded in memory at mem_loc and write the values to the failboxes. Values for spilled vars and registers are stored on stack at frame_loc """ @@ -539,9 +527,10 @@ DESCR_FLOAT = 0x02 DESCR_SPECIAL = 0x03 CODE_FROMSTACK = 64 - CODE_STOP = 0 | DESCR_SPECIAL - CODE_HOLE = 4 | DESCR_SPECIAL - CODE_INPUTARG = 8 | DESCR_SPECIAL + CODE_STOP = 0 | DESCR_SPECIAL + CODE_HOLE = 4 | DESCR_SPECIAL + CODE_INPUTARG = 8 | DESCR_SPECIAL + CODE_FORCED = 12 | DESCR_SPECIAL #XXX where should this be written? def gen_descr_encoding(self, descr, failargs, locs): assert self.mc is not None @@ -579,9 +568,6 @@ self.mc.write32(fdescr) self.align() - # assert that the fail_boxes lists are big enough - assert len(failargs) <= self.fail_boxes_int.SIZE - def _gen_path_to_exit_path(self, descr, args, arglocs, save_exc, fcond=c.AL): assert isinstance(save_exc, bool) @@ -860,7 +846,7 @@ descr._arm_failure_recovery_block = failure_recovery_pos relative_offset = tok.pos_recovery_stub - tok.offset guard_pos = block_start + tok.offset - if not tok.is_invalidate: + if not tok.is_guard_not_invalidated: # patch the guard jumpt to the stub # overwrite the generate NOP with a B_offs to the pos of the # stub @@ -1255,11 +1241,6 @@ else: raise AssertionError('Trying to pop to an invalid location') - def leave_jitted_hook(self): - ptrs = self.fail_boxes_ptr.ar - llop.gc_assume_young_pointers(lltype.Void, - llmemory.cast_ptr_to_adr(ptrs)) - def malloc_cond(self, nursery_free_adr, nursery_top_adr, size): assert size & (WORD-1) == 0 # must be correctly aligned diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py --- a/pypy/jit/backend/arm/opassembler.py +++ b/pypy/jit/backend/arm/opassembler.py @@ -33,12 +33,13 @@ class GuardToken(object): - def __init__(self, descr, failargs, faillocs, offset, - save_exc, fcond=c.AL, is_invalidate=False): + def __init__(self, descr, failargs, faillocs, offset, save_exc, fcond=c.AL, + is_guard_not_invalidated=False, is_guard_not_forced=False): assert isinstance(save_exc, bool) self.descr = descr self.offset = offset - self.is_invalidate = is_invalidate + self.is_guard_not_invalidated = is_guard_not_invalidated + self.is_guard_not_forced = is_guard_not_forced self.failargs = failargs self.faillocs = faillocs self.save_exc = save_exc @@ -190,7 +191,8 @@ return fcond def _emit_guard(self, op, arglocs, fcond, save_exc, - is_guard_not_invalidated=False): + is_guard_not_invalidated=False, + is_guard_not_forced=False): assert isinstance(save_exc, bool) assert isinstance(fcond, int) descr = op.getdescr() @@ -210,7 +212,8 @@ faillocs=arglocs, offset=pos, save_exc=save_exc, - is_invalidate=is_guard_not_invalidated, + is_guard_not_invalidated=is_guard_not_invalidated, + is_guard_not_forced=is_guard_not_forced, fcond=fcond)) return c.AL @@ -312,49 +315,11 @@ return fcond def emit_op_finish(self, op, arglocs, regalloc, fcond): - for i in range(len(arglocs) - 1): - loc = arglocs[i] - box = op.getarg(i) - if loc is None: - continue - if loc.is_reg(): - if box.type == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(i) - elif box.type == INT: - adr = self.fail_boxes_int.get_addr_for_num(i) - else: - assert 0 - self.mc.gen_load_int(r.ip.value, adr) - self.mc.STR_ri(loc.value, r.ip.value) - elif loc.is_vfp_reg(): - assert box.type == FLOAT - adr = self.fail_boxes_float.get_addr_for_num(i) - self.mc.gen_load_int(r.ip.value, adr) - self.mc.VSTR(loc.value, r.ip.value) - elif loc.is_stack() or loc.is_imm() or loc.is_imm_float(): - if box.type == FLOAT: - adr = self.fail_boxes_float.get_addr_for_num(i) - self.mov_loc_loc(loc, r.vfp_ip) - self.mc.gen_load_int(r.ip.value, adr) - self.mc.VSTR(r.vfp_ip.value, r.ip.value) - elif box.type == REF or box.type == INT: - if box.type == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(i) - elif box.type == INT: - adr = self.fail_boxes_int.get_addr_for_num(i) - else: - assert 0 - self.mov_loc_loc(loc, r.ip) - self.mc.gen_load_int(r.lr.value, adr) - self.mc.STR_ri(r.ip.value, r.lr.value) - else: - assert 0 - # note: no exception should currently be set in llop.get_exception_addr - # even if this finish may be an exit_frame_with_exception (in this case - # the exception instance is in arglocs[0]). - addr = self.cpu.get_on_leave_jitted_int(save_exception=False) - self.mc.BL(addr) - self.mc.gen_load_int(r.r0.value, arglocs[-1].value) + [argloc] = arglocs + if argloc is not r.r0: #XXX verify this + self.mov(argloc, eax) + self.mov_loc_loc(arg_loc, r.r0, fcond) + # exit function self.gen_func_epilog() return fcond @@ -1266,7 +1231,8 @@ self.mc.LDR_ri(r.ip.value, r.fp.value) self.mc.CMP_ri(r.ip.value, 0) - self._emit_guard(guard_op, arglocs[1 + numargs:], c.GE, save_exc=True) + self._emit_guard(guard_op, arglocs[1 + numargs:], c.GE, + save_exc=True, is_guard_not_forced=True) return fcond def emit_guard_call_release_gil(self, op, guard_op, arglocs, regalloc, diff --git a/pypy/jit/backend/arm/regalloc.py b/pypy/jit/backend/arm/regalloc.py --- a/pypy/jit/backend/arm/regalloc.py +++ b/pypy/jit/backend/arm/regalloc.py @@ -654,15 +654,9 @@ return args def prepare_op_finish(self, op, fcond): - args = [None] * (op.numargs() + 1) - for i in range(op.numargs()): - arg = op.getarg(i) - if arg: - args[i] = self.loc(arg) - self.possibly_free_var(arg) - n = self.cpu.get_fail_descr_number(op.getdescr()) - args[-1] = imm(n) - return args + loc = self.loc(op.getarg(0)) + self.possibly_free_var(op.getarg(0)) + return [loc] def prepare_op_guard_true(self, op, fcond): l0 = self._ensure_value_is_boxed(op.getarg(0)) diff --git a/pypy/jit/backend/arm/runner.py b/pypy/jit/backend/arm/runner.py --- a/pypy/jit/backend/arm/runner.py +++ b/pypy/jit/backend/arm/runner.py @@ -23,15 +23,21 @@ AbstractLLCPU.__init__(self, rtyper, stats, opts, translate_support_code, gcdescr) + from pypy.jit.backend.llsupport import jitframe + self.deadframe_size_max = llmemory.sizeof(jitframe.DEADFRAME, + self.get_failargs_limit()) + def set_debug(self, flag): return self.assembler.set_debug(flag) + def get_failargs_limit(self): + if self.opts is not None: + return self.opts.failargs_limit + else: + return 1000 + def setup(self): - if self.opts is not None: - failargs_limit = self.opts.failargs_limit - else: - failargs_limit = 1000 - self.assembler = AssemblerARM(self, failargs_limit=failargs_limit) + self.assembler = AssemblerARM(self, self.translate_support_code) def setup_once(self): self.assembler.setup_once() @@ -51,24 +57,6 @@ return self.assembler.assemble_bridge(faildescr, inputargs, operations, original_loop_token, log=log) - def get_latest_value_float(self, index): - return self.assembler.fail_boxes_float.getitem(index) - - def get_latest_value_int(self, index): - return self.assembler.fail_boxes_int.getitem(index) - - def get_latest_value_ref(self, index): - return self.assembler.fail_boxes_ptr.getitem(index) - - def get_latest_value_count(self): - return self.assembler.fail_boxes_count - - def get_latest_force_token(self): - return self.assembler.fail_force_index - - def get_on_leave_jitted_hook(self): - return self.assembler.leave_jitted_hook - def clear_latest_values(self, count): setitem = self.assembler.fail_boxes_ptr.setitem null = lltype.nullptr(llmemory.GCREF.TO) @@ -76,7 +64,7 @@ setitem(index, null) def make_execute_token(self, *ARGS): - FUNCPTR = lltype.Ptr(lltype.FuncType(ARGS, lltype.Signed)) + FUNCPTR = lltype.Ptr(lltype.FuncType(ARGS, llmemory.GCREF)) def execute_token(executable_token, *args): clt = executable_token.compiled_loop_token @@ -91,12 +79,13 @@ prev_interpreter = LLInterpreter.current_interpreter LLInterpreter.current_interpreter = self.debug_ll_interpreter try: - fail_index = func(*args) + deadframe = func(*args) finally: if not self.translate_support_code: LLInterpreter.current_interpreter = prev_interpreter #llop.debug_print(lltype.Void, "<<<< Back") - return self.get_fail_descr_from_number(fail_index) + self.gc_set_extra_threshold() + return deadframe return execute_token def cast_ptr_to_int(x): @@ -110,6 +99,7 @@ flavor='raw', zero=True, immortal=True) def force(self, addr_of_force_index): + assert 0, 'refactor ME' TP = rffi.CArrayPtr(lltype.Signed) fail_index = rffi.cast(TP, addr_of_force_index)[0] assert fail_index >= 0, "already forced!" From noreply at buildbot.pypy.org Thu Jan 3 12:51:49 2013 From: noreply at buildbot.pypy.org (bivab) Date: Thu, 3 Jan 2013 12:51:49 +0100 (CET) Subject: [pypy-commit] pypy remove-globals-in-jit: (wip) continue porting deadframe creation Message-ID: <20130103115149.D3ED21C078F@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: remove-globals-in-jit Changeset: r59672:4494a20539bb Date: 2012-12-21 15:05 +0100 http://bitbucket.org/pypy/pypy/changeset/4494a20539bb/ Log: (wip) continue porting deadframe creation diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py --- a/pypy/jit/backend/arm/assembler.py +++ b/pypy/jit/backend/arm/assembler.py @@ -1,5 +1,6 @@ from __future__ import with_statement import os +from pypy.jit.backend.llsupport import jitframe from pypy.jit.backend.arm.helper.assembler import saved_registers from pypy.jit.backend.arm import conditions as c from pypy.jit.backend.arm import registers as r @@ -326,7 +327,7 @@ def setup_failure_recovery(self): - @rgc.no_collect + #@rgc.no_collect -- XXX still true, but hacked gc_set_extra_threshold def failure_recovery_func(mem_loc, frame_pointer, stack_pointer): """mem_loc is a structure in memory describing where the values for the failargs are stored. frame loc is the address of the frame @@ -345,21 +346,62 @@ @staticmethod def decode_registers_and_descr(cpu, mem_loc, frame_pointer, registers, vfp_registers): - # XXX rewrite - """Decode locations encoded in memory at mem_loc and write the values - to the failboxes. Values for spilled vars and registers are stored on - stack at frame_loc """ - assert frame_pointer & 1 == 0 - self.fail_force_index = frame_pointer + # no malloc allowed here!! xxx apart from one, hacking a lot + #self.fail_ebp = allregisters[16 + ebp.value] + num = 0 + deadframe = lltype.nullptr(jitframe.DEADFRAME) bytecode = rffi.cast(rffi.UCHARP, mem_loc) + # step 1: lots of mess just to count the final value of 'num' + bytecode1 = bytecode + while 1: + code = rffi.cast(lltype.Signed, bytecode1[0]) + bytecode1 = rffi.ptradd(bytecode1, 1) + if code >= AssemblerARM.CODE_FROMSTACK: + while code > 0x7F: + code = rffi.cast(lltype.Signed, bytecode1[0]) + bytecode1 = rffi.ptradd(bytecode1, 1) + else: + kind = code & 3 + if kind == AssemblerARM.DESCR_SPECIAL: + if code == AssemblerARM.CODE_HOLE: + num += 1 + continue + if code == AssemblerARM.CODE_INPUTARG: + continue + if code == AssemblerARM.CODE_FORCED: + # resuming from a GUARD_NOT_FORCED + xxx + token = allregisters[16 + ebp.value] + deadframe = ( + cpu.assembler.force_token_to_dead_frame.pop(token)) + deadframe = lltype.cast_opaque_ptr( + jitframe.DEADFRAMEPTR, deadframe) + continue + assert code == AssemblerARM.CODE_STOP + break + num += 1 + + # allocate the deadframe + if not deadframe: + # Remove the "reserve" at the end of the nursery. This means + # that it is guaranteed that the following malloc() works + # without requiring a collect(), but it needs to be re-added + # as soon as possible. + cpu.gc_clear_extra_threshold() + assert num <= cpu.get_failargs_limit() + try: + deadframe = lltype.malloc(jitframe.DEADFRAME, num) + except MemoryError: + fatalerror("memory usage error in grab_frame_values") + # fill it + code_inputarg = False num = 0 - value = 0 - fvalue = 0 - code_inputarg = False - while True: + value_hi = 0 + while 1: + # decode the next instruction from the bytecode code = rffi.cast(lltype.Signed, bytecode[0]) bytecode = rffi.ptradd(bytecode, 1) - if code >= self.CODE_FROMSTACK: + if code >= AssemblerARM.CODE_FROMSTACK: if code > 0x7F: shift = 7 code &= 0x7F @@ -372,54 +414,63 @@ break # load the value from the stack kind = code & 3 - code = int((code - self.CODE_FROMSTACK) >> 2) + code = (code - AssemblerARM.CODE_FROMSTACK) >> 2 if code_inputarg: code = ~code code_inputarg = False - if kind == self.DESCR_FLOAT: - # we use code + 1 to get the hi word of the double worded float - stackloc = frame_pointer - get_fp_offset(int(code) + 1) - assert stackloc & 3 == 0 - fvalue = rffi.cast(rffi.LONGLONGP, stackloc)[0] - else: - stackloc = frame_pointer - get_fp_offset(int(code)) - assert stackloc & 1 == 0 - value = rffi.cast(rffi.LONGP, stackloc)[0] + stackloc = frame_addr + get_fp_offset(int(code)) + value = rffi.cast(rffi.LONGP, stackloc)[0] + if kind == AssemblerARM.DESCR_FLOAT: + assert WORD == 4 + value_hi = value + value = rffi.cast(rffi.LONGP, stackloc - WORD)[0] else: - # 'code' identifies a register: load its value kind = code & 3 - if kind == self.DESCR_SPECIAL: - if code == self.CODE_HOLE: + if kind == AssemblerARM.DESCR_SPECIAL: + if code == AssemblerARM.CODE_HOLE: num += 1 continue - if code == self.CODE_INPUTARG: + if code == AssemblerARM.CODE_INPUTARG: code_inputarg = True continue - assert code == self.CODE_STOP + if code == AssemblerARM.CODE_FORCED: + continue + assert code == AssemblerARM.CODE_STOP break + # 'code' identifies a register: load its value code >>= 2 - if kind == self.DESCR_FLOAT: - fvalue = vfp_registers[code] + if kind == AssemblerARM.DESCR_FLOAT: + if WORD == 4: + value = vfp_registers[2*code] + value_hi = vfp_registers[2*code + 1] + else: + value = allregisters[code] else: value = registers[code] # store the loaded value into fail_boxes_ - if kind == self.DESCR_FLOAT: - tgt = self.fail_boxes_float.get_addr_for_num(num) - rffi.cast(rffi.LONGLONGP, tgt)[0] = fvalue + if kind == AssemblerARM.DESCR_INT: + deadframe.jf_values[num].int = value + elif kind == AssemblerARM.DESCR_REF: + deadframe.jf_values[num].ref = rffi.cast(llmemory.GCREF, value) + elif kind == AssemblerARM.DESCR_FLOAT: + assert WORD == 4 + assert not longlong.is_64_bit + floatvalue = rffi.cast(lltype.SignedLongLong, value_hi) + floatvalue <<= 32 + floatvalue |= rffi.cast(lltype.SignedLongLong, + rffi.cast(lltype.Unsigned, value)) + deadframe.jf_values[num].float = floatvalue else: - if kind == self.DESCR_INT: - tgt = self.fail_boxes_int.get_addr_for_num(num) - elif kind == self.DESCR_REF: - assert (value & 3) == 0, "misaligned pointer" - tgt = self.fail_boxes_ptr.get_addr_for_num(num) - else: - assert 0, "bogus kind" - rffi.cast(rffi.LONGP, tgt)[0] = value + assert 0, "bogus kind" num += 1 - self.fail_boxes_count = num + # + assert num == len(deadframe.jf_values) + if not we_are_translated(): + assert bytecode[4] == 0xCC fail_index = rffi.cast(rffi.INTP, bytecode)[0] - fail_index = rffi.cast(lltype.Signed, fail_index) - return fail_index + fail_descr = cpu.get_fail_descr_from_number(fail_index) + deadframe.jf_descr = fail_descr.hide(cpu) + return lltype.cast_opaque_ptr(llmemory.GCREF, deadframe) def decode_inputargs(self, code): descr_to_box_type = [REF, INT, FLOAT] @@ -508,6 +559,16 @@ self.failure_recovery_func) self._insert_checks(mc) with saved_registers(mc, r.all_regs, r.all_vfp_regs): + if exc: + # We might have an exception pending. Load it into r4 + # (this is a register saved across calls) + mc.gen_load_int(r.r5.value, self.cpu.pos_exc_value()) + mc.LDR_ri(r.r4.value, self.cpu.pos_exc_value()) + # clear the exc flags + mc.gen_load_int(r.r6.value, 0) + mc.STR_ri(r.r6.value, r.r5.value) + mc.gen_load_int(r.r5.value, self.cpu.pos_exception()) + mc.STR_ri(r.r6.value, r.r5.value) # move mem block address, to r0 to pass as mc.MOV_rr(r.r0.value, r.lr.value) # pass the current frame pointer as second param @@ -516,6 +577,12 @@ mc.MOV_rr(r.r2.value, r.sp.value) self._insert_checks(mc) mc.BL(rffi.cast(lltype.Signed, decode_registers_addr)) + if exc: + # save ebx into 'jf_guard_exc' + from pypy.jit.backend.llsupport.descr import unpack_fielddescr + descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) + offset, size, _ = unpack_fielddescr(descrs.jf_guard_exc) + mc.STR_rr(r.r4.value, r.r0.value, offset, cond=c.AL) mc.MOV_rr(r.ip.value, r.r0.value) mc.MOV_rr(r.r0.value, r.ip.value) self.gen_func_epilog(mc=mc) From noreply at buildbot.pypy.org Thu Jan 3 12:51:51 2013 From: noreply at buildbot.pypy.org (bivab) Date: Thu, 3 Jan 2013 12:51:51 +0100 (CET) Subject: [pypy-commit] pypy remove-globals-in-jit: refactor to follow more closely the x86 backend structure Message-ID: <20130103115151.07E991C078F@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: remove-globals-in-jit Changeset: r59673:ccc1942a4a7d Date: 2012-12-21 15:06 +0100 http://bitbucket.org/pypy/pypy/changeset/ccc1942a4a7d/ Log: refactor to follow more closely the x86 backend structure diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py --- a/pypy/jit/backend/arm/assembler.py +++ b/pypy/jit/backend/arm/assembler.py @@ -108,9 +108,13 @@ gc_ll_descr.initialize() self._build_wb_slowpath(False) self._build_wb_slowpath(True) + self._build_failure_recovery(exc=True, withfloats=False) + self._build_failure_recovery(exc=False, withfloats=False) if self.cpu.supports_floats: self._build_wb_slowpath(False, withfloats=True) self._build_wb_slowpath(True, withfloats=True) + self._build_failure_recovery(exc=True, withfloats=True) + self._build_failure_recovery(exc=False, withfloats=True) self._build_propagate_exception_path() if gc_ll_descr.get_malloc_slowpath_addr is not None: self._build_malloc_slowpath() @@ -118,7 +122,7 @@ if gc_ll_descr.gcrootmap and gc_ll_descr.gcrootmap.is_shadow_stack: self._build_release_gil(gc_ll_descr.gcrootmap) self.memcpy_addr = self.cpu.cast_ptr_to_int(memcpy_fn) - self._exit_code_addr = self._gen_exit_path() + if not self._debug: # if self._debug is already set it means that someone called # set_debug by hand before initializing the assembler. Leave it @@ -213,20 +217,6 @@ self.releasegil_addr = rffi.cast(lltype.Signed, releasegil_func) self.reacqgil_addr = rffi.cast(lltype.Signed, reacqgil_func) - def _gen_leave_jitted_hook_code(self, save_exc): - mc = ARMv7Builder() - if self.cpu.supports_floats: - floats = r.caller_vfp_resp - else: - floats = [] - with saved_registers(mc, r.caller_resp + [r.lr], floats): - addr = self.cpu.get_on_leave_jitted_int(save_exception=save_exc) - mc.BL(addr) - assert self._exit_code_addr != 0 - mc.B(self._exit_code_addr) - return mc.materialize(self.cpu.asmmemmgr, [], - self.cpu.gc_ll_descr.gcrootmap) - def _build_propagate_exception_path(self): if self.cpu.propagate_exception_v < 0: return # not supported (for tests, or non-translated) @@ -332,15 +322,16 @@ """mem_loc is a structure in memory describing where the values for the failargs are stored. frame loc is the address of the frame pointer for the frame to be decoded frame """ - vfp_registers = rffi.cast(rffi.LONGLONGP, stack_pointer) - registers = rffi.ptradd(vfp_registers, len(r.all_vfp_regs)) + vfp_registers = rffi.cast(rffi.LONGP, stack_pointer) + registers = rffi.ptradd(vfp_registers, 2*len(r.all_vfp_regs)) registers = rffi.cast(rffi.LONGP, registers) return self.decode_registers_and_descr(self.cpu, mem_loc, frame_pointer, registers, vfp_registers) + self.failure_recovery_code = [0, 0, 0, 0] self.failure_recovery_func = failure_recovery_func - recovery_func_sign = lltype.Ptr(lltype.FuncType([lltype.Signed] * 3, + _FAILURE_RECOVERY_FUNC = lltype.Ptr(lltype.FuncType([rffi.LONGP] * 3, llmemory.GCREF)) @staticmethod @@ -553,12 +544,16 @@ self.mc.CMP_ri(r.r0.value, 0) self.mc.B(self.propagate_exception_path, c=c.EQ) - def _gen_exit_path(self): + def _build_failure_recovery(self, exc, withfloats=False): mc = ARMv7Builder() - decode_registers_addr = llhelper(self.recovery_func_sign, + decode_registers_addr = llhelper(self._FAILURE_RECOVERY_FUNC, self.failure_recovery_func) self._insert_checks(mc) - with saved_registers(mc, r.all_regs, r.all_vfp_regs): + if withfloats: + f = r.all_vfp_regs + else: + f = [] + with saved_registers(mc, r.all_regs, f): if exc: # We might have an exception pending. Load it into r4 # (this is a register saved across calls) @@ -586,8 +581,10 @@ mc.MOV_rr(r.ip.value, r.r0.value) mc.MOV_rr(r.r0.value, r.ip.value) self.gen_func_epilog(mc=mc) - return mc.materialize(self.cpu.asmmemmgr, [], + rawstart = mc.materialize(self.cpu.asmmemmgr, [], self.cpu.gc_ll_descr.gcrootmap) + self.failure_recovery_code[exc + 2 * withfloats] = rawstart + self.mc = None DESCR_REF = 0x00 DESCR_INT = 0x01 @@ -599,7 +596,7 @@ CODE_INPUTARG = 8 | DESCR_SPECIAL CODE_FORCED = 12 | DESCR_SPECIAL #XXX where should this be written? - def gen_descr_encoding(self, descr, failargs, locs): + def write_failure_recovery_description(self, descr, failargs, locs): assert self.mc is not None for i in range(len(failargs)): arg = failargs[i] @@ -631,23 +628,33 @@ self.mc.writechar(chr(n)) self.mc.writechar(chr(self.CODE_STOP)) - fdescr = self.cpu.get_fail_descr_number(descr) - self.mc.write32(fdescr) + + def generate_quick_failure(self, guardtok, fcond=c.AL): + assert isinstance(guardtok.save_exc, bool) + fail_index = self.cpu.get_fail_descr_number(guardtok.descr) + startpos = self.mc.currpos() + withfloats = False + for box in guardtok.failargs: + if box is not None and box.type == FLOAT: + withfloats = True + break + exc = guardtok.save_exc + target = self.failure_recovery_code[exc + 2 * withfloats] + assert target != 0 + self.mc.BL(target) + # write tight data that describes the failure recovery + if guardtok.is_guard_not_forced: + mc.writechar(chr(self.CODE_FORCED)) + self.write_failure_recovery_description(guardtok.descr, + guardtok.failargs, guardtok.faillocs[1:]) + self.mc.write32(fail_index) + # for testing the decoding, write a final byte 0xCC + if not we_are_translated(): + self.mc.writechar('\xCC') + faillocs = [loc for loc in guardtok.faillocs if loc is not None] + guardtok.descr._arm_debug_faillocs = faillocs self.align() - - def _gen_path_to_exit_path(self, descr, args, arglocs, - save_exc, fcond=c.AL): - assert isinstance(save_exc, bool) - self.gen_exit_code(self.mc, save_exc, fcond) - self.gen_descr_encoding(descr, args, arglocs[1:]) - - def gen_exit_code(self, mc, save_exc, fcond=c.AL): - assert isinstance(save_exc, bool) - if save_exc: - path = self._leave_jitted_hook_save_exc - else: - path = self._leave_jitted_hook - mc.BL(path) + return startpos def align(self): while(self.mc.currpos() % FUNC_ALIGN != 0): @@ -894,15 +901,10 @@ def write_pending_failure_recoveries(self): for tok in self.pending_guards: - descr = tok.descr #generate the exit stub and the encoded representation - pos = self.mc.currpos() - tok.pos_recovery_stub = pos - - self._gen_path_to_exit_path(descr, tok.failargs, - tok.faillocs, save_exc=tok.save_exc) + tok.pos_recovery_stub = self.generate_quick_failure(tok) # store info on the descr - descr._arm_current_frame_depth = tok.faillocs[0].getint() + tok.descr._arm_current_frame_depth = tok.faillocs[0].getint() def process_pending_guards(self, block_start): clt = self.current_clt From noreply at buildbot.pypy.org Thu Jan 3 12:51:52 2013 From: noreply at buildbot.pypy.org (bivab) Date: Thu, 3 Jan 2013 12:51:52 +0100 (CET) Subject: [pypy-commit] pypy remove-globals-in-jit: update test Message-ID: <20130103115152.263801C078F@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: remove-globals-in-jit Changeset: r59674:89c0a9d2f482 Date: 2012-12-28 18:50 +0100 http://bitbucket.org/pypy/pypy/changeset/89c0a9d2f482/ Log: update test diff --git a/pypy/jit/backend/arm/test/test_runner.py b/pypy/jit/backend/arm/test/test_runner.py --- a/pypy/jit/backend/arm/test/test_runner.py +++ b/pypy/jit/backend/arm/test/test_runner.py @@ -202,8 +202,8 @@ ARGS = [lltype.Signed] * numargs RES = lltype.Signed args = [i+1 for i in range(numargs)] - res = self.cpu.execute_token(looptoken, *args) - assert self.cpu.get_latest_value_int(0) == sum(args) + deadframe = self.cpu.execute_token(looptoken, *args) + assert self.cpu.get_latest_value_int(deadframe, 0) == sum(args) def test_debugger_on(self): from pypy.rlib import debug From noreply at buildbot.pypy.org Thu Jan 3 12:51:53 2013 From: noreply at buildbot.pypy.org (bivab) Date: Thu, 3 Jan 2013 12:51:53 +0100 (CET) Subject: [pypy-commit] pypy remove-globals-in-jit: merge heads Message-ID: <20130103115153.508471C078F@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: remove-globals-in-jit Changeset: r59675:f7e4a76050c6 Date: 2012-12-31 14:22 +0100 http://bitbucket.org/pypy/pypy/changeset/f7e4a76050c6/ Log: merge heads diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py --- a/pypy/jit/backend/arm/assembler.py +++ b/pypy/jit/backend/arm/assembler.py @@ -1,5 +1,6 @@ from __future__ import with_statement import os +from pypy.jit.backend.llsupport import jitframe from pypy.jit.backend.arm.helper.assembler import saved_registers from pypy.jit.backend.arm import conditions as c from pypy.jit.backend.arm import registers as r @@ -30,7 +31,7 @@ from pypy.rlib.objectmodel import compute_unique_id # XXX Move to llsupport -from pypy.jit.backend.x86.support import values_array, memcpy_fn +from pypy.jit.backend.x86.support import memcpy_fn DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', ('i', lltype.Signed), ('type', lltype.Char), # 'b'ridge, 'l'abel or @@ -44,14 +45,8 @@ debug = True - def __init__(self, cpu, failargs_limit=1000): + def __init__(self, cpu, translate_support_code=False): self.cpu = cpu - self.fail_boxes_int = values_array(lltype.Signed, failargs_limit) - self.fail_boxes_float = values_array(longlong.FLOATSTORAGE, - failargs_limit) - self.fail_boxes_ptr = values_array(llmemory.GCREF, failargs_limit) - self.fail_boxes_count = 0 - self.fail_force_index = 0 self.setup_failure_recovery() self.mc = None self.memcpy_addr = 0 @@ -68,6 +63,7 @@ self._debug = False self.loop_run_counters = [] self.debug_counter_descr = cpu.fielddescrof(DEBUG_COUNTER, 'i') + self.force_token_to_dead_frame = {} # XXX temporary hack def set_debug(self, v): r = self._debug @@ -112,9 +108,13 @@ gc_ll_descr.initialize() self._build_wb_slowpath(False) self._build_wb_slowpath(True) + self._build_failure_recovery(exc=True, withfloats=False) + self._build_failure_recovery(exc=False, withfloats=False) if self.cpu.supports_floats: self._build_wb_slowpath(False, withfloats=True) self._build_wb_slowpath(True, withfloats=True) + self._build_failure_recovery(exc=True, withfloats=True) + self._build_failure_recovery(exc=False, withfloats=True) self._build_propagate_exception_path() if gc_ll_descr.get_malloc_slowpath_addr is not None: self._build_malloc_slowpath() @@ -122,10 +122,7 @@ if gc_ll_descr.gcrootmap and gc_ll_descr.gcrootmap.is_shadow_stack: self._build_release_gil(gc_ll_descr.gcrootmap) self.memcpy_addr = self.cpu.cast_ptr_to_int(memcpy_fn) - self._exit_code_addr = self._gen_exit_path() - self._leave_jitted_hook_save_exc = \ - self._gen_leave_jitted_hook_code(True) - self._leave_jitted_hook = self._gen_leave_jitted_hook_code(False) + if not self._debug: # if self._debug is already set it means that someone called # set_debug by hand before initializing the assembler. Leave it @@ -220,36 +217,17 @@ self.releasegil_addr = rffi.cast(lltype.Signed, releasegil_func) self.reacqgil_addr = rffi.cast(lltype.Signed, reacqgil_func) - def _gen_leave_jitted_hook_code(self, save_exc): - mc = ARMv7Builder() - if self.cpu.supports_floats: - floats = r.caller_vfp_resp - else: - floats = [] - with saved_registers(mc, r.caller_resp + [r.lr], floats): - addr = self.cpu.get_on_leave_jitted_int(save_exception=save_exc) - mc.BL(addr) - assert self._exit_code_addr != 0 - mc.B(self._exit_code_addr) - return mc.materialize(self.cpu.asmmemmgr, [], - self.cpu.gc_ll_descr.gcrootmap) - def _build_propagate_exception_path(self): if self.cpu.propagate_exception_v < 0: return # not supported (for tests, or non-translated) # mc = ARMv7Builder() - # call on_leave_jitted_save_exc() - if self.cpu.supports_floats: - floats = r.caller_vfp_resp - else: - floats = [] - with saved_registers(mc, r.caller_resp + [r.lr], floats): - addr = self.cpu.get_on_leave_jitted_int(save_exception=True, - default_to_memoryerror=True) - mc.BL(addr) - mc.gen_load_int(r.ip.value, self.cpu.propagate_exception_v) - mc.MOV_rr(r.r0.value, r.ip.value) + # + # Call the helper, which will return a dead frame object with + # the correct exception set, or MemoryError by default + # XXX make sure we return the correct value here + addr = rffi.cast(lltype.Signed, self.cpu.get_propagate_exception()) + mc.BL(addr) self.gen_func_epilog(mc=mc) self.propagate_exception_path = mc.materialize(self.cpu.asmmemmgr, []) @@ -282,12 +260,12 @@ # restore registers and return # We check for c.EQ here, meaning all bits zero in this case mc.POP([reg.value for reg in r.argument_regs] + [r.pc.value], cond=c.EQ) - # call on_leave_jitted_save_exc() - addr = self.cpu.get_on_leave_jitted_int(save_exception=True) + # + # Call the helper, which will return a dead frame object with + # the correct exception set, or MemoryError by default + addr = rffi.cast(lltype.Signed, self.cpu.get_propagate_exception()) mc.BL(addr) # - mc.gen_load_int(r.r0.value, self.cpu.propagate_exception_v) - # # footer -- note the ADD, which skips the return address of this # function, and will instead return to the caller's caller. Note # also that we completely ignore the saved arguments, because we @@ -339,39 +317,82 @@ def setup_failure_recovery(self): - @rgc.no_collect + #@rgc.no_collect -- XXX still true, but hacked gc_set_extra_threshold def failure_recovery_func(mem_loc, frame_pointer, stack_pointer): """mem_loc is a structure in memory describing where the values for the failargs are stored. frame loc is the address of the frame pointer for the frame to be decoded frame """ - vfp_registers = rffi.cast(rffi.LONGLONGP, stack_pointer) - registers = rffi.ptradd(vfp_registers, len(r.all_vfp_regs)) + vfp_registers = rffi.cast(rffi.LONGP, stack_pointer) + registers = rffi.ptradd(vfp_registers, 2*len(r.all_vfp_regs)) registers = rffi.cast(rffi.LONGP, registers) - return self.decode_registers_and_descr(mem_loc, frame_pointer, + return self.decode_registers_and_descr(self.cpu, mem_loc, frame_pointer, registers, vfp_registers) + self.failure_recovery_code = [0, 0, 0, 0] self.failure_recovery_func = failure_recovery_func - recovery_func_sign = lltype.Ptr(lltype.FuncType([lltype.Signed] * 3, - lltype.Signed)) + _FAILURE_RECOVERY_FUNC = lltype.Ptr(lltype.FuncType([rffi.LONGP] * 3, + llmemory.GCREF)) - @rgc.no_collect - def decode_registers_and_descr(self, mem_loc, frame_pointer, + @staticmethod + def decode_registers_and_descr(cpu, mem_loc, frame_pointer, registers, vfp_registers): - """Decode locations encoded in memory at mem_loc and write the values - to the failboxes. Values for spilled vars and registers are stored on - stack at frame_loc """ - assert frame_pointer & 1 == 0 - self.fail_force_index = frame_pointer + # no malloc allowed here!! xxx apart from one, hacking a lot + #self.fail_ebp = allregisters[16 + ebp.value] + num = 0 + deadframe = lltype.nullptr(jitframe.DEADFRAME) bytecode = rffi.cast(rffi.UCHARP, mem_loc) + # step 1: lots of mess just to count the final value of 'num' + bytecode1 = bytecode + while 1: + code = rffi.cast(lltype.Signed, bytecode1[0]) + bytecode1 = rffi.ptradd(bytecode1, 1) + if code >= AssemblerARM.CODE_FROMSTACK: + while code > 0x7F: + code = rffi.cast(lltype.Signed, bytecode1[0]) + bytecode1 = rffi.ptradd(bytecode1, 1) + else: + kind = code & 3 + if kind == AssemblerARM.DESCR_SPECIAL: + if code == AssemblerARM.CODE_HOLE: + num += 1 + continue + if code == AssemblerARM.CODE_INPUTARG: + continue + if code == AssemblerARM.CODE_FORCED: + # resuming from a GUARD_NOT_FORCED + xxx + token = allregisters[16 + ebp.value] + deadframe = ( + cpu.assembler.force_token_to_dead_frame.pop(token)) + deadframe = lltype.cast_opaque_ptr( + jitframe.DEADFRAMEPTR, deadframe) + continue + assert code == AssemblerARM.CODE_STOP + break + num += 1 + + # allocate the deadframe + if not deadframe: + # Remove the "reserve" at the end of the nursery. This means + # that it is guaranteed that the following malloc() works + # without requiring a collect(), but it needs to be re-added + # as soon as possible. + cpu.gc_clear_extra_threshold() + assert num <= cpu.get_failargs_limit() + try: + deadframe = lltype.malloc(jitframe.DEADFRAME, num) + except MemoryError: + fatalerror("memory usage error in grab_frame_values") + # fill it + code_inputarg = False num = 0 - value = 0 - fvalue = 0 - code_inputarg = False - while True: + value_hi = 0 + while 1: + # decode the next instruction from the bytecode code = rffi.cast(lltype.Signed, bytecode[0]) bytecode = rffi.ptradd(bytecode, 1) - if code >= self.CODE_FROMSTACK: + if code >= AssemblerARM.CODE_FROMSTACK: if code > 0x7F: shift = 7 code &= 0x7F @@ -384,54 +405,63 @@ break # load the value from the stack kind = code & 3 - code = int((code - self.CODE_FROMSTACK) >> 2) + code = (code - AssemblerARM.CODE_FROMSTACK) >> 2 if code_inputarg: code = ~code code_inputarg = False - if kind == self.DESCR_FLOAT: - # we use code + 1 to get the hi word of the double worded float - stackloc = frame_pointer - get_fp_offset(int(code) + 1) - assert stackloc & 3 == 0 - fvalue = rffi.cast(rffi.LONGLONGP, stackloc)[0] - else: - stackloc = frame_pointer - get_fp_offset(int(code)) - assert stackloc & 1 == 0 - value = rffi.cast(rffi.LONGP, stackloc)[0] + stackloc = frame_addr + get_fp_offset(int(code)) + value = rffi.cast(rffi.LONGP, stackloc)[0] + if kind == AssemblerARM.DESCR_FLOAT: + assert WORD == 4 + value_hi = value + value = rffi.cast(rffi.LONGP, stackloc - WORD)[0] else: - # 'code' identifies a register: load its value kind = code & 3 - if kind == self.DESCR_SPECIAL: - if code == self.CODE_HOLE: + if kind == AssemblerARM.DESCR_SPECIAL: + if code == AssemblerARM.CODE_HOLE: num += 1 continue - if code == self.CODE_INPUTARG: + if code == AssemblerARM.CODE_INPUTARG: code_inputarg = True continue - assert code == self.CODE_STOP + if code == AssemblerARM.CODE_FORCED: + continue + assert code == AssemblerARM.CODE_STOP break + # 'code' identifies a register: load its value code >>= 2 - if kind == self.DESCR_FLOAT: - fvalue = vfp_registers[code] + if kind == AssemblerARM.DESCR_FLOAT: + if WORD == 4: + value = vfp_registers[2*code] + value_hi = vfp_registers[2*code + 1] + else: + value = allregisters[code] else: value = registers[code] # store the loaded value into fail_boxes_ - if kind == self.DESCR_FLOAT: - tgt = self.fail_boxes_float.get_addr_for_num(num) - rffi.cast(rffi.LONGLONGP, tgt)[0] = fvalue + if kind == AssemblerARM.DESCR_INT: + deadframe.jf_values[num].int = value + elif kind == AssemblerARM.DESCR_REF: + deadframe.jf_values[num].ref = rffi.cast(llmemory.GCREF, value) + elif kind == AssemblerARM.DESCR_FLOAT: + assert WORD == 4 + assert not longlong.is_64_bit + floatvalue = rffi.cast(lltype.SignedLongLong, value_hi) + floatvalue <<= 32 + floatvalue |= rffi.cast(lltype.SignedLongLong, + rffi.cast(lltype.Unsigned, value)) + deadframe.jf_values[num].float = floatvalue else: - if kind == self.DESCR_INT: - tgt = self.fail_boxes_int.get_addr_for_num(num) - elif kind == self.DESCR_REF: - assert (value & 3) == 0, "misaligned pointer" - tgt = self.fail_boxes_ptr.get_addr_for_num(num) - else: - assert 0, "bogus kind" - rffi.cast(rffi.LONGP, tgt)[0] = value + assert 0, "bogus kind" num += 1 - self.fail_boxes_count = num + # + assert num == len(deadframe.jf_values) + if not we_are_translated(): + assert bytecode[4] == 0xCC fail_index = rffi.cast(rffi.INTP, bytecode)[0] - fail_index = rffi.cast(lltype.Signed, fail_index) - return fail_index + fail_descr = cpu.get_fail_descr_from_number(fail_index) + deadframe.jf_descr = fail_descr.hide(cpu) + return lltype.cast_opaque_ptr(llmemory.GCREF, deadframe) def decode_inputargs(self, code): descr_to_box_type = [REF, INT, FLOAT] @@ -514,12 +544,26 @@ self.mc.CMP_ri(r.r0.value, 0) self.mc.B(self.propagate_exception_path, c=c.EQ) - def _gen_exit_path(self): + def _build_failure_recovery(self, exc, withfloats=False): mc = ARMv7Builder() - decode_registers_addr = llhelper(self.recovery_func_sign, + decode_registers_addr = llhelper(self._FAILURE_RECOVERY_FUNC, self.failure_recovery_func) self._insert_checks(mc) - with saved_registers(mc, r.all_regs, r.all_vfp_regs): + if withfloats: + f = r.all_vfp_regs + else: + f = [] + with saved_registers(mc, r.all_regs, f): + if exc: + # We might have an exception pending. Load it into r4 + # (this is a register saved across calls) + mc.gen_load_int(r.r5.value, self.cpu.pos_exc_value()) + mc.LDR_ri(r.r4.value, self.cpu.pos_exc_value()) + # clear the exc flags + mc.gen_load_int(r.r6.value, 0) + mc.STR_ri(r.r6.value, r.r5.value) + mc.gen_load_int(r.r5.value, self.cpu.pos_exception()) + mc.STR_ri(r.r6.value, r.r5.value) # move mem block address, to r0 to pass as mc.MOV_rr(r.r0.value, r.lr.value) # pass the current frame pointer as second param @@ -528,22 +572,31 @@ mc.MOV_rr(r.r2.value, r.sp.value) self._insert_checks(mc) mc.BL(rffi.cast(lltype.Signed, decode_registers_addr)) + if exc: + # save ebx into 'jf_guard_exc' + from pypy.jit.backend.llsupport.descr import unpack_fielddescr + descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) + offset, size, _ = unpack_fielddescr(descrs.jf_guard_exc) + mc.STR_rr(r.r4.value, r.r0.value, offset, cond=c.AL) mc.MOV_rr(r.ip.value, r.r0.value) mc.MOV_rr(r.r0.value, r.ip.value) self.gen_func_epilog(mc=mc) - return mc.materialize(self.cpu.asmmemmgr, [], + rawstart = mc.materialize(self.cpu.asmmemmgr, [], self.cpu.gc_ll_descr.gcrootmap) + self.failure_recovery_code[exc + 2 * withfloats] = rawstart + self.mc = None DESCR_REF = 0x00 DESCR_INT = 0x01 DESCR_FLOAT = 0x02 DESCR_SPECIAL = 0x03 CODE_FROMSTACK = 64 - CODE_STOP = 0 | DESCR_SPECIAL - CODE_HOLE = 4 | DESCR_SPECIAL - CODE_INPUTARG = 8 | DESCR_SPECIAL + CODE_STOP = 0 | DESCR_SPECIAL + CODE_HOLE = 4 | DESCR_SPECIAL + CODE_INPUTARG = 8 | DESCR_SPECIAL + CODE_FORCED = 12 | DESCR_SPECIAL #XXX where should this be written? - def gen_descr_encoding(self, descr, failargs, locs): + def write_failure_recovery_description(self, descr, failargs, locs): assert self.mc is not None for i in range(len(failargs)): arg = failargs[i] @@ -575,26 +628,33 @@ self.mc.writechar(chr(n)) self.mc.writechar(chr(self.CODE_STOP)) - fdescr = self.cpu.get_fail_descr_number(descr) - self.mc.write32(fdescr) + + def generate_quick_failure(self, guardtok, fcond=c.AL): + assert isinstance(guardtok.save_exc, bool) + fail_index = self.cpu.get_fail_descr_number(guardtok.descr) + startpos = self.mc.currpos() + withfloats = False + for box in guardtok.failargs: + if box is not None and box.type == FLOAT: + withfloats = True + break + exc = guardtok.save_exc + target = self.failure_recovery_code[exc + 2 * withfloats] + assert target != 0 + self.mc.BL(target) + # write tight data that describes the failure recovery + if guardtok.is_guard_not_forced: + mc.writechar(chr(self.CODE_FORCED)) + self.write_failure_recovery_description(guardtok.descr, + guardtok.failargs, guardtok.faillocs[1:]) + self.mc.write32(fail_index) + # for testing the decoding, write a final byte 0xCC + if not we_are_translated(): + self.mc.writechar('\xCC') + faillocs = [loc for loc in guardtok.faillocs if loc is not None] + guardtok.descr._arm_debug_faillocs = faillocs self.align() - - # assert that the fail_boxes lists are big enough - assert len(failargs) <= self.fail_boxes_int.SIZE - - def _gen_path_to_exit_path(self, descr, args, arglocs, - save_exc, fcond=c.AL): - assert isinstance(save_exc, bool) - self.gen_exit_code(self.mc, save_exc, fcond) - self.gen_descr_encoding(descr, args, arglocs[1:]) - - def gen_exit_code(self, mc, save_exc, fcond=c.AL): - assert isinstance(save_exc, bool) - if save_exc: - path = self._leave_jitted_hook_save_exc - else: - path = self._leave_jitted_hook - mc.BL(path) + return startpos def align(self): while(self.mc.currpos() % FUNC_ALIGN != 0): @@ -841,15 +901,10 @@ def write_pending_failure_recoveries(self): for tok in self.pending_guards: - descr = tok.descr #generate the exit stub and the encoded representation - pos = self.mc.currpos() - tok.pos_recovery_stub = pos - - self._gen_path_to_exit_path(descr, tok.failargs, - tok.faillocs, save_exc=tok.save_exc) + tok.pos_recovery_stub = self.generate_quick_failure(tok) # store info on the descr - descr._arm_current_frame_depth = tok.faillocs[0].getint() + tok.descr._arm_current_frame_depth = tok.faillocs[0].getint() def process_pending_guards(self, block_start): clt = self.current_clt @@ -860,7 +915,7 @@ descr._arm_failure_recovery_block = failure_recovery_pos relative_offset = tok.pos_recovery_stub - tok.offset guard_pos = block_start + tok.offset - if not tok.is_invalidate: + if not tok.is_guard_not_invalidated: # patch the guard jumpt to the stub # overwrite the generate NOP with a B_offs to the pos of the # stub @@ -1255,11 +1310,6 @@ else: raise AssertionError('Trying to pop to an invalid location') - def leave_jitted_hook(self): - ptrs = self.fail_boxes_ptr.ar - llop.gc_assume_young_pointers(lltype.Void, - llmemory.cast_ptr_to_adr(ptrs)) - def malloc_cond(self, nursery_free_adr, nursery_top_adr, size): assert size & (WORD-1) == 0 # must be correctly aligned diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py --- a/pypy/jit/backend/arm/opassembler.py +++ b/pypy/jit/backend/arm/opassembler.py @@ -33,12 +33,13 @@ class GuardToken(object): - def __init__(self, descr, failargs, faillocs, offset, - save_exc, fcond=c.AL, is_invalidate=False): + def __init__(self, descr, failargs, faillocs, offset, save_exc, fcond=c.AL, + is_guard_not_invalidated=False, is_guard_not_forced=False): assert isinstance(save_exc, bool) self.descr = descr self.offset = offset - self.is_invalidate = is_invalidate + self.is_guard_not_invalidated = is_guard_not_invalidated + self.is_guard_not_forced = is_guard_not_forced self.failargs = failargs self.faillocs = faillocs self.save_exc = save_exc @@ -190,7 +191,8 @@ return fcond def _emit_guard(self, op, arglocs, fcond, save_exc, - is_guard_not_invalidated=False): + is_guard_not_invalidated=False, + is_guard_not_forced=False): assert isinstance(save_exc, bool) assert isinstance(fcond, int) descr = op.getdescr() @@ -210,7 +212,8 @@ faillocs=arglocs, offset=pos, save_exc=save_exc, - is_invalidate=is_guard_not_invalidated, + is_guard_not_invalidated=is_guard_not_invalidated, + is_guard_not_forced=is_guard_not_forced, fcond=fcond)) return c.AL @@ -312,49 +315,11 @@ return fcond def emit_op_finish(self, op, arglocs, regalloc, fcond): - for i in range(len(arglocs) - 1): - loc = arglocs[i] - box = op.getarg(i) - if loc is None: - continue - if loc.is_reg(): - if box.type == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(i) - elif box.type == INT: - adr = self.fail_boxes_int.get_addr_for_num(i) - else: - assert 0 - self.mc.gen_load_int(r.ip.value, adr) - self.mc.STR_ri(loc.value, r.ip.value) - elif loc.is_vfp_reg(): - assert box.type == FLOAT - adr = self.fail_boxes_float.get_addr_for_num(i) - self.mc.gen_load_int(r.ip.value, adr) - self.mc.VSTR(loc.value, r.ip.value) - elif loc.is_stack() or loc.is_imm() or loc.is_imm_float(): - if box.type == FLOAT: - adr = self.fail_boxes_float.get_addr_for_num(i) - self.mov_loc_loc(loc, r.vfp_ip) - self.mc.gen_load_int(r.ip.value, adr) - self.mc.VSTR(r.vfp_ip.value, r.ip.value) - elif box.type == REF or box.type == INT: - if box.type == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(i) - elif box.type == INT: - adr = self.fail_boxes_int.get_addr_for_num(i) - else: - assert 0 - self.mov_loc_loc(loc, r.ip) - self.mc.gen_load_int(r.lr.value, adr) - self.mc.STR_ri(r.ip.value, r.lr.value) - else: - assert 0 - # note: no exception should currently be set in llop.get_exception_addr - # even if this finish may be an exit_frame_with_exception (in this case - # the exception instance is in arglocs[0]). - addr = self.cpu.get_on_leave_jitted_int(save_exception=False) - self.mc.BL(addr) - self.mc.gen_load_int(r.r0.value, arglocs[-1].value) + [argloc] = arglocs + if argloc is not r.r0: #XXX verify this + self.mov(argloc, eax) + self.mov_loc_loc(arg_loc, r.r0, fcond) + # exit function self.gen_func_epilog() return fcond @@ -1266,7 +1231,8 @@ self.mc.LDR_ri(r.ip.value, r.fp.value) self.mc.CMP_ri(r.ip.value, 0) - self._emit_guard(guard_op, arglocs[1 + numargs:], c.GE, save_exc=True) + self._emit_guard(guard_op, arglocs[1 + numargs:], c.GE, + save_exc=True, is_guard_not_forced=True) return fcond def emit_guard_call_release_gil(self, op, guard_op, arglocs, regalloc, diff --git a/pypy/jit/backend/arm/regalloc.py b/pypy/jit/backend/arm/regalloc.py --- a/pypy/jit/backend/arm/regalloc.py +++ b/pypy/jit/backend/arm/regalloc.py @@ -644,15 +644,9 @@ return args def prepare_op_finish(self, op, fcond): - args = [None] * (op.numargs() + 1) - for i in range(op.numargs()): - arg = op.getarg(i) - if arg: - args[i] = self.loc(arg) - self.possibly_free_var(arg) - n = self.cpu.get_fail_descr_number(op.getdescr()) - args[-1] = imm(n) - return args + loc = self.loc(op.getarg(0)) + self.possibly_free_var(op.getarg(0)) + return [loc] def prepare_op_guard_true(self, op, fcond): l0 = self.make_sure_var_in_reg(op.getarg(0)) diff --git a/pypy/jit/backend/arm/runner.py b/pypy/jit/backend/arm/runner.py --- a/pypy/jit/backend/arm/runner.py +++ b/pypy/jit/backend/arm/runner.py @@ -23,15 +23,21 @@ AbstractLLCPU.__init__(self, rtyper, stats, opts, translate_support_code, gcdescr) + from pypy.jit.backend.llsupport import jitframe + self.deadframe_size_max = llmemory.sizeof(jitframe.DEADFRAME, + self.get_failargs_limit()) + def set_debug(self, flag): return self.assembler.set_debug(flag) + def get_failargs_limit(self): + if self.opts is not None: + return self.opts.failargs_limit + else: + return 1000 + def setup(self): - if self.opts is not None: - failargs_limit = self.opts.failargs_limit - else: - failargs_limit = 1000 - self.assembler = AssemblerARM(self, failargs_limit=failargs_limit) + self.assembler = AssemblerARM(self, self.translate_support_code) def setup_once(self): self.assembler.setup_once() @@ -51,24 +57,6 @@ return self.assembler.assemble_bridge(faildescr, inputargs, operations, original_loop_token, log=log) - def get_latest_value_float(self, index): - return self.assembler.fail_boxes_float.getitem(index) - - def get_latest_value_int(self, index): - return self.assembler.fail_boxes_int.getitem(index) - - def get_latest_value_ref(self, index): - return self.assembler.fail_boxes_ptr.getitem(index) - - def get_latest_value_count(self): - return self.assembler.fail_boxes_count - - def get_latest_force_token(self): - return self.assembler.fail_force_index - - def get_on_leave_jitted_hook(self): - return self.assembler.leave_jitted_hook - def clear_latest_values(self, count): setitem = self.assembler.fail_boxes_ptr.setitem null = lltype.nullptr(llmemory.GCREF.TO) @@ -76,7 +64,7 @@ setitem(index, null) def make_execute_token(self, *ARGS): - FUNCPTR = lltype.Ptr(lltype.FuncType(ARGS, lltype.Signed)) + FUNCPTR = lltype.Ptr(lltype.FuncType(ARGS, llmemory.GCREF)) def execute_token(executable_token, *args): clt = executable_token.compiled_loop_token @@ -91,12 +79,13 @@ prev_interpreter = LLInterpreter.current_interpreter LLInterpreter.current_interpreter = self.debug_ll_interpreter try: - fail_index = func(*args) + deadframe = func(*args) finally: if not self.translate_support_code: LLInterpreter.current_interpreter = prev_interpreter #llop.debug_print(lltype.Void, "<<<< Back") - return self.get_fail_descr_from_number(fail_index) + self.gc_set_extra_threshold() + return deadframe return execute_token def cast_ptr_to_int(x): @@ -110,6 +99,7 @@ flavor='raw', zero=True, immortal=True) def force(self, addr_of_force_index): + assert 0, 'refactor ME' TP = rffi.CArrayPtr(lltype.Signed) fail_index = rffi.cast(TP, addr_of_force_index)[0] assert fail_index >= 0, "already forced!" diff --git a/pypy/jit/backend/arm/test/test_runner.py b/pypy/jit/backend/arm/test/test_runner.py --- a/pypy/jit/backend/arm/test/test_runner.py +++ b/pypy/jit/backend/arm/test/test_runner.py @@ -60,8 +60,8 @@ ] cpu.compile_loop(inp, operations, looptoken) args = [i for i in range(1, 15)] - self.cpu.execute_token(looptoken, *args) - output = [self.cpu.get_latest_value_int(i - 1) for i in range(1, 15)] + deadframe = self.cpu.execute_token(looptoken, *args) + output = [self.cpu.get_latest_value_int(deadframe, i - 1) for i in range(1, 15)] expected = [3, 7, 11, 15, 19, 23, 27, 3, 7, 11, 15, 19, 23, 27] assert output == expected @@ -202,8 +202,8 @@ ARGS = [lltype.Signed] * numargs RES = lltype.Signed args = [i+1 for i in range(numargs)] - res = self.cpu.execute_token(looptoken, *args) - assert self.cpu.get_latest_value_int(0) == sum(args) + deadframe = self.cpu.execute_token(looptoken, *args) + assert self.cpu.get_latest_value_int(deadframe, 0) == sum(args) def test_debugger_on(self): from pypy.rlib import debug diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py --- a/pypy/jit/backend/llsupport/llmodel.py +++ b/pypy/jit/backend/llsupport/llmodel.py @@ -193,6 +193,12 @@ # ------------------- helpers and descriptions -------------------- + def gc_set_extra_threshold(self): + llop.gc_set_extra_threshold(lltype.Void, self.deadframe_size_max) + + def gc_clear_extra_threshold(self): + llop.gc_set_extra_threshold(lltype.Void, 0) + @staticmethod def _cast_int_to_gcref(x): # dangerous! only use if you are sure no collection could occur diff --git a/pypy/jit/backend/x86/runner.py b/pypy/jit/backend/x86/runner.py --- a/pypy/jit/backend/x86/runner.py +++ b/pypy/jit/backend/x86/runner.py @@ -1,6 +1,5 @@ import py from pypy.rpython.lltypesystem import lltype, llmemory, rffi -from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.llinterp import LLInterpreter from pypy.rlib.objectmodel import we_are_translated, specialize from pypy.rlib.jit_hooks import LOOP_RUN_CONTAINER @@ -58,12 +57,6 @@ else: return 1000 - def gc_set_extra_threshold(self): - llop.gc_set_extra_threshold(lltype.Void, self.deadframe_size_max) - - def gc_clear_extra_threshold(self): - llop.gc_set_extra_threshold(lltype.Void, 0) - def setup(self): self.assembler = Assembler386(self, self.translate_support_code) From noreply at buildbot.pypy.org Thu Jan 3 12:51:54 2013 From: noreply at buildbot.pypy.org (bivab) Date: Thu, 3 Jan 2013 12:51:54 +0100 (CET) Subject: [pypy-commit] pypy remove-globals-in-jit: update call_assembler Message-ID: <20130103115154.6F6921C078F@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: remove-globals-in-jit Changeset: r59676:3cab3d19acec Date: 2012-12-31 16:15 +0100 http://bitbucket.org/pypy/pypy/changeset/3cab3d19acec/ Log: update call_assembler diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py --- a/pypy/jit/backend/arm/opassembler.py +++ b/pypy/jit/backend/arm/opassembler.py @@ -16,6 +16,7 @@ gen_emit_unary_float_op, saved_registers, count_reg_args) +from pypy.jit.backend.arm.helper.regalloc import check_imm_arg from pypy.jit.backend.arm.codebuilder import ARMv7Builder, OverwritingBuilder from pypy.jit.backend.arm.jump import remap_frame_layout from pypy.jit.backend.arm.regalloc import TempInt, TempPtr @@ -27,7 +28,8 @@ from pypy.jit.metainterp.history import JitCellToken, TargetToken from pypy.jit.metainterp.resoperation import rop from pypy.rlib.objectmodel import we_are_translated -from pypy.rpython.lltypesystem import rstr +from pypy.rlib import rgc +from pypy.rpython.lltypesystem import rstr, rffi, lltype, llmemory NO_FORCE_INDEX = -1 @@ -1117,8 +1119,26 @@ value = self.cpu.done_with_this_frame_float_v else: raise AssertionError(kind) - self.mc.gen_load_int(r.ip.value, value) - self.mc.CMP_rr(tmploc.value, r.ip.value) + from pypy.jit.backend.llsupport.descr import unpack_fielddescr + from pypy.jit.backend.llsupport.descr import unpack_interiorfielddescr + descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) + _offset, _size, _ = unpack_fielddescr(descrs.jf_descr) + fail_descr = self.cpu.get_fail_descr_from_number(value) + value = fail_descr.hide(self.cpu) + rgc._make_sure_does_not_move(value) + value = rffi.cast(lltype.Signed, value) + + if check_imm_arg(_offset): + self.mc.LDR_ri(r.ip.value, tmploc.value, imm=_offset) + else: + self.mc.gen_load_int(r.ip.value, _offset) + self.mc.LDR_rr(r.ip.value, tmploc.value, r.ip.value) + if check_imm_arg(value): + self.mc.CMP_ri(r.ip.value, imm=value) + else: + self.mc.gen_load_int(r.lr.value, value) + self.mc.CMP_rr(r.lr.value, r.ip.value) + #if values are equal we take the fast path # Slow path, calling helper @@ -1145,19 +1165,31 @@ if op.result is not None: # load the return value from fail_boxes_xxx[0] kind = op.result.type - if kind == INT: - adr = self.fail_boxes_int.get_addr_for_num(0) - elif kind == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(0) - elif kind == FLOAT: - adr = self.fail_boxes_float.get_addr_for_num(0) + if kind == FLOAT: + t = unpack_interiorfielddescr(descrs.as_float)[0] + if not check_imm_arg(t): + self.mc.gen_load_int(r.ip.value, t, cond=fast_path_cond) + self.mc.ADD_rr(r.ip.value, r.r0.value, r.ip.value, + cond=fast_path_cond) + t = 0 + base = r.ip + else: + base = r.r0 + self.mc.VLDR(resloc.value, base.value, imm=t, + cond=fast_path_cond) else: - raise AssertionError(kind) - self.mc.gen_load_int(r.ip.value, adr, cond=fast_path_cond) - if op.result.type == FLOAT: - self.mc.VLDR(resloc.value, r.ip.value, cond=fast_path_cond) - else: - self.mc.LDR_ri(resloc.value, r.ip.value, cond=fast_path_cond) + assert resloc is r.r0 + if kind == INT: + t = unpack_interiorfielddescr(descrs.as_int)[0] + else: + t = unpack_interiorfielddescr(descrs.as_ref)[0] + if not check_imm_arg(t): + self.mc.gen_load_int(r.ip.value, t, cond=fast_path_cond) + self.mc.LDR_rr(resloc.value, resloc.value, r.ip.value, + cond=fast_path_cond) + else: + self.mc.LDR_ri(resloc.value, resloc.value, imm=t, + cond=fast_path_cond) # jump to merge point jmp_pos = self.mc.currpos() self.mc.BKPT() diff --git a/pypy/jit/backend/arm/test/test_runner.py b/pypy/jit/backend/arm/test/test_runner.py --- a/pypy/jit/backend/arm/test/test_runner.py +++ b/pypy/jit/backend/arm/test/test_runner.py @@ -65,11 +65,13 @@ expected = [3, 7, 11, 15, 19, 23, 27, 3, 7, 11, 15, 19, 23, 27] assert output == expected - def test_redirect_call_assember2(self): - def assembler_helper(failindex, virtualizable): - return self.cpu.get_latest_value_int(0) + def test_redirect_call_assembler2(self): + def assembler_helper(deadframe, virtualizable): + x = self.cpu.get_latest_value_int(deadframe, 0) + assert x == 11 + return 7 - FUNCPTR = lltype.Ptr(lltype.FuncType([lltype.Signed, llmemory.GCREF], + FUNCPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF, llmemory.GCREF], lltype.Signed)) class FakeJitDriverSD: @@ -101,12 +103,12 @@ self.cpu.compile_loop(loop2.inputargs, loop2.operations, lt2) self.cpu.compile_loop(loop3.inputargs, loop3.operations, lt3) self.cpu.compile_loop(loop1.inputargs, loop1.operations, lt1) - self.cpu.execute_token(lt1, 11) - assert self.cpu.get_latest_value_int(0) == 12 + df = self.cpu.execute_token(lt1, 10) + assert self.cpu.get_latest_value_int(df, 0) == 7 self.cpu.redirect_call_assembler(lt2, lt3) - self.cpu.execute_token(lt1, 11) - assert self.cpu.get_latest_value_int(0) == 10 + df = self.cpu.execute_token(lt1, 12) + assert self.cpu.get_latest_value_int(df, 0) == 7 SFloat = lltype.GcForwardReference() SFloat.become(lltype.GcStruct('SFloat', ('parent', rclass.OBJECT), From noreply at buildbot.pypy.org Thu Jan 3 12:51:55 2013 From: noreply at buildbot.pypy.org (bivab) Date: Thu, 3 Jan 2013 12:51:55 +0100 (CET) Subject: [pypy-commit] pypy remove-globals-in-jit: rename decode_registers_and_descr method to grab_frame_values Message-ID: <20130103115155.8AF5B1C078F@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: remove-globals-in-jit Changeset: r59677:a629fa5404be Date: 2013-01-02 14:53 +0100 http://bitbucket.org/pypy/pypy/changeset/a629fa5404be/ Log: rename decode_registers_and_descr method to grab_frame_values diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py --- a/pypy/jit/backend/arm/assembler.py +++ b/pypy/jit/backend/arm/assembler.py @@ -325,7 +325,7 @@ vfp_registers = rffi.cast(rffi.LONGP, stack_pointer) registers = rffi.ptradd(vfp_registers, 2*len(r.all_vfp_regs)) registers = rffi.cast(rffi.LONGP, registers) - return self.decode_registers_and_descr(self.cpu, mem_loc, frame_pointer, + return self.grab_frame_values(self.cpu, mem_loc, frame_pointer, registers, vfp_registers) self.failure_recovery_code = [0, 0, 0, 0] @@ -335,7 +335,7 @@ llmemory.GCREF)) @staticmethod - def decode_registers_and_descr(cpu, mem_loc, frame_pointer, + def grab_frame_values(cpu, mem_loc, frame_pointer, registers, vfp_registers): # no malloc allowed here!! xxx apart from one, hacking a lot #self.fail_ebp = allregisters[16 + ebp.value] From noreply at buildbot.pypy.org Thu Jan 3 12:51:56 2013 From: noreply at buildbot.pypy.org (bivab) Date: Thu, 3 Jan 2013 12:51:56 +0100 (CET) Subject: [pypy-commit] pypy remove-globals-in-jit: fixes Message-ID: <20130103115156.A9AB81C078F@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: remove-globals-in-jit Changeset: r59678:bacf4f0236c6 Date: 2013-01-02 14:54 +0100 http://bitbucket.org/pypy/pypy/changeset/bacf4f0236c6/ Log: fixes diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py --- a/pypy/jit/backend/arm/assembler.py +++ b/pypy/jit/backend/arm/assembler.py @@ -409,7 +409,7 @@ if code_inputarg: code = ~code code_inputarg = False - stackloc = frame_addr + get_fp_offset(int(code)) + stackloc = force_index - get_fp_offset(int(code)) value = rffi.cast(rffi.LONGP, stackloc)[0] if kind == AssemblerARM.DESCR_FLOAT: assert WORD == 4 @@ -435,7 +435,7 @@ value = vfp_registers[2*code] value_hi = vfp_registers[2*code + 1] else: - value = allregisters[code] + value = registers[code] else: value = registers[code] # store the loaded value into fail_boxes_ @@ -644,7 +644,7 @@ self.mc.BL(target) # write tight data that describes the failure recovery if guardtok.is_guard_not_forced: - mc.writechar(chr(self.CODE_FORCED)) + self.mc.writechar(chr(self.CODE_FORCED)) self.write_failure_recovery_description(guardtok.descr, guardtok.failargs, guardtok.faillocs[1:]) self.mc.write32(fail_index) From noreply at buildbot.pypy.org Thu Jan 3 12:51:57 2013 From: noreply at buildbot.pypy.org (bivab) Date: Thu, 3 Jan 2013 12:51:57 +0100 (CET) Subject: [pypy-commit] pypy remove-globals-in-jit: WIP: port forcing support Message-ID: <20130103115157.C55561C078F@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: remove-globals-in-jit Changeset: r59679:655e6658b685 Date: 2013-01-02 14:55 +0100 http://bitbucket.org/pypy/pypy/changeset/655e6658b685/ Log: WIP: port forcing support diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py --- a/pypy/jit/backend/arm/assembler.py +++ b/pypy/jit/backend/arm/assembler.py @@ -338,7 +338,7 @@ def grab_frame_values(cpu, mem_loc, frame_pointer, registers, vfp_registers): # no malloc allowed here!! xxx apart from one, hacking a lot - #self.fail_ebp = allregisters[16 + ebp.value] + force_index = rffi.cast(lltype.Signed, frame_pointer) num = 0 deadframe = lltype.nullptr(jitframe.DEADFRAME) bytecode = rffi.cast(rffi.UCHARP, mem_loc) @@ -361,8 +361,7 @@ continue if code == AssemblerARM.CODE_FORCED: # resuming from a GUARD_NOT_FORCED - xxx - token = allregisters[16 + ebp.value] + token = force_index deadframe = ( cpu.assembler.force_token_to_dead_frame.pop(token)) deadframe = lltype.cast_opaque_ptr( diff --git a/pypy/jit/backend/arm/runner.py b/pypy/jit/backend/arm/runner.py --- a/pypy/jit/backend/arm/runner.py +++ b/pypy/jit/backend/arm/runner.py @@ -99,23 +99,27 @@ flavor='raw', zero=True, immortal=True) def force(self, addr_of_force_index): - assert 0, 'refactor ME' TP = rffi.CArrayPtr(lltype.Signed) fail_index = rffi.cast(TP, addr_of_force_index)[0] assert fail_index >= 0, "already forced!" faildescr = self.get_fail_descr_from_number(fail_index) rffi.cast(TP, addr_of_force_index)[0] = ~fail_index - bytecode = self.assembler._find_failure_recovery_bytecode(faildescr) + frb = self.assembler._find_failure_recovery_bytecode(faildescr) + bytecode = rffi.cast(rffi.UCHARP, frb) addr_all_null_regsiters = rffi.cast(rffi.LONG, self.all_null_registers) - # start of "no gc operation!" block - fail_index_2 = self.assembler.failure_recovery_func( - bytecode, - addr_of_force_index, - addr_all_null_regsiters) - self.assembler.leave_jitted_hook() - # end of "no gc operation!" block - assert fail_index == fail_index_2 - return faildescr + # + assert (rffi.cast(lltype.Signed, bytecode[0]) == + self.assembler.CODE_FORCED) + bytecode = rffi.ptradd(bytecode, 1) + deadframe = self.assembler.failure_recovery_func(bytecode, + addr_of_force_index, + self.all_null_registers) + + # + assert self.get_latest_descr(deadframe) is faildescr + self.assembler.force_token_to_dead_frame[addr_of_force_index] = ( + deadframe) + return deadframe def redirect_call_assembler(self, oldlooptoken, newlooptoken): self.assembler.redirect_call_assembler(oldlooptoken, newlooptoken) From noreply at buildbot.pypy.org Thu Jan 3 12:52:00 2013 From: noreply at buildbot.pypy.org (bivab) Date: Thu, 3 Jan 2013 12:52:00 +0100 (CET) Subject: [pypy-commit] pypy remove-globals-in-jit: merge default Message-ID: <20130103115200.13F291C078F@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: remove-globals-in-jit Changeset: r59680:62ae5659ff10 Date: 2012-12-31 12:58 +0100 http://bitbucket.org/pypy/pypy/changeset/62ae5659ff10/ Log: merge default diff too long, truncating to 2000 out of 3575 lines diff --git a/lib-python/2.7/ctypes/test/test_internals.py b/lib-python/2.7/ctypes/test/test_internals.py --- a/lib-python/2.7/ctypes/test/test_internals.py +++ b/lib-python/2.7/ctypes/test/test_internals.py @@ -1,7 +1,10 @@ # This tests the internal _objects attribute import unittest from ctypes import * -from sys import getrefcount as grc +try: + from sys import getrefcount as grc +except ImportError: + grc = None # e.g. PyPy # XXX This test must be reviewed for correctness!!! @@ -22,6 +25,8 @@ self.assertEqual(id(a), id(b)) def test_ints(self): + if grc is None: + return unittest.skip("no sys.getrefcount()") i = 42000123 refcnt = grc(i) ci = c_int(i) @@ -29,6 +34,8 @@ self.assertEqual(ci._objects, None) def test_c_char_p(self): + if grc is None: + return unittest.skip("no sys.getrefcount()") s = "Hello, World" refcnt = grc(s) cs = c_char_p(s) diff --git a/lib-python/2.7/ctypes/test/test_memfunctions.py b/lib-python/2.7/ctypes/test/test_memfunctions.py --- a/lib-python/2.7/ctypes/test/test_memfunctions.py +++ b/lib-python/2.7/ctypes/test/test_memfunctions.py @@ -53,7 +53,8 @@ s = string_at("foo bar") # XXX The following may be wrong, depending on how Python # manages string instances - self.assertEqual(2, sys.getrefcount(s)) + if hasattr(sys, 'getrefcount'): + self.assertEqual(2, sys.getrefcount(s)) self.assertTrue(s, "foo bar") self.assertEqual(string_at("foo bar", 8), "foo bar\0") diff --git a/lib-python/2.7/ctypes/test/test_python_api.py b/lib-python/2.7/ctypes/test/test_python_api.py --- a/lib-python/2.7/ctypes/test/test_python_api.py +++ b/lib-python/2.7/ctypes/test/test_python_api.py @@ -9,7 +9,10 @@ ################################################################ -from sys import getrefcount as grc +try: + from sys import getrefcount as grc +except ImportError: + grc = None # e.g. PyPy if sys.version_info > (2, 4): c_py_ssize_t = c_size_t else: diff --git a/lib-python/2.7/ctypes/test/test_refcounts.py b/lib-python/2.7/ctypes/test/test_refcounts.py --- a/lib-python/2.7/ctypes/test/test_refcounts.py +++ b/lib-python/2.7/ctypes/test/test_refcounts.py @@ -11,7 +11,10 @@ class RefcountTestCase(unittest.TestCase): def test_1(self): - from sys import getrefcount as grc + try: + from sys import getrefcount as grc + except ImportError: + return unittest.skip("no sys.getrefcount()") f = dll._testfunc_callback_i_if f.restype = ctypes.c_int @@ -35,7 +38,10 @@ def test_refcount(self): - from sys import getrefcount as grc + try: + from sys import getrefcount as grc + except ImportError: + return unittest.skip("no sys.getrefcount()") def func(*args): pass # this is the standard refcount for func @@ -84,6 +90,10 @@ class AnotherLeak(unittest.TestCase): def test_callback(self): import sys + try: + from sys import getrefcount + except ImportError: + return unittest.skip("no sys.getrefcount()") proto = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int, ctypes.c_int) def func(a, b): diff --git a/lib-python/2.7/timeit.py b/lib-python/2.7/timeit.py --- a/lib-python/2.7/timeit.py +++ b/lib-python/2.7/timeit.py @@ -190,7 +190,8 @@ else: it = [None] * number gcold = gc.isenabled() - gc.disable() + if '__pypy__' not in sys.builtin_module_names: + gc.disable() # only do that on CPython try: timing = self.inner(it, self.timer) finally: diff --git a/lib_pypy/_ctypes_test.py b/lib_pypy/_ctypes_test.py --- a/lib_pypy/_ctypes_test.py +++ b/lib_pypy/_ctypes_test.py @@ -2,10 +2,6 @@ import tempfile import gc -# Monkeypatch & hacks to let ctypes.tests import. -# This should be removed at some point. -sys.getrefcount = lambda x: len(gc.get_referrers(x)) - 1 - def compile_shared(): """Compile '_ctypes_test.c' into an extension module, and import it """ diff --git a/lib_pypy/numpypy/core/arrayprint.py b/lib_pypy/numpypy/core/arrayprint.py --- a/lib_pypy/numpypy/core/arrayprint.py +++ b/lib_pypy/numpypy/core/arrayprint.py @@ -248,9 +248,9 @@ 'int' : IntegerFormat(data), 'float' : FloatFormat(data, precision, suppress_small), 'longfloat' : LongFloatFormat(precision), - #'complexfloat' : ComplexFormat(data, precision, - # suppress_small), - #'longcomplexfloat' : LongComplexFormat(precision), + 'complexfloat' : ComplexFormat(data, precision, + suppress_small), + 'longcomplexfloat' : LongComplexFormat(precision), 'datetime' : DatetimeFormat(data), 'timedelta' : TimedeltaFormat(data), 'numpystr' : repr_format, @@ -294,19 +294,19 @@ #else: format_function = formatdict['int'] elif issubclass(dtypeobj, _nt.floating): - #if issubclass(dtypeobj, _nt.longfloat): - # format_function = formatdict['longfloat'] - #else: - format_function = formatdict['float'] - #elif issubclass(dtypeobj, _nt.complexfloating): - # if issubclass(dtypeobj, _nt.clongfloat): - # format_function = formatdict['longcomplexfloat'] - # else: - # format_function = formatdict['complexfloat'] + if issubclass(dtypeobj, _nt.longfloat): + format_function = formatdict['longfloat'] + else: + format_function = formatdict['float'] + elif issubclass(dtypeobj, _nt.complexfloating): + if issubclass(dtypeobj, _nt.clongfloat): + format_function = formatdict['longcomplexfloat'] + else: + format_function = formatdict['complexfloat'] elif issubclass(dtypeobj, (_nt.unicode_, _nt.string_)): format_function = formatdict['numpystr'] - elif issubclass(dtypeobj, _nt.datetime64): - format_function = formatdict['datetime'] + #elif issubclass(dtypeobj, _nt.datetime64): + # format_function = formatdict['datetime'] else: format_function = formatdict['str'] diff --git a/pypy/annotation/annrpython.py b/pypy/annotation/annrpython.py --- a/pypy/annotation/annrpython.py +++ b/pypy/annotation/annrpython.py @@ -8,7 +8,7 @@ from pypy.objspace.flow.model import (Variable, Constant, FunctionGraph, c_last_exception, checkgraph) from pypy.translator import simplify, transform -from pypy.annotation import model as annmodel, signature +from pypy.annotation import model as annmodel, signature, unaryop, binaryop from pypy.annotation.bookkeeper import Bookkeeper import py log = py.log.Producer("annrpython") @@ -453,12 +453,12 @@ # occour for this specific, typed operation. if block.exitswitch == c_last_exception: op = block.operations[-1] - if op.opname in annmodel.BINARY_OPERATIONS: + if op.opname in binaryop.BINARY_OPERATIONS: arg1 = self.binding(op.args[0]) arg2 = self.binding(op.args[1]) binop = getattr(pair(arg1, arg2), op.opname, None) can_only_throw = annmodel.read_can_only_throw(binop, arg1, arg2) - elif op.opname in annmodel.UNARY_OPERATIONS: + elif op.opname in unaryop.UNARY_OPERATIONS: arg1 = self.binding(op.args[0]) opname = op.opname if opname == 'contains': opname = 'op_contains' @@ -629,10 +629,10 @@ return self.bookkeeper.newdict() - def _registeroperations(cls, model): + def _registeroperations(cls, unary_ops, binary_ops): # All unary operations d = {} - for opname in model.UNARY_OPERATIONS: + for opname in unary_ops: fnname = 'consider_op_' + opname exec py.code.Source(""" def consider_op_%s(self, arg, *args): @@ -640,7 +640,7 @@ """ % (opname, opname)).compile() in globals(), d setattr(cls, fnname, d[fnname]) # All binary operations - for opname in model.BINARY_OPERATIONS: + for opname in binary_ops: fnname = 'consider_op_' + opname exec py.code.Source(""" def consider_op_%s(self, arg1, arg2, *args): @@ -650,7 +650,7 @@ _registeroperations = classmethod(_registeroperations) # register simple operations handling -RPythonAnnotator._registeroperations(annmodel) +RPythonAnnotator._registeroperations(unaryop.UNARY_OPERATIONS, binaryop.BINARY_OPERATIONS) class BlockedInference(Exception): diff --git a/pypy/annotation/binaryop.py b/pypy/annotation/binaryop.py --- a/pypy/annotation/binaryop.py +++ b/pypy/annotation/binaryop.py @@ -7,10 +7,10 @@ from pypy.tool.pairtype import pair, pairtype from pypy.annotation.model import SomeObject, SomeInteger, SomeBool, s_Bool from pypy.annotation.model import SomeString, SomeChar, SomeList, SomeDict -from pypy.annotation.model import SomeUnicodeCodePoint, SomeStringOrUnicode +from pypy.annotation.model import SomeUnicodeCodePoint, SomeUnicodeString from pypy.annotation.model import SomeTuple, SomeImpossibleValue, s_ImpossibleValue from pypy.annotation.model import SomeInstance, SomeBuiltin, SomeIterator -from pypy.annotation.model import SomePBC, SomeFloat, s_None +from pypy.annotation.model import SomePBC, SomeFloat, s_None, SomeByteArray from pypy.annotation.model import SomeExternalObject, SomeWeakRef from pypy.annotation.model import SomeAddress, SomeTypedAddressAccess from pypy.annotation.model import SomeSingleFloat, SomeLongFloat, SomeType @@ -19,7 +19,6 @@ from pypy.annotation.model import read_can_only_throw from pypy.annotation.model import add_knowntypedata, merge_knowntypedata from pypy.annotation.model import SomeGenericCallable -from pypy.annotation.model import SomeUnicodeString from pypy.annotation.bookkeeper import getbookkeeper from pypy.objspace.flow.model import Variable, Constant from pypy.rlib import rarithmetic @@ -416,6 +415,34 @@ result.const = str1.const + str2.const return result +class __extend__(pairtype(SomeByteArray, SomeByteArray)): + def union((b1, b2)): + can_be_None = b1.can_be_None or b2.can_be_None + return SomeByteArray(can_be_None=can_be_None) + + def add((b1, b2)): + result = SomeByteArray() + if b1.is_immutable_constant() and b2.is_immutable_constant(): + result.const = b1.const + b2.const + return result + +class __extend__(pairtype(SomeByteArray, SomeInteger)): + def getitem((s_b, s_i)): + return SomeInteger() + + def setitem((s_b, s_i), s_i2): + assert isinstance(s_i2, SomeInteger) + +class __extend__(pairtype(SomeString, SomeByteArray), + pairtype(SomeByteArray, SomeString), + pairtype(SomeChar, SomeByteArray), + pairtype(SomeByteArray, SomeChar)): + def add((b1, b2)): + result = SomeByteArray() + if b1.is_immutable_constant() and b2.is_immutable_constant(): + result.const = b1.const + b2.const + return result + class __extend__(pairtype(SomeChar, SomeChar)): def union((chr1, chr2)): @@ -458,7 +485,8 @@ for s_item in s_tuple.items: if isinstance(s_item, SomeFloat): pass # or s_item is a subclass, like SomeInteger - elif isinstance(s_item, SomeStringOrUnicode) and s_item.no_nul: + elif (isinstance(s_item, SomeString) or + isinstance(s_item, SomeUnicodeString)) and s_item.no_nul: pass else: no_nul = False diff --git a/pypy/annotation/bookkeeper.py b/pypy/annotation/bookkeeper.py --- a/pypy/annotation/bookkeeper.py +++ b/pypy/annotation/bookkeeper.py @@ -13,7 +13,7 @@ SomeUnicodeCodePoint, SomeOOStaticMeth, s_None, s_ImpossibleValue, \ SomeLLADTMeth, SomeBool, SomeTuple, SomeOOClass, SomeImpossibleValue, \ SomeUnicodeString, SomeList, SomeObject, HarmlesslyBlocked, \ - SomeWeakRef, lltype_to_annotation, SomeType + SomeWeakRef, lltype_to_annotation, SomeType, SomeByteArray from pypy.annotation.classdef import InstanceSource, ClassDef from pypy.annotation.listdef import ListDef, ListItem from pypy.annotation.dictdef import DictDef @@ -349,6 +349,8 @@ result = SomeUnicodeCodePoint() else: result = SomeUnicodeString() + elif tp is bytearray: + result = SomeByteArray() elif tp is tuple: result = SomeTuple(items = [self.immutablevalue(e, need_const) for e in x]) elif tp is float: diff --git a/pypy/annotation/builtin.py b/pypy/annotation/builtin.py --- a/pypy/annotation/builtin.py +++ b/pypy/annotation/builtin.py @@ -9,7 +9,7 @@ from pypy.annotation.model import SomeFloat, unionof, SomeUnicodeString from pypy.annotation.model import SomePBC, SomeInstance, SomeDict, SomeList from pypy.annotation.model import SomeWeakRef, SomeIterator -from pypy.annotation.model import SomeOOObject +from pypy.annotation.model import SomeOOObject, SomeByteArray from pypy.annotation.model import annotation_to_lltype, lltype_to_annotation, ll_to_annotation from pypy.annotation.model import add_knowntypedata from pypy.annotation.model import s_ImpossibleValue @@ -119,6 +119,9 @@ def builtin_unicode(s_unicode): return constpropagate(unicode, [s_unicode], SomeUnicodeString()) +def builtin_bytearray(s_str): + return constpropagate(bytearray, [s_str], SomeByteArray()) + def our_issubclass(cls1, cls2): """ we're going to try to be less silly in the face of old-style classes""" from pypy.annotation.classdef import ClassDef @@ -253,24 +256,6 @@ s = SomeInteger(nonneg=True, knowntype=s.knowntype) return s -def builtin_apply(*stuff): - getbookkeeper().warning("ignoring apply%r" % (stuff,)) - return SomeObject() - -##def builtin_slice(*args): -## bk = getbookkeeper() -## if len(args) == 1: -## return SomeSlice( -## bk.immutablevalue(None), args[0], bk.immutablevalue(None)) -## elif len(args) == 2: -## return SomeSlice( -## args[0], args[1], bk.immutablevalue(None)) -## elif len(args) == 3: -## return SomeSlice( -## args[0], args[1], args[2]) -## else: -## raise Exception, "bogus call to slice()" - def OSError_init(s_self, *args): pass diff --git a/pypy/annotation/model.py b/pypy/annotation/model.py --- a/pypy/annotation/model.py +++ b/pypy/annotation/model.py @@ -202,11 +202,16 @@ self.knowntypedata = knowntypedata class SomeStringOrUnicode(SomeObject): + """Base class for shared implementation of SomeString and SomeUnicodeString. + + Cannot be an annotation.""" + immutable = True can_be_None=False no_nul = False # No NUL character in the string. def __init__(self, can_be_None=False, no_nul=False): + assert type(self) is not SomeStringOrUnicode if can_be_None: self.can_be_None = True if no_nul: @@ -225,19 +230,19 @@ d2 = d2.copy(); d2['no_nul'] = 0 # ignored return d1 == d2 + def nonnoneify(self): + return self.__class__(can_be_None=False, no_nul=self.no_nul) + class SomeString(SomeStringOrUnicode): "Stands for an object which is known to be a string." knowntype = str - def nonnoneify(self): - return SomeString(can_be_None=False, no_nul=self.no_nul) - class SomeUnicodeString(SomeStringOrUnicode): "Stands for an object which is known to be an unicode string" knowntype = unicode - def nonnoneify(self): - return SomeUnicodeString(can_be_None=False, no_nul=self.no_nul) +class SomeByteArray(SomeStringOrUnicode): + knowntype = bytearray class SomeChar(SomeString): "Stands for an object known to be a string of length 1." @@ -773,7 +778,3 @@ else: raise RuntimeError("The annotator relies on 'assert' statements from the\n" "\tannotated program: you cannot run it with 'python -O'.") - -# this has the side-effect of registering the unary and binary operations -from pypy.annotation.unaryop import UNARY_OPERATIONS -from pypy.annotation.binaryop import BINARY_OPERATIONS diff --git a/pypy/annotation/signature.py b/pypy/annotation/signature.py --- a/pypy/annotation/signature.py +++ b/pypy/annotation/signature.py @@ -151,4 +151,9 @@ actualtypes[:] = params_s def enforce_signature_return(funcdesc, sigtype, inferredtype): - return finish_type(sigtype, funcdesc.bookkeeper, funcdesc.pyobj) + s_sigret = finish_type(sigtype, funcdesc.bookkeeper, funcdesc.pyobj) + if not s_sigret.contains(inferredtype): + raise Exception("%r return value:\n" + "expected %s,\n" + " got %s" % (funcdesc, s_sigret, inferredtype)) + return s_sigret diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py --- a/pypy/annotation/test/test_annrpython.py +++ b/pypy/annotation/test/test_annrpython.py @@ -3819,6 +3819,37 @@ a = self.RPythonAnnotator() a.build_types(f, []) # assert did not explode + def test_bytearray(self): + def f(): + return bytearray("xyz") + + a = self.RPythonAnnotator() + assert isinstance(a.build_types(f, []), annmodel.SomeByteArray) + + def test_bytearray_add(self): + def f(a): + return a + bytearray("xyz") + + a = self.RPythonAnnotator() + assert isinstance(a.build_types(f, [annmodel.SomeByteArray()]), + annmodel.SomeByteArray) + a = self.RPythonAnnotator() + assert isinstance(a.build_types(f, [str]), + annmodel.SomeByteArray) + a = self.RPythonAnnotator() + assert isinstance(a.build_types(f, [annmodel.SomeChar()]), + annmodel.SomeByteArray) + + def test_bytearray_setitem_getitem(self): + def f(b, i, c): + b[i] = c + return b[i + 1] + + a = self.RPythonAnnotator() + assert isinstance(a.build_types(f, [annmodel.SomeByteArray(), + int, int]), + annmodel.SomeInteger) + def g(n): return [0,1,2,n] diff --git a/pypy/doc/config/objspace.usemodules.time.txt b/pypy/doc/config/objspace.usemodules.time.txt --- a/pypy/doc/config/objspace.usemodules.time.txt +++ b/pypy/doc/config/objspace.usemodules.time.txt @@ -1,4 +1,5 @@ Use the 'time' module. Obsolete; use :config:`objspace.usemodules.rctime` for our up-to-date version -of the application-level 'time' module. +of the application-level 'time' module, at least for C-like targets (the C +and LLVM backends). diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -8,8 +8,21 @@ .. branch: length-hint Implement __lenght_hint__ according to PEP 424 +.. branch: numpypy-longdouble +Long double support for numpypy + +.. branch: signatures +Improved RPython typing + +.. branch: rpython-bytearray +Rudimentary support for bytearray in RPython + .. branches we don't care about .. branch: autoreds +.. branch: reflex-support +.. branch: kill-faking +.. branch: improved_ebnfparse_error +.. branch: task-decorator .. branch: release-2.0-beta1 diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py --- a/pypy/jit/backend/arm/assembler.py +++ b/pypy/jit/backend/arm/assembler.py @@ -8,7 +8,7 @@ from pypy.jit.backend.arm.codebuilder import ARMv7Builder, OverwritingBuilder from pypy.jit.backend.arm.locations import get_fp_offset from pypy.jit.backend.arm.regalloc import (Regalloc, ARMFrameManager, - ARMv7RegisterManager, check_imm_arg, + CoreRegisterManager, check_imm_arg, operations as regalloc_operations, operations_with_guard as regalloc_operations_with_guard) from pypy.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper @@ -492,10 +492,10 @@ # are stored in r0 and r1. mc.SUB_rr(r.r0.value, r.r1.value, r.r0.value) addr = self.cpu.gc_ll_descr.get_malloc_slowpath_addr() - for reg, ofs in ARMv7RegisterManager.REGLOC_TO_COPY_AREA_OFS.items(): + for reg, ofs in CoreRegisterManager.REGLOC_TO_COPY_AREA_OFS.items(): mc.STR_ri(reg.value, r.fp.value, imm=ofs) mc.BL(addr) - for reg, ofs in ARMv7RegisterManager.REGLOC_TO_COPY_AREA_OFS.items(): + for reg, ofs in CoreRegisterManager.REGLOC_TO_COPY_AREA_OFS.items(): mc.LDR_ri(reg.value, r.fp.value, imm=ofs) mc.CMP_ri(r.r0.value, 0) diff --git a/pypy/jit/backend/arm/helper/regalloc.py b/pypy/jit/backend/arm/helper/regalloc.py --- a/pypy/jit/backend/arm/helper/regalloc.py +++ b/pypy/jit/backend/arm/helper/regalloc.py @@ -32,14 +32,14 @@ imm_a0 = check_imm_box(a0, imm_size, allow_zero=allow_zero) imm_a1 = check_imm_box(a1, imm_size, allow_zero=allow_zero) if not imm_a0 and imm_a1: - l0 = self._ensure_value_is_boxed(a0) + l0 = self.make_sure_var_in_reg(a0) l1 = self.convert_to_imm(a1) elif commutative and imm_a0 and not imm_a1: l1 = self.convert_to_imm(a0) - l0 = self._ensure_value_is_boxed(a1, boxes) + l0 = self.make_sure_var_in_reg(a1, boxes) else: - l0 = self._ensure_value_is_boxed(a0, boxes) - l1 = self._ensure_value_is_boxed(a1, boxes) + l0 = self.make_sure_var_in_reg(a0, boxes) + l1 = self.make_sure_var_in_reg(a1, boxes) self.possibly_free_vars_for_op(op) self.free_temp_vars() res = self.force_allocate_reg(op.result, boxes) @@ -52,10 +52,10 @@ if guard: def f(self, op, guard_op, fcond): locs = [] - loc1 = self._ensure_value_is_boxed(op.getarg(0)) + loc1 = self.make_sure_var_in_reg(op.getarg(0)) locs.append(loc1) if base: - loc2 = self._ensure_value_is_boxed(op.getarg(1)) + loc2 = self.make_sure_var_in_reg(op.getarg(1)) locs.append(loc2) self.possibly_free_vars_for_op(op) self.free_temp_vars() @@ -70,10 +70,10 @@ else: def f(self, op, fcond): locs = [] - loc1 = self._ensure_value_is_boxed(op.getarg(0)) + loc1 = self.make_sure_var_in_reg(op.getarg(0)) locs.append(loc1) if base: - loc2 = self._ensure_value_is_boxed(op.getarg(1)) + loc2 = self.make_sure_var_in_reg(op.getarg(1)) locs.append(loc2) self.possibly_free_vars_for_op(op) self.free_temp_vars() @@ -111,11 +111,11 @@ arg0, arg1 = boxes imm_a1 = check_imm_box(arg1) - l0 = self._ensure_value_is_boxed(arg0, forbidden_vars=boxes) + l0 = self.make_sure_var_in_reg(arg0, forbidden_vars=boxes) if imm_a1: l1 = self.convert_to_imm(arg1) else: - l1 = self._ensure_value_is_boxed(arg1, forbidden_vars=boxes) + l1 = self.make_sure_var_in_reg(arg1, forbidden_vars=boxes) self.possibly_free_vars_for_op(op) self.free_temp_vars() @@ -134,7 +134,7 @@ assert fcond is not None a0 = op.getarg(0) assert isinstance(a0, Box) - reg = self._ensure_value_is_boxed(a0) + reg = self.make_sure_var_in_reg(a0) self.possibly_free_vars_for_op(op) if guard_op is None: res = self.force_allocate_reg(op.result, [a0]) diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py --- a/pypy/jit/backend/arm/opassembler.py +++ b/pypy/jit/backend/arm/opassembler.py @@ -988,8 +988,8 @@ def _emit_copystrcontent(self, op, regalloc, fcond, is_unicode): # compute the source address args = op.getarglist() - base_loc = regalloc._ensure_value_is_boxed(args[0], args) - ofs_loc = regalloc._ensure_value_is_boxed(args[2], args) + base_loc = regalloc.make_sure_var_in_reg(args[0], args) + ofs_loc = regalloc.make_sure_var_in_reg(args[2], args) assert args[0] is not args[1] # forbidden case of aliasing regalloc.possibly_free_var(args[0]) regalloc.free_temp_vars() @@ -1009,8 +1009,8 @@ dstaddr_loc = regalloc.force_allocate_reg(dstaddr_box, selected_reg=r.r0) forbidden_vars.append(dstaddr_box) - base_loc = regalloc._ensure_value_is_boxed(args[1], forbidden_vars) - ofs_loc = regalloc._ensure_value_is_boxed(args[3], forbidden_vars) + base_loc = regalloc.make_sure_var_in_reg(args[1], forbidden_vars) + ofs_loc = regalloc.make_sure_var_in_reg(args[3], forbidden_vars) assert base_loc.is_reg() assert ofs_loc.is_reg() regalloc.possibly_free_var(args[1]) @@ -1026,7 +1026,7 @@ # need the box here if isinstance(args[4], Box): length_box = args[4] - length_loc = regalloc._ensure_value_is_boxed(args[4], + length_loc = regalloc.make_sure_var_in_reg(args[4], forbidden_vars) else: length_box = TempInt() diff --git a/pypy/jit/backend/arm/regalloc.py b/pypy/jit/backend/arm/regalloc.py --- a/pypy/jit/backend/arm/regalloc.py +++ b/pypy/jit/backend/arm/regalloc.py @@ -84,8 +84,28 @@ def void(self, op, fcond): return [] +class ARMRegisterManager(RegisterManager): + def return_constant(self, v, forbidden_vars=[], selected_reg=None): + self._check_type(v) + if isinstance(v, Const): + if isinstance(v, ConstPtr): + tp = REF + elif isinstance(v, ConstFloat): + tp = FLOAT + else: + tp = INT + loc = self.get_scratch_reg(tp, + self.temp_boxes + forbidden_vars, + selected_reg=selected_reg) + immvalue = self.convert_to_imm(v) + self.assembler.load(loc, immvalue) + return loc + else: + return RegisterManager.return_constant(self, v, + forbidden_vars, selected_reg) -class VFPRegisterManager(RegisterManager): + +class VFPRegisterManager(ARMRegisterManager): all_regs = r.all_vfp_regs box_types = [FLOAT] save_around_call_regs = r.all_vfp_regs @@ -107,20 +127,7 @@ reg = self.force_allocate_reg(v, selected_reg=r.d0) return reg - def ensure_value_is_boxed(self, thing, forbidden_vars=[]): - loc = None - if isinstance(thing, Const): - assert isinstance(thing, ConstFloat) - loc = self.get_scratch_reg(FLOAT, self.temp_boxes + forbidden_vars) - immvalue = self.convert_to_imm(thing) - self.assembler.load(loc, immvalue) - else: - loc = self.make_sure_var_in_reg(thing, - forbidden_vars=self.temp_boxes + forbidden_vars) - return loc - - def get_scratch_reg(self, type=FLOAT, forbidden_vars=[], - selected_reg=None): + def get_scratch_reg(self, type=FLOAT, forbidden_vars=[], selected_reg=None): assert type == FLOAT # for now box = TempFloat() self.temp_boxes.append(box) @@ -129,7 +136,7 @@ return reg -class ARMv7RegisterManager(RegisterManager): +class CoreRegisterManager(ARMRegisterManager): all_regs = r.all_regs box_types = None # or a list of acceptable types no_lower_byte_regs = all_regs @@ -162,22 +169,6 @@ return locations.ImmLocation(rffi.cast(lltype.Signed, c.value)) assert 0 - def ensure_value_is_boxed(self, thing, forbidden_vars=None): - loc = None - if isinstance(thing, Const): - if isinstance(thing, ConstPtr): - tp = REF - else: - tp = INT - loc = self.get_scratch_reg(tp, forbidden_vars=self.temp_boxes - + forbidden_vars) - immvalue = self.convert_to_imm(thing) - self.assembler.load(loc, immvalue) - else: - loc = self.make_sure_var_in_reg(thing, - forbidden_vars=self.temp_boxes + forbidden_vars) - return loc - def get_scratch_reg(self, type=INT, forbidden_vars=[], selected_reg=None): assert type == INT or type == REF box = TempBox() @@ -277,7 +268,12 @@ def make_sure_var_in_reg(self, var, forbidden_vars=[], selected_reg=None, need_lower_byte=False): - assert 0, 'should not be called directly' + if var.type == FLOAT: + return self.vfprm.make_sure_var_in_reg(var, forbidden_vars, + selected_reg, need_lower_byte) + else: + return self.rm.make_sure_var_in_reg(var, forbidden_vars, + selected_reg, need_lower_byte) def convert_to_imm(self, value): if isinstance(value, ConstInt): @@ -294,7 +290,7 @@ fm = self.frame_manager asm = self.assembler self.vfprm = VFPRegisterManager(longevity, fm, asm) - self.rm = ARMv7RegisterManager(longevity, fm, asm) + self.rm = CoreRegisterManager(longevity, fm, asm) def prepare_loop(self, inputargs, operations): self._prepare(inputargs, operations) @@ -426,12 +422,6 @@ self.rm.before_call(force_store, save_all_regs) self.vfprm.before_call(force_store, save_all_regs) - def _ensure_value_is_boxed(self, thing, forbidden_vars=[]): - if thing.type == FLOAT: - return self.vfprm.ensure_value_is_boxed(thing, forbidden_vars) - else: - return self.rm.ensure_value_is_boxed(thing, forbidden_vars) - def _sync_var(self, v): if v.type == FLOAT: self.vfprm._sync_var(v) @@ -444,14 +434,14 @@ imm_a0 = check_imm_box(a0) imm_a1 = check_imm_box(a1) if not imm_a0 and imm_a1: - l0 = self._ensure_value_is_boxed(a0, boxes) + l0 = self.make_sure_var_in_reg(a0, boxes) l1 = self.convert_to_imm(a1) elif imm_a0 and not imm_a1: l0 = self.convert_to_imm(a0) - l1 = self._ensure_value_is_boxed(a1, boxes) + l1 = self.make_sure_var_in_reg(a1, boxes) else: - l0 = self._ensure_value_is_boxed(a0, boxes) - l1 = self._ensure_value_is_boxed(a1, boxes) + l0 = self.make_sure_var_in_reg(a0, boxes) + l1 = self.make_sure_var_in_reg(a1, boxes) return [l0, l1] def prepare_op_int_add(self, op, fcond): @@ -466,14 +456,14 @@ imm_a0 = check_imm_box(a0) imm_a1 = check_imm_box(a1) if not imm_a0 and imm_a1: - l0 = self._ensure_value_is_boxed(a0, boxes) + l0 = self.make_sure_var_in_reg(a0, boxes) l1 = self.convert_to_imm(a1) elif imm_a0 and not imm_a1: l0 = self.convert_to_imm(a0) - l1 = self._ensure_value_is_boxed(a1, boxes) + l1 = self.make_sure_var_in_reg(a1, boxes) else: - l0 = self._ensure_value_is_boxed(a0, boxes) - l1 = self._ensure_value_is_boxed(a1, boxes) + l0 = self.make_sure_var_in_reg(a0, boxes) + l1 = self.make_sure_var_in_reg(a1, boxes) return [l0, l1] def prepare_op_int_sub(self, op, fcond): @@ -487,8 +477,8 @@ boxes = op.getarglist() a0, a1 = boxes - reg1 = self._ensure_value_is_boxed(a0, forbidden_vars=boxes) - reg2 = self._ensure_value_is_boxed(a1, forbidden_vars=boxes) + reg1 = self.make_sure_var_in_reg(a0, forbidden_vars=boxes) + reg2 = self.make_sure_var_in_reg(a1, forbidden_vars=boxes) self.possibly_free_vars(boxes) self.possibly_free_vars_for_op(op) @@ -497,14 +487,14 @@ return [reg1, reg2, res] def prepare_op_int_force_ge_zero(self, op, fcond): - argloc = self._ensure_value_is_boxed(op.getarg(0)) + argloc = self.make_sure_var_in_reg(op.getarg(0)) resloc = self.force_allocate_reg(op.result, [op.getarg(0)]) return [argloc, resloc] def prepare_guard_int_mul_ovf(self, op, guard, fcond): boxes = op.getarglist() - reg1 = self._ensure_value_is_boxed(boxes[0], forbidden_vars=boxes) - reg2 = self._ensure_value_is_boxed(boxes[1], forbidden_vars=boxes) + reg1 = self.make_sure_var_in_reg(boxes[0], forbidden_vars=boxes) + reg2 = self.make_sure_var_in_reg(boxes[1], forbidden_vars=boxes) res = self.force_allocate_reg(op.result) return self._prepare_guard(guard, [reg1, reg2, res]) @@ -576,7 +566,7 @@ prepare_guard_int_is_zero = prepare_op_unary_cmp('int_is_zero') def prepare_op_int_neg(self, op, fcond): - l0 = self._ensure_value_is_boxed(op.getarg(0)) + l0 = self.make_sure_var_in_reg(op.getarg(0)) self.possibly_free_vars_for_op(op) self.free_temp_vars() resloc = self.force_allocate_reg(op.result) @@ -629,15 +619,15 @@ def _prepare_llong_binop_xx(self, op, fcond): # arg 0 is the address of the function - loc0 = self._ensure_value_is_boxed(op.getarg(1)) - loc1 = self._ensure_value_is_boxed(op.getarg(2)) + loc0 = self.make_sure_var_in_reg(op.getarg(1)) + loc1 = self.make_sure_var_in_reg(op.getarg(2)) self.possibly_free_vars_for_op(op) self.free_temp_vars() res = self.vfprm.force_allocate_reg(op.result) return [loc0, loc1, res] def _prepare_llong_to_int(self, op, fcond): - loc0 = self._ensure_value_is_boxed(op.getarg(1)) + loc0 = self.make_sure_var_in_reg(op.getarg(1)) res = self.force_allocate_reg(op.result) return [loc0, res] @@ -665,7 +655,7 @@ return args def prepare_op_guard_true(self, op, fcond): - l0 = self._ensure_value_is_boxed(op.getarg(0)) + l0 = self.make_sure_var_in_reg(op.getarg(0)) args = self._prepare_guard(op, [l0]) return args @@ -677,9 +667,9 @@ boxes = op.getarglist() a0, a1 = boxes imm_a1 = check_imm_box(a1) - l0 = self._ensure_value_is_boxed(a0, boxes) + l0 = self.make_sure_var_in_reg(a0, boxes) if not imm_a1: - l1 = self._ensure_value_is_boxed(a1, boxes) + l1 = self.make_sure_var_in_reg(a1, boxes) else: l1 = self.convert_to_imm(a1) assert op.result is None @@ -699,7 +689,7 @@ def prepare_op_guard_exception(self, op, fcond): boxes = op.getarglist() arg0 = ConstInt(rffi.cast(lltype.Signed, op.getarg(0).getint())) - loc = self._ensure_value_is_boxed(arg0) + loc = self.make_sure_var_in_reg(arg0) loc1 = self.get_scratch_reg(INT, boxes) if op.result in self.longevity: resloc = self.force_allocate_reg(op.result, boxes) @@ -713,7 +703,7 @@ return arglocs def prepare_op_guard_no_exception(self, op, fcond): - loc = self._ensure_value_is_boxed( + loc = self.make_sure_var_in_reg( ConstInt(self.cpu.pos_exception())) arglocs = self._prepare_guard(op, [loc]) return arglocs @@ -727,7 +717,7 @@ assert isinstance(op.getarg(0), Box) boxes = op.getarglist() - x = self._ensure_value_is_boxed(boxes[0], boxes) + x = self.make_sure_var_in_reg(boxes[0], boxes) y_val = rffi.cast(lltype.Signed, op.getarg(1).getint()) arglocs = [x, None, None] @@ -837,8 +827,8 @@ boxes = op.getarglist() a0, a1 = boxes ofs, size, sign = unpack_fielddescr(op.getdescr()) - base_loc = self._ensure_value_is_boxed(a0, boxes) - value_loc = self._ensure_value_is_boxed(a1, boxes) + base_loc = self.make_sure_var_in_reg(a0, boxes) + value_loc = self.make_sure_var_in_reg(a1, boxes) if check_imm_arg(ofs): ofs_loc = imm(ofs) else: @@ -851,7 +841,7 @@ def prepare_op_getfield_gc(self, op, fcond): a0 = op.getarg(0) ofs, size, sign = unpack_fielddescr(op.getdescr()) - base_loc = self._ensure_value_is_boxed(a0) + base_loc = self.make_sure_var_in_reg(a0) immofs = imm(ofs) if check_imm_arg(ofs): ofs_loc = immofs @@ -871,8 +861,8 @@ t = unpack_interiorfielddescr(op.getdescr()) ofs, itemsize, fieldsize, sign = t args = op.getarglist() - base_loc = self._ensure_value_is_boxed(op.getarg(0), args) - index_loc = self._ensure_value_is_boxed(op.getarg(1), args) + base_loc = self.make_sure_var_in_reg(op.getarg(0), args) + index_loc = self.make_sure_var_in_reg(op.getarg(1), args) immofs = imm(ofs) if check_imm_arg(ofs): ofs_loc = immofs @@ -889,9 +879,9 @@ t = unpack_interiorfielddescr(op.getdescr()) ofs, itemsize, fieldsize, sign = t args = op.getarglist() - base_loc = self._ensure_value_is_boxed(op.getarg(0), args) - index_loc = self._ensure_value_is_boxed(op.getarg(1), args) - value_loc = self._ensure_value_is_boxed(op.getarg(2), args) + base_loc = self.make_sure_var_in_reg(op.getarg(0), args) + index_loc = self.make_sure_var_in_reg(op.getarg(1), args) + value_loc = self.make_sure_var_in_reg(op.getarg(2), args) immofs = imm(ofs) if check_imm_arg(ofs): ofs_loc = immofs @@ -907,7 +897,7 @@ assert isinstance(arraydescr, ArrayDescr) ofs = arraydescr.lendescr.offset arg = op.getarg(0) - base_loc = self._ensure_value_is_boxed(arg) + base_loc = self.make_sure_var_in_reg(arg) self.possibly_free_vars_for_op(op) self.free_temp_vars() res = self.force_allocate_reg(op.result) @@ -917,9 +907,9 @@ size, ofs, _ = unpack_arraydescr(op.getdescr()) scale = get_scale(size) args = op.getarglist() - base_loc = self._ensure_value_is_boxed(args[0], args) - ofs_loc = self._ensure_value_is_boxed(args[1], args) - value_loc = self._ensure_value_is_boxed(args[2], args) + base_loc = self.make_sure_var_in_reg(args[0], args) + ofs_loc = self.make_sure_var_in_reg(args[1], args) + value_loc = self.make_sure_var_in_reg(args[2], args) assert check_imm_arg(ofs) return [value_loc, base_loc, ofs_loc, imm(scale), imm(ofs)] prepare_op_setarrayitem_raw = prepare_op_setarrayitem_gc @@ -929,8 +919,8 @@ boxes = op.getarglist() size, ofs, _ = unpack_arraydescr(op.getdescr()) scale = get_scale(size) - base_loc = self._ensure_value_is_boxed(boxes[0], boxes) - ofs_loc = self._ensure_value_is_boxed(boxes[1], boxes) + base_loc = self.make_sure_var_in_reg(boxes[0], boxes) + ofs_loc = self.make_sure_var_in_reg(boxes[1], boxes) self.possibly_free_vars_for_op(op) self.free_temp_vars() res = self.force_allocate_reg(op.result) @@ -944,7 +934,7 @@ def prepare_op_strlen(self, op, fcond): args = op.getarglist() - l0 = self._ensure_value_is_boxed(op.getarg(0)) + l0 = self.make_sure_var_in_reg(op.getarg(0)) basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, self.cpu.translate_support_code) immofs = imm(ofs_length) @@ -963,14 +953,14 @@ def prepare_op_strgetitem(self, op, fcond): boxes = op.getarglist() - base_loc = self._ensure_value_is_boxed(boxes[0]) + base_loc = self.make_sure_var_in_reg(boxes[0]) a1 = boxes[1] imm_a1 = check_imm_box(a1) if imm_a1: ofs_loc = self.convert_to_imm(a1) else: - ofs_loc = self._ensure_value_is_boxed(a1, boxes) + ofs_loc = self.make_sure_var_in_reg(a1, boxes) self.possibly_free_vars_for_op(op) self.free_temp_vars() @@ -983,9 +973,9 @@ def prepare_op_strsetitem(self, op, fcond): boxes = op.getarglist() - base_loc = self._ensure_value_is_boxed(boxes[0], boxes) - ofs_loc = self._ensure_value_is_boxed(boxes[1], boxes) - value_loc = self._ensure_value_is_boxed(boxes[2], boxes) + base_loc = self.make_sure_var_in_reg(boxes[0], boxes) + ofs_loc = self.make_sure_var_in_reg(boxes[1], boxes) + value_loc = self.make_sure_var_in_reg(boxes[2], boxes) basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, self.cpu.translate_support_code) assert itemsize == 1 @@ -995,7 +985,7 @@ prepare_op_copyunicodecontent = void def prepare_op_unicodelen(self, op, fcond): - l0 = self._ensure_value_is_boxed(op.getarg(0)) + l0 = self.make_sure_var_in_reg(op.getarg(0)) basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, self.cpu.translate_support_code) immofs = imm(ofs_length) @@ -1012,8 +1002,8 @@ def prepare_op_unicodegetitem(self, op, fcond): boxes = op.getarglist() - base_loc = self._ensure_value_is_boxed(boxes[0], boxes) - ofs_loc = self._ensure_value_is_boxed(boxes[1], boxes) + base_loc = self.make_sure_var_in_reg(boxes[0], boxes) + ofs_loc = self.make_sure_var_in_reg(boxes[1], boxes) self.possibly_free_vars_for_op(op) self.free_temp_vars() @@ -1027,9 +1017,9 @@ def prepare_op_unicodesetitem(self, op, fcond): boxes = op.getarglist() - base_loc = self._ensure_value_is_boxed(boxes[0], boxes) - ofs_loc = self._ensure_value_is_boxed(boxes[1], boxes) - value_loc = self._ensure_value_is_boxed(boxes[2], boxes) + base_loc = self.make_sure_var_in_reg(boxes[0], boxes) + ofs_loc = self.make_sure_var_in_reg(boxes[1], boxes) + value_loc = self.make_sure_var_in_reg(boxes[2], boxes) basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, self.cpu.translate_support_code) scale = itemsize / 2 @@ -1042,7 +1032,7 @@ if imm_arg: argloc = self.convert_to_imm(arg) else: - argloc = self._ensure_value_is_boxed(arg) + argloc = self.make_sure_var_in_reg(arg) self.possibly_free_vars_for_op(op) self.free_temp_vars() resloc = self.force_allocate_reg(op.result) @@ -1093,7 +1083,7 @@ # twice from the memory. N = op.numargs() args = op.getarglist() - arglocs = [self._ensure_value_is_boxed(op.getarg(i), args) + arglocs = [self.make_sure_var_in_reg(op.getarg(i), args) for i in range(N)] tmp = self.get_scratch_reg(INT, args) assert tmp not in arglocs @@ -1215,7 +1205,7 @@ float_result=False, name='prepare_guard_float_ge') def prepare_op_math_sqrt(self, op, fcond): - loc = self._ensure_value_is_boxed(op.getarg(1)) + loc = self.make_sure_var_in_reg(op.getarg(1)) self.possibly_free_vars_for_op(op) self.free_temp_vars() res = self.vfprm.force_allocate_reg(op.result) @@ -1223,12 +1213,12 @@ return [loc, res] def prepare_op_cast_float_to_int(self, op, fcond): - loc1 = self._ensure_value_is_boxed(op.getarg(0)) + loc1 = self.make_sure_var_in_reg(op.getarg(0)) res = self.rm.force_allocate_reg(op.result) return [loc1, res] def prepare_op_cast_int_to_float(self, op, fcond): - loc1 = self._ensure_value_is_boxed(op.getarg(0)) + loc1 = self.make_sure_var_in_reg(op.getarg(0)) res = self.vfprm.force_allocate_reg(op.result) return [loc1, res] @@ -1247,12 +1237,12 @@ return [loc, res] def prepare_op_cast_float_to_singlefloat(self, op, fcond): - loc1 = self._ensure_value_is_boxed(op.getarg(0)) + loc1 = self.make_sure_var_in_reg(op.getarg(0)) res = self.force_allocate_reg(op.result) return [loc1, res] - + def prepare_op_cast_singlefloat_to_float(self, op, fcond): - loc1 = self._ensure_value_is_boxed(op.getarg(0)) + loc1 = self.make_sure_var_in_reg(op.getarg(0)) res = self.force_allocate_reg(op.result) return [loc1, res] diff --git a/pypy/jit/backend/arm/test/test_gc_integration.py b/pypy/jit/backend/arm/test/test_gc_integration.py --- a/pypy/jit/backend/arm/test/test_gc_integration.py +++ b/pypy/jit/backend/arm/test/test_gc_integration.py @@ -20,7 +20,7 @@ from pypy.jit.backend.arm.test.test_regalloc import BaseTestRegalloc from pypy.jit.backend.arm.regalloc import ARMFrameManager, VFPRegisterManager from pypy.jit.codewriter.effectinfo import EffectInfo -from pypy.jit.backend.arm.regalloc import Regalloc, ARMv7RegisterManager +from pypy.jit.backend.arm.regalloc import Regalloc CPU = getcpuclass() diff --git a/pypy/jit/backend/llsupport/regalloc.py b/pypy/jit/backend/llsupport/regalloc.py --- a/pypy/jit/backend/llsupport/regalloc.py +++ b/pypy/jit/backend/llsupport/regalloc.py @@ -479,7 +479,7 @@ """ raise NotImplementedError("Abstract") - def get_scratch_reg(self, forbidden_vars=[]): + def get_scratch_reg(self, type, forbidden_vars=[], selected_reg=None): """ Platform specific - Allocates a temporary register """ raise NotImplementedError("Abstract") diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py --- a/pypy/jit/codewriter/jtransform.py +++ b/pypy/jit/codewriter/jtransform.py @@ -304,6 +304,8 @@ rewrite_op_convert_float_bytes_to_longlong = _noop_rewrite rewrite_op_convert_longlong_bytes_to_float = _noop_rewrite + cast_ptr_to_weakrefptr = _noop_rewrite + cast_weakrefptr_to_ptr = _noop_rewrite # ---------- # Various kinds of calls @@ -450,6 +452,7 @@ resulttype, extra, extrakey) return SpaceOperation('direct_call', [c_func] + args, op.result) + def _do_builtin_call(self, op, oopspec_name=None, args=None, extra=None, extrakey=None): if oopspec_name is None: oopspec_name = op.opname @@ -482,6 +485,9 @@ rewrite_op_uint_mod = _do_builtin_call rewrite_op_cast_float_to_uint = _do_builtin_call rewrite_op_cast_uint_to_float = _do_builtin_call + rewrite_op_weakref_create = _do_builtin_call + rewrite_op_weakref_deref = _do_builtin_call + rewrite_op_gc_add_memory_pressure = _do_builtin_call # ---------- # getfield/setfield/mallocs etc. diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py --- a/pypy/jit/codewriter/support.py +++ b/pypy/jit/codewriter/support.py @@ -1,15 +1,13 @@ import sys -from pypy.rpython.lltypesystem import lltype, rclass, rffi +from pypy.rpython.lltypesystem import lltype, rclass, rffi, llmemory from pypy.rpython.ootypesystem import ootype from pypy.rpython import rlist from pypy.rpython.lltypesystem import rstr as ll_rstr, rdict as ll_rdict -from pypy.rpython.lltypesystem import rlist as lltypesystem_rlist from pypy.rpython.lltypesystem.module import ll_math from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.ootypesystem import rdict as oo_rdict from pypy.rpython.llinterp import LLInterpreter from pypy.rpython.extregistry import ExtRegistryEntry -from pypy.rpython.annlowlevel import cast_base_ptr_to_instance from pypy.translator.simplify import get_funcobj from pypy.translator.unsimplify import split_block from pypy.objspace.flow.model import Constant @@ -648,6 +646,15 @@ build_ll_1_raw_free_no_track_allocation = ( build_raw_free_builder(track_allocation=False)) + def _ll_1_weakref_create(obj): + return llop.weakref_create(llmemory.WeakRefPtr, obj) + + def _ll_1_weakref_deref(TP, obj): + return llop.weakref_deref(lltype.Ptr(TP), obj) + _ll_1_weakref_deref.need_result_type = True + + def _ll_1_gc_add_memory_pressure(num): + llop.gc_add_memory_pressure(lltype.Void, num) class OOtypeHelpers: diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py --- a/pypy/jit/metainterp/test/test_ajit.py +++ b/pypy/jit/metainterp/test/test_ajit.py @@ -1,24 +1,19 @@ -import math import sys import py -from pypy import conftest -from pypy.jit.codewriter import longlong -from pypy.jit.codewriter.policy import JitPolicy, StopAtXPolicy -from pypy.jit.metainterp import pyjitpl, history -from pypy.jit.metainterp.optimizeopt import ALL_OPTS_DICT +from pypy.jit.codewriter.policy import StopAtXPolicy +from pypy.jit.metainterp import history from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin, noConst -from pypy.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper from pypy.jit.metainterp.warmspot import get_stats from pypy.rlib import rerased from pypy.rlib.jit import (JitDriver, we_are_jitted, hint, dont_look_inside, loop_invariant, elidable, promote, jit_debug, assert_green, AssertGreenFailed, unroll_safe, current_trace_length, look_inside_iff, - isconstant, isvirtual, promote_string, set_param, record_known_class) + isconstant, isvirtual, set_param, record_known_class) from pypy.rlib.longlong2float import float2longlong, longlong2float from pypy.rlib.rarithmetic import ovfcheck, is_valid_int -from pypy.rpython.lltypesystem import lltype, llmemory, rffi +from pypy.rpython.lltypesystem import lltype, rffi from pypy.rpython.ootypesystem import ootype @@ -3962,3 +3957,27 @@ return 42 self.interp_operations(f, [1, 2, 3]) self.check_operations_history(call=1, guard_no_exception=0) + + def test_weakref(self): + import weakref + + class A(object): + def __init__(self, x): + self.x = x + + def f(i): + a = A(i) + w = weakref.ref(a) + return w().x + a.x + + assert self.interp_operations(f, [3]) == 6 + + def test_gc_add_memory_pressure(self): + from pypy.rlib import rgc + + def f(): + rgc.add_memory_pressure(1234) + return 3 + + self.interp_operations(f, []) + diff --git a/pypy/jit/metainterp/test/test_string.py b/pypy/jit/metainterp/test/test_string.py --- a/pypy/jit/metainterp/test/test_string.py +++ b/pypy/jit/metainterp/test/test_string.py @@ -620,3 +620,13 @@ return result res = self.meta_interp(main, [9]) assert res == main(9) + + def test_bytearray(self): + py.test.skip("implement it") + def f(i): + b = bytearray("abc") + b[1] = i + return b[1] + + res = self.interp_operations(f, [13]) + assert res == 13 diff --git a/pypy/module/_cffi_backend/ctypeenum.py b/pypy/module/_cffi_backend/ctypeenum.py --- a/pypy/module/_cffi_backend/ctypeenum.py +++ b/pypy/module/_cffi_backend/ctypeenum.py @@ -67,7 +67,7 @@ except OperationError, e: if not e.match(space, space.w_TypeError): raise - if space.isinstance_w(w_ob, space.w_str): + if space.isinstance_w(w_ob, space.w_basestring): value = self.convert_enum_string_to_int(space.str_w(w_ob)) value = r_ulonglong(value) misc.write_raw_integer_data(cdata, value, self.size) @@ -78,11 +78,14 @@ space = self.space return self.convert_enum_string_to_int(space.str_w(w_ob)) + def cast_unicode(self, w_ob): + return self.cast_str(w_ob) + def convert_enum_string_to_int(self, s): space = self.space if s.startswith('#'): try: - return int(s[1:]) # xxx is it RPython? + return int(s[1:]) except ValueError: raise OperationError(space.w_ValueError, space.wrap("invalid literal after '#'")) diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py --- a/pypy/module/_cffi_backend/misc.py +++ b/pypy/module/_cffi_backend/misc.py @@ -1,7 +1,7 @@ from __future__ import with_statement from pypy.interpreter.error import OperationError, operationerrfmt from pypy.rpython.lltypesystem import lltype, llmemory, rffi -from pypy.rlib.rarithmetic import r_uint, r_ulonglong +from pypy.rlib.rarithmetic import r_uint, r_ulonglong, is_signed_integer_type from pypy.rlib.unroll import unrolling_iterable from pypy.rlib.objectmodel import keepalive_until_here, specialize from pypy.rlib import jit @@ -64,10 +64,16 @@ @specialize.argtype(1) def write_raw_integer_data(target, source, size): - for TP, TPP in _prim_unsigned_types: - if size == rffi.sizeof(TP): - rffi.cast(TPP, target)[0] = rffi.cast(TP, source) - return + if is_signed_integer_type(lltype.typeOf(source)): + for TP, TPP in _prim_signed_types: + if size == rffi.sizeof(TP): + rffi.cast(TPP, target)[0] = rffi.cast(TP, source) + return + else: + for TP, TPP in _prim_unsigned_types: + if size == rffi.sizeof(TP): + rffi.cast(TPP, target)[0] = rffi.cast(TP, source) + return raise NotImplementedError("bad integer size") def write_raw_float_data(target, source, size): diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1309,6 +1309,12 @@ assert p.a1 == "c" e = py.test.raises(TypeError, newp, BStructPtr, [None]) assert "must be a str or int, not NoneType" in str(e.value) + if sys.version_info < (3,): + p.a1 = unicode("def") + assert p.a1 == "def" and type(p.a1) is str + py.test.raises(UnicodeEncodeError, "p.a1 = unichr(1234)") + BEnum2 = new_enum_type(unicode("foo"), (unicode('abc'),), (5,)) + assert string(cast(BEnum2, unicode('abc'))) == 'abc' def test_enum_overflow(): for ovf in (2**63, -2**63-1, 2**31, -2**31-1): diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py --- a/pypy/module/_hashlib/interp_hashlib.py +++ b/pypy/module/_hashlib/interp_hashlib.py @@ -20,9 +20,10 @@ + rffi.sizeof(ropenssl.EVP_MD) * 2 + 208 class W_Hash(Wrappable): - ctx = lltype.nullptr(ropenssl.EVP_MD_CTX.TO) + NULL_CTX = lltype.nullptr(ropenssl.EVP_MD_CTX.TO) + ctx = NULL_CTX - def __init__(self, space, name): + def __init__(self, space, name, copy_from=NULL_CTX): self.name = name digest_type = self.digest_type_by_name(space) self.digest_size = rffi.getintfield(digest_type, 'c_md_size') @@ -35,14 +36,16 @@ ctx = lltype.malloc(ropenssl.EVP_MD_CTX.TO, flavor='raw') rgc.add_memory_pressure(HASH_MALLOC_SIZE + self.digest_size) try: - ropenssl.EVP_DigestInit(ctx, digest_type) + if copy_from: + ropenssl.EVP_MD_CTX_copy(ctx, copy_from) + else: + ropenssl.EVP_DigestInit(ctx, digest_type) self.ctx = ctx except: lltype.free(ctx, flavor='raw') raise def __del__(self): - # self.lock.free() if self.ctx: ropenssl.EVP_MD_CTX_cleanup(self.ctx) lltype.free(self.ctx, flavor='raw') @@ -68,10 +71,8 @@ def copy(self, space): "Return a copy of the hash object." - w_hash = W_Hash(space, self.name) with self.lock: - ropenssl.EVP_MD_CTX_copy(w_hash.ctx, self.ctx) - return w_hash + return W_Hash(space, self.name, copy_from=self.ctx) def digest(self, space): "Return the digest value as a string of binary data." diff --git a/pypy/module/fcntl/test/test_fcntl.py b/pypy/module/fcntl/test/test_fcntl.py --- a/pypy/module/fcntl/test/test_fcntl.py +++ b/pypy/module/fcntl/test/test_fcntl.py @@ -11,7 +11,7 @@ os.unlink(i) class AppTestFcntl: - spaceconfig = dict(usemodules=('fcntl', 'array', 'struct', 'termios', 'select', 'time')) + spaceconfig = dict(usemodules=('fcntl', 'array', 'struct', 'termios', 'select', 'rctime')) def setup_class(cls): tmpprefix = str(udir.ensure('test_fcntl', dir=1).join('tmp_')) cls.w_tmp = cls.space.wrap(tmpprefix) @@ -191,8 +191,11 @@ if child_pid == 0: # We're the child time.sleep(1) - return + os._exit(0) try: + # We're the parent, we want TIOCGPGRP calls after child started but before it dies + time.sleep(0.5) + buf = array.array('i', [0]) res = fcntl.ioctl(mfd, TIOCGPGRP, buf, True) assert res == 0 diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py --- a/pypy/module/imp/test/test_import.py +++ b/pypy/module/imp/test/test_import.py @@ -1268,7 +1268,7 @@ class AppTestMultithreadedImp(object): - spaceconfig = dict(usemodules=['thread', 'time']) + spaceconfig = dict(usemodules=['thread', 'rctime']) def setup_class(cls): #if not conftest.option.runappdirect: diff --git a/pypy/module/micronumpy/__init__.py b/pypy/module/micronumpy/__init__.py --- a/pypy/module/micronumpy/__init__.py +++ b/pypy/module/micronumpy/__init__.py @@ -1,4 +1,5 @@ from pypy.interpreter.mixedmodule import MixedModule +from pypy.module.micronumpy.interp_boxes import long_double_size class Module(MixedModule): @@ -58,6 +59,8 @@ 'float16': 'interp_boxes.W_Float16Box', 'float32': 'interp_boxes.W_Float32Box', 'float64': 'interp_boxes.W_Float64Box', + 'longdouble': 'interp_boxes.W_LongDoubleBox', + 'longfloat': 'interp_boxes.W_LongDoubleBox', 'intp': 'types.IntP.BoxType', 'uintp': 'types.UIntP.BoxType', 'flexible': 'interp_boxes.W_FlexibleBox', @@ -70,6 +73,8 @@ 'complex_': 'interp_boxes.W_Complex128Box', 'complex128': 'interp_boxes.W_Complex128Box', 'complex64': 'interp_boxes.W_Complex64Box', + 'clongdouble': 'interp_boxes.W_CLongDoubleBox', + 'clongfloat': 'interp_boxes.W_CLongDoubleBox', } # ufuncs @@ -163,3 +168,10 @@ 'max': 'app_numpy.max', 'arange': 'app_numpy.arange', } + +if long_double_size == 16: + Module.interpleveldefs['float128'] = 'interp_boxes.W_Float128Box' + Module.interpleveldefs['complex256'] = 'interp_boxes.W_Complex256Box' +elif long_double_size == 12: + Module.interpleveldefs['float96'] = 'interp_boxes.W_Float96Box' + Module.interpleveldefs['complex192'] = 'interp_boxes.W_Complex192Box' diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py --- a/pypy/module/micronumpy/interp_boxes.py +++ b/pypy/module/micronumpy/interp_boxes.py @@ -8,12 +8,20 @@ from pypy.objspace.std.inttype import int_typedef from pypy.objspace.std.complextype import complex_typedef from pypy.rlib.rarithmetic import LONG_BIT +from pypy.rpython.lltypesystem import rffi from pypy.tool.sourcetools import func_with_new_name from pypy.module.micronumpy.arrayimpl.voidbox import VoidBoxStorage MIXIN_32 = (int_typedef,) if LONG_BIT == 32 else () MIXIN_64 = (int_typedef,) if LONG_BIT == 64 else () +# Is this the proper place for this? +long_double_size = rffi.sizeof_c_type('long double', ignore_errors=True) +import os +if long_double_size == 8 and os.name == 'nt': + # this is a lie, or maybe a wish + long_double_size = 12 + def new_dtype_getter(name): def _get_dtype(space): @@ -226,7 +234,6 @@ class W_Float64Box(W_FloatingBox, PrimitiveBox): descr__new__, _get_dtype = new_dtype_getter("float64") - class W_FlexibleBox(W_GenericBox): def __init__(self, arr, ofs, dtype): self.arr = arr # we have to keep array alive @@ -315,6 +322,33 @@ descr__new__, _get_dtype = new_dtype_getter("complex128") _COMPONENTS_BOX = W_Float64Box +if long_double_size == 12: + class W_Float96Box(W_FloatingBox, PrimitiveBox): + descr__new__, _get_dtype = new_dtype_getter("float96") + + W_LongDoubleBox = W_Float96Box + + class W_Complex192Box(ComplexBox, W_ComplexFloatingBox): + descr__new__, _get_dtype = new_dtype_getter("complex192") + _COMPONENTS_BOX = W_Float96Box + + W_CLongDoubleBox = W_Complex192Box + +elif long_double_size == 16: + class W_Float128Box(W_FloatingBox, PrimitiveBox): + descr__new__, _get_dtype = new_dtype_getter("float128") + W_LongDoubleBox = W_Float128Box + + class W_Complex256Box(ComplexBox, W_ComplexFloatingBox): + descr__new__, _get_dtype = new_dtype_getter("complex256") + _COMPONENTS_BOX = W_Float128Box + + W_CLongDoubleBox = W_Complex256Box + +else: + W_LongDoubleBox = W_Float64Box + W_CLongDoubleBox = W_Complex64Box + W_GenericBox.typedef = TypeDef("generic", __module__ = "numpypy", @@ -479,6 +513,33 @@ __new__ = interp2app(W_Float64Box.descr__new__.im_func), ) +if long_double_size == 12: + W_Float96Box.typedef = TypeDef("float96", (W_FloatingBox.typedef), + __module__ = "numpypy", + + __new__ = interp2app(W_Float96Box.descr__new__.im_func), + ) + + W_Complex192Box.typedef = TypeDef("complex192", (W_ComplexFloatingBox.typedef, complex_typedef), + __module__ = "numpypy", + __new__ = interp2app(W_Complex192Box.descr__new__.im_func), + real = GetSetProperty(W_ComplexFloatingBox.descr_get_real), + imag = GetSetProperty(W_ComplexFloatingBox.descr_get_imag), + ) + +elif long_double_size == 16: + W_Float128Box.typedef = TypeDef("float128", (W_FloatingBox.typedef), + __module__ = "numpypy", + + __new__ = interp2app(W_Float128Box.descr__new__.im_func), + ) + + W_Complex256Box.typedef = TypeDef("complex256", (W_ComplexFloatingBox.typedef, complex_typedef), + __module__ = "numpypy", + __new__ = interp2app(W_Complex256Box.descr__new__.im_func), + real = GetSetProperty(W_ComplexFloatingBox.descr_get_real), + imag = GetSetProperty(W_ComplexFloatingBox.descr_get_imag), + ) W_FlexibleBox.typedef = TypeDef("flexible", W_GenericBox.typedef, __module__ = "numpypy", diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -15,6 +15,7 @@ SIGNEDLTR = "i" BOOLLTR = "b" FLOATINGLTR = "f" +COMPLEXLTR = "c" VOIDLTR = 'V' STRINGLTR = 'S' UNICODELTR = 'U' @@ -135,7 +136,7 @@ return self.kind == SIGNEDLTR def is_complex_type(self): - return (self.num == 14 or self.num == 15) + return (self.num == 14 or self.num == 15 or self.num == 16) def is_bool_type(self): return self.kind == BOOLLTR @@ -412,17 +413,10 @@ alternate_constructors=[space.w_float], aliases=["float"], ) - # self.w_float128dtype = W_Dtype( - # types.Float128(), - # num=13, - # kind=FLOATINGLTR, - # name="float128", - # ... - # ) self.w_complex64dtype = W_Dtype( types.Complex64(), num=14, - kind=FLOATINGLTR, + kind=COMPLEXLTR, name="complex64", char="F", w_box_type = space.gettypefor(interp_boxes.W_Complex64Box), @@ -430,13 +424,64 @@ self.w_complex128dtype = W_Dtype( types.Complex128(), num=15, - kind=FLOATINGLTR, + kind=COMPLEXLTR, name="complex128", char="D", w_box_type = space.gettypefor(interp_boxes.W_Complex128Box), alternate_constructors=[space.w_complex], aliases=["complex"], ) + if interp_boxes.long_double_size == 12: + self.w_float96dtype = W_Dtype( + types.Float96(), + num=13, + kind=FLOATINGLTR, + name="float96", + char="g", + w_box_type=space.gettypefor(interp_boxes.W_Float96Box), + aliases=["longfloat", "longdouble"], + ) + self.w_longdouble = self.w_float96dtype + + self.w_complex192dtype = W_Dtype( + types.Complex192(), + num=16, + kind=COMPLEXLTR, + name="complex192", + char="G", + w_box_type = space.gettypefor(interp_boxes.W_Complex192Box), + alternate_constructors=[space.w_complex], + aliases=["clongdouble", "clongfloat"], + ) + self.w_clongdouble = self.w_complex192dtype + + elif interp_boxes.long_double_size == 16: + self.w_float128dtype = W_Dtype( + types.Float128(), + num=13, + kind=FLOATINGLTR, + name="float128", + char="g", + w_box_type=space.gettypefor(interp_boxes.W_Float128Box), + aliases=["longfloat", "longdouble"], + ) + self.w_longdouble = self.w_float128dtype + + self.w_complex256dtype = W_Dtype( + types.Complex256(), + num=16, + kind=COMPLEXLTR, + name="complex256", + char="G", + w_box_type = space.gettypefor(interp_boxes.W_Complex256Box), + alternate_constructors=[space.w_complex], + aliases=["clongdouble", "clongfloat"], + ) + self.w_clongdouble = self.w_complex256dtype + else: + self.w_float64dtype.aliases += ["longfloat", "longdouble"] + self.w_longdouble = self.w_float64dtype + self.w_clongdouble = self.w_complex64dtype self.w_stringdtype = W_Dtype( types.StringType(1), num=18, @@ -507,14 +552,16 @@ self.w_int16dtype, self.w_uint16dtype, self.w_int32dtype, self.w_uint32dtype, self.w_longdtype, self.w_ulongdtype, self.w_int64dtype, self.w_uint64dtype, - self.w_float16dtype, self.w_float32dtype, self.w_float64dtype, self.w_complex64dtype, - self.w_complex128dtype, + self.w_float16dtype, self.w_float32dtype, self.w_float64dtype, + self.w_longdouble, + self.w_complex64dtype, self.w_complex128dtype, self.w_clongdouble, self.w_stringdtype, self.w_unicodedtype, self.w_voiddtype, self.w_intpdtype, self.w_uintpdtype, ] self.float_dtypes_by_num_bytes = sorted( (dtype.itemtype.get_element_size(), dtype) - for dtype in [self.w_float16dtype, self.w_float32dtype, self.w_float64dtype] + for dtype in [self.w_float16dtype, self.w_float32dtype, + self.w_float64dtype, self.w_longdouble] ) self.dtypes_by_name = {} # we reverse, so the stuff with lower numbers override stuff with @@ -540,7 +587,7 @@ 'LONGLONG': self.w_int64dtype, 'SHORT': self.w_int16dtype, 'VOID': self.w_voiddtype, - #'LONGDOUBLE':, + 'LONGDOUBLE': self.w_longdouble, 'UBYTE': self.w_uint8dtype, 'UINTP': self.w_ulongdtype, 'ULONG': self.w_ulongdtype, @@ -549,7 +596,7 @@ #'OBJECT', 'ULONGLONG': self.w_uint64dtype, 'STRING': self.w_stringdtype, - #'CDOUBLE', + 'CDOUBLE': self.w_complex64dtype, #'DATETIME', 'UINT': self.w_uint32dtype, 'INTP': self.w_intpdtype, @@ -563,7 +610,7 @@ 'USHORT': self.w_uint16dtype, 'FLOAT': self.w_float32dtype, 'BOOL': self.w_booldtype, - #, 'CLONGDOUBLE'] + 'CLONGDOUBLE': self.w_clongdouble, } typeinfo_partial = { 'Generic': interp_boxes.W_GenericBox, @@ -573,7 +620,7 @@ 'Integer': interp_boxes.W_IntegerBox, 'SignedInteger': interp_boxes.W_SignedIntegerBox, 'UnsignedInteger': interp_boxes.W_UnsignedIntegerBox, - #'ComplexFloating', + 'ComplexFloating': interp_boxes.W_ComplexFloatingBox, 'Number': interp_boxes.W_NumberBox, 'Floating': interp_boxes.W_FloatingBox } diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py --- a/pypy/module/micronumpy/interp_ufuncs.py +++ b/pypy/module/micronumpy/interp_ufuncs.py @@ -375,12 +375,17 @@ if promote_bools and (dt1.kind == dt2.kind == interp_dtype.BOOLLTR): return interp_dtype.get_dtype_cache(space).w_int8dtype - # Everything promotes to complex - if dt2.num == 14 or dt2.num == 15 or dt1.num == 14 or dt2.num == 15: - if dt2.num == 15 or dt1.num == 15: + # Everything numeric promotes to complex + if dt2.is_complex_type() or dt1.is_complex_type(): + if dt2.num == 14: + return interp_dtype.get_dtype_cache(space).w_complex64dtype + elif dt2.num == 15: return interp_dtype.get_dtype_cache(space).w_complex128dtype + elif dt2.num == 16: + return interp_dtype.get_dtype_cache(space).w_clongdouble else: - return interp_dtype.get_dtype_cache(space).w_complex64dtype + raise OperationError(space.w_TypeError, space.wrap("Unsupported types")) + if promote_to_float: return find_unaryop_result_dtype(space, dt2, promote_to_float=True) @@ -431,7 +436,7 @@ if not allow_complex and (dt.is_complex_type()): raise OperationError(space.w_TypeError, space.wrap("Unsupported types")) if promote_to_float: - if dt.kind == interp_dtype.FLOATINGLTR: + if dt.kind == interp_dtype.FLOATINGLTR or dt.kind==interp_dtype.COMPLEXLTR: return dt if dt.num >= 5: return interp_dtype.get_dtype_cache(space).w_float64dtype diff --git a/pypy/module/micronumpy/test/test_complex.py b/pypy/module/micronumpy/test/test_complex.py --- a/pypy/module/micronumpy/test/test_complex.py +++ b/pypy/module/micronumpy/test/test_complex.py @@ -139,7 +139,6 @@ def test_fmax(self): from _numpypy import fmax, array - import math nnan, nan, inf, ninf = float('-nan'), float('nan'), float('inf'), float('-inf') a = array((complex(ninf, 10), complex(10, ninf), complex( inf, 10), complex(10, inf), @@ -167,7 +166,6 @@ def test_fmin(self): from _numpypy import fmin, array - import math nnan, nan, inf, ninf = float('-nan'), float('nan'), float('inf'), float('-inf') a = array((complex(ninf, 10), complex(10, ninf), complex( inf, 10), complex(10, inf), @@ -198,7 +196,7 @@ raises(TypeError, signbit, complex(1,1)) def test_reciprocal(self): - from _numpypy import array, reciprocal, complex64, complex128 + from _numpypy import array, reciprocal, complex64, complex128, clongdouble inf = float('inf') nan = float('nan') @@ -214,7 +212,7 @@ complex(-r, i), -0j, 0j, cnan, cnan, cnan, cnan] - for c, rel_err in ((complex64, 2e-7), (complex128, 2e-15), ): + for c, rel_err in ((complex64, 2e-7), (complex128, 2e-15), (clongdouble, 2e-15)): actual = reciprocal(array([orig], dtype=c)) for b, a, e in zip(orig, actual, expected): assert (a[0].real - e.real) < rel_err @@ -234,13 +232,12 @@ raises(TypeError, copysign, a, b) def test_exp2(self): - import math - from _numpypy import array, exp2, complex128, complex64 + from _numpypy import array, exp2, complex128, complex64, clongfloat inf = float('inf') ninf = -float('inf') nan = float('nan') cmpl = complex - for c,rel_err in ((complex128, 2e-15), (complex64, 1e-7)): + for c,rel_err in ((complex128, 2e-15), (complex64, 1e-7), (clongfloat, 2e-15)): a = [cmpl(-5., 0), cmpl(-5., -5.), cmpl(-5., 5.), cmpl(0., -5.), cmpl(0., 0.), cmpl(0., 5.), cmpl(-0., -5.), cmpl(-0., 0.), cmpl(-0., 5.), @@ -499,7 +496,7 @@ def test_basic(self): from _numpypy import (complex128, complex64, add, array, dtype, subtract as sub, multiply, divide, negative, abs, floor_divide, - real, imag, sign) + real, imag, sign, clongfloat) from _numpypy import (equal, not_equal, greater, greater_equal, less, less_equal, isnan) assert real(4.0) == 4.0 @@ -507,7 +504,7 @@ a = array([complex(3.0, 4.0)]) b = a.real assert b.dtype == dtype(float) - for complex_ in complex64, complex128: + for complex_ in complex64, complex128, clongfloat: O = complex(0, 0) c0 = complex_(complex(2.5, 0)) diff --git a/pypy/module/micronumpy/test/test_dtypes.py b/pypy/module/micronumpy/test/test_dtypes.py --- a/pypy/module/micronumpy/test/test_dtypes.py +++ b/pypy/module/micronumpy/test/test_dtypes.py @@ -116,8 +116,11 @@ def test_bool_binop_types(self): from _numpypy import array, dtype types = [ - '?', 'b', 'B', 'h', 'H', 'i', 'I', 'l', 'L', 'q', 'Q', 'f', 'd', 'e', + '?', 'b', 'B', 'h', 'H', 'i', 'I', 'l', 'L', 'q', 'Q', 'f', 'd', + 'e' ] + if array([0], dtype='longdouble').itemsize > 8: + types += ['g', 'G'] a = array([True], '?') for t in types: assert (a + array([0], t)).dtype is dtype(t) @@ -233,6 +236,7 @@ (numpy.float16, 10.), (numpy.float32, 2.0), (numpy.float64, 4.32), + (numpy.longdouble, 4.32), ]: assert hash(tp(value)) == hash(value) @@ -469,6 +473,19 @@ assert numpy.float64('23.4') == numpy.float64(23.4) raises(ValueError, numpy.float64, '23.2df') + def test_longfloat(self): + import _numpypy as numpy + # it can be float96 or float128 + if numpy.longfloat != numpy.float64: + assert numpy.longfloat.mro()[1:] == [numpy.floating, + numpy.inexact, numpy.number, + numpy.generic, object] + a = numpy.array([1, 2, 3], numpy.longdouble) + assert repr(type(a[1])) == repr(numpy.longdouble) + assert numpy.float64(12) == numpy.longdouble(12) + assert numpy.float64(12) == numpy.longfloat(12) + raises(ValueError, numpy.longfloat, '23.2df') + def test_complex_floating(self): import _numpypy as numpy @@ -526,6 +543,12 @@ assert numpy.dtype(complex).type is numpy.complex128 assert numpy.dtype("complex").type is numpy.complex128 + d = numpy.dtype('complex64') + assert d.kind == 'c' + assert d.num == 14 + assert d.char == 'F' + + def test_subclass_type(self): import _numpypy as numpy diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -2110,7 +2110,7 @@ def test_fromstring_types(self): from _numpypy import (fromstring, int8, int16, int32, int64, uint8, - uint16, uint32, float16, float32, float64) + uint16, uint32, float16, float32, float64, longfloat, array) a = fromstring('\xFF', dtype=int8) assert a[0] == -1 b = fromstring('\xFF', dtype=uint8) @@ -2133,9 +2133,21 @@ assert j[0] == 12 k = fromstring(self.float16val, dtype=float16) assert k[0] == float16(5.) + dt = array([5],dtype=longfloat).dtype + if dt.itemsize == 12: + from _numpypy import float96 + m = fromstring('\x00\x00\x00\x00\x00\x00\x00\xa0\x01@\x00\x00', dtype=float96) + elif dt.itemsize==16: + from _numpypy import float128 + m = fromstring('\x00\x00\x00\x00\x00\x00\x00\xa0\x01@\x00\x00\x00\x00\x00\x00', dtype=float128) + elif dt.itemsize == 8: + skip('longfloat is float64') + else: + skip('unknown itemsize for longfloat') + assert m[0] == longfloat(5.) def test_fromstring_invalid(self): - from _numpypy import fromstring, uint16, uint8, int32 + from _numpypy import fromstring, uint16, uint8 #default dtype is 64-bit float, so 3 bytes should fail raises(ValueError, fromstring, "\x01\x02\x03") #3 bytes is not modulo 2 bytes (int16) diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -14,7 +14,8 @@ from pypy.rpython.lltypesystem import lltype, rffi from pypy.rlib.rstruct.runpack import runpack from pypy.rlib.rstruct.nativefmttable import native_is_bigendian -from pypy.rlib.rstruct.ieee import float_pack, float_unpack, unpack_float +from pypy.rlib.rstruct.ieee import (float_pack, float_unpack, + unpack_float, unpack_float128) from pypy.tool.sourcetools import func_with_new_name from pypy.rlib import jit from pypy.rlib.rstring import StringBuilder @@ -1496,7 +1497,6 @@ ComponentBoxType = interp_boxes.W_Float32Box - NonNativeComplex64 = Complex64 class Complex128(ComplexFloating, BaseType): @@ -1510,6 +1510,60 @@ NonNativeComplex128 = Complex128 +if interp_boxes.long_double_size == 12: + class Float96(BaseType, Float): + _attrs_ = () + + T = rffi.LONGDOUBLE + BoxType = interp_boxes.W_Float96Box + format_code = "q" + + def runpack_str(self, s): + assert len(s) == 12 + fval = unpack_float128(s, native_is_bigendian) + return self.box(fval) + + class NonNativeFloat96(Float96): + pass + + class Complex192(ComplexFloating, BaseType): + _attrs_ = () + + T = rffi.CHAR + _COMPONENTS_T = rffi.LONGDOUBLE + BoxType = interp_boxes.W_Complex192Box + ComponentBoxType = interp_boxes.W_Float96Box + + NonNativeComplex192 = Complex192 + + +elif interp_boxes.long_double_size == 16: + class Float128(BaseType, Float): + _attrs_ = () + + T = rffi.LONGDOUBLE + BoxType = interp_boxes.W_Float128Box + format_code = "q" + + def runpack_str(self, s): + assert len(s) == 16 From noreply at buildbot.pypy.org Thu Jan 3 12:52:01 2013 From: noreply at buildbot.pypy.org (bivab) Date: Thu, 3 Jan 2013 12:52:01 +0100 (CET) Subject: [pypy-commit] pypy remove-globals-in-jit: merge Message-ID: <20130103115201.802F51C078F@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: remove-globals-in-jit Changeset: r59681:a61e0bd22263 Date: 2013-01-02 15:07 +0100 http://bitbucket.org/pypy/pypy/changeset/a61e0bd22263/ Log: merge diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py --- a/pypy/jit/backend/arm/assembler.py +++ b/pypy/jit/backend/arm/assembler.py @@ -1,5 +1,6 @@ from __future__ import with_statement import os +from pypy.jit.backend.llsupport import jitframe from pypy.jit.backend.arm.helper.assembler import saved_registers from pypy.jit.backend.arm import conditions as c from pypy.jit.backend.arm import registers as r @@ -30,7 +31,7 @@ from pypy.rlib.objectmodel import compute_unique_id # XXX Move to llsupport -from pypy.jit.backend.x86.support import values_array, memcpy_fn +from pypy.jit.backend.x86.support import memcpy_fn DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', ('i', lltype.Signed), ('type', lltype.Char), # 'b'ridge, 'l'abel or @@ -44,14 +45,8 @@ debug = True - def __init__(self, cpu, failargs_limit=1000): + def __init__(self, cpu, translate_support_code=False): self.cpu = cpu - self.fail_boxes_int = values_array(lltype.Signed, failargs_limit) - self.fail_boxes_float = values_array(longlong.FLOATSTORAGE, - failargs_limit) - self.fail_boxes_ptr = values_array(llmemory.GCREF, failargs_limit) - self.fail_boxes_count = 0 - self.fail_force_index = 0 self.setup_failure_recovery() self.mc = None self.memcpy_addr = 0 @@ -68,6 +63,7 @@ self._debug = False self.loop_run_counters = [] self.debug_counter_descr = cpu.fielddescrof(DEBUG_COUNTER, 'i') + self.force_token_to_dead_frame = {} # XXX temporary hack def set_debug(self, v): r = self._debug @@ -112,9 +108,13 @@ gc_ll_descr.initialize() self._build_wb_slowpath(False) self._build_wb_slowpath(True) + self._build_failure_recovery(exc=True, withfloats=False) + self._build_failure_recovery(exc=False, withfloats=False) if self.cpu.supports_floats: self._build_wb_slowpath(False, withfloats=True) self._build_wb_slowpath(True, withfloats=True) + self._build_failure_recovery(exc=True, withfloats=True) + self._build_failure_recovery(exc=False, withfloats=True) self._build_propagate_exception_path() if gc_ll_descr.get_malloc_slowpath_addr is not None: self._build_malloc_slowpath() @@ -122,10 +122,7 @@ if gc_ll_descr.gcrootmap and gc_ll_descr.gcrootmap.is_shadow_stack: self._build_release_gil(gc_ll_descr.gcrootmap) self.memcpy_addr = self.cpu.cast_ptr_to_int(memcpy_fn) - self._exit_code_addr = self._gen_exit_path() - self._leave_jitted_hook_save_exc = \ - self._gen_leave_jitted_hook_code(True) - self._leave_jitted_hook = self._gen_leave_jitted_hook_code(False) + if not self._debug: # if self._debug is already set it means that someone called # set_debug by hand before initializing the assembler. Leave it @@ -220,36 +217,17 @@ self.releasegil_addr = rffi.cast(lltype.Signed, releasegil_func) self.reacqgil_addr = rffi.cast(lltype.Signed, reacqgil_func) - def _gen_leave_jitted_hook_code(self, save_exc): - mc = ARMv7Builder() - if self.cpu.supports_floats: - floats = r.caller_vfp_resp - else: - floats = [] - with saved_registers(mc, r.caller_resp + [r.lr], floats): - addr = self.cpu.get_on_leave_jitted_int(save_exception=save_exc) - mc.BL(addr) - assert self._exit_code_addr != 0 - mc.B(self._exit_code_addr) - return mc.materialize(self.cpu.asmmemmgr, [], - self.cpu.gc_ll_descr.gcrootmap) - def _build_propagate_exception_path(self): if self.cpu.propagate_exception_v < 0: return # not supported (for tests, or non-translated) # mc = ARMv7Builder() - # call on_leave_jitted_save_exc() - if self.cpu.supports_floats: - floats = r.caller_vfp_resp - else: - floats = [] - with saved_registers(mc, r.caller_resp + [r.lr], floats): - addr = self.cpu.get_on_leave_jitted_int(save_exception=True, - default_to_memoryerror=True) - mc.BL(addr) - mc.gen_load_int(r.ip.value, self.cpu.propagate_exception_v) - mc.MOV_rr(r.r0.value, r.ip.value) + # + # Call the helper, which will return a dead frame object with + # the correct exception set, or MemoryError by default + # XXX make sure we return the correct value here + addr = rffi.cast(lltype.Signed, self.cpu.get_propagate_exception()) + mc.BL(addr) self.gen_func_epilog(mc=mc) self.propagate_exception_path = mc.materialize(self.cpu.asmmemmgr, []) @@ -282,12 +260,12 @@ # restore registers and return # We check for c.EQ here, meaning all bits zero in this case mc.POP([reg.value for reg in r.argument_regs] + [r.pc.value], cond=c.EQ) - # call on_leave_jitted_save_exc() - addr = self.cpu.get_on_leave_jitted_int(save_exception=True) + # + # Call the helper, which will return a dead frame object with + # the correct exception set, or MemoryError by default + addr = rffi.cast(lltype.Signed, self.cpu.get_propagate_exception()) mc.BL(addr) # - mc.gen_load_int(r.r0.value, self.cpu.propagate_exception_v) - # # footer -- note the ADD, which skips the return address of this # function, and will instead return to the caller's caller. Note # also that we completely ignore the saved arguments, because we @@ -339,39 +317,81 @@ def setup_failure_recovery(self): - @rgc.no_collect + #@rgc.no_collect -- XXX still true, but hacked gc_set_extra_threshold def failure_recovery_func(mem_loc, frame_pointer, stack_pointer): """mem_loc is a structure in memory describing where the values for the failargs are stored. frame loc is the address of the frame pointer for the frame to be decoded frame """ - vfp_registers = rffi.cast(rffi.LONGLONGP, stack_pointer) - registers = rffi.ptradd(vfp_registers, len(r.all_vfp_regs)) + vfp_registers = rffi.cast(rffi.LONGP, stack_pointer) + registers = rffi.ptradd(vfp_registers, 2*len(r.all_vfp_regs)) registers = rffi.cast(rffi.LONGP, registers) - return self.decode_registers_and_descr(mem_loc, frame_pointer, + return self.grab_frame_values(self.cpu, mem_loc, frame_pointer, registers, vfp_registers) + self.failure_recovery_code = [0, 0, 0, 0] self.failure_recovery_func = failure_recovery_func - recovery_func_sign = lltype.Ptr(lltype.FuncType([lltype.Signed] * 3, - lltype.Signed)) + _FAILURE_RECOVERY_FUNC = lltype.Ptr(lltype.FuncType([rffi.LONGP] * 3, + llmemory.GCREF)) - @rgc.no_collect - def decode_registers_and_descr(self, mem_loc, frame_pointer, + @staticmethod + def grab_frame_values(cpu, mem_loc, frame_pointer, registers, vfp_registers): - """Decode locations encoded in memory at mem_loc and write the values - to the failboxes. Values for spilled vars and registers are stored on - stack at frame_loc """ - assert frame_pointer & 1 == 0 - self.fail_force_index = frame_pointer + # no malloc allowed here!! xxx apart from one, hacking a lot + force_index = rffi.cast(lltype.Signed, frame_pointer) + num = 0 + deadframe = lltype.nullptr(jitframe.DEADFRAME) bytecode = rffi.cast(rffi.UCHARP, mem_loc) + # step 1: lots of mess just to count the final value of 'num' + bytecode1 = bytecode + while 1: + code = rffi.cast(lltype.Signed, bytecode1[0]) + bytecode1 = rffi.ptradd(bytecode1, 1) + if code >= AssemblerARM.CODE_FROMSTACK: + while code > 0x7F: + code = rffi.cast(lltype.Signed, bytecode1[0]) + bytecode1 = rffi.ptradd(bytecode1, 1) + else: + kind = code & 3 + if kind == AssemblerARM.DESCR_SPECIAL: + if code == AssemblerARM.CODE_HOLE: + num += 1 + continue + if code == AssemblerARM.CODE_INPUTARG: + continue + if code == AssemblerARM.CODE_FORCED: + # resuming from a GUARD_NOT_FORCED + token = force_index + deadframe = ( + cpu.assembler.force_token_to_dead_frame.pop(token)) + deadframe = lltype.cast_opaque_ptr( + jitframe.DEADFRAMEPTR, deadframe) + continue + assert code == AssemblerARM.CODE_STOP + break + num += 1 + + # allocate the deadframe + if not deadframe: + # Remove the "reserve" at the end of the nursery. This means + # that it is guaranteed that the following malloc() works + # without requiring a collect(), but it needs to be re-added + # as soon as possible. + cpu.gc_clear_extra_threshold() + assert num <= cpu.get_failargs_limit() + try: + deadframe = lltype.malloc(jitframe.DEADFRAME, num) + except MemoryError: + fatalerror("memory usage error in grab_frame_values") + # fill it + code_inputarg = False num = 0 - value = 0 - fvalue = 0 - code_inputarg = False - while True: + value_hi = 0 + while 1: + # decode the next instruction from the bytecode code = rffi.cast(lltype.Signed, bytecode[0]) bytecode = rffi.ptradd(bytecode, 1) - if code >= self.CODE_FROMSTACK: + if code >= AssemblerARM.CODE_FROMSTACK: if code > 0x7F: shift = 7 code &= 0x7F @@ -384,54 +404,63 @@ break # load the value from the stack kind = code & 3 - code = int((code - self.CODE_FROMSTACK) >> 2) + code = (code - AssemblerARM.CODE_FROMSTACK) >> 2 if code_inputarg: code = ~code code_inputarg = False - if kind == self.DESCR_FLOAT: - # we use code + 1 to get the hi word of the double worded float - stackloc = frame_pointer - get_fp_offset(int(code) + 1) - assert stackloc & 3 == 0 - fvalue = rffi.cast(rffi.LONGLONGP, stackloc)[0] - else: - stackloc = frame_pointer - get_fp_offset(int(code)) - assert stackloc & 1 == 0 - value = rffi.cast(rffi.LONGP, stackloc)[0] + stackloc = force_index - get_fp_offset(int(code)) + value = rffi.cast(rffi.LONGP, stackloc)[0] + if kind == AssemblerARM.DESCR_FLOAT: + assert WORD == 4 + value_hi = value + value = rffi.cast(rffi.LONGP, stackloc - WORD)[0] else: - # 'code' identifies a register: load its value kind = code & 3 - if kind == self.DESCR_SPECIAL: - if code == self.CODE_HOLE: + if kind == AssemblerARM.DESCR_SPECIAL: + if code == AssemblerARM.CODE_HOLE: num += 1 continue - if code == self.CODE_INPUTARG: + if code == AssemblerARM.CODE_INPUTARG: code_inputarg = True continue - assert code == self.CODE_STOP + if code == AssemblerARM.CODE_FORCED: + continue + assert code == AssemblerARM.CODE_STOP break + # 'code' identifies a register: load its value code >>= 2 - if kind == self.DESCR_FLOAT: - fvalue = vfp_registers[code] + if kind == AssemblerARM.DESCR_FLOAT: + if WORD == 4: + value = vfp_registers[2*code] + value_hi = vfp_registers[2*code + 1] + else: + value = registers[code] else: value = registers[code] # store the loaded value into fail_boxes_ - if kind == self.DESCR_FLOAT: - tgt = self.fail_boxes_float.get_addr_for_num(num) - rffi.cast(rffi.LONGLONGP, tgt)[0] = fvalue + if kind == AssemblerARM.DESCR_INT: + deadframe.jf_values[num].int = value + elif kind == AssemblerARM.DESCR_REF: + deadframe.jf_values[num].ref = rffi.cast(llmemory.GCREF, value) + elif kind == AssemblerARM.DESCR_FLOAT: + assert WORD == 4 + assert not longlong.is_64_bit + floatvalue = rffi.cast(lltype.SignedLongLong, value_hi) + floatvalue <<= 32 + floatvalue |= rffi.cast(lltype.SignedLongLong, + rffi.cast(lltype.Unsigned, value)) + deadframe.jf_values[num].float = floatvalue else: - if kind == self.DESCR_INT: - tgt = self.fail_boxes_int.get_addr_for_num(num) - elif kind == self.DESCR_REF: - assert (value & 3) == 0, "misaligned pointer" - tgt = self.fail_boxes_ptr.get_addr_for_num(num) - else: - assert 0, "bogus kind" - rffi.cast(rffi.LONGP, tgt)[0] = value + assert 0, "bogus kind" num += 1 - self.fail_boxes_count = num + # + assert num == len(deadframe.jf_values) + if not we_are_translated(): + assert bytecode[4] == 0xCC fail_index = rffi.cast(rffi.INTP, bytecode)[0] - fail_index = rffi.cast(lltype.Signed, fail_index) - return fail_index + fail_descr = cpu.get_fail_descr_from_number(fail_index) + deadframe.jf_descr = fail_descr.hide(cpu) + return lltype.cast_opaque_ptr(llmemory.GCREF, deadframe) def decode_inputargs(self, code): descr_to_box_type = [REF, INT, FLOAT] @@ -514,12 +543,26 @@ self.mc.CMP_ri(r.r0.value, 0) self.mc.B(self.propagate_exception_path, c=c.EQ) - def _gen_exit_path(self): + def _build_failure_recovery(self, exc, withfloats=False): mc = ARMv7Builder() - decode_registers_addr = llhelper(self.recovery_func_sign, + decode_registers_addr = llhelper(self._FAILURE_RECOVERY_FUNC, self.failure_recovery_func) self._insert_checks(mc) - with saved_registers(mc, r.all_regs, r.all_vfp_regs): + if withfloats: + f = r.all_vfp_regs + else: + f = [] + with saved_registers(mc, r.all_regs, f): + if exc: + # We might have an exception pending. Load it into r4 + # (this is a register saved across calls) + mc.gen_load_int(r.r5.value, self.cpu.pos_exc_value()) + mc.LDR_ri(r.r4.value, self.cpu.pos_exc_value()) + # clear the exc flags + mc.gen_load_int(r.r6.value, 0) + mc.STR_ri(r.r6.value, r.r5.value) + mc.gen_load_int(r.r5.value, self.cpu.pos_exception()) + mc.STR_ri(r.r6.value, r.r5.value) # move mem block address, to r0 to pass as mc.MOV_rr(r.r0.value, r.lr.value) # pass the current frame pointer as second param @@ -528,22 +571,31 @@ mc.MOV_rr(r.r2.value, r.sp.value) self._insert_checks(mc) mc.BL(rffi.cast(lltype.Signed, decode_registers_addr)) + if exc: + # save ebx into 'jf_guard_exc' + from pypy.jit.backend.llsupport.descr import unpack_fielddescr + descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) + offset, size, _ = unpack_fielddescr(descrs.jf_guard_exc) + mc.STR_rr(r.r4.value, r.r0.value, offset, cond=c.AL) mc.MOV_rr(r.ip.value, r.r0.value) mc.MOV_rr(r.r0.value, r.ip.value) self.gen_func_epilog(mc=mc) - return mc.materialize(self.cpu.asmmemmgr, [], + rawstart = mc.materialize(self.cpu.asmmemmgr, [], self.cpu.gc_ll_descr.gcrootmap) + self.failure_recovery_code[exc + 2 * withfloats] = rawstart + self.mc = None DESCR_REF = 0x00 DESCR_INT = 0x01 DESCR_FLOAT = 0x02 DESCR_SPECIAL = 0x03 CODE_FROMSTACK = 64 - CODE_STOP = 0 | DESCR_SPECIAL - CODE_HOLE = 4 | DESCR_SPECIAL - CODE_INPUTARG = 8 | DESCR_SPECIAL + CODE_STOP = 0 | DESCR_SPECIAL + CODE_HOLE = 4 | DESCR_SPECIAL + CODE_INPUTARG = 8 | DESCR_SPECIAL + CODE_FORCED = 12 | DESCR_SPECIAL #XXX where should this be written? - def gen_descr_encoding(self, descr, failargs, locs): + def write_failure_recovery_description(self, descr, failargs, locs): assert self.mc is not None for i in range(len(failargs)): arg = failargs[i] @@ -575,26 +627,33 @@ self.mc.writechar(chr(n)) self.mc.writechar(chr(self.CODE_STOP)) - fdescr = self.cpu.get_fail_descr_number(descr) - self.mc.write32(fdescr) + + def generate_quick_failure(self, guardtok, fcond=c.AL): + assert isinstance(guardtok.save_exc, bool) + fail_index = self.cpu.get_fail_descr_number(guardtok.descr) + startpos = self.mc.currpos() + withfloats = False + for box in guardtok.failargs: + if box is not None and box.type == FLOAT: + withfloats = True + break + exc = guardtok.save_exc + target = self.failure_recovery_code[exc + 2 * withfloats] + assert target != 0 + self.mc.BL(target) + # write tight data that describes the failure recovery + if guardtok.is_guard_not_forced: + self.mc.writechar(chr(self.CODE_FORCED)) + self.write_failure_recovery_description(guardtok.descr, + guardtok.failargs, guardtok.faillocs[1:]) + self.mc.write32(fail_index) + # for testing the decoding, write a final byte 0xCC + if not we_are_translated(): + self.mc.writechar('\xCC') + faillocs = [loc for loc in guardtok.faillocs if loc is not None] + guardtok.descr._arm_debug_faillocs = faillocs self.align() - - # assert that the fail_boxes lists are big enough - assert len(failargs) <= self.fail_boxes_int.SIZE - - def _gen_path_to_exit_path(self, descr, args, arglocs, - save_exc, fcond=c.AL): - assert isinstance(save_exc, bool) - self.gen_exit_code(self.mc, save_exc, fcond) - self.gen_descr_encoding(descr, args, arglocs[1:]) - - def gen_exit_code(self, mc, save_exc, fcond=c.AL): - assert isinstance(save_exc, bool) - if save_exc: - path = self._leave_jitted_hook_save_exc - else: - path = self._leave_jitted_hook - mc.BL(path) + return startpos def align(self): while(self.mc.currpos() % FUNC_ALIGN != 0): @@ -841,15 +900,10 @@ def write_pending_failure_recoveries(self): for tok in self.pending_guards: - descr = tok.descr #generate the exit stub and the encoded representation - pos = self.mc.currpos() - tok.pos_recovery_stub = pos - - self._gen_path_to_exit_path(descr, tok.failargs, - tok.faillocs, save_exc=tok.save_exc) + tok.pos_recovery_stub = self.generate_quick_failure(tok) # store info on the descr - descr._arm_current_frame_depth = tok.faillocs[0].getint() + tok.descr._arm_current_frame_depth = tok.faillocs[0].getint() def process_pending_guards(self, block_start): clt = self.current_clt @@ -860,7 +914,7 @@ descr._arm_failure_recovery_block = failure_recovery_pos relative_offset = tok.pos_recovery_stub - tok.offset guard_pos = block_start + tok.offset - if not tok.is_invalidate: + if not tok.is_guard_not_invalidated: # patch the guard jumpt to the stub # overwrite the generate NOP with a B_offs to the pos of the # stub @@ -1255,11 +1309,6 @@ else: raise AssertionError('Trying to pop to an invalid location') - def leave_jitted_hook(self): - ptrs = self.fail_boxes_ptr.ar - llop.gc_assume_young_pointers(lltype.Void, - llmemory.cast_ptr_to_adr(ptrs)) - def malloc_cond(self, nursery_free_adr, nursery_top_adr, size): assert size & (WORD-1) == 0 # must be correctly aligned diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py --- a/pypy/jit/backend/arm/opassembler.py +++ b/pypy/jit/backend/arm/opassembler.py @@ -16,6 +16,7 @@ gen_emit_unary_float_op, saved_registers, count_reg_args) +from pypy.jit.backend.arm.helper.regalloc import check_imm_arg from pypy.jit.backend.arm.codebuilder import ARMv7Builder, OverwritingBuilder from pypy.jit.backend.arm.jump import remap_frame_layout from pypy.jit.backend.arm.regalloc import TempInt, TempPtr @@ -27,18 +28,20 @@ from pypy.jit.metainterp.history import JitCellToken, TargetToken from pypy.jit.metainterp.resoperation import rop from pypy.rlib.objectmodel import we_are_translated -from pypy.rpython.lltypesystem import rstr +from pypy.rlib import rgc +from pypy.rpython.lltypesystem import rstr, rffi, lltype, llmemory NO_FORCE_INDEX = -1 class GuardToken(object): - def __init__(self, descr, failargs, faillocs, offset, - save_exc, fcond=c.AL, is_invalidate=False): + def __init__(self, descr, failargs, faillocs, offset, save_exc, fcond=c.AL, + is_guard_not_invalidated=False, is_guard_not_forced=False): assert isinstance(save_exc, bool) self.descr = descr self.offset = offset - self.is_invalidate = is_invalidate + self.is_guard_not_invalidated = is_guard_not_invalidated + self.is_guard_not_forced = is_guard_not_forced self.failargs = failargs self.faillocs = faillocs self.save_exc = save_exc @@ -190,7 +193,8 @@ return fcond def _emit_guard(self, op, arglocs, fcond, save_exc, - is_guard_not_invalidated=False): + is_guard_not_invalidated=False, + is_guard_not_forced=False): assert isinstance(save_exc, bool) assert isinstance(fcond, int) descr = op.getdescr() @@ -210,7 +214,8 @@ faillocs=arglocs, offset=pos, save_exc=save_exc, - is_invalidate=is_guard_not_invalidated, + is_guard_not_invalidated=is_guard_not_invalidated, + is_guard_not_forced=is_guard_not_forced, fcond=fcond)) return c.AL @@ -312,49 +317,11 @@ return fcond def emit_op_finish(self, op, arglocs, regalloc, fcond): - for i in range(len(arglocs) - 1): - loc = arglocs[i] - box = op.getarg(i) - if loc is None: - continue - if loc.is_reg(): - if box.type == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(i) - elif box.type == INT: - adr = self.fail_boxes_int.get_addr_for_num(i) - else: - assert 0 - self.mc.gen_load_int(r.ip.value, adr) - self.mc.STR_ri(loc.value, r.ip.value) - elif loc.is_vfp_reg(): - assert box.type == FLOAT - adr = self.fail_boxes_float.get_addr_for_num(i) - self.mc.gen_load_int(r.ip.value, adr) - self.mc.VSTR(loc.value, r.ip.value) - elif loc.is_stack() or loc.is_imm() or loc.is_imm_float(): - if box.type == FLOAT: - adr = self.fail_boxes_float.get_addr_for_num(i) - self.mov_loc_loc(loc, r.vfp_ip) - self.mc.gen_load_int(r.ip.value, adr) - self.mc.VSTR(r.vfp_ip.value, r.ip.value) - elif box.type == REF or box.type == INT: - if box.type == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(i) - elif box.type == INT: - adr = self.fail_boxes_int.get_addr_for_num(i) - else: - assert 0 - self.mov_loc_loc(loc, r.ip) - self.mc.gen_load_int(r.lr.value, adr) - self.mc.STR_ri(r.ip.value, r.lr.value) - else: - assert 0 - # note: no exception should currently be set in llop.get_exception_addr - # even if this finish may be an exit_frame_with_exception (in this case - # the exception instance is in arglocs[0]). - addr = self.cpu.get_on_leave_jitted_int(save_exception=False) - self.mc.BL(addr) - self.mc.gen_load_int(r.r0.value, arglocs[-1].value) + [argloc] = arglocs + if argloc is not r.r0: #XXX verify this + self.mov(argloc, eax) + self.mov_loc_loc(arg_loc, r.r0, fcond) + # exit function self.gen_func_epilog() return fcond @@ -1152,8 +1119,26 @@ value = self.cpu.done_with_this_frame_float_v else: raise AssertionError(kind) - self.mc.gen_load_int(r.ip.value, value) - self.mc.CMP_rr(tmploc.value, r.ip.value) + from pypy.jit.backend.llsupport.descr import unpack_fielddescr + from pypy.jit.backend.llsupport.descr import unpack_interiorfielddescr + descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) + _offset, _size, _ = unpack_fielddescr(descrs.jf_descr) + fail_descr = self.cpu.get_fail_descr_from_number(value) + value = fail_descr.hide(self.cpu) + rgc._make_sure_does_not_move(value) + value = rffi.cast(lltype.Signed, value) + + if check_imm_arg(_offset): + self.mc.LDR_ri(r.ip.value, tmploc.value, imm=_offset) + else: + self.mc.gen_load_int(r.ip.value, _offset) + self.mc.LDR_rr(r.ip.value, tmploc.value, r.ip.value) + if check_imm_arg(value): + self.mc.CMP_ri(r.ip.value, imm=value) + else: + self.mc.gen_load_int(r.lr.value, value) + self.mc.CMP_rr(r.lr.value, r.ip.value) + #if values are equal we take the fast path # Slow path, calling helper @@ -1180,19 +1165,31 @@ if op.result is not None: # load the return value from fail_boxes_xxx[0] kind = op.result.type - if kind == INT: - adr = self.fail_boxes_int.get_addr_for_num(0) - elif kind == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(0) - elif kind == FLOAT: - adr = self.fail_boxes_float.get_addr_for_num(0) + if kind == FLOAT: + t = unpack_interiorfielddescr(descrs.as_float)[0] + if not check_imm_arg(t): + self.mc.gen_load_int(r.ip.value, t, cond=fast_path_cond) + self.mc.ADD_rr(r.ip.value, r.r0.value, r.ip.value, + cond=fast_path_cond) + t = 0 + base = r.ip + else: + base = r.r0 + self.mc.VLDR(resloc.value, base.value, imm=t, + cond=fast_path_cond) else: - raise AssertionError(kind) - self.mc.gen_load_int(r.ip.value, adr, cond=fast_path_cond) - if op.result.type == FLOAT: - self.mc.VLDR(resloc.value, r.ip.value, cond=fast_path_cond) - else: - self.mc.LDR_ri(resloc.value, r.ip.value, cond=fast_path_cond) + assert resloc is r.r0 + if kind == INT: + t = unpack_interiorfielddescr(descrs.as_int)[0] + else: + t = unpack_interiorfielddescr(descrs.as_ref)[0] + if not check_imm_arg(t): + self.mc.gen_load_int(r.ip.value, t, cond=fast_path_cond) + self.mc.LDR_rr(resloc.value, resloc.value, r.ip.value, + cond=fast_path_cond) + else: + self.mc.LDR_ri(resloc.value, resloc.value, imm=t, + cond=fast_path_cond) # jump to merge point jmp_pos = self.mc.currpos() self.mc.BKPT() @@ -1266,7 +1263,8 @@ self.mc.LDR_ri(r.ip.value, r.fp.value) self.mc.CMP_ri(r.ip.value, 0) - self._emit_guard(guard_op, arglocs[1 + numargs:], c.GE, save_exc=True) + self._emit_guard(guard_op, arglocs[1 + numargs:], c.GE, + save_exc=True, is_guard_not_forced=True) return fcond def emit_guard_call_release_gil(self, op, guard_op, arglocs, regalloc, diff --git a/pypy/jit/backend/arm/regalloc.py b/pypy/jit/backend/arm/regalloc.py --- a/pypy/jit/backend/arm/regalloc.py +++ b/pypy/jit/backend/arm/regalloc.py @@ -644,15 +644,9 @@ return args def prepare_op_finish(self, op, fcond): - args = [None] * (op.numargs() + 1) - for i in range(op.numargs()): - arg = op.getarg(i) - if arg: - args[i] = self.loc(arg) - self.possibly_free_var(arg) - n = self.cpu.get_fail_descr_number(op.getdescr()) - args[-1] = imm(n) - return args + loc = self.loc(op.getarg(0)) + self.possibly_free_var(op.getarg(0)) + return [loc] def prepare_op_guard_true(self, op, fcond): l0 = self.make_sure_var_in_reg(op.getarg(0)) diff --git a/pypy/jit/backend/arm/runner.py b/pypy/jit/backend/arm/runner.py --- a/pypy/jit/backend/arm/runner.py +++ b/pypy/jit/backend/arm/runner.py @@ -23,15 +23,21 @@ AbstractLLCPU.__init__(self, rtyper, stats, opts, translate_support_code, gcdescr) + from pypy.jit.backend.llsupport import jitframe + self.deadframe_size_max = llmemory.sizeof(jitframe.DEADFRAME, + self.get_failargs_limit()) + def set_debug(self, flag): return self.assembler.set_debug(flag) + def get_failargs_limit(self): + if self.opts is not None: + return self.opts.failargs_limit + else: + return 1000 + def setup(self): - if self.opts is not None: - failargs_limit = self.opts.failargs_limit - else: - failargs_limit = 1000 - self.assembler = AssemblerARM(self, failargs_limit=failargs_limit) + self.assembler = AssemblerARM(self, self.translate_support_code) def setup_once(self): self.assembler.setup_once() @@ -51,24 +57,6 @@ return self.assembler.assemble_bridge(faildescr, inputargs, operations, original_loop_token, log=log) - def get_latest_value_float(self, index): - return self.assembler.fail_boxes_float.getitem(index) - - def get_latest_value_int(self, index): - return self.assembler.fail_boxes_int.getitem(index) - - def get_latest_value_ref(self, index): - return self.assembler.fail_boxes_ptr.getitem(index) - - def get_latest_value_count(self): - return self.assembler.fail_boxes_count - - def get_latest_force_token(self): - return self.assembler.fail_force_index - - def get_on_leave_jitted_hook(self): - return self.assembler.leave_jitted_hook - def clear_latest_values(self, count): setitem = self.assembler.fail_boxes_ptr.setitem null = lltype.nullptr(llmemory.GCREF.TO) @@ -76,7 +64,7 @@ setitem(index, null) def make_execute_token(self, *ARGS): - FUNCPTR = lltype.Ptr(lltype.FuncType(ARGS, lltype.Signed)) + FUNCPTR = lltype.Ptr(lltype.FuncType(ARGS, llmemory.GCREF)) def execute_token(executable_token, *args): clt = executable_token.compiled_loop_token @@ -91,12 +79,13 @@ prev_interpreter = LLInterpreter.current_interpreter LLInterpreter.current_interpreter = self.debug_ll_interpreter try: - fail_index = func(*args) + deadframe = func(*args) finally: if not self.translate_support_code: LLInterpreter.current_interpreter = prev_interpreter #llop.debug_print(lltype.Void, "<<<< Back") - return self.get_fail_descr_from_number(fail_index) + self.gc_set_extra_threshold() + return deadframe return execute_token def cast_ptr_to_int(x): @@ -115,17 +104,22 @@ assert fail_index >= 0, "already forced!" faildescr = self.get_fail_descr_from_number(fail_index) rffi.cast(TP, addr_of_force_index)[0] = ~fail_index - bytecode = self.assembler._find_failure_recovery_bytecode(faildescr) + frb = self.assembler._find_failure_recovery_bytecode(faildescr) + bytecode = rffi.cast(rffi.UCHARP, frb) addr_all_null_regsiters = rffi.cast(rffi.LONG, self.all_null_registers) - # start of "no gc operation!" block - fail_index_2 = self.assembler.failure_recovery_func( - bytecode, - addr_of_force_index, - addr_all_null_regsiters) - self.assembler.leave_jitted_hook() - # end of "no gc operation!" block - assert fail_index == fail_index_2 - return faildescr + # + assert (rffi.cast(lltype.Signed, bytecode[0]) == + self.assembler.CODE_FORCED) + bytecode = rffi.ptradd(bytecode, 1) + deadframe = self.assembler.failure_recovery_func(bytecode, + addr_of_force_index, + self.all_null_registers) + + # + assert self.get_latest_descr(deadframe) is faildescr + self.assembler.force_token_to_dead_frame[addr_of_force_index] = ( + deadframe) + return deadframe def redirect_call_assembler(self, oldlooptoken, newlooptoken): self.assembler.redirect_call_assembler(oldlooptoken, newlooptoken) diff --git a/pypy/jit/backend/arm/test/test_runner.py b/pypy/jit/backend/arm/test/test_runner.py --- a/pypy/jit/backend/arm/test/test_runner.py +++ b/pypy/jit/backend/arm/test/test_runner.py @@ -60,16 +60,18 @@ ] cpu.compile_loop(inp, operations, looptoken) args = [i for i in range(1, 15)] - self.cpu.execute_token(looptoken, *args) - output = [self.cpu.get_latest_value_int(i - 1) for i in range(1, 15)] + deadframe = self.cpu.execute_token(looptoken, *args) + output = [self.cpu.get_latest_value_int(deadframe, i - 1) for i in range(1, 15)] expected = [3, 7, 11, 15, 19, 23, 27, 3, 7, 11, 15, 19, 23, 27] assert output == expected - def test_redirect_call_assember2(self): - def assembler_helper(failindex, virtualizable): - return self.cpu.get_latest_value_int(0) + def test_redirect_call_assembler2(self): + def assembler_helper(deadframe, virtualizable): + x = self.cpu.get_latest_value_int(deadframe, 0) + assert x == 11 + return 7 - FUNCPTR = lltype.Ptr(lltype.FuncType([lltype.Signed, llmemory.GCREF], + FUNCPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF, llmemory.GCREF], lltype.Signed)) class FakeJitDriverSD: @@ -101,12 +103,12 @@ self.cpu.compile_loop(loop2.inputargs, loop2.operations, lt2) self.cpu.compile_loop(loop3.inputargs, loop3.operations, lt3) self.cpu.compile_loop(loop1.inputargs, loop1.operations, lt1) - self.cpu.execute_token(lt1, 11) - assert self.cpu.get_latest_value_int(0) == 12 + df = self.cpu.execute_token(lt1, 10) + assert self.cpu.get_latest_value_int(df, 0) == 7 self.cpu.redirect_call_assembler(lt2, lt3) - self.cpu.execute_token(lt1, 11) - assert self.cpu.get_latest_value_int(0) == 10 + df = self.cpu.execute_token(lt1, 12) + assert self.cpu.get_latest_value_int(df, 0) == 7 SFloat = lltype.GcForwardReference() SFloat.become(lltype.GcStruct('SFloat', ('parent', rclass.OBJECT), @@ -202,8 +204,8 @@ ARGS = [lltype.Signed] * numargs RES = lltype.Signed args = [i+1 for i in range(numargs)] - res = self.cpu.execute_token(looptoken, *args) - assert self.cpu.get_latest_value_int(0) == sum(args) + deadframe = self.cpu.execute_token(looptoken, *args) + assert self.cpu.get_latest_value_int(deadframe, 0) == sum(args) def test_debugger_on(self): from pypy.rlib import debug diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py --- a/pypy/jit/backend/llsupport/llmodel.py +++ b/pypy/jit/backend/llsupport/llmodel.py @@ -193,6 +193,12 @@ # ------------------- helpers and descriptions -------------------- + def gc_set_extra_threshold(self): + llop.gc_set_extra_threshold(lltype.Void, self.deadframe_size_max) + + def gc_clear_extra_threshold(self): + llop.gc_set_extra_threshold(lltype.Void, 0) + @staticmethod def _cast_int_to_gcref(x): # dangerous! only use if you are sure no collection could occur diff --git a/pypy/jit/backend/llsupport/rewrite.py b/pypy/jit/backend/llsupport/rewrite.py --- a/pypy/jit/backend/llsupport/rewrite.py +++ b/pypy/jit/backend/llsupport/rewrite.py @@ -68,7 +68,7 @@ # ---------- self.newops.append(op) # ---------- FINISH ---------- - if self.newops[-1].getopnum() == rop.FINISH: + if len(self.newops) != 0 and self.newops[-1].getopnum() == rop.FINISH: self.handle_finish(self.newops.pop()) # ---------- return self.newops diff --git a/pypy/jit/backend/test/test_random.py b/pypy/jit/backend/test/test_random.py --- a/pypy/jit/backend/test/test_random.py +++ b/pypy/jit/backend/test/test_random.py @@ -515,8 +515,8 @@ def get_cpu(): if pytest.config.option.backend == 'llgraph': - from pypy.jit.backend.llgraph.runner import LLtypeCPU - return LLtypeCPU(None) + from pypy.jit.backend.llgraph.runner import LLGraphCPU + return LLGraphCPU(None) elif pytest.config.option.backend == 'cpu': from pypy.jit.backend.detect_cpu import getcpuclass return getcpuclass()(None, None) @@ -674,25 +674,27 @@ def run_loop(self): cpu = self.builder.cpu self.clear_state() - exc = cpu.grab_exc_value() - assert not exc + # disable check for now + # exc = cpu.grab_exc_value() + # assert not exc arguments = [box.value for box in self.loop.inputargs] - fail = cpu.execute_token(self.runjitcelltoken(), *arguments) + deadframe = cpu.execute_token(self.runjitcelltoken(), *arguments) + fail = cpu.get_latest_descr(deadframe) do_assert(fail is self.should_fail_by.getdescr(), "Got %r, expected %r" % (fail, self.should_fail_by.getdescr())) for i, v in enumerate(self.get_fail_args()): if isinstance(v, (BoxFloat, ConstFloat)): - value = cpu.get_latest_value_float(i) + value = cpu.get_latest_value_float(deadframe, i) else: - value = cpu.get_latest_value_int(i) + value = cpu.get_latest_value_int(deadframe, i) do_assert(value == self.expected[v], "Got %r, expected %r for value #%d" % (value, self.expected[v], i) ) - exc = cpu.grab_exc_value() + exc = cpu.grab_exc_value(deadframe) if (self.guard_op is not None and self.guard_op.is_guard_exception()): if self.guard_op.getopnum() == rop.GUARD_NO_EXCEPTION: From noreply at buildbot.pypy.org Thu Jan 3 12:52:02 2013 From: noreply at buildbot.pypy.org (bivab) Date: Thu, 3 Jan 2013 12:52:02 +0100 (CET) Subject: [pypy-commit] pypy remove-globals-in-jit: remove methods (moved to base class) Message-ID: <20130103115202.A25961C078F@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: remove-globals-in-jit Changeset: r59682:e5d04164f720 Date: 2013-01-03 12:45 +0100 http://bitbucket.org/pypy/pypy/changeset/e5d04164f720/ Log: remove methods (moved to base class) diff --git a/pypy/jit/backend/x86/runner.py b/pypy/jit/backend/x86/runner.py --- a/pypy/jit/backend/x86/runner.py +++ b/pypy/jit/backend/x86/runner.py @@ -58,12 +58,6 @@ else: return 1000 - def gc_set_extra_threshold(self): - llop.gc_set_extra_threshold(lltype.Void, self.deadframe_size_max) - - def gc_clear_extra_threshold(self): - llop.gc_set_extra_threshold(lltype.Void, 0) - def setup(self): self.assembler = Assembler386(self, self.translate_support_code) From noreply at buildbot.pypy.org Thu Jan 3 12:52:03 2013 From: noreply at buildbot.pypy.org (bivab) Date: Thu, 3 Jan 2013 12:52:03 +0100 (CET) Subject: [pypy-commit] pypy remove-globals-in-jit: call grab_frame_values directly when forcing Message-ID: <20130103115203.C677F1C078F@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: remove-globals-in-jit Changeset: r59683:b11bbf39387b Date: 2013-01-03 12:46 +0100 http://bitbucket.org/pypy/pypy/changeset/b11bbf39387b/ Log: call grab_frame_values directly when forcing diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py --- a/pypy/jit/backend/arm/assembler.py +++ b/pypy/jit/backend/arm/assembler.py @@ -26,7 +26,7 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.jit.backend.arm.opassembler import ResOpAssembler from pypy.rlib.debug import (debug_print, debug_start, debug_stop, - have_debug_prints) + have_debug_prints, fatalerror) from pypy.rlib.jit import AsmInfo from pypy.rlib.objectmodel import compute_unique_id @@ -325,7 +325,8 @@ vfp_registers = rffi.cast(rffi.LONGP, stack_pointer) registers = rffi.ptradd(vfp_registers, 2*len(r.all_vfp_regs)) registers = rffi.cast(rffi.LONGP, registers) - return self.grab_frame_values(self.cpu, mem_loc, frame_pointer, + bytecode = rffi.cast(rffi.UCHARP, mem_loc) + return self.grab_frame_values(self.cpu, bytecode, frame_pointer, registers, vfp_registers) self.failure_recovery_code = [0, 0, 0, 0] @@ -335,13 +336,13 @@ llmemory.GCREF)) @staticmethod - def grab_frame_values(cpu, mem_loc, frame_pointer, + #@rgc.no_collect -- XXX still true, but hacked gc_set_extra_threshold + def grab_frame_values(cpu, bytecode, frame_pointer, registers, vfp_registers): # no malloc allowed here!! xxx apart from one, hacking a lot force_index = rffi.cast(lltype.Signed, frame_pointer) num = 0 deadframe = lltype.nullptr(jitframe.DEADFRAME) - bytecode = rffi.cast(rffi.UCHARP, mem_loc) # step 1: lots of mess just to count the final value of 'num' bytecode1 = bytecode while 1: @@ -545,7 +546,7 @@ def _build_failure_recovery(self, exc, withfloats=False): mc = ARMv7Builder() - decode_registers_addr = llhelper(self._FAILURE_RECOVERY_FUNC, + failure_recovery = llhelper(self._FAILURE_RECOVERY_FUNC, self.failure_recovery_func) self._insert_checks(mc) if withfloats: @@ -570,7 +571,7 @@ # pass the current stack pointer as third param mc.MOV_rr(r.r2.value, r.sp.value) self._insert_checks(mc) - mc.BL(rffi.cast(lltype.Signed, decode_registers_addr)) + mc.BL(rffi.cast(lltype.Signed, failure_recovery)) if exc: # save ebx into 'jf_guard_exc' from pypy.jit.backend.llsupport.descr import unpack_fielddescr diff --git a/pypy/jit/backend/arm/runner.py b/pypy/jit/backend/arm/runner.py --- a/pypy/jit/backend/arm/runner.py +++ b/pypy/jit/backend/arm/runner.py @@ -111,10 +111,10 @@ assert (rffi.cast(lltype.Signed, bytecode[0]) == self.assembler.CODE_FORCED) bytecode = rffi.ptradd(bytecode, 1) - deadframe = self.assembler.failure_recovery_func(bytecode, - addr_of_force_index, - self.all_null_registers) - + deadframe = self.assembler.grab_frame_values(self, + bytecode, addr_of_force_index, + self.all_null_registers, + self.all_null_registers) # assert self.get_latest_descr(deadframe) is faildescr self.assembler.force_token_to_dead_frame[addr_of_force_index] = ( From noreply at buildbot.pypy.org Thu Jan 3 12:52:04 2013 From: noreply at buildbot.pypy.org (bivab) Date: Thu, 3 Jan 2013 12:52:04 +0100 (CET) Subject: [pypy-commit] pypy remove-globals-in-jit: fix Message-ID: <20130103115204.E5DE91C078F@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: remove-globals-in-jit Changeset: r59684:0ce75b404895 Date: 2013-01-03 12:47 +0100 http://bitbucket.org/pypy/pypy/changeset/0ce75b404895/ Log: fix diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py --- a/pypy/jit/backend/arm/opassembler.py +++ b/pypy/jit/backend/arm/opassembler.py @@ -319,8 +319,7 @@ def emit_op_finish(self, op, arglocs, regalloc, fcond): [argloc] = arglocs if argloc is not r.r0: #XXX verify this - self.mov(argloc, eax) - self.mov_loc_loc(arg_loc, r.r0, fcond) + self.mov_loc_loc(argloc, r.r0, fcond) # exit function self.gen_func_epilog() return fcond From noreply at buildbot.pypy.org Thu Jan 3 12:52:06 2013 From: noreply at buildbot.pypy.org (bivab) Date: Thu, 3 Jan 2013 12:52:06 +0100 (CET) Subject: [pypy-commit] pypy remove-globals-in-jit: merge heads Message-ID: <20130103115206.264781C078F@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: remove-globals-in-jit Changeset: r59685:6a1b904bc6b1 Date: 2013-01-03 12:49 +0100 http://bitbucket.org/pypy/pypy/changeset/6a1b904bc6b1/ Log: merge heads diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -216,40 +216,6 @@ """ self.optimize_loop(ops, expected) - def test_constfold_all(self): - from pypy.jit.backend.llgraph.llimpl import TYPES # xxx fish - from pypy.jit.metainterp.executor import execute_nonspec - from pypy.jit.metainterp.history import BoxInt - import random - for opnum in range(rop.INT_ADD, rop.SAME_AS+1): - try: - op = opname[opnum] - except KeyError: - continue - if 'FLOAT' in op: - continue - argtypes, restype = TYPES[op.lower()] - args = [] - for argtype in argtypes: - assert argtype in ('int', 'bool') - args.append(random.randrange(1, 20)) - assert restype in ('int', 'bool') - ops = """ - [] - i1 = %s(%s) - escape(i1) - jump() - """ % (op.lower(), ', '.join(map(str, args))) - argboxes = [BoxInt(a) for a in args] - expected_value = execute_nonspec(self.cpu, None, opnum, - argboxes).getint() - expected = """ - [] - escape(%d) - jump() - """ % expected_value - self.optimize_loop(ops, expected) - # ---------- def test_remove_guard_class_1(self): @@ -658,8 +624,8 @@ escape(i3) p1 = new_with_vtable(ConstClass(node_vtable)) p1sub = new_with_vtable(ConstClass(node_vtable2)) + setfield_gc(p1sub, i1, descr=valuedescr) setfield_gc(p1, i1, descr=valuedescr) - setfield_gc(p1sub, i1, descr=valuedescr) setfield_gc(p1, p1sub, descr=nextdescr) jump(i1, p1, p2) """ @@ -994,10 +960,10 @@ ops = """ [f0, f1, f2, f3] p0 = new_array(2, descr=complexarraydescr) + setinteriorfield_gc(p0, 0, f1, descr=compleximagdescr) setinteriorfield_gc(p0, 0, f0, descr=complexrealdescr) - setinteriorfield_gc(p0, 0, f1, descr=compleximagdescr) + setinteriorfield_gc(p0, 1, f3, descr=compleximagdescr) setinteriorfield_gc(p0, 1, f2, descr=complexrealdescr) - setinteriorfield_gc(p0, 1, f3, descr=compleximagdescr) f4 = getinteriorfield_gc(p0, 0, descr=complexrealdescr) f5 = getinteriorfield_gc(p0, 1, descr=complexrealdescr) f6 = float_mul(f4, f5) @@ -1032,8 +998,8 @@ [f0, f1] f2 = float_mul(f0, f1) p0 = new_array(1, descr=complexarraydescr) + setinteriorfield_gc(p0, 0, f1, descr=compleximagdescr) setinteriorfield_gc(p0, 0, f0, descr=complexrealdescr) - setinteriorfield_gc(p0, 0, f1, descr=compleximagdescr) i0 = escape(f2, p0) finish(i0) """ diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -1,21 +1,14 @@ import py from pypy.rlib.objectmodel import instantiate from pypy.jit.metainterp.optimizeopt.test.test_util import ( - LLtypeMixin, BaseTest, Storage, _sortboxes, convert_old_style_to_targets) -import pypy.jit.metainterp.optimizeopt.optimizer as optimizeopt -import pypy.jit.metainterp.optimizeopt.virtualize as virtualize -from pypy.jit.metainterp.optimizeopt import ALL_OPTS_DICT, build_opt_chain + LLtypeMixin, BaseTest, convert_old_style_to_targets) +from pypy.jit.metainterp.optimizeopt import build_opt_chain from pypy.jit.metainterp.optimize import InvalidLoop from pypy.jit.metainterp.history import AbstractDescr, ConstInt, BoxInt -from pypy.jit.metainterp.history import TreeLoop, JitCellToken, TargetToken -from pypy.jit.metainterp.jitprof import EmptyProfiler -from pypy.jit.metainterp import executor, compile, resume, history -from pypy.jit.metainterp.resoperation import rop, opname, ResOperation -from pypy.jit.tool.oparser import pure_parse -from pypy.jit.metainterp.optimizeopt.util import args_dict +from pypy.jit.metainterp.history import TreeLoop +from pypy.jit.metainterp import compile, resume +from pypy.jit.metainterp.resoperation import rop, opname, opargnum from pypy.jit.metainterp.optimizeopt.test.test_optimizebasic import FakeMetaInterpStaticData -from pypy.config.pypyoption import get_pypy_config -from pypy.jit.metainterp.optimizeopt.unroll import Inliner def test_build_opt_chain(): def check(chain, expected_names): @@ -185,9 +178,7 @@ self.optimize_loop(ops, expected) def test_constfold_all(self): - from pypy.jit.backend.llgraph.llimpl import TYPES # xxx fish from pypy.jit.metainterp.executor import execute_nonspec - from pypy.jit.metainterp.history import BoxInt import random for opnum in range(rop.INT_ADD, rop.SAME_AS+1): try: @@ -196,12 +187,9 @@ continue if 'FLOAT' in op: continue - argtypes, restype = TYPES[op.lower()] args = [] - for argtype in argtypes: - assert argtype in ('int', 'bool') + for _ in range(opargnum[opnum]): args.append(random.randrange(1, 20)) - assert restype in ('int', 'bool') ops = """ [] i1 = %s(%s) @@ -836,8 +824,8 @@ escape(i3) p4 = new_with_vtable(ConstClass(node_vtable)) p1sub = new_with_vtable(ConstClass(node_vtable2)) + setfield_gc(p1sub, i1, descr=valuedescr) setfield_gc(p4, i1, descr=valuedescr) - setfield_gc(p1sub, i1, descr=valuedescr) setfield_gc(p4, p1sub, descr=nextdescr) jump(i1, p4) """ @@ -1168,13 +1156,13 @@ i2 = int_sub(i1, 1) i3 = int_add(i0, i1) i4 = same_as(i2) # This same_as should be killed by backend - jump(i3, i2, i1) + jump(i3, i1, i2) """ expected = """ [i0, i1, i1bis] - i2 = int_sub(i1, 1) - i3 = int_add(i0, i1) - jump(i3, i2, i1) + i2 = int_sub(i1bis, 1) + i3 = int_add(i0, i1bis) + jump(i3, i1bis, i2) """ self.optimize_loop(ops, expected, preamble) @@ -1226,8 +1214,8 @@ i1 = int_add(i0, 1) p1 = new_with_vtable(ConstClass(node_vtable2)) p2 = new_with_vtable(ConstClass(node_vtable2)) + setfield_gc(p2, p1, descr=nextdescr) setfield_gc(p2, i1, descr=valuedescr) - setfield_gc(p2, p1, descr=nextdescr) setfield_gc(p1, p2, descr=nextdescr) setfield_gc(p0, p1, descr=nextdescr) jump(p1) @@ -1239,8 +1227,8 @@ i1 = int_add(i0, 1) p1 = new_with_vtable(ConstClass(node_vtable2)) p2 = new_with_vtable(ConstClass(node_vtable2)) + setfield_gc(p2, p1, descr=nextdescr) setfield_gc(p2, i1, descr=valuedescr) - setfield_gc(p2, p1, descr=nextdescr) setfield_gc(p1, p2, descr=nextdescr) setfield_gc(p0, p1, descr=nextdescr) jump(p1) diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py --- a/pypy/jit/metainterp/resoperation.py +++ b/pypy/jit/metainterp/resoperation.py @@ -539,6 +539,7 @@ pass opclasses = [] # mapping numbers to the concrete ResOp class +opargnum = [] # mapping numbers to number or args (or -1) opname = {} # mapping numbers to the original names, for debugging oparity = [] # mapping numbers to the arity of the operation or -1 opwithdescr = [] # mapping numbers to a flag "takes a descr" @@ -567,6 +568,7 @@ else: cls = None opclasses.append(cls) + opargnum.append(arity) oparity.append(arity) opwithdescr.append(withdescr) opboolresult.append(boolresult) From noreply at buildbot.pypy.org Thu Jan 3 20:48:27 2013 From: noreply at buildbot.pypy.org (antocuni) Date: Thu, 3 Jan 2013 20:48:27 +0100 (CET) Subject: [pypy-commit] pypy virtual-raw-mallocs: fix the tests in test_optimizebasic, which failed because Whatever() compared == INT even when it should not Message-ID: <20130103194827.2E1FB1C11F4@cobra.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: virtual-raw-mallocs Changeset: r59686:a9496762e5d8 Date: 2013-01-03 18:26 +0100 http://bitbucket.org/pypy/pypy/changeset/a9496762e5d8/ Log: fix the tests in test_optimizebasic, which failed because Whatever() compared == INT even when it should not diff --git a/pypy/jit/metainterp/resume.py b/pypy/jit/metainterp/resume.py --- a/pypy/jit/metainterp/resume.py +++ b/pypy/jit/metainterp/resume.py @@ -453,6 +453,8 @@ return self.liveboxes_from_env[box] return self.liveboxes[box] +class WrongVirtualKind(Exception): + pass class AbstractVirtualInfo(object): kind = REF @@ -467,6 +469,15 @@ def debug_prints(self): raise NotImplementedError + # subclasses should override one and only one of these two methods, + # depending on the kind + def allocate(self, decoder, index): + raise WrongVirtualKind + + def allocate_int(self, decoder, index): + raise WrongVirtualKind + + class AbstractVirtualStructInfo(AbstractVirtualInfo): def __init__(self, fielddescrs): self.fielddescrs = fielddescrs diff --git a/pypy/jit/metainterp/test/test_resume.py b/pypy/jit/metainterp/test/test_resume.py --- a/pypy/jit/metainterp/test/test_resume.py +++ b/pypy/jit/metainterp/test/test_resume.py @@ -1005,7 +1005,12 @@ @staticmethod def enumerate_vars(callback_i, callback_r, callback_f, _): for index, tagged in enumerate(self.cur_numb.nums): - box = self.decode_box(tagged, Whatever()) + _, tag = untag(tagged) + if tag == TAGVIRTUAL: + kind = REF + else: + kind = Whatever() + box = self.decode_box(tagged, kind) if box.type == INT: callback_i(index, index) elif box.type == REF: From noreply at buildbot.pypy.org Thu Jan 3 20:48:28 2013 From: noreply at buildbot.pypy.org (antocuni) Date: Thu, 3 Jan 2013 20:48:28 +0100 (CET) Subject: [pypy-commit] pypy virtual-raw-mallocs: fix translation Message-ID: <20130103194828.717FF1C11F5@cobra.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: virtual-raw-mallocs Changeset: r59687:06b5f4994e9c Date: 2013-01-03 18:36 +0100 http://bitbucket.org/pypy/pypy/changeset/06b5f4994e9c/ Log: fix translation diff --git a/pypy/jit/backend/llgraph/runner.py b/pypy/jit/backend/llgraph/runner.py --- a/pypy/jit/backend/llgraph/runner.py +++ b/pypy/jit/backend/llgraph/runner.py @@ -408,6 +408,7 @@ # so far this is used only by optimizeopt.virtualize for # {GET,SET}ARRAYITEM_RAW: for now we just return dummy values for # basesize and is_signed + assert isinstance(arraydescr, Descr) return 0, arraydescr.width, True # ---------- the backend-dependent operations ---------- From noreply at buildbot.pypy.org Thu Jan 3 20:48:29 2013 From: noreply at buildbot.pypy.org (antocuni) Date: Thu, 3 Jan 2013 20:48:29 +0100 (CET) Subject: [pypy-commit] pypy virtual-raw-mallocs: we cannot move these methods here, else we get an annotation error Message-ID: <20130103194829.9C8871C11F6@cobra.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: virtual-raw-mallocs Changeset: r59688:e07c123a9879 Date: 2013-01-03 18:37 +0100 http://bitbucket.org/pypy/pypy/changeset/e07c123a9879/ Log: we cannot move these methods here, else we get an annotation error diff --git a/pypy/jit/metainterp/resume.py b/pypy/jit/metainterp/resume.py --- a/pypy/jit/metainterp/resume.py +++ b/pypy/jit/metainterp/resume.py @@ -453,9 +453,6 @@ return self.liveboxes_from_env[box] return self.liveboxes[box] -class WrongVirtualKind(Exception): - pass - class AbstractVirtualInfo(object): kind = REF #def allocate(self, decoder, index): @@ -468,14 +465,6 @@ def debug_prints(self): raise NotImplementedError - - # subclasses should override one and only one of these two methods, - # depending on the kind - def allocate(self, decoder, index): - raise WrongVirtualKind - - def allocate_int(self, decoder, index): - raise WrongVirtualKind class AbstractVirtualStructInfo(AbstractVirtualInfo): From noreply at buildbot.pypy.org Thu Jan 3 20:48:30 2013 From: noreply at buildbot.pypy.org (antocuni) Date: Thu, 3 Jan 2013 20:48:30 +0100 (CET) Subject: [pypy-commit] pypy virtual-raw-mallocs: fix Message-ID: <20130103194830.C422E1C11F7@cobra.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: virtual-raw-mallocs Changeset: r59689:b5f22503190c Date: 2013-01-03 18:38 +0100 http://bitbucket.org/pypy/pypy/changeset/b5f22503190c/ Log: fix diff --git a/pypy/jit/codewriter/test/test_jtransform.py b/pypy/jit/codewriter/test/test_jtransform.py --- a/pypy/jit/codewriter/test/test_jtransform.py +++ b/pypy/jit/codewriter/test/test_jtransform.py @@ -605,7 +605,7 @@ assert op1.args == [] def test_raw_free(): - S = rffi.CArray(lltype.Signed) + S = rffi.CArray(lltype.Char) flags = Constant({'flavor': 'raw', 'track_allocation': True}, lltype.Void) op = SpaceOperation('free', [varoftype(lltype.Ptr(S)), flags], From noreply at buildbot.pypy.org Thu Jan 3 20:48:32 2013 From: noreply at buildbot.pypy.org (antocuni) Date: Thu, 3 Jan 2013 20:48:32 +0100 (CET) Subject: [pypy-commit] pypy virtual-raw-mallocs: bah, we need to play this dance to fix the annotator in a couple of tests in which VRawBuffer would never be instantiated, thus leading to blocked blocks Message-ID: <20130103194832.06A791C11F8@cobra.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: virtual-raw-mallocs Changeset: r59690:b128c43355bf Date: 2013-01-03 20:47 +0100 http://bitbucket.org/pypy/pypy/changeset/b128c43355bf/ Log: bah, we need to play this dance to fix the annotator in a couple of tests in which VRawBuffer would never be instantiated, thus leading to blocked blocks diff --git a/pypy/jit/backend/x86/test/test_ztranslation.py b/pypy/jit/backend/x86/test/test_ztranslation.py --- a/pypy/jit/backend/x86/test/test_ztranslation.py +++ b/pypy/jit/backend/x86/test/test_ztranslation.py @@ -13,6 +13,18 @@ from pypy.config.translationoption import DEFL_GC from pypy.rlib import rgc +def fix_annotator_for_vrawbuffer(monkeypatch): + from pypy.rlib.nonconst import NonConstant + from pypy.jit.metainterp.optimizeopt.virtualize import VRawBufferValue + from pypy.jit.metainterp import warmspot + + def my_hook_for_tests(cpu): + # this is needed so that the annotator can see it + if NonConstant(False): + v = VRawBufferValue(cpu, None, -1, None, None) + monkeypatch.setattr(warmspot, 'hook_for_tests', my_hook_for_tests) + + class TestTranslationX86(CCompiledMixin): CPUClass = getcpuclass() @@ -22,7 +34,8 @@ assert '-msse2' in cbuilder.eci.compile_extra assert '-mfpmath=sse' in cbuilder.eci.compile_extra - def test_stuff_translates(self): + def test_stuff_translates(self, monkeypatch): + fix_annotator_for_vrawbuffer(monkeypatch) # this is a basic test that tries to hit a number of features and their # translation: # - jitting of loops and bridges @@ -95,9 +108,10 @@ res = self.meta_interp(main, [40, -49]) assert res == expected - def test_direct_assembler_call_translates(self): + def test_direct_assembler_call_translates(self, monkeypatch): """Test CALL_ASSEMBLER and the recursion limit""" from pypy.rlib.rstackovf import StackOverflow + fix_annotator_for_vrawbuffer(monkeypatch) class Thing(object): def __init__(self, val): @@ -171,7 +185,8 @@ assert 1024 <= bound <= 131072 assert bound & (bound-1) == 0 # a power of two - def test_jit_get_stats(self): + def test_jit_get_stats(self, monkeypatch): + fix_annotator_for_vrawbuffer(monkeypatch) driver = JitDriver(greens = [], reds = ['i']) def f(): @@ -201,7 +216,8 @@ t.config.translation.gcremovetypeptr = True return t - def test_external_exception_handling_translates(self): + def test_external_exception_handling_translates(self, monkeypatch): + fix_annotator_for_vrawbuffer(monkeypatch) jitdriver = JitDriver(greens = [], reds = ['n', 'total']) class ImDone(Exception): diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py --- a/pypy/jit/metainterp/warmspot.py +++ b/pypy/jit/metainterp/warmspot.py @@ -861,8 +861,11 @@ ts = self.cpu.ts state = jd.warmstate maybe_compile_and_run = jd._maybe_compile_and_run_fn + cpu = jd.warmstate.cpu def ll_portal_runner(*args): + hook_for_tests(cpu) # usually it's empty, but tests can monkeypatch + # it to fix the annotator start = True while 1: try: @@ -1096,3 +1099,10 @@ graphs = self.translator.graphs for graph, block, i in find_force_quasi_immutable(graphs): self.replace_force_quasiimmut_with_direct_call(block.operations[i]) + +def hook_for_tests(): + """ + This function is empty and does nothing. Its only role is to be + monkey-patched by tests to "fix" the annotator if needed (see + e.g. x86/test/test_ztranslation::test_external_exception_handling_translates + """ From noreply at buildbot.pypy.org Fri Jan 4 06:55:30 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Fri, 4 Jan 2013 06:55:30 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments: Random style fixes. Note: you cannot use rgc.ll_arraycopy in resize because its an array of structs Message-ID: <20130104055530.9ABCA1C01EE@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: rdict-experiments Changeset: r59691:7ad06da1bb76 Date: 2013-01-03 21:55 -0800 http://bitbucket.org/pypy/pypy/changeset/7ad06da1bb76/ Log: Random style fixes. Note: you cannot use rgc.ll_arraycopy in resize because its an array of structs diff --git a/pypy/rlib/dict.py b/pypy/rlib/dict.py --- a/pypy/rlib/dict.py +++ b/pypy/rlib/dict.py @@ -1,4 +1,3 @@ - from pypy.rlib.jit_hooks import _cast_to_gcref from pypy.rpython.lltypesystem import lltype, llmemory, rclass from pypy.rpython.annlowlevel import cast_base_ptr_to_instance @@ -13,6 +12,7 @@ ('value', llmemory.GCREF))) MAIN_TP = lltype.GcArray(lltype.Signed) + class Dict(object): 'Space efficient dictionary with fast iteration and cheap resizes.' @@ -26,7 +26,7 @@ else: perturb = hashvalue n = len(self.indices) - i = perturb & (n-1) + i = perturb & (n - 1) while True: index = self.indices[i] if index == FREE: @@ -36,7 +36,7 @@ elif self.values[index].key == key: return (index, i) i = 5 * i + perturb + 1 - i = i & (n-1) + i = i & (n - 1) perturb >>= PERTURB_SHIFT _lookup._always_inline_ = True @@ -68,12 +68,12 @@ perturb = -hashvalue else: perturb = hashvalue - i = hashvalue & (n-1) + i = hashvalue & (n - 1) while True: if self.indices[i] == FREE: break i = 5 * i + perturb + 1 - i = i & (n-1) + i = i & (n - 1) perturb >>= PERTURB_SHIFT self.indices[i] = index self.filled = self.used @@ -100,7 +100,6 @@ ptr = lltype.cast_opaque_ptr(rclass.OBJECTPTR, llref) return cast_base_ptr_to_instance(CLS, ptr) - def __setitem__(self, key, value): hashvalue = key # hash index, i = self._lookup(key, hashvalue) @@ -180,7 +179,7 @@ key = self.keylist[-1] value = self.valuelist[-1] except IndexError: - raise KeyError( 'popitem(): dictionary is empty') + raise KeyError('popitem(): dictionary is empty') del self[key] return key, value From noreply at buildbot.pypy.org Fri Jan 4 07:47:04 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Fri, 4 Jan 2013 07:47:04 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments: collapse this into one loop, now seems faster than current dict at 25000000 iterations Message-ID: <20130104064704.274941C0DD9@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: rdict-experiments Changeset: r59692:37ae37148e6a Date: 2013-01-03 22:46 -0800 http://bitbucket.org/pypy/pypy/changeset/37ae37148e6a/ Log: collapse this into one loop, now seems faster than current dict at 25000000 iterations diff --git a/pypy/rlib/dict.py b/pypy/rlib/dict.py --- a/pypy/rlib/dict.py +++ b/pypy/rlib/dict.py @@ -62,6 +62,7 @@ n = new_size self.indices = self._make_index(n) PERTURB_SHIFT = 5 + new_values = lltype.malloc(TP, new_size * 2 / 3 + 1) for index in range(self.used): hashvalue = self.values[index].key if hashvalue < 0: @@ -76,12 +77,10 @@ i = i & (n - 1) perturb >>= PERTURB_SHIFT self.indices[i] = index + new_values[index].key = self.values[index].key + new_values[index].value = self.values[index].value self.filled = self.used - old_values = self.values - self.values = lltype.malloc(TP, new_size * 2 / 3 + 1) - for i in range(self.used): - self.values[i].key = old_values[i].key - self.values[i].value = old_values[i].value + self.values = new_values def clear(self): self.indices = self._make_index(8) From noreply at buildbot.pypy.org Fri Jan 4 11:39:08 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 4 Jan 2013 11:39:08 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments: use ll_arraycopy Message-ID: <20130104103908.6A36C1C01EE@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments Changeset: r59693:bbda3b169422 Date: 2013-01-04 12:31 +0200 http://bitbucket.org/pypy/pypy/changeset/bbda3b169422/ Log: use ll_arraycopy diff --git a/pypy/rlib/dict.py b/pypy/rlib/dict.py --- a/pypy/rlib/dict.py +++ b/pypy/rlib/dict.py @@ -1,5 +1,6 @@ from pypy.rlib.jit_hooks import _cast_to_gcref +from pypy.rlib import rgc from pypy.rpython.lltypesystem import lltype, llmemory, rclass from pypy.rpython.annlowlevel import cast_base_ptr_to_instance @@ -79,9 +80,7 @@ self.filled = self.used old_values = self.values self.values = lltype.malloc(TP, new_size * 2 / 3 + 1) - for i in range(self.used): - self.values[i].key = old_values[i].key - self.values[i].value = old_values[i].value + rgc.ll_arraycopy(old_values, self.values, 0, 0, self.used) def clear(self): self.indices = self._make_index(8) From noreply at buildbot.pypy.org Fri Jan 4 11:39:09 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 4 Jan 2013 11:39:09 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments: merge Message-ID: <20130104103909.9622B1C01EE@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments Changeset: r59694:c9a49e752464 Date: 2013-01-04 12:38 +0200 http://bitbucket.org/pypy/pypy/changeset/c9a49e752464/ Log: merge diff --git a/pypy/rlib/dict.py b/pypy/rlib/dict.py --- a/pypy/rlib/dict.py +++ b/pypy/rlib/dict.py @@ -1,4 +1,3 @@ - from pypy.rlib.jit_hooks import _cast_to_gcref from pypy.rlib import rgc from pypy.rpython.lltypesystem import lltype, llmemory, rclass @@ -14,6 +13,7 @@ ('value', llmemory.GCREF))) MAIN_TP = lltype.GcArray(lltype.Signed) + class Dict(object): 'Space efficient dictionary with fast iteration and cheap resizes.' @@ -27,7 +27,7 @@ else: perturb = hashvalue n = len(self.indices) - i = perturb & (n-1) + i = perturb & (n - 1) while True: index = self.indices[i] if index == FREE: @@ -37,7 +37,7 @@ elif self.values[index].key == key: return (index, i) i = 5 * i + perturb + 1 - i = i & (n-1) + i = i & (n - 1) perturb >>= PERTURB_SHIFT _lookup._always_inline_ = True @@ -69,12 +69,12 @@ perturb = -hashvalue else: perturb = hashvalue - i = hashvalue & (n-1) + i = hashvalue & (n - 1) while True: if self.indices[i] == FREE: break i = 5 * i + perturb + 1 - i = i & (n-1) + i = i & (n - 1) perturb >>= PERTURB_SHIFT self.indices[i] = index self.filled = self.used @@ -99,7 +99,6 @@ ptr = lltype.cast_opaque_ptr(rclass.OBJECTPTR, llref) return cast_base_ptr_to_instance(CLS, ptr) - def __setitem__(self, key, value): hashvalue = key # hash index, i = self._lookup(key, hashvalue) @@ -179,7 +178,7 @@ key = self.keylist[-1] value = self.valuelist[-1] except IndexError: - raise KeyError( 'popitem(): dictionary is empty') + raise KeyError('popitem(): dictionary is empty') del self[key] return key, value From noreply at buildbot.pypy.org Fri Jan 4 22:30:50 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Fri, 4 Jan 2013 22:30:50 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Initial package rename: Message-ID: <20130104213050.7C9BE1C01EE@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59695:f86b274daa88 Date: 2012-12-30 21:54 +0100 http://bitbucket.org/pypy/pypy/changeset/f86b274daa88/ Log: Initial package rename: pypy.rpython -> rpython.rtyper pypy.translator -> rpython.translator pypy.rlib -> rpython.rlib pypy.jit -> rpython.jit pypy.objspace.flow -> rpython.flowspace pypy.annotation -> rpython.annotator diff too long, truncating to 2000 out of 38413 lines diff --git a/dotviewer/test/test_translator.py b/dotviewer/test/test_translator.py --- a/dotviewer/test/test_translator.py +++ b/dotviewer/test/test_translator.py @@ -21,7 +21,7 @@ def test_annotated(): - from pypy.translator.interactive import Translation + from rpython.translator.interactive import Translation t = Translation(is_prime) t.annotate([int]) t.viewcg() diff --git a/lib_pypy/ctypes_config_cache/dumpcache.py b/lib_pypy/ctypes_config_cache/dumpcache.py --- a/lib_pypy/ctypes_config_cache/dumpcache.py +++ b/lib_pypy/ctypes_config_cache/dumpcache.py @@ -1,6 +1,6 @@ import os from ctypes_configure import dumpcache -from pypy.jit.backend import detect_cpu +from rpython.jit.backend import detect_cpu def dumpcache2(basename, config): model = detect_cpu.autodetect_main_model_and_size() @@ -14,7 +14,7 @@ try: from __pypy__ import cpumodel except ImportError: - from pypy.jit.backend import detect_cpu + from rpython.jit.backend import detect_cpu cpumodel = detect_cpu.autodetect_main_model_and_size() # XXX relative import, should be removed together with # XXX the relative imports done e.g. by lib_pypy/pypy_test/test_hashlib diff --git a/lib_pypy/ctypes_config_cache/rebuild.py b/lib_pypy/ctypes_config_cache/rebuild.py --- a/lib_pypy/ctypes_config_cache/rebuild.py +++ b/lib_pypy/ctypes_config_cache/rebuild.py @@ -31,7 +31,7 @@ sys.path[:] = path def try_rebuild(): - from pypy.jit.backend import detect_cpu + from rpython.jit.backend import detect_cpu model = detect_cpu.autodetect_main_model_and_size() # remove the files '_*_model_.py' left = {} diff --git a/pypy/bin/py.py b/pypy/bin/py.py --- a/pypy/bin/py.py +++ b/pypy/bin/py.py @@ -57,7 +57,7 @@ def set_compiler(option, opt, value, parser): - from pypy.translator.platform import set_platform + from rpython.translator.platform import set_platform set_platform('host', value) def main_(argv=None): diff --git a/pypy/bin/reportstaticdata.py b/pypy/bin/reportstaticdata.py --- a/pypy/bin/reportstaticdata.py +++ b/pypy/bin/reportstaticdata.py @@ -28,7 +28,7 @@ """ import autopath -from pypy.translator.tool.staticsizereport import print_report +from rpython.translator.tool.staticsizereport import print_report def parse_options(argv): kwds = {} diff --git a/pypy/bin/translatorshell.py b/pypy/bin/translatorshell.py --- a/pypy/bin/translatorshell.py +++ b/pypy/bin/translatorshell.py @@ -24,9 +24,9 @@ """ import autopath, os, sys -from pypy.translator.interactive import Translation -from pypy.rpython.rtyper import * -from pypy.rlib.rarithmetic import * +from rpython.translator.interactive import Translation +from rpython.rtyper.rtyper import * +from rpython.rlib.rarithmetic import * import py @@ -53,8 +53,8 @@ setup_readline() except ImportError, err: print "Disabling readline support (%s)" % err - from pypy.translator.test import snippet - from pypy.rpython.rtyper import RPythonTyper + from rpython.translator.test import snippet + from rpython.rtyper.rtyper import RPythonTyper if (os.getcwd() not in sys.path and os.path.curdir not in sys.path): diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -93,25 +93,25 @@ } module_import_dependencies = { - # no _rawffi if importing pypy.rlib.clibffi raises ImportError + # no _rawffi if importing rpython.rlib.clibffi raises ImportError # or CompilationError or py.test.skip.Exception - "_rawffi" : ["pypy.rlib.clibffi"], - "_ffi" : ["pypy.rlib.clibffi"], + "_rawffi" : ["rpython.rlib.clibffi"], + "_ffi" : ["rpython.rlib.clibffi"], - "zlib" : ["pypy.rlib.rzlib"], + "zlib" : ["rpython.rlib.rzlib"], "bz2" : ["pypy.module.bz2.interp_bz2"], "pyexpat" : ["pypy.module.pyexpat.interp_pyexpat"], "_ssl" : ["pypy.module._ssl.interp_ssl"], "_hashlib" : ["pypy.module._ssl.interp_ssl"], "_minimal_curses": ["pypy.module._minimal_curses.fficurses"], - "_continuation": ["pypy.rlib.rstacklet"], + "_continuation": ["rpython.rlib.rstacklet"], } def get_module_validator(modname): if modname in module_import_dependencies: modlist = module_import_dependencies[modname] def validator(config): - from pypy.rpython.tool.rffi_platform import CompilationError + from rpython.rtyper.tool.rffi_platform import CompilationError try: for name in modlist: __import__(name) diff --git a/pypy/config/test/test_config.py b/pypy/config/test/test_config.py --- a/pypy/config/test/test_config.py +++ b/pypy/config/test/test_config.py @@ -90,7 +90,7 @@ assert c1.a == [1] def test_annotator_folding(): - from pypy.translator.interactive import Translation + from rpython.translator.interactive import Translation gcoption = ChoiceOption('name', 'GC name', ['ref', 'framework'], 'ref') gcgroup = OptionDescription('gc', '', [gcoption]) diff --git a/pypy/config/translationoption.py b/pypy/config/translationoption.py --- a/pypy/config/translationoption.py +++ b/pypy/config/translationoption.py @@ -198,7 +198,7 @@ default=DEFL_INLINE_THRESHOLD, cmdline="--inline-threshold"), StrOption("inline_heuristic", "Dotted name of an heuristic function " "for inlining", - default="pypy.translator.backendopt.inline.inlining_heuristic", + default="rpython.translator.backendopt.inline.inlining_heuristic", cmdline="--inline-heuristic"), BoolOption("print_statistics", "Print statistics while optimizing", @@ -226,7 +226,7 @@ StrOption("profile_based_inline_heuristic", "Dotted name of an heuristic function " "for profile based inlining", - default="pypy.translator.backendopt.inline.inlining_heuristic", + default="rpython.translator.backendopt.inline.inlining_heuristic", ), # cmdline="--prof-based-inline-heuristic" fix me # control clever malloc removal BoolOption("clever_malloc_removal", @@ -241,7 +241,7 @@ StrOption("clever_malloc_removal_heuristic", "Dotted name of an heuristic function " "for inlining in clever malloc removal", - default="pypy.translator.backendopt.inline.inlining_heuristic", + default="rpython.translator.backendopt.inline.inlining_heuristic", cmdline="--clever-malloc-removal-heuristic"), BoolOption("remove_asserts", @@ -339,7 +339,7 @@ def final_check_config(config): # XXX: this should be a real config option, but it is hard to refactor it; # instead, we "just" patch it from here - from pypy.rlib import rfloat + from rpython.rlib import rfloat if config.translation.type_system == 'ootype': rfloat.USE_SHORT_FLOAT_REPR = False @@ -393,11 +393,11 @@ # ---------------------------------------------------------------- def set_platform(config): - from pypy.translator.platform import set_platform + from rpython.translator.platform import set_platform set_platform(config.translation.platform, config.translation.cc) def get_platform(config): - from pypy.translator.platform import pick_platform + from rpython.translator.platform import pick_platform opt = config.translation.platform cc = config.translation.cc return pick_platform(opt, cc) diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -36,7 +36,7 @@ def _set_platform(opt, opt_str, value, parser): from pypy.config.translationoption import PLATFORMS - from pypy.translator.platform import set_platform + from rpython.translator.platform import set_platform if value not in PLATFORMS: raise ValueError("%s not in %s" % (value, PLATFORMS)) set_platform(value, None) diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py --- a/pypy/interpreter/argument.py +++ b/pypy/interpreter/argument.py @@ -3,8 +3,8 @@ """ from pypy.interpreter.error import OperationError, operationerrfmt -from pypy.rlib.debug import make_sure_not_resized -from pypy.rlib import jit +from rpython.rlib.debug import make_sure_not_resized +from rpython.rlib import jit class Signature(object): diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py --- a/pypy/interpreter/astcompiler/assemble.py +++ b/pypy/interpreter/astcompiler/assemble.py @@ -7,8 +7,8 @@ from pypy.tool import stdlib_opcode as ops from pypy.interpreter.error import OperationError -from pypy.rlib.objectmodel import we_are_translated -from pypy.rlib import rfloat +from rpython.rlib.objectmodel import we_are_translated +from rpython.rlib import rfloat class Instruction(object): diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py --- a/pypy/interpreter/astcompiler/ast.py +++ b/pypy/interpreter/astcompiler/ast.py @@ -3,7 +3,7 @@ from pypy.interpreter import typedef from pypy.interpreter.gateway import interp2app from pypy.interpreter.error import OperationError, operationerrfmt -from pypy.rlib.unroll import unrolling_iterable +from rpython.rlib.unroll import unrolling_iterable from pypy.tool.pairtype import extendabletype from pypy.tool.sourcetools import func_with_new_name diff --git a/pypy/interpreter/astcompiler/astbuilder.py b/pypy/interpreter/astcompiler/astbuilder.py --- a/pypy/interpreter/astcompiler/astbuilder.py +++ b/pypy/interpreter/astcompiler/astbuilder.py @@ -4,7 +4,7 @@ from pypy.interpreter.pyparser.pygram import syms, tokens from pypy.interpreter.pyparser.error import SyntaxError from pypy.interpreter.pyparser import parsestring -from pypy.rlib.objectmodel import specialize +from rpython.rlib.objectmodel import specialize def ast_from_node(space, node, compile_info): diff --git a/pypy/interpreter/astcompiler/misc.py b/pypy/interpreter/astcompiler/misc.py --- a/pypy/interpreter/astcompiler/misc.py +++ b/pypy/interpreter/astcompiler/misc.py @@ -1,7 +1,7 @@ from pypy.interpreter import gateway from pypy.interpreter.astcompiler import ast -from pypy.rlib.objectmodel import we_are_translated -from pypy.rlib.unroll import unrolling_iterable +from rpython.rlib.objectmodel import we_are_translated +from rpython.rlib.unroll import unrolling_iterable app = gateway.applevel(""" diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py --- a/pypy/interpreter/astcompiler/optimize.py +++ b/pypy/interpreter/astcompiler/optimize.py @@ -4,8 +4,8 @@ from pypy.interpreter.astcompiler import ast, consts, misc from pypy.tool import stdlib_opcode as ops from pypy.interpreter.error import OperationError -from pypy.rlib.unroll import unrolling_iterable -from pypy.rlib.runicode import MAXUNICODE +from rpython.rlib.unroll import unrolling_iterable +from rpython.rlib.runicode import MAXUNICODE def optimize_ast(space, tree, compile_info): diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py --- a/pypy/interpreter/astcompiler/tools/asdl_py.py +++ b/pypy/interpreter/astcompiler/tools/asdl_py.py @@ -542,7 +542,7 @@ from pypy.interpreter import typedef from pypy.interpreter.gateway import interp2app from pypy.interpreter.error import OperationError, operationerrfmt -from pypy.rlib.unroll import unrolling_iterable +from rpython.rlib.unroll import unrolling_iterable from pypy.tool.pairtype import extendabletype from pypy.tool.sourcetools import func_with_new_name diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -8,11 +8,11 @@ from pypy.interpreter.miscutils import ThreadLocals from pypy.tool.cache import Cache from pypy.tool.uid import HUGEVAL_BYTES -from pypy.rlib import jit -from pypy.rlib.debug import make_sure_not_resized -from pypy.rlib.objectmodel import we_are_translated, newlist_hint,\ +from rpython.rlib import jit +from rpython.rlib.debug import make_sure_not_resized +from rpython.rlib.objectmodel import we_are_translated, newlist_hint,\ compute_unique_id -from pypy.rlib.rarithmetic import r_uint +from rpython.rlib.rarithmetic import r_uint __all__ = ['ObjSpace', 'OperationError', 'Wrappable', 'W_Root'] @@ -1327,7 +1327,7 @@ def str0_w(self, w_obj): "Like str_w, but rejects strings with NUL bytes." - from pypy.rlib import rstring + from rpython.rlib import rstring result = w_obj.str_w(self) if '\x00' in result: raise OperationError(self.w_TypeError, self.wrap( @@ -1355,7 +1355,7 @@ def unicode0_w(self, w_obj): "Like unicode_w, but rejects strings with NUL bytes." - from pypy.rlib import rstring + from rpython.rlib import rstring result = w_obj.unicode_w(self) if u'\x00' in result: raise OperationError(self.w_TypeError, self.wrap( @@ -1451,7 +1451,7 @@ except OperationError, e: if not e.match(self, self.w_OverflowError): raise - from pypy.rlib.rarithmetic import intmask + from rpython.rlib.rarithmetic import intmask return intmask(self.bigint_w(w_obj).uintmask()) def truncatedlonglong_w(self, w_obj): @@ -1462,7 +1462,7 @@ except OperationError, e: if not e.match(self, self.w_OverflowError): raise - from pypy.rlib.rarithmetic import longlongmask + from rpython.rlib.rarithmetic import longlongmask return longlongmask(self.bigint_w(w_obj).ulonglongmask()) def c_filedescriptor_w(self, w_fd): @@ -1529,7 +1529,7 @@ ## Table describing the regular part of the interface of object spaces, ## namely all methods which only take w_ arguments and return a w_ result -## (if any). Note: keep in sync with pypy.objspace.flow.operation.Table. +## (if any). Note: keep in sync with rpython.flowspace.operation.Table. ObjSpace.MethodTable = [ # method name # symbol # number of arguments # special method name(s) diff --git a/pypy/interpreter/buffer.py b/pypy/interpreter/buffer.py --- a/pypy/interpreter/buffer.py +++ b/pypy/interpreter/buffer.py @@ -19,8 +19,8 @@ from pypy.interpreter.typedef import TypeDef from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.error import OperationError -from pypy.rlib.objectmodel import compute_hash -from pypy.rlib.rstring import StringBuilder +from rpython.rlib.objectmodel import compute_hash +from rpython.rlib.rstring import StringBuilder class Buffer(Wrappable): @@ -155,7 +155,7 @@ if space.isinstance_w(w_object, space.w_unicode): # unicode objects support the old buffer interface # but not the new buffer interface (change in python 2.7) - from pypy.rlib.rstruct.unichar import pack_unichar, UNICODE_SIZE + from rpython.rlib.rstruct.unichar import pack_unichar, UNICODE_SIZE unistr = space.unicode_w(w_object) builder = StringBuilder(len(unistr) * UNICODE_SIZE) for unich in unistr: diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py --- a/pypy/interpreter/error.py +++ b/pypy/interpreter/error.py @@ -1,6 +1,6 @@ import os, sys -from pypy.rlib import jit -from pypy.rlib.objectmodel import we_are_translated +from rpython.rlib import jit +from rpython.rlib.objectmodel import we_are_translated from errno import EINTR AUTO_DEBUG = os.getenv('PYPY_DEBUG') @@ -324,7 +324,7 @@ try: OpErrFmt = _fmtcache2[formats] except KeyError: - from pypy.rlib.unroll import unrolling_iterable + from rpython.rlib.unroll import unrolling_iterable attrs = ['x%d' % i for i in range(len(formats))] entries = unrolling_iterable(enumerate(attrs)) # @@ -382,7 +382,7 @@ _WINDOWS = True def wrap_windowserror(space, e, w_filename=None): - from pypy.rlib import rwin32 + from rpython.rlib import rwin32 winerror = e.winerror try: @@ -440,7 +440,7 @@ wrap_oserror._annspecialcase_ = 'specialize:arg(3)' def exception_from_errno(space, w_type): - from pypy.rlib.rposix import get_errno + from rpython.rlib.rposix import get_errno errno = get_errno() msg = os.strerror(errno) diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -1,8 +1,8 @@ import sys from pypy.interpreter.error import OperationError -from pypy.rlib.rarithmetic import LONG_BIT -from pypy.rlib.unroll import unrolling_iterable -from pypy.rlib import jit +from rpython.rlib.rarithmetic import LONG_BIT +from rpython.rlib.unroll import unrolling_iterable +from rpython.rlib import jit TICK_COUNTER_STEP = 100 diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py --- a/pypy/interpreter/function.py +++ b/pypy/interpreter/function.py @@ -6,13 +6,13 @@ attribute. """ -from pypy.rlib.unroll import unrolling_iterable +from rpython.rlib.unroll import unrolling_iterable from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.eval import Code from pypy.interpreter.argument import Arguments -from pypy.rlib import jit -from pypy.rlib.debug import make_sure_not_resized +from rpython.rlib import jit +from rpython.rlib.debug import make_sure_not_resized funccallunrolling = unrolling_iterable(range(4)) diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py --- a/pypy/interpreter/gateway.py +++ b/pypy/interpreter/gateway.py @@ -19,9 +19,9 @@ SpaceCache, DescrMismatch) from pypy.interpreter.error import OperationError from pypy.interpreter.function import ClassMethod, FunctionWithFixedCode -from pypy.rlib import rstackovf -from pypy.rlib.objectmodel import we_are_translated -from pypy.rlib.rarithmetic import r_longlong, r_int, r_ulonglong, r_uint +from rpython.rlib import rstackovf +from rpython.rlib.objectmodel import we_are_translated +from rpython.rlib.rarithmetic import r_longlong, r_int, r_ulonglong, r_uint from pypy.tool.sourcetools import func_with_new_name, compile2 diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py --- a/pypy/interpreter/generator.py +++ b/pypy/interpreter/generator.py @@ -1,7 +1,7 @@ from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.error import OperationError from pypy.interpreter.pyopcode import LoopBlock -from pypy.rlib import jit +from rpython.rlib import jit class GeneratorIterator(Wrappable): diff --git a/pypy/interpreter/module.py b/pypy/interpreter/module.py --- a/pypy/interpreter/module.py +++ b/pypy/interpreter/module.py @@ -4,7 +4,7 @@ from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.error import OperationError -from pypy.rlib.objectmodel import we_are_translated +from rpython.rlib.objectmodel import we_are_translated class Module(Wrappable): """A module.""" diff --git a/pypy/interpreter/nestedscope.py b/pypy/interpreter/nestedscope.py --- a/pypy/interpreter/nestedscope.py +++ b/pypy/interpreter/nestedscope.py @@ -3,7 +3,7 @@ from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.mixedmodule import MixedModule from pypy.interpreter.astcompiler import consts -from pypy.rlib import jit +from rpython.rlib import jit from pypy.tool.uid import uid class Cell(Wrappable): diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -13,10 +13,10 @@ from pypy.interpreter.astcompiler.consts import ( CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS, CO_NESTED, CO_GENERATOR, CO_CONTAINSGLOBALS) -from pypy.rlib.rarithmetic import intmask -from pypy.rlib.debug import make_sure_not_resized -from pypy.rlib import jit -from pypy.rlib.objectmodel import compute_hash +from rpython.rlib.rarithmetic import intmask +from rpython.rlib.debug import make_sure_not_resized +from rpython.rlib import jit +from rpython.rlib.objectmodel import compute_hash from pypy.tool.stdlib_opcode import opcodedesc, HAVE_ARGUMENT diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py --- a/pypy/interpreter/pyframe.py +++ b/pypy/interpreter/pyframe.py @@ -7,11 +7,11 @@ from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.executioncontext import ExecutionContext from pypy.interpreter import pytraceback -from pypy.rlib.objectmodel import we_are_translated, instantiate -from pypy.rlib.jit import hint -from pypy.rlib.debug import make_sure_not_resized, check_nonneg -from pypy.rlib.rarithmetic import intmask, r_uint -from pypy.rlib import jit +from rpython.rlib.objectmodel import we_are_translated, instantiate +from rpython.rlib.jit import hint +from rpython.rlib.debug import make_sure_not_resized, check_nonneg +from rpython.rlib.rarithmetic import intmask, r_uint +from rpython.rlib import jit from pypy.tool import stdlib_opcode from pypy.tool.stdlib_opcode import host_bytecode_spec diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -10,11 +10,11 @@ from pypy.interpreter import gateway, function, eval, pyframe, pytraceback from pypy.interpreter.pycode import PyCode, BytecodeCorruption from pypy.tool.sourcetools import func_with_new_name -from pypy.rlib.objectmodel import we_are_translated -from pypy.rlib import jit, rstackovf -from pypy.rlib.rarithmetic import r_uint, intmask -from pypy.rlib.unroll import unrolling_iterable -from pypy.rlib.debug import check_nonneg +from rpython.rlib.objectmodel import we_are_translated +from rpython.rlib import jit, rstackovf +from rpython.rlib.rarithmetic import r_uint, intmask +from rpython.rlib.unroll import unrolling_iterable +from rpython.rlib.debug import check_nonneg from pypy.tool.stdlib_opcode import (bytecode_spec, unrolling_all_opcode_descs) diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py --- a/pypy/interpreter/pyparser/parsestring.py +++ b/pypy/interpreter/pyparser/parsestring.py @@ -1,6 +1,6 @@ from pypy.interpreter.error import OperationError from pypy.interpreter import unicodehelper -from pypy.rlib.rstring import StringBuilder +from rpython.rlib.rstring import StringBuilder def parsestr(space, encoding, s, unicode_literal=False): """Parses a string or unicode literal, and return a wrapped value. diff --git a/pypy/interpreter/streamutil.py b/pypy/interpreter/streamutil.py --- a/pypy/interpreter/streamutil.py +++ b/pypy/interpreter/streamutil.py @@ -1,4 +1,4 @@ -from pypy.rlib.streamio import StreamError +from rpython.rlib.streamio import StreamError from pypy.interpreter.error import OperationError, wrap_oserror2 def wrap_streamerror(space, e, w_filename=None): diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py --- a/pypy/interpreter/test/test_gateway.py +++ b/pypy/interpreter/test/test_gateway.py @@ -197,7 +197,7 @@ space.call_function, w_app_g, space.wrap(-1)) def test_interp2app_unwrap_spec_c_int(self): - from pypy.rlib.rarithmetic import r_longlong + from rpython.rlib.rarithmetic import r_longlong space = self.space w = space.wrap def g(space, x): diff --git a/pypy/interpreter/test/test_objspace.py b/pypy/interpreter/test/test_objspace.py --- a/pypy/interpreter/test/test_objspace.py +++ b/pypy/interpreter/test/test_objspace.py @@ -2,7 +2,7 @@ from pypy.interpreter.error import OperationError from pypy.interpreter.function import Function from pypy.interpreter.pycode import PyCode -from pypy.rlib.rarithmetic import r_longlong, r_ulonglong +from rpython.rlib.rarithmetic import r_longlong, r_ulonglong import sys # this test isn't so much to test that the objspace interface *works* diff --git a/pypy/interpreter/test/test_zzpickle_and_slow.py b/pypy/interpreter/test/test_zzpickle_and_slow.py --- a/pypy/interpreter/test/test_zzpickle_and_slow.py +++ b/pypy/interpreter/test/test_zzpickle_and_slow.py @@ -1,7 +1,7 @@ import py from pypy import conftest from pypy.interpreter import gateway -from pypy.rlib.jit import non_virtual_ref, vref_None +from rpython.rlib.jit import non_virtual_ref, vref_None class AppTestSlow: spaceconfig = dict(usemodules=['itertools']) diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py --- a/pypy/interpreter/typedef.py +++ b/pypy/interpreter/typedef.py @@ -9,8 +9,8 @@ from pypy.interpreter.baseobjspace import Wrappable, DescrMismatch from pypy.interpreter.error import OperationError, operationerrfmt from pypy.tool.sourcetools import compile2, func_with_new_name -from pypy.rlib.objectmodel import instantiate, compute_identity_hash, specialize -from pypy.rlib.jit import promote +from rpython.rlib.objectmodel import instantiate, compute_identity_hash, specialize +from rpython.rlib.jit import promote class TypeDef: def __init__(self, __name, __base=None, __total_ordering__=None, **rawdict): diff --git a/pypy/interpreter/unicodehelper.py b/pypy/interpreter/unicodehelper.py --- a/pypy/interpreter/unicodehelper.py +++ b/pypy/interpreter/unicodehelper.py @@ -1,6 +1,6 @@ from pypy.interpreter.error import OperationError -from pypy.rlib.objectmodel import specialize -from pypy.rlib import runicode +from rpython.rlib.objectmodel import specialize +from rpython.rlib import runicode from pypy.module._codecs import interp_codecs @specialize.memo() diff --git a/pypy/module/__builtin__/abstractinst.py b/pypy/module/__builtin__/abstractinst.py --- a/pypy/module/__builtin__/abstractinst.py +++ b/pypy/module/__builtin__/abstractinst.py @@ -7,7 +7,7 @@ addition to checking for instances and subtypes in the normal way. """ -from pypy.rlib import jit +from rpython.rlib import jit from pypy.interpreter.error import OperationError from pypy.module.__builtin__.interp_classobj import W_ClassObject from pypy.module.__builtin__.interp_classobj import W_InstanceObject diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py --- a/pypy/module/__builtin__/functional.py +++ b/pypy/module/__builtin__/functional.py @@ -7,10 +7,10 @@ from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault from pypy.interpreter.typedef import TypeDef -from pypy.rlib import jit -from pypy.rlib.objectmodel import specialize -from pypy.rlib.rarithmetic import r_uint, intmask -from pypy.rlib.rbigint import rbigint +from rpython.rlib import jit +from rpython.rlib.objectmodel import specialize +from rpython.rlib.rarithmetic import r_uint, intmask +from rpython.rlib.rbigint import rbigint def get_len_of_range(space, lo, hi, step): diff --git a/pypy/module/__builtin__/interp_classobj.py b/pypy/module/__builtin__/interp_classobj.py --- a/pypy/module/__builtin__/interp_classobj.py +++ b/pypy/module/__builtin__/interp_classobj.py @@ -4,9 +4,9 @@ from pypy.interpreter.typedef import TypeDef, make_weakref_descr from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.typedef import GetSetProperty, descr_get_dict, descr_set_dict -from pypy.rlib.objectmodel import compute_identity_hash -from pypy.rlib.debug import make_sure_not_resized -from pypy.rlib import jit +from rpython.rlib.objectmodel import compute_identity_hash +from rpython.rlib.debug import make_sure_not_resized +from rpython.rlib import jit def raise_type_err(space, argument, expected, w_obj): diff --git a/pypy/module/__builtin__/operation.py b/pypy/module/__builtin__/operation.py --- a/pypy/module/__builtin__/operation.py +++ b/pypy/module/__builtin__/operation.py @@ -5,9 +5,9 @@ from pypy.interpreter import gateway from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import unwrap_spec, WrappedDefault -from pypy.rlib.runicode import UNICHR -from pypy.rlib.rfloat import isnan, isinf, round_double -from pypy.rlib import rfloat +from rpython.rlib.runicode import UNICHR +from rpython.rlib.rfloat import isnan, isinf, round_double +from rpython.rlib import rfloat import __builtin__ def abs(space, w_val): diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -71,6 +71,6 @@ PYC_MAGIC = get_pyc_magic(self.space) self.extra_interpdef('PYC_MAGIC', 'space.wrap(%d)' % PYC_MAGIC) # - from pypy.jit.backend import detect_cpu + from rpython.jit.backend import detect_cpu model = detect_cpu.autodetect_main_model_and_size() self.extra_interpdef('cpumodel', 'space.wrap(%r)' % model) diff --git a/pypy/module/__pypy__/interp_builders.py b/pypy/module/__pypy__/interp_builders.py --- a/pypy/module/__pypy__/interp_builders.py +++ b/pypy/module/__pypy__/interp_builders.py @@ -2,7 +2,7 @@ from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import TypeDef -from pypy.rlib.rstring import UnicodeBuilder, StringBuilder +from rpython.rlib.rstring import UnicodeBuilder, StringBuilder from pypy.tool.sourcetools import func_with_new_name diff --git a/pypy/module/__pypy__/interp_debug.py b/pypy/module/__pypy__/interp_debug.py --- a/pypy/module/__pypy__/interp_debug.py +++ b/pypy/module/__pypy__/interp_debug.py @@ -1,5 +1,5 @@ from pypy.interpreter.gateway import unwrap_spec -from pypy.rlib import debug, jit +from rpython.rlib import debug, jit @jit.dont_look_inside diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -1,11 +1,11 @@ from pypy.interpreter.baseobjspace import ObjSpace, W_Root from pypy.interpreter.error import OperationError, wrap_oserror from pypy.interpreter.gateway import unwrap_spec -from pypy.rlib.objectmodel import we_are_translated +from rpython.rlib.objectmodel import we_are_translated from pypy.objspace.std.listobject import W_ListObject from pypy.objspace.std.typeobject import MethodCache from pypy.objspace.std.mapdict import IndexCache -from pypy.rlib import rposix +from rpython.rlib import rposix def internal_repr(space, w_object): @@ -85,7 +85,7 @@ raise wrap_oserror(space, e) def get_console_cp(space): - from pypy.rlib import rwin32 # Windows only + from rpython.rlib import rwin32 # Windows only return space.newtuple([ space.wrap('cp%d' % rwin32.GetConsoleCP()), space.wrap('cp%d' % rwin32.GetConsoleOutputCP()), diff --git a/pypy/module/__pypy__/interp_time.py b/pypy/module/__pypy__/interp_time.py --- a/pypy/module/__pypy__/interp_time.py +++ b/pypy/module/__pypy__/interp_time.py @@ -3,9 +3,9 @@ from pypy.interpreter.error import exception_from_errno from pypy.interpreter.gateway import unwrap_spec -from pypy.rpython.lltypesystem import rffi, lltype -from pypy.rpython.tool import rffi_platform -from pypy.translator.tool.cbuild import ExternalCompilationInfo +from rpython.rtyper.lltypesystem import rffi, lltype +from rpython.rtyper.tool import rffi_platform +from rpython.translator.tool.cbuild import ExternalCompilationInfo if sys.platform == 'linux2': libraries = ["rt"] diff --git a/pypy/module/__pypy__/test/test_debug.py b/pypy/module/__pypy__/test/test_debug.py --- a/pypy/module/__pypy__/test/test_debug.py +++ b/pypy/module/__pypy__/test/test_debug.py @@ -1,7 +1,7 @@ import py from pypy.interpreter.gateway import interp2app -from pypy.rlib import debug +from rpython.rlib import debug class AppTestDebug: diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -1,5 +1,5 @@ from pypy.interpreter.mixedmodule import MixedModule -from pypy.rlib import rdynload +from rpython.rlib import rdynload class Module(MixedModule): diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py --- a/pypy/module/_cffi_backend/cbuffer.py +++ b/pypy/module/_cffi_backend/cbuffer.py @@ -3,7 +3,7 @@ from pypy.interpreter.buffer import RWBuffer from pypy.interpreter.gateway import unwrap_spec, interp2app from pypy.interpreter.typedef import TypeDef -from pypy.rpython.lltypesystem import rffi +from rpython.rtyper.lltypesystem import rffi from pypy.module._cffi_backend import cdataobj, ctypeptr, ctypearray diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py --- a/pypy/module/_cffi_backend/ccallback.py +++ b/pypy/module/_cffi_backend/ccallback.py @@ -3,10 +3,10 @@ """ import os from pypy.interpreter.error import OperationError, operationerrfmt -from pypy.rpython.lltypesystem import lltype, llmemory, rffi -from pypy.rlib.objectmodel import compute_unique_id, keepalive_until_here -from pypy.rlib import clibffi, rweakref, rgc -from pypy.rlib.rarithmetic import r_ulonglong +from rpython.rtyper.lltypesystem import lltype, llmemory, rffi +from rpython.rlib.objectmodel import compute_unique_id, keepalive_until_here +from rpython.rlib import clibffi, rweakref, rgc +from rpython.rlib.rarithmetic import r_ulonglong from pypy.module._cffi_backend.cdataobj import W_CData from pypy.module._cffi_backend.ctypefunc import SIZE_OF_FFI_ARG, BIG_ENDIAN diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py --- a/pypy/module/_cffi_backend/cdataobj.py +++ b/pypy/module/_cffi_backend/cdataobj.py @@ -3,9 +3,9 @@ from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import TypeDef, make_weakref_descr -from pypy.rpython.lltypesystem import lltype, rffi -from pypy.rlib.objectmodel import keepalive_until_here, specialize -from pypy.rlib import objectmodel, rgc +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rlib.objectmodel import keepalive_until_here, specialize +from rpython.rlib import objectmodel, rgc from pypy.tool.sourcetools import func_with_new_name from pypy.module._cffi_backend import misc diff --git a/pypy/module/_cffi_backend/cerrno.py b/pypy/module/_cffi_backend/cerrno.py --- a/pypy/module/_cffi_backend/cerrno.py +++ b/pypy/module/_cffi_backend/cerrno.py @@ -1,11 +1,11 @@ import sys -from pypy.rlib import rposix +from rpython.rlib import rposix from pypy.interpreter.executioncontext import ExecutionContext from pypy.interpreter.gateway import unwrap_spec WIN32 = sys.platform == 'win32' if WIN32: - from pypy.rlib import rwin32 + from rpython.rlib import rwin32 ExecutionContext._cffi_saved_errno = 0 diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py --- a/pypy/module/_cffi_backend/ctypearray.py +++ b/pypy/module/_cffi_backend/ctypearray.py @@ -6,9 +6,9 @@ from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.gateway import interp2app from pypy.interpreter.typedef import TypeDef -from pypy.rpython.lltypesystem import rffi -from pypy.rlib.objectmodel import keepalive_until_here -from pypy.rlib.rarithmetic import ovfcheck +from rpython.rtyper.lltypesystem import rffi +from rpython.rlib.objectmodel import keepalive_until_here +from rpython.rlib.rarithmetic import ovfcheck from pypy.module._cffi_backend.ctypeprim import W_CTypePrimitiveChar from pypy.module._cffi_backend.ctypeprim import W_CTypePrimitiveUniChar diff --git a/pypy/module/_cffi_backend/ctypeenum.py b/pypy/module/_cffi_backend/ctypeenum.py --- a/pypy/module/_cffi_backend/ctypeenum.py +++ b/pypy/module/_cffi_backend/ctypeenum.py @@ -3,9 +3,9 @@ """ from pypy.interpreter.error import OperationError, operationerrfmt -from pypy.rpython.lltypesystem import rffi -from pypy.rlib.rarithmetic import intmask, r_ulonglong -from pypy.rlib.objectmodel import keepalive_until_here +from rpython.rtyper.lltypesystem import rffi +from rpython.rlib.rarithmetic import intmask, r_ulonglong +from rpython.rlib.objectmodel import keepalive_until_here from pypy.module._cffi_backend.ctypeprim import W_CTypePrimitiveSigned from pypy.module._cffi_backend import misc diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py --- a/pypy/module/_cffi_backend/ctypefunc.py +++ b/pypy/module/_cffi_backend/ctypefunc.py @@ -4,13 +4,13 @@ import sys from pypy.interpreter.error import OperationError, operationerrfmt -from pypy.rpython.lltypesystem import lltype, llmemory, rffi -from pypy.rlib import jit, clibffi, jit_libffi -from pypy.rlib.jit_libffi import CIF_DESCRIPTION, CIF_DESCRIPTION_P -from pypy.rlib.jit_libffi import FFI_TYPE, FFI_TYPE_P, FFI_TYPE_PP -from pypy.rlib.jit_libffi import SIZE_OF_FFI_ARG -from pypy.rlib.objectmodel import we_are_translated, instantiate -from pypy.rlib.objectmodel import keepalive_until_here +from rpython.rtyper.lltypesystem import lltype, llmemory, rffi +from rpython.rlib import jit, clibffi, jit_libffi +from rpython.rlib.jit_libffi import CIF_DESCRIPTION, CIF_DESCRIPTION_P +from rpython.rlib.jit_libffi import FFI_TYPE, FFI_TYPE_P, FFI_TYPE_PP +from rpython.rlib.jit_libffi import SIZE_OF_FFI_ARG +from rpython.rlib.objectmodel import we_are_translated, instantiate +from rpython.rlib.objectmodel import keepalive_until_here from pypy.module._cffi_backend.ctypeobj import W_CType from pypy.module._cffi_backend.ctypeptr import W_CTypePtrBase, W_CTypePointer diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py --- a/pypy/module/_cffi_backend/ctypeobj.py +++ b/pypy/module/_cffi_backend/ctypeobj.py @@ -4,8 +4,8 @@ from pypy.interpreter.typedef import TypeDef from pypy.interpreter.typedef import make_weakref_descr from pypy.interpreter.typedef import GetSetProperty, interp_attrproperty -from pypy.rpython.lltypesystem import lltype, llmemory, rffi -from pypy.rlib.objectmodel import we_are_translated +from rpython.rtyper.lltypesystem import lltype, llmemory, rffi +from rpython.rlib.objectmodel import we_are_translated from pypy.module._cffi_backend import cdataobj diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py --- a/pypy/module/_cffi_backend/ctypeprim.py +++ b/pypy/module/_cffi_backend/ctypeprim.py @@ -3,10 +3,10 @@ """ from pypy.interpreter.error import operationerrfmt -from pypy.rpython.lltypesystem import lltype, rffi -from pypy.rlib.rarithmetic import r_uint, r_ulonglong, intmask -from pypy.rlib.objectmodel import keepalive_until_here -from pypy.rlib import jit +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rlib.rarithmetic import r_uint, r_ulonglong, intmask +from rpython.rlib.objectmodel import keepalive_until_here +from rpython.rlib import jit from pypy.module._cffi_backend.ctypeobj import W_CType from pypy.module._cffi_backend import cdataobj, misc diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py --- a/pypy/module/_cffi_backend/ctypeptr.py +++ b/pypy/module/_cffi_backend/ctypeptr.py @@ -4,10 +4,10 @@ from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.error import wrap_oserror -from pypy.rpython.lltypesystem import lltype, rffi -from pypy.rlib.objectmodel import keepalive_until_here -from pypy.rlib.rarithmetic import ovfcheck -from pypy.rlib import rposix +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rlib.objectmodel import keepalive_until_here +from rpython.rlib.rarithmetic import ovfcheck +from rpython.rlib import rposix from pypy.module._cffi_backend.ctypeobj import W_CType from pypy.module._cffi_backend import cdataobj, misc, ctypeprim, ctypevoid diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py --- a/pypy/module/_cffi_backend/ctypestruct.py +++ b/pypy/module/_cffi_backend/ctypestruct.py @@ -3,12 +3,12 @@ """ from pypy.interpreter.error import OperationError, operationerrfmt -from pypy.rpython.lltypesystem import lltype, rffi +from rpython.rtyper.lltypesystem import lltype, rffi from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.typedef import TypeDef, interp_attrproperty -from pypy.rlib.objectmodel import keepalive_until_here -from pypy.rlib.rarithmetic import r_uint, r_ulonglong, r_longlong, intmask -from pypy.rlib import jit +from rpython.rlib.objectmodel import keepalive_until_here +from rpython.rlib.rarithmetic import r_uint, r_ulonglong, r_longlong, intmask +from rpython.rlib import jit from pypy.module._cffi_backend.ctypeobj import W_CType from pypy.module._cffi_backend import cdataobj, ctypeprim, misc diff --git a/pypy/module/_cffi_backend/libraryobj.py b/pypy/module/_cffi_backend/libraryobj.py --- a/pypy/module/_cffi_backend/libraryobj.py +++ b/pypy/module/_cffi_backend/libraryobj.py @@ -3,8 +3,8 @@ from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import TypeDef -from pypy.rpython.lltypesystem import lltype, rffi -from pypy.rlib.rdynload import DLLHANDLE, dlopen, dlsym, dlclose, DLOpenError +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rlib.rdynload import DLLHANDLE, dlopen, dlsym, dlclose, DLOpenError from pypy.module._cffi_backend.cdataobj import W_CData from pypy.module._cffi_backend.ctypeobj import W_CType diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py --- a/pypy/module/_cffi_backend/misc.py +++ b/pypy/module/_cffi_backend/misc.py @@ -1,11 +1,11 @@ from __future__ import with_statement from pypy.interpreter.error import OperationError, operationerrfmt -from pypy.rpython.lltypesystem import lltype, llmemory, rffi -from pypy.rlib.rarithmetic import r_uint, r_ulonglong, is_signed_integer_type -from pypy.rlib.unroll import unrolling_iterable -from pypy.rlib.objectmodel import keepalive_until_here, specialize -from pypy.rlib import jit -from pypy.translator.tool.cbuild import ExternalCompilationInfo +from rpython.rtyper.lltypesystem import lltype, llmemory, rffi +from rpython.rlib.rarithmetic import r_uint, r_ulonglong, is_signed_integer_type +from rpython.rlib.unroll import unrolling_iterable +from rpython.rlib.objectmodel import keepalive_until_here, specialize +from rpython.rlib import jit +from rpython.translator.tool.cbuild import ExternalCompilationInfo # ____________________________________________________________ diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py --- a/pypy/module/_cffi_backend/newtype.py +++ b/pypy/module/_cffi_backend/newtype.py @@ -1,8 +1,8 @@ from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.gateway import unwrap_spec -from pypy.rpython.lltypesystem import lltype, rffi -from pypy.rlib.rarithmetic import ovfcheck -from pypy.rlib.objectmodel import specialize +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rlib.rarithmetic import ovfcheck +from rpython.rlib.objectmodel import specialize from pypy.module._cffi_backend import ctypeobj, ctypeprim, ctypeptr, ctypearray from pypy.module._cffi_backend import ctypestruct, ctypevoid, ctypeenum diff --git a/pypy/module/_cffi_backend/test/test_c.py b/pypy/module/_cffi_backend/test/test_c.py --- a/pypy/module/_cffi_backend/test/test_c.py +++ b/pypy/module/_cffi_backend/test/test_c.py @@ -22,8 +22,8 @@ from pypy.tool.udir import udir from pypy.interpreter import gateway from pypy.module._cffi_backend import Module -from pypy.translator.platform import host -from pypy.translator.tool.cbuild import ExternalCompilationInfo +from rpython.translator.platform import host +from rpython.translator.tool.cbuild import ExternalCompilationInfo class AppTestC(object): diff --git a/pypy/module/_cffi_backend/test/test_ztranslation.py b/pypy/module/_cffi_backend/test/test_ztranslation.py --- a/pypy/module/_cffi_backend/test/test_ztranslation.py +++ b/pypy/module/_cffi_backend/test/test_ztranslation.py @@ -1,6 +1,6 @@ from pypy.objspace.fake.checkmodule import checkmodule from pypy.module._cffi_backend import ctypeptr -from pypy.rpython.lltypesystem import lltype, rffi +from rpython.rtyper.lltypesystem import lltype, rffi # side-effect: FORMAT_LONGDOUBLE must be built before test_checkmodule() from pypy.module._cffi_backend import misc diff --git a/pypy/module/_codecs/__init__.py b/pypy/module/_codecs/__init__.py --- a/pypy/module/_codecs/__init__.py +++ b/pypy/module/_codecs/__init__.py @@ -1,5 +1,5 @@ from pypy.interpreter.mixedmodule import MixedModule -from pypy.rlib import runicode +from rpython.rlib import runicode from pypy.module._codecs import interp_codecs class Module(MixedModule): diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py --- a/pypy/module/_codecs/interp_codecs.py +++ b/pypy/module/_codecs/interp_codecs.py @@ -1,7 +1,7 @@ from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault -from pypy.rlib.rstring import UnicodeBuilder -from pypy.rlib.objectmodel import we_are_translated +from rpython.rlib.rstring import UnicodeBuilder +from rpython.rlib.objectmodel import we_are_translated class CodecState(object): def __init__(self, space): @@ -352,7 +352,7 @@ # ____________________________________________________________ # delegation to runicode -from pypy.rlib import runicode +from rpython.rlib import runicode def make_encoder_wrapper(name): rname = "unicode_encode_%s" % (name.replace("_encode", ""), ) diff --git a/pypy/module/_collections/interp_deque.py b/pypy/module/_collections/interp_deque.py --- a/pypy/module/_collections/interp_deque.py +++ b/pypy/module/_collections/interp_deque.py @@ -5,7 +5,7 @@ from pypy.interpreter.typedef import GetSetProperty from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.error import OperationError -from pypy.rlib.debug import check_nonneg +from rpython.rlib.debug import check_nonneg # A `dequeobject` is composed of a doubly-linked list of `block` nodes. diff --git a/pypy/module/_continuation/interp_continuation.py b/pypy/module/_continuation/interp_continuation.py --- a/pypy/module/_continuation/interp_continuation.py +++ b/pypy/module/_continuation/interp_continuation.py @@ -1,5 +1,5 @@ -from pypy.rlib.rstacklet import StackletThread -from pypy.rlib import jit +from rpython.rlib.rstacklet import StackletThread +from rpython.rlib import jit from pypy.interpreter.error import OperationError from pypy.interpreter.executioncontext import ExecutionContext from pypy.interpreter.baseobjspace import Wrappable @@ -206,7 +206,7 @@ self.space = space self.ec = ec # for unpickling - from pypy.rlib.rweakref import RWeakKeyDictionary + from rpython.rlib.rweakref import RWeakKeyDictionary self.frame2continulet = RWeakKeyDictionary(PyFrame, W_Continulet) ExecutionContext.stacklet_thread = None diff --git a/pypy/module/_continuation/interp_pickle.py b/pypy/module/_continuation/interp_pickle.py --- a/pypy/module/_continuation/interp_pickle.py +++ b/pypy/module/_continuation/interp_pickle.py @@ -1,5 +1,5 @@ from pypy.tool import stdlib_opcode as pythonopcode -from pypy.rlib import jit +from rpython.rlib import jit from pypy.interpreter.error import OperationError from pypy.interpreter.pyframe import PyFrame from pypy.module._continuation.interp_continuation import State, global_state diff --git a/pypy/module/_continuation/test/support.py b/pypy/module/_continuation/test/support.py --- a/pypy/module/_continuation/test/support.py +++ b/pypy/module/_continuation/test/support.py @@ -1,5 +1,5 @@ import py -from pypy.rpython.tool.rffi_platform import CompilationError +from rpython.rtyper.tool.rffi_platform import CompilationError class BaseAppTest: @@ -7,7 +7,7 @@ def setup_class(cls): try: - import pypy.rlib.rstacklet + import rpython.rlib.rstacklet except CompilationError, e: py.test.skip("cannot import rstacklet: %s" % e) diff --git a/pypy/module/_csv/interp_reader.py b/pypy/module/_csv/interp_reader.py --- a/pypy/module/_csv/interp_reader.py +++ b/pypy/module/_csv/interp_reader.py @@ -1,4 +1,4 @@ -from pypy.rlib.rstring import StringBuilder +from rpython.rlib.rstring import StringBuilder from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import unwrap_spec diff --git a/pypy/module/_csv/interp_writer.py b/pypy/module/_csv/interp_writer.py --- a/pypy/module/_csv/interp_writer.py +++ b/pypy/module/_csv/interp_writer.py @@ -1,4 +1,4 @@ -from pypy.rlib.rstring import StringBuilder +from rpython.rlib.rstring import StringBuilder from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.error import OperationError from pypy.interpreter.typedef import TypeDef, interp2app diff --git a/pypy/module/_demo/demo.py b/pypy/module/_demo/demo.py --- a/pypy/module/_demo/demo.py +++ b/pypy/module/_demo/demo.py @@ -2,9 +2,9 @@ from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import TypeDef, GetSetProperty -from pypy.rpython.lltypesystem import rffi, lltype -from pypy.rpython.tool import rffi_platform -from pypy.translator.tool.cbuild import ExternalCompilationInfo +from rpython.rtyper.lltypesystem import rffi, lltype +from rpython.rtyper.tool import rffi_platform +from rpython.translator.tool.cbuild import ExternalCompilationInfo import sys, math time_t = rffi_platform.getsimpletype('time_t', '#include ', rffi.LONG) diff --git a/pypy/module/_ffi/interp_ffitype.py b/pypy/module/_ffi/interp_ffitype.py --- a/pypy/module/_ffi/interp_ffitype.py +++ b/pypy/module/_ffi/interp_ffitype.py @@ -1,6 +1,6 @@ -from pypy.rlib import libffi, clibffi -from pypy.rlib.rarithmetic import intmask -from pypy.rlib import jit +from rpython.rlib import libffi, clibffi +from rpython.rlib.rarithmetic import intmask +from rpython.rlib import jit from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.typedef import TypeDef, interp_attrproperty from pypy.interpreter.gateway import interp2app diff --git a/pypy/module/_ffi/interp_funcptr.py b/pypy/module/_ffi/interp_funcptr.py --- a/pypy/module/_ffi/interp_funcptr.py +++ b/pypy/module/_ffi/interp_funcptr.py @@ -5,14 +5,14 @@ from pypy.interpreter.typedef import TypeDef from pypy.module._ffi.interp_ffitype import W_FFIType # -from pypy.rpython.lltypesystem import lltype, rffi +from rpython.rtyper.lltypesystem import lltype, rffi # -from pypy.rlib import jit -from pypy.rlib import libffi -from pypy.rlib.clibffi import get_libc_name, StackCheckError, LibFFIError -from pypy.rlib.rdynload import DLOpenError -from pypy.rlib.rarithmetic import intmask, r_uint -from pypy.rlib.objectmodel import we_are_translated +from rpython.rlib import jit +from rpython.rlib import libffi +from rpython.rlib.clibffi import get_libc_name, StackCheckError, LibFFIError +from rpython.rlib.rdynload import DLOpenError +from rpython.rlib.rarithmetic import intmask, r_uint +from rpython.rlib.objectmodel import we_are_translated from pypy.module._ffi.type_converter import FromAppLevelConverter, ToAppLevelConverter from pypy.module._rawffi.interp_rawffi import got_libffi_error diff --git a/pypy/module/_ffi/interp_struct.py b/pypy/module/_ffi/interp_struct.py --- a/pypy/module/_ffi/interp_struct.py +++ b/pypy/module/_ffi/interp_struct.py @@ -1,9 +1,9 @@ -from pypy.rpython.lltypesystem import lltype, rffi -from pypy.rlib import clibffi -from pypy.rlib import libffi -from pypy.rlib import jit -from pypy.rlib.rgc import must_be_light_finalizer -from pypy.rlib.rarithmetic import r_uint, r_ulonglong, r_singlefloat, intmask +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rlib import clibffi +from rpython.rlib import libffi +from rpython.rlib import jit +from rpython.rlib.rgc import must_be_light_finalizer +from rpython.rlib.rarithmetic import r_uint, r_ulonglong, r_singlefloat, intmask from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.typedef import TypeDef, interp_attrproperty from pypy.interpreter.gateway import interp2app, unwrap_spec diff --git a/pypy/module/_ffi/test/test_funcptr.py b/pypy/module/_ffi/test/test_funcptr.py --- a/pypy/module/_ffi/test/test_funcptr.py +++ b/pypy/module/_ffi/test/test_funcptr.py @@ -1,8 +1,8 @@ -from pypy.rpython.lltypesystem import rffi -from pypy.rlib.clibffi import get_libc_name -from pypy.rlib.libffi import types -from pypy.rlib.libffi import CDLL -from pypy.rlib.test.test_clibffi import get_libm_name +from rpython.rtyper.lltypesystem import rffi +from rpython.rlib.clibffi import get_libc_name +from rpython.rlib.libffi import types +from rpython.rlib.libffi import CDLL +from rpython.rlib.test.test_clibffi import get_libm_name import sys, py @@ -12,8 +12,8 @@ @classmethod def prepare_c_example(cls): from pypy.tool.udir import udir - from pypy.translator.tool.cbuild import ExternalCompilationInfo - from pypy.translator.platform import platform + from rpython.translator.tool.cbuild import ExternalCompilationInfo + from rpython.translator.platform import platform c_file = udir.ensure("test__ffi", dir=1).join("foolib.c") # automatically collect the C source from the docstrings of the tests diff --git a/pypy/module/_ffi/test/test_struct.py b/pypy/module/_ffi/test/test_struct.py --- a/pypy/module/_ffi/test/test_struct.py +++ b/pypy/module/_ffi/test/test_struct.py @@ -58,9 +58,9 @@ else: cls.w_read_raw_mem = cls.space.wrap(interp2app(read_raw_mem)) # - from pypy.rlib import clibffi - from pypy.rlib.rarithmetic import r_uint - from pypy.rpython.lltypesystem import lltype, rffi + from rpython.rlib import clibffi + from rpython.rlib.rarithmetic import r_uint + from rpython.rtyper.lltypesystem import lltype, rffi dummy_type = lltype.malloc(clibffi.FFI_TYPE_P.TO, flavor='raw') dummy_type.c_size = r_uint(123) dummy_type.c_alignment = rffi.cast(rffi.USHORT, 0) diff --git a/pypy/module/_ffi/test/test_type_converter.py b/pypy/module/_ffi/test/test_type_converter.py --- a/pypy/module/_ffi/test/test_type_converter.py +++ b/pypy/module/_ffi/test/test_type_converter.py @@ -1,6 +1,6 @@ import sys -from pypy.rlib.rarithmetic import r_uint, r_singlefloat, r_longlong, r_ulonglong -from pypy.rlib.libffi import IS_32_BIT +from rpython.rlib.rarithmetic import r_uint, r_singlefloat, r_longlong, r_ulonglong +from rpython.rlib.libffi import IS_32_BIT from pypy.module._ffi.interp_ffitype import app_types, descr_new_pointer from pypy.module._ffi.type_converter import FromAppLevelConverter, ToAppLevelConverter diff --git a/pypy/module/_ffi/type_converter.py b/pypy/module/_ffi/type_converter.py --- a/pypy/module/_ffi/type_converter.py +++ b/pypy/module/_ffi/type_converter.py @@ -1,7 +1,7 @@ -from pypy.rlib import libffi -from pypy.rlib import jit -from pypy.rlib.rarithmetic import intmask, r_uint -from pypy.rpython.lltypesystem import rffi +from rpython.rlib import libffi +from rpython.rlib import jit +from rpython.rlib.rarithmetic import intmask, r_uint +from rpython.rtyper.lltypesystem import rffi from pypy.interpreter.error import operationerrfmt, OperationError from pypy.module._rawffi.structure import W_StructureInstance, W_Structure from pypy.module._ffi.interp_ffitype import app_types @@ -73,7 +73,7 @@ def _singlefloat(self, w_ffitype, w_obj): # a separate function, which can be seen by the jit or not, # depending on whether singlefloats are supported - from pypy.rlib.rarithmetic import r_singlefloat + from rpython.rlib.rarithmetic import r_singlefloat floatval = self.space.float_w(w_obj) singlefloatval = r_singlefloat(floatval) self.handle_singlefloat(w_ffitype, w_obj, singlefloatval) diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py --- a/pypy/module/_file/interp_file.py +++ b/pypy/module/_file/interp_file.py @@ -2,9 +2,9 @@ import os import stat import errno -from pypy.rlib import streamio -from pypy.rlib.rarithmetic import r_longlong -from pypy.rlib.rstring import StringBuilder +from rpython.rlib import streamio +from rpython.rlib.rarithmetic import r_longlong +from rpython.rlib.rstring import StringBuilder from pypy.module._file.interp_stream import W_AbstractStream, StreamErrors from pypy.module.posix.interp_posix import dispatch_filename from pypy.interpreter.error import OperationError, operationerrfmt diff --git a/pypy/module/_file/interp_stream.py b/pypy/module/_file/interp_stream.py --- a/pypy/module/_file/interp_stream.py +++ b/pypy/module/_file/interp_stream.py @@ -1,6 +1,6 @@ import py -from pypy.rlib import streamio -from pypy.rlib.streamio import StreamErrors +from rpython.rlib import streamio +from rpython.rlib.streamio import StreamErrors from pypy.interpreter.error import OperationError from pypy.interpreter.baseobjspace import ObjSpace, Wrappable diff --git a/pypy/module/_hashlib/__init__.py b/pypy/module/_hashlib/__init__.py --- a/pypy/module/_hashlib/__init__.py +++ b/pypy/module/_hashlib/__init__.py @@ -14,5 +14,5 @@ interpleveldefs['openssl_' + name] = 'interp_hashlib.new_' + name def startup(self, space): - from pypy.rlib.ropenssl import init_digests + from rpython.rlib.ropenssl import init_digests init_digests() diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py --- a/pypy/module/_hashlib/interp_hashlib.py +++ b/pypy/module/_hashlib/interp_hashlib.py @@ -4,10 +4,10 @@ from pypy.interpreter.error import OperationError from pypy.tool.sourcetools import func_renamer from pypy.interpreter.baseobjspace import Wrappable -from pypy.rpython.lltypesystem import lltype, llmemory, rffi -from pypy.rlib import rgc, ropenssl -from pypy.rlib.objectmodel import keepalive_until_here -from pypy.rlib.rstring import StringBuilder +from rpython.rtyper.lltypesystem import lltype, llmemory, rffi +from rpython.rlib import rgc, ropenssl +from rpython.rlib.objectmodel import keepalive_until_here +from rpython.rlib.rstring import StringBuilder from pypy.module.thread.os_lock import Lock algorithms = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512') diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py --- a/pypy/module/_io/interp_bufferedio.py +++ b/pypy/module/_io/interp_bufferedio.py @@ -4,9 +4,9 @@ from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.buffer import RWBuffer -from pypy.rlib.rstring import StringBuilder -from pypy.rlib.rarithmetic import r_longlong, intmask -from pypy.rlib import rposix +from rpython.rlib.rstring import StringBuilder +from rpython.rlib.rarithmetic import r_longlong, intmask +from rpython.rlib import rposix from pypy.tool.sourcetools import func_renamer from pypy.module._io.interp_iobase import ( W_IOBase, DEFAULT_BUFFER_SIZE, convert_size, diff --git a/pypy/module/_io/interp_bytesio.py b/pypy/module/_io/interp_bytesio.py --- a/pypy/module/_io/interp_bytesio.py +++ b/pypy/module/_io/interp_bytesio.py @@ -2,13 +2,13 @@ TypeDef, generic_new_descr, GetSetProperty) from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.error import OperationError, operationerrfmt -from pypy.rlib.rarithmetic import r_longlong +from rpython.rlib.rarithmetic import r_longlong from pypy.module._io.interp_bufferedio import W_BufferedIOBase from pypy.module._io.interp_iobase import convert_size import sys def buffer2string(buffer, start, end): - from pypy.rlib.rstring import StringBuilder + from rpython.rlib.rstring import StringBuilder builder = StringBuilder(end - start) for i in range(start, end): builder.append(buffer[i]) diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py --- a/pypy/module/_io/interp_fileio.py +++ b/pypy/module/_io/interp_fileio.py @@ -1,8 +1,8 @@ from pypy.interpreter.typedef import TypeDef, interp_attrproperty, GetSetProperty from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.error import OperationError, wrap_oserror, wrap_oserror2 -from pypy.rlib.rarithmetic import r_longlong -from pypy.rlib.rstring import StringBuilder +from rpython.rlib.rarithmetic import r_longlong +from rpython.rlib.rstring import StringBuilder from os import O_RDONLY, O_WRONLY, O_RDWR, O_CREAT, O_TRUNC import sys, os, stat, errno from pypy.module._io.interp_iobase import W_RawIOBase, convert_size @@ -401,7 +401,7 @@ if sys.platform == "win32": def _truncate(self, size): - from pypy.rlib.streamio import ftruncate_win32 + from rpython.rlib.streamio import ftruncate_win32 ftruncate_win32(self.fd, size) else: def _truncate(self, size): diff --git a/pypy/module/_io/interp_io.py b/pypy/module/_io/interp_io.py --- a/pypy/module/_io/interp_io.py +++ b/pypy/module/_io/interp_io.py @@ -7,7 +7,7 @@ from pypy.module.exceptions.interp_exceptions import W_IOError from pypy.module._io.interp_fileio import W_FileIO from pypy.module._io.interp_textio import W_TextIOWrapper -from pypy.rpython.module.ll_os_stat import STAT_FIELD_TYPES +from rpython.rtyper.module.ll_os_stat import STAT_FIELD_TYPES class W_BlockingIOError(W_IOError): diff --git a/pypy/module/_io/interp_iobase.py b/pypy/module/_io/interp_iobase.py --- a/pypy/module/_io/interp_iobase.py +++ b/pypy/module/_io/interp_iobase.py @@ -4,8 +4,8 @@ make_weakref_descr) from pypy.interpreter.gateway import interp2app from pypy.interpreter.error import OperationError, operationerrfmt -from pypy.rlib.rstring import StringBuilder -from pypy.rlib import rweakref +from rpython.rlib.rstring import StringBuilder +from rpython.rlib import rweakref DEFAULT_BUFFER_SIZE = 8192 diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py --- a/pypy/module/_io/interp_textio.py +++ b/pypy/module/_io/interp_textio.py @@ -5,9 +5,9 @@ from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.error import OperationError -from pypy.rlib.rarithmetic import intmask, r_ulonglong, r_uint -from pypy.rlib.rbigint import rbigint -from pypy.rlib.rstring import UnicodeBuilder +from rpython.rlib.rarithmetic import intmask, r_ulonglong, r_uint +from rpython.rlib.rbigint import rbigint +from rpython.rlib.rstring import UnicodeBuilder from pypy.module._codecs import interp_codecs from pypy.module._io.interp_iobase import convert_size import sys diff --git a/pypy/module/_locale/__init__.py b/pypy/module/_locale/__init__.py --- a/pypy/module/_locale/__init__.py +++ b/pypy/module/_locale/__init__.py @@ -1,5 +1,5 @@ from pypy.interpreter.mixedmodule import MixedModule -from pypy.rlib import rlocale +from rpython.rlib import rlocale import sys class Module(MixedModule): diff --git a/pypy/module/_locale/interp_locale.py b/pypy/module/_locale/interp_locale.py --- a/pypy/module/_locale/interp_locale.py +++ b/pypy/module/_locale/interp_locale.py @@ -1,12 +1,12 @@ -from pypy.rlib import rposix -from pypy.rlib.rarithmetic import intmask +from rpython.rlib import rposix +from rpython.rlib.rarithmetic import intmask from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import unwrap_spec -from pypy.rlib import rlocale +from rpython.rlib import rlocale from pypy.module.exceptions.interp_exceptions import _new_exception, W_Exception -from pypy.rpython.lltypesystem import lltype, rffi +from rpython.rtyper.lltypesystem import lltype, rffi W_Error = _new_exception('Error', W_Exception, 'locale error') diff --git a/pypy/module/_lsprof/interp_lsprof.py b/pypy/module/_lsprof/interp_lsprof.py --- a/pypy/module/_lsprof/interp_lsprof.py +++ b/pypy/module/_lsprof/interp_lsprof.py @@ -6,13 +6,13 @@ from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import (TypeDef, GetSetProperty, interp_attrproperty) -from pypy.rlib import jit -from pypy.rlib.objectmodel import we_are_translated -from pypy.rlib.rtimer import read_timestamp, _is_64_bit -from pypy.rpython.lltypesystem import rffi, lltype -from pypy.translator.tool.cbuild import ExternalCompilationInfo +from rpython.rlib import jit +from rpython.rlib.objectmodel import we_are_translated +from rpython.rlib.rtimer import read_timestamp, _is_64_bit +from rpython.rtyper.lltypesystem import rffi, lltype +from rpython.translator.tool.cbuild import ExternalCompilationInfo from pypy.tool.autopath import pypydir -from pypy.rlib.rarithmetic import r_longlong +from rpython.rlib.rarithmetic import r_longlong import time, sys diff --git a/pypy/module/_md5/interp_md5.py b/pypy/module/_md5/interp_md5.py --- a/pypy/module/_md5/interp_md5.py +++ b/pypy/module/_md5/interp_md5.py @@ -1,4 +1,4 @@ -from pypy.rlib import rmd5 +from rpython.rlib import rmd5 from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.typedef import TypeDef from pypy.interpreter.gateway import interp2app, unwrap_spec diff --git a/pypy/module/_minimal_curses/__init__.py b/pypy/module/_minimal_curses/__init__.py --- a/pypy/module/_minimal_curses/__init__.py +++ b/pypy/module/_minimal_curses/__init__.py @@ -10,7 +10,7 @@ from pypy.interpreter.mixedmodule import MixedModule from pypy.module._minimal_curses import fficurses from pypy.module._minimal_curses import interp_curses -from pypy.rlib.nonconst import NonConstant +from rpython.rlib.nonconst import NonConstant class Module(MixedModule): diff --git a/pypy/module/_minimal_curses/fficurses.py b/pypy/module/_minimal_curses/fficurses.py --- a/pypy/module/_minimal_curses/fficurses.py +++ b/pypy/module/_minimal_curses/fficurses.py @@ -2,12 +2,12 @@ """ The ffi for rpython, need to be imported for side effects """ -from pypy.rpython.lltypesystem import rffi -from pypy.rpython.lltypesystem import lltype -from pypy.rpython.tool import rffi_platform -from pypy.rpython.extfunc import register_external +from rpython.rtyper.lltypesystem import rffi +from rpython.rtyper.lltypesystem import lltype +from rpython.rtyper.tool import rffi_platform +from rpython.rtyper.extfunc import register_external from pypy.module._minimal_curses import interp_curses -from pypy.translator.tool.cbuild import ExternalCompilationInfo +from rpython.translator.tool.cbuild import ExternalCompilationInfo from sys import platform import os.path diff --git a/pypy/module/_minimal_curses/interp_curses.py b/pypy/module/_minimal_curses/interp_curses.py --- a/pypy/module/_minimal_curses/interp_curses.py +++ b/pypy/module/_minimal_curses/interp_curses.py @@ -13,8 +13,8 @@ def __init__(self, msg): self.msg = msg -from pypy.annotation.classdef import FORCE_ATTRIBUTES_INTO_CLASSES -from pypy.annotation.model import SomeString +from rpython.annotator.classdef import FORCE_ATTRIBUTES_INTO_CLASSES +from rpython.annotator.model import SomeString # this is necessary due to annmixlevel FORCE_ATTRIBUTES_INTO_CLASSES[curses_error] = {'msg': SomeString()} diff --git a/pypy/module/_minimal_curses/test/test_curses.py b/pypy/module/_minimal_curses/test/test_curses.py --- a/pypy/module/_minimal_curses/test/test_curses.py +++ b/pypy/module/_minimal_curses/test/test_curses.py @@ -75,7 +75,7 @@ """ Test compiled version """ def test_csetupterm(self): - from pypy.translator.c.test.test_genc import compile + from rpython.translator.c.test.test_genc import compile from pypy.module._minimal_curses import interp_curses def runs_setupterm(): interp_curses._curses_setupterm_null(1) @@ -84,7 +84,7 @@ fn() def test_ctgetstr(self): - from pypy.translator.c.test.test_genc import compile + from rpython.translator.c.test.test_genc import compile from pypy.module._minimal_curses import interp_curses def runs_ctgetstr(): interp_curses._curses_setupterm("xterm", 1) @@ -95,7 +95,7 @@ assert res == '\x1b[%i%p1%d;%p2%dH' def test_ctparm(self): - from pypy.translator.c.test.test_genc import compile + from rpython.translator.c.test.test_genc import compile from pypy.module._minimal_curses import interp_curses def runs_tparm(): interp_curses._curses_setupterm("xterm", 1) diff --git a/pypy/module/_multibytecodec/c_codecs.py b/pypy/module/_multibytecodec/c_codecs.py --- a/pypy/module/_multibytecodec/c_codecs.py +++ b/pypy/module/_multibytecodec/c_codecs.py @@ -1,6 +1,6 @@ import py -from pypy.rpython.lltypesystem import lltype, rffi -from pypy.translator.tool.cbuild import ExternalCompilationInfo +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.translator.tool.cbuild import ExternalCompilationInfo from pypy.tool.autopath import pypydir UNICODE_REPLACEMENT_CHARACTER = u'\uFFFD' diff --git a/pypy/module/_multibytecodec/interp_incremental.py b/pypy/module/_multibytecodec/interp_incremental.py --- a/pypy/module/_multibytecodec/interp_incremental.py +++ b/pypy/module/_multibytecodec/interp_incremental.py @@ -1,4 +1,4 @@ -from pypy.rpython.lltypesystem import lltype +from rpython.rtyper.lltypesystem import lltype from pypy.module._multibytecodec import c_codecs from pypy.module._multibytecodec.interp_multibytecodec import ( MultibyteCodec, wrap_unicodedecodeerror, wrap_runtimeerror, diff --git a/pypy/module/_multibytecodec/test/test_translation.py b/pypy/module/_multibytecodec/test/test_translation.py --- a/pypy/module/_multibytecodec/test/test_translation.py +++ b/pypy/module/_multibytecodec/test/test_translation.py @@ -1,5 +1,5 @@ from pypy.module._multibytecodec import c_codecs -from pypy.translator.c.test import test_standalone +from rpython.translator.c.test import test_standalone class TestTranslation(test_standalone.StandaloneTests): diff --git a/pypy/module/_multiprocessing/interp_connection.py b/pypy/module/_multiprocessing/interp_connection.py --- a/pypy/module/_multiprocessing/interp_connection.py +++ b/pypy/module/_multiprocessing/interp_connection.py @@ -4,9 +4,9 @@ from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault from pypy.interpreter.error import ( OperationError, wrap_oserror, operationerrfmt) -from pypy.rpython.lltypesystem import rffi, lltype -from pypy.rlib.rarithmetic import intmask -from pypy.rlib import rpoll, rsocket +from rpython.rtyper.lltypesystem import rffi, lltype +from rpython.rlib.rarithmetic import intmask +from rpython.rlib import rpoll, rsocket import sys READABLE = 1 @@ -194,20 +194,20 @@ if sys.platform == 'win32': def WRITE(self, data): - from pypy.rlib._rsocket_rffi import send, geterrno + from rpython.rlib._rsocket_rffi import send, geterrno length = send(self.fd, data, len(data), 0) if length < 0: raise WindowsError(geterrno(), "send") return length def READ(self, size): - from pypy.rlib._rsocket_rffi import socketrecv, geterrno + from rpython.rlib._rsocket_rffi import socketrecv, geterrno with rffi.scoped_alloc_buffer(size) as buf: length = socketrecv(self.fd, buf.raw, buf.size, 0) if length < 0: raise WindowsError(geterrno(), "recv") return buf.str(length) def CLOSE(self): - from pypy.rlib._rsocket_rffi import socketclose + from rpython.rlib._rsocket_rffi import socketclose socketclose(self.fd) else: def WRITE(self, data): @@ -343,7 +343,7 @@ class W_PipeConnection(W_BaseConnection): if sys.platform == 'win32': - from pypy.rlib.rwin32 import INVALID_HANDLE_VALUE + from rpython.rlib.rwin32 import INVALID_HANDLE_VALUE def __init__(self, handle, flags): W_BaseConnection.__init__(self, flags) @@ -372,7 +372,7 @@ return w_handle(space, self.handle) def do_close(self): - from pypy.rlib.rwin32 import CloseHandle + from rpython.rlib.rwin32 import CloseHandle if self.is_valid(): CloseHandle(self.handle) self.handle = self.INVALID_HANDLE_VALUE @@ -380,7 +380,7 @@ def do_send_string(self, space, buffer, offset, size): from pypy.module._multiprocessing.interp_win32 import ( _WriteFile, ERROR_NO_SYSTEM_RESOURCES) - from pypy.rlib import rwin32 + from rpython.rlib import rwin32 charp = rffi.str2charp(buffer) written_ptr = lltype.malloc(rffi.CArrayPtr(rwin32.DWORD).TO, 1, @@ -402,7 +402,7 @@ def do_recv_string(self, space, buflength, maxlength): from pypy.module._multiprocessing.interp_win32 import ( _ReadFile, _PeekNamedPipe, ERROR_BROKEN_PIPE, ERROR_MORE_DATA) - from pypy.rlib import rwin32 + from rpython.rlib import rwin32 from pypy.interpreter.error import wrap_windowserror read_ptr = lltype.malloc(rffi.CArrayPtr(rwin32.DWORD).TO, 1, @@ -457,7 +457,7 @@ def do_poll(self, space, timeout): from pypy.module._multiprocessing.interp_win32 import ( _PeekNamedPipe, _GetTickCount, _Sleep) - from pypy.rlib import rwin32 + from rpython.rlib import rwin32 from pypy.interpreter.error import wrap_windowserror bytes_ptr = lltype.malloc(rffi.CArrayPtr(rwin32.DWORD).TO, 1, flavor='raw') diff --git a/pypy/module/_multiprocessing/interp_memory.py b/pypy/module/_multiprocessing/interp_memory.py --- a/pypy/module/_multiprocessing/interp_memory.py +++ b/pypy/module/_multiprocessing/interp_memory.py @@ -1,5 +1,5 @@ from pypy.interpreter.error import OperationError -from pypy.rpython.lltypesystem import rffi +from rpython.rtyper.lltypesystem import rffi from pypy.module.mmap.interp_mmap import W_MMap def address_of_buffer(space, w_obj): diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py --- a/pypy/module/_multiprocessing/interp_semaphore.py +++ b/pypy/module/_multiprocessing/interp_semaphore.py @@ -3,11 +3,11 @@ from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.error import wrap_oserror, OperationError -from pypy.rpython.lltypesystem import rffi, lltype -from pypy.rlib import rgc -from pypy.rlib.rarithmetic import r_uint -from pypy.translator.tool.cbuild import ExternalCompilationInfo -from pypy.rpython.tool import rffi_platform as platform +from rpython.rtyper.lltypesystem import rffi, lltype +from rpython.rlib import rgc +from rpython.rlib.rarithmetic import r_uint +from rpython.translator.tool.cbuild import ExternalCompilationInfo +from rpython.rtyper.tool import rffi_platform as platform from pypy.module.thread import ll_thread from pypy.module._multiprocessing.interp_connection import w_handle import sys, os, time, errno @@ -15,7 +15,7 @@ RECURSIVE_MUTEX, SEMAPHORE = range(2) if sys.platform == 'win32': - from pypy.rlib import rwin32 + from rpython.rlib import rwin32 from pypy.module._multiprocessing.interp_win32 import ( handle_w, _GetTickCount) @@ -31,7 +31,7 @@ rwin32.BOOL) else: - from pypy.rlib import rposix + from rpython.rlib import rposix if sys.platform == 'darwin': libraries = [] diff --git a/pypy/module/_multiprocessing/interp_win32.py b/pypy/module/_multiprocessing/interp_win32.py --- a/pypy/module/_multiprocessing/interp_win32.py +++ b/pypy/module/_multiprocessing/interp_win32.py @@ -1,11 +1,11 @@ from pypy.interpreter.gateway import unwrap_spec, interp2app from pypy.interpreter.function import StaticMethod from pypy.interpreter.error import wrap_windowserror, OperationError -from pypy.rlib import rwin32 -from pypy.rlib.rarithmetic import r_uint -from pypy.rpython.lltypesystem import rffi, lltype -from pypy.translator.tool.cbuild import ExternalCompilationInfo -from pypy.rpython.tool import rffi_platform +from rpython.rlib import rwin32 +from rpython.rlib.rarithmetic import r_uint +from rpython.rtyper.lltypesystem import rffi, lltype +from rpython.translator.tool.cbuild import ExternalCompilationInfo +from rpython.rtyper.tool import rffi_platform from pypy.module._multiprocessing.interp_connection import w_handle CONSTANTS = """ diff --git a/pypy/module/_pickle_support/maker.py b/pypy/module/_pickle_support/maker.py --- a/pypy/module/_pickle_support/maker.py +++ b/pypy/module/_pickle_support/maker.py @@ -6,7 +6,7 @@ from pypy.interpreter.pyframe import PyFrame from pypy.interpreter.pytraceback import PyTraceback from pypy.interpreter.generator import GeneratorIterator -from pypy.rlib.objectmodel import instantiate +from rpython.rlib.objectmodel import instantiate from pypy.interpreter.gateway import unwrap_spec from pypy.objspace.std.iterobject import W_SeqIterObject, W_ReverseSeqIterObject diff --git a/pypy/module/_random/interp_random.py b/pypy/module/_random/interp_random.py --- a/pypy/module/_random/interp_random.py +++ b/pypy/module/_random/interp_random.py @@ -2,8 +2,8 @@ from pypy.interpreter.typedef import TypeDef from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.baseobjspace import Wrappable -from pypy.rlib.rarithmetic import r_uint, intmask -from pypy.rlib import rbigint, rrandom, rstring +from rpython.rlib.rarithmetic import r_uint, intmask +from rpython.rlib import rbigint, rrandom, rstring import time diff --git a/pypy/module/_rawffi/__init__.py b/pypy/module/_rawffi/__init__.py --- a/pypy/module/_rawffi/__init__.py +++ b/pypy/module/_rawffi/__init__.py @@ -4,7 +4,7 @@ from pypy.interpreter.mixedmodule import MixedModule from pypy.module._rawffi.interp_rawffi import W_CDLL -from pypy.rpython.lltypesystem import lltype, rffi +from rpython.rtyper.lltypesystem import lltype, rffi from pypy.module._rawffi.tracker import Tracker import sys @@ -46,7 +46,7 @@ if hasattr(interp_rawffi, 'check_HRESULT'): Module.interpleveldefs['check_HRESULT'] = 'interp_rawffi.check_HRESULT' - from pypy.rlib import clibffi + from rpython.rlib import clibffi for name in ['FUNCFLAG_STDCALL', 'FUNCFLAG_CDECL', 'FUNCFLAG_PYTHONAPI', 'FUNCFLAG_USE_ERRNO', 'FUNCFLAG_USE_LASTERROR', ]: diff --git a/pypy/module/_rawffi/array.py b/pypy/module/_rawffi/array.py --- a/pypy/module/_rawffi/array.py +++ b/pypy/module/_rawffi/array.py @@ -5,7 +5,7 @@ from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import TypeDef, GetSetProperty, interp_attrproperty -from pypy.rpython.lltypesystem import lltype, rffi +from rpython.rtyper.lltypesystem import lltype, rffi from pypy.interpreter.error import OperationError from pypy.module._rawffi.interp_rawffi import segfault_exception from pypy.module._rawffi.interp_rawffi import W_DataShape, W_DataInstance @@ -13,7 +13,7 @@ from pypy.module._rawffi.interp_rawffi import TYPEMAP from pypy.module._rawffi.interp_rawffi import size_alignment from pypy.module._rawffi.interp_rawffi import unpack_shape_with_length -from pypy.rlib.rarithmetic import r_uint +from rpython.rlib.rarithmetic import r_uint def push_elem(ll_array, pos, value): TP = lltype.typeOf(value) diff --git a/pypy/module/_rawffi/callback.py b/pypy/module/_rawffi/callback.py --- a/pypy/module/_rawffi/callback.py +++ b/pypy/module/_rawffi/callback.py @@ -1,14 +1,14 @@ from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import TypeDef, GetSetProperty -from pypy.rpython.lltypesystem import lltype, rffi +from rpython.rtyper.lltypesystem import lltype, rffi from pypy.module._rawffi.array import push_elem from pypy.module._rawffi.structure import W_Structure from pypy.module._rawffi.interp_rawffi import (W_DataInstance, letter2tp, unwrap_value, unpack_argshapes, got_libffi_error) -from pypy.rlib.clibffi import USERDATA_P, CallbackFuncPtr, FUNCFLAG_CDECL -from pypy.rlib.clibffi import ffi_type_void, LibFFIError -from pypy.rlib import rweakref +from rpython.rlib.clibffi import USERDATA_P, CallbackFuncPtr, FUNCFLAG_CDECL +from rpython.rlib.clibffi import ffi_type_void, LibFFIError +from rpython.rlib import rweakref from pypy.module._rawffi.tracker import tracker from pypy.interpreter.error import OperationError from pypy.interpreter import gateway diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py --- a/pypy/module/_rawffi/interp_rawffi.py +++ b/pypy/module/_rawffi/interp_rawffi.py @@ -3,18 +3,18 @@ from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import TypeDef, GetSetProperty -from pypy.rlib.clibffi import * -from pypy.rpython.lltypesystem import lltype, rffi -from pypy.rlib.unroll import unrolling_iterable -import pypy.rlib.rposix as rposix +from rpython.rlib.clibffi import * +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rlib.unroll import unrolling_iterable +import rpython.rlib.rposix as rposix _MS_WINDOWS = os.name == "nt" if _MS_WINDOWS: - from pypy.rlib import rwin32 + from rpython.rlib import rwin32 from pypy.tool.sourcetools import func_with_new_name -from pypy.rlib.rarithmetic import intmask, r_uint +from rpython.rlib.rarithmetic import intmask, r_uint from pypy.module._rawffi.tracker import tracker TYPEMAP = { @@ -553,11 +553,11 @@ if sys.platform == 'win32': def get_last_error(space): - from pypy.rlib.rwin32 import GetLastError + from rpython.rlib.rwin32 import GetLastError return space.wrap(GetLastError()) @unwrap_spec(error=int) def set_last_error(space, error): - from pypy.rlib.rwin32 import SetLastError + from rpython.rlib.rwin32 import SetLastError SetLastError(error) else: # always have at least a dummy version of these functions diff --git a/pypy/module/_rawffi/structure.py b/pypy/module/_rawffi/structure.py --- a/pypy/module/_rawffi/structure.py +++ b/pypy/module/_rawffi/structure.py @@ -6,7 +6,7 @@ from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import interp_attrproperty from pypy.interpreter.typedef import TypeDef, GetSetProperty -from pypy.rpython.lltypesystem import lltype, rffi +from rpython.rtyper.lltypesystem import lltype, rffi from pypy.interpreter.error import OperationError, operationerrfmt from pypy.module._rawffi.interp_rawffi import segfault_exception, _MS_WINDOWS from pypy.module._rawffi.interp_rawffi import W_DataShape, W_DataInstance @@ -14,8 +14,8 @@ from pypy.module._rawffi.interp_rawffi import unpack_shape_with_length from pypy.module._rawffi.interp_rawffi import size_alignment, LL_TYPEMAP from pypy.module._rawffi.interp_rawffi import unroll_letters_for_numbers -from pypy.rlib import clibffi -from pypy.rlib.rarithmetic import intmask, r_uint, signedtype, widen +from rpython.rlib import clibffi +from rpython.rlib.rarithmetic import intmask, r_uint, signedtype, widen def unpack_fields(space, w_fields): fields_w = space.unpackiterable(w_fields) diff --git a/pypy/module/_rawffi/test/test__rawffi.py b/pypy/module/_rawffi/test/test__rawffi.py --- a/pypy/module/_rawffi/test/test__rawffi.py +++ b/pypy/module/_rawffi/test/test__rawffi.py @@ -1,5 +1,5 @@ -from pypy.translator.platform import platform -from pypy.translator.tool.cbuild import ExternalCompilationInfo +from rpython.translator.platform import platform +from rpython.translator.tool.cbuild import ExternalCompilationInfo from pypy.module._rawffi.interp_rawffi import TYPEMAP from pypy.module._rawffi.tracker import Tracker @@ -197,7 +197,7 @@ def setup_class(cls): space = cls.space - from pypy.rlib.clibffi import get_libc_name + from rpython.rlib.clibffi import get_libc_name cls.w_lib_name = space.wrap(cls.prepare_c_example()) cls.w_libc_name = space.wrap(get_libc_name()) if sys.platform == 'win32': diff --git a/pypy/module/_sha/interp_sha.py b/pypy/module/_sha/interp_sha.py --- a/pypy/module/_sha/interp_sha.py +++ b/pypy/module/_sha/interp_sha.py @@ -1,4 +1,4 @@ -from pypy.rlib import rsha +from rpython.rlib import rsha from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.typedef import TypeDef From noreply at buildbot.pypy.org Fri Jan 4 22:30:56 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Fri, 4 Jan 2013 22:30:56 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Moved ansi modules and udir to rpython Message-ID: <20130104213056.77C981C11F3@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59699:81f81b215fcd Date: 2013-01-01 23:02 +0100 http://bitbucket.org/pypy/pypy/changeset/81f81b215fcd/ Log: Moved ansi modules and udir to rpython diff --git a/lib_pypy/ctypes_config_cache/rebuild.py b/lib_pypy/ctypes_config_cache/rebuild.py --- a/lib_pypy/ctypes_config_cache/rebuild.py +++ b/lib_pypy/ctypes_config_cache/rebuild.py @@ -15,7 +15,7 @@ _dirpath = os.path.dirname(__file__) or os.curdir -from pypy.tool.ansi_print import ansi_log +from rpython.tool.ansi_print import ansi_log log = py.log.Producer("ctypes_config_cache") py.log.setconsumer("ctypes_config_cache", ansi_log) diff --git a/lib_pypy/ctypes_config_cache/test/test_cache.py b/lib_pypy/ctypes_config_cache/test/test_cache.py --- a/lib_pypy/ctypes_config_cache/test/test_cache.py +++ b/lib_pypy/ctypes_config_cache/test/test_cache.py @@ -1,6 +1,6 @@ import py import sys, os -from pypy.tool.udir import udir +from rpython.tool.udir import udir dirpath = py.path.local(__file__).dirpath().dirpath() diff --git a/lib_pypy/pypy_test/test_dbm_extra.py b/lib_pypy/pypy_test/test_dbm_extra.py --- a/lib_pypy/pypy_test/test_dbm_extra.py +++ b/lib_pypy/pypy_test/test_dbm_extra.py @@ -1,6 +1,6 @@ from __future__ import absolute_import import py -from pypy.tool.udir import udir +from rpython.tool.udir import udir try: from lib_pypy import dbm except ImportError, e: diff --git a/lib_pypy/pypy_test/test_marshal_extra.py b/lib_pypy/pypy_test/test_marshal_extra.py --- a/lib_pypy/pypy_test/test_marshal_extra.py +++ b/lib_pypy/pypy_test/test_marshal_extra.py @@ -4,7 +4,7 @@ import marshal as cpy_marshal from lib_pypy import _marshal as marshal -from pypy.tool.udir import udir +from rpython.tool.udir import udir hello = "he" hello += "llo" diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py --- a/pypy/interpreter/error.py +++ b/pypy/interpreter/error.py @@ -368,7 +368,7 @@ # ____________________________________________________________ # Utilities -from pypy.tool.ansi_print import ansi_print +from rpython.tool.ansi_print import ansi_print def debug_print(text, file=None, newline=True): # 31: ANSI color code "red" diff --git a/pypy/interpreter/test/test_buffer.py b/pypy/interpreter/test/test_buffer.py --- a/pypy/interpreter/test/test_buffer.py +++ b/pypy/interpreter/test/test_buffer.py @@ -1,6 +1,6 @@ import py from pypy.interpreter.buffer import Buffer -from pypy.tool.udir import udir +from rpython.tool.udir import udir testdir = udir.ensure('test_buffer', dir=1) diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py --- a/pypy/interpreter/test/test_compiler.py +++ b/pypy/interpreter/test/test_compiler.py @@ -622,7 +622,7 @@ assert space.float_w(w_result) == 0 def test_dont_inherit_across_import(self): - from pypy.tool.udir import udir + from rpython.tool.udir import udir udir.join('test_dont_inherit_across_import.py').write('x = 1/2\n') space = self.space s1 = str(py.code.Source(""" diff --git a/pypy/interpreter/test/test_exec.py b/pypy/interpreter/test/test_exec.py --- a/pypy/interpreter/test/test_exec.py +++ b/pypy/interpreter/test/test_exec.py @@ -2,7 +2,7 @@ New for PyPy - Could be incorporated into CPython regression tests. """ -from pypy.tool.udir import udir +from rpython.tool.udir import udir def test_file(space): diff --git a/pypy/interpreter/test/test_executioncontext.py b/pypy/interpreter/test/test_executioncontext.py --- a/pypy/interpreter/test/test_executioncontext.py +++ b/pypy/interpreter/test/test_executioncontext.py @@ -262,7 +262,7 @@ def setup_method(self, meth): if not self.runappdirect: py.test.skip("test is meant for running with py.test -A") - from pypy.tool.udir import udir + from rpython.tool.udir import udir tmpfile = udir.join('test_execution_context') tmpfile.write(""" import gc diff --git a/pypy/interpreter/test/test_extmodules.py b/pypy/interpreter/test/test_extmodules.py --- a/pypy/interpreter/test/test_extmodules.py +++ b/pypy/interpreter/test/test_extmodules.py @@ -3,7 +3,7 @@ from pypy.config.pypyoption import get_pypy_config from pypy.objspace.std import StdObjSpace -from pypy.tool.udir import udir +from rpython.tool.udir import udir mod_init = """ from pypy.interpreter.mixedmodule import MixedModule diff --git a/pypy/interpreter/test/test_main.py b/pypy/interpreter/test/test_main.py --- a/pypy/interpreter/test/test_main.py +++ b/pypy/interpreter/test/test_main.py @@ -1,7 +1,7 @@ from cStringIO import StringIO import py -from pypy.tool.udir import udir +from rpython.tool.udir import udir from pypy.interpreter.baseobjspace import OperationError from pypy.interpreter import main diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py --- a/pypy/interpreter/test/test_pyframe.py +++ b/pypy/interpreter/test/test_pyframe.py @@ -1,4 +1,4 @@ -from pypy.tool import udir +from rpython.tool import udir from pypy.conftest import option diff --git a/pypy/interpreter/test/test_typedef.py b/pypy/interpreter/test/test_typedef.py --- a/pypy/interpreter/test/test_typedef.py +++ b/pypy/interpreter/test/test_typedef.py @@ -1,6 +1,6 @@ import gc from pypy.interpreter import typedef -from pypy.tool.udir import udir +from rpython.tool.udir import udir from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.gateway import ObjSpace, interp2app diff --git a/pypy/interpreter/test/test_zpy.py b/pypy/interpreter/test/test_zpy.py --- a/pypy/interpreter/test/test_zpy.py +++ b/pypy/interpreter/test/test_zpy.py @@ -1,5 +1,5 @@ -from pypy.tool.udir import udir +from rpython.tool.udir import udir import py import sys import pypy diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py --- a/pypy/module/__builtin__/test/test_builtin.py +++ b/pypy/module/__builtin__/test/test_builtin.py @@ -670,7 +670,7 @@ class TestInternal: def test_execfile(self, space): - from pypy.tool.udir import udir + from rpython.tool.udir import udir fn = str(udir.join('test_execfile')) f = open(fn, 'w') print >>f, "i=42" @@ -684,7 +684,7 @@ assert space.eq_w(w_value, space.wrap(42)) def test_execfile_different_lineendings(self, space): - from pypy.tool.udir import udir + from rpython.tool.udir import udir d = udir.ensure('lineending', dir=1) dos = d.join('dos.py') f = dos.open('wb') diff --git a/pypy/module/_cffi_backend/test/test_c.py b/pypy/module/_cffi_backend/test/test_c.py --- a/pypy/module/_cffi_backend/test/test_c.py +++ b/pypy/module/_cffi_backend/test/test_c.py @@ -19,7 +19,7 @@ if sys.version_info < (2, 6): py.test.skip("requires the b'' literal syntax") -from pypy.tool.udir import udir +from rpython.tool.udir import udir from pypy.interpreter import gateway from pypy.module._cffi_backend import Module from rpython.translator.platform import host diff --git a/pypy/module/_ffi/test/test_funcptr.py b/pypy/module/_ffi/test/test_funcptr.py --- a/pypy/module/_ffi/test/test_funcptr.py +++ b/pypy/module/_ffi/test/test_funcptr.py @@ -11,7 +11,7 @@ @classmethod def prepare_c_example(cls): - from pypy.tool.udir import udir + from rpython.tool.udir import udir from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.translator.platform import platform diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py --- a/pypy/module/_file/test/test_file.py +++ b/pypy/module/_file/test/test_file.py @@ -444,7 +444,7 @@ def test_flush_at_exit(): from pypy import conftest from pypy.tool.option import make_config, make_objspace - from pypy.tool.udir import udir + from rpython.tool.udir import udir tmpfile = udir.join('test_flush_at_exit') config = make_config(conftest.option) diff --git a/pypy/module/_file/test/test_file_extra.py b/pypy/module/_file/test/test_file_extra.py --- a/pypy/module/_file/test/test_file_extra.py +++ b/pypy/module/_file/test/test_file_extra.py @@ -1,8 +1,8 @@ import os, random, sys -import pypy.tool.udir +import rpython.tool.udir import py -udir = pypy.tool.udir.udir.ensure('test_file_extra', dir=1) +udir = rpython.tool.udir.udir.ensure('test_file_extra', dir=1) # XXX this file is a random test. It may only fail occasionally diff --git a/pypy/module/_io/test/test_bufferedio.py b/pypy/module/_io/test/test_bufferedio.py --- a/pypy/module/_io/test/test_bufferedio.py +++ b/pypy/module/_io/test/test_bufferedio.py @@ -1,6 +1,6 @@ from __future__ import with_statement from pypy.interpreter.gateway import interp2app -from pypy.tool.udir import udir +from rpython.tool.udir import udir from pypy.module._io import interp_bufferedio from pypy.interpreter.error import OperationError import py.test diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -1,4 +1,4 @@ -from pypy.tool.udir import udir +from rpython.tool.udir import udir import os class AppTestFileIO: @@ -163,7 +163,7 @@ def test_flush_at_exit(): from pypy import conftest from pypy.tool.option import make_config, make_objspace - from pypy.tool.udir import udir + from rpython.tool.udir import udir tmpfile = udir.join('test_flush_at_exit') config = make_config(conftest.option) diff --git a/pypy/module/_io/test/test_io.py b/pypy/module/_io/test/test_io.py --- a/pypy/module/_io/test/test_io.py +++ b/pypy/module/_io/test/test_io.py @@ -1,6 +1,6 @@ from __future__ import with_statement -from pypy.tool.udir import udir +from rpython.tool.udir import udir class AppTestIoModule: diff --git a/pypy/module/_minimal_curses/test/test_curses.py b/pypy/module/_minimal_curses/test/test_curses.py --- a/pypy/module/_minimal_curses/test/test_curses.py +++ b/pypy/module/_minimal_curses/test/test_curses.py @@ -1,5 +1,5 @@ from pypy.tool.autopath import pypydir -from pypy.tool.udir import udir +from rpython.tool.udir import udir import py import sys # tests here are run as snippets through a pexpected python subprocess diff --git a/pypy/module/_rawffi/test/test__rawffi.py b/pypy/module/_rawffi/test/test__rawffi.py --- a/pypy/module/_rawffi/test/test__rawffi.py +++ b/pypy/module/_rawffi/test/test__rawffi.py @@ -9,7 +9,7 @@ spaceconfig = dict(usemodules=['_rawffi', 'struct']) def prepare_c_example(): - from pypy.tool.udir import udir + from rpython.tool.udir import udir c_file = udir.ensure("test__rawffi", dir=1).join("xlib.c") c_file.write(py.code.Source(''' #include diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -1,7 +1,7 @@ import sys import py from pypy.tool.pytest.objspace import gettestobjspace -from pypy.tool.udir import udir +from rpython.tool.udir import udir from rpython.rlib import rsocket from rpython.rtyper.lltypesystem import lltype, rffi diff --git a/pypy/module/_winreg/test/test_winreg.py b/pypy/module/_winreg/test/test_winreg.py --- a/pypy/module/_winreg/test/test_winreg.py +++ b/pypy/module/_winreg/test/test_winreg.py @@ -1,4 +1,4 @@ -from pypy.tool.udir import udir +from rpython.tool.udir import udir import os, sys, py diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -12,7 +12,7 @@ from rpython.rlib.objectmodel import we_are_translated from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.translator.gensupp import NameManager -from pypy.tool.udir import udir +from rpython.tool.udir import udir from rpython.translator import platform from pypy.module.cpyext.state import State from pypy.interpreter.error import OperationError, operationerrfmt diff --git a/pypy/module/cpyext/presetup.py b/pypy/module/cpyext/presetup.py --- a/pypy/module/cpyext/presetup.py +++ b/pypy/module/cpyext/presetup.py @@ -11,7 +11,7 @@ dn = os.path.dirname rootdir = dn(dn(dn(dn(__file__)))) sys.path.insert(0, rootdir) -from pypy.tool.udir import udir +from rpython.tool.udir import udir pypydir = os.path.join(rootdir, 'pypy') f = open(os.path.join(str(udir), 'pyconfig.h'), "w") f.write("\n") diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -10,7 +10,7 @@ from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.translator import platform from rpython.translator.gensupp import uniquemodulename -from pypy.tool.udir import udir +from rpython.tool.udir import udir from pypy.module.cpyext import api from pypy.module.cpyext.state import State from pypy.module.cpyext.pyobject import RefcountState diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py --- a/pypy/module/cpyext/test/test_eval.py +++ b/pypy/module/cpyext/test/test_eval.py @@ -6,7 +6,7 @@ from pypy.module.cpyext.api import fopen, fclose, fileno, Py_ssize_tP from pypy.interpreter.gateway import interp2app from pypy.interpreter.astcompiler import consts -from pypy.tool.udir import udir +from rpython.tool.udir import udir import sys, os class TestEval(BaseApiTest): diff --git a/pypy/module/cpyext/test/test_pyfile.py b/pypy/module/cpyext/test/test_pyfile.py --- a/pypy/module/cpyext/test/test_pyfile.py +++ b/pypy/module/cpyext/test/test_pyfile.py @@ -2,7 +2,7 @@ from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.object import Py_PRINT_RAW from rpython.rtyper.lltypesystem import rffi, lltype -from pypy.tool.udir import udir +from rpython.tool.udir import udir import pytest class TestFile(BaseApiTest): diff --git a/pypy/module/fcntl/test/test_fcntl.py b/pypy/module/fcntl/test/test_fcntl.py --- a/pypy/module/fcntl/test/test_fcntl.py +++ b/pypy/module/fcntl/test/test_fcntl.py @@ -1,5 +1,5 @@ import os -from pypy.tool.udir import udir +from rpython.tool.udir import udir if os.name == "nt": from py.test import skip diff --git a/pypy/module/gc/test/test_app_referents.py b/pypy/module/gc/test/test_app_referents.py --- a/pypy/module/gc/test/test_app_referents.py +++ b/pypy/module/gc/test/test_app_referents.py @@ -1,5 +1,5 @@ import py, os -from pypy.tool.udir import udir +from rpython.tool.udir import udir def test_interface_to_dump_rpy_heap_str(space): diff --git a/pypy/module/gc/test/test_gc.py b/pypy/module/gc/test/test_gc.py --- a/pypy/module/gc/test/test_gc.py +++ b/pypy/module/gc/test/test_gc.py @@ -73,7 +73,7 @@ def setup_class(cls): import py - from pypy.tool.udir import udir + from rpython.tool.udir import udir from rpython.rlib import rgc class X(object): def __init__(self, count, size, links): diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py --- a/pypy/module/imp/test/test_import.py +++ b/pypy/module/imp/test/test_import.py @@ -3,7 +3,7 @@ from pypy.interpreter import gateway from pypy.interpreter.error import OperationError import pypy.interpreter.pycode -from pypy.tool.udir import udir +from rpython.tool.udir import udir from rpython.rlib import streamio from pypy.tool.option import make_config from pypy.tool.pytest.objspace import maketestobjspace diff --git a/pypy/module/marshal/test/test_marshal.py b/pypy/module/marshal/test/test_marshal.py --- a/pypy/module/marshal/test/test_marshal.py +++ b/pypy/module/marshal/test/test_marshal.py @@ -1,4 +1,4 @@ -from pypy.tool.udir import udir +from rpython.tool.udir import udir class AppTestMarshal: diff --git a/pypy/module/mmap/test/test_mmap.py b/pypy/module/mmap/test/test_mmap.py --- a/pypy/module/mmap/test/test_mmap.py +++ b/pypy/module/mmap/test/test_mmap.py @@ -1,5 +1,5 @@ from __future__ import with_statement -from pypy.tool.udir import udir +from rpython.tool.udir import udir import os class AppTestMMap: diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -3,7 +3,7 @@ from __future__ import with_statement from pypy.objspace.std import StdObjSpace -from pypy.tool.udir import udir +from rpython.tool.udir import udir from pypy.tool.pytest.objspace import gettestobjspace from pypy.tool.autopath import pypydir from rpython.rtyper.module.ll_os import RegisterOs diff --git a/pypy/module/posix/test/test_posix_libfile.py b/pypy/module/posix/test/test_posix_libfile.py --- a/pypy/module/posix/test/test_posix_libfile.py +++ b/pypy/module/posix/test/test_posix_libfile.py @@ -1,6 +1,6 @@ import os -from pypy.tool.udir import udir +from rpython.tool.udir import udir def setup_module(mod): diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py --- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py +++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py @@ -4,7 +4,7 @@ import subprocess import py from lib_pypy import disassembler -from pypy.tool.udir import udir +from rpython.tool.udir import udir from pypy.tool import logparser from rpython.jit.tool.jitoutput import parse_prof from pypy.module.pypyjit.test_pypy_c.model import (Log, find_ids_range, diff --git a/pypy/module/sys/state.py b/pypy/module/sys/state.py --- a/pypy/module/sys/state.py +++ b/pypy/module/sys/state.py @@ -57,6 +57,6 @@ def pypy_getudir(space): """NOT_RPYTHON (should be removed from interpleveldefs before translation)""" - from pypy.tool.udir import udir + from rpython.tool.udir import udir return space.wrap(str(udir)) diff --git a/pypy/module/termios/test/test_termios.py b/pypy/module/termios/test/test_termios.py --- a/pypy/module/termios/test/test_termios.py +++ b/pypy/module/termios/test/test_termios.py @@ -2,7 +2,7 @@ import py import sys from pypy.tool.autopath import pypydir -from pypy.tool.udir import udir +from rpython.tool.udir import udir class TestTermios(object): def setup_class(cls): diff --git a/pypy/module/test_lib_pypy/test_msvcrt.py b/pypy/module/test_lib_pypy/test_msvcrt.py --- a/pypy/module/test_lib_pypy/test_msvcrt.py +++ b/pypy/module/test_lib_pypy/test_msvcrt.py @@ -1,4 +1,4 @@ -from pypy.tool.udir import udir +from rpython.tool.udir import udir import py import sys diff --git a/pypy/module/thread/test/test_import_lock.py b/pypy/module/thread/test/test_import_lock.py --- a/pypy/module/thread/test/test_import_lock.py +++ b/pypy/module/thread/test/test_import_lock.py @@ -1,4 +1,4 @@ -from pypy.tool.udir import udir +from rpython.tool.udir import udir from pypy.module.thread.test.support import GenericTestThread diff --git a/pypy/module/zipimport/test/test_zipimport.py b/pypy/module/zipimport/test/test_zipimport.py --- a/pypy/module/zipimport/test/test_zipimport.py +++ b/pypy/module/zipimport/test/test_zipimport.py @@ -6,7 +6,7 @@ from pypy.module.imp.importing import get_pyc_magic, _w_long from StringIO import StringIO -from pypy.tool.udir import udir +from rpython.tool.udir import udir from zipfile import ZIP_STORED, ZIP_DEFLATED diff --git a/pypy/tool/error.py b/pypy/tool/error.py --- a/pypy/tool/error.py +++ b/pypy/tool/error.py @@ -2,7 +2,7 @@ """ error handling features, just a way of displaying errors """ -from pypy.tool.ansi_print import ansi_log +from rpython.tool.ansi_print import ansi_log from rpython.flowspace.model import Variable import sys diff --git a/pypy/tool/pytest/expecttest.py b/pypy/tool/pytest/expecttest.py --- a/pypy/tool/pytest/expecttest.py +++ b/pypy/tool/pytest/expecttest.py @@ -8,7 +8,7 @@ import py import os, sys -from pypy.tool.udir import udir +from rpython.tool.udir import udir from pypy.tool.autopath import pypydir diff --git a/pypy/tool/pytest/test/test_pytestsupport.py b/pypy/tool/pytest/test/test_pytestsupport.py --- a/pypy/tool/pytest/test/test_pytestsupport.py +++ b/pypy/tool/pytest/test/test_pytestsupport.py @@ -5,7 +5,7 @@ from pypy.tool.pytest.appsupport import (AppFrame, build_pytest_assertion, AppExceptionInfo, interpret) import py -from pypy.tool.udir import udir +from rpython.tool.udir import udir import os import sys import pypy diff --git a/pypy/tool/release/make_release.py b/pypy/tool/release/make_release.py --- a/pypy/tool/release/make_release.py +++ b/pypy/tool/release/make_release.py @@ -14,7 +14,7 @@ from xml.dom import minidom import re import py -from pypy.tool.udir import udir +from rpython.tool.udir import udir from pypy.tool.release.package import package import tarfile import os diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -15,7 +15,7 @@ import py import os import fnmatch -from pypy.tool.udir import udir +from rpython.tool.udir import udir if sys.version_info < (2,6): py.test.skip("requires 2.6 so far") diff --git a/pypy/tool/test/test_gcc_cache.py b/pypy/tool/test/test_gcc_cache.py --- a/pypy/tool/test/test_gcc_cache.py +++ b/pypy/tool/test/test_gcc_cache.py @@ -1,6 +1,6 @@ import sys from pypy.tool.gcc_cache import * -from pypy.tool.udir import udir +from rpython.tool.udir import udir import md5, cStringIO from rpython.translator.tool.cbuild import ExternalCompilationInfo diff --git a/pypy/tool/test/test_logparser.py b/pypy/tool/test/test_logparser.py --- a/pypy/tool/test/test_logparser.py +++ b/pypy/tool/test/test_logparser.py @@ -1,4 +1,4 @@ -from pypy.tool.udir import udir +from rpython.tool.udir import udir from pypy.tool.logparser import * diff --git a/pypy/tool/test/test_udir.py b/pypy/tool/test/test_udir.py --- a/pypy/tool/test/test_udir.py +++ b/pypy/tool/test/test_udir.py @@ -1,5 +1,5 @@ -from pypy.tool import udir +from rpython.tool import udir def test_make_udir(): root = str(udir.udir.ensure('make_udir1', dir=1)) diff --git a/pypy/tool/version.py b/pypy/tool/version.py --- a/pypy/tool/version.py +++ b/pypy/tool/version.py @@ -10,7 +10,7 @@ if not err: return - from pypy.tool.ansi_print import ansi_log + from rpython.tool.ansi_print import ansi_log log = py.log.Producer("version") py.log.setconsumer("version", ansi_log) log.WARNING('Errors getting %s information: %s' % (repo_type, err)) diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py --- a/rpython/annotator/annrpython.py +++ b/rpython/annotator/annrpython.py @@ -1,7 +1,7 @@ from __future__ import absolute_import import types -from pypy.tool.ansi_print import ansi_log +from rpython.tool.ansi_print import ansi_log from rpython.tool.pairtype import pair from pypy.tool.error import (format_blocked_annotation_error, AnnotatorError, gather_error, ErrorWrapper) diff --git a/rpython/jit/backend/arm/codebuilder.py b/rpython/jit/backend/arm/codebuilder.py --- a/rpython/jit/backend/arm/codebuilder.py +++ b/rpython/jit/backend/arm/codebuilder.py @@ -6,7 +6,7 @@ from rpython.jit.backend.llsupport.asmmemmgr import BlockBuilderMixin from rpython.rlib.objectmodel import we_are_translated from rpython.rtyper.lltypesystem import lltype, rffi, llmemory -from pypy.tool.udir import udir +from rpython.tool.udir import udir clear_cache = rffi.llexternal( "__clear_cache", diff --git a/rpython/jit/backend/arm/test/gen.py b/rpython/jit/backend/arm/test/gen.py --- a/rpython/jit/backend/arm/test/gen.py +++ b/rpython/jit/backend/arm/test/gen.py @@ -1,5 +1,5 @@ import os -from pypy.tool.udir import udir +from rpython.tool.udir import udir import tempfile from rpython.jit.backend.arm.test.support import AS class ASMInstruction(object): diff --git a/rpython/jit/backend/arm/test/test_zrpy_gc.py b/rpython/jit/backend/arm/test/test_zrpy_gc.py --- a/rpython/jit/backend/arm/test/test_zrpy_gc.py +++ b/rpython/jit/backend/arm/test/test_zrpy_gc.py @@ -12,7 +12,7 @@ from rpython.rlib.jit import JitDriver, dont_look_inside from rpython.rlib.jit import elidable, unroll_safe from rpython.jit.backend.llsupport.gc import GcLLDescr_framework -from pypy.tool.udir import udir +from rpython.tool.udir import udir from pypy.config.translationoption import DEFL_GC from rpython.jit.backend.arm.test.support import skip_unless_run_slow_tests skip_unless_run_slow_tests() diff --git a/rpython/jit/backend/arm/test/test_ztranslation.py b/rpython/jit/backend/arm/test/test_ztranslation.py --- a/rpython/jit/backend/arm/test/test_ztranslation.py +++ b/rpython/jit/backend/arm/test/test_ztranslation.py @@ -1,5 +1,5 @@ import py, os, sys -from pypy.tool.udir import udir +from rpython.tool.udir import udir from rpython.rlib.jit import JitDriver, unroll_parameters, set_param from rpython.rlib.jit import PARAMETERS, dont_look_inside from rpython.rlib.jit import promote diff --git a/rpython/jit/backend/arm/tool/viewcode.py b/rpython/jit/backend/arm/tool/viewcode.py --- a/rpython/jit/backend/arm/tool/viewcode.py +++ b/rpython/jit/backend/arm/tool/viewcode.py @@ -59,7 +59,7 @@ def get_tmp_file(): - # don't use pypy.tool.udir here to avoid removing old usessions which + # don't use rpython.tool.udir here to avoid removing old usessions which # might still contain interesting executables udir = py.path.local.make_numbered_dir(prefix='viewcode-', keep=2) tmpfile = str(udir.join('dump.tmp')) diff --git a/rpython/jit/backend/llgraph/llimpl.py b/rpython/jit/backend/llgraph/llimpl.py --- a/rpython/jit/backend/llgraph/llimpl.py +++ b/rpython/jit/backend/llgraph/llimpl.py @@ -28,7 +28,7 @@ from rpython.rlib.rtimer import read_timestamp import py -from pypy.tool.ansi_print import ansi_log +from rpython.tool.ansi_print import ansi_log log = py.log.Producer('runner') py.log.setconsumer('runner', ansi_log) diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py --- a/rpython/jit/backend/x86/runner.py +++ b/rpython/jit/backend/x86/runner.py @@ -13,7 +13,7 @@ from rpython.jit.backend.x86 import regloc import sys -from pypy.tool.ansi_print import ansi_log +from rpython.tool.ansi_print import ansi_log log = py.log.Producer('jitbackend') py.log.setconsumer('jitbackend', ansi_log) diff --git a/rpython/jit/backend/x86/test/test_runner.py b/rpython/jit/backend/x86/test/test_runner.py --- a/rpython/jit/backend/x86/test/test_runner.py +++ b/rpython/jit/backend/x86/test/test_runner.py @@ -13,7 +13,7 @@ from rpython.jit.metainterp.executor import execute from rpython.jit.backend.test.runner_test import LLtypeBackendTest from rpython.jit.tool.oparser import parse -from pypy.tool.udir import udir +from rpython.tool.udir import udir import ctypes import sys import os diff --git a/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py b/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py --- a/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py +++ b/rpython/jit/backend/x86/test/test_rx86_32_auto_encoding.py @@ -2,7 +2,7 @@ import py from rpython.jit.backend.x86 import rx86 from rpython.rlib.rarithmetic import intmask -from pypy.tool.udir import udir +from rpython.tool.udir import udir INPUTNAME = 'checkfile_%s.s' FILENAME = 'checkfile_%s.o' diff --git a/rpython/jit/backend/x86/test/test_zrpy_gc.py b/rpython/jit/backend/x86/test/test_zrpy_gc.py --- a/rpython/jit/backend/x86/test/test_zrpy_gc.py +++ b/rpython/jit/backend/x86/test/test_zrpy_gc.py @@ -12,7 +12,7 @@ from rpython.rlib.jit import JitDriver, dont_look_inside from rpython.rlib.jit import elidable, unroll_safe from rpython.jit.backend.llsupport.gc import GcLLDescr_framework -from pypy.tool.udir import udir +from rpython.tool.udir import udir from pypy.config.translationoption import DEFL_GC class X(object): diff --git a/rpython/jit/backend/x86/test/test_ztranslation.py b/rpython/jit/backend/x86/test/test_ztranslation.py --- a/rpython/jit/backend/x86/test/test_ztranslation.py +++ b/rpython/jit/backend/x86/test/test_ztranslation.py @@ -1,5 +1,5 @@ import py, os, sys -from pypy.tool.udir import udir +from rpython.tool.udir import udir from rpython.rlib.jit import JitDriver, unroll_parameters, set_param from rpython.rlib.jit import PARAMETERS, dont_look_inside from rpython.rlib.jit import promote diff --git a/rpython/jit/backend/x86/tool/test/test_viewcode.py b/rpython/jit/backend/x86/tool/test/test_viewcode.py --- a/rpython/jit/backend/x86/tool/test/test_viewcode.py +++ b/rpython/jit/backend/x86/tool/test/test_viewcode.py @@ -4,7 +4,7 @@ import os import py import tempfile -from pypy.tool.udir import udir +from rpython.tool.udir import udir def test_format_code_dump_with_labels(): lines = StringIO(""" diff --git a/rpython/jit/backend/x86/tool/viewcode.py b/rpython/jit/backend/x86/tool/viewcode.py --- a/rpython/jit/backend/x86/tool/viewcode.py +++ b/rpython/jit/backend/x86/tool/viewcode.py @@ -17,17 +17,17 @@ import subprocess from bisect import bisect_left -# don't use pypy.tool.udir here to avoid removing old usessions which +# don't use rpython.tool.udir here to avoid removing old usessions which # might still contain interesting executables udir = py.path.local.make_numbered_dir(prefix='viewcode-', keep=2) tmpfile = str(udir.join('dump.tmp')) # hack hack import pypy.tool -mod = new.module('pypy.tool.udir') +mod = new.module('rpython.tool.udir') mod.udir = udir -sys.modules['pypy.tool.udir'] = mod -pypy.tool.udir = mod +sys.modules['rpython.tool.udir'] = mod +rpython.tool.udir = mod # ____________________________________________________________ # Some support code from Psyco. There is more over there, diff --git a/rpython/jit/codewriter/codewriter.py b/rpython/jit/codewriter/codewriter.py --- a/rpython/jit/codewriter/codewriter.py +++ b/rpython/jit/codewriter/codewriter.py @@ -8,7 +8,7 @@ from rpython.jit.codewriter.call import CallControl from rpython.jit.codewriter.policy import log from rpython.flowspace.model import copygraph -from pypy.tool.udir import udir +from rpython.tool.udir import udir class CodeWriter(object): diff --git a/rpython/jit/codewriter/policy.py b/rpython/jit/codewriter/policy.py --- a/rpython/jit/codewriter/policy.py +++ b/rpython/jit/codewriter/policy.py @@ -1,8 +1,8 @@ from rpython.jit.metainterp import history -from pypy.tool.udir import udir +from rpython.tool.udir import udir import py -from pypy.tool.ansi_print import ansi_log +from rpython.tool.ansi_print import ansi_log log = py.log.Producer('jitcodewriter') py.log.setconsumer('jitcodewriter', ansi_log) diff --git a/rpython/rlib/clibffi.py b/rpython/rlib/clibffi.py --- a/rpython/rlib/clibffi.py +++ b/rpython/rlib/clibffi.py @@ -20,7 +20,7 @@ import sys import ctypes.util -from pypy.tool.ansi_print import ansi_log +from rpython.tool.ansi_print import ansi_log log = py.log.Producer("libffi") py.log.setconsumer("libffi", ansi_log) diff --git a/rpython/rlib/rwin32.py b/rpython/rlib/rwin32.py --- a/rpython/rlib/rwin32.py +++ b/rpython/rlib/rwin32.py @@ -3,7 +3,7 @@ """ from rpython.rtyper.tool import rffi_platform -from pypy.tool.udir import udir +from rpython.tool.udir import udir from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.translator.platform import CompilationError from rpython.rtyper.lltypesystem import lltype, rffi diff --git a/rpython/rlib/test/test_clibffi.py b/rpython/rlib/test/test_clibffi.py --- a/rpython/rlib/test/test_clibffi.py +++ b/rpython/rlib/test/test_clibffi.py @@ -264,7 +264,7 @@ def test_struct_by_val(self): from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.translator.platform import platform - from pypy.tool.udir import udir + from rpython.tool.udir import udir c_file = udir.ensure("test_libffi", dir=1).join("xlib.c") c_file.write(py.code.Source(''' @@ -315,7 +315,7 @@ def test_ret_struct_val(self): from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.translator.platform import platform - from pypy.tool.udir import udir + from rpython.tool.udir import udir c_file = udir.ensure("test_libffi", dir=1).join("xlib.c") c_file.write(py.code.Source(''' @@ -391,7 +391,7 @@ def test_cdll_life_time(self): from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.translator.platform import platform - from pypy.tool.udir import udir + from rpython.tool.udir import udir c_file = udir.ensure("test_libffi", dir=1).join("xlib.c") c_file.write(py.code.Source(''' diff --git a/rpython/rlib/test/test_libffi.py b/rpython/rlib/test/test_libffi.py --- a/rpython/rlib/test/test_libffi.py +++ b/rpython/rlib/test/test_libffi.py @@ -202,7 +202,7 @@ @classmethod def setup_class(cls): - from pypy.tool.udir import udir + from rpython.tool.udir import udir from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.translator.tool.cbuild import STANDARD_DEFINES from rpython.translator.platform import platform diff --git a/rpython/rlib/test/test_rmmap.py b/rpython/rlib/test/test_rmmap.py --- a/rpython/rlib/test/test_rmmap.py +++ b/rpython/rlib/test/test_rmmap.py @@ -1,4 +1,4 @@ -from pypy.tool.udir import udir +from rpython.tool.udir import udir import os, sys from rpython.rtyper.test.test_llinterp import interpret from rpython.rlib.rarithmetic import intmask diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py --- a/rpython/rlib/test/test_rposix.py +++ b/rpython/rlib/test/test_rposix.py @@ -1,5 +1,5 @@ from rpython.rtyper.test.test_llinterp import interpret -from pypy.tool.udir import udir +from rpython.tool.udir import udir from rpython.rlib import rposix import os, sys import py diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py --- a/rpython/rlib/test/test_rsocket.py +++ b/rpython/rlib/test/test_rsocket.py @@ -406,7 +406,7 @@ def test_unix_socket_connect(): if getattr(rsocket, 'AF_UNIX', None) is None: py.test.skip('AF_UNIX not supported.') - from pypy.tool.udir import udir + from rpython.tool.udir import udir sockpath = str(udir.join('test_unix_socket_connect')) a = UNIXAddress(sockpath) diff --git a/rpython/rlib/test/test_rwin32.py b/rpython/rlib/test/test_rwin32.py --- a/rpython/rlib/test/test_rwin32.py +++ b/rpython/rlib/test/test_rwin32.py @@ -3,7 +3,7 @@ skip('tests for win32 only') from rpython.rlib import rwin32 -from pypy.tool.udir import udir +from rpython.tool.udir import udir def test_get_osfhandle(): diff --git a/rpython/rlib/test/test_rzipfile.py b/rpython/rlib/test/test_rzipfile.py --- a/rpython/rlib/test/test_rzipfile.py +++ b/rpython/rlib/test/test_rzipfile.py @@ -1,7 +1,7 @@ import py from rpython.rlib.rzipfile import RZipFile -from pypy.tool.udir import udir +from rpython.tool.udir import udir from zipfile import ZIP_STORED, ZIP_DEFLATED, ZipInfo, ZipFile from rpython.rtyper.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin import os diff --git a/rpython/rlib/test/test_streamio.py b/rpython/rlib/test/test_streamio.py --- a/rpython/rlib/test/test_streamio.py +++ b/rpython/rlib/test/test_streamio.py @@ -3,7 +3,7 @@ import os import time import random -from pypy.tool.udir import udir +from rpython.tool.udir import udir from rpython.rlib import streamio diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py --- a/rpython/rtyper/llinterp.py +++ b/rpython/rtyper/llinterp.py @@ -1248,7 +1248,7 @@ # start of a dump file if not self.TRACE: return - from pypy.tool.udir import udir + from rpython.tool.udir import udir n = Tracer.Counter Tracer.Counter += 1 filename = 'llinterp_trace_%d.html' % n @@ -1400,5 +1400,5 @@ # by default we route all logging messages to nothingness # e.g. tests can then switch on logging to get more help # for failing tests -from pypy.tool.ansi_print import ansi_log +from rpython.tool.ansi_print import ansi_log py.log.setconsumer('llinterp', ansi_log) diff --git a/rpython/rtyper/lltypesystem/test/test_ll2ctypes.py b/rpython/rtyper/lltypesystem/test/test_ll2ctypes.py --- a/rpython/rtyper/lltypesystem/test/test_ll2ctypes.py +++ b/rpython/rtyper/lltypesystem/test/test_ll2ctypes.py @@ -12,7 +12,7 @@ from rpython.rtyper.annlowlevel import llhelper from rpython.rlib import rposix from rpython.translator.tool.cbuild import ExternalCompilationInfo -from pypy.tool.udir import udir +from rpython.tool.udir import udir from rpython.rtyper.test.test_llinterp import interpret from rpython.annotator.annrpython import RPythonAnnotator from rpython.rtyper.rtyper import RPythonTyper diff --git a/rpython/rtyper/lltypesystem/test/test_rffi.py b/rpython/rtyper/lltypesystem/test/test_rffi.py --- a/rpython/rtyper/lltypesystem/test/test_rffi.py +++ b/rpython/rtyper/lltypesystem/test/test_rffi.py @@ -8,7 +8,7 @@ from rpython.rtyper.lltypesystem.lltype import Signed, Ptr, Char, malloc from rpython.rtyper.lltypesystem.rstr import STR from rpython.rtyper.lltypesystem import lltype -from pypy.tool.udir import udir +from rpython.tool.udir import udir from rpython.rtyper.test.test_llinterp import interpret from rpython.rtyper.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin from rpython.annotator.annrpython import RPythonAnnotator diff --git a/rpython/rtyper/memory/gc/test/test_env.py b/rpython/rtyper/memory/gc/test/test_env.py --- a/rpython/rtyper/memory/gc/test/test_env.py +++ b/rpython/rtyper/memory/gc/test/test_env.py @@ -1,7 +1,7 @@ import os, py from rpython.rtyper.memory.gc import env from rpython.rlib.rarithmetic import r_uint -from pypy.tool.udir import udir +from rpython.tool.udir import udir class FakeEnviron: diff --git a/rpython/rtyper/memory/gc/test/test_inspector.py b/rpython/rtyper/memory/gc/test/test_inspector.py --- a/rpython/rtyper/memory/gc/test/test_inspector.py +++ b/rpython/rtyper/memory/gc/test/test_inspector.py @@ -1,5 +1,5 @@ import os -from pypy.tool.udir import udir +from rpython.tool.udir import udir from rpython.rtyper.memory.gc.test.test_direct import BaseDirectGCTest, S from rpython.rtyper.memory.gc import inspector from rpython.rtyper.lltypesystem import llmemory diff --git a/rpython/rtyper/memory/gctransform/framework.py b/rpython/rtyper/memory/gctransform/framework.py --- a/rpython/rtyper/memory/gctransform/framework.py +++ b/rpython/rtyper/memory/gctransform/framework.py @@ -587,7 +587,7 @@ def write_typeid_list(self): """write out the list of type ids together with some info""" - from pypy.tool.udir import udir + from rpython.tool.udir import udir # XXX not ideal since it is not per compilation, but per run # XXX argh argh, this only gives the member index but not the # real typeid, which is a complete mess to obtain now... diff --git a/rpython/rtyper/memory/gctransform/log.py b/rpython/rtyper/memory/gctransform/log.py --- a/rpython/rtyper/memory/gctransform/log.py +++ b/rpython/rtyper/memory/gctransform/log.py @@ -1,4 +1,4 @@ import py -from pypy.tool.ansi_print import ansi_log +from rpython.tool.ansi_print import ansi_log log = py.log.Producer("gctransform") py.log.setconsumer("gctransform", ansi_log) diff --git a/rpython/rtyper/memory/test/snippet.py b/rpython/rtyper/memory/test/snippet.py --- a/rpython/rtyper/memory/test/snippet.py +++ b/rpython/rtyper/memory/test/snippet.py @@ -1,5 +1,5 @@ import os, py -from pypy.tool.udir import udir +from rpython.tool.udir import udir from rpython.rtyper.lltypesystem import lltype from rpython.rtyper.lltypesystem.lloperation import llop diff --git a/rpython/rtyper/module/test/test_ll_os.py b/rpython/rtyper/module/test/test_ll_os.py --- a/rpython/rtyper/module/test/test_ll_os.py +++ b/rpython/rtyper/module/test/test_ll_os.py @@ -2,7 +2,7 @@ from py.path import local import pypy -from pypy.tool.udir import udir +from rpython.tool.udir import udir from rpython.translator.c.test.test_genc import compile from rpython.rtyper.module import ll_os #has side effect of registering functions diff --git a/rpython/rtyper/module/test/test_ll_os_path.py b/rpython/rtyper/module/test/test_ll_os_path.py --- a/rpython/rtyper/module/test/test_ll_os_path.py +++ b/rpython/rtyper/module/test/test_ll_os_path.py @@ -8,7 +8,7 @@ from rpython.rtyper.lltypesystem.module.ll_os_path import Implementation as impl from rpython.rtyper.module.support import ll_strcpy from rpython.rtyper.test.test_llinterp import interpret -from pypy.tool.udir import udir +from rpython.tool.udir import udir def test_exists(): filename = impl.to_rstr(str(py.path.local(__file__))) diff --git a/rpython/rtyper/module/test/test_ll_termios.py b/rpython/rtyper/module/test/test_ll_termios.py --- a/rpython/rtyper/module/test/test_ll_termios.py +++ b/rpython/rtyper/module/test/test_ll_termios.py @@ -1,5 +1,5 @@ import py, re, sys -from pypy.tool.udir import udir +from rpython.tool.udir import udir # tests here are run as snippets through a pexpected python subprocess def setup_module(mod): diff --git a/rpython/rtyper/module/test/test_posix.py b/rpython/rtyper/module/test/test_posix.py --- a/rpython/rtyper/module/test/test_posix.py +++ b/rpython/rtyper/module/test/test_posix.py @@ -1,6 +1,6 @@ import py from rpython.rtyper.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin -from pypy.tool.udir import udir +from rpython.tool.udir import udir from rpython.rlib.rarithmetic import is_valid_int import os diff --git a/rpython/rtyper/rmodel.py b/rpython/rtyper/rmodel.py --- a/rpython/rtyper/rmodel.py +++ b/rpython/rtyper/rmodel.py @@ -480,7 +480,7 @@ # logging/warning import py -from pypy.tool.ansi_print import ansi_log +from rpython.tool.ansi_print import ansi_log log = py.log.Producer("rtyper") py.log.setconsumer("rtyper", ansi_log) diff --git a/rpython/rtyper/test/test_rbuiltin.py b/rpython/rtyper/test/test_rbuiltin.py --- a/rpython/rtyper/test/test_rbuiltin.py +++ b/rpython/rtyper/test/test_rbuiltin.py @@ -12,7 +12,7 @@ from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rtyper.test import test_llinterp from rpython.rtyper.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin -from pypy.tool import udir +from rpython.tool import udir from rpython.translator.translator import graphof diff --git a/rpython/rtyper/tool/rffi_platform.py b/rpython/rtyper/tool/rffi_platform.py --- a/rpython/rtyper/tool/rffi_platform.py +++ b/rpython/rtyper/tool/rffi_platform.py @@ -10,7 +10,7 @@ from pypy.tool.gcc_cache import build_executable_cache, try_compile_cache from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.translator.platform import CompilationError -from pypy.tool.udir import udir +from rpython.tool.udir import udir from pypy.tool.autopath import pypydir from rpython.rlib.rarithmetic import r_uint, r_longlong, r_ulonglong, intmask diff --git a/rpython/rtyper/tool/rfficache.py b/rpython/rtyper/tool/rfficache.py --- a/rpython/rtyper/tool/rfficache.py +++ b/rpython/rtyper/tool/rfficache.py @@ -5,7 +5,7 @@ import py import os from rpython.translator.tool.cbuild import ExternalCompilationInfo -from pypy.tool.udir import udir +from rpython.tool.udir import udir from rpython.rlib import rarithmetic from rpython.rtyper.lltypesystem import lltype from pypy.tool.gcc_cache import build_executable_cache diff --git a/rpython/rtyper/tool/test/test_mkrffi.py b/rpython/rtyper/tool/test/test_mkrffi.py --- a/rpython/rtyper/tool/test/test_mkrffi.py +++ b/rpython/rtyper/tool/test/test_mkrffi.py @@ -56,7 +56,7 @@ class TestMkrffi(object): def setup_class(cls): import ctypes - from pypy.tool.udir import udir + from rpython.tool.udir import udir from rpython.translator.platform import platform from rpython.translator.tool.cbuild import ExternalCompilationInfo diff --git a/rpython/rtyper/tool/test/test_rffi_platform.py b/rpython/rtyper/tool/test/test_rffi_platform.py --- a/rpython/rtyper/tool/test/test_rffi_platform.py +++ b/rpython/rtyper/tool/test/test_rffi_platform.py @@ -2,7 +2,7 @@ from rpython.rtyper.tool import rffi_platform from rpython.rtyper.lltypesystem import lltype from rpython.rtyper.lltypesystem import rffi -from pypy.tool.udir import udir +from rpython.tool.udir import udir from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.translator.platform import platform from rpython.rlib.rarithmetic import r_uint, r_longlong, r_ulonglong diff --git a/rpython/rtyper/tool/test/test_rfficache.py b/rpython/rtyper/tool/test/test_rfficache.py --- a/rpython/rtyper/tool/test/test_rfficache.py +++ b/rpython/rtyper/tool/test/test_rfficache.py @@ -1,7 +1,7 @@ from rpython.rtyper.tool.rfficache import * from rpython.rtyper.lltypesystem import rffi -from pypy.tool.udir import udir +from rpython.tool.udir import udir def test_sizeof_c_type(): sizeofchar = sizeof_c_type('char') diff --git a/pypy/tool/ansi_mandelbrot.py b/rpython/tool/ansi_mandelbrot.py rename from pypy/tool/ansi_mandelbrot.py rename to rpython/tool/ansi_mandelbrot.py diff --git a/pypy/tool/ansi_print.py b/rpython/tool/ansi_print.py rename from pypy/tool/ansi_print.py rename to rpython/tool/ansi_print.py --- a/pypy/tool/ansi_print.py +++ b/rpython/tool/ansi_print.py @@ -4,7 +4,7 @@ import sys from py.io import ansi_print -from pypy.tool.ansi_mandelbrot import Driver +from rpython.tool.ansi_mandelbrot import Driver class AnsiLog: wrote_dot = False # XXX sharing state with all instances diff --git a/pypy/tool/ansicolor.py b/rpython/tool/ansicolor.py rename from pypy/tool/ansicolor.py rename to rpython/tool/ansicolor.py diff --git a/pypy/tool/udir.py b/rpython/tool/udir.py rename from pypy/tool/udir.py rename to rpython/tool/udir.py diff --git a/rpython/translator/backendopt/canraise.py b/rpython/translator/backendopt/canraise.py --- a/rpython/translator/backendopt/canraise.py +++ b/rpython/translator/backendopt/canraise.py @@ -1,7 +1,7 @@ import py from rpython.rtyper.lltypesystem.lloperation import LL_OPERATIONS -from pypy.tool.ansi_print import ansi_log +from rpython.tool.ansi_print import ansi_log from rpython.translator.backendopt import graphanalyze from rpython.translator.simplify import get_funcobj diff --git a/rpython/translator/backendopt/support.py b/rpython/translator/backendopt/support.py --- a/rpython/translator/backendopt/support.py +++ b/rpython/translator/backendopt/support.py @@ -2,7 +2,7 @@ from rpython.rtyper.lltypesystem import lltype from rpython.rtyper.rmodel import inputconst -from pypy.tool.ansi_print import ansi_log +from rpython.tool.ansi_print import ansi_log from rpython.translator.simplify import get_graph diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py --- a/rpython/translator/c/genc.py +++ b/rpython/translator/c/genc.py @@ -6,7 +6,7 @@ from rpython.rtyper.typesystem import getfunctionptr from pypy.tool import runsubprocess from pypy.tool.nullpath import NullPyPathLocal -from pypy.tool.udir import udir +from rpython.tool.udir import udir from rpython.translator.c import gc from rpython.translator.c.database import LowLevelDatabase from rpython.translator.c.extfunc import pre_include_code_lines diff --git a/rpython/translator/c/support.py b/rpython/translator/c/support.py --- a/rpython/translator/c/support.py +++ b/rpython/translator/c/support.py @@ -165,6 +165,6 @@ # logging import py -from pypy.tool.ansi_print import ansi_log +from rpython.tool.ansi_print import ansi_log log = py.log.Producer("c") py.log.setconsumer("c", ansi_log) diff --git a/rpython/translator/c/test/test_extfunc.py b/rpython/translator/c/test/test_extfunc.py --- a/rpython/translator/c/test/test_extfunc.py +++ b/rpython/translator/c/test/test_extfunc.py @@ -1,7 +1,7 @@ import autopath import py import os, time, sys -from pypy.tool.udir import udir +from rpython.tool.udir import udir from rpython.rlib.rarithmetic import r_longlong from rpython.annotator import model as annmodel from rpython.translator.c.test.test_genc import compile diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py --- a/rpython/translator/c/test/test_newgc.py +++ b/rpython/translator/c/test/test_newgc.py @@ -12,7 +12,7 @@ from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.rtyper.lltypesystem.lloperation import llop from rpython.rtyper.memory.test import snippet -from pypy.tool.udir import udir +from rpython.tool.udir import udir from rpython.translator.interactive import Translation diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py --- a/rpython/translator/c/test/test_standalone.py +++ b/rpython/translator/c/test/test_standalone.py @@ -9,7 +9,7 @@ from rpython.translator.backendopt import all from rpython.translator.c.genc import CStandaloneBuilder, ExternalCompilationInfo from rpython.annotator.listdef import s_list_of_strings -from pypy.tool.udir import udir +from rpython.tool.udir import udir from pypy.tool.autopath import pypydir from pypy.conftest import option diff --git a/rpython/translator/cli/cts.py b/rpython/translator/cli/cts.py --- a/rpython/translator/cli/cts.py +++ b/rpython/translator/cli/cts.py @@ -11,7 +11,7 @@ from rpython.translator.cli.option import getoption from rpython.translator.cli import oopspec -from pypy.tool.ansi_print import ansi_log +from rpython.tool.ansi_print import ansi_log import py log = py.log.Producer("cli") py.log.setconsumer("cli", ansi_log) diff --git a/rpython/translator/cli/query.py b/rpython/translator/cli/query.py --- a/rpython/translator/cli/query.py +++ b/rpython/translator/cli/query.py @@ -3,7 +3,7 @@ import os.path import py import subprocess -from pypy.tool.udir import udir +from rpython.tool.udir import udir from rpython.rtyper.ootypesystem import ootype from rpython.translator.cli.rte import Query from rpython.translator.cli.sdk import SDK diff --git a/rpython/translator/cli/rte.py b/rpython/translator/cli/rte.py --- a/rpython/translator/cli/rte.py +++ b/rpython/translator/cli/rte.py @@ -9,7 +9,7 @@ import py import subprocess from rpython.translator.cli.sdk import SDK -from pypy.tool.ansi_print import ansi_log +from rpython.tool.ansi_print import ansi_log log = py.log.Producer("cli") py.log.setconsumer("cli", ansi_log) diff --git a/rpython/translator/cli/support.py b/rpython/translator/cli/support.py --- a/rpython/translator/cli/support.py +++ b/rpython/translator/cli/support.py @@ -3,7 +3,7 @@ from rpython.rtyper.ootypesystem import ootype from rpython.translator.cli.rte import Support -from pypy.tool.ansi_print import ansi_log +from rpython.tool.ansi_print import ansi_log log = py.log.Producer("cli") py.log.setconsumer("cli", ansi_log) diff --git a/rpython/translator/cli/test/runtest.py b/rpython/translator/cli/test/runtest.py --- a/rpython/translator/cli/test/runtest.py +++ b/rpython/translator/cli/test/runtest.py @@ -3,7 +3,7 @@ import py import subprocess -from pypy.tool.udir import udir +from rpython.tool.udir import udir from rpython.translator.translator import TranslationContext from rpython.rtyper.test.tool import BaseRtypingTest, OORtypeMixin from rpython.rtyper.lltypesystem.lltype import typeOf diff --git a/rpython/translator/driver.py b/rpython/translator/driver.py --- a/rpython/translator/driver.py +++ b/rpython/translator/driver.py @@ -8,12 +8,12 @@ from rpython.translator.goal.timing import Timer from rpython.annotator.listdef import s_list_of_strings from rpython.annotator import policy as annpolicy -from pypy.tool.udir import udir +from rpython.tool.udir import udir from pypy.tool.debug_print import debug_start, debug_print, debug_stop from rpython.rlib.entrypoint import secondary_entrypoints import py -from pypy.tool.ansi_print import ansi_log +from rpython.tool.ansi_print import ansi_log log = py.log.Producer("translation") py.log.setconsumer("translation", ansi_log) diff --git a/rpython/translator/goal/old_queries.py b/rpython/translator/goal/old_queries.py --- a/rpython/translator/goal/old_queries.py +++ b/rpython/translator/goal/old_queries.py @@ -464,7 +464,7 @@ # def worstblocks_topten(t, n=10): - from pypy.tool.ansi_print import ansi_print + from rpython.tool.ansi_print import ansi_print ann = t.annotator h = [(count, block) for block, count in ann.reflowcounter.iteritems()] h.sort() diff --git a/rpython/translator/goal/test2/test_app_main.py b/rpython/translator/goal/test2/test_app_main.py --- a/rpython/translator/goal/test2/test_app_main.py +++ b/rpython/translator/goal/test2/test_app_main.py @@ -5,7 +5,7 @@ import py import sys, os, re, runpy, subprocess import autopath -from pypy.tool.udir import udir +from rpython.tool.udir import udir from contextlib import contextmanager banner = sys.version.splitlines()[0] diff --git a/rpython/translator/goal/timing.py b/rpython/translator/goal/timing.py --- a/rpython/translator/goal/timing.py +++ b/rpython/translator/goal/timing.py @@ -5,7 +5,7 @@ import time import py -from pypy.tool.ansi_print import ansi_log +from rpython.tool.ansi_print import ansi_log log = py.log.Producer("Timer") py.log.setconsumer("Timer", ansi_log) diff --git a/rpython/translator/goal/translate.py b/rpython/translator/goal/translate.py --- a/rpython/translator/goal/translate.py +++ b/rpython/translator/goal/translate.py @@ -85,7 +85,7 @@ ]) import optparse -from pypy.tool.ansi_print import ansi_log +from rpython.tool.ansi_print import ansi_log log = py.log.Producer("translation") py.log.setconsumer("translation", ansi_log) diff --git a/rpython/translator/jvm/genjvm.py b/rpython/translator/jvm/genjvm.py --- a/rpython/translator/jvm/genjvm.py +++ b/rpython/translator/jvm/genjvm.py @@ -10,7 +10,7 @@ import py from rpython.rtyper.ootypesystem import ootype -from pypy.tool.udir import udir +from rpython.tool.udir import udir from rpython.translator.backendopt.all import backend_optimizations from rpython.translator.backendopt.checkvirtual import check_virtual_methods from rpython.translator.oosupport.genoo import GenOO diff --git a/rpython/translator/jvm/log.py b/rpython/translator/jvm/log.py --- a/rpython/translator/jvm/log.py +++ b/rpython/translator/jvm/log.py @@ -4,7 +4,7 @@ from rpython.translator.jvm.log import log """ -from pypy.tool.ansi_print import ansi_log +from rpython.tool.ansi_print import ansi_log import py log = py.log.Producer("jvm") py.log.setconsumer("jvm", ansi_log) diff --git a/rpython/translator/jvm/test/runtest.py b/rpython/translator/jvm/test/runtest.py --- a/rpython/translator/jvm/test/runtest.py +++ b/rpython/translator/jvm/test/runtest.py @@ -3,7 +3,7 @@ import py import subprocess -from pypy.tool.udir import udir +from rpython.tool.udir import udir from rpython.rtyper.test.tool import BaseRtypingTest, OORtypeMixin from rpython.rtyper.lltypesystem.lltype import typeOf from rpython.rtyper.ootypesystem import ootype diff --git a/rpython/translator/oosupport/function.py b/rpython/translator/oosupport/function.py --- a/rpython/translator/oosupport/function.py +++ b/rpython/translator/oosupport/function.py @@ -1,5 +1,5 @@ import py -from pypy.tool.ansi_print import ansi_log +from rpython.tool.ansi_print import ansi_log log = py.log.Producer("oosupport") py.log.setconsumer("oosupport", ansi_log) diff --git a/rpython/translator/oosupport/test_template/builtin.py b/rpython/translator/oosupport/test_template/builtin.py --- a/rpython/translator/oosupport/test_template/builtin.py +++ b/rpython/translator/oosupport/test_template/builtin.py @@ -2,7 +2,7 @@ import errno import stat from py.builtin import sorted -from pypy.tool import udir +from rpython.tool import udir from rpython.rtyper.test.test_rbuiltin import BaseTestRbuiltin from rpython.rtyper.module.test.test_ll_time import BaseTestTime as llBaseTestTime diff --git a/rpython/translator/platform/__init__.py b/rpython/translator/platform/__init__.py --- a/rpython/translator/platform/__init__.py +++ b/rpython/translator/platform/__init__.py @@ -2,9 +2,9 @@ import py, os, sys -from pypy.tool.ansi_print import ansi_log +from rpython.tool.ansi_print import ansi_log from pypy.tool.runsubprocess import run_subprocess as _run_subprocess -from pypy.tool.udir import udir +from rpython.tool.udir import udir log = py.log.Producer("platform") diff --git a/rpython/translator/platform/arm.py b/rpython/translator/platform/arm.py --- a/rpython/translator/platform/arm.py +++ b/rpython/translator/platform/arm.py @@ -2,7 +2,7 @@ from rpython.translator.platform.linux import Linux from rpython.translator.platform.posix import _run_subprocess, GnuMakefile from rpython.translator.platform import ExecutionResult, log -from pypy.tool.udir import udir +from rpython.tool.udir import udir from pypy.tool import autopath from os import getenv diff --git a/rpython/translator/platform/maemo.py b/rpython/translator/platform/maemo.py --- a/rpython/translator/platform/maemo.py +++ b/rpython/translator/platform/maemo.py @@ -2,7 +2,7 @@ import py, os -from pypy.tool.udir import udir +from rpython.tool.udir import udir from rpython.translator.platform import ExecutionResult, log from rpython.translator.platform.linux import Linux from rpython.translator.platform.posix import GnuMakefile, _run_subprocess diff --git a/rpython/translator/platform/test/test_darwin.py b/rpython/translator/platform/test/test_darwin.py --- a/rpython/translator/platform/test/test_darwin.py +++ b/rpython/translator/platform/test/test_darwin.py @@ -6,7 +6,7 @@ if sys.platform != 'darwin': py.test.skip("Darwin only") -from pypy.tool.udir import udir +from rpython.tool.udir import udir from rpython.translator.platform.darwin import Darwin_i386, Darwin_x86_64, Darwin_PowerPC from rpython.translator.platform.test.test_platform import TestPlatform as BasicTest from rpython.translator.tool.cbuild import ExternalCompilationInfo diff --git a/rpython/translator/platform/test/test_maemo.py b/rpython/translator/platform/test/test_maemo.py --- a/rpython/translator/platform/test/test_maemo.py +++ b/rpython/translator/platform/test/test_maemo.py @@ -3,7 +3,7 @@ """ import py -from pypy.tool.udir import udir +from rpython.tool.udir import udir from rpython.translator.platform.maemo import Maemo, check_scratchbox from rpython.translator.platform.test.test_platform import TestPlatform as BasicTest from rpython.translator.tool.cbuild import ExternalCompilationInfo diff --git a/rpython/translator/platform/test/test_platform.py b/rpython/translator/platform/test/test_platform.py --- a/rpython/translator/platform/test/test_platform.py +++ b/rpython/translator/platform/test/test_platform.py @@ -1,6 +1,6 @@ import py, sys, ctypes, os -from pypy.tool.udir import udir +from rpython.tool.udir import udir from rpython.translator.platform import CompilationError, Platform from rpython.translator.platform import host from rpython.translator.tool.cbuild import ExternalCompilationInfo diff --git a/rpython/translator/platform/test/test_posix.py b/rpython/translator/platform/test/test_posix.py --- a/rpython/translator/platform/test/test_posix.py +++ b/rpython/translator/platform/test/test_posix.py @@ -1,7 +1,7 @@ from rpython.translator.platform import host, CompilationError from rpython.translator.tool.cbuild import ExternalCompilationInfo -from pypy.tool.udir import udir +from rpython.tool.udir import udir from StringIO import StringIO import sys, os diff --git a/rpython/translator/sandbox/rsandbox.py b/rpython/translator/sandbox/rsandbox.py --- a/rpython/translator/sandbox/rsandbox.py +++ b/rpython/translator/sandbox/rsandbox.py @@ -16,7 +16,7 @@ from rpython.rlib.objectmodel import CDefinedIntSymbolic from rpython.tool.sourcetools import func_with_new_name from rpython.rtyper.annlowlevel import MixLevelHelperAnnotator -from pypy.tool.ansi_print import ansi_log +from rpython.tool.ansi_print import ansi_log import py log = py.log.Producer("sandbox") py.log.setconsumer("sandbox", ansi_log) diff --git a/rpython/translator/sandbox/sandlib.py b/rpython/translator/sandbox/sandlib.py --- a/rpython/translator/sandbox/sandlib.py +++ b/rpython/translator/sandbox/sandlib.py @@ -13,7 +13,7 @@ def create_log(): """Make and return a log for the sandbox to use, if needed.""" # These imports are local to avoid importing pypy if we don't need to. - from pypy.tool.ansi_print import AnsiLog + from rpython.tool.ansi_print import AnsiLog class MyAnsiLog(AnsiLog): KW_TO_COLOR = { diff --git a/rpython/translator/sandbox/test/test_vfs.py b/rpython/translator/sandbox/test/test_vfs.py --- a/rpython/translator/sandbox/test/test_vfs.py +++ b/rpython/translator/sandbox/test/test_vfs.py @@ -1,7 +1,7 @@ import py import sys, stat, os from rpython.translator.sandbox.vfs import * -from pypy.tool.udir import udir +from rpython.tool.udir import udir HASLINK = hasattr(os, 'symlink') diff --git a/rpython/translator/test/test_unsimplify.py b/rpython/translator/test/test_unsimplify.py --- a/rpython/translator/test/test_unsimplify.py +++ b/rpython/translator/test/test_unsimplify.py @@ -5,7 +5,7 @@ from rpython.rtyper.llinterp import LLInterpreter from rpython.flowspace.model import checkgraph from rpython.rlib.objectmodel import we_are_translated -from pypy.tool.udir import udir +from rpython.tool.udir import udir def translate(func, argtypes, type_system="lltype"): t = TranslationContext() diff --git a/rpython/translator/tool/benchmark.py b/rpython/translator/tool/benchmark.py --- a/rpython/translator/tool/benchmark.py +++ b/rpython/translator/tool/benchmark.py @@ -1,6 +1,6 @@ import autopath from pypy.tool import testit -from pypy.tool.udir import udir +from rpython.tool.udir import udir from rpython.translator.tool.cbuild import build_cfunc from rpython.translator.test.test_cltrans import global_cl, make_cl_func diff --git a/rpython/translator/tool/cbuild.py b/rpython/translator/tool/cbuild.py --- a/rpython/translator/tool/cbuild.py +++ b/rpython/translator/tool/cbuild.py @@ -3,7 +3,7 @@ from pypy.tool.autopath import pypydir from rpython.translator.platform import host -from pypy.tool.udir import udir +from rpython.tool.udir import udir class ExternalCompilationInfo(object): diff --git a/rpython/translator/tool/graphpage.py b/rpython/translator/tool/graphpage.py --- a/rpython/translator/tool/graphpage.py +++ b/rpython/translator/tool/graphpage.py @@ -6,7 +6,7 @@ from rpython.annotator.description import MethodDesc from rpython.annotator.classdef import ClassDef from pypy.tool.uid import uid -from pypy.tool.udir import udir +from rpython.tool.udir import udir from dotviewer.graphpage import GraphPage as BaseGraphPage diff --git a/rpython/translator/tool/make_dot.py b/rpython/translator/tool/make_dot.py --- a/rpython/translator/tool/make_dot.py +++ b/rpython/translator/tool/make_dot.py @@ -2,7 +2,7 @@ import inspect, linecache from rpython.flowspace.model import * from rpython.flowspace.objspace import FlowObjSpace as Space -from pypy.tool.udir import udir +from rpython.tool.udir import udir from py.process import cmdexec from pypy.interpreter.pytraceback import offset2lineno diff --git a/rpython/translator/tool/staticsizereport.py b/rpython/translator/tool/staticsizereport.py --- a/rpython/translator/tool/staticsizereport.py +++ b/rpython/translator/tool/staticsizereport.py @@ -1,7 +1,7 @@ from __future__ import division import cPickle as pickle -from pypy.tool.ansicolor import red, yellow, green +from rpython.tool.ansicolor import red, yellow, green from rpython.rtyper.lltypesystem.lltype import typeOf, _ptr, Ptr, ContainerType from rpython.rtyper.lltypesystem import llmemory from rpython.rtyper.memory.lltypelayout import convert_offset_to_int diff --git a/rpython/translator/tool/test/test_cbuild.py b/rpython/translator/tool/test/test_cbuild.py --- a/rpython/translator/tool/test/test_cbuild.py +++ b/rpython/translator/tool/test/test_cbuild.py @@ -1,6 +1,6 @@ import py -from pypy.tool.udir import udir +from rpython.tool.udir import udir from rpython.translator.tool.cbuild import ExternalCompilationInfo from subprocess import Popen, PIPE, STDOUT diff --git a/rpython/translator/translator.py b/rpython/translator/translator.py --- a/rpython/translator/translator.py +++ b/rpython/translator/translator.py @@ -9,7 +9,7 @@ from rpython.translator import simplify from rpython.flowspace.model import FunctionGraph, checkgraph, Block from rpython.flowspace.objspace import FlowObjSpace -from pypy.tool.ansi_print import ansi_log +from rpython.tool.ansi_print import ansi_log from rpython.tool.sourcetools import nice_repr_for_func from pypy.config.pypyoption import pypy_optiondescription from pypy.config.translationoption import get_combined_translation_config From noreply at buildbot.pypy.org Fri Jan 4 22:30:51 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Fri, 4 Jan 2013 22:30:51 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Created rpython/__init__.py Message-ID: <20130104213051.AB9F61C0DD9@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59696:8ba366364abe Date: 2012-12-31 01:08 +0100 http://bitbucket.org/pypy/pypy/changeset/8ba366364abe/ Log: Created rpython/__init__.py diff --git a/rpython/__init__.py b/rpython/__init__.py new file mode 100644 From noreply at buildbot.pypy.org Fri Jan 4 22:30:57 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Fri, 4 Jan 2013 22:30:57 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Moved pypy.tool.identity_dict and pypy.tool.uid to rpython Message-ID: <20130104213057.B468B1C01EE@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59700:24e99ed7dda1 Date: 2013-01-02 03:04 +0100 http://bitbucket.org/pypy/pypy/changeset/24e99ed7dda1/ Log: Moved pypy.tool.identity_dict and pypy.tool.uid to rpython diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -7,7 +7,7 @@ from pypy.interpreter.argument import Arguments from pypy.interpreter.miscutils import ThreadLocals from pypy.tool.cache import Cache -from pypy.tool.uid import HUGEVAL_BYTES +from rpython.tool.uid import HUGEVAL_BYTES from rpython.rlib import jit from rpython.rlib.debug import make_sure_not_resized from rpython.rlib.objectmodel import we_are_translated, newlist_hint,\ diff --git a/pypy/interpreter/nestedscope.py b/pypy/interpreter/nestedscope.py --- a/pypy/interpreter/nestedscope.py +++ b/pypy/interpreter/nestedscope.py @@ -4,7 +4,7 @@ from pypy.interpreter.mixedmodule import MixedModule from pypy.interpreter.astcompiler import consts from rpython.rlib import jit -from pypy.tool.uid import uid +from rpython.tool.uid import uid class Cell(Wrappable): "A simple container for a wrapped value." diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -16,7 +16,7 @@ from pypy.module.cpyext.pyobject import RefcountState from pypy.module.cpyext.pyobject import Py_DecRef, InvalidPointerException from rpython.translator.goal import autopath -from pypy.tool.identity_dict import identity_dict +from rpython.tool.identity_dict import identity_dict from pypy.tool import leakfinder @api.cpython_api([], api.PyObject) diff --git a/pypy/tool/algo/graphlib.py b/pypy/tool/algo/graphlib.py --- a/pypy/tool/algo/graphlib.py +++ b/pypy/tool/algo/graphlib.py @@ -6,7 +6,7 @@ 'edges' is a dict mapping vertices to a list of edges with its source. Note that we can usually use 'edges' as the set of 'vertices' too. """ -from pypy.tool.identity_dict import identity_dict +from rpython.tool.identity_dict import identity_dict class Edge: diff --git a/pypy/tool/test/test_identitydict.py b/pypy/tool/test/test_identitydict.py --- a/pypy/tool/test/test_identitydict.py +++ b/pypy/tool/test/test_identitydict.py @@ -1,5 +1,5 @@ import py -from pypy.tool.identity_dict import identity_dict, IdentityDictPurePython +from rpython.tool.identity_dict import identity_dict, IdentityDictPurePython class TestIdentityDictNative: identity_dict = identity_dict diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py --- a/rpython/annotator/bookkeeper.py +++ b/rpython/annotator/bookkeeper.py @@ -25,7 +25,7 @@ from rpython.rtyper.lltypesystem import lltype, llmemory from rpython.rtyper.ootypesystem import ootype from rpython.rtyper import extregistry -from pypy.tool.identity_dict import identity_dict +from rpython.tool.identity_dict import identity_dict class Stats(object): diff --git a/rpython/annotator/specialize.py b/rpython/annotator/specialize.py --- a/rpython/annotator/specialize.py +++ b/rpython/annotator/specialize.py @@ -1,6 +1,6 @@ # specialization support import py -from pypy.tool.uid import uid +from rpython.tool.uid import uid from rpython.tool.sourcetools import func_with_new_name from pypy.tool.algo.unionfind import UnionFind from rpython.flowspace.model import Block, Link, Variable, SpaceOperation diff --git a/rpython/flowspace/model.py b/rpython/flowspace/model.py --- a/rpython/flowspace/model.py +++ b/rpython/flowspace/model.py @@ -4,7 +4,7 @@ # the below object/attribute model evolved from # a discussion in Berlin, 4th of october 2003 import py -from pypy.tool.uid import uid, Hashable +from rpython.tool.uid import uid, Hashable from rpython.tool.sourcetools import PY_IDENTIFIER, nice_repr_for_func from rpython.rlib.rarithmetic import is_valid_int, r_longlong, r_ulonglong, r_uint diff --git a/rpython/rlib/unroll.py b/rpython/rlib/unroll.py --- a/rpython/rlib/unroll.py +++ b/rpython/rlib/unroll.py @@ -1,4 +1,4 @@ -from pypy.tool.uid import uid +from rpython.tool.uid import uid # Support for explicit specialization: in code using global constants # that are instances of SpecTag, code paths are not merged when diff --git a/rpython/rtyper/extregistry.py b/rpython/rtyper/extregistry.py --- a/rpython/rtyper/extregistry.py +++ b/rpython/rtyper/extregistry.py @@ -1,6 +1,6 @@ import weakref import UserDict -from pypy.tool.uid import Hashable +from rpython.tool.uid import Hashable class AutoRegisteringType(type): diff --git a/rpython/rtyper/lltypesystem/ll2ctypes.py b/rpython/rtyper/lltypesystem/ll2ctypes.py --- a/rpython/rtyper/lltypesystem/ll2ctypes.py +++ b/rpython/rtyper/lltypesystem/ll2ctypes.py @@ -19,7 +19,7 @@ from rpython.rtyper.lltypesystem import lltype, llmemory from rpython.rtyper.extfunc import ExtRegistryEntry from rpython.rlib.objectmodel import Symbolic, ComputedIntSymbolic -from pypy.tool.uid import fixid +from rpython.tool.uid import fixid from rpython.rlib.rarithmetic import r_singlefloat, r_longfloat, base_int, intmask from rpython.rlib.rarithmetic import is_emulated_long, maxint from rpython.annotator import model as annmodel diff --git a/rpython/rtyper/lltypesystem/llmemory.py b/rpython/rtyper/lltypesystem/llmemory.py --- a/rpython/rtyper/lltypesystem/llmemory.py +++ b/rpython/rtyper/lltypesystem/llmemory.py @@ -7,7 +7,7 @@ import weakref from rpython.rlib.objectmodel import Symbolic from rpython.rtyper.lltypesystem import lltype -from pypy.tool.uid import uid +from rpython.tool.uid import uid from rpython.rlib.rarithmetic import is_valid_int diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py --- a/rpython/rtyper/lltypesystem/lltype.py +++ b/rpython/rtyper/lltypesystem/lltype.py @@ -2,7 +2,7 @@ r_ulonglong, r_longlong, r_longfloat, r_longlonglong, base_int, normalizedinttype, longlongmask, longlonglongmask) from rpython.rlib.objectmodel import Symbolic -from pypy.tool.identity_dict import identity_dict +from rpython.tool.identity_dict import identity_dict from pypy.tool import leakfinder from types import NoneType from rpython.rlib.rarithmetic import maxint, is_valid_int, is_emulated_long diff --git a/rpython/rtyper/lltypesystem/rclass.py b/rpython/rtyper/lltypesystem/rclass.py --- a/rpython/rtyper/lltypesystem/rclass.py +++ b/rpython/rtyper/lltypesystem/rclass.py @@ -20,7 +20,7 @@ from rpython.annotator import model as annmodel from rpython.rlib.rarithmetic import intmask from rpython.rlib import objectmodel -from pypy.tool.identity_dict import identity_dict +from rpython.tool.identity_dict import identity_dict from rpython.rtyper.lltypesystem.lloperation import llop # diff --git a/rpython/rtyper/lltypesystem/test/test_lltype.py b/rpython/rtyper/lltypesystem/test/test_lltype.py --- a/rpython/rtyper/lltypesystem/test/test_lltype.py +++ b/rpython/rtyper/lltypesystem/test/test_lltype.py @@ -2,7 +2,7 @@ import py from rpython.rtyper.lltypesystem.lltype import * from rpython.rtyper.lltypesystem import lltype, rffi -from pypy.tool.identity_dict import identity_dict +from rpython.tool.identity_dict import identity_dict from pypy.tool import leakfinder def isweak(p, T): diff --git a/rpython/rtyper/memory/gctypelayout.py b/rpython/rtyper/memory/gctypelayout.py --- a/rpython/rtyper/memory/gctypelayout.py +++ b/rpython/rtyper/memory/gctypelayout.py @@ -3,7 +3,7 @@ from rpython.rtyper.lltypesystem.lloperation import llop from rpython.rlib.debug import ll_assert from rpython.rlib.rarithmetic import intmask -from pypy.tool.identity_dict import identity_dict +from rpython.tool.identity_dict import identity_dict class GCData(object): diff --git a/rpython/rtyper/memory/support.py b/rpython/rtyper/memory/support.py --- a/rpython/rtyper/memory/support.py +++ b/rpython/rtyper/memory/support.py @@ -2,7 +2,7 @@ from rpython.rlib.objectmodel import free_non_gc_object, we_are_translated from rpython.rlib.rarithmetic import r_uint, LONG_BIT from rpython.rlib.debug import ll_assert -from pypy.tool.identity_dict import identity_dict +from rpython.tool.identity_dict import identity_dict def mangle_hash(i): diff --git a/rpython/rtyper/ootypesystem/ootype.py b/rpython/rtyper/ootypesystem/ootype.py --- a/rpython/rtyper/ootypesystem/ootype.py +++ b/rpython/rtyper/ootypesystem/ootype.py @@ -7,7 +7,7 @@ from rpython.rtyper.lltypesystem.lltype import identityhash from rpython.rlib.rarithmetic import intmask from rpython.rlib import objectmodel -from pypy.tool.uid import uid +from rpython.tool.uid import uid STATICNESS = True diff --git a/rpython/rtyper/ootypesystem/rclass.py b/rpython/rtyper/ootypesystem/rclass.py --- a/rpython/rtyper/ootypesystem/rclass.py +++ b/rpython/rtyper/ootypesystem/rclass.py @@ -10,7 +10,7 @@ from rpython.rtyper.exceptiondata import standardexceptions from rpython.tool.pairtype import pairtype from rpython.tool.sourcetools import func_with_new_name -from pypy.tool.identity_dict import identity_dict +from rpython.tool.identity_dict import identity_dict OBJECT = ootype.ROOT META = ootype.Instance("Meta", ootype.ROOT, diff --git a/pypy/tool/identity_dict.py b/rpython/tool/identity_dict.py rename from pypy/tool/identity_dict.py rename to rpython/tool/identity_dict.py diff --git a/pypy/tool/uid.py b/rpython/tool/uid.py rename from pypy/tool/uid.py rename to rpython/tool/uid.py diff --git a/rpython/translator/backendopt/escape.py b/rpython/translator/backendopt/escape.py --- a/rpython/translator/backendopt/escape.py +++ b/rpython/translator/backendopt/escape.py @@ -1,7 +1,7 @@ from rpython.flowspace.model import Variable from rpython.rtyper.lltypesystem import lltype from rpython.translator.simplify import get_graph -from pypy.tool.uid import uid +from rpython.tool.uid import uid class CreationPoint(object): diff --git a/rpython/translator/c/database.py b/rpython/translator/c/database.py --- a/rpython/translator/c/database.py +++ b/rpython/translator/c/database.py @@ -15,7 +15,7 @@ from rpython.translator.c.support import log, barebonearray from rpython.translator.c.extfunc import do_the_getting from rpython.translator.c import gc -from pypy.tool.identity_dict import identity_dict +from rpython.tool.identity_dict import identity_dict class NoCorrespondingNode(Exception): diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py --- a/rpython/translator/c/funcgen.py +++ b/rpython/translator/c/funcgen.py @@ -12,7 +12,7 @@ from rpython.rtyper.lltypesystem.llmemory import Address from rpython.translator.backendopt.ssa import SSI_to_SSA from rpython.translator.backendopt.innerloop import find_inner_loops -from pypy.tool.identity_dict import identity_dict +from rpython.tool.identity_dict import identity_dict LOCALVAR = 'l_%s' diff --git a/rpython/translator/oosupport/function.py b/rpython/translator/oosupport/function.py --- a/rpython/translator/oosupport/function.py +++ b/rpython/translator/oosupport/function.py @@ -7,7 +7,7 @@ from rpython.rtyper.ootypesystem import ootype from rpython.translator.oosupport.treebuilder import SubOperation from rpython.translator.oosupport.metavm import InstructionList, StoreResult -from pypy.tool.identity_dict import identity_dict +from rpython.tool.identity_dict import identity_dict def render_sub_op(sub_op, db, generator): diff --git a/rpython/translator/tool/graphpage.py b/rpython/translator/tool/graphpage.py --- a/rpython/translator/tool/graphpage.py +++ b/rpython/translator/tool/graphpage.py @@ -5,7 +5,7 @@ from rpython.annotator.model import SomePBC from rpython.annotator.description import MethodDesc from rpython.annotator.classdef import ClassDef -from pypy.tool.uid import uid +from rpython.tool.uid import uid from rpython.tool.udir import udir from dotviewer.graphpage import GraphPage as BaseGraphPage diff --git a/rpython/translator/tool/lltracker.py b/rpython/translator/tool/lltracker.py --- a/rpython/translator/tool/lltracker.py +++ b/rpython/translator/tool/lltracker.py @@ -7,8 +7,8 @@ from rpython.rtyper.lltypesystem import lltype, llmemory from rpython.rtyper.memory.gcheader import header2obj from rpython.translator.tool.reftracker import BaseRefTrackerPage, MARKER -from pypy.tool.uid import uid -from pypy.tool.identity_dict import identity_dict +from rpython.tool.uid import uid +from rpython.tool.identity_dict import identity_dict class LLRefTrackerPage(BaseRefTrackerPage): diff --git a/rpython/translator/tool/reftracker.py b/rpython/translator/tool/reftracker.py --- a/rpython/translator/tool/reftracker.py +++ b/rpython/translator/tool/reftracker.py @@ -6,7 +6,7 @@ import autopath, sys, os, types import gc from rpython.translator.tool.graphpage import GraphPage, DotGen -from pypy.tool.uid import uid +from rpython.tool.uid import uid MARKER = object() From noreply at buildbot.pypy.org Fri Jan 4 22:30:58 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Fri, 4 Jan 2013 22:30:58 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Added __init__.py in rpython/tool. Fixed missing rename in pypy/objspace/std/newformat.py Message-ID: <20130104213058.DD99F1C01EE@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59701:0b0c7a97fa59 Date: 2013-01-02 16:19 +0100 http://bitbucket.org/pypy/pypy/changeset/0b0c7a97fa59/ Log: Added __init__.py in rpython/tool. Fixed missing rename in pypy/objspace/std/newformat.py diff --git a/pypy/objspace/std/newformat.py b/pypy/objspace/std/newformat.py --- a/pypy/objspace/std/newformat.py +++ b/pypy/objspace/std/newformat.py @@ -6,7 +6,7 @@ from rpython.rlib import rstring, runicode, rlocale, rarithmetic, rfloat, jit from rpython.rlib.objectmodel import specialize from rpython.rlib.rfloat import copysign, formatd -from pypy.tool import sourcetools +from rpython.tool import sourcetools @specialize.argtype(1) diff --git a/rpython/tool/__init__.py b/rpython/tool/__init__.py new file mode 100644 From noreply at buildbot.pypy.org Fri Jan 4 22:30:52 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Fri, 4 Jan 2013 22:30:52 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Adjust autopath.py to new directory structure Message-ID: <20130104213052.F13F71C0DDA@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59697:e60833b188c1 Date: 2012-12-31 23:30 +0100 http://bitbucket.org/pypy/pypy/changeset/e60833b188c1/ Log: Adjust autopath.py to new directory structure diff --git a/rpython/annotator/test/autopath.py b/rpython/annotator/test/autopath.py --- a/rpython/annotator/test/autopath.py +++ b/rpython/annotator/test/autopath.py @@ -125,7 +125,7 @@ # set guaranteed attributes -pypydir, this_dir = __dirinfo('pypy') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/jit/backend/autopath.py b/rpython/jit/backend/autopath.py --- a/rpython/jit/backend/autopath.py +++ b/rpython/jit/backend/autopath.py @@ -125,7 +125,7 @@ # set guaranteed attributes -pypydir, this_dir = __dirinfo('pypy') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/jit/backend/x86/autopath.py b/rpython/jit/backend/x86/autopath.py --- a/rpython/jit/backend/x86/autopath.py +++ b/rpython/jit/backend/x86/autopath.py @@ -125,7 +125,7 @@ # set guaranteed attributes -pypydir, this_dir = __dirinfo('pypy') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/jit/backend/x86/tool/autopath.py b/rpython/jit/backend/x86/tool/autopath.py --- a/rpython/jit/backend/x86/tool/autopath.py +++ b/rpython/jit/backend/x86/tool/autopath.py @@ -125,7 +125,7 @@ # set guaranteed attributes -pypydir, this_dir = __dirinfo('pypy') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/jit/tl/autopath.py b/rpython/jit/tl/autopath.py --- a/rpython/jit/tl/autopath.py +++ b/rpython/jit/tl/autopath.py @@ -125,7 +125,7 @@ # set guaranteed attributes -pypydir, this_dir = __dirinfo('pypy') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/jit/tl/spli/autopath.py b/rpython/jit/tl/spli/autopath.py --- a/rpython/jit/tl/spli/autopath.py +++ b/rpython/jit/tl/spli/autopath.py @@ -125,7 +125,7 @@ # set guaranteed attributes -pypydir, this_dir = __dirinfo('pypy') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/jit/tool/autopath.py b/rpython/jit/tool/autopath.py --- a/rpython/jit/tool/autopath.py +++ b/rpython/jit/tool/autopath.py @@ -125,7 +125,7 @@ # set guaranteed attributes -pypydir, this_dir = __dirinfo('pypy') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/rlib/parsing/test/autopath.py b/rpython/rlib/parsing/test/autopath.py --- a/rpython/rlib/parsing/test/autopath.py +++ b/rpython/rlib/parsing/test/autopath.py @@ -125,7 +125,7 @@ # set guaranteed attributes -pypydir, this_dir = __dirinfo('pypy') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/rlib/test/autopath.py b/rpython/rlib/test/autopath.py --- a/rpython/rlib/test/autopath.py +++ b/rpython/rlib/test/autopath.py @@ -125,7 +125,7 @@ # set guaranteed attributes -pypydir, this_dir = __dirinfo('pypy') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/translator/autopath.py b/rpython/translator/autopath.py --- a/rpython/translator/autopath.py +++ b/rpython/translator/autopath.py @@ -125,7 +125,7 @@ # set guaranteed attributes -pypydir, this_dir = __dirinfo('pypy') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/translator/c/autopath.py b/rpython/translator/c/autopath.py --- a/rpython/translator/c/autopath.py +++ b/rpython/translator/c/autopath.py @@ -125,7 +125,7 @@ # set guaranteed attributes -pypydir, this_dir = __dirinfo('pypy') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/translator/c/gcc/autopath.py b/rpython/translator/c/gcc/autopath.py --- a/rpython/translator/c/gcc/autopath.py +++ b/rpython/translator/c/gcc/autopath.py @@ -125,7 +125,7 @@ # set guaranteed attributes -pypydir, this_dir = __dirinfo('pypy') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/translator/c/test/autopath.py b/rpython/translator/c/test/autopath.py --- a/rpython/translator/c/test/autopath.py +++ b/rpython/translator/c/test/autopath.py @@ -125,7 +125,7 @@ # set guaranteed attributes -pypydir, this_dir = __dirinfo('pypy') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/translator/cli/test/autopath.py b/rpython/translator/cli/test/autopath.py --- a/rpython/translator/cli/test/autopath.py +++ b/rpython/translator/cli/test/autopath.py @@ -125,7 +125,7 @@ # set guaranteed attributes -pypydir, this_dir = __dirinfo('pypy') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/translator/goal/autopath.py b/rpython/translator/goal/autopath.py --- a/rpython/translator/goal/autopath.py +++ b/rpython/translator/goal/autopath.py @@ -125,7 +125,7 @@ # set guaranteed attributes -pypydir, this_dir = __dirinfo('pypy') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/translator/goal/test2/autopath.py b/rpython/translator/goal/test2/autopath.py --- a/rpython/translator/goal/test2/autopath.py +++ b/rpython/translator/goal/test2/autopath.py @@ -125,7 +125,7 @@ # set guaranteed attributes -pypydir, this_dir = __dirinfo('pypy') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/translator/sandbox/autopath.py b/rpython/translator/sandbox/autopath.py --- a/rpython/translator/sandbox/autopath.py +++ b/rpython/translator/sandbox/autopath.py @@ -125,7 +125,7 @@ # set guaranteed attributes -pypydir, this_dir = __dirinfo('pypy') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/translator/sandbox/test/autopath.py b/rpython/translator/sandbox/test/autopath.py --- a/rpython/translator/sandbox/test/autopath.py +++ b/rpython/translator/sandbox/test/autopath.py @@ -125,7 +125,7 @@ # set guaranteed attributes -pypydir, this_dir = __dirinfo('pypy') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/translator/test/autopath.py b/rpython/translator/test/autopath.py --- a/rpython/translator/test/autopath.py +++ b/rpython/translator/test/autopath.py @@ -125,7 +125,7 @@ # set guaranteed attributes -pypydir, this_dir = __dirinfo('pypy') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/translator/tool/autopath.py b/rpython/translator/tool/autopath.py --- a/rpython/translator/tool/autopath.py +++ b/rpython/translator/tool/autopath.py @@ -125,7 +125,7 @@ # set guaranteed attributes -pypydir, this_dir = __dirinfo('pypy') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() From noreply at buildbot.pypy.org Fri Jan 4 22:31:00 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Fri, 4 Jan 2013 22:31:00 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Moved pypy.tool.algo to rpython Message-ID: <20130104213100.263451C01EE@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59702:5e96a470b4e0 Date: 2013-01-02 16:24 +0100 http://bitbucket.org/pypy/pypy/changeset/5e96a470b4e0/ Log: Moved pypy.tool.algo to rpython diff --git a/pypy/tool/importfun.py b/pypy/tool/importfun.py --- a/pypy/tool/importfun.py +++ b/pypy/tool/importfun.py @@ -385,7 +385,7 @@ print ' ', k, v def find_cycles(system): - from pypy.tool.algo import graphlib + from rpython.tool.algo import graphlib vertices = dict.fromkeys(system.modules) edges = {} for m in system.modules: diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py --- a/rpython/annotator/bookkeeper.py +++ b/rpython/annotator/bookkeeper.py @@ -21,7 +21,7 @@ from rpython.annotator.signature import annotationoftype from rpython.flowspace.argument import ArgumentsForTranslation from rpython.rlib.objectmodel import r_dict, Symbolic -from pypy.tool.algo.unionfind import UnionFind +from rpython.tool.algo.unionfind import UnionFind from rpython.rtyper.lltypesystem import lltype, llmemory from rpython.rtyper.ootypesystem import ootype from rpython.rtyper import extregistry diff --git a/rpython/annotator/specialize.py b/rpython/annotator/specialize.py --- a/rpython/annotator/specialize.py +++ b/rpython/annotator/specialize.py @@ -2,7 +2,7 @@ import py from rpython.tool.uid import uid from rpython.tool.sourcetools import func_with_new_name -from pypy.tool.algo.unionfind import UnionFind +from rpython.tool.algo.unionfind import UnionFind from rpython.flowspace.model import Block, Link, Variable, SpaceOperation from rpython.flowspace.model import Constant, checkgraph from rpython.annotator import model as annmodel diff --git a/rpython/jit/codewriter/regalloc.py b/rpython/jit/codewriter/regalloc.py --- a/rpython/jit/codewriter/regalloc.py +++ b/rpython/jit/codewriter/regalloc.py @@ -1,4 +1,4 @@ -from pypy.tool.algo import regalloc +from rpython.tool.algo import regalloc from rpython.jit.metainterp.history import getkind from rpython.jit.codewriter.flatten import ListOfKind diff --git a/rpython/rtyper/memory/test/snippet.py b/rpython/rtyper/memory/test/snippet.py --- a/rpython/rtyper/memory/test/snippet.py +++ b/rpython/rtyper/memory/test/snippet.py @@ -9,7 +9,7 @@ def definestr_finalizer_order(cls): import random - from pypy.tool.algo import graphlib + from rpython.tool.algo import graphlib cls.finalizer_order_examples = examples = [] if cls.large_tests_ok: diff --git a/pypy/tool/algo/BB.sml b/rpython/tool/algo/BB.sml rename from pypy/tool/algo/BB.sml rename to rpython/tool/algo/BB.sml diff --git a/pypy/tool/algo/__init__.py b/rpython/tool/algo/__init__.py rename from pypy/tool/algo/__init__.py rename to rpython/tool/algo/__init__.py diff --git a/pypy/tool/algo/color.py b/rpython/tool/algo/color.py rename from pypy/tool/algo/color.py rename to rpython/tool/algo/color.py diff --git a/pypy/tool/algo/fset.py b/rpython/tool/algo/fset.py rename from pypy/tool/algo/fset.py rename to rpython/tool/algo/fset.py diff --git a/pypy/tool/algo/graphlib.py b/rpython/tool/algo/graphlib.py rename from pypy/tool/algo/graphlib.py rename to rpython/tool/algo/graphlib.py diff --git a/pypy/tool/algo/multiweakdict.py b/rpython/tool/algo/multiweakdict.py rename from pypy/tool/algo/multiweakdict.py rename to rpython/tool/algo/multiweakdict.py diff --git a/pypy/tool/algo/regalloc.py b/rpython/tool/algo/regalloc.py rename from pypy/tool/algo/regalloc.py rename to rpython/tool/algo/regalloc.py --- a/pypy/tool/algo/regalloc.py +++ b/rpython/tool/algo/regalloc.py @@ -1,7 +1,7 @@ import sys from rpython.flowspace.model import Variable -from pypy.tool.algo.color import DependencyGraph -from pypy.tool.algo.unionfind import UnionFind +from rpython.tool.algo.color import DependencyGraph +from rpython.tool.algo.unionfind import UnionFind def perform_register_allocation(graph, consider_var, ListOfKind=()): """Perform register allocation for the Variables of the given 'kind' diff --git a/pypy/tool/algo/sparsemat.py b/rpython/tool/algo/sparsemat.py rename from pypy/tool/algo/sparsemat.py rename to rpython/tool/algo/sparsemat.py diff --git a/pypy/tool/algo/test/__init__.py b/rpython/tool/algo/test/__init__.py rename from pypy/tool/algo/test/__init__.py rename to rpython/tool/algo/test/__init__.py diff --git a/pypy/tool/algo/test/autopath.py b/rpython/tool/algo/test/autopath.py rename from pypy/tool/algo/test/autopath.py rename to rpython/tool/algo/test/autopath.py diff --git a/pypy/tool/algo/test/test_color.py b/rpython/tool/algo/test/test_color.py rename from pypy/tool/algo/test/test_color.py rename to rpython/tool/algo/test/test_color.py --- a/pypy/tool/algo/test/test_color.py +++ b/rpython/tool/algo/test/test_color.py @@ -1,4 +1,4 @@ -from pypy.tool.algo.color import DependencyGraph +from rpython.tool.algo.color import DependencyGraph def graph1(): diff --git a/pypy/tool/algo/test/test_fset.py b/rpython/tool/algo/test/test_fset.py rename from pypy/tool/algo/test/test_fset.py rename to rpython/tool/algo/test/test_fset.py --- a/pypy/tool/algo/test/test_fset.py +++ b/rpython/tool/algo/test/test_fset.py @@ -1,4 +1,4 @@ -from pypy.tool.algo.fset import FSet, checktree, emptyset +from rpython.tool.algo.fset import FSet, checktree, emptyset import random diff --git a/pypy/tool/algo/test/test_graphlib.py b/rpython/tool/algo/test/test_graphlib.py rename from pypy/tool/algo/test/test_graphlib.py rename to rpython/tool/algo/test/test_graphlib.py --- a/pypy/tool/algo/test/test_graphlib.py +++ b/rpython/tool/algo/test/test_graphlib.py @@ -1,6 +1,6 @@ import autopath import random -from pypy.tool.algo.graphlib import * +from rpython.tool.algo.graphlib import * def copy_edges(edges): result = {} diff --git a/pypy/tool/algo/test/test_multiweakdict.py b/rpython/tool/algo/test/test_multiweakdict.py rename from pypy/tool/algo/test/test_multiweakdict.py rename to rpython/tool/algo/test/test_multiweakdict.py --- a/pypy/tool/algo/test/test_multiweakdict.py +++ b/rpython/tool/algo/test/test_multiweakdict.py @@ -1,5 +1,5 @@ import py, gc -from pypy.tool.algo.multiweakdict import MultiWeakKeyDictionary +from rpython.tool.algo.multiweakdict import MultiWeakKeyDictionary class A(object): diff --git a/pypy/tool/algo/test/test_sparsemat.py b/rpython/tool/algo/test/test_sparsemat.py rename from pypy/tool/algo/test/test_sparsemat.py rename to rpython/tool/algo/test/test_sparsemat.py --- a/pypy/tool/algo/test/test_sparsemat.py +++ b/rpython/tool/algo/test/test_sparsemat.py @@ -1,5 +1,5 @@ import autopath -from pypy.tool.algo.sparsemat import * +from rpython.tool.algo.sparsemat import * def test_sparsemat1(): diff --git a/pypy/tool/algo/test/test_unionfind.py b/rpython/tool/algo/test/test_unionfind.py rename from pypy/tool/algo/test/test_unionfind.py rename to rpython/tool/algo/test/test_unionfind.py --- a/pypy/tool/algo/test/test_unionfind.py +++ b/rpython/tool/algo/test/test_unionfind.py @@ -1,4 +1,4 @@ -from pypy.tool.algo.unionfind import UnionFind +from rpython.tool.algo.unionfind import UnionFind def test_cleanup(): diff --git a/pypy/tool/algo/unionfind.py b/rpython/tool/algo/unionfind.py rename from pypy/tool/algo/unionfind.py rename to rpython/tool/algo/unionfind.py diff --git a/rpython/translator/backendopt/graphanalyze.py b/rpython/translator/backendopt/graphanalyze.py --- a/rpython/translator/backendopt/graphanalyze.py +++ b/rpython/translator/backendopt/graphanalyze.py @@ -1,5 +1,5 @@ from rpython.translator.simplify import get_graph, get_funcobj -from pypy.tool.algo.unionfind import UnionFind +from rpython.tool.algo.unionfind import UnionFind class GraphAnalyzer(object): diff --git a/rpython/translator/backendopt/inline.py b/rpython/translator/backendopt/inline.py --- a/rpython/translator/backendopt/inline.py +++ b/rpython/translator/backendopt/inline.py @@ -4,7 +4,7 @@ SpaceOperation, c_last_exception, FunctionGraph, mkentrymap) from rpython.rtyper.lltypesystem.lltype import Bool, Signed, typeOf, Void, Ptr, normalizeptr from rpython.rtyper.ootypesystem import ootype -from pypy.tool.algo import sparsemat +from rpython.tool.algo import sparsemat from rpython.translator.backendopt import removenoops from rpython.translator.backendopt.canraise import RaiseAnalyzer from rpython.translator.backendopt.support import log, find_loop_blocks diff --git a/rpython/translator/backendopt/innerloop.py b/rpython/translator/backendopt/innerloop.py --- a/rpython/translator/backendopt/innerloop.py +++ b/rpython/translator/backendopt/innerloop.py @@ -2,7 +2,7 @@ This is optional support code for backends: it finds which cycles in a graph are likely to correspond to source-level 'inner loops'. """ -from pypy.tool.algo import graphlib +from rpython.tool.algo import graphlib from rpython.flowspace.model import Variable from rpython.translator.backendopt.ssa import DataFlowFamilyBuilder diff --git a/rpython/translator/backendopt/malloc.py b/rpython/translator/backendopt/malloc.py --- a/rpython/translator/backendopt/malloc.py +++ b/rpython/translator/backendopt/malloc.py @@ -1,5 +1,5 @@ from rpython.flowspace.model import Variable, Constant, SpaceOperation -from pypy.tool.algo.unionfind import UnionFind +from rpython.tool.algo.unionfind import UnionFind from rpython.rtyper.lltypesystem import lltype from rpython.rtyper.ootypesystem import ootype from rpython.translator import simplify diff --git a/rpython/translator/backendopt/ssa.py b/rpython/translator/backendopt/ssa.py --- a/rpython/translator/backendopt/ssa.py +++ b/rpython/translator/backendopt/ssa.py @@ -1,5 +1,5 @@ from rpython.flowspace.model import Variable, mkentrymap -from pypy.tool.algo.unionfind import UnionFind +from rpython.tool.algo.unionfind import UnionFind class DataFlowFamilyBuilder: """Follow the flow of the data in the graph. Builds a UnionFind grouping diff --git a/rpython/translator/transform.py b/rpython/translator/transform.py --- a/rpython/translator/transform.py +++ b/rpython/translator/transform.py @@ -205,7 +205,7 @@ def insert_ll_stackcheck(translator): from rpython.translator.backendopt.support import find_calls_from from rpython.rlib.rstack import stack_check - from pypy.tool.algo.graphlib import Edge, make_edge_dict, break_cycles_v + from rpython.tool.algo.graphlib import Edge, make_edge_dict, break_cycles_v rtyper = translator.rtyper graph = rtyper.annotate_helper(stack_check, []) rtyper.specialize_more_blocks() From noreply at buildbot.pypy.org Fri Jan 4 22:30:54 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Fri, 4 Jan 2013 22:30:54 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Moved pypy.tool.pairtype and pypy.tool.sourcetools to rpython. Fixed missing rename in bin/rpython Message-ID: <20130104213054.901D31C0F8A@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59698:af5aee973ab3 Date: 2013-01-01 18:31 +0100 http://bitbucket.org/pypy/pypy/changeset/af5aee973ab3/ Log: Moved pypy.tool.pairtype and pypy.tool.sourcetools to rpython. Fixed missing rename in bin/rpython diff --git a/pypy/bin/rpython b/pypy/bin/rpython --- a/pypy/bin/rpython +++ b/pypy/bin/rpython @@ -8,7 +8,7 @@ """ import sys -from pypy.translator.goal.translate import main +from rpython.translator.goal.translate import main # no implicit targets if len(sys.argv) == 1: diff --git a/pypy/config/config.py b/pypy/config/config.py --- a/pypy/config/config.py +++ b/pypy/config/config.py @@ -1,6 +1,6 @@ import py import optparse -from pypy.tool.pairtype import extendabletype +from rpython.tool.pairtype import extendabletype SUPPRESS_USAGE = optparse.SUPPRESS_USAGE diff --git a/pypy/interpreter/astcompiler/ast.py b/pypy/interpreter/astcompiler/ast.py --- a/pypy/interpreter/astcompiler/ast.py +++ b/pypy/interpreter/astcompiler/ast.py @@ -4,8 +4,8 @@ from pypy.interpreter.gateway import interp2app from pypy.interpreter.error import OperationError, operationerrfmt from rpython.rlib.unroll import unrolling_iterable -from pypy.tool.pairtype import extendabletype -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.pairtype import extendabletype +from rpython.tool.sourcetools import func_with_new_name def check_string(space, w_obj): diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py b/pypy/interpreter/astcompiler/tools/asdl_py.py --- a/pypy/interpreter/astcompiler/tools/asdl_py.py +++ b/pypy/interpreter/astcompiler/tools/asdl_py.py @@ -543,8 +543,8 @@ from pypy.interpreter.gateway import interp2app from pypy.interpreter.error import OperationError, operationerrfmt from rpython.rlib.unroll import unrolling_iterable -from pypy.tool.pairtype import extendabletype -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.pairtype import extendabletype +from rpython.tool.sourcetools import func_with_new_name def check_string(space, w_obj): diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py --- a/pypy/interpreter/gateway.py +++ b/pypy/interpreter/gateway.py @@ -22,7 +22,7 @@ from rpython.rlib import rstackovf from rpython.rlib.objectmodel import we_are_translated from rpython.rlib.rarithmetic import r_longlong, r_int, r_ulonglong, r_uint -from pypy.tool.sourcetools import func_with_new_name, compile2 +from rpython.tool.sourcetools import func_with_new_name, compile2 # internal non-translatable parts: diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py --- a/pypy/interpreter/pyframe.py +++ b/pypy/interpreter/pyframe.py @@ -1,7 +1,7 @@ """ PyFrame class implementation with the interpreter main loop. """ -from pypy.tool.pairtype import extendabletype +from rpython.tool.pairtype import extendabletype from pypy.interpreter import eval, baseobjspace, pycode from pypy.interpreter.argument import Arguments from pypy.interpreter.error import OperationError, operationerrfmt diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -9,7 +9,7 @@ from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter import gateway, function, eval, pyframe, pytraceback from pypy.interpreter.pycode import PyCode, BytecodeCorruption -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name from rpython.rlib.objectmodel import we_are_translated from rpython.rlib import jit, rstackovf from rpython.rlib.rarithmetic import r_uint, intmask diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py --- a/pypy/interpreter/typedef.py +++ b/pypy/interpreter/typedef.py @@ -8,7 +8,7 @@ from pypy.interpreter.argument import Arguments from pypy.interpreter.baseobjspace import Wrappable, DescrMismatch from pypy.interpreter.error import OperationError, operationerrfmt -from pypy.tool.sourcetools import compile2, func_with_new_name +from rpython.tool.sourcetools import compile2, func_with_new_name from rpython.rlib.objectmodel import instantiate, compute_identity_hash, specialize from rpython.rlib.jit import promote diff --git a/pypy/module/__pypy__/interp_builders.py b/pypy/module/__pypy__/interp_builders.py --- a/pypy/module/__pypy__/interp_builders.py +++ b/pypy/module/__pypy__/interp_builders.py @@ -3,7 +3,7 @@ from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import TypeDef from rpython.rlib.rstring import UnicodeBuilder, StringBuilder -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name def create_builder(name, strtype, builder_cls): diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py --- a/pypy/module/_cffi_backend/cdataobj.py +++ b/pypy/module/_cffi_backend/cdataobj.py @@ -6,7 +6,7 @@ from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rlib.objectmodel import keepalive_until_here, specialize from rpython.rlib import objectmodel, rgc -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name from pypy.module._cffi_backend import misc diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py --- a/pypy/module/_hashlib/interp_hashlib.py +++ b/pypy/module/_hashlib/interp_hashlib.py @@ -2,7 +2,7 @@ from pypy.interpreter.gateway import unwrap_spec, interp2app from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.interpreter.error import OperationError -from pypy.tool.sourcetools import func_renamer +from rpython.tool.sourcetools import func_renamer from pypy.interpreter.baseobjspace import Wrappable from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.rlib import rgc, ropenssl diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py --- a/pypy/module/_io/interp_bufferedio.py +++ b/pypy/module/_io/interp_bufferedio.py @@ -7,7 +7,7 @@ from rpython.rlib.rstring import StringBuilder from rpython.rlib.rarithmetic import r_longlong, intmask from rpython.rlib import rposix -from pypy.tool.sourcetools import func_renamer +from rpython.tool.sourcetools import func_renamer from pypy.module._io.interp_iobase import ( W_IOBase, DEFAULT_BUFFER_SIZE, convert_size, check_readable_w, check_writable_w, check_seekable_w) diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py --- a/pypy/module/_rawffi/interp_rawffi.py +++ b/pypy/module/_rawffi/interp_rawffi.py @@ -13,7 +13,7 @@ if _MS_WINDOWS: from rpython.rlib import rwin32 -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name from rpython.rlib.rarithmetic import intmask, r_uint from pypy.module._rawffi.tracker import tracker diff --git a/pypy/module/clr/boxing_rules.py b/pypy/module/clr/boxing_rules.py --- a/pypy/module/clr/boxing_rules.py +++ b/pypy/module/clr/boxing_rules.py @@ -1,4 +1,4 @@ -from pypy.tool.pairtype import extendabletype +from rpython.tool.pairtype import extendabletype from pypy.interpreter.baseobjspace import W_Root from pypy.objspace.std.intobject import W_IntObject from pypy.objspace.std.floatobject import W_FloatObject diff --git a/pypy/module/cmath/interp_cmath.py b/pypy/module/cmath/interp_cmath.py --- a/pypy/module/cmath/interp_cmath.py +++ b/pypy/module/cmath/interp_cmath.py @@ -1,6 +1,6 @@ import math from rpython.rlib.objectmodel import specialize -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name from pypy.interpreter.error import OperationError from pypy.module.cmath import names_and_docstrings from rpython.rlib import rcomplex diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -34,7 +34,7 @@ from pypy.module.exceptions import interp_exceptions # CPython 2.4 compatibility from py.builtin import BaseException -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name from rpython.rtyper.lltypesystem.lloperation import llop DEBUG_WRAPPER = True diff --git a/pypy/module/cpyext/cdatetime.py b/pypy/module/cpyext/cdatetime.py --- a/pypy/module/cpyext/cdatetime.py +++ b/pypy/module/cpyext/cdatetime.py @@ -5,7 +5,7 @@ from pypy.module.cpyext.import_ import PyImport_Import from pypy.module.cpyext.typeobject import PyTypeObjectPtr from pypy.interpreter.error import OperationError -from pypy.tool.sourcetools import func_renamer +from rpython.tool.sourcetools import func_renamer # API import function diff --git a/pypy/module/cpyext/number.py b/pypy/module/cpyext/number.py --- a/pypy/module/cpyext/number.py +++ b/pypy/module/cpyext/number.py @@ -2,7 +2,7 @@ from pypy.module.cpyext.api import cpython_api, CANNOT_FAIL, Py_ssize_t from pypy.module.cpyext.pyobject import PyObject from rpython.rtyper.lltypesystem import rffi, lltype -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL) def PyIndex_Check(space, w_obj): diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -19,7 +19,7 @@ from pypy.interpreter.argument import Arguments from rpython.rlib.unroll import unrolling_iterable from rpython.rlib.objectmodel import specialize -from pypy.tool.sourcetools import func_renamer +from rpython.tool.sourcetools import func_renamer from rpython.rtyper.annlowlevel import llhelper # XXX: Also defined in object.h diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -15,7 +15,7 @@ from pypy.module._codecs.interp_codecs import CodecState from pypy.objspace.std import unicodeobject, unicodetype, stringtype from rpython.rlib import runicode -from pypy.tool.sourcetools import func_renamer +from rpython.tool.sourcetools import func_renamer import sys ## See comment in stringobject.py. diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py --- a/pypy/module/micronumpy/base.py +++ b/pypy/module/micronumpy/base.py @@ -1,6 +1,6 @@ from pypy.interpreter.baseobjspace import Wrappable -from pypy.tool.pairtype import extendabletype +from rpython.tool.pairtype import extendabletype from pypy.module.micronumpy.support import calc_strides def issequence_w(space, w_obj): diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py --- a/pypy/module/micronumpy/interp_boxes.py +++ b/pypy/module/micronumpy/interp_boxes.py @@ -9,7 +9,7 @@ from pypy.objspace.std.complextype import complex_typedef from rpython.rlib.rarithmetic import LONG_BIT from rpython.rtyper.lltypesystem import rffi -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name from pypy.module.micronumpy.arrayimpl.voidbox import VoidBoxStorage MIXIN_32 = (int_typedef,) if LONG_BIT == 32 else () diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -13,7 +13,7 @@ from pypy.module.micronumpy import loop from pypy.module.micronumpy.dot import match_dot_shapes from pypy.module.micronumpy.interp_arrayops import repeat -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name from rpython.rlib import jit from rpython.rlib.rstring import StringBuilder diff --git a/pypy/module/micronumpy/interp_ufuncs.py b/pypy/module/micronumpy/interp_ufuncs.py --- a/pypy/module/micronumpy/interp_ufuncs.py +++ b/pypy/module/micronumpy/interp_ufuncs.py @@ -5,7 +5,7 @@ from pypy.module.micronumpy import interp_boxes, interp_dtype, loop from rpython.rlib import jit from rpython.rlib.rarithmetic import LONG_BIT -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name from pypy.module.micronumpy.interp_support import unwrap_axis_arg from pypy.module.micronumpy.strides import shape_agreement from pypy.module.micronumpy.base import convert_to_array, W_NDimArray diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -16,7 +16,7 @@ from rpython.rlib.rstruct.nativefmttable import native_is_bigendian from rpython.rlib.rstruct.ieee import (float_pack, float_unpack, unpack_float, unpack_float128) -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name from rpython.rlib import jit from rpython.rlib.rstring import StringBuilder diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py --- a/pypy/module/pypyjit/interp_jit.py +++ b/pypy/module/pypyjit/interp_jit.py @@ -3,7 +3,7 @@ This is transformed to become a JIT by code elsewhere: pypy/jit/* """ -from pypy.tool.pairtype import extendabletype +from rpython.tool.pairtype import extendabletype from rpython.rlib.rarithmetic import r_uint, intmask from rpython.rlib.jit import JitDriver, hint, we_are_jitted, dont_look_inside from rpython.rlib import jit diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py --- a/pypy/objspace/descroperation.py +++ b/pypy/objspace/descroperation.py @@ -4,7 +4,7 @@ from pypy.interpreter.function import Function, Method, FunctionWithFixedCode from pypy.interpreter.argument import Arguments from pypy.interpreter.typedef import default_identity_hash -from pypy.tool.sourcetools import compile2, func_with_new_name +from rpython.tool.sourcetools import compile2, func_with_new_name from pypy.module.__builtin__.interp_classobj import W_InstanceObject from rpython.rlib.objectmodel import specialize diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py --- a/pypy/objspace/fake/objspace.py +++ b/pypy/objspace/fake/objspace.py @@ -8,7 +8,7 @@ from rpython.rtyper.extregistry import ExtRegistryEntry from rpython.rtyper.lltypesystem import lltype from pypy.tool.option import make_config -from pypy.tool.sourcetools import compile2, func_with_new_name +from rpython.tool.sourcetools import compile2, func_with_new_name from rpython.translator.translator import TranslationContext diff --git a/pypy/objspace/std/builtinshortcut.py b/pypy/objspace/std/builtinshortcut.py --- a/pypy/objspace/std/builtinshortcut.py +++ b/pypy/objspace/std/builtinshortcut.py @@ -3,7 +3,7 @@ from pypy.objspace.descroperation import DescrOperation from pypy.objspace.std.multimethod import FailedToImplement from pypy.objspace.std.boolobject import W_BoolObject -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name # ____________________________________________________________ # diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py --- a/pypy/objspace/std/bytearrayobject.py +++ b/pypy/objspace/std/bytearrayobject.py @@ -25,7 +25,7 @@ makebytearraydata_w, getbytevalue, new_bytearray ) -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name class W_BytearrayObject(W_Object): diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py --- a/pypy/objspace/std/dictmultiobject.py +++ b/pypy/objspace/std/dictmultiobject.py @@ -10,7 +10,7 @@ from rpython.rlib.objectmodel import r_dict, we_are_translated, specialize,\ newlist_hint from rpython.rlib.debug import mark_dict_non_null -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name from rpython.rlib import rerased from rpython.rlib import jit diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py --- a/pypy/objspace/std/floatobject.py +++ b/pypy/objspace/std/floatobject.py @@ -13,7 +13,7 @@ DTSF_ADD_DOT_0, DTSF_STR_PRECISION) from rpython.rlib.rbigint import rbigint from rpython.rlib import rfloat -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name import math diff --git a/pypy/objspace/std/formatting.py b/pypy/objspace/std/formatting.py --- a/pypy/objspace/std/formatting.py +++ b/pypy/objspace/std/formatting.py @@ -8,7 +8,7 @@ from rpython.rlib.rfloat import formatd, DTSF_ALT, isnan, isinf from rpython.rlib.rstring import StringBuilder, UnicodeBuilder from rpython.rlib.unroll import unrolling_iterable -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name class BaseStringFormatter(object): diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py --- a/pypy/objspace/std/intobject.py +++ b/pypy/objspace/std/intobject.py @@ -84,7 +84,7 @@ def declare_new_int_comparison(opname): import operator - from pypy.tool.sourcetools import func_with_new_name + from rpython.tool.sourcetools import func_with_new_name op = getattr(operator, opname) def f(space, w_int1, w_int2): i = w_int1.intval diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py --- a/pypy/objspace/std/listobject.py +++ b/pypy/objspace/std/listobject.py @@ -13,7 +13,7 @@ from rpython.rlib.listsort import make_timsort_class from rpython.rlib import rerased, jit, debug from pypy.interpreter.argument import Signature -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name UNROLL_CUTOFF = 5 diff --git a/pypy/objspace/std/multimethod.py b/pypy/objspace/std/multimethod.py --- a/pypy/objspace/std/multimethod.py +++ b/pypy/objspace/std/multimethod.py @@ -1,5 +1,5 @@ -from pypy.tool.sourcetools import compile2 +from rpython.tool.sourcetools import compile2 # This provide two compatible implementations of "multimethods". A # multimethod is a callable object which chooses and calls a real diff --git a/pypy/objspace/std/proxy_helpers.py b/pypy/objspace/std/proxy_helpers.py --- a/pypy/objspace/std/proxy_helpers.py +++ b/pypy/objspace/std/proxy_helpers.py @@ -6,7 +6,7 @@ from pypy.objspace.std.model import W_ANY, W_Object from pypy.interpreter import baseobjspace from pypy.interpreter.argument import Arguments -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name def create_mm_names(classname, mm, is_local): s = "" diff --git a/pypy/objspace/std/smallintobject.py b/pypy/objspace/std/smallintobject.py --- a/pypy/objspace/std/smallintobject.py +++ b/pypy/objspace/std/smallintobject.py @@ -11,7 +11,7 @@ from rpython.rlib.objectmodel import UnboxedValue from rpython.rlib.rbigint import rbigint from rpython.rlib.rarithmetic import r_uint -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name from pypy.objspace.std.inttype import wrapint class W_SmallIntObject(W_AbstractIntObject, UnboxedValue): diff --git a/pypy/objspace/std/smalltupleobject.py b/pypy/objspace/std/smalltupleobject.py --- a/pypy/objspace/std/smalltupleobject.py +++ b/pypy/objspace/std/smalltupleobject.py @@ -9,7 +9,7 @@ from pypy.interpreter import gateway from rpython.rlib.debug import make_sure_not_resized from rpython.rlib.unroll import unrolling_iterable -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name from pypy.objspace.std.tupleobject import W_AbstractTupleObject, W_TupleObject class W_SmallTupleObject(W_AbstractTupleObject): diff --git a/pypy/objspace/std/specialisedtupleobject.py b/pypy/objspace/std/specialisedtupleobject.py --- a/pypy/objspace/std/specialisedtupleobject.py +++ b/pypy/objspace/std/specialisedtupleobject.py @@ -8,7 +8,7 @@ from rpython.rlib.rarithmetic import intmask from rpython.rlib.objectmodel import compute_hash from rpython.rlib.unroll import unrolling_iterable -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name class NotSpecialised(Exception): pass diff --git a/pypy/objspace/std/stdtypedef.py b/pypy/objspace/std/stdtypedef.py --- a/pypy/objspace/std/stdtypedef.py +++ b/pypy/objspace/std/stdtypedef.py @@ -8,7 +8,7 @@ from pypy.objspace.std.model import StdObjSpaceMultiMethod from pypy.objspace.std.multimethod import FailedToImplement from rpython.rlib import jit -from pypy.tool.sourcetools import compile2 +from rpython.tool.sourcetools import compile2 __all__ = ['StdTypeDef', 'SMM'] diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py --- a/pypy/objspace/std/stringobject.py +++ b/pypy/objspace/std/stringobject.py @@ -349,7 +349,7 @@ return space.newlist(res_w) def make_rsplit_with_delim(funcname, sliced): - from pypy.tool.sourcetools import func_with_new_name + from rpython.tool.sourcetools import func_with_new_name def fn(space, w_self, w_by, w_maxsplit=-1): maxsplit = space.int_w(w_maxsplit) diff --git a/pypy/objspace/std/tupleobject.py b/pypy/objspace/std/tupleobject.py --- a/pypy/objspace/std/tupleobject.py +++ b/pypy/objspace/std/tupleobject.py @@ -8,7 +8,7 @@ from pypy.objspace.std import slicetype from rpython.rlib.debug import make_sure_not_resized from rpython.rlib import jit -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name # Tuples of known length up to UNROLL_TUPLE_LIMIT have unrolled certain methods UNROLL_TUPLE_LIMIT = 10 diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py --- a/pypy/objspace/std/unicodeobject.py +++ b/pypy/objspace/std/unicodeobject.py @@ -14,7 +14,7 @@ from rpython.rlib.rstring import UnicodeBuilder from rpython.rlib.runicode import make_unicode_escape_function from pypy.module.unicodedata import unicodedb -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name from rpython.rlib import jit from pypy.objspace.std.formatting import mod_format diff --git a/pypy/tool/frozenlist.py b/pypy/tool/frozenlist.py --- a/pypy/tool/frozenlist.py +++ b/pypy/tool/frozenlist.py @@ -1,4 +1,4 @@ -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name def forbid(*args): raise TypeError, "cannot mutate a frozenlist" diff --git a/pypy/tool/test/test_pairtype.py b/pypy/tool/test/test_pairtype.py --- a/pypy/tool/test/test_pairtype.py +++ b/pypy/tool/test/test_pairtype.py @@ -1,5 +1,5 @@ -from pypy.tool.pairtype import pairtype, pair, extendabletype +from rpython.tool.pairtype import pairtype, pair, extendabletype def test_binop(): ### Binary operation example diff --git a/pypy/tool/test/test_sourcetools.py b/pypy/tool/test/test_sourcetools.py --- a/pypy/tool/test/test_sourcetools.py +++ b/pypy/tool/test/test_sourcetools.py @@ -1,4 +1,4 @@ -from pypy.tool.sourcetools import func_with_new_name, func_renamer, rpython_wrapper +from rpython.tool.sourcetools import func_with_new_name, func_renamer, rpython_wrapper def test_rename(): def f(x, y=5): diff --git a/pypy/tool/test/test_template.py b/pypy/tool/test/test_template.py --- a/pypy/tool/test/test_template.py +++ b/pypy/tool/test/test_template.py @@ -1,5 +1,5 @@ import autopath -from pypy.tool.sourcetools import compile_template +from rpython.tool.sourcetools import compile_template some_global = 5 diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py --- a/rpython/annotator/annrpython.py +++ b/rpython/annotator/annrpython.py @@ -2,7 +2,7 @@ import types from pypy.tool.ansi_print import ansi_log -from pypy.tool.pairtype import pair +from rpython.tool.pairtype import pair from pypy.tool.error import (format_blocked_annotation_error, AnnotatorError, gather_error, ErrorWrapper) from rpython.flowspace.model import (Variable, Constant, FunctionGraph, diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py --- a/rpython/annotator/binaryop.py +++ b/rpython/annotator/binaryop.py @@ -4,7 +4,7 @@ import py import operator -from pypy.tool.pairtype import pair, pairtype +from rpython.tool.pairtype import pair, pairtype from rpython.annotator.model import SomeObject, SomeInteger, SomeBool, s_Bool from rpython.annotator.model import SomeString, SomeChar, SomeList, SomeDict from rpython.annotator.model import SomeUnicodeCodePoint, SomeUnicodeString diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py --- a/rpython/annotator/description.py +++ b/rpython/annotator/description.py @@ -4,8 +4,8 @@ from rpython.flowspace.model import Constant, FunctionGraph from rpython.flowspace.bytecode import cpython_code_signature from rpython.flowspace.argument import rawshape, ArgErr -from pypy.tool.sourcetools import valid_identifier -from pypy.tool.pairtype import extendabletype +from rpython.tool.sourcetools import valid_identifier +from rpython.tool.pairtype import extendabletype class CallFamily(object): """A family of Desc objects that could be called from common call sites. diff --git a/rpython/annotator/model.py b/rpython/annotator/model.py --- a/rpython/annotator/model.py +++ b/rpython/annotator/model.py @@ -32,7 +32,7 @@ from types import BuiltinFunctionType, MethodType, FunctionType import pypy from pypy.tool import descriptor -from pypy.tool.pairtype import pair, extendabletype +from rpython.tool.pairtype import pair, extendabletype from rpython.rlib.rarithmetic import r_uint, r_ulonglong, base_int from rpython.rlib.rarithmetic import r_singlefloat, r_longfloat import inspect, weakref diff --git a/rpython/annotator/specialize.py b/rpython/annotator/specialize.py --- a/rpython/annotator/specialize.py +++ b/rpython/annotator/specialize.py @@ -1,7 +1,7 @@ # specialization support import py from pypy.tool.uid import uid -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name from pypy.tool.algo.unionfind import UnionFind from rpython.flowspace.model import Block, Link, Variable, SpaceOperation from rpython.flowspace.model import Constant, checkgraph diff --git a/rpython/flowspace/generator.py b/rpython/flowspace/generator.py --- a/rpython/flowspace/generator.py +++ b/rpython/flowspace/generator.py @@ -5,7 +5,7 @@ from rpython.translator.unsimplify import insert_empty_startblock from rpython.translator.unsimplify import split_block from rpython.translator.simplify import eliminate_empty_blocks, simplify_graph -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name from rpython.flowspace.argument import Signature diff --git a/rpython/flowspace/model.py b/rpython/flowspace/model.py --- a/rpython/flowspace/model.py +++ b/rpython/flowspace/model.py @@ -5,7 +5,7 @@ # a discussion in Berlin, 4th of october 2003 import py from pypy.tool.uid import uid, Hashable -from pypy.tool.sourcetools import PY_IDENTIFIER, nice_repr_for_func +from rpython.tool.sourcetools import PY_IDENTIFIER, nice_repr_for_func from rpython.rlib.rarithmetic import is_valid_int, r_longlong, r_ulonglong, r_uint @@ -57,7 +57,7 @@ def source(self): if hasattr(self, "_source"): return self._source - from pypy.tool.sourcetools import getsource + from rpython.tool.sourcetools import getsource self.func # can raise AttributeError src = getsource(self.func) if src is None: diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py --- a/rpython/flowspace/operation.py +++ b/rpython/flowspace/operation.py @@ -6,7 +6,7 @@ import __builtin__ import __future__ import operator -from pypy.tool.sourcetools import compile2 +from rpython.tool.sourcetools import compile2 from rpython.rlib.rarithmetic import ovfcheck # this is a copy that should be shared with standard objspace diff --git a/rpython/jit/backend/cli/method.py b/rpython/jit/backend/cli/method.py --- a/rpython/jit/backend/cli/method.py +++ b/rpython/jit/backend/cli/method.py @@ -1,7 +1,7 @@ import py import os from rpython.rlib.debug import debug_start, debug_stop -from pypy.tool.pairtype import extendabletype +from rpython.tool.pairtype import extendabletype from rpython.rtyper.ootypesystem import ootype from rpython.translator.cli import dotnet from rpython.translator.cli.dotnet import CLR diff --git a/rpython/jit/backend/cli/runner.py b/rpython/jit/backend/cli/runner.py --- a/rpython/jit/backend/cli/runner.py +++ b/rpython/jit/backend/cli/runner.py @@ -1,4 +1,4 @@ -from pypy.tool.pairtype import extendabletype +from rpython.tool.pairtype import extendabletype from rpython.rtyper.ootypesystem import ootype from rpython.rlib.objectmodel import we_are_translated from rpython.jit.metainterp import history diff --git a/rpython/jit/backend/x86/jump.py b/rpython/jit/backend/x86/jump.py --- a/rpython/jit/backend/x86/jump.py +++ b/rpython/jit/backend/x86/jump.py @@ -1,5 +1,5 @@ import sys -from pypy.tool.pairtype import extendabletype +from rpython.tool.pairtype import extendabletype from rpython.jit.backend.x86.regloc import ImmediateAssemblerLocation, StackLoc def remap_frame_layout(assembler, src_locations, dst_locations, tmpreg): diff --git a/rpython/jit/backend/x86/regloc.py b/rpython/jit/backend/x86/regloc.py --- a/rpython/jit/backend/x86/regloc.py +++ b/rpython/jit/backend/x86/regloc.py @@ -2,7 +2,7 @@ from rpython.jit.backend.x86 import rx86 from rpython.rlib.unroll import unrolling_iterable from rpython.jit.backend.x86.arch import WORD, IS_X86_32, IS_X86_64 -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name from rpython.rlib.objectmodel import specialize, instantiate from rpython.rlib.rarithmetic import intmask from rpython.jit.metainterp.history import FLOAT diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -7,7 +7,7 @@ from rpython.rlib import rstack from rpython.rlib.jit import JitDebugInfo, Counters from pypy.conftest import option -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name from rpython.jit.metainterp.resoperation import ResOperation, rop, get_deep_immutable_oplist from rpython.jit.metainterp.history import TreeLoop, Box, History, JitCellToken, TargetToken diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py --- a/rpython/jit/metainterp/optimizeopt/optimizer.py +++ b/rpython/jit/metainterp/optimizeopt/optimizer.py @@ -7,7 +7,7 @@ from rpython.jit.metainterp.optimizeopt.util import make_dispatcher_method from rpython.jit.metainterp.resoperation import rop, ResOperation, AbstractResOp from rpython.jit.metainterp.typesystem import llhelper, oohelper -from pypy.tool.pairtype import extendabletype +from rpython.tool.pairtype import extendabletype from rpython.rlib.debug import debug_print from rpython.rlib.objectmodel import specialize diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py --- a/rpython/jit/metainterp/warmspot.py +++ b/rpython/jit/metainterp/warmspot.py @@ -1,5 +1,5 @@ import sys, py -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name from rpython.rtyper.lltypesystem import lltype, llmemory from rpython.rtyper.annlowlevel import llhelper, MixLevelHelperAnnotator,\ cast_base_ptr_to_instance, hlstr diff --git a/rpython/jit/metainterp/warmstate.py b/rpython/jit/metainterp/warmstate.py --- a/rpython/jit/metainterp/warmstate.py +++ b/rpython/jit/metainterp/warmstate.py @@ -12,7 +12,7 @@ from rpython.rlib.debug import debug_start, debug_stop, debug_print from rpython.jit.metainterp import history from rpython.jit.codewriter import support, heaptracker, longlong -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name # ____________________________________________________________ diff --git a/rpython/rlib/_jit_vref.py b/rpython/rlib/_jit_vref.py --- a/rpython/rlib/_jit_vref.py +++ b/rpython/rlib/_jit_vref.py @@ -1,5 +1,5 @@ from rpython.annotator import model as annmodel -from pypy.tool.pairtype import pairtype +from rpython.tool.pairtype import pairtype from rpython.rtyper.extregistry import ExtRegistryEntry from rpython.rtyper.rclass import getinstancerepr from rpython.rtyper.rmodel import Repr diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -6,7 +6,7 @@ from rpython.rlib.objectmodel import CDefinedIntSymbolic, keepalive_until_here, specialize from rpython.rlib.unroll import unrolling_iterable from rpython.rtyper.extregistry import ExtRegistryEntry -from pypy.tool.sourcetools import rpython_wrapper +from rpython.tool.sourcetools import rpython_wrapper DEBUG_ELIDABLE_FUNCTIONS = False diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py --- a/rpython/rlib/objectmodel.py +++ b/rpython/rlib/objectmodel.py @@ -10,7 +10,7 @@ import types import math import inspect -from pypy.tool.sourcetools import rpython_wrapper +from rpython.tool.sourcetools import rpython_wrapper # specialize is a decorator factory for attaching _annspecialcase_ # attributes to functions: for example diff --git a/rpython/rlib/parsing/parsing.py b/rpython/rlib/parsing/parsing.py --- a/rpython/rlib/parsing/parsing.py +++ b/rpython/rlib/parsing/parsing.py @@ -219,7 +219,7 @@ self.made = {} def compile(self): - from pypy.tool.sourcetools import func_with_new_name + from rpython.tool.sourcetools import func_with_new_name self.allcode.append("class CompileableParser(baseclass):") self.make_matcher(self.parser.startsymbol) self.make_fixed() diff --git a/rpython/rlib/rerased.py b/rpython/rlib/rerased.py --- a/rpython/rlib/rerased.py +++ b/rpython/rlib/rerased.py @@ -16,7 +16,7 @@ import sys from rpython.annotator import model as annmodel -from pypy.tool.pairtype import pairtype +from rpython.tool.pairtype import pairtype from rpython.rtyper.extregistry import ExtRegistryEntry from rpython.rtyper.rclass import getinstancerepr from rpython.rtyper.rmodel import Repr diff --git a/rpython/rlib/rmarshal.py b/rpython/rlib/rmarshal.py --- a/rpython/rlib/rmarshal.py +++ b/rpython/rlib/rmarshal.py @@ -5,7 +5,7 @@ from rpython.annotator import model as annmodel from rpython.annotator.signature import annotation from rpython.annotator.listdef import ListDef, TooLateForChange -from pypy.tool.pairtype import pair, pairtype +from rpython.tool.pairtype import pair, pairtype from rpython.rlib.rarithmetic import r_longlong, intmask, LONG_BIT from rpython.rlib.rfloat import formatd, rstring_to_float from rpython.rlib.unroll import unrolling_iterable diff --git a/rpython/rlib/rsre/rsre_core.py b/rpython/rlib/rsre/rsre_core.py --- a/rpython/rlib/rsre/rsre_core.py +++ b/rpython/rlib/rsre/rsre_core.py @@ -2,7 +2,7 @@ from rpython.rlib.debug import check_nonneg from rpython.rlib.unroll import unrolling_iterable from rpython.rlib.rsre import rsre_char -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name from rpython.rlib.objectmodel import we_are_translated from rpython.rlib import jit from rpython.rlib.rsre.rsre_jit import install_jitdriver, install_jitdriver_spec diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py --- a/rpython/rlib/rstring.py +++ b/rpython/rlib/rstring.py @@ -4,7 +4,7 @@ from rpython.annotator.model import (SomeObject, SomeString, s_None, SomeChar, SomeInteger, SomeUnicodeCodePoint, SomeUnicodeString, SomePtr, SomePBC) from rpython.rlib.rarithmetic import ovfcheck -from pypy.tool.pairtype import pair, pairtype +from rpython.tool.pairtype import pair, pairtype from rpython.rtyper.extregistry import ExtRegistryEntry diff --git a/rpython/rlib/rweakref.py b/rpython/rlib/rweakref.py --- a/rpython/rlib/rweakref.py +++ b/rpython/rlib/rweakref.py @@ -69,7 +69,7 @@ from rpython.rtyper import extregistry from rpython.annotator import model as annmodel from rpython.annotator.bookkeeper import getbookkeeper -from pypy.tool.pairtype import pairtype +from rpython.tool.pairtype import pairtype class Entry(extregistry.ExtRegistryEntry): _about_ = has_weakref_support diff --git a/rpython/rtyper/annlowlevel.py b/rpython/rtyper/annlowlevel.py --- a/rpython/rtyper/annlowlevel.py +++ b/rpython/rtyper/annlowlevel.py @@ -3,7 +3,7 @@ """ import types -from pypy.tool.sourcetools import valid_identifier +from rpython.tool.sourcetools import valid_identifier from rpython.annotator import model as annmodel from rpython.annotator.policy import AnnotatorPolicy, Sig from rpython.annotator.specialize import flatten_star_args diff --git a/rpython/rtyper/controllerentry.py b/rpython/rtyper/controllerentry.py --- a/rpython/rtyper/controllerentry.py +++ b/rpython/rtyper/controllerentry.py @@ -1,5 +1,5 @@ from rpython.annotator import model as annmodel -from pypy.tool.pairtype import pairtype +from rpython.tool.pairtype import pairtype from rpython.annotator.binaryop import _make_none_union, SomePBC # SomePBC needed by _make_none_union from rpython.annotator.bookkeeper import getbookkeeper from rpython.rtyper.extregistry import ExtRegistryEntry diff --git a/rpython/rtyper/extfunc.py b/rpython/rtyper/extfunc.py --- a/rpython/rtyper/extfunc.py +++ b/rpython/rtyper/extfunc.py @@ -188,7 +188,7 @@ if hasattr(self, fake_method_name): # If we have both an {ll,oo}impl and a {ll,oo}fakeimpl, # we need a wrapper that selects the proper one and calls it - from pypy.tool.sourcetools import func_with_new_name + from rpython.tool.sourcetools import func_with_new_name # Using '*args' is delicate because this wrapper is also # created for init-time functions like llarena.arena_malloc # which are called before the GC is fully initialized diff --git a/rpython/rtyper/extfuncregistry.py b/rpython/rtyper/extfuncregistry.py --- a/rpython/rtyper/extfuncregistry.py +++ b/rpython/rtyper/extfuncregistry.py @@ -71,7 +71,7 @@ # ___________________________ # os.path functions -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name import os.path # os.path.join is RPython, but we don't want to compile it directly diff --git a/rpython/rtyper/lltypesystem/module/ll_math.py b/rpython/rtyper/lltypesystem/module/ll_math.py --- a/rpython/rtyper/lltypesystem/module/ll_math.py +++ b/rpython/rtyper/lltypesystem/module/ll_math.py @@ -4,7 +4,7 @@ import sys from rpython.rtyper.lltypesystem import lltype, rffi -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name from pypy.tool.autopath import pypydir from rpython.rlib import jit, rposix from rpython.translator.tool.cbuild import ExternalCompilationInfo diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py --- a/rpython/rtyper/lltypesystem/opimpl.py +++ b/rpython/rtyper/lltypesystem/opimpl.py @@ -2,7 +2,7 @@ from rpython.rlib import debug from rpython.rlib.rarithmetic import is_valid_int from rpython.rtyper.lltypesystem import lltype, llmemory -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name # ____________________________________________________________ # Implementation of the 'canfold' operations diff --git a/rpython/rtyper/lltypesystem/rbuilder.py b/rpython/rtyper/lltypesystem/rbuilder.py --- a/rpython/rtyper/lltypesystem/rbuilder.py +++ b/rpython/rtyper/lltypesystem/rbuilder.py @@ -7,7 +7,7 @@ from rpython.rtyper.lltypesystem.rstr import (STR, UNICODE, char_repr, string_repr, unichar_repr, unicode_repr) from rpython.rtyper.rbuilder import AbstractStringBuilderRepr -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name # Think about heuristics below, maybe we can come up with something # better or at least compare it with list heuristics diff --git a/rpython/rtyper/lltypesystem/rclass.py b/rpython/rtyper/lltypesystem/rclass.py --- a/rpython/rtyper/lltypesystem/rclass.py +++ b/rpython/rtyper/lltypesystem/rclass.py @@ -1,6 +1,6 @@ import sys import types -from pypy.tool.pairtype import pairtype, pair +from rpython.tool.pairtype import pairtype, pair from rpython.flowspace.model import Constant from rpython.rtyper.error import TyperError from rpython.rtyper.rmodel import Repr, inputconst, warning, mangle diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py --- a/rpython/rtyper/lltypesystem/rdict.py +++ b/rpython/rtyper/lltypesystem/rdict.py @@ -1,4 +1,4 @@ -from pypy.tool.pairtype import pairtype +from rpython.tool.pairtype import pairtype from rpython.flowspace.model import Constant from rpython.rtyper.rdict import (AbstractDictRepr, AbstractDictIteratorRepr, rtype_newdict) diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py --- a/rpython/rtyper/lltypesystem/rffi.py +++ b/rpython/rtyper/lltypesystem/rffi.py @@ -5,7 +5,7 @@ from rpython.rtyper.lltypesystem.llmemory import cast_adr_to_ptr, cast_ptr_to_adr from rpython.rtyper.lltypesystem.llmemory import itemoffsetof, raw_memcopy from rpython.annotator.model import lltype_to_annotation -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name from rpython.rlib.objectmodel import Symbolic, CDefinedIntSymbolic from rpython.rlib.objectmodel import keepalive_until_here from rpython.rlib import rarithmetic, rgc diff --git a/rpython/rtyper/lltypesystem/rlist.py b/rpython/rtyper/lltypesystem/rlist.py --- a/rpython/rtyper/lltypesystem/rlist.py +++ b/rpython/rtyper/lltypesystem/rlist.py @@ -8,7 +8,7 @@ AbstractFixedSizeListRepr, AbstractListIteratorRepr, ll_setitem_nonneg, ADTIList, ADTIFixedList, dum_nocheck) from rpython.rtyper.rmodel import Repr, inputconst, externalvsinternal -from pypy.tool.pairtype import pairtype, pair +from rpython.tool.pairtype import pairtype, pair # ____________________________________________________________ diff --git a/rpython/rtyper/lltypesystem/rpbc.py b/rpython/rtyper/lltypesystem/rpbc.py --- a/rpython/rtyper/lltypesystem/rpbc.py +++ b/rpython/rtyper/lltypesystem/rpbc.py @@ -1,6 +1,6 @@ import types import sys -from pypy.tool.pairtype import pairtype, pair +from rpython.tool.pairtype import pairtype, pair from rpython.annotator import model as annmodel from rpython.annotator import description from rpython.flowspace.model import Constant, Variable @@ -15,7 +15,7 @@ AbstractFunctionsPBCRepr, AbstractMultipleUnrelatedFrozenPBCRepr, \ SingleFrozenPBCRepr, none_frozen_pbc_repr, get_concrete_calltable from rpython.rtyper.lltypesystem import rclass, llmemory -from pypy.tool.sourcetools import has_varargs +from rpython.tool.sourcetools import has_varargs from rpython.rlib.unroll import unrolling_iterable from rpython.rlib.debug import ll_assert diff --git a/rpython/rtyper/lltypesystem/rstr.py b/rpython/rtyper/lltypesystem/rstr.py --- a/rpython/rtyper/lltypesystem/rstr.py +++ b/rpython/rtyper/lltypesystem/rstr.py @@ -18,7 +18,7 @@ staticAdtMethod, GcForwardReference, malloc from rpython.rtyper.rmodel import Repr from rpython.rtyper.lltypesystem import llmemory -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name # ____________________________________________________________ # diff --git a/rpython/rtyper/lltypesystem/rtuple.py b/rpython/rtyper/lltypesystem/rtuple.py --- a/rpython/rtyper/lltypesystem/rtuple.py +++ b/rpython/rtyper/lltypesystem/rtuple.py @@ -1,4 +1,4 @@ -from pypy.tool.pairtype import pairtype +from rpython.tool.pairtype import pairtype from rpython.rtyper.rmodel import inputconst from rpython.rtyper.rtuple import AbstractTupleRepr, AbstractTupleIteratorRepr from rpython.rtyper.lltypesystem.lltype import \ diff --git a/rpython/rtyper/memory/gc/minimark.py b/rpython/rtyper/memory/gc/minimark.py --- a/rpython/rtyper/memory/gc/minimark.py +++ b/rpython/rtyper/memory/gc/minimark.py @@ -53,7 +53,7 @@ from rpython.rlib.rarithmetic import LONG_BIT_SHIFT from rpython.rlib.debug import ll_assert, debug_print, debug_start, debug_stop from rpython.rlib.objectmodel import we_are_translated -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name # # Handles the objects in 2 generations: diff --git a/rpython/rtyper/memory/gctransform/framework.py b/rpython/rtyper/memory/gctransform/framework.py --- a/rpython/rtyper/memory/gctransform/framework.py +++ b/rpython/rtyper/memory/gctransform/framework.py @@ -9,7 +9,7 @@ from rpython.rtyper.memory.gctransform.transform import GCTransformer from rpython.rtyper.memory.gctypelayout import ll_weakref_deref, WEAKREF, \ WEAKREFPTR -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name from rpython.translator.backendopt import graphanalyze from rpython.translator.backendopt.finalizer import FinalizerAnalyzer from rpython.translator.backendopt.support import var_needsgc diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py --- a/rpython/rtyper/module/ll_os.py +++ b/rpython/rtyper/module/ll_os.py @@ -8,7 +8,7 @@ import os, sys, errno import py from rpython.rtyper.module.support import OOSupport -from pypy.tool.sourcetools import func_renamer +from rpython.tool.sourcetools import func_renamer from rpython.rlib.rarithmetic import r_longlong from rpython.rtyper.extfunc import ( BaseLazyRegistering, register_external) diff --git a/rpython/rtyper/module/ll_os_stat.py b/rpython/rtyper/module/ll_os_stat.py --- a/rpython/rtyper/module/ll_os_stat.py +++ b/rpython/rtyper/module/ll_os_stat.py @@ -4,8 +4,8 @@ """ import os, sys from rpython.annotator import model as annmodel -from pypy.tool.pairtype import pairtype -from pypy.tool.sourcetools import func_with_new_name, func_renamer +from rpython.tool.pairtype import pairtype +from rpython.tool.sourcetools import func_with_new_name, func_renamer from rpython.rtyper import extregistry from rpython.rtyper.extfunc import register_external, extdef from rpython.rtyper.lltypesystem import rffi, lltype diff --git a/rpython/rtyper/module/ll_win32file.py b/rpython/rtyper/module/ll_win32file.py --- a/rpython/rtyper/module/ll_win32file.py +++ b/rpython/rtyper/module/ll_win32file.py @@ -7,7 +7,7 @@ from rpython.rtyper.lltypesystem import lltype, rffi from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.rtyper.tool import rffi_platform as platform -from pypy.tool.sourcetools import func_renamer +from rpython.tool.sourcetools import func_renamer from rpython.rlib.objectmodel import specialize def make_win32_traits(traits): diff --git a/rpython/rtyper/module/r_os_stat.py b/rpython/rtyper/module/r_os_stat.py --- a/rpython/rtyper/module/r_os_stat.py +++ b/rpython/rtyper/module/r_os_stat.py @@ -9,7 +9,7 @@ """ from rpython.annotator import model as annmodel from rpython.flowspace.model import Constant -from pypy.tool.pairtype import pairtype +from rpython.tool.pairtype import pairtype from rpython.rtyper.rmodel import Repr, IntegerRepr from rpython.rtyper.error import TyperError from rpython.rtyper.module import ll_os_stat diff --git a/rpython/rtyper/normalizecalls.py b/rpython/rtyper/normalizecalls.py --- a/rpython/rtyper/normalizecalls.py +++ b/rpython/rtyper/normalizecalls.py @@ -5,7 +5,7 @@ from rpython.rlib.objectmodel import ComputedIntSymbolic from rpython.rtyper.error import TyperError from rpython.rtyper.rmodel import getgcflavor -from pypy.tool.sourcetools import valid_identifier +from rpython.tool.sourcetools import valid_identifier def normalize_call_familes(annotator): diff --git a/rpython/rtyper/ootypesystem/rclass.py b/rpython/rtyper/ootypesystem/rclass.py --- a/rpython/rtyper/ootypesystem/rclass.py +++ b/rpython/rtyper/ootypesystem/rclass.py @@ -8,8 +8,8 @@ getinstancerepr, getclassrepr, get_type_repr from rpython.rtyper.ootypesystem import ootype from rpython.rtyper.exceptiondata import standardexceptions -from pypy.tool.pairtype import pairtype -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.pairtype import pairtype +from rpython.tool.sourcetools import func_with_new_name from pypy.tool.identity_dict import identity_dict OBJECT = ootype.ROOT diff --git a/rpython/rtyper/ootypesystem/rdict.py b/rpython/rtyper/ootypesystem/rdict.py --- a/rpython/rtyper/ootypesystem/rdict.py +++ b/rpython/rtyper/ootypesystem/rdict.py @@ -1,5 +1,5 @@ from rpython.rtyper.error import TyperError -from pypy.tool.pairtype import pairtype +from rpython.tool.pairtype import pairtype from rpython.flowspace.model import Constant from rpython.rtyper.rdict import AbstractDictRepr, AbstractDictIteratorRepr from rpython.rtyper.rpbc import (MethodOfFrozenPBCRepr, AbstractFunctionsPBCRepr, diff --git a/rpython/rtyper/ootypesystem/rlist.py b/rpython/rtyper/ootypesystem/rlist.py --- a/rpython/rtyper/ootypesystem/rlist.py +++ b/rpython/rtyper/ootypesystem/rlist.py @@ -1,4 +1,4 @@ -from pypy.tool.pairtype import pairtype +from rpython.tool.pairtype import pairtype from rpython.rtyper.rlist import AbstractBaseListRepr, AbstractListRepr, \ AbstractListIteratorRepr, AbstractFixedSizeListRepr, rtype_newlist, rtype_alloc_and_set from rpython.rtyper.rmodel import Repr, IntegerRepr diff --git a/rpython/rtyper/ootypesystem/rootype.py b/rpython/rtyper/ootypesystem/rootype.py --- a/rpython/rtyper/ootypesystem/rootype.py +++ b/rpython/rtyper/ootypesystem/rootype.py @@ -3,7 +3,7 @@ from rpython.rtyper.rmodel import Repr from rpython.rtyper.ootypesystem import ootype from rpython.rtyper.ootypesystem.ootype import Void, Class, Object -from pypy.tool.pairtype import pairtype +from rpython.tool.pairtype import pairtype class __extend__(annmodel.SomeOOObject): def rtyper_makerepr(self, rtyper): diff --git a/rpython/rtyper/ootypesystem/rpbc.py b/rpython/rtyper/ootypesystem/rpbc.py --- a/rpython/rtyper/ootypesystem/rpbc.py +++ b/rpython/rtyper/ootypesystem/rpbc.py @@ -12,7 +12,7 @@ from rpython.rtyper.ootypesystem.rclass import mangle, META from rpython.annotator import model as annmodel from rpython.annotator import description -from pypy.tool.pairtype import pairtype +from rpython.tool.pairtype import pairtype from rpython.flowspace.model import Constant, Variable import types diff --git a/rpython/rtyper/ootypesystem/rstr.py b/rpython/rtyper/ootypesystem/rstr.py --- a/rpython/rtyper/ootypesystem/rstr.py +++ b/rpython/rtyper/ootypesystem/rstr.py @@ -1,4 +1,4 @@ -from pypy.tool.pairtype import pairtype +from rpython.tool.pairtype import pairtype from rpython.annotator import model as annmodel from rpython.rlib.objectmodel import specialize from rpython.rlib.rarithmetic import ovfcheck diff --git a/rpython/rtyper/raddress.py b/rpython/rtyper/raddress.py --- a/rpython/rtyper/raddress.py +++ b/rpython/rtyper/raddress.py @@ -1,5 +1,5 @@ # rtyping of memory address operations -from pypy.tool.pairtype import pairtype +from rpython.tool.pairtype import pairtype from rpython.annotator import model as annmodel from rpython.rtyper.lltypesystem.llmemory import NULL, Address, \ cast_adr_to_int, fakeaddress, sizeof diff --git a/rpython/rtyper/rbool.py b/rpython/rtyper/rbool.py --- a/rpython/rtyper/rbool.py +++ b/rpython/rtyper/rbool.py @@ -1,4 +1,4 @@ -from pypy.tool.pairtype import pairtype +from rpython.tool.pairtype import pairtype from rpython.annotator import model as annmodel from rpython.rtyper.lltypesystem.lltype import Signed, Unsigned, Bool, Float from rpython.rtyper.error import TyperError diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py --- a/rpython/rtyper/rbuiltin.py +++ b/rpython/rtyper/rbuiltin.py @@ -5,7 +5,7 @@ from rpython.rtyper.error import TyperError from rpython.rtyper.lltypesystem import lltype, llmemory from rpython.rtyper.rmodel import Repr -from pypy.tool.pairtype import pairtype +from rpython.tool.pairtype import pairtype class __extend__(annmodel.SomeBuiltin): diff --git a/rpython/rtyper/rbytearray.py b/rpython/rtyper/rbytearray.py --- a/rpython/rtyper/rbytearray.py +++ b/rpython/rtyper/rbytearray.py @@ -1,6 +1,6 @@ from rpython.annotator import model as annmodel -from pypy.tool.pairtype import pairtype +from rpython.tool.pairtype import pairtype from rpython.rtyper.rstr import AbstractStringRepr from rpython.rtyper.rmodel import IntegerRepr from rpython.rtyper.lltypesystem import lltype diff --git a/rpython/rtyper/rcontrollerentry.py b/rpython/rtyper/rcontrollerentry.py --- a/rpython/rtyper/rcontrollerentry.py +++ b/rpython/rtyper/rcontrollerentry.py @@ -1,4 +1,4 @@ -from pypy.tool.pairtype import pairtype +from rpython.tool.pairtype import pairtype from rpython.flowspace.model import Constant from rpython.rtyper.rmodel import Repr from rpython.rtyper.error import TyperError diff --git a/rpython/rtyper/rexternalobj.py b/rpython/rtyper/rexternalobj.py --- a/rpython/rtyper/rexternalobj.py +++ b/rpython/rtyper/rexternalobj.py @@ -6,7 +6,7 @@ from rpython.flowspace.model import Constant, Variable from rpython.rtyper import extregistry from rpython.annotator.signature import annotation -from pypy.tool.pairtype import pairtype +from rpython.tool.pairtype import pairtype # ExternalObjects diff --git a/rpython/rtyper/rfloat.py b/rpython/rtyper/rfloat.py --- a/rpython/rtyper/rfloat.py +++ b/rpython/rtyper/rfloat.py @@ -1,4 +1,4 @@ -from pypy.tool.pairtype import pairtype +from rpython.tool.pairtype import pairtype from rpython.annotator import model as annmodel from rpython.rtyper.lltypesystem.lltype import (Signed, Unsigned, SignedLongLong, UnsignedLongLong, Bool, Float) diff --git a/rpython/rtyper/rgeneric.py b/rpython/rtyper/rgeneric.py --- a/rpython/rtyper/rgeneric.py +++ b/rpython/rtyper/rgeneric.py @@ -2,7 +2,7 @@ from rpython.rtyper.rmodel import Repr from rpython.rtyper.rpbc import AbstractFunctionsPBCRepr,\ AbstractMethodsPBCRepr -from pypy.tool.pairtype import pairtype +from rpython.tool.pairtype import pairtype from rpython.rtyper.lltypesystem import lltype class AbstractGenericCallableRepr(Repr): diff --git a/rpython/rtyper/rint.py b/rpython/rtyper/rint.py --- a/rpython/rtyper/rint.py +++ b/rpython/rtyper/rint.py @@ -1,5 +1,5 @@ import sys -from pypy.tool.pairtype import pairtype +from rpython.tool.pairtype import pairtype from rpython.annotator import model as annmodel from rpython.flowspace.operation import op_appendices from rpython.rtyper.lltypesystem.lltype import Signed, Unsigned, Bool, Float, \ diff --git a/rpython/rtyper/rlist.py b/rpython/rtyper/rlist.py --- a/rpython/rtyper/rlist.py +++ b/rpython/rtyper/rlist.py @@ -1,4 +1,4 @@ -from pypy.tool.pairtype import pairtype, pair +from rpython.tool.pairtype import pairtype, pair from rpython.flowspace.model import Constant from rpython.annotator import model as annmodel from rpython.rtyper.error import TyperError diff --git a/rpython/rtyper/rmodel.py b/rpython/rtyper/rmodel.py --- a/rpython/rtyper/rmodel.py +++ b/rpython/rtyper/rmodel.py @@ -1,4 +1,4 @@ -from pypy.tool.pairtype import pairtype, extendabletype, pair +from rpython.tool.pairtype import pairtype, extendabletype, pair from rpython.annotator import model as annmodel, unaryop, binaryop from rpython.annotator import description from rpython.flowspace.model import Constant diff --git a/rpython/rtyper/rpbc.py b/rpython/rtyper/rpbc.py --- a/rpython/rtyper/rpbc.py +++ b/rpython/rtyper/rpbc.py @@ -1,6 +1,6 @@ import types import sys -from pypy.tool.pairtype import pair, pairtype +from rpython.tool.pairtype import pair, pairtype from rpython.annotator import model as annmodel from rpython.annotator import description from rpython.flowspace.model import Constant diff --git a/rpython/rtyper/rptr.py b/rpython/rtyper/rptr.py --- a/rpython/rtyper/rptr.py +++ b/rpython/rtyper/rptr.py @@ -1,4 +1,4 @@ -from pypy.tool.pairtype import pairtype +from rpython.tool.pairtype import pairtype from rpython.annotator import model as annmodel from rpython.flowspace import model as flowmodel from rpython.rtyper.lltypesystem import lltype diff --git a/rpython/rtyper/rrange.py b/rpython/rtyper/rrange.py --- a/rpython/rtyper/rrange.py +++ b/rpython/rtyper/rrange.py @@ -1,4 +1,4 @@ -from pypy.tool.pairtype import pairtype +from rpython.tool.pairtype import pairtype from rpython.rtyper.error import TyperError from rpython.rtyper.lltypesystem.lltype import Signed, Void, Ptr from rpython.rtyper.rmodel import Repr, IntegerRepr, IteratorRepr diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py --- a/rpython/rtyper/rstr.py +++ b/rpython/rtyper/rstr.py @@ -1,6 +1,6 @@ from pypy.tool.staticmethods import StaticMethods -from pypy.tool.pairtype import pairtype, pair -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.pairtype import pairtype, pair +from rpython.tool.sourcetools import func_with_new_name from rpython.annotator import model as annmodel from rpython.rlib import jit from rpython.rlib.nonconst import NonConstant diff --git a/rpython/rtyper/rtuple.py b/rpython/rtyper/rtuple.py --- a/rpython/rtyper/rtuple.py +++ b/rpython/rtyper/rtuple.py @@ -1,5 +1,5 @@ import operator -from pypy.tool.pairtype import pairtype +from rpython.tool.pairtype import pairtype from rpython.annotator import model as annmodel from rpython.flowspace.model import Constant from rpython.rtyper.error import TyperError diff --git a/rpython/rtyper/rtyper.py b/rpython/rtyper/rtyper.py --- a/rpython/rtyper/rtyper.py +++ b/rpython/rtyper/rtyper.py @@ -13,7 +13,7 @@ import os import py -from pypy.tool.pairtype import pair +from rpython.tool.pairtype import pair from rpython.annotator import model as annmodel, unaryop, binaryop from rpython.annotator.annrpython import FAIL from rpython.flowspace.model import Variable, Constant diff --git a/rpython/rtyper/typesystem.py b/rpython/rtyper/typesystem.py --- a/rpython/rtyper/typesystem.py +++ b/rpython/rtyper/typesystem.py @@ -1,7 +1,7 @@ """typesystem.py -- Typesystem-specific operations for RTyper.""" -from pypy.tool.pairtype import extendabletype +from rpython.tool.pairtype import extendabletype from rpython.rtyper.ootypesystem import ootype from rpython.rtyper.lltypesystem import lltype @@ -189,7 +189,7 @@ # Multiple dispatch on type system and high-level annotation -from pypy.tool.pairtype import pairtype +from rpython.tool.pairtype import pairtype from rpython.annotator.model import SomeObject class __extend__(pairtype(TypeSystem, SomeObject)): diff --git a/pypy/tool/pairtype.py b/rpython/tool/pairtype.py rename from pypy/tool/pairtype.py rename to rpython/tool/pairtype.py diff --git a/pypy/tool/sourcetools.py b/rpython/tool/sourcetools.py rename from pypy/tool/sourcetools.py rename to rpython/tool/sourcetools.py diff --git a/rpython/translator/c/database.py b/rpython/translator/c/database.py --- a/rpython/translator/c/database.py +++ b/rpython/translator/c/database.py @@ -6,7 +6,7 @@ from rpython.rtyper.lltypesystem.llmemory import WeakRef, _WeakRefType, GCREF from rpython.rtyper.lltypesystem.rffi import CConstant from rpython.rtyper.lltypesystem import llgroup -from pypy.tool.sourcetools import valid_identifier +from rpython.tool.sourcetools import valid_identifier from rpython.translator.c.primitive import PrimitiveName, PrimitiveType from rpython.translator.c.node import StructDefNode, ArrayDefNode from rpython.translator.c.node import FixedSizeArrayDefNode, BareBoneArrayDefNode diff --git a/rpython/translator/c/test/test_lltyped.py b/rpython/translator/c/test/test_lltyped.py --- a/rpython/translator/c/test/test_lltyped.py +++ b/rpython/translator/c/test/test_lltyped.py @@ -1,7 +1,7 @@ import py from rpython.rtyper.lltypesystem.lltype import * from rpython.translator.c.test.test_genc import compile -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name class TestLowLevelType(object): diff --git a/rpython/translator/cli/dotnet.py b/rpython/translator/cli/dotnet.py --- a/rpython/translator/cli/dotnet.py +++ b/rpython/translator/cli/dotnet.py @@ -1,6 +1,6 @@ import types -from pypy.tool.pairtype import pair, pairtype +from rpython.tool.pairtype import pair, pairtype from rpython.annotator.model import SomeObject, SomeInstance, SomeOOInstance, SomeInteger, s_None,\ s_ImpossibleValue, lltype_to_annotation, annotation_to_lltype, SomeChar, SomeString, SomeOOStaticMeth from rpython.annotator.unaryop import immutablevalue diff --git a/rpython/translator/exceptiontransform.py b/rpython/translator/exceptiontransform.py --- a/rpython/translator/exceptiontransform.py +++ b/rpython/translator/exceptiontransform.py @@ -15,7 +15,7 @@ from rpython.rlib.debug import ll_assert from rpython.annotator import model as annmodel from rpython.rtyper.annlowlevel import MixLevelHelperAnnotator -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name PrimitiveErrorValue = {lltype.Signed: -1, lltype.Unsigned: r_uint(-1), diff --git a/rpython/translator/sandbox/rsandbox.py b/rpython/translator/sandbox/rsandbox.py --- a/rpython/translator/sandbox/rsandbox.py +++ b/rpython/translator/sandbox/rsandbox.py @@ -14,7 +14,7 @@ from rpython.annotator import model as annmodel from rpython.rlib.unroll import unrolling_iterable from rpython.rlib.objectmodel import CDefinedIntSymbolic -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name from rpython.rtyper.annlowlevel import MixLevelHelperAnnotator from pypy.tool.ansi_print import ansi_log import py diff --git a/rpython/translator/sandbox/test/test_sandlib.py b/rpython/translator/sandbox/test/test_sandlib.py --- a/rpython/translator/sandbox/test/test_sandlib.py +++ b/rpython/translator/sandbox/test/test_sandlib.py @@ -1,6 +1,6 @@ import py import errno, os, StringIO -from pypy.tool.sourcetools import func_with_new_name +from rpython.tool.sourcetools import func_with_new_name from rpython.rtyper.lltypesystem import rffi from rpython.translator.sandbox.sandlib import SandboxedProc from rpython.translator.sandbox.sandlib import SimpleIOSandboxedProc diff --git a/rpython/translator/tool/pdbplus.py b/rpython/translator/tool/pdbplus.py --- a/rpython/translator/tool/pdbplus.py +++ b/rpython/translator/tool/pdbplus.py @@ -422,7 +422,7 @@ def pdbcatch(f): "A decorator that throws you in a pdbplus if the given function raises." - from pypy.tool.sourcetools import func_with_new_name + from rpython.tool.sourcetools import func_with_new_name def wrapper(*args, **kwds): try: return f(*args, **kwds) diff --git a/rpython/translator/translator.py b/rpython/translator/translator.py --- a/rpython/translator/translator.py +++ b/rpython/translator/translator.py @@ -10,7 +10,7 @@ from rpython.flowspace.model import FunctionGraph, checkgraph, Block from rpython.flowspace.objspace import FlowObjSpace from pypy.tool.ansi_print import ansi_log -from pypy.tool.sourcetools import nice_repr_for_func +from rpython.tool.sourcetools import nice_repr_for_func from pypy.config.pypyoption import pypy_optiondescription from pypy.config.translationoption import get_combined_translation_config from pypy.config.translationoption import get_platform From noreply at buildbot.pypy.org Fri Jan 4 22:31:01 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Fri, 4 Jan 2013 22:31:01 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Changed autopath.py in rpython to search for rpython directory Message-ID: <20130104213101.6C8461C01EE@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59703:dfa35f50ad74 Date: 2013-01-02 17:47 +0100 http://bitbucket.org/pypy/pypy/changeset/dfa35f50ad74/ Log: Changed autopath.py in rpython to search for rpython directory diff --git a/rpython/annotator/test/autopath.py b/rpython/annotator/test/autopath.py --- a/rpython/annotator/test/autopath.py +++ b/rpython/annotator/test/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - pypydir pypy root directory path + rpythondir rpython root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) + "'%s'" % join(rpythondir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) + s = open(join(rpythondir, 'tool', _myname), 'rb').read() + walk(rpythondir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -pypydir, this_dir = __dirinfo('rpython') +rpythondir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/jit/backend/autopath.py b/rpython/jit/backend/autopath.py --- a/rpython/jit/backend/autopath.py +++ b/rpython/jit/backend/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - pypydir pypy root directory path + rpythondir rpython root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) + "'%s'" % join(rpythondir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) + s = open(join(rpythondir, 'tool', _myname), 'rb').read() + walk(rpythondir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -pypydir, this_dir = __dirinfo('rpython') +rpythondir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/jit/backend/x86/autopath.py b/rpython/jit/backend/x86/autopath.py --- a/rpython/jit/backend/x86/autopath.py +++ b/rpython/jit/backend/x86/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - pypydir pypy root directory path + rpythondir rpython root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) + "'%s'" % join(rpythondir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) + s = open(join(rpythondir, 'tool', _myname), 'rb').read() + walk(rpythondir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -pypydir, this_dir = __dirinfo('rpython') +rpythondir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/jit/backend/x86/tool/autopath.py b/rpython/jit/backend/x86/tool/autopath.py --- a/rpython/jit/backend/x86/tool/autopath.py +++ b/rpython/jit/backend/x86/tool/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - pypydir pypy root directory path + rpythondir rpython root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) + "'%s'" % join(rpythondir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) + s = open(join(rpythondir, 'tool', _myname), 'rb').read() + walk(rpythondir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -pypydir, this_dir = __dirinfo('rpython') +rpythondir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/jit/tl/autopath.py b/rpython/jit/tl/autopath.py --- a/rpython/jit/tl/autopath.py +++ b/rpython/jit/tl/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - pypydir pypy root directory path + rpythondir rpython root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) + "'%s'" % join(rpythondir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) + s = open(join(rpythondir, 'tool', _myname), 'rb').read() + walk(rpythondir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -pypydir, this_dir = __dirinfo('rpython') +rpythondir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/jit/tl/spli/autopath.py b/rpython/jit/tl/spli/autopath.py --- a/rpython/jit/tl/spli/autopath.py +++ b/rpython/jit/tl/spli/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - pypydir pypy root directory path + rpythondir rpython root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) + "'%s'" % join(rpythondir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) + s = open(join(rpythondir, 'tool', _myname), 'rb').read() + walk(rpythondir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -pypydir, this_dir = __dirinfo('rpython') +rpythondir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/jit/tool/autopath.py b/rpython/jit/tool/autopath.py --- a/rpython/jit/tool/autopath.py +++ b/rpython/jit/tool/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - pypydir pypy root directory path + rpythondir rpython root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) + "'%s'" % join(rpythondir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) + s = open(join(rpythondir, 'tool', _myname), 'rb').read() + walk(rpythondir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -pypydir, this_dir = __dirinfo('rpython') +rpythondir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/rlib/_rffi_stacklet.py b/rpython/rlib/_rffi_stacklet.py --- a/rpython/rlib/_rffi_stacklet.py +++ b/rpython/rlib/_rffi_stacklet.py @@ -1,5 +1,5 @@ import py -from pypy.tool.autopath import pypydir +from rpython.tool.autopath import rpythondir from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.rtyper.tool import rffi_platform @@ -7,7 +7,7 @@ import sys -cdir = py.path.local(pypydir) / 'translator' / 'c' +cdir = py.path.local(rpythondir) / 'translator' / 'c' eci = ExternalCompilationInfo( include_dirs = [cdir], diff --git a/rpython/rlib/clibffi.py b/rpython/rlib/clibffi.py --- a/rpython/rlib/clibffi.py +++ b/rpython/rlib/clibffi.py @@ -12,7 +12,7 @@ from rpython.rlib.rdynload import DLOpenError, DLLHANDLE from rpython.rlib import jit from rpython.rlib.objectmodel import specialize -from pypy.tool.autopath import pypydir +from rpython.tool.autopath import rpythondir from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.translator.platform import platform import py @@ -122,7 +122,7 @@ ]) else: USE_C_LIBFFI_MSVC = True - libffidir = py.path.local(pypydir).join('translator', 'c', 'src', 'libffi_msvc') + libffidir = py.path.local(rpythondir).join('translator', 'c', 'src', 'libffi_msvc') if not _WIN64: asm_ifc = 'win32.c' else: diff --git a/rpython/rlib/parsing/test/autopath.py b/rpython/rlib/parsing/test/autopath.py --- a/rpython/rlib/parsing/test/autopath.py +++ b/rpython/rlib/parsing/test/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - pypydir pypy root directory path + rpythondir rpython root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) + "'%s'" % join(rpythondir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) + s = open(join(rpythondir, 'tool', _myname), 'rb').read() + walk(rpythondir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -pypydir, this_dir = __dirinfo('rpython') +rpythondir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/rlib/rdtoa.py b/rpython/rlib/rdtoa.py --- a/rpython/rlib/rdtoa.py +++ b/rpython/rlib/rdtoa.py @@ -1,13 +1,13 @@ from __future__ import with_statement from rpython.rlib import rfloat from rpython.translator.tool.cbuild import ExternalCompilationInfo -from pypy.tool.autopath import pypydir +from rpython.tool.autopath import rpythondir from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rlib import jit from rpython.rlib.rstring import StringBuilder import py, sys -cdir = py.path.local(pypydir) / 'translator' / 'c' +cdir = py.path.local(rpythondir) / 'translator' / 'c' include_dirs = [cdir] # set the word endianness based on the host's endianness diff --git a/rpython/rlib/rstack.py b/rpython/rlib/rstack.py --- a/rpython/rlib/rstack.py +++ b/rpython/rlib/rstack.py @@ -14,12 +14,12 @@ from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rtyper.lltypesystem.lloperation import llop from rpython.rtyper.controllerentry import Controller, SomeControlledInstance -from pypy.tool.autopath import pypydir +from rpython.tool.autopath import rpythondir from rpython.translator.tool.cbuild import ExternalCompilationInfo # ____________________________________________________________ -srcdir = py.path.local(pypydir) / 'translator' / 'c' / 'src' +srcdir = py.path.local(rpythondir) / 'translator' / 'c' / 'src' compilation_info = ExternalCompilationInfo( includes=['src/stack.h'], separate_module_files=[srcdir / 'stack.c', srcdir / 'threadlocal.c']) diff --git a/rpython/rlib/test/autopath.py b/rpython/rlib/test/autopath.py --- a/rpython/rlib/test/autopath.py +++ b/rpython/rlib/test/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - pypydir pypy root directory path + rpythondir rpython root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) + "'%s'" % join(rpythondir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) + s = open(join(rpythondir, 'tool', _myname), 'rb').read() + walk(rpythondir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -pypydir, this_dir = __dirinfo('rpython') +rpythondir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/rtyper/lltypesystem/module/ll_math.py b/rpython/rtyper/lltypesystem/module/ll_math.py --- a/rpython/rtyper/lltypesystem/module/ll_math.py +++ b/rpython/rtyper/lltypesystem/module/ll_math.py @@ -5,7 +5,7 @@ from rpython.rtyper.lltypesystem import lltype, rffi from rpython.tool.sourcetools import func_with_new_name -from pypy.tool.autopath import pypydir +from rpython.tool.autopath import rpythondir from rpython.rlib import jit, rposix from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.translator.platform import platform @@ -25,7 +25,7 @@ else: eci = ExternalCompilationInfo() # Some math functions are C99 and not defined by the Microsoft compiler - cdir = py.path.local(pypydir).join('translator', 'c') + cdir = py.path.local(rpythondir).join('translator', 'c') math_eci = ExternalCompilationInfo( include_dirs = [cdir], includes = ['src/ll_math.h'], diff --git a/rpython/rtyper/module/ll_strtod.py b/rpython/rtyper/module/ll_strtod.py --- a/rpython/rtyper/module/ll_strtod.py +++ b/rpython/rtyper/module/ll_strtod.py @@ -3,17 +3,17 @@ from rpython.rtyper.extfunc import BaseLazyRegistering, extdef, registering from rpython.rlib import rfloat from rpython.rtyper.lltypesystem import lltype, rffi -from pypy.tool.autopath import pypydir +from rpython.tool.autopath import rpythondir from rpython.rtyper.ootypesystem import ootype from rpython.rlib import rposix from rpython.translator.tool.cbuild import ExternalCompilationInfo -from pypy.tool.autopath import pypydir +from rpython.tool.autopath import rpythondir from rpython.annotator.model import SomeString class CConfig: _compilation_info_ = ExternalCompilationInfo( includes = ['src/ll_strtod.h'], - include_dirs = [str(py.path.local(pypydir).join('translator', 'c'))], + include_dirs = [str(py.path.local(rpythondir).join('translator', 'c'))], separate_module_sources = ['#include '], export_symbols = ['LL_strtod_formatd', 'LL_strtod_parts_to_float'], ) diff --git a/rpython/rtyper/tool/rffi_platform.py b/rpython/rtyper/tool/rffi_platform.py --- a/rpython/rtyper/tool/rffi_platform.py +++ b/rpython/rtyper/tool/rffi_platform.py @@ -11,7 +11,7 @@ from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.translator.platform import CompilationError from rpython.tool.udir import udir -from pypy.tool.autopath import pypydir +from rpython.tool.autopath import rpythondir from rpython.rlib.rarithmetic import r_uint, r_longlong, r_ulonglong, intmask # ____________________________________________________________ @@ -738,7 +738,7 @@ # ____________________________________________________________ -PYPY_EXTERNAL_DIR = py.path.local(pypydir).join('..', '..') +PYPY_EXTERNAL_DIR = py.path.local(rpythondir).join('..', '..') # XXX make this configurable if sys.platform == 'win32': for libdir in [ diff --git a/rpython/tool/algo/test/autopath.py b/rpython/tool/algo/test/autopath.py --- a/rpython/tool/algo/test/autopath.py +++ b/rpython/tool/algo/test/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - pypydir pypy root directory path + rpythondir rpython root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) + "'%s'" % join(rpythondir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) + s = open(join(rpythondir, 'tool', _myname), 'rb').read() + walk(rpythondir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -pypydir, this_dir = __dirinfo('pypy') +rpythondir, this_dir = __dirinfo('pypy') if __name__ == '__main__': __clone() diff --git a/pypy/tool/autopath.py b/rpython/tool/autopath.py copy from pypy/tool/autopath.py copy to rpython/tool/autopath.py --- a/pypy/tool/autopath.py +++ b/rpython/tool/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - pypydir pypy root directory path + rpythondir rpython root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) + "'%s'" % join(rpythondir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) + s = open(join(rpythondir, 'tool', _myname), 'rb').read() + walk(rpythondir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -pypydir, this_dir = __dirinfo('pypy') +rpythondir, this_dir = __dirinfo('pypy') if __name__ == '__main__': __clone() diff --git a/rpython/translator/autopath.py b/rpython/translator/autopath.py --- a/rpython/translator/autopath.py +++ b/rpython/translator/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - pypydir pypy root directory path + rpythondir rpython root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) + "'%s'" % join(rpythondir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) + s = open(join(rpythondir, 'tool', _myname), 'rb').read() + walk(rpythondir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -pypydir, this_dir = __dirinfo('rpython') +rpythondir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/translator/c/autopath.py b/rpython/translator/c/autopath.py --- a/rpython/translator/c/autopath.py +++ b/rpython/translator/c/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - pypydir pypy root directory path + rpythondir rpython root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) + "'%s'" % join(rpythondir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) + s = open(join(rpythondir, 'tool', _myname), 'rb').read() + walk(rpythondir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -pypydir, this_dir = __dirinfo('rpython') +rpythondir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/translator/c/gcc/autopath.py b/rpython/translator/c/gcc/autopath.py --- a/rpython/translator/c/gcc/autopath.py +++ b/rpython/translator/c/gcc/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - pypydir pypy root directory path + rpythondir rpython root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) + "'%s'" % join(rpythondir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) + s = open(join(rpythondir, 'tool', _myname), 'rb').read() + walk(rpythondir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -pypydir, this_dir = __dirinfo('rpython') +rpythondir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py --- a/rpython/translator/c/genc.py +++ b/rpython/translator/c/genc.py @@ -132,7 +132,7 @@ self.secondary_entrypoints = secondary_entrypoints def get_eci(self): - pypy_include_dir = py.path.local(autopath.pypydir).join('translator', 'c') + pypy_include_dir = py.path.local(autopath.rpythondir).join('translator', 'c') include_dirs = [pypy_include_dir] return ExternalCompilationInfo(include_dirs=include_dirs) @@ -754,7 +754,7 @@ defines['PYPY_LONGLONG_BIT'] = LONGLONG_BIT def add_extra_files(eci): - srcdir = py.path.local(autopath.pypydir).join('translator', 'c', 'src') + srcdir = py.path.local(autopath.rpythondir).join('translator', 'c', 'src') files = [ srcdir / 'entrypoint.c', # ifdef PYPY_STANDALONE srcdir / 'allocator.c', # ifdef PYPY_STANDALONE diff --git a/rpython/translator/c/test/autopath.py b/rpython/translator/c/test/autopath.py --- a/rpython/translator/c/test/autopath.py +++ b/rpython/translator/c/test/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - pypydir pypy root directory path + rpythondir rpython root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) + "'%s'" % join(rpythondir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) + s = open(join(rpythondir, 'tool', _myname), 'rb').read() + walk(rpythondir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -pypydir, this_dir = __dirinfo('rpython') +rpythondir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py --- a/rpython/translator/c/test/test_standalone.py +++ b/rpython/translator/c/test/test_standalone.py @@ -10,7 +10,7 @@ from rpython.translator.c.genc import CStandaloneBuilder, ExternalCompilationInfo from rpython.annotator.listdef import s_list_of_strings from rpython.tool.udir import udir -from pypy.tool.autopath import pypydir +from rpython.tool.autopath import rpythondir from pypy.conftest import option @@ -240,7 +240,7 @@ def test_separate_files(self): # One file in translator/c/src - fname = py.path.local(pypydir).join( + fname = py.path.local(rpythondir).join( 'translator', 'c', 'src', 'll_strtod.c') # One file in (another) subdir of the temp directory diff --git a/rpython/translator/cli/gencli.py b/rpython/translator/cli/gencli.py --- a/rpython/translator/cli/gencli.py +++ b/rpython/translator/cli/gencli.py @@ -92,7 +92,7 @@ if timeout and not sys.platform.startswith('win'): import os from pypy.tool import autopath - watchdog = os.path.join(autopath.pypydir, 'tool', 'watchdog.py') + watchdog = os.path.join(autopath.rpythondir, 'tool', 'watchdog.py') args[:0] = [sys.executable, watchdog, str(float(timeout))] proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = proc.communicate() diff --git a/rpython/translator/cli/test/autopath.py b/rpython/translator/cli/test/autopath.py --- a/rpython/translator/cli/test/autopath.py +++ b/rpython/translator/cli/test/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - pypydir pypy root directory path + rpythondir rpython root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) + "'%s'" % join(rpythondir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) + s = open(join(rpythondir, 'tool', _myname), 'rb').read() + walk(rpythondir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -pypydir, this_dir = __dirinfo('rpython') +rpythondir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/translator/goal/autopath.py b/rpython/translator/goal/autopath.py --- a/rpython/translator/goal/autopath.py +++ b/rpython/translator/goal/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - pypydir pypy root directory path + rpythondir rpython root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) + "'%s'" % join(rpythondir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) + s = open(join(rpythondir, 'tool', _myname), 'rb').read() + walk(rpythondir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -pypydir, this_dir = __dirinfo('rpython') +rpythondir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/translator/goal/test2/autopath.py b/rpython/translator/goal/test2/autopath.py --- a/rpython/translator/goal/test2/autopath.py +++ b/rpython/translator/goal/test2/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - pypydir pypy root directory path + rpythondir rpython root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) + "'%s'" % join(rpythondir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) + s = open(join(rpythondir, 'tool', _myname), 'rb').read() + walk(rpythondir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -pypydir, this_dir = __dirinfo('rpython') +rpythondir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/translator/goal/test2/test_app_main.py b/rpython/translator/goal/test2/test_app_main.py --- a/rpython/translator/goal/test2/test_app_main.py +++ b/rpython/translator/goal/test2/test_app_main.py @@ -523,7 +523,7 @@ assert line.rstrip() == 'Not at all. They could be carried.' print 'A five ounce bird could not carry a one pound coconut.' """) - py_py = os.path.join(autopath.pypydir, 'bin', 'py.py') + py_py = os.path.join(autopath.rpythondir, 'bin', 'py.py') child = self._spawn(sys.executable, [py_py, '-S', path]) child.expect('Are you suggesting coconuts migrate?', timeout=120) child.sendline('Not at all. They could be carried.') @@ -850,7 +850,7 @@ self.w_goal_dir = self.space.wrap(goal_dir) self.w_fake_exe = self.space.wrap(str(fake_exe)) self.w_expected_path = self.space.wrap(expected_path) - self.w_trunkdir = self.space.wrap(os.path.dirname(autopath.pypydir)) + self.w_trunkdir = self.space.wrap(os.path.dirname(autopath.rpythondir)) # foo_py = prefix.join('foo.py').write("pass") self.w_foo_py = self.space.wrap(str(foo_py)) diff --git a/rpython/translator/goal/translate.py b/rpython/translator/goal/translate.py --- a/rpython/translator/goal/translate.py +++ b/rpython/translator/goal/translate.py @@ -12,7 +12,7 @@ import py # clean up early pypy/_cache try: - py.path.local(autopath.pypydir).join('_cache').remove() + py.path.local(autopath.rpythondir).join('_cache').remove() except Exception: pass diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py --- a/rpython/translator/platform/posix.py +++ b/rpython/translator/platform/posix.py @@ -88,7 +88,7 @@ if path is None: path = cfiles[0].dirpath() - pypypath = py.path.local(autopath.pypydir) + pypypath = py.path.local(autopath.rpythondir) if exe_name is None: exe_name = cfiles[0].new(ext=self.exe_ext) @@ -139,7 +139,7 @@ m.comment('automatically generated makefile') definitions = [ - ('PYPYDIR', '"%s"' % autopath.pypydir), + ('PYPYDIR', '"%s"' % autopath.rpythondir), ('TARGET', target_name), ('DEFAULT_TARGET', exe_name.basename), ('SOURCES', rel_cfiles), diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -252,7 +252,7 @@ if path is None: path = cfiles[0].dirpath() - pypypath = py.path.local(autopath.pypydir) + pypypath = py.path.local(autopath.rpythondir) if exe_name is None: exe_name = cfiles[0].new(ext=self.exe_ext) @@ -294,7 +294,7 @@ m.comment('automatically generated makefile') definitions = [ - ('PYPYDIR', autopath.pypydir), + ('PYPYDIR', autopath.rpythondir), ('TARGET', target_name), ('DEFAULT_TARGET', exe_name.basename), ('SOURCES', rel_cfiles), diff --git a/rpython/translator/sandbox/autopath.py b/rpython/translator/sandbox/autopath.py --- a/rpython/translator/sandbox/autopath.py +++ b/rpython/translator/sandbox/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - pypydir pypy root directory path + rpythondir rpython root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) + "'%s'" % join(rpythondir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) + s = open(join(rpythondir, 'tool', _myname), 'rb').read() + walk(rpythondir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -pypydir, this_dir = __dirinfo('rpython') +rpythondir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/translator/sandbox/test/autopath.py b/rpython/translator/sandbox/test/autopath.py --- a/rpython/translator/sandbox/test/autopath.py +++ b/rpython/translator/sandbox/test/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - pypydir pypy root directory path + rpythondir rpython root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) + "'%s'" % join(rpythondir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) + s = open(join(rpythondir, 'tool', _myname), 'rb').read() + walk(rpythondir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -pypydir, this_dir = __dirinfo('rpython') +rpythondir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/translator/test/autopath.py b/rpython/translator/test/autopath.py --- a/rpython/translator/test/autopath.py +++ b/rpython/translator/test/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - pypydir pypy root directory path + rpythondir rpython root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) + "'%s'" % join(rpythondir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) + s = open(join(rpythondir, 'tool', _myname), 'rb').read() + walk(rpythondir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -pypydir, this_dir = __dirinfo('rpython') +rpythondir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/translator/tool/autopath.py b/rpython/translator/tool/autopath.py --- a/rpython/translator/tool/autopath.py +++ b/rpython/translator/tool/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - pypydir pypy root directory path + rpythondir rpython root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) + "'%s'" % join(rpythondir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) + s = open(join(rpythondir, 'tool', _myname), 'rb').read() + walk(rpythondir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -pypydir, this_dir = __dirinfo('rpython') +rpythondir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/translator/tool/cbuild.py b/rpython/translator/tool/cbuild.py --- a/rpython/translator/tool/cbuild.py +++ b/rpython/translator/tool/cbuild.py @@ -1,7 +1,7 @@ import py import sys -from pypy.tool.autopath import pypydir +from rpython.tool.autopath import rpythondir from rpython.translator.platform import host from rpython.tool.udir import udir From noreply at buildbot.pypy.org Fri Jan 4 22:31:02 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Fri, 4 Jan 2013 22:31:02 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Breaks more than it fixes Message-ID: <20130104213102.B27631C01EE@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59704:e92b34ba5faa Date: 2013-01-02 18:12 +0100 http://bitbucket.org/pypy/pypy/changeset/e92b34ba5faa/ Log: Breaks more than it fixes diff --git a/rpython/annotator/test/autopath.py b/rpython/annotator/test/autopath.py --- a/rpython/annotator/test/autopath.py +++ b/rpython/annotator/test/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - rpythondir rpython root directory path + pypydir pypy root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(rpythondir, 'tool',_myname)) + "'%s'" % join(pypydir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(rpythondir, 'tool', _myname), 'rb').read() - walk(rpythondir, sync_walker, s) + s = open(join(pypydir, 'tool', _myname), 'rb').read() + walk(pypydir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -rpythondir, this_dir = __dirinfo('rpython') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/jit/backend/autopath.py b/rpython/jit/backend/autopath.py --- a/rpython/jit/backend/autopath.py +++ b/rpython/jit/backend/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - rpythondir rpython root directory path + pypydir pypy root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(rpythondir, 'tool',_myname)) + "'%s'" % join(pypydir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(rpythondir, 'tool', _myname), 'rb').read() - walk(rpythondir, sync_walker, s) + s = open(join(pypydir, 'tool', _myname), 'rb').read() + walk(pypydir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -rpythondir, this_dir = __dirinfo('rpython') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/jit/backend/x86/autopath.py b/rpython/jit/backend/x86/autopath.py --- a/rpython/jit/backend/x86/autopath.py +++ b/rpython/jit/backend/x86/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - rpythondir rpython root directory path + pypydir pypy root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(rpythondir, 'tool',_myname)) + "'%s'" % join(pypydir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(rpythondir, 'tool', _myname), 'rb').read() - walk(rpythondir, sync_walker, s) + s = open(join(pypydir, 'tool', _myname), 'rb').read() + walk(pypydir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -rpythondir, this_dir = __dirinfo('rpython') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/jit/backend/x86/tool/autopath.py b/rpython/jit/backend/x86/tool/autopath.py --- a/rpython/jit/backend/x86/tool/autopath.py +++ b/rpython/jit/backend/x86/tool/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - rpythondir rpython root directory path + pypydir pypy root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(rpythondir, 'tool',_myname)) + "'%s'" % join(pypydir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(rpythondir, 'tool', _myname), 'rb').read() - walk(rpythondir, sync_walker, s) + s = open(join(pypydir, 'tool', _myname), 'rb').read() + walk(pypydir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -rpythondir, this_dir = __dirinfo('rpython') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/jit/tl/autopath.py b/rpython/jit/tl/autopath.py --- a/rpython/jit/tl/autopath.py +++ b/rpython/jit/tl/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - rpythondir rpython root directory path + pypydir pypy root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(rpythondir, 'tool',_myname)) + "'%s'" % join(pypydir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(rpythondir, 'tool', _myname), 'rb').read() - walk(rpythondir, sync_walker, s) + s = open(join(pypydir, 'tool', _myname), 'rb').read() + walk(pypydir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -rpythondir, this_dir = __dirinfo('rpython') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/jit/tl/spli/autopath.py b/rpython/jit/tl/spli/autopath.py --- a/rpython/jit/tl/spli/autopath.py +++ b/rpython/jit/tl/spli/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - rpythondir rpython root directory path + pypydir pypy root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(rpythondir, 'tool',_myname)) + "'%s'" % join(pypydir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(rpythondir, 'tool', _myname), 'rb').read() - walk(rpythondir, sync_walker, s) + s = open(join(pypydir, 'tool', _myname), 'rb').read() + walk(pypydir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -rpythondir, this_dir = __dirinfo('rpython') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/jit/tool/autopath.py b/rpython/jit/tool/autopath.py --- a/rpython/jit/tool/autopath.py +++ b/rpython/jit/tool/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - rpythondir rpython root directory path + pypydir pypy root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(rpythondir, 'tool',_myname)) + "'%s'" % join(pypydir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(rpythondir, 'tool', _myname), 'rb').read() - walk(rpythondir, sync_walker, s) + s = open(join(pypydir, 'tool', _myname), 'rb').read() + walk(pypydir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -rpythondir, this_dir = __dirinfo('rpython') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/rlib/_rffi_stacklet.py b/rpython/rlib/_rffi_stacklet.py --- a/rpython/rlib/_rffi_stacklet.py +++ b/rpython/rlib/_rffi_stacklet.py @@ -1,5 +1,5 @@ import py -from rpython.tool.autopath import rpythondir +from pypy.tool.autopath import pypydir from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.rtyper.tool import rffi_platform @@ -7,7 +7,7 @@ import sys -cdir = py.path.local(rpythondir) / 'translator' / 'c' +cdir = py.path.local(pypydir) / 'translator' / 'c' eci = ExternalCompilationInfo( include_dirs = [cdir], diff --git a/rpython/rlib/clibffi.py b/rpython/rlib/clibffi.py --- a/rpython/rlib/clibffi.py +++ b/rpython/rlib/clibffi.py @@ -12,7 +12,7 @@ from rpython.rlib.rdynload import DLOpenError, DLLHANDLE from rpython.rlib import jit from rpython.rlib.objectmodel import specialize -from rpython.tool.autopath import rpythondir +from pypy.tool.autopath import pypydir from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.translator.platform import platform import py @@ -122,7 +122,7 @@ ]) else: USE_C_LIBFFI_MSVC = True - libffidir = py.path.local(rpythondir).join('translator', 'c', 'src', 'libffi_msvc') + libffidir = py.path.local(pypydir).join('translator', 'c', 'src', 'libffi_msvc') if not _WIN64: asm_ifc = 'win32.c' else: diff --git a/rpython/rlib/parsing/test/autopath.py b/rpython/rlib/parsing/test/autopath.py --- a/rpython/rlib/parsing/test/autopath.py +++ b/rpython/rlib/parsing/test/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - rpythondir rpython root directory path + pypydir pypy root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(rpythondir, 'tool',_myname)) + "'%s'" % join(pypydir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(rpythondir, 'tool', _myname), 'rb').read() - walk(rpythondir, sync_walker, s) + s = open(join(pypydir, 'tool', _myname), 'rb').read() + walk(pypydir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -rpythondir, this_dir = __dirinfo('rpython') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/rlib/rdtoa.py b/rpython/rlib/rdtoa.py --- a/rpython/rlib/rdtoa.py +++ b/rpython/rlib/rdtoa.py @@ -1,13 +1,13 @@ from __future__ import with_statement from rpython.rlib import rfloat from rpython.translator.tool.cbuild import ExternalCompilationInfo -from rpython.tool.autopath import rpythondir +from pypy.tool.autopath import pypydir from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rlib import jit from rpython.rlib.rstring import StringBuilder import py, sys -cdir = py.path.local(rpythondir) / 'translator' / 'c' +cdir = py.path.local(pypydir) / 'translator' / 'c' include_dirs = [cdir] # set the word endianness based on the host's endianness diff --git a/rpython/rlib/rstack.py b/rpython/rlib/rstack.py --- a/rpython/rlib/rstack.py +++ b/rpython/rlib/rstack.py @@ -14,12 +14,12 @@ from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rtyper.lltypesystem.lloperation import llop from rpython.rtyper.controllerentry import Controller, SomeControlledInstance -from rpython.tool.autopath import rpythondir +from pypy.tool.autopath import pypydir from rpython.translator.tool.cbuild import ExternalCompilationInfo # ____________________________________________________________ -srcdir = py.path.local(rpythondir) / 'translator' / 'c' / 'src' +srcdir = py.path.local(pypydir) / 'translator' / 'c' / 'src' compilation_info = ExternalCompilationInfo( includes=['src/stack.h'], separate_module_files=[srcdir / 'stack.c', srcdir / 'threadlocal.c']) diff --git a/rpython/rlib/test/autopath.py b/rpython/rlib/test/autopath.py --- a/rpython/rlib/test/autopath.py +++ b/rpython/rlib/test/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - rpythondir rpython root directory path + pypydir pypy root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(rpythondir, 'tool',_myname)) + "'%s'" % join(pypydir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(rpythondir, 'tool', _myname), 'rb').read() - walk(rpythondir, sync_walker, s) + s = open(join(pypydir, 'tool', _myname), 'rb').read() + walk(pypydir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -rpythondir, this_dir = __dirinfo('rpython') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/rtyper/lltypesystem/module/ll_math.py b/rpython/rtyper/lltypesystem/module/ll_math.py --- a/rpython/rtyper/lltypesystem/module/ll_math.py +++ b/rpython/rtyper/lltypesystem/module/ll_math.py @@ -5,7 +5,7 @@ from rpython.rtyper.lltypesystem import lltype, rffi from rpython.tool.sourcetools import func_with_new_name -from rpython.tool.autopath import rpythondir +from pypy.tool.autopath import pypydir from rpython.rlib import jit, rposix from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.translator.platform import platform @@ -25,7 +25,7 @@ else: eci = ExternalCompilationInfo() # Some math functions are C99 and not defined by the Microsoft compiler - cdir = py.path.local(rpythondir).join('translator', 'c') + cdir = py.path.local(pypydir).join('translator', 'c') math_eci = ExternalCompilationInfo( include_dirs = [cdir], includes = ['src/ll_math.h'], diff --git a/rpython/rtyper/module/ll_strtod.py b/rpython/rtyper/module/ll_strtod.py --- a/rpython/rtyper/module/ll_strtod.py +++ b/rpython/rtyper/module/ll_strtod.py @@ -3,17 +3,17 @@ from rpython.rtyper.extfunc import BaseLazyRegistering, extdef, registering from rpython.rlib import rfloat from rpython.rtyper.lltypesystem import lltype, rffi -from rpython.tool.autopath import rpythondir +from pypy.tool.autopath import pypydir from rpython.rtyper.ootypesystem import ootype from rpython.rlib import rposix from rpython.translator.tool.cbuild import ExternalCompilationInfo -from rpython.tool.autopath import rpythondir +from pypy.tool.autopath import pypydir from rpython.annotator.model import SomeString class CConfig: _compilation_info_ = ExternalCompilationInfo( includes = ['src/ll_strtod.h'], - include_dirs = [str(py.path.local(rpythondir).join('translator', 'c'))], + include_dirs = [str(py.path.local(pypydir).join('translator', 'c'))], separate_module_sources = ['#include '], export_symbols = ['LL_strtod_formatd', 'LL_strtod_parts_to_float'], ) diff --git a/rpython/rtyper/tool/rffi_platform.py b/rpython/rtyper/tool/rffi_platform.py --- a/rpython/rtyper/tool/rffi_platform.py +++ b/rpython/rtyper/tool/rffi_platform.py @@ -11,7 +11,7 @@ from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.translator.platform import CompilationError from rpython.tool.udir import udir -from rpython.tool.autopath import rpythondir +from pypy.tool.autopath import pypydir from rpython.rlib.rarithmetic import r_uint, r_longlong, r_ulonglong, intmask # ____________________________________________________________ @@ -738,7 +738,7 @@ # ____________________________________________________________ -PYPY_EXTERNAL_DIR = py.path.local(rpythondir).join('..', '..') +PYPY_EXTERNAL_DIR = py.path.local(pypydir).join('..', '..') # XXX make this configurable if sys.platform == 'win32': for libdir in [ diff --git a/rpython/tool/algo/test/autopath.py b/rpython/tool/algo/test/autopath.py --- a/rpython/tool/algo/test/autopath.py +++ b/rpython/tool/algo/test/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - rpythondir rpython root directory path + pypydir pypy root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(rpythondir, 'tool',_myname)) + "'%s'" % join(pypydir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(rpythondir, 'tool', _myname), 'rb').read() - walk(rpythondir, sync_walker, s) + s = open(join(pypydir, 'tool', _myname), 'rb').read() + walk(pypydir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -rpythondir, this_dir = __dirinfo('pypy') +pypydir, this_dir = __dirinfo('pypy') if __name__ == '__main__': __clone() diff --git a/rpython/tool/autopath.py b/rpython/tool/autopath.py deleted file mode 100644 --- a/rpython/tool/autopath.py +++ /dev/null @@ -1,131 +0,0 @@ -""" -self cloning, automatic path configuration - -copy this into any subdirectory of pypy from which scripts need -to be run, typically all of the test subdirs. -The idea is that any such script simply issues - - import autopath - -and this will make sure that the parent directory containing "pypy" -is in sys.path. - -If you modify the master "autopath.py" version (in pypy/tool/autopath.py) -you can directly run it which will copy itself on all autopath.py files -it finds under the pypy root directory. - -This module always provides these attributes: - - rpythondir rpython root directory path - this_dir directory where this autopath.py resides - -""" - -def __dirinfo(part): - """ return (partdir, this_dir) and insert parent of partdir - into sys.path. If the parent directories don't have the part - an EnvironmentError is raised.""" - - import sys, os - try: - head = this_dir = os.path.realpath(os.path.dirname(__file__)) - except NameError: - head = this_dir = os.path.realpath(os.path.dirname(sys.argv[0])) - - error = None - while head: - partdir = head - head, tail = os.path.split(head) - if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') - if not os.path.exists(checkfile): - error = "Cannot find %r" % (os.path.normpath(checkfile),) - break - else: - error = "Cannot find the parent directory %r of the path %r" % ( - partdir, this_dir) - if not error: - # check for bogus end-of-line style (e.g. files checked out on - # Windows and moved to Unix) - f = open(__file__.replace('.pyc', '.py'), 'r') - data = f.read() - f.close() - if data.endswith('\r\n') or data.endswith('\r'): - error = ("Bad end-of-line style in the .py files. Typically " - "caused by a zip file or a checkout done on Windows and " - "moved to Unix or vice-versa.") - if error: - raise EnvironmentError("Invalid source tree - bogus checkout! " + - error) - - pypy_root = os.path.join(head, '') - try: - sys.path.remove(head) - except ValueError: - pass - sys.path.insert(0, head) - - munged = {} - for name, mod in sys.modules.items(): - if '.' in name: - continue - fn = getattr(mod, '__file__', None) - if not isinstance(fn, str): - continue - newname = os.path.splitext(os.path.basename(fn))[0] - if not newname.startswith(part + '.'): - continue - path = os.path.join(os.path.dirname(os.path.realpath(fn)), '') - if path.startswith(pypy_root) and newname != part: - modpaths = os.path.normpath(path[len(pypy_root):]).split(os.sep) - if newname != '__init__': - modpaths.append(newname) - modpath = '.'.join(modpaths) - if modpath not in sys.modules: - munged[modpath] = mod - - for name, mod in munged.iteritems(): - if name not in sys.modules: - sys.modules[name] = mod - if '.' in name: - prename = name[:name.rfind('.')] - postname = name[len(prename)+1:] - if prename not in sys.modules: - __import__(prename) - if not hasattr(sys.modules[prename], postname): - setattr(sys.modules[prename], postname, mod) - - return partdir, this_dir - -def __clone(): - """ clone master version of autopath.py into all subdirs """ - from os.path import join, walk - if not this_dir.endswith(join('pypy','tool')): - raise EnvironmentError("can only clone master version " - "'%s'" % join(rpythondir, 'tool',_myname)) - - - def sync_walker(arg, dirname, fnames): - if _myname in fnames: - fn = join(dirname, _myname) - f = open(fn, 'rwb+') - try: - if f.read() == arg: - print "checkok", fn - else: - print "syncing", fn - f = open(fn, 'w') - f.write(arg) - finally: - f.close() - s = open(join(rpythondir, 'tool', _myname), 'rb').read() - walk(rpythondir, sync_walker, s) - -_myname = 'autopath.py' - -# set guaranteed attributes - -rpythondir, this_dir = __dirinfo('pypy') - -if __name__ == '__main__': - __clone() diff --git a/rpython/translator/autopath.py b/rpython/translator/autopath.py --- a/rpython/translator/autopath.py +++ b/rpython/translator/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - rpythondir rpython root directory path + pypydir pypy root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(rpythondir, 'tool',_myname)) + "'%s'" % join(pypydir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(rpythondir, 'tool', _myname), 'rb').read() - walk(rpythondir, sync_walker, s) + s = open(join(pypydir, 'tool', _myname), 'rb').read() + walk(pypydir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -rpythondir, this_dir = __dirinfo('rpython') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/translator/c/autopath.py b/rpython/translator/c/autopath.py --- a/rpython/translator/c/autopath.py +++ b/rpython/translator/c/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - rpythondir rpython root directory path + pypydir pypy root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(rpythondir, 'tool',_myname)) + "'%s'" % join(pypydir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(rpythondir, 'tool', _myname), 'rb').read() - walk(rpythondir, sync_walker, s) + s = open(join(pypydir, 'tool', _myname), 'rb').read() + walk(pypydir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -rpythondir, this_dir = __dirinfo('rpython') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/translator/c/gcc/autopath.py b/rpython/translator/c/gcc/autopath.py --- a/rpython/translator/c/gcc/autopath.py +++ b/rpython/translator/c/gcc/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - rpythondir rpython root directory path + pypydir pypy root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(rpythondir, 'tool',_myname)) + "'%s'" % join(pypydir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(rpythondir, 'tool', _myname), 'rb').read() - walk(rpythondir, sync_walker, s) + s = open(join(pypydir, 'tool', _myname), 'rb').read() + walk(pypydir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -rpythondir, this_dir = __dirinfo('rpython') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py --- a/rpython/translator/c/genc.py +++ b/rpython/translator/c/genc.py @@ -132,7 +132,7 @@ self.secondary_entrypoints = secondary_entrypoints def get_eci(self): - pypy_include_dir = py.path.local(autopath.rpythondir).join('translator', 'c') + pypy_include_dir = py.path.local(autopath.pypydir).join('translator', 'c') include_dirs = [pypy_include_dir] return ExternalCompilationInfo(include_dirs=include_dirs) @@ -754,7 +754,7 @@ defines['PYPY_LONGLONG_BIT'] = LONGLONG_BIT def add_extra_files(eci): - srcdir = py.path.local(autopath.rpythondir).join('translator', 'c', 'src') + srcdir = py.path.local(autopath.pypydir).join('translator', 'c', 'src') files = [ srcdir / 'entrypoint.c', # ifdef PYPY_STANDALONE srcdir / 'allocator.c', # ifdef PYPY_STANDALONE diff --git a/rpython/translator/c/test/autopath.py b/rpython/translator/c/test/autopath.py --- a/rpython/translator/c/test/autopath.py +++ b/rpython/translator/c/test/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - rpythondir rpython root directory path + pypydir pypy root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(rpythondir, 'tool',_myname)) + "'%s'" % join(pypydir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(rpythondir, 'tool', _myname), 'rb').read() - walk(rpythondir, sync_walker, s) + s = open(join(pypydir, 'tool', _myname), 'rb').read() + walk(pypydir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -rpythondir, this_dir = __dirinfo('rpython') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py --- a/rpython/translator/c/test/test_standalone.py +++ b/rpython/translator/c/test/test_standalone.py @@ -10,7 +10,7 @@ from rpython.translator.c.genc import CStandaloneBuilder, ExternalCompilationInfo from rpython.annotator.listdef import s_list_of_strings from rpython.tool.udir import udir -from rpython.tool.autopath import rpythondir +from pypy.tool.autopath import pypydir from pypy.conftest import option @@ -240,7 +240,7 @@ def test_separate_files(self): # One file in translator/c/src - fname = py.path.local(rpythondir).join( + fname = py.path.local(pypydir).join( 'translator', 'c', 'src', 'll_strtod.c') # One file in (another) subdir of the temp directory diff --git a/rpython/translator/cli/gencli.py b/rpython/translator/cli/gencli.py --- a/rpython/translator/cli/gencli.py +++ b/rpython/translator/cli/gencli.py @@ -92,7 +92,7 @@ if timeout and not sys.platform.startswith('win'): import os from pypy.tool import autopath - watchdog = os.path.join(autopath.rpythondir, 'tool', 'watchdog.py') + watchdog = os.path.join(autopath.pypydir, 'tool', 'watchdog.py') args[:0] = [sys.executable, watchdog, str(float(timeout))] proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = proc.communicate() diff --git a/rpython/translator/cli/test/autopath.py b/rpython/translator/cli/test/autopath.py --- a/rpython/translator/cli/test/autopath.py +++ b/rpython/translator/cli/test/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - rpythondir rpython root directory path + pypydir pypy root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(rpythondir, 'tool',_myname)) + "'%s'" % join(pypydir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(rpythondir, 'tool', _myname), 'rb').read() - walk(rpythondir, sync_walker, s) + s = open(join(pypydir, 'tool', _myname), 'rb').read() + walk(pypydir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -rpythondir, this_dir = __dirinfo('rpython') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/translator/goal/autopath.py b/rpython/translator/goal/autopath.py --- a/rpython/translator/goal/autopath.py +++ b/rpython/translator/goal/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - rpythondir rpython root directory path + pypydir pypy root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(rpythondir, 'tool',_myname)) + "'%s'" % join(pypydir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(rpythondir, 'tool', _myname), 'rb').read() - walk(rpythondir, sync_walker, s) + s = open(join(pypydir, 'tool', _myname), 'rb').read() + walk(pypydir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -rpythondir, this_dir = __dirinfo('rpython') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/translator/goal/test2/autopath.py b/rpython/translator/goal/test2/autopath.py --- a/rpython/translator/goal/test2/autopath.py +++ b/rpython/translator/goal/test2/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - rpythondir rpython root directory path + pypydir pypy root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(rpythondir, 'tool',_myname)) + "'%s'" % join(pypydir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(rpythondir, 'tool', _myname), 'rb').read() - walk(rpythondir, sync_walker, s) + s = open(join(pypydir, 'tool', _myname), 'rb').read() + walk(pypydir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -rpythondir, this_dir = __dirinfo('rpython') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/translator/goal/test2/test_app_main.py b/rpython/translator/goal/test2/test_app_main.py --- a/rpython/translator/goal/test2/test_app_main.py +++ b/rpython/translator/goal/test2/test_app_main.py @@ -523,7 +523,7 @@ assert line.rstrip() == 'Not at all. They could be carried.' print 'A five ounce bird could not carry a one pound coconut.' """) - py_py = os.path.join(autopath.rpythondir, 'bin', 'py.py') + py_py = os.path.join(autopath.pypydir, 'bin', 'py.py') child = self._spawn(sys.executable, [py_py, '-S', path]) child.expect('Are you suggesting coconuts migrate?', timeout=120) child.sendline('Not at all. They could be carried.') @@ -850,7 +850,7 @@ self.w_goal_dir = self.space.wrap(goal_dir) self.w_fake_exe = self.space.wrap(str(fake_exe)) self.w_expected_path = self.space.wrap(expected_path) - self.w_trunkdir = self.space.wrap(os.path.dirname(autopath.rpythondir)) + self.w_trunkdir = self.space.wrap(os.path.dirname(autopath.pypydir)) # foo_py = prefix.join('foo.py').write("pass") self.w_foo_py = self.space.wrap(str(foo_py)) diff --git a/rpython/translator/goal/translate.py b/rpython/translator/goal/translate.py --- a/rpython/translator/goal/translate.py +++ b/rpython/translator/goal/translate.py @@ -12,7 +12,7 @@ import py # clean up early pypy/_cache try: - py.path.local(autopath.rpythondir).join('_cache').remove() + py.path.local(autopath.pypydir).join('_cache').remove() except Exception: pass diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py --- a/rpython/translator/platform/posix.py +++ b/rpython/translator/platform/posix.py @@ -88,7 +88,7 @@ if path is None: path = cfiles[0].dirpath() - pypypath = py.path.local(autopath.rpythondir) + pypypath = py.path.local(autopath.pypydir) if exe_name is None: exe_name = cfiles[0].new(ext=self.exe_ext) @@ -139,7 +139,7 @@ m.comment('automatically generated makefile') definitions = [ - ('PYPYDIR', '"%s"' % autopath.rpythondir), + ('PYPYDIR', '"%s"' % autopath.pypydir), ('TARGET', target_name), ('DEFAULT_TARGET', exe_name.basename), ('SOURCES', rel_cfiles), diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -252,7 +252,7 @@ if path is None: path = cfiles[0].dirpath() - pypypath = py.path.local(autopath.rpythondir) + pypypath = py.path.local(autopath.pypydir) if exe_name is None: exe_name = cfiles[0].new(ext=self.exe_ext) @@ -294,7 +294,7 @@ m.comment('automatically generated makefile') definitions = [ - ('PYPYDIR', autopath.rpythondir), + ('PYPYDIR', autopath.pypydir), ('TARGET', target_name), ('DEFAULT_TARGET', exe_name.basename), ('SOURCES', rel_cfiles), diff --git a/rpython/translator/sandbox/autopath.py b/rpython/translator/sandbox/autopath.py --- a/rpython/translator/sandbox/autopath.py +++ b/rpython/translator/sandbox/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - rpythondir rpython root directory path + pypydir pypy root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(rpythondir, 'tool',_myname)) + "'%s'" % join(pypydir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(rpythondir, 'tool', _myname), 'rb').read() - walk(rpythondir, sync_walker, s) + s = open(join(pypydir, 'tool', _myname), 'rb').read() + walk(pypydir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -rpythondir, this_dir = __dirinfo('rpython') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/translator/sandbox/test/autopath.py b/rpython/translator/sandbox/test/autopath.py --- a/rpython/translator/sandbox/test/autopath.py +++ b/rpython/translator/sandbox/test/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - rpythondir rpython root directory path + pypydir pypy root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(rpythondir, 'tool',_myname)) + "'%s'" % join(pypydir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(rpythondir, 'tool', _myname), 'rb').read() - walk(rpythondir, sync_walker, s) + s = open(join(pypydir, 'tool', _myname), 'rb').read() + walk(pypydir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -rpythondir, this_dir = __dirinfo('rpython') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/translator/test/autopath.py b/rpython/translator/test/autopath.py --- a/rpython/translator/test/autopath.py +++ b/rpython/translator/test/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - rpythondir rpython root directory path + pypydir pypy root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(rpythondir, 'tool',_myname)) + "'%s'" % join(pypydir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(rpythondir, 'tool', _myname), 'rb').read() - walk(rpythondir, sync_walker, s) + s = open(join(pypydir, 'tool', _myname), 'rb').read() + walk(pypydir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -rpythondir, this_dir = __dirinfo('rpython') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/translator/tool/autopath.py b/rpython/translator/tool/autopath.py --- a/rpython/translator/tool/autopath.py +++ b/rpython/translator/tool/autopath.py @@ -16,7 +16,7 @@ This module always provides these attributes: - rpythondir rpython root directory path + pypydir pypy root directory path this_dir directory where this autopath.py resides """ @@ -37,7 +37,7 @@ partdir = head head, tail = os.path.split(head) if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'rpython', '__init__.py') + checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') if not os.path.exists(checkfile): error = "Cannot find %r" % (os.path.normpath(checkfile),) break @@ -102,7 +102,7 @@ from os.path import join, walk if not this_dir.endswith(join('pypy','tool')): raise EnvironmentError("can only clone master version " - "'%s'" % join(rpythondir, 'tool',_myname)) + "'%s'" % join(pypydir, 'tool',_myname)) def sync_walker(arg, dirname, fnames): @@ -118,14 +118,14 @@ f.write(arg) finally: f.close() - s = open(join(rpythondir, 'tool', _myname), 'rb').read() - walk(rpythondir, sync_walker, s) + s = open(join(pypydir, 'tool', _myname), 'rb').read() + walk(pypydir, sync_walker, s) _myname = 'autopath.py' # set guaranteed attributes -rpythondir, this_dir = __dirinfo('rpython') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/translator/tool/cbuild.py b/rpython/translator/tool/cbuild.py --- a/rpython/translator/tool/cbuild.py +++ b/rpython/translator/tool/cbuild.py @@ -1,7 +1,7 @@ import py import sys -from rpython.tool.autopath import rpythondir +from pypy.tool.autopath import pypydir from rpython.translator.platform import host from rpython.tool.udir import udir From noreply at buildbot.pypy.org Fri Jan 4 22:31:04 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Fri, 4 Jan 2013 22:31:04 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Moved pypy.tool.error, pypy.tool.leakfinder, pypy.tool.logparser, pypy.tool.progressbar and pypy.tool.stdlib_opcode to rpython Message-ID: <20130104213104.06DEB1C01EE@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59705:4f3d2f54abd4 Date: 2013-01-04 16:41 +0100 http://bitbucket.org/pypy/pypy/changeset/4f3d2f54abd4/ Log: Moved pypy.tool.error, pypy.tool.leakfinder, pypy.tool.logparser, pypy.tool.progressbar and pypy.tool.stdlib_opcode to rpython diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py --- a/pypy/interpreter/astcompiler/assemble.py +++ b/pypy/interpreter/astcompiler/assemble.py @@ -4,7 +4,7 @@ from pypy.interpreter.astcompiler import ast, symtable from pypy.interpreter import pycode -from pypy.tool import stdlib_opcode as ops +from rpython.tool import stdlib_opcode as ops from pypy.interpreter.error import OperationError from rpython.rlib.objectmodel import we_are_translated diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py --- a/pypy/interpreter/astcompiler/codegen.py +++ b/pypy/interpreter/astcompiler/codegen.py @@ -10,7 +10,7 @@ from pypy.interpreter.astcompiler import ast, assemble, symtable, consts, misc from pypy.interpreter.astcompiler import optimize # For side effects from pypy.interpreter.pyparser.error import SyntaxError -from pypy.tool import stdlib_opcode as ops +from rpython.tool import stdlib_opcode as ops from pypy.interpreter.error import OperationError diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py --- a/pypy/interpreter/astcompiler/optimize.py +++ b/pypy/interpreter/astcompiler/optimize.py @@ -2,7 +2,7 @@ import sys from pypy.interpreter.astcompiler import ast, consts, misc -from pypy.tool import stdlib_opcode as ops +from rpython.tool import stdlib_opcode as ops from pypy.interpreter.error import OperationError from rpython.rlib.unroll import unrolling_iterable from rpython.rlib.runicode import MAXUNICODE diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py --- a/pypy/interpreter/astcompiler/test/test_compiler.py +++ b/pypy/interpreter/astcompiler/test/test_compiler.py @@ -4,7 +4,7 @@ from pypy.interpreter.pyparser.test import expressions from pypy.interpreter.pycode import PyCode from pypy.interpreter.pyparser.error import SyntaxError, IndentationError -from pypy.tool import stdlib_opcode as ops +from rpython.tool import stdlib_opcode as ops def compile_with_astcompiler(expr, mode, space): p = pyparse.PythonParser(space) diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -17,7 +17,7 @@ from rpython.rlib.debug import make_sure_not_resized from rpython.rlib import jit from rpython.rlib.objectmodel import compute_hash -from pypy.tool.stdlib_opcode import opcodedesc, HAVE_ARGUMENT +from rpython.tool.stdlib_opcode import opcodedesc, HAVE_ARGUMENT class BytecodeCorruption(Exception): diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py --- a/pypy/interpreter/pyframe.py +++ b/pypy/interpreter/pyframe.py @@ -12,8 +12,8 @@ from rpython.rlib.debug import make_sure_not_resized, check_nonneg from rpython.rlib.rarithmetic import intmask, r_uint from rpython.rlib import jit -from pypy.tool import stdlib_opcode -from pypy.tool.stdlib_opcode import host_bytecode_spec +from rpython.tool import stdlib_opcode +from rpython.tool.stdlib_opcode import host_bytecode_spec # Define some opcodes used g = globals() diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -15,7 +15,7 @@ from rpython.rlib.rarithmetic import r_uint, intmask from rpython.rlib.unroll import unrolling_iterable from rpython.rlib.debug import check_nonneg -from pypy.tool.stdlib_opcode import (bytecode_spec, +from rpython.tool.stdlib_opcode import (bytecode_spec, unrolling_all_opcode_descs) def unaryoperation(operationname): diff --git a/pypy/module/_continuation/interp_pickle.py b/pypy/module/_continuation/interp_pickle.py --- a/pypy/module/_continuation/interp_pickle.py +++ b/pypy/module/_continuation/interp_pickle.py @@ -1,4 +1,4 @@ -from pypy.tool import stdlib_opcode as pythonopcode +from rpython.tool import stdlib_opcode as pythonopcode from rpython.rlib import jit from pypy.interpreter.error import OperationError from pypy.interpreter.pyframe import PyFrame diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -17,7 +17,7 @@ from pypy.module.cpyext.pyobject import Py_DecRef, InvalidPointerException from rpython.translator.goal import autopath from rpython.tool.identity_dict import identity_dict -from pypy.tool import leakfinder +from rpython.tool import leakfinder @api.cpython_api([], api.PyObject) def PyPy_Crash1(space): diff --git a/pypy/module/cpyext/test/test_pystate.py b/pypy/module/cpyext/test/test_pystate.py --- a/pypy/module/cpyext/test/test_pystate.py +++ b/pypy/module/cpyext/test/test_pystate.py @@ -6,7 +6,7 @@ from rpython.rtyper.lltypesystem import lltype from pypy.module.cpyext.test.test_cpyext import LeakCheckingTest, freeze_refcnts from pypy.module.cpyext.pystate import PyThreadState_Get, PyInterpreterState_Head -from pypy.tool import leakfinder +from rpython.tool import leakfinder class AppTestThreads(AppTestCpythonExtensionBase): def test_allow_threads(self): diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py --- a/pypy/module/pypyjit/interp_jit.py +++ b/pypy/module/pypyjit/interp_jit.py @@ -27,7 +27,7 @@ JUMP_ABSOLUTE = opmap['JUMP_ABSOLUTE'] def get_printable_location(next_instr, is_being_profiled, bytecode): - from pypy.tool.stdlib_opcode import opcode_method_names + from rpython.tool.stdlib_opcode import opcode_method_names name = opcode_method_names[ord(bytecode.co_code[next_instr])] return '%s #%d %s' % (bytecode.get_repr(), next_instr, name) diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py --- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py +++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py @@ -5,7 +5,7 @@ import py from lib_pypy import disassembler from rpython.tool.udir import udir -from pypy.tool import logparser +from rpython.tool import logparser from rpython.jit.tool.jitoutput import parse_prof from pypy.module.pypyjit.test_pypy_c.model import (Log, find_ids_range, find_ids, diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py --- a/pypy/module/sys/vm.py +++ b/pypy/module/sys/vm.py @@ -106,7 +106,7 @@ space.w_None]) def exc_info_direct(space, frame): - from pypy.tool import stdlib_opcode + from rpython.tool import stdlib_opcode # In order to make the JIT happy, we try to return (exc, val, None) # instead of (exc, val, tb). We can do that only if we recognize # the following pattern in the bytecode: diff --git a/pypy/tool/jitlogparser/parser.py b/pypy/tool/jitlogparser/parser.py --- a/pypy/tool/jitlogparser/parser.py +++ b/pypy/tool/jitlogparser/parser.py @@ -2,7 +2,7 @@ from rpython.jit.metainterp.resoperation import opname from rpython.jit.tool.oparser import OpParser -from pypy.tool.logparser import parse_log_file, extract_category +from rpython.tool.logparser import parse_log_file, extract_category from copy import copy def parse_code_data(arg): diff --git a/pypy/tool/pydis.py b/pypy/tool/pydis.py --- a/pypy/tool/pydis.py +++ b/pypy/tool/pydis.py @@ -8,8 +8,8 @@ import autopath import sys -from pypy.tool import stdlib_opcode -from pypy.tool.stdlib_opcode import * +from rpython.tool import stdlib_opcode +from rpython.tool.stdlib_opcode import * __all__ = ["dis","pydisassemble","distb","disco"] + stdlib_opcode.__all__ diff --git a/pypy/tool/pytest/plugins.py b/pypy/tool/pytest/plugins.py --- a/pypy/tool/pytest/plugins.py +++ b/pypy/tool/pytest/plugins.py @@ -1,7 +1,7 @@ # pytest hooks, installed by pypy.conftest. import py -from pypy.tool import leakfinder +from rpython.tool import leakfinder class LeakFinder: """Track memory allocations during test execution. diff --git a/pypy/tool/test/test_error.py b/pypy/tool/test/test_error.py --- a/pypy/tool/test/test_error.py +++ b/pypy/tool/test/test_error.py @@ -3,7 +3,7 @@ """ from rpython.translator.translator import TranslationContext -from pypy.tool.error import AnnotatorError +from rpython.tool.error import AnnotatorError from rpython.annotator.model import UnionError import py diff --git a/pypy/tool/test/test_leakfinder.py b/pypy/tool/test/test_leakfinder.py --- a/pypy/tool/test/test_leakfinder.py +++ b/pypy/tool/test/test_leakfinder.py @@ -1,5 +1,5 @@ import py -from pypy.tool import leakfinder +from rpython.tool import leakfinder def test_start_stop(): leakfinder.start_tracking_allocations() diff --git a/pypy/tool/test/test_logparser.py b/pypy/tool/test/test_logparser.py --- a/pypy/tool/test/test_logparser.py +++ b/pypy/tool/test/test_logparser.py @@ -1,5 +1,5 @@ from rpython.tool.udir import udir -from pypy.tool.logparser import * +from rpython.tool.logparser import * globalpath = udir.join('test_logparser.log') diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py --- a/rpython/annotator/annrpython.py +++ b/rpython/annotator/annrpython.py @@ -3,7 +3,7 @@ import types from rpython.tool.ansi_print import ansi_log from rpython.tool.pairtype import pair -from pypy.tool.error import (format_blocked_annotation_error, +from rpython.tool.error import (format_blocked_annotation_error, AnnotatorError, gather_error, ErrorWrapper) from rpython.flowspace.model import (Variable, Constant, FunctionGraph, c_last_exception, checkgraph) diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py --- a/rpython/annotator/binaryop.py +++ b/rpython/annotator/binaryop.py @@ -22,7 +22,7 @@ from rpython.annotator.bookkeeper import getbookkeeper from rpython.flowspace.model import Variable, Constant from rpython.rlib import rarithmetic -from pypy.tool.error import AnnotatorError +from rpython.tool.error import AnnotatorError # convenience only! def immutablevalue(x): diff --git a/rpython/annotator/builtin.py b/rpython/annotator/builtin.py --- a/rpython/annotator/builtin.py +++ b/rpython/annotator/builtin.py @@ -16,7 +16,7 @@ from rpython.annotator.bookkeeper import getbookkeeper from rpython.annotator import description from rpython.flowspace.model import Constant -from pypy.tool.error import AnnotatorError +from rpython.tool.error import AnnotatorError import rpython.rlib.rarithmetic import rpython.rlib.objectmodel diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py --- a/rpython/annotator/unaryop.py +++ b/rpython/annotator/unaryop.py @@ -17,7 +17,7 @@ from rpython.annotator import builtin from rpython.annotator.binaryop import _clone ## XXX where to put this? from rpython.rtyper import extregistry -from pypy.tool.error import AnnotatorError +from rpython.tool.error import AnnotatorError # convenience only! def immutablevalue(x): diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py --- a/rpython/flowspace/bytecode.py +++ b/rpython/flowspace/bytecode.py @@ -1,7 +1,7 @@ """ Bytecode handling classes and functions for use by the flow space. """ -from pypy.tool.stdlib_opcode import (host_bytecode_spec, EXTENDED_ARG, +from rpython.tool.stdlib_opcode import (host_bytecode_spec, EXTENDED_ARG, HAVE_ARGUMENT) from rpython.flowspace.argument import Signature from rpython.flowspace.flowcontext import BytecodeCorruption diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py --- a/rpython/flowspace/flowcontext.py +++ b/rpython/flowspace/flowcontext.py @@ -5,8 +5,8 @@ import sys import collections -from pypy.tool.error import source_lines -from pypy.tool.stdlib_opcode import host_bytecode_spec +from rpython.tool.error import source_lines +from rpython.tool.stdlib_opcode import host_bytecode_spec from rpython.flowspace.argument import ArgumentsForTranslation from rpython.flowspace.model import (Constant, Variable, Block, Link, UnwrapException, c_last_exception) diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py --- a/rpython/flowspace/test/test_objspace.py +++ b/rpython/flowspace/test/test_objspace.py @@ -8,7 +8,7 @@ from rpython.flowspace.objspace import FlowObjSpace from rpython.flowspace.flowcontext import FlowingError, FlowSpaceFrame from pypy import conftest -from pypy.tool.stdlib_opcode import bytecode_spec, host_bytecode_spec +from rpython.tool.stdlib_opcode import bytecode_spec, host_bytecode_spec import os import operator diff --git a/rpython/jit/backend/x86/test/test_runner.py b/rpython/jit/backend/x86/test/test_runner.py --- a/rpython/jit/backend/x86/test/test_runner.py +++ b/rpython/jit/backend/x86/test/test_runner.py @@ -522,7 +522,7 @@ self.cpu.setup_once() def test_debugger_on(self): - from pypy.tool.logparser import parse_log_file, extract_category + from rpython.tool.logparser import parse_log_file, extract_category from rpython.rlib import debug targettoken, preambletoken = TargetToken(), TargetToken() diff --git a/rpython/jit/backend/x86/tool/viewcode.py b/rpython/jit/backend/x86/tool/viewcode.py --- a/rpython/jit/backend/x86/tool/viewcode.py +++ b/rpython/jit/backend/x86/tool/viewcode.py @@ -434,7 +434,7 @@ sys.exit(2) # import cStringIO - from pypy.tool import logparser + from rpython.tool import logparser log1 = logparser.parse_log_file(sys.argv[1]) text1 = logparser.extract_category(log1, catprefix='jit-backend-dump') f = cStringIO.StringIO() diff --git a/rpython/jit/tl/spli/interpreter.py b/rpython/jit/tl/spli/interpreter.py --- a/rpython/jit/tl/spli/interpreter.py +++ b/rpython/jit/tl/spli/interpreter.py @@ -1,5 +1,5 @@ import os -from pypy.tool import stdlib_opcode +from rpython.tool import stdlib_opcode from rpython.jit.tl.spli import objects, pycode from rpython.rlib.unroll import unrolling_iterable from rpython.rlib.jit import JitDriver, promote, dont_look_inside diff --git a/rpython/jit/tool/findadrinlog.py b/rpython/jit/tool/findadrinlog.py --- a/rpython/jit/tool/findadrinlog.py +++ b/rpython/jit/tool/findadrinlog.py @@ -1,6 +1,6 @@ import autopath import sys, re -from pypy.tool import logparser +from rpython.tool import logparser # fflush(pypy_debug_file) diff --git a/rpython/jit/tool/loopviewer.py b/rpython/jit/tool/loopviewer.py --- a/rpython/jit/tool/loopviewer.py +++ b/rpython/jit/tool/loopviewer.py @@ -8,7 +8,7 @@ import sys import optparse from pprint import pprint -from pypy.tool import logparser +from rpython.tool import logparser from rpython.jit.tool.oparser import parse from rpython.jit.metainterp.history import ConstInt from rpython.rtyper.lltypesystem import llmemory, lltype diff --git a/rpython/jit/tool/showstats.py b/rpython/jit/tool/showstats.py --- a/rpython/jit/tool/showstats.py +++ b/rpython/jit/tool/showstats.py @@ -3,7 +3,7 @@ import autopath import sys, py -from pypy.tool import logparser +from rpython.tool import logparser from rpython.jit.tool.oparser import parse from rpython.jit.metainterp.resoperation import rop from rpython.rtyper.lltypesystem import lltype, llmemory diff --git a/rpython/jit/tool/test/test_jitoutput.py b/rpython/jit/tool/test/test_jitoutput.py --- a/rpython/jit/tool/test/test_jitoutput.py +++ b/rpython/jit/tool/test/test_jitoutput.py @@ -5,7 +5,7 @@ from rpython.jit.backend.llgraph import runner from rpython.jit.metainterp.jitprof import Profiler, JITPROF_LINES from rpython.jit.tool.jitoutput import parse_prof -from pypy.tool.logparser import parse_log, extract_category +from rpython.tool.logparser import parse_log, extract_category def test_really_run(): """ This test checks whether output of jitprof did not change. diff --git a/rpython/jit/tool/traceviewer.py b/rpython/jit/tool/traceviewer.py --- a/rpython/jit/tool/traceviewer.py +++ b/rpython/jit/tool/traceviewer.py @@ -11,8 +11,8 @@ import autopath from rpython.translator.tool.graphpage import GraphPage from rpython.translator.tool.make_dot import DotGen -from pypy.tool import logparser -from pypy.tool import progressbar +from rpython.tool import logparser +from rpython.tool import progressbar class SubPage(GraphPage): def compute(self, graph): diff --git a/rpython/rtyper/lltypesystem/lltype.py b/rpython/rtyper/lltypesystem/lltype.py --- a/rpython/rtyper/lltypesystem/lltype.py +++ b/rpython/rtyper/lltypesystem/lltype.py @@ -3,7 +3,7 @@ base_int, normalizedinttype, longlongmask, longlonglongmask) from rpython.rlib.objectmodel import Symbolic from rpython.tool.identity_dict import identity_dict -from pypy.tool import leakfinder +from rpython.tool import leakfinder from types import NoneType from rpython.rlib.rarithmetic import maxint, is_valid_int, is_emulated_long import weakref diff --git a/rpython/rtyper/lltypesystem/test/test_lltype.py b/rpython/rtyper/lltypesystem/test/test_lltype.py --- a/rpython/rtyper/lltypesystem/test/test_lltype.py +++ b/rpython/rtyper/lltypesystem/test/test_lltype.py @@ -3,7 +3,7 @@ from rpython.rtyper.lltypesystem.lltype import * from rpython.rtyper.lltypesystem import lltype, rffi from rpython.tool.identity_dict import identity_dict -from pypy.tool import leakfinder +from rpython.tool import leakfinder def isweak(p, T): try: diff --git a/rpython/rtyper/ootypesystem/test/test_oortype.py b/rpython/rtyper/ootypesystem/test/test_oortype.py --- a/rpython/rtyper/ootypesystem/test/test_oortype.py +++ b/rpython/rtyper/ootypesystem/test/test_oortype.py @@ -9,7 +9,7 @@ from rpython.translator.translator import TranslationContext, graphof from rpython.rtyper.test.test_llinterp import interpret from rpython.rlib.objectmodel import r_dict -from pypy.tool.error import AnnotatorError +from rpython.tool.error import AnnotatorError from rpython.rtyper.ootypesystem import ooregistry # side effects def gengraph(f, args=[], viewBefore=False, viewAfter=False, mangle=True): diff --git a/rpython/rtyper/test/test_llinterp.py b/rpython/rtyper/test/test_llinterp.py --- a/rpython/rtyper/test/test_llinterp.py +++ b/rpython/rtyper/test/test_llinterp.py @@ -11,7 +11,7 @@ from rpython.annotator import model as annmodel from rpython.annotator.model import lltype_to_annotation from rpython.rlib.rarithmetic import r_uint, ovfcheck -from pypy.tool import leakfinder +from rpython.tool import leakfinder from pypy import conftest diff --git a/pypy/tool/error.py b/rpython/tool/error.py rename from pypy/tool/error.py rename to rpython/tool/error.py diff --git a/pypy/tool/leakfinder.py b/rpython/tool/leakfinder.py rename from pypy/tool/leakfinder.py rename to rpython/tool/leakfinder.py diff --git a/pypy/tool/logparser.py b/rpython/tool/logparser.py rename from pypy/tool/logparser.py rename to rpython/tool/logparser.py --- a/pypy/tool/logparser.py +++ b/rpython/tool/logparser.py @@ -10,7 +10,7 @@ import autopath import sys, re from rpython.rlib.debug import DebugLog -from pypy.tool import progressbar +from rpython.tool import progressbar def parse_log_file(filename, verbose=True): f = open(filename, 'r') diff --git a/pypy/tool/progressbar.py b/rpython/tool/progressbar.py rename from pypy/tool/progressbar.py rename to rpython/tool/progressbar.py diff --git a/pypy/tool/stdlib_opcode.py b/rpython/tool/stdlib_opcode.py rename from pypy/tool/stdlib_opcode.py rename to rpython/tool/stdlib_opcode.py From noreply at buildbot.pypy.org Fri Jan 4 22:31:05 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Fri, 4 Jan 2013 22:31:05 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Moved targetpypystandalone to toplevel and added rpython.tool.autopath Message-ID: <20130104213105.2D49F1C01EE@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59706:82b01d80249e Date: 2013-01-04 20:16 +0100 http://bitbucket.org/pypy/pypy/changeset/82b01d80249e/ Log: Moved targetpypystandalone to toplevel and added rpython.tool.autopath diff --git a/pypy/tool/autopath.py b/rpython/tool/autopath.py copy from pypy/tool/autopath.py copy to rpython/tool/autopath.py --- a/pypy/tool/autopath.py +++ b/rpython/tool/autopath.py @@ -125,7 +125,7 @@ # set guaranteed attributes -pypydir, this_dir = __dirinfo('pypy') +pypydir, this_dir = __dirinfo('rpython') if __name__ == '__main__': __clone() diff --git a/rpython/translator/goal/targetpypystandalone.py b/targetpypystandalone.py rename from rpython/translator/goal/targetpypystandalone.py rename to targetpypystandalone.py From noreply at buildbot.pypy.org Fri Jan 4 22:31:07 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Fri, 4 Jan 2013 22:31:07 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Killing the beast named autopath Message-ID: <20130104213107.3A3671C01EE@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59707:8be506609828 Date: 2013-01-04 22:21 +0100 http://bitbucket.org/pypy/pypy/changeset/8be506609828/ Log: Killing the beast named autopath diff too long, truncating to 2000 out of 5894 lines diff --git a/pypy/bin/autopath.py b/pypy/bin/autopath.py deleted file mode 100644 --- a/pypy/bin/autopath.py +++ /dev/null @@ -1,131 +0,0 @@ -""" -self cloning, automatic path configuration - -copy this into any subdirectory of pypy from which scripts need -to be run, typically all of the test subdirs. -The idea is that any such script simply issues - - import autopath - -and this will make sure that the parent directory containing "pypy" -is in sys.path. - -If you modify the master "autopath.py" version (in pypy/tool/autopath.py) -you can directly run it which will copy itself on all autopath.py files -it finds under the pypy root directory. - -This module always provides these attributes: - - pypydir pypy root directory path - this_dir directory where this autopath.py resides - -""" - -def __dirinfo(part): - """ return (partdir, this_dir) and insert parent of partdir - into sys.path. If the parent directories don't have the part - an EnvironmentError is raised.""" - - import sys, os - try: - head = this_dir = os.path.realpath(os.path.dirname(__file__)) - except NameError: - head = this_dir = os.path.realpath(os.path.dirname(sys.argv[0])) - - error = None - while head: - partdir = head - head, tail = os.path.split(head) - if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') - if not os.path.exists(checkfile): - error = "Cannot find %r" % (os.path.normpath(checkfile),) - break - else: - error = "Cannot find the parent directory %r of the path %r" % ( - partdir, this_dir) - if not error: - # check for bogus end-of-line style (e.g. files checked out on - # Windows and moved to Unix) - f = open(__file__.replace('.pyc', '.py'), 'r') - data = f.read() - f.close() - if data.endswith('\r\n') or data.endswith('\r'): - error = ("Bad end-of-line style in the .py files. Typically " - "caused by a zip file or a checkout done on Windows and " - "moved to Unix or vice-versa.") - if error: - raise EnvironmentError("Invalid source tree - bogus checkout! " + - error) - - pypy_root = os.path.join(head, '') - try: - sys.path.remove(head) - except ValueError: - pass - sys.path.insert(0, head) - - munged = {} - for name, mod in sys.modules.items(): - if '.' in name: - continue - fn = getattr(mod, '__file__', None) - if not isinstance(fn, str): - continue - newname = os.path.splitext(os.path.basename(fn))[0] - if not newname.startswith(part + '.'): - continue - path = os.path.join(os.path.dirname(os.path.realpath(fn)), '') - if path.startswith(pypy_root) and newname != part: - modpaths = os.path.normpath(path[len(pypy_root):]).split(os.sep) - if newname != '__init__': - modpaths.append(newname) - modpath = '.'.join(modpaths) - if modpath not in sys.modules: - munged[modpath] = mod - - for name, mod in munged.iteritems(): - if name not in sys.modules: - sys.modules[name] = mod - if '.' in name: - prename = name[:name.rfind('.')] - postname = name[len(prename)+1:] - if prename not in sys.modules: - __import__(prename) - if not hasattr(sys.modules[prename], postname): - setattr(sys.modules[prename], postname, mod) - - return partdir, this_dir - -def __clone(): - """ clone master version of autopath.py into all subdirs """ - from os.path import join, walk - if not this_dir.endswith(join('pypy','tool')): - raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) - - - def sync_walker(arg, dirname, fnames): - if _myname in fnames: - fn = join(dirname, _myname) - f = open(fn, 'rwb+') - try: - if f.read() == arg: - print "checkok", fn - else: - print "syncing", fn - f = open(fn, 'w') - f.write(arg) - finally: - f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) - -_myname = 'autopath.py' - -# set guaranteed attributes - -pypydir, this_dir = __dirinfo('pypy') - -if __name__ == '__main__': - __clone() diff --git a/pypy/bin/checkmodule.py b/pypy/bin/checkmodule.py --- a/pypy/bin/checkmodule.py +++ b/pypy/bin/checkmodule.py @@ -6,8 +6,8 @@ pypy/module//. Useful for testing whether a modules compiles without doing a full translation. """ -import autopath import sys, os +sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..'))) from pypy.objspace.fake.checkmodule import checkmodule diff --git a/pypy/bin/dotviewer.py b/pypy/bin/dotviewer.py --- a/pypy/bin/dotviewer.py +++ b/pypy/bin/dotviewer.py @@ -4,7 +4,8 @@ Run with no arguments for help. """ -import autopath +import sys +sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..'))) from dotviewer.dotviewer import main if __name__ == '__main__': diff --git a/pypy/bin/reportstaticdata.py b/pypy/bin/reportstaticdata.py --- a/pypy/bin/reportstaticdata.py +++ b/pypy/bin/reportstaticdata.py @@ -27,7 +27,9 @@ --help Show this help message """ -import autopath +import sys +sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..'))) + from rpython.translator.tool.staticsizereport import print_report def parse_options(argv): diff --git a/pypy/bin/translatorshell.py b/pypy/bin/translatorshell.py --- a/pypy/bin/translatorshell.py +++ b/pypy/bin/translatorshell.py @@ -23,7 +23,9 @@ Try dir(snippet) for list of current snippets. """ -import autopath, os, sys +import os, sys +sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..'))) + from rpython.translator.interactive import Translation from rpython.rtyper.rtyper import * from rpython.rlib.rarithmetic import * diff --git a/pypy/config/autopath.py b/pypy/config/autopath.py deleted file mode 100644 --- a/pypy/config/autopath.py +++ /dev/null @@ -1,131 +0,0 @@ -""" -self cloning, automatic path configuration - -copy this into any subdirectory of pypy from which scripts need -to be run, typically all of the test subdirs. -The idea is that any such script simply issues - - import autopath - -and this will make sure that the parent directory containing "pypy" -is in sys.path. - -If you modify the master "autopath.py" version (in pypy/tool/autopath.py) -you can directly run it which will copy itself on all autopath.py files -it finds under the pypy root directory. - -This module always provides these attributes: - - pypydir pypy root directory path - this_dir directory where this autopath.py resides - -""" - -def __dirinfo(part): - """ return (partdir, this_dir) and insert parent of partdir - into sys.path. If the parent directories don't have the part - an EnvironmentError is raised.""" - - import sys, os - try: - head = this_dir = os.path.realpath(os.path.dirname(__file__)) - except NameError: - head = this_dir = os.path.realpath(os.path.dirname(sys.argv[0])) - - error = None - while head: - partdir = head - head, tail = os.path.split(head) - if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') - if not os.path.exists(checkfile): - error = "Cannot find %r" % (os.path.normpath(checkfile),) - break - else: - error = "Cannot find the parent directory %r of the path %r" % ( - partdir, this_dir) - if not error: - # check for bogus end-of-line style (e.g. files checked out on - # Windows and moved to Unix) - f = open(__file__.replace('.pyc', '.py'), 'r') - data = f.read() - f.close() - if data.endswith('\r\n') or data.endswith('\r'): - error = ("Bad end-of-line style in the .py files. Typically " - "caused by a zip file or a checkout done on Windows and " - "moved to Unix or vice-versa.") - if error: - raise EnvironmentError("Invalid source tree - bogus checkout! " + - error) - - pypy_root = os.path.join(head, '') - try: - sys.path.remove(head) - except ValueError: - pass - sys.path.insert(0, head) - - munged = {} - for name, mod in sys.modules.items(): - if '.' in name: - continue - fn = getattr(mod, '__file__', None) - if not isinstance(fn, str): - continue - newname = os.path.splitext(os.path.basename(fn))[0] - if not newname.startswith(part + '.'): - continue - path = os.path.join(os.path.dirname(os.path.realpath(fn)), '') - if path.startswith(pypy_root) and newname != part: - modpaths = os.path.normpath(path[len(pypy_root):]).split(os.sep) - if newname != '__init__': - modpaths.append(newname) - modpath = '.'.join(modpaths) - if modpath not in sys.modules: - munged[modpath] = mod - - for name, mod in munged.iteritems(): - if name not in sys.modules: - sys.modules[name] = mod - if '.' in name: - prename = name[:name.rfind('.')] - postname = name[len(prename)+1:] - if prename not in sys.modules: - __import__(prename) - if not hasattr(sys.modules[prename], postname): - setattr(sys.modules[prename], postname, mod) - - return partdir, this_dir - -def __clone(): - """ clone master version of autopath.py into all subdirs """ - from os.path import join, walk - if not this_dir.endswith(join('pypy','tool')): - raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) - - - def sync_walker(arg, dirname, fnames): - if _myname in fnames: - fn = join(dirname, _myname) - f = open(fn, 'rwb+') - try: - if f.read() == arg: - print "checkok", fn - else: - print "syncing", fn - f = open(fn, 'w') - f.write(arg) - finally: - f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) - -_myname = 'autopath.py' - -# set guaranteed attributes - -pypydir, this_dir = __dirinfo('pypy') - -if __name__ == '__main__': - __clone() diff --git a/pypy/config/translationoption.py b/pypy/config/translationoption.py --- a/pypy/config/translationoption.py +++ b/pypy/config/translationoption.py @@ -1,4 +1,3 @@ -import autopath import py, os, sys from pypy.config.config import OptionDescription, BoolOption, IntOption, ArbitraryOption, FloatOption from pypy.config.config import ChoiceOption, StrOption, to_optparse, Config diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -10,6 +10,7 @@ # option = None +pypydir = os.path.realpath(os.path.join(os.path.dirname(__file__), '..')) def braindead_deindent(self): """monkeypatch that wont end up doing stupid in the python tokenizer""" diff --git a/pypy/doc/config/autopath.py b/pypy/doc/config/autopath.py deleted file mode 100644 --- a/pypy/doc/config/autopath.py +++ /dev/null @@ -1,131 +0,0 @@ -""" -self cloning, automatic path configuration - -copy this into any subdirectory of pypy from which scripts need -to be run, typically all of the test subdirs. -The idea is that any such script simply issues - - import autopath - -and this will make sure that the parent directory containing "pypy" -is in sys.path. - -If you modify the master "autopath.py" version (in pypy/tool/autopath.py) -you can directly run it which will copy itself on all autopath.py files -it finds under the pypy root directory. - -This module always provides these attributes: - - pypydir pypy root directory path - this_dir directory where this autopath.py resides - -""" - -def __dirinfo(part): - """ return (partdir, this_dir) and insert parent of partdir - into sys.path. If the parent directories don't have the part - an EnvironmentError is raised.""" - - import sys, os - try: - head = this_dir = os.path.realpath(os.path.dirname(__file__)) - except NameError: - head = this_dir = os.path.realpath(os.path.dirname(sys.argv[0])) - - error = None - while head: - partdir = head - head, tail = os.path.split(head) - if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') - if not os.path.exists(checkfile): - error = "Cannot find %r" % (os.path.normpath(checkfile),) - break - else: - error = "Cannot find the parent directory %r of the path %r" % ( - partdir, this_dir) - if not error: - # check for bogus end-of-line style (e.g. files checked out on - # Windows and moved to Unix) - f = open(__file__.replace('.pyc', '.py'), 'r') - data = f.read() - f.close() - if data.endswith('\r\n') or data.endswith('\r'): - error = ("Bad end-of-line style in the .py files. Typically " - "caused by a zip file or a checkout done on Windows and " - "moved to Unix or vice-versa.") - if error: - raise EnvironmentError("Invalid source tree - bogus checkout! " + - error) - - pypy_root = os.path.join(head, '') - try: - sys.path.remove(head) - except ValueError: - pass - sys.path.insert(0, head) - - munged = {} - for name, mod in sys.modules.items(): - if '.' in name: - continue - fn = getattr(mod, '__file__', None) - if not isinstance(fn, str): - continue - newname = os.path.splitext(os.path.basename(fn))[0] - if not newname.startswith(part + '.'): - continue - path = os.path.join(os.path.dirname(os.path.realpath(fn)), '') - if path.startswith(pypy_root) and newname != part: - modpaths = os.path.normpath(path[len(pypy_root):]).split(os.sep) - if newname != '__init__': - modpaths.append(newname) - modpath = '.'.join(modpaths) - if modpath not in sys.modules: - munged[modpath] = mod - - for name, mod in munged.iteritems(): - if name not in sys.modules: - sys.modules[name] = mod - if '.' in name: - prename = name[:name.rfind('.')] - postname = name[len(prename)+1:] - if prename not in sys.modules: - __import__(prename) - if not hasattr(sys.modules[prename], postname): - setattr(sys.modules[prename], postname, mod) - - return partdir, this_dir - -def __clone(): - """ clone master version of autopath.py into all subdirs """ - from os.path import join, walk - if not this_dir.endswith(join('pypy','tool')): - raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) - - - def sync_walker(arg, dirname, fnames): - if _myname in fnames: - fn = join(dirname, _myname) - f = open(fn, 'rwb+') - try: - if f.read() == arg: - print "checkok", fn - else: - print "syncing", fn - f = open(fn, 'w') - f.write(arg) - finally: - f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) - -_myname = 'autopath.py' - -# set guaranteed attributes - -pypydir, this_dir = __dirinfo('pypy') - -if __name__ == '__main__': - __clone() diff --git a/pypy/doc/config/generate.py b/pypy/doc/config/generate.py --- a/pypy/doc/config/generate.py +++ b/pypy/doc/config/generate.py @@ -1,4 +1,3 @@ -import autopath import py from pypy.config import pypyoption, translationoption, config, makerestdoc from pypy.doc.config.confrest import all_optiondescrs diff --git a/pypy/doc/config/makemodules.py b/pypy/doc/config/makemodules.py --- a/pypy/doc/config/makemodules.py +++ b/pypy/doc/config/makemodules.py @@ -1,4 +1,3 @@ -import autopath import py from pypy.config import pypyoption, translationoption, config diff --git a/pypy/interpreter/pyparser/autopath.py b/pypy/interpreter/pyparser/autopath.py deleted file mode 100644 --- a/pypy/interpreter/pyparser/autopath.py +++ /dev/null @@ -1,131 +0,0 @@ -""" -self cloning, automatic path configuration - -copy this into any subdirectory of pypy from which scripts need -to be run, typically all of the test subdirs. -The idea is that any such script simply issues - - import autopath - -and this will make sure that the parent directory containing "pypy" -is in sys.path. - -If you modify the master "autopath.py" version (in pypy/tool/autopath.py) -you can directly run it which will copy itself on all autopath.py files -it finds under the pypy root directory. - -This module always provides these attributes: - - pypydir pypy root directory path - this_dir directory where this autopath.py resides - -""" - -def __dirinfo(part): - """ return (partdir, this_dir) and insert parent of partdir - into sys.path. If the parent directories don't have the part - an EnvironmentError is raised.""" - - import sys, os - try: - head = this_dir = os.path.realpath(os.path.dirname(__file__)) - except NameError: - head = this_dir = os.path.realpath(os.path.dirname(sys.argv[0])) - - error = None - while head: - partdir = head - head, tail = os.path.split(head) - if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') - if not os.path.exists(checkfile): - error = "Cannot find %r" % (os.path.normpath(checkfile),) - break - else: - error = "Cannot find the parent directory %r of the path %r" % ( - partdir, this_dir) - if not error: - # check for bogus end-of-line style (e.g. files checked out on - # Windows and moved to Unix) - f = open(__file__.replace('.pyc', '.py'), 'r') - data = f.read() - f.close() - if data.endswith('\r\n') or data.endswith('\r'): - error = ("Bad end-of-line style in the .py files. Typically " - "caused by a zip file or a checkout done on Windows and " - "moved to Unix or vice-versa.") - if error: - raise EnvironmentError("Invalid source tree - bogus checkout! " + - error) - - pypy_root = os.path.join(head, '') - try: - sys.path.remove(head) - except ValueError: - pass - sys.path.insert(0, head) - - munged = {} - for name, mod in sys.modules.items(): - if '.' in name: - continue - fn = getattr(mod, '__file__', None) - if not isinstance(fn, str): - continue - newname = os.path.splitext(os.path.basename(fn))[0] - if not newname.startswith(part + '.'): - continue - path = os.path.join(os.path.dirname(os.path.realpath(fn)), '') - if path.startswith(pypy_root) and newname != part: - modpaths = os.path.normpath(path[len(pypy_root):]).split(os.sep) - if newname != '__init__': - modpaths.append(newname) - modpath = '.'.join(modpaths) - if modpath not in sys.modules: - munged[modpath] = mod - - for name, mod in munged.iteritems(): - if name not in sys.modules: - sys.modules[name] = mod - if '.' in name: - prename = name[:name.rfind('.')] - postname = name[len(prename)+1:] - if prename not in sys.modules: - __import__(prename) - if not hasattr(sys.modules[prename], postname): - setattr(sys.modules[prename], postname, mod) - - return partdir, this_dir - -def __clone(): - """ clone master version of autopath.py into all subdirs """ - from os.path import join, walk - if not this_dir.endswith(join('pypy','tool')): - raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) - - - def sync_walker(arg, dirname, fnames): - if _myname in fnames: - fn = join(dirname, _myname) - f = open(fn, 'rwb+') - try: - if f.read() == arg: - print "checkok", fn - else: - print "syncing", fn - f = open(fn, 'w') - f.write(arg) - finally: - f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) - -_myname = 'autopath.py' - -# set guaranteed attributes - -pypydir, this_dir = __dirinfo('pypy') - -if __name__ == '__main__': - __clone() diff --git a/pypy/interpreter/pyparser/genpytokenize.py b/pypy/interpreter/pyparser/genpytokenize.py old mode 100644 new mode 100755 --- a/pypy/interpreter/pyparser/genpytokenize.py +++ b/pypy/interpreter/pyparser/genpytokenize.py @@ -10,7 +10,6 @@ $Id: genPytokenize.py,v 1.1 2003/10/02 17:37:17 jriehl Exp $ """ -import autopath from pypy.interpreter.pyparser.pylexer import * from pypy.interpreter.pyparser.automata import NonGreedyDFA, DFA, DEFAULT diff --git a/pypy/module/__builtin__/test/autopath.py b/pypy/module/__builtin__/test/autopath.py deleted file mode 100644 --- a/pypy/module/__builtin__/test/autopath.py +++ /dev/null @@ -1,131 +0,0 @@ -""" -self cloning, automatic path configuration - -copy this into any subdirectory of pypy from which scripts need -to be run, typically all of the test subdirs. -The idea is that any such script simply issues - - import autopath - -and this will make sure that the parent directory containing "pypy" -is in sys.path. - -If you modify the master "autopath.py" version (in pypy/tool/autopath.py) -you can directly run it which will copy itself on all autopath.py files -it finds under the pypy root directory. - -This module always provides these attributes: - - pypydir pypy root directory path - this_dir directory where this autopath.py resides - -""" - -def __dirinfo(part): - """ return (partdir, this_dir) and insert parent of partdir - into sys.path. If the parent directories don't have the part - an EnvironmentError is raised.""" - - import sys, os - try: - head = this_dir = os.path.realpath(os.path.dirname(__file__)) - except NameError: - head = this_dir = os.path.realpath(os.path.dirname(sys.argv[0])) - - error = None - while head: - partdir = head - head, tail = os.path.split(head) - if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') - if not os.path.exists(checkfile): - error = "Cannot find %r" % (os.path.normpath(checkfile),) - break - else: - error = "Cannot find the parent directory %r of the path %r" % ( - partdir, this_dir) - if not error: - # check for bogus end-of-line style (e.g. files checked out on - # Windows and moved to Unix) - f = open(__file__.replace('.pyc', '.py'), 'r') - data = f.read() - f.close() - if data.endswith('\r\n') or data.endswith('\r'): - error = ("Bad end-of-line style in the .py files. Typically " - "caused by a zip file or a checkout done on Windows and " - "moved to Unix or vice-versa.") - if error: - raise EnvironmentError("Invalid source tree - bogus checkout! " + - error) - - pypy_root = os.path.join(head, '') - try: - sys.path.remove(head) - except ValueError: - pass - sys.path.insert(0, head) - - munged = {} - for name, mod in sys.modules.items(): - if '.' in name: - continue - fn = getattr(mod, '__file__', None) - if not isinstance(fn, str): - continue - newname = os.path.splitext(os.path.basename(fn))[0] - if not newname.startswith(part + '.'): - continue - path = os.path.join(os.path.dirname(os.path.realpath(fn)), '') - if path.startswith(pypy_root) and newname != part: - modpaths = os.path.normpath(path[len(pypy_root):]).split(os.sep) - if newname != '__init__': - modpaths.append(newname) - modpath = '.'.join(modpaths) - if modpath not in sys.modules: - munged[modpath] = mod - - for name, mod in munged.iteritems(): - if name not in sys.modules: - sys.modules[name] = mod - if '.' in name: - prename = name[:name.rfind('.')] - postname = name[len(prename)+1:] - if prename not in sys.modules: - __import__(prename) - if not hasattr(sys.modules[prename], postname): - setattr(sys.modules[prename], postname, mod) - - return partdir, this_dir - -def __clone(): - """ clone master version of autopath.py into all subdirs """ - from os.path import join, walk - if not this_dir.endswith(join('pypy','tool')): - raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) - - - def sync_walker(arg, dirname, fnames): - if _myname in fnames: - fn = join(dirname, _myname) - f = open(fn, 'rwb+') - try: - if f.read() == arg: - print "checkok", fn - else: - print "syncing", fn - f = open(fn, 'w') - f.write(arg) - finally: - f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) - -_myname = 'autopath.py' - -# set guaranteed attributes - -pypydir, this_dir = __dirinfo('pypy') - -if __name__ == '__main__': - __clone() diff --git a/pypy/module/__builtin__/test/test_apply.py b/pypy/module/__builtin__/test/test_apply.py --- a/pypy/module/__builtin__/test/test_apply.py +++ b/pypy/module/__builtin__/test/test_apply.py @@ -1,6 +1,3 @@ -import autopath - - # This is a very trivial series of tests. If apply is subtlely broken, # we will have to find out some other way. diff --git a/pypy/module/__builtin__/test/test_buffer.py b/pypy/module/__builtin__/test/test_buffer.py --- a/pypy/module/__builtin__/test/test_buffer.py +++ b/pypy/module/__builtin__/test/test_buffer.py @@ -1,6 +1,5 @@ """Tests some behaviour of the buffer type that is not tested in lib-python/2.5.2/test/test_types.py where the stdlib buffer tests live.""" -import autopath class AppTestBuffer: spaceconfig = dict(usemodules=['array']) diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py --- a/pypy/module/__builtin__/test/test_builtin.py +++ b/pypy/module/__builtin__/test/test_builtin.py @@ -1,4 +1,3 @@ -import autopath import sys class AppTestBuiltinApp: diff --git a/pypy/module/__builtin__/test/test_descriptor.py b/pypy/module/__builtin__/test/test_descriptor.py --- a/pypy/module/__builtin__/test/test_descriptor.py +++ b/pypy/module/__builtin__/test/test_descriptor.py @@ -1,6 +1,3 @@ -import autopath - - class AppTestBuiltinApp: def test_staticmethod(self): class C(object): diff --git a/pypy/module/__builtin__/test/test_filter.py b/pypy/module/__builtin__/test/test_filter.py --- a/pypy/module/__builtin__/test/test_filter.py +++ b/pypy/module/__builtin__/test/test_filter.py @@ -1,5 +1,3 @@ -import autopath - # trivial functions for testing class AppTestFilter: diff --git a/pypy/module/__builtin__/test/test_functional.py b/pypy/module/__builtin__/test/test_functional.py --- a/pypy/module/__builtin__/test/test_functional.py +++ b/pypy/module/__builtin__/test/test_functional.py @@ -1,6 +1,3 @@ -import autopath - - class AppTestMap: def test_trivial_map_one_seq(self): diff --git a/pypy/module/__builtin__/test/test_minmax.py b/pypy/module/__builtin__/test/test_minmax.py --- a/pypy/module/__builtin__/test/test_minmax.py +++ b/pypy/module/__builtin__/test/test_minmax.py @@ -1,5 +1,3 @@ -import autopath - class AppTestMin: def test_min_notseq(self): diff --git a/pypy/module/__builtin__/test/test_range.py b/pypy/module/__builtin__/test/test_range.py --- a/pypy/module/__builtin__/test/test_range.py +++ b/pypy/module/__builtin__/test/test_range.py @@ -1,5 +1,3 @@ -import autopath - class AppTestRange: def test_range_toofew(self): diff --git a/pypy/module/__builtin__/test/test_rawinput.py b/pypy/module/__builtin__/test/test_rawinput.py --- a/pypy/module/__builtin__/test/test_rawinput.py +++ b/pypy/module/__builtin__/test/test_rawinput.py @@ -1,6 +1,3 @@ -import autopath - - class AppTestRawInput(): def test_input_and_raw_input(self): diff --git a/pypy/module/__builtin__/test/test_reduce.py b/pypy/module/__builtin__/test/test_reduce.py --- a/pypy/module/__builtin__/test/test_reduce.py +++ b/pypy/module/__builtin__/test/test_reduce.py @@ -1,6 +1,3 @@ -import autopath - - class AppTestReduce: def test_None(self): raises(TypeError, reduce, lambda x, y: x+y, [1,2,3], None) diff --git a/pypy/module/__builtin__/test/test_vars.py b/pypy/module/__builtin__/test/test_vars.py --- a/pypy/module/__builtin__/test/test_vars.py +++ b/pypy/module/__builtin__/test/test_vars.py @@ -1,5 +1,3 @@ -import autopath - class AppTestVars: def test_vars_no_arguments(self): diff --git a/pypy/module/__builtin__/test/test_zip.py b/pypy/module/__builtin__/test/test_zip.py --- a/pypy/module/__builtin__/test/test_zip.py +++ b/pypy/module/__builtin__/test/test_zip.py @@ -1,5 +1,3 @@ -import autopath - class AppTestZip: def test_zip_no_arguments(self): diff --git a/pypy/module/_codecs/test/autopath.py b/pypy/module/_codecs/test/autopath.py deleted file mode 100644 --- a/pypy/module/_codecs/test/autopath.py +++ /dev/null @@ -1,131 +0,0 @@ -""" -self cloning, automatic path configuration - -copy this into any subdirectory of pypy from which scripts need -to be run, typically all of the test subdirs. -The idea is that any such script simply issues - - import autopath - -and this will make sure that the parent directory containing "pypy" -is in sys.path. - -If you modify the master "autopath.py" version (in pypy/tool/autopath.py) -you can directly run it which will copy itself on all autopath.py files -it finds under the pypy root directory. - -This module always provides these attributes: - - pypydir pypy root directory path - this_dir directory where this autopath.py resides - -""" - -def __dirinfo(part): - """ return (partdir, this_dir) and insert parent of partdir - into sys.path. If the parent directories don't have the part - an EnvironmentError is raised.""" - - import sys, os - try: - head = this_dir = os.path.realpath(os.path.dirname(__file__)) - except NameError: - head = this_dir = os.path.realpath(os.path.dirname(sys.argv[0])) - - error = None - while head: - partdir = head - head, tail = os.path.split(head) - if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') - if not os.path.exists(checkfile): - error = "Cannot find %r" % (os.path.normpath(checkfile),) - break - else: - error = "Cannot find the parent directory %r of the path %r" % ( - partdir, this_dir) - if not error: - # check for bogus end-of-line style (e.g. files checked out on - # Windows and moved to Unix) - f = open(__file__.replace('.pyc', '.py'), 'r') - data = f.read() - f.close() - if data.endswith('\r\n') or data.endswith('\r'): - error = ("Bad end-of-line style in the .py files. Typically " - "caused by a zip file or a checkout done on Windows and " - "moved to Unix or vice-versa.") - if error: - raise EnvironmentError("Invalid source tree - bogus checkout! " + - error) - - pypy_root = os.path.join(head, '') - try: - sys.path.remove(head) - except ValueError: - pass - sys.path.insert(0, head) - - munged = {} - for name, mod in sys.modules.items(): - if '.' in name: - continue - fn = getattr(mod, '__file__', None) - if not isinstance(fn, str): - continue - newname = os.path.splitext(os.path.basename(fn))[0] - if not newname.startswith(part + '.'): - continue - path = os.path.join(os.path.dirname(os.path.realpath(fn)), '') - if path.startswith(pypy_root) and newname != part: - modpaths = os.path.normpath(path[len(pypy_root):]).split(os.sep) - if newname != '__init__': - modpaths.append(newname) - modpath = '.'.join(modpaths) - if modpath not in sys.modules: - munged[modpath] = mod - - for name, mod in munged.iteritems(): - if name not in sys.modules: - sys.modules[name] = mod - if '.' in name: - prename = name[:name.rfind('.')] - postname = name[len(prename)+1:] - if prename not in sys.modules: - __import__(prename) - if not hasattr(sys.modules[prename], postname): - setattr(sys.modules[prename], postname, mod) - - return partdir, this_dir - -def __clone(): - """ clone master version of autopath.py into all subdirs """ - from os.path import join, walk - if not this_dir.endswith(join('pypy','tool')): - raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) - - - def sync_walker(arg, dirname, fnames): - if _myname in fnames: - fn = join(dirname, _myname) - f = open(fn, 'rwb+') - try: - if f.read() == arg: - print "checkok", fn - else: - print "syncing", fn - f = open(fn, 'w') - f.write(arg) - finally: - f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) - -_myname = 'autopath.py' - -# set guaranteed attributes - -pypydir, this_dir = __dirinfo('pypy') - -if __name__ == '__main__': - __clone() diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -1,6 +1,3 @@ -import autopath - - class AppTestCodecs: spaceconfig = { "usemodules": ['unicodedata', 'struct', 'binascii'], diff --git a/pypy/module/_lsprof/interp_lsprof.py b/pypy/module/_lsprof/interp_lsprof.py --- a/pypy/module/_lsprof/interp_lsprof.py +++ b/pypy/module/_lsprof/interp_lsprof.py @@ -11,7 +11,7 @@ from rpython.rlib.rtimer import read_timestamp, _is_64_bit from rpython.rtyper.lltypesystem import rffi, lltype from rpython.translator.tool.cbuild import ExternalCompilationInfo -from pypy.tool.autopath import pypydir +from pypy.conftest import pypydir from rpython.rlib.rarithmetic import r_longlong import time, sys diff --git a/pypy/module/_minimal_curses/test/test_curses.py b/pypy/module/_minimal_curses/test/test_curses.py --- a/pypy/module/_minimal_curses/test/test_curses.py +++ b/pypy/module/_minimal_curses/test/test_curses.py @@ -1,4 +1,4 @@ -from pypy.tool.autopath import pypydir +from pypy.conftest import pypydir from rpython.tool.udir import udir import py import sys diff --git a/pypy/module/_multibytecodec/c_codecs.py b/pypy/module/_multibytecodec/c_codecs.py --- a/pypy/module/_multibytecodec/c_codecs.py +++ b/pypy/module/_multibytecodec/c_codecs.py @@ -1,7 +1,7 @@ import py from rpython.rtyper.lltypesystem import lltype, rffi from rpython.translator.tool.cbuild import ExternalCompilationInfo -from pypy.tool.autopath import pypydir +from pypy.conftest import pypydir UNICODE_REPLACEMENT_CHARACTER = u'\uFFFD' diff --git a/pypy/module/_sre/test/autopath.py b/pypy/module/_sre/test/autopath.py deleted file mode 100644 --- a/pypy/module/_sre/test/autopath.py +++ /dev/null @@ -1,131 +0,0 @@ -""" -self cloning, automatic path configuration - -copy this into any subdirectory of pypy from which scripts need -to be run, typically all of the test subdirs. -The idea is that any such script simply issues - - import autopath - -and this will make sure that the parent directory containing "pypy" -is in sys.path. - -If you modify the master "autopath.py" version (in pypy/tool/autopath.py) -you can directly run it which will copy itself on all autopath.py files -it finds under the pypy root directory. - -This module always provides these attributes: - - pypydir pypy root directory path - this_dir directory where this autopath.py resides - -""" - -def __dirinfo(part): - """ return (partdir, this_dir) and insert parent of partdir - into sys.path. If the parent directories don't have the part - an EnvironmentError is raised.""" - - import sys, os - try: - head = this_dir = os.path.realpath(os.path.dirname(__file__)) - except NameError: - head = this_dir = os.path.realpath(os.path.dirname(sys.argv[0])) - - error = None - while head: - partdir = head - head, tail = os.path.split(head) - if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') - if not os.path.exists(checkfile): - error = "Cannot find %r" % (os.path.normpath(checkfile),) - break - else: - error = "Cannot find the parent directory %r of the path %r" % ( - partdir, this_dir) - if not error: - # check for bogus end-of-line style (e.g. files checked out on - # Windows and moved to Unix) - f = open(__file__.replace('.pyc', '.py'), 'r') - data = f.read() - f.close() - if data.endswith('\r\n') or data.endswith('\r'): - error = ("Bad end-of-line style in the .py files. Typically " - "caused by a zip file or a checkout done on Windows and " - "moved to Unix or vice-versa.") - if error: - raise EnvironmentError("Invalid source tree - bogus checkout! " + - error) - - pypy_root = os.path.join(head, '') - try: - sys.path.remove(head) - except ValueError: - pass - sys.path.insert(0, head) - - munged = {} - for name, mod in sys.modules.items(): - if '.' in name: - continue - fn = getattr(mod, '__file__', None) - if not isinstance(fn, str): - continue - newname = os.path.splitext(os.path.basename(fn))[0] - if not newname.startswith(part + '.'): - continue - path = os.path.join(os.path.dirname(os.path.realpath(fn)), '') - if path.startswith(pypy_root) and newname != part: - modpaths = os.path.normpath(path[len(pypy_root):]).split(os.sep) - if newname != '__init__': - modpaths.append(newname) - modpath = '.'.join(modpaths) - if modpath not in sys.modules: - munged[modpath] = mod - - for name, mod in munged.iteritems(): - if name not in sys.modules: - sys.modules[name] = mod - if '.' in name: - prename = name[:name.rfind('.')] - postname = name[len(prename)+1:] - if prename not in sys.modules: - __import__(prename) - if not hasattr(sys.modules[prename], postname): - setattr(sys.modules[prename], postname, mod) - - return partdir, this_dir - -def __clone(): - """ clone master version of autopath.py into all subdirs """ - from os.path import join, walk - if not this_dir.endswith(join('pypy','tool')): - raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) - - - def sync_walker(arg, dirname, fnames): - if _myname in fnames: - fn = join(dirname, _myname) - f = open(fn, 'rwb+') - try: - if f.read() == arg: - print "checkok", fn - else: - print "syncing", fn - f = open(fn, 'w') - f.write(arg) - finally: - f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) - -_myname = 'autopath.py' - -# set guaranteed attributes - -pypydir, this_dir = __dirinfo('pypy') - -if __name__ == '__main__': - __clone() diff --git a/pypy/module/_sre/test/test_app_sre.py b/pypy/module/_sre/test/test_app_sre.py --- a/pypy/module/_sre/test/test_app_sre.py +++ b/pypy/module/_sre/test/test_app_sre.py @@ -1,11 +1,10 @@ """Regular expression tests specific to _sre.py and accumulated during TDD.""" -import autopath import py from py.test import raises, skip from pypy.interpreter.gateway import app2interp_temp def init_app_test(cls, space): - cls.w_s = space.appexec([space.wrap(autopath.this_dir)], + cls.w_s = space.appexec([space.wrap(os.path.realpath(os.path.dirname(__file__)))], """(this_dir): import sys # Uh-oh, ugly hack diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -4,7 +4,7 @@ import py -from rpython.translator.goal import autopath +from pypy.conftest import pypydir from rpython.rtyper.lltypesystem import rffi, lltype from rpython.rtyper.tool import rffi_platform from rpython.rtyper.lltypesystem import ll2ctypes @@ -45,7 +45,7 @@ size_t = rffi.ULONG ADDR = lltype.Signed -pypydir = py.path.local(autopath.pypydir) +pypydir = py.path.local(pypydir) include_dir = pypydir / 'module' / 'cpyext' / 'include' source_dir = pypydir / 'module' / 'cpyext' / 'src' translator_c_dir = pypydir / 'translator' / 'c' diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -4,6 +4,7 @@ import py +from pypy.conftest import pypydir from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import interp2app, unwrap_spec from rpython.rtyper.lltypesystem import rffi, lltype, ll2ctypes @@ -15,7 +16,6 @@ from pypy.module.cpyext.state import State from pypy.module.cpyext.pyobject import RefcountState from pypy.module.cpyext.pyobject import Py_DecRef, InvalidPointerException -from rpython.translator.goal import autopath from rpython.tool.identity_dict import identity_dict from rpython.tool import leakfinder @@ -235,7 +235,7 @@ else: if filename is None: filename = name - filename = py.path.local(autopath.pypydir) / 'module' \ + filename = py.path.local(pypydir) / 'module' \ / 'cpyext'/ 'test' / (filename + ".c") kwds = dict(separate_module_files=[filename]) @@ -304,7 +304,7 @@ self.w_record_imported_module = self.space.wrap( interp2app(record_imported_module)) self.w_here = self.space.wrap( - str(py.path.local(autopath.pypydir)) + '/module/cpyext/test/') + str(py.path.local(pypydir)) + '/module/cpyext/test/') # create the file lock before we count allocations diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -5,7 +5,7 @@ from pypy.objspace.std import StdObjSpace from rpython.tool.udir import udir from pypy.tool.pytest.objspace import gettestobjspace -from pypy.tool.autopath import pypydir +from pypy.conftest import pypydir from rpython.rtyper.module.ll_os import RegisterOs import os import py diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py --- a/pypy/module/signal/interp_signal.py +++ b/pypy/module/signal/interp_signal.py @@ -9,7 +9,7 @@ from rpython.translator.tool.cbuild import ExternalCompilationInfo import py import sys -from pypy.tool import autopath +from pypy.conftest import pypydir from rpython.rlib import jit, rposix from rpython.rlib.rarithmetic import intmask, is_valid_int @@ -41,7 +41,7 @@ if sys.platform != 'win32': includes.append('sys/time.h') -cdir = py.path.local(autopath.pypydir).join('translator', 'c') +cdir = py.path.local(pypydir).join('translator', 'c') eci = ExternalCompilationInfo( includes = includes, diff --git a/pypy/module/sys/test/autopath.py b/pypy/module/sys/test/autopath.py deleted file mode 100644 --- a/pypy/module/sys/test/autopath.py +++ /dev/null @@ -1,131 +0,0 @@ -""" -self cloning, automatic path configuration - -copy this into any subdirectory of pypy from which scripts need -to be run, typically all of the test subdirs. -The idea is that any such script simply issues - - import autopath - -and this will make sure that the parent directory containing "pypy" -is in sys.path. - -If you modify the master "autopath.py" version (in pypy/tool/autopath.py) -you can directly run it which will copy itself on all autopath.py files -it finds under the pypy root directory. - -This module always provides these attributes: - - pypydir pypy root directory path - this_dir directory where this autopath.py resides - -""" - -def __dirinfo(part): - """ return (partdir, this_dir) and insert parent of partdir - into sys.path. If the parent directories don't have the part - an EnvironmentError is raised.""" - - import sys, os - try: - head = this_dir = os.path.realpath(os.path.dirname(__file__)) - except NameError: - head = this_dir = os.path.realpath(os.path.dirname(sys.argv[0])) - - error = None - while head: - partdir = head - head, tail = os.path.split(head) - if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') - if not os.path.exists(checkfile): - error = "Cannot find %r" % (os.path.normpath(checkfile),) - break - else: - error = "Cannot find the parent directory %r of the path %r" % ( - partdir, this_dir) - if not error: - # check for bogus end-of-line style (e.g. files checked out on - # Windows and moved to Unix) - f = open(__file__.replace('.pyc', '.py'), 'r') - data = f.read() - f.close() - if data.endswith('\r\n') or data.endswith('\r'): - error = ("Bad end-of-line style in the .py files. Typically " - "caused by a zip file or a checkout done on Windows and " - "moved to Unix or vice-versa.") - if error: - raise EnvironmentError("Invalid source tree - bogus checkout! " + - error) - - pypy_root = os.path.join(head, '') - try: - sys.path.remove(head) - except ValueError: - pass - sys.path.insert(0, head) - - munged = {} - for name, mod in sys.modules.items(): - if '.' in name: - continue - fn = getattr(mod, '__file__', None) - if not isinstance(fn, str): - continue - newname = os.path.splitext(os.path.basename(fn))[0] - if not newname.startswith(part + '.'): - continue - path = os.path.join(os.path.dirname(os.path.realpath(fn)), '') - if path.startswith(pypy_root) and newname != part: - modpaths = os.path.normpath(path[len(pypy_root):]).split(os.sep) - if newname != '__init__': - modpaths.append(newname) - modpath = '.'.join(modpaths) - if modpath not in sys.modules: - munged[modpath] = mod - - for name, mod in munged.iteritems(): - if name not in sys.modules: - sys.modules[name] = mod - if '.' in name: - prename = name[:name.rfind('.')] - postname = name[len(prename)+1:] - if prename not in sys.modules: - __import__(prename) - if not hasattr(sys.modules[prename], postname): - setattr(sys.modules[prename], postname, mod) - - return partdir, this_dir - -def __clone(): - """ clone master version of autopath.py into all subdirs """ - from os.path import join, walk - if not this_dir.endswith(join('pypy','tool')): - raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) - - - def sync_walker(arg, dirname, fnames): - if _myname in fnames: - fn = join(dirname, _myname) - f = open(fn, 'rwb+') - try: - if f.read() == arg: - print "checkok", fn - else: - print "syncing", fn - f = open(fn, 'w') - f.write(arg) - finally: - f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) - -_myname = 'autopath.py' - -# set guaranteed attributes - -pypydir, this_dir = __dirinfo('pypy') - -if __name__ == '__main__': - __clone() diff --git a/pypy/module/termios/test/test_termios.py b/pypy/module/termios/test/test_termios.py --- a/pypy/module/termios/test/test_termios.py +++ b/pypy/module/termios/test/test_termios.py @@ -1,7 +1,7 @@ import py import sys -from pypy.tool.autopath import pypydir +from pypy.conftest import pypydir from rpython.tool.udir import udir class TestTermios(object): diff --git a/pypy/module/thread/ll_thread.py b/pypy/module/thread/ll_thread.py --- a/pypy/module/thread/ll_thread.py +++ b/pypy/module/thread/ll_thread.py @@ -7,12 +7,12 @@ from rpython.rlib.objectmodel import we_are_translated, specialize from rpython.rtyper.lltypesystem.lloperation import llop from rpython.rtyper.tool import rffi_platform -from pypy.tool import autopath +from pypy.conftest import pypydir class error(Exception): pass -pypydir = py.path.local(autopath.pypydir) +pypydir = py.path.local(pypydir) translator_c_dir = pypydir / 'translator' / 'c' eci = ExternalCompilationInfo( diff --git a/pypy/test_all.py b/pypy/test_all.py old mode 100644 new mode 100755 --- a/pypy/test_all.py +++ b/pypy/test_all.py @@ -24,7 +24,6 @@ print >> sys.stderr, __doc__ sys.exit(2) - import tool.autopath import pytest import pytest_cov sys.exit(pytest.main(plugins=[pytest_cov])) diff --git a/pypy/tool/autopath.py b/pypy/tool/autopath.py deleted file mode 100644 --- a/pypy/tool/autopath.py +++ /dev/null @@ -1,131 +0,0 @@ -""" -self cloning, automatic path configuration - -copy this into any subdirectory of pypy from which scripts need -to be run, typically all of the test subdirs. -The idea is that any such script simply issues - - import autopath - -and this will make sure that the parent directory containing "pypy" -is in sys.path. - -If you modify the master "autopath.py" version (in pypy/tool/autopath.py) -you can directly run it which will copy itself on all autopath.py files -it finds under the pypy root directory. - -This module always provides these attributes: - - pypydir pypy root directory path - this_dir directory where this autopath.py resides - -""" - -def __dirinfo(part): - """ return (partdir, this_dir) and insert parent of partdir - into sys.path. If the parent directories don't have the part - an EnvironmentError is raised.""" - - import sys, os - try: - head = this_dir = os.path.realpath(os.path.dirname(__file__)) - except NameError: - head = this_dir = os.path.realpath(os.path.dirname(sys.argv[0])) - - error = None - while head: - partdir = head - head, tail = os.path.split(head) - if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') - if not os.path.exists(checkfile): - error = "Cannot find %r" % (os.path.normpath(checkfile),) - break - else: - error = "Cannot find the parent directory %r of the path %r" % ( - partdir, this_dir) - if not error: - # check for bogus end-of-line style (e.g. files checked out on - # Windows and moved to Unix) - f = open(__file__.replace('.pyc', '.py'), 'r') - data = f.read() - f.close() - if data.endswith('\r\n') or data.endswith('\r'): - error = ("Bad end-of-line style in the .py files. Typically " - "caused by a zip file or a checkout done on Windows and " - "moved to Unix or vice-versa.") - if error: - raise EnvironmentError("Invalid source tree - bogus checkout! " + - error) - - pypy_root = os.path.join(head, '') - try: - sys.path.remove(head) - except ValueError: - pass - sys.path.insert(0, head) - - munged = {} - for name, mod in sys.modules.items(): - if '.' in name: - continue - fn = getattr(mod, '__file__', None) - if not isinstance(fn, str): - continue - newname = os.path.splitext(os.path.basename(fn))[0] - if not newname.startswith(part + '.'): - continue - path = os.path.join(os.path.dirname(os.path.realpath(fn)), '') - if path.startswith(pypy_root) and newname != part: - modpaths = os.path.normpath(path[len(pypy_root):]).split(os.sep) - if newname != '__init__': - modpaths.append(newname) - modpath = '.'.join(modpaths) - if modpath not in sys.modules: - munged[modpath] = mod - - for name, mod in munged.iteritems(): - if name not in sys.modules: - sys.modules[name] = mod - if '.' in name: - prename = name[:name.rfind('.')] - postname = name[len(prename)+1:] - if prename not in sys.modules: - __import__(prename) - if not hasattr(sys.modules[prename], postname): - setattr(sys.modules[prename], postname, mod) - - return partdir, this_dir - -def __clone(): - """ clone master version of autopath.py into all subdirs """ - from os.path import join, walk - if not this_dir.endswith(join('pypy','tool')): - raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) - - - def sync_walker(arg, dirname, fnames): - if _myname in fnames: - fn = join(dirname, _myname) - f = open(fn, 'rwb+') - try: - if f.read() == arg: - print "checkok", fn - else: - print "syncing", fn - f = open(fn, 'w') - f.write(arg) - finally: - f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) - -_myname = 'autopath.py' - -# set guaranteed attributes - -pypydir, this_dir = __dirinfo('pypy') - -if __name__ == '__main__': - __clone() diff --git a/pypy/tool/gcc_cache.py b/pypy/tool/gcc_cache.py --- a/pypy/tool/gcc_cache.py +++ b/pypy/tool/gcc_cache.py @@ -1,4 +1,4 @@ -from pypy.tool.autopath import pypydir +from pypy.conftest import pypydir from rpython.translator.platform import CompilationError from rpython.translator.tool.cbuild import ExternalCompilationInfo from pypy.tool.compat import md5 diff --git a/pypy/tool/genstatistic.py b/pypy/tool/genstatistic.py --- a/pypy/tool/genstatistic.py +++ b/pypy/tool/genstatistic.py @@ -1,10 +1,10 @@ -import autopath import py from py._cmdline import pycountloc as countloc from py.xml import raw +from pypy import conftest -pypydir = py.path.local(autopath.pypydir) +pypydir = py.path.local(conftest.pypydir) def isdocfile(p): return (p.ext in ('.txt', '.rst') or @@ -25,7 +25,7 @@ def __init__(self, rel): self.rel = rel def __call__(self, p): - return p.relto(autopath.pypydir).startswith(self.rel) + return p.relto(conftest.pypydir).startswith(self.rel) def isfile(p): return p.check(file=1) and p.ext in ('.py', '.txt', '') @@ -42,7 +42,7 @@ def getpypycounter(): filecounter = countloc.FileCounter() - root = py.path.local(autopath.pypydir) + root = py.path.local(conftest.pypydir) filecounter.addrecursive(root, isfile, rec=recpypy) return filecounter @@ -121,7 +121,7 @@ print "writing source statistics to", target pypycounter = getpypycounter() model = CounterModel(pypycounter) - rev = py.path.svnwc(autopath.pypydir).info().rev + rev = py.path.svnwc(conftest.pypydir).info().rev html = py.xml.html doc = html.html( html.head( diff --git a/pypy/tool/getdocstrings.py b/pypy/tool/getdocstrings.py --- a/pypy/tool/getdocstrings.py +++ b/pypy/tool/getdocstrings.py @@ -1,9 +1,9 @@ -import autopath import re from os import listdir from sys import stdin, stdout, stderr +from pypy.conftest import pypydir -where = autopath.pypydir + '/objspace/std/' +where = pypydir + '/objspace/std/' quote = '(' + "'" + '|' + '"' + ')' triplequotes = '(' + "'''" + '|' + '"""' + ')' # Note: this will produce erroneous result if you nest triple quotes diff --git a/pypy/tool/import_graph.py b/pypy/tool/import_graph.py --- a/pypy/tool/import_graph.py +++ b/pypy/tool/import_graph.py @@ -1,11 +1,10 @@ from __future__ import division -import autopath import py import math import random -exclude_files = ["__init__.py", "autopath.py", "conftest.py"] +exclude_files = ["__init__.py", "conftest.py"] def include_file(path): if ("test" in str(path) or "tool" in str(path) or diff --git a/pypy/tool/isolate_slave.py b/pypy/tool/isolate_slave.py --- a/pypy/tool/isolate_slave.py +++ b/pypy/tool/isolate_slave.py @@ -1,4 +1,3 @@ -import autopath import sys, imp from pypy.tool import slaveproc diff --git a/pypy/tool/pydis.py b/pypy/tool/pydis.py --- a/pypy/tool/pydis.py +++ b/pypy/tool/pydis.py @@ -5,7 +5,6 @@ """ -import autopath import sys from rpython.tool import stdlib_opcode diff --git a/pypy/tool/pytest/appsupport.py b/pypy/tool/pytest/appsupport.py --- a/pypy/tool/pytest/appsupport.py +++ b/pypy/tool/pytest/appsupport.py @@ -1,4 +1,3 @@ -import autopath import py from pypy.interpreter import gateway, pycode from pypy.interpreter.error import OperationError diff --git a/pypy/tool/pytest/autopath.py b/pypy/tool/pytest/autopath.py deleted file mode 100644 --- a/pypy/tool/pytest/autopath.py +++ /dev/null @@ -1,131 +0,0 @@ -""" -self cloning, automatic path configuration - -copy this into any subdirectory of pypy from which scripts need -to be run, typically all of the test subdirs. -The idea is that any such script simply issues - - import autopath - -and this will make sure that the parent directory containing "pypy" -is in sys.path. - -If you modify the master "autopath.py" version (in pypy/tool/autopath.py) -you can directly run it which will copy itself on all autopath.py files -it finds under the pypy root directory. - -This module always provides these attributes: - - pypydir pypy root directory path - this_dir directory where this autopath.py resides - -""" - -def __dirinfo(part): - """ return (partdir, this_dir) and insert parent of partdir - into sys.path. If the parent directories don't have the part - an EnvironmentError is raised.""" - - import sys, os - try: - head = this_dir = os.path.realpath(os.path.dirname(__file__)) - except NameError: - head = this_dir = os.path.realpath(os.path.dirname(sys.argv[0])) - - error = None - while head: - partdir = head - head, tail = os.path.split(head) - if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') - if not os.path.exists(checkfile): - error = "Cannot find %r" % (os.path.normpath(checkfile),) - break - else: - error = "Cannot find the parent directory %r of the path %r" % ( - partdir, this_dir) - if not error: - # check for bogus end-of-line style (e.g. files checked out on - # Windows and moved to Unix) - f = open(__file__.replace('.pyc', '.py'), 'r') - data = f.read() - f.close() - if data.endswith('\r\n') or data.endswith('\r'): - error = ("Bad end-of-line style in the .py files. Typically " - "caused by a zip file or a checkout done on Windows and " - "moved to Unix or vice-versa.") - if error: - raise EnvironmentError("Invalid source tree - bogus checkout! " + - error) - - pypy_root = os.path.join(head, '') - try: - sys.path.remove(head) - except ValueError: - pass - sys.path.insert(0, head) - - munged = {} - for name, mod in sys.modules.items(): - if '.' in name: - continue - fn = getattr(mod, '__file__', None) - if not isinstance(fn, str): - continue - newname = os.path.splitext(os.path.basename(fn))[0] - if not newname.startswith(part + '.'): - continue - path = os.path.join(os.path.dirname(os.path.realpath(fn)), '') - if path.startswith(pypy_root) and newname != part: - modpaths = os.path.normpath(path[len(pypy_root):]).split(os.sep) - if newname != '__init__': - modpaths.append(newname) - modpath = '.'.join(modpaths) - if modpath not in sys.modules: - munged[modpath] = mod - - for name, mod in munged.iteritems(): - if name not in sys.modules: - sys.modules[name] = mod - if '.' in name: - prename = name[:name.rfind('.')] - postname = name[len(prename)+1:] - if prename not in sys.modules: - __import__(prename) - if not hasattr(sys.modules[prename], postname): - setattr(sys.modules[prename], postname, mod) - - return partdir, this_dir - -def __clone(): - """ clone master version of autopath.py into all subdirs """ - from os.path import join, walk - if not this_dir.endswith(join('pypy','tool')): - raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) - - - def sync_walker(arg, dirname, fnames): - if _myname in fnames: - fn = join(dirname, _myname) - f = open(fn, 'rwb+') - try: - if f.read() == arg: - print "checkok", fn - else: - print "syncing", fn - f = open(fn, 'w') - f.write(arg) - finally: - f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) - -_myname = 'autopath.py' - -# set guaranteed attributes - -pypydir, this_dir = __dirinfo('pypy') - -if __name__ == '__main__': - __clone() diff --git a/pypy/tool/pytest/confpath.py b/pypy/tool/pytest/confpath.py --- a/pypy/tool/pytest/confpath.py +++ b/pypy/tool/pytest/confpath.py @@ -1,4 +1,3 @@ -import autopath import py import pypy from pypy.tool import lib_pypy diff --git a/pypy/tool/pytest/expecttest.py b/pypy/tool/pytest/expecttest.py --- a/pypy/tool/pytest/expecttest.py +++ b/pypy/tool/pytest/expecttest.py @@ -9,7 +9,7 @@ import py import os, sys from rpython.tool.udir import udir -from pypy.tool.autopath import pypydir +from pypy.conftest import pypydir class ExpectTestMethod(py.test.collect.Function): diff --git a/pypy/tool/pytest/genreportdata.py b/pypy/tool/pytest/genreportdata.py --- a/pypy/tool/pytest/genreportdata.py +++ b/pypy/tool/pytest/genreportdata.py @@ -1,5 +1,4 @@ #! /usr/bin/env python -import autopath import py import sys diff --git a/pypy/tool/release/autopath.py b/pypy/tool/release/autopath.py deleted file mode 100644 --- a/pypy/tool/release/autopath.py +++ /dev/null @@ -1,131 +0,0 @@ -""" -self cloning, automatic path configuration - -copy this into any subdirectory of pypy from which scripts need -to be run, typically all of the test subdirs. -The idea is that any such script simply issues - - import autopath - -and this will make sure that the parent directory containing "pypy" -is in sys.path. - -If you modify the master "autopath.py" version (in pypy/tool/autopath.py) -you can directly run it which will copy itself on all autopath.py files -it finds under the pypy root directory. - -This module always provides these attributes: - - pypydir pypy root directory path - this_dir directory where this autopath.py resides - -""" - -def __dirinfo(part): - """ return (partdir, this_dir) and insert parent of partdir - into sys.path. If the parent directories don't have the part - an EnvironmentError is raised.""" - - import sys, os - try: - head = this_dir = os.path.realpath(os.path.dirname(__file__)) - except NameError: - head = this_dir = os.path.realpath(os.path.dirname(sys.argv[0])) - - error = None - while head: - partdir = head - head, tail = os.path.split(head) - if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') - if not os.path.exists(checkfile): - error = "Cannot find %r" % (os.path.normpath(checkfile),) - break - else: - error = "Cannot find the parent directory %r of the path %r" % ( - partdir, this_dir) - if not error: - # check for bogus end-of-line style (e.g. files checked out on - # Windows and moved to Unix) - f = open(__file__.replace('.pyc', '.py'), 'r') - data = f.read() - f.close() - if data.endswith('\r\n') or data.endswith('\r'): - error = ("Bad end-of-line style in the .py files. Typically " - "caused by a zip file or a checkout done on Windows and " - "moved to Unix or vice-versa.") - if error: - raise EnvironmentError("Invalid source tree - bogus checkout! " + - error) - - pypy_root = os.path.join(head, '') - try: - sys.path.remove(head) - except ValueError: - pass - sys.path.insert(0, head) - - munged = {} - for name, mod in sys.modules.items(): - if '.' in name: - continue - fn = getattr(mod, '__file__', None) - if not isinstance(fn, str): - continue - newname = os.path.splitext(os.path.basename(fn))[0] - if not newname.startswith(part + '.'): - continue - path = os.path.join(os.path.dirname(os.path.realpath(fn)), '') - if path.startswith(pypy_root) and newname != part: - modpaths = os.path.normpath(path[len(pypy_root):]).split(os.sep) - if newname != '__init__': - modpaths.append(newname) - modpath = '.'.join(modpaths) - if modpath not in sys.modules: - munged[modpath] = mod - - for name, mod in munged.iteritems(): - if name not in sys.modules: From noreply at buildbot.pypy.org Sat Jan 5 00:53:09 2013 From: noreply at buildbot.pypy.org (Trundle) Date: Sat, 5 Jan 2013 00:53:09 +0100 (CET) Subject: [pypy-commit] pyrepl default: Pass correct number of arguments to commands. Message-ID: <20130104235309.89EBA1C0DD9@cobra.cs.uni-duesseldorf.de> Author: Andreas St?hrk Branch: Changeset: r204:3b7b8ac1ca43 Date: 2013-01-05 00:46 +0100 http://bitbucket.org/pypy/pyrepl/changeset/3b7b8ac1ca43/ Log: Pass correct number of arguments to commands. Fixes issue #4. diff --git a/pyrepl/reader.py b/pyrepl/reader.py --- a/pyrepl/reader.py +++ b/pyrepl/reader.py @@ -526,7 +526,7 @@ cmd = self.commands.get(cmd[0], commands.invalid_command)(self, *cmd) elif isinstance(cmd[0], type): - cmd = cmd[0](self, cmd) + cmd = cmd[0](self, *cmd) else: return # nothing to do diff --git a/testing/test_bugs.py b/testing/test_bugs.py --- a/testing/test_bugs.py +++ b/testing/test_bugs.py @@ -17,15 +17,26 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -from .infrastructure import EA, read_spec +from pyrepl.historical_reader import HistoricalReader +from .infrastructure import EA, TestReader, read_spec # this test case should contain as-verbatim-as-possible versions of # (applicable) bug reports import pytest +class HistoricalTestReader(HistoricalReader, TestReader): + pass + @pytest.mark.xfail(reason='event missing', run=False) def test_transpose_at_start(): read_spec([( 'transpose', [EA, '']), ( 'accept', [''])]) +def test_cmd_instantiation_crash(): + spec = [ + ('reverse-history-isearch', ["(r-search `') "]), + (('key', 'left'), ['']), + ('accept', ['']) + ] + read_spec(spec, HistoricalTestReader) From noreply at buildbot.pypy.org Sat Jan 5 00:55:07 2013 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Sat, 5 Jan 2013 00:55:07 +0100 (CET) Subject: [pypy-commit] pyrepl default: add the fixed #4 to the changelog Message-ID: <20130104235507.0A04F1C0DD9@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: Changeset: r205:5504f4c783a1 Date: 2013-01-05 00:54 +0100 http://bitbucket.org/pypy/pyrepl/changeset/5504f4c783a1/ Log: add the fixed #4 to the changelog diff --git a/CHANGES b/CHANGES --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,11 @@ This file contains more verbose descriptions of the changes between versions than those in README. +New in Version 0.8.x: + + * propper command usage + fix one wrong call (thanks Trundle) + + New in 0.7.3: + ^T at the start of a line should not crash. From noreply at buildbot.pypy.org Sat Jan 5 00:58:23 2013 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Sat, 5 Jan 2013 00:58:23 +0100 (CET) Subject: [pypy-commit] pyrepl default: kill the unused c utils Message-ID: <20130104235823.360201C0DD9@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: Changeset: r206:33b744b9b942 Date: 2013-01-05 00:58 +0100 http://bitbucket.org/pypy/pyrepl/changeset/33b744b9b942/ Log: kill the unused c utils diff --git a/pyrepl_utilsmodule.c b/pyrepl_utilsmodule.c deleted file mode 100644 --- a/pyrepl_utilsmodule.c +++ /dev/null @@ -1,175 +0,0 @@ - -/* Copyright 2000-2001 Michael Hudson mwh at python.net - * - * All Rights Reserved - * - * - * Permission to use, copy, modify, and distribute this software and - * its documentation for any purpose is hereby granted without fee, - * provided that the above copyright notice appear in all copies and - * that both that copyright notice and this permission notice appear in - * supporting documentation. - * - * THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO - * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY - * AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, - * INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER - * RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF - * CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN - * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -#include - -/* nothing here that can't be done in Python, but it helps to be able - to do it quicker */ - -static char* _unctrl_map[255]; - -static PyObject* -pyrepl_utils_init_unctrl_map(PyObject* self, PyObject* args) -{ - PyObject* dict; - PyObject* pyc; - PyObject* pys; - int c = 0; - char cc; - - if (!PyArg_ParseTuple(args, "O", &dict)) { - return NULL; - } - - if (!PyDict_Check(dict)) { - PyErr_SetString(PyExc_TypeError, - "init_unctrl_map: must be dict"); - } - - for (c = 0; c < 256; c++) { - cc = c; - pyc = PyString_FromStringAndSize(&cc,1); - if (!pyc) return NULL; - pys = PyDict_GetItem(dict, pyc); - if (!pys) { - PyErr_Format(PyExc_KeyError, - "%c",cc); - _unctrl_map[0] = NULL; - return NULL; - } - if (!PyString_Check(pys)) { - PyErr_SetString(PyExc_TypeError, - "init_unctrl_map: found non-string"); - } - Py_INCREF(pys); /* this ain't going away */ - _unctrl_map[c] = PyString_AS_STRING(pys); - Py_DECREF(pyc); - } - - Py_INCREF(Py_None); - return Py_None; -} - -static char pyrepl_utils_init_unctrl_map_doc[] = -" init_unctrl_map(unctrl_map:dict) -> None\n\ -\n\ -Call this before calling disp_str."; - -static PyObject* -pyrepl_utils_disp_str(PyObject* self, PyObject* args) -{ - char *s; - char *r; - char **temp; - int slen = 0, rlen = 0; - int i, j, k, n; - PyObject *list, *ret; - - if (!PyArg_ParseTuple(args, "s#", &s, &slen)) { - return NULL; - } - - if (!_unctrl_map[0]) { - PyErr_SetString(PyExc_RuntimeError, - "bad boy!"); - return NULL; - } - - temp = malloc(sizeof(char*)*slen); - if (!temp) { - PyErr_NoMemory(); - return NULL; - } - - for (i = 0; i < slen; i++) { - temp[i] = _unctrl_map[(unsigned char)s[i]]; - rlen += strlen(temp[i]); - } - - r = malloc(rlen + 1); - if (!r) { - free(temp); - PyErr_NoMemory(); - return NULL; - } - - list = PyList_New(rlen); - if (!list) { - free(r); - free(temp); - return NULL; - } - - for (i = 0, j = 0; i < slen; i++) { - n = strlen(temp[i]); - memcpy(&r[j], temp[i], n); - PyList_SET_ITEM(list, j, PyInt_FromLong(1)); - k = j + 1; - j += n; - while (k < j) { - PyList_SET_ITEM(list, k, PyInt_FromLong(0)); - k++; - } - } - - free(temp); - r[rlen] = '\000'; - - ret = Py_BuildValue("(sN)", r, list); - - free(r); - - return ret; -} - -static char pyrepl_utils_disp_str_doc[] = -" disp_str(buffer:string) -> (string, [int])\n\ -\n\ -Return the string that should be the printed represenation of\n\ -|buffer| and a list detailing where the characters of |buffer|\n\ -get used up. E.g:\n\ -\n\ ->>> disp_str('\\003')\n\ -('^C', [1, 0])\n\ -\n\ -the list always contains 0s or 1s at present; it could conceivably\n\ -go higher as and when unicode support happens.\n\ -\n\ -You MUST call init_unctrl_map before using this version."; - - -PyMethodDef pyrepl_utils_methods[] = { - { "init_unctrl_map", pyrepl_utils_init_unctrl_map, - METH_VARARGS, pyrepl_utils_init_unctrl_map_doc }, - { "disp_str", pyrepl_utils_disp_str, - METH_VARARGS, pyrepl_utils_disp_str_doc }, - { NULL, NULL } -}; - -static char pyrepl_utils_doc[] = -"Utilities to help speed up pyrepl."; - -void init_pyrepl_utils(void) -{ - Py_InitModule3("_pyrepl_utils", - pyrepl_utils_methods, - pyrepl_utils_doc); -} diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -42,7 +42,6 @@ description = "A library for building flexible command line interfaces", platforms = ["unix", "linux"], packages = ["pyrepl" ], - #ext_modules = [Extension("_pyrepl_utils", ["pyrepl_utilsmodule.c"])], scripts = ["pythoni", "pythoni1"], long_description = long_desc, ) From noreply at buildbot.pypy.org Sat Jan 5 03:55:25 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Sat, 5 Jan 2013 03:55:25 +0100 (CET) Subject: [pypy-commit] pypy default: Optimize match.group('name') by making it a module dict. Message-ID: <20130105025525.54A641C01EE@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: Changeset: r59708:3d2ff1e85bf5 Date: 2013-01-04 18:55 -0800 http://bitbucket.org/pypy/pypy/changeset/3d2ff1e85bf5/ Log: Optimize match.group('name') by making it a module dict. diff --git a/lib-python/2.7/sre_parse.py b/lib-python/2.7/sre_parse.py --- a/lib-python/2.7/sre_parse.py +++ b/lib-python/2.7/sre_parse.py @@ -16,6 +16,12 @@ from sre_constants import * +try: + from __pypy__ import newdict +except ImportError: + def newdict(tp): + return {} + SPECIAL_CHARS = ".\\[{()*+?^$|" REPEAT_CHARS = "*+?{" @@ -68,7 +74,7 @@ self.flags = 0 self.open = [] self.groups = 1 - self.groupdict = {} + self.groupdict = newdict("module") def opengroup(self, name=None): gid = self.groups self.groups = gid + 1 diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py --- a/pypy/module/_sre/interp_sre.py +++ b/pypy/module/_sre/interp_sre.py @@ -90,7 +90,7 @@ # SRE_Pattern class class W_SRE_Pattern(Wrappable): - _immutable_fields_ = ["code", "flags", "num_groups"] + _immutable_fields_ = ["code", "flags", "num_groups", "w_indexgroup"] def cannot_copy_w(self): space = self.space From noreply at buildbot.pypy.org Sat Jan 5 07:12:23 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sat, 5 Jan 2013 07:12:23 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Replaced pypy.conftest.pypydir with rpython.translator.translator.cdir when possible Message-ID: <20130105061223.6B3B41C01EE@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59709:ae3cb6821f4a Date: 2013-01-05 06:58 +0100 http://bitbucket.org/pypy/pypy/changeset/ae3cb6821f4a/ Log: Replaced pypy.conftest.pypydir with rpython.translator.translator.cdir when possible diff --git a/pypy/tool/isolate_slave.py b/pypy/tool/isolate_slave.py --- a/pypy/tool/isolate_slave.py +++ b/pypy/tool/isolate_slave.py @@ -1,4 +1,5 @@ import sys, imp +sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..'))) from pypy.tool import slaveproc class IsolateSlave(slaveproc.Slave): diff --git a/rpython/rlib/_rffi_stacklet.py b/rpython/rlib/_rffi_stacklet.py --- a/rpython/rlib/_rffi_stacklet.py +++ b/rpython/rlib/_rffi_stacklet.py @@ -1,13 +1,13 @@ import py -from pypy.conftest import pypydir from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.rtyper.tool import rffi_platform from rpython.rlib.rarithmetic import is_emulated_long +from rpython.translator.translator import cdir import sys -cdir = py.path.local(pypydir) / 'translator' / 'c' +cdir = py.path.local(cdir) eci = ExternalCompilationInfo( include_dirs = [cdir], diff --git a/rpython/rlib/clibffi.py b/rpython/rlib/clibffi.py --- a/rpython/rlib/clibffi.py +++ b/rpython/rlib/clibffi.py @@ -12,9 +12,9 @@ from rpython.rlib.rdynload import DLOpenError, DLLHANDLE from rpython.rlib import jit from rpython.rlib.objectmodel import specialize -from pypy.conftest import pypydir from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.translator.platform import platform +from rpython.translator.translator import cdir import py import os import sys @@ -122,7 +122,7 @@ ]) else: USE_C_LIBFFI_MSVC = True - libffidir = py.path.local(pypydir).join('translator', 'c', 'src', 'libffi_msvc') + libffidir = py.path.local(cdir).join('src', 'libffi_msvc') if not _WIN64: asm_ifc = 'win32.c' else: diff --git a/rpython/rlib/rdtoa.py b/rpython/rlib/rdtoa.py --- a/rpython/rlib/rdtoa.py +++ b/rpython/rlib/rdtoa.py @@ -1,13 +1,13 @@ from __future__ import with_statement from rpython.rlib import rfloat from rpython.translator.tool.cbuild import ExternalCompilationInfo -from pypy.conftest import pypydir +from rpython.translator.translator import cdir from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rlib import jit from rpython.rlib.rstring import StringBuilder import py, sys -cdir = py.path.local(pypydir) / 'translator' / 'c' +cdir = py.path.local(cdir) include_dirs = [cdir] # set the word endianness based on the host's endianness diff --git a/rpython/rlib/rstack.py b/rpython/rlib/rstack.py --- a/rpython/rlib/rstack.py +++ b/rpython/rlib/rstack.py @@ -14,12 +14,12 @@ from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rtyper.lltypesystem.lloperation import llop from rpython.rtyper.controllerentry import Controller, SomeControlledInstance -from pypy.conftest import pypydir +from rpython.translator.translator import cdir from rpython.translator.tool.cbuild import ExternalCompilationInfo # ____________________________________________________________ -srcdir = py.path.local(pypydir) / 'translator' / 'c' / 'src' +srcdir = py.path.local(cdir) / 'src' compilation_info = ExternalCompilationInfo( includes=['src/stack.h'], separate_module_files=[srcdir / 'stack.c', srcdir / 'threadlocal.c']) diff --git a/rpython/rtyper/lltypesystem/module/ll_math.py b/rpython/rtyper/lltypesystem/module/ll_math.py --- a/rpython/rtyper/lltypesystem/module/ll_math.py +++ b/rpython/rtyper/lltypesystem/module/ll_math.py @@ -5,7 +5,7 @@ from rpython.rtyper.lltypesystem import lltype, rffi from rpython.tool.sourcetools import func_with_new_name -from pypy.conftest import pypydir +from rpython.translator.translator import cdir from rpython.rlib import jit, rposix from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.translator.platform import platform @@ -25,7 +25,7 @@ else: eci = ExternalCompilationInfo() # Some math functions are C99 and not defined by the Microsoft compiler - cdir = py.path.local(pypydir).join('translator', 'c') + cdir = py.path.local(cdir) math_eci = ExternalCompilationInfo( include_dirs = [cdir], includes = ['src/ll_math.h'], diff --git a/rpython/rtyper/module/ll_strtod.py b/rpython/rtyper/module/ll_strtod.py --- a/rpython/rtyper/module/ll_strtod.py +++ b/rpython/rtyper/module/ll_strtod.py @@ -3,17 +3,16 @@ from rpython.rtyper.extfunc import BaseLazyRegistering, extdef, registering from rpython.rlib import rfloat from rpython.rtyper.lltypesystem import lltype, rffi -from pypy.conftest import pypydir from rpython.rtyper.ootypesystem import ootype from rpython.rlib import rposix from rpython.translator.tool.cbuild import ExternalCompilationInfo -from pypy.conftest import pypydir +from rpython.translator.translator import cdir from rpython.annotator.model import SomeString class CConfig: _compilation_info_ = ExternalCompilationInfo( includes = ['src/ll_strtod.h'], - include_dirs = [str(py.path.local(pypydir).join('translator', 'c'))], + include_dirs = [str(py.path.local(cdir))], separate_module_sources = ['#include '], export_symbols = ['LL_strtod_formatd', 'LL_strtod_parts_to_float'], ) diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py --- a/rpython/translator/c/test/test_standalone.py +++ b/rpython/translator/c/test/test_standalone.py @@ -5,12 +5,11 @@ from rpython.rlib.rarithmetic import r_longlong from rpython.rlib.debug import ll_assert, have_debug_prints, debug_flush from rpython.rlib.debug import debug_print, debug_start, debug_stop, debug_offset -from rpython.translator.translator import TranslationContext +from rpython.translator.translator import TranslationContext, cdir from rpython.translator.backendopt import all from rpython.translator.c.genc import CStandaloneBuilder, ExternalCompilationInfo from rpython.annotator.listdef import s_list_of_strings from rpython.tool.udir import udir -from pypy.conftest import pypydir from pypy.conftest import option @@ -240,8 +239,7 @@ def test_separate_files(self): # One file in translator/c/src - fname = py.path.local(pypydir).join( - 'translator', 'c', 'src', 'll_strtod.c') + fname = py.path.local(cdir).join('src', 'll_strtod.c') # One file in (another) subdir of the temp directory dirname = udir.join("test_dir").ensure(dir=1) diff --git a/rpython/translator/translator.py b/rpython/translator/translator.py --- a/rpython/translator/translator.py +++ b/rpython/translator/translator.py @@ -6,6 +6,8 @@ """ import os, sys, types, copy +cdir = os.path.join(os.path.dirname(__file__), 'c') + from rpython.translator import simplify from rpython.flowspace.model import FunctionGraph, checkgraph, Block from rpython.flowspace.objspace import FlowObjSpace From noreply at buildbot.pypy.org Sat Jan 5 07:39:27 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sat, 5 Jan 2013 07:39:27 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Using cdir instead of pypydir in pypy module when possible Message-ID: <20130105063927.B435D1C01A6@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59710:6fa7a8c8236b Date: 2013-01-05 07:31 +0100 http://bitbucket.org/pypy/pypy/changeset/6fa7a8c8236b/ Log: Using cdir instead of pypydir in pypy module when possible diff --git a/pypy/module/_lsprof/interp_lsprof.py b/pypy/module/_lsprof/interp_lsprof.py --- a/pypy/module/_lsprof/interp_lsprof.py +++ b/pypy/module/_lsprof/interp_lsprof.py @@ -11,14 +11,14 @@ from rpython.rlib.rtimer import read_timestamp, _is_64_bit from rpython.rtyper.lltypesystem import rffi, lltype from rpython.translator.tool.cbuild import ExternalCompilationInfo -from pypy.conftest import pypydir +from rpython.translator.translator import cdir from rpython.rlib.rarithmetic import r_longlong import time, sys # cpu affinity settings -srcdir = py.path.local(pypydir).join('translator', 'c', 'src') +srcdir = py.path.local(cdir).join('src') eci = ExternalCompilationInfo( separate_module_files=[srcdir.join('profiling.c')], export_symbols=['pypy_setup_profiling', 'pypy_teardown_profiling']) diff --git a/pypy/module/_multibytecodec/c_codecs.py b/pypy/module/_multibytecodec/c_codecs.py --- a/pypy/module/_multibytecodec/c_codecs.py +++ b/pypy/module/_multibytecodec/c_codecs.py @@ -1,7 +1,7 @@ import py from rpython.rtyper.lltypesystem import lltype, rffi from rpython.translator.tool.cbuild import ExternalCompilationInfo -from pypy.conftest import pypydir +from rpython.translator.translator import cdir UNICODE_REPLACEMENT_CHARACTER = u'\uFFFD' @@ -15,7 +15,7 @@ return 'EncodeDecodeError(%r, %r, %r)' % (self.start, self.end, self.reason) -srcdir = py.path.local(pypydir).join('translator', 'c') +srcdir = py.path.local(cdir) codecs = [ # _codecs_cn diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -10,6 +10,7 @@ from rpython.rtyper.lltypesystem import ll2ctypes from rpython.rtyper.annlowlevel import llhelper from rpython.rlib.objectmodel import we_are_translated +from rpython.translator.translator import cdir from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.translator.gensupp import NameManager from rpython.tool.udir import udir @@ -48,7 +49,7 @@ pypydir = py.path.local(pypydir) include_dir = pypydir / 'module' / 'cpyext' / 'include' source_dir = pypydir / 'module' / 'cpyext' / 'src' -translator_c_dir = pypydir / 'translator' / 'c' +translator_c_dir = py.path.local(cdir) include_dirs = [ include_dir, translator_c_dir, diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py --- a/pypy/module/signal/interp_signal.py +++ b/pypy/module/signal/interp_signal.py @@ -7,9 +7,9 @@ from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rtyper.tool import rffi_platform from rpython.translator.tool.cbuild import ExternalCompilationInfo +from rpython.translator.translator import cdir import py import sys -from pypy.conftest import pypydir from rpython.rlib import jit, rposix from rpython.rlib.rarithmetic import intmask, is_valid_int @@ -41,7 +41,7 @@ if sys.platform != 'win32': includes.append('sys/time.h') -cdir = py.path.local(pypydir).join('translator', 'c') +cdir = py.path.local(cdir) eci = ExternalCompilationInfo( includes = includes, diff --git a/pypy/module/thread/ll_thread.py b/pypy/module/thread/ll_thread.py --- a/pypy/module/thread/ll_thread.py +++ b/pypy/module/thread/ll_thread.py @@ -1,6 +1,7 @@ from rpython.rtyper.lltypesystem import rffi, lltype, llmemory from rpython.translator.tool.cbuild import ExternalCompilationInfo +from rpython.translator.translator import cdir import py from rpython.rlib import jit, rgc from rpython.rlib.debug import ll_assert @@ -13,7 +14,7 @@ pass pypydir = py.path.local(pypydir) -translator_c_dir = pypydir / 'translator' / 'c' +translator_c_dir = py.path.local(cdir) eci = ExternalCompilationInfo( includes = ['src/thread.h'], From noreply at buildbot.pypy.org Sat Jan 5 08:36:05 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sat, 5 Jan 2013 08:36:05 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed imports Message-ID: <20130105073605.492C61C0DDA@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59711:31d1f6120bd2 Date: 2013-01-05 08:35 +0100 http://bitbucket.org/pypy/pypy/changeset/31d1f6120bd2/ Log: Fixed imports diff --git a/pypy/bin/py.py b/pypy/bin/py.py --- a/pypy/bin/py.py +++ b/pypy/bin/py.py @@ -6,10 +6,10 @@ """ -try: - import autopath -except ImportError: - pass +import os, sys +import time + +sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..'))) import pypy from pypy.tool import option @@ -18,8 +18,7 @@ from pypy.config.config import OptionDescription, BoolOption, StrOption from pypy.config.config import Config, to_optparse from pypy.config import pypyoption -import os, sys -import time + cmdline_optiondescr = OptionDescription("interactive", "the options of py.py", [ BoolOption("verbose", "show verbose interpreter-level traceback", diff --git a/pypy/module/_sre/test/test_app_sre.py b/pypy/module/_sre/test/test_app_sre.py --- a/pypy/module/_sre/test/test_app_sre.py +++ b/pypy/module/_sre/test/test_app_sre.py @@ -1,4 +1,5 @@ """Regular expression tests specific to _sre.py and accumulated during TDD.""" +import os import py from py.test import raises, skip from pypy.interpreter.gateway import app2interp_temp diff --git a/rpython/annotator/model.py b/rpython/annotator/model.py --- a/rpython/annotator/model.py +++ b/rpython/annotator/model.py @@ -30,7 +30,7 @@ from __future__ import absolute_import from types import BuiltinFunctionType, MethodType, FunctionType -import pypy +import rpython from pypy.tool import descriptor from rpython.tool.pairtype import pair, extendabletype from rpython.rlib.rarithmetic import r_uint, r_ulonglong, base_int diff --git a/rpython/jit/backend/x86/tool/viewcode.py b/rpython/jit/backend/x86/tool/viewcode.py --- a/rpython/jit/backend/x86/tool/viewcode.py +++ b/rpython/jit/backend/x86/tool/viewcode.py @@ -23,7 +23,7 @@ tmpfile = str(udir.join('dump.tmp')) # hack hack -import pypy.tool +import rpython.tool mod = new.module('rpython.tool.udir') mod.udir = udir sys.modules['rpython.tool.udir'] = mod From noreply at buildbot.pypy.org Sat Jan 5 09:22:07 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Sat, 5 Jan 2013 09:22:07 +0100 (CET) Subject: [pypy-commit] pypy default: fix a typo Message-ID: <20130105082207.D8BFA1C01EE@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: Changeset: r59712:83f1c0026c4e Date: 2013-01-05 00:21 -0800 http://bitbucket.org/pypy/pypy/changeset/83f1c0026c4e/ Log: fix a typo diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py --- a/pypy/module/_sre/interp_sre.py +++ b/pypy/module/_sre/interp_sre.py @@ -90,7 +90,7 @@ # SRE_Pattern class class W_SRE_Pattern(Wrappable): - _immutable_fields_ = ["code", "flags", "num_groups", "w_indexgroup"] + _immutable_fields_ = ["code", "flags", "num_groups", "w_groupindex"] def cannot_copy_w(self): space = self.space From noreply at buildbot.pypy.org Sat Jan 5 09:24:45 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Sat, 5 Jan 2013 09:24:45 +0100 (CET) Subject: [pypy-commit] pypy default: promote this Message-ID: <20130105082445.A193E1C0C35@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: Changeset: r59713:7f214ef7fd3f Date: 2013-01-05 00:24 -0800 http://bitbucket.org/pypy/pypy/changeset/7f214ef7fd3f/ Log: promote this diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py --- a/pypy/module/_sre/interp_sre.py +++ b/pypy/module/_sre/interp_sre.py @@ -54,7 +54,7 @@ result = [-1] * (2 * num_groups) mark = ctx.match_marks while mark is not None: - index = mark.gid + index = jit.promote(mark.gid) if result[index] == -1: result[index] = mark.position mark = mark.prev From noreply at buildbot.pypy.org Sat Jan 5 09:37:26 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 5 Jan 2013 09:37:26 +0100 (CET) Subject: [pypy-commit] pypy default: Fix more urls. We should at some point grep for 'codespeak' in pypy/doc Message-ID: <20130105083726.C9C801C0F8A@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r59714:ef9572dfacca Date: 2013-01-05 09:37 +0100 http://bitbucket.org/pypy/pypy/changeset/ef9572dfacca/ Log: Fix more urls. We should at some point grep for 'codespeak' in pypy/doc and cry. diff --git a/pypy/module/README.txt b/pypy/module/README.txt --- a/pypy/module/README.txt +++ b/pypy/module/README.txt @@ -3,7 +3,7 @@ that require access to interpreter level. See here for more information: - http://codespeak.net/pypy/dist/pypy/doc/coding-guide.html#modules-in-pypy + http://doc.pypy.org/en/latest/coding-guide.html#modules-in-pypy ATTENTION: don't put any '.py' files directly into pypy/module because you can easily get import mixups on e.g. "import sys" diff --git a/pypy/tool/option.py b/pypy/tool/option.py --- a/pypy/tool/option.py +++ b/pypy/tool/option.py @@ -6,7 +6,7 @@ import optparse extra_useage = """For detailed descriptions of all the options see -http://codespeak.net/pypy/dist/pypy/doc/config/commandline.html""" +http://doc.pypy.org/en/latest/config/commandline.html""" def get_standard_options(): config = get_pypy_config() diff --git a/pypy/tool/release/make_release.py b/pypy/tool/release/make_release.py deleted file mode 100755 --- a/pypy/tool/release/make_release.py +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env python - -""" A tool to download correct pypy-c's from nightly build run and package them -into release packages. Note: you must run apropriate buildbots first and -make sure there are no failures. Use force-builds.py from the same directory. - -Usage: make_release.py - e.g.: make_release.py release-1.4.1 1.4.1 -""" - -import autopath -import sys -import urllib2 -from xml.dom import minidom -import re -import py -from pypy.tool.udir import udir -from pypy.tool.release.package import package -import tarfile -import os -import shutil - -BASEURL = 'http://buildbot.pypy.org/nightly/' -PAUSE = False - -def browse_nightly(branch, - baseurl=BASEURL, - override_xml=None): - if override_xml is None: - url = baseurl + branch + '/' - xml = urllib2.urlopen(url).read() - else: - xml = override_xml - dom = minidom.parseString(xml) - refs = [node.getAttribute('href') for node in dom.getElementsByTagName('a') - if 'pypy' in node.getAttribute('href')] - # all refs are of form: pypy-c-{type}-{revnum}-{hghash}-{platform}.tar.bz2 - r = re.compile('pypy-c-([\w\d]+)-(\d+)-([0-9a-f]+)-([\w\d]+).tar.bz2$') - d = {} - for ref in refs: - kind, revnum, hghash, platform = r.match(ref).groups() - rev = int(revnum) - try: - lastrev, _ = d[(kind, platform)] - except KeyError: - lastrev = -1 - if rev > lastrev: - d[(kind, platform)] = rev, ref - return d - -def main(branch, release): - to_download = browse_nightly(branch) - tmpdir = udir.join('download') - tmpdir.ensure(dir=True) - alltars = [] - olddir = os.getcwd() - try: - os.chdir(str(tmpdir)) - print 'Using tmpdir', str(tmpdir) - for (kind, platform), (rev, name) in to_download.iteritems(): - if platform == 'win32': - print 'Ignoring %s, windows unsupported' % name - else: - print "Downloading %s at rev %d" % (name, rev) - url = BASEURL + branch + "/" + name - data = urllib2.urlopen(url).read() - tmpdir.join(name).write(data, mode="wb") - t = tarfile.open(str(tmpdir.join(name))) - dirname = t.getmembers()[0].name - t.extractall(path=str(tmpdir)) - if kind == 'jit': - kind = '' - else: - kind = '-' + kind - topdirname = 'pypy-%s-%s%s' % (release, platform, kind) - os.system('mv %s %s' % (str(tmpdir.join(dirname)), - str(tmpdir.join(topdirname)))) - if PAUSE: - print 'Pausing, press Enter...' - raw_input() - name = '%s.tar.bz2' % topdirname - print "Building %s" % name - t = tarfile.open(name, 'w:bz2') - t.add(topdirname) - alltars.append(name) - t.close() - shutil.rmtree(str(tmpdir.join(topdirname))) - for name in alltars: - print "Uploading %s" % name - os.system('scp %s codespeak.net:/www/pypy.org/htdocs/download' % name) - finally: - os.chdir(olddir) - -if __name__ == '__main__': - if len(sys.argv) != 3: - print __doc__ - sys.exit(1) - main(sys.argv[1], release=sys.argv[2]) - diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -122,7 +122,7 @@ try: os.chdir(str(builddir)) # - # 'strip' fun: see https://codespeak.net/issue/pypy-dev/issue587 + # 'strip' fun: see issue #587 for source, target in binaries: if sys.platform == 'win32': pass From noreply at buildbot.pypy.org Sat Jan 5 11:17:30 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 5 Jan 2013 11:17:30 +0100 (CET) Subject: [pypy-commit] pypy callback-jit: failing test Message-ID: <20130105101730.CE1361C01EE@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: callback-jit Changeset: r59715:9a1a38217dfe Date: 2013-01-05 12:16 +0200 http://bitbucket.org/pypy/pypy/changeset/9a1a38217dfe/ Log: failing test diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py --- a/pypy/jit/metainterp/test/test_ajit.py +++ b/pypy/jit/metainterp/test/test_ajit.py @@ -3981,3 +3981,24 @@ self.interp_operations(f, []) + def test_external_call(self): + from pypy.rlib.objectmodel import invoke_around_extcall + + T = rffi.CArrayPtr(rffi.TIME_T) + external = rffi.llexternal("time", [T], rffi.TIME_T) + + l = [] + + def before(): + l.append("before") + + def after(): + l.append("after") + + def f(): + external(lltype.nullptr(T.TO)) + return 1 + + invoke_around_extcall(before, after) + self.interp_operations(f, []) + assert len(l) == 2 From noreply at buildbot.pypy.org Sat Jan 5 13:59:18 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 5 Jan 2013 13:59:18 +0100 (CET) Subject: [pypy-commit] pypy default: failing test Message-ID: <20130105125918.5D2471C0DDA@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r59716:9b6da2d2d5c6 Date: 2013-01-05 12:16 +0200 http://bitbucket.org/pypy/pypy/changeset/9b6da2d2d5c6/ Log: failing test diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py --- a/pypy/jit/metainterp/test/test_ajit.py +++ b/pypy/jit/metainterp/test/test_ajit.py @@ -3979,5 +3979,26 @@ rgc.add_memory_pressure(1234) return 3 + def test_external_call(self): + from pypy.rlib.objectmodel import invoke_around_extcall + + T = rffi.CArrayPtr(rffi.TIME_T) + external = rffi.llexternal("time", [T], rffi.TIME_T) + + l = [] + + def before(): + l.append("before") + + def after(): + l.append("after") + + def f(): + external(lltype.nullptr(T.TO)) + return 1 + + invoke_around_extcall(before, after) + self.interp_operations(f, []) + assert len(l) == 2 self.interp_operations(f, []) From noreply at buildbot.pypy.org Sat Jan 5 13:59:19 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 5 Jan 2013 13:59:19 +0100 (CET) Subject: [pypy-commit] pypy default: Initial fix for rffi functions that release the GIL: the JIT must Message-ID: <20130105125919.C2E561C0DDA@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r59717:c760f8e27156 Date: 2013-01-05 12:15 +0100 http://bitbucket.org/pypy/pypy/changeset/c760f8e27156/ Log: Initial fix for rffi functions that release the GIL: the JIT must never look inside _gctransformer_hint_close_stack_ functions. diff --git a/pypy/jit/codewriter/call.py b/pypy/jit/codewriter/call.py --- a/pypy/jit/codewriter/call.py +++ b/pypy/jit/codewriter/call.py @@ -134,9 +134,12 @@ if getattr(funcobj, 'graph', None) is None: return 'residual' targetgraph = funcobj.graph - if (hasattr(targetgraph, 'func') and - hasattr(targetgraph.func, 'oopspec')): - return 'builtin' + if hasattr(targetgraph, 'func'): + if getattr(targetgraph.func, + '_gctransformer_hint_close_stack_', False): + return 'residual' + if hasattr(targetgraph.func, 'oopspec'): + return 'builtin' elif op.opname == 'oosend': SELFTYPE, methname, opargs = support.decompose_oosend(op) if SELFTYPE.oopspec_name is not None: diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py --- a/pypy/jit/metainterp/test/test_ajit.py +++ b/pypy/jit/metainterp/test/test_ajit.py @@ -3985,20 +3985,29 @@ T = rffi.CArrayPtr(rffi.TIME_T) external = rffi.llexternal("time", [T], rffi.TIME_T) - l = [] + class Oups(Exception): + pass + class State: + pass + state = State() def before(): - l.append("before") + if we_are_jitted(): + raise Oups + state.l.append("before") def after(): - l.append("after") + if we_are_jitted(): + raise Oups + state.l.append("after") def f(): + state.l = [] + invoke_around_extcall(before, after) external(lltype.nullptr(T.TO)) - return 1 - - invoke_around_extcall(before, after) - self.interp_operations(f, []) - assert len(l) == 2 - self.interp_operations(f, []) - + return len(state.l) + + res = self.interp_operations(f, []) + assert res == 2 + res = self.interp_operations(f, []) + assert res == 2 From noreply at buildbot.pypy.org Sat Jan 5 13:59:20 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 5 Jan 2013 13:59:20 +0100 (CET) Subject: [pypy-commit] pypy default: Check more in-depth that we don't produce a JitCode for a function with Message-ID: <20130105125920.E7D281C0DDA@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r59718:1c7011a1b5ed Date: 2013-01-05 12:19 +0100 http://bitbucket.org/pypy/pypy/changeset/1c7011a1b5ed/ Log: Check more in-depth that we don't produce a JitCode for a function with _gctransformer_hint_close_stack_ set diff --git a/pypy/jit/codewriter/call.py b/pypy/jit/codewriter/call.py --- a/pypy/jit/codewriter/call.py +++ b/pypy/jit/codewriter/call.py @@ -135,6 +135,8 @@ return 'residual' targetgraph = funcobj.graph if hasattr(targetgraph, 'func'): + # must never produce JitCode for a function with + # _gctransformer_hint_close_stack_ set! if getattr(targetgraph.func, '_gctransformer_hint_close_stack_', False): return 'residual' @@ -167,6 +169,13 @@ try: return self.jitcodes[graph] except KeyError: + # must never produce JitCode for a function with + # _gctransformer_hint_close_stack_ set! + if hasattr(graph, 'func') and getattr(graph.func, + '_gctransformer_hint_close_stack_', False): + raise AssertionError( + '%s has _gctransformer_hint_close_stack_' % (graph,)) + # fnaddr, calldescr = self.get_jitcode_calldescr(graph) jitcode = JitCode(graph.name, fnaddr, calldescr, called_from=called_from) From noreply at buildbot.pypy.org Sat Jan 5 13:59:22 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 5 Jan 2013 13:59:22 +0100 (CET) Subject: [pypy-commit] pypy remove-globals-in-jit: hg merge default Message-ID: <20130105125922.4D1201C0DDA@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: remove-globals-in-jit Changeset: r59719:cbb1b76ebfa1 Date: 2013-01-05 13:51 +0100 http://bitbucket.org/pypy/pypy/changeset/cbb1b76ebfa1/ Log: hg merge default diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -28,7 +28,7 @@ DEALINGS IN THE SOFTWARE. -PyPy Copyright holders 2003-2012 +PyPy Copyright holders 2003-2013 ----------------------------------- Except when otherwise stated (look for LICENSE files or information at diff --git a/lib-python/2.7/sre_parse.py b/lib-python/2.7/sre_parse.py --- a/lib-python/2.7/sre_parse.py +++ b/lib-python/2.7/sre_parse.py @@ -16,6 +16,12 @@ from sre_constants import * +try: + from __pypy__ import newdict +except ImportError: + def newdict(tp): + return {} + SPECIAL_CHARS = ".\\[{()*+?^$|" REPEAT_CHARS = "*+?{" @@ -68,7 +74,7 @@ self.flags = 0 self.open = [] self.groups = 1 - self.groupdict = {} + self.groupdict = newdict("module") def opengroup(self, name=None): gid = self.groups self.groups = gid + 1 diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst --- a/pypy/doc/index.rst +++ b/pypy/doc/index.rst @@ -67,7 +67,7 @@ .. _Python: http://docs.python.org/index.html .. _`more...`: architecture.html#mission-statement .. _`PyPy blog`: http://morepypy.blogspot.com/ -.. _`development bug/feature tracker`: https://codespeak.net/issue/pypy-dev/ +.. _`development bug/feature tracker`: https://bugs.pypy.org .. _here: http://tismerysoft.de/pypy/irc-logs/pypy .. _`Mercurial commit mailing list`: http://python.org/mailman/listinfo/pypy-commit .. _`development mailing list`: http://python.org/mailman/listinfo/pypy-dev diff --git a/pypy/jit/backend/arm/test/test_loop_unroll.py b/pypy/jit/backend/arm/test/test_loop_unroll.py --- a/pypy/jit/backend/arm/test/test_loop_unroll.py +++ b/pypy/jit/backend/arm/test/test_loop_unroll.py @@ -1,8 +1,8 @@ import py -from pypy.jit.backend.x86.test.test_basic import Jit386Mixin +from pypy.jit.backend.arm.test.support import JitARMMixin from pypy.jit.metainterp.test import test_loop_unroll -class TestLoopSpec(Jit386Mixin, test_loop_unroll.LoopUnrollTest): +class TestLoopSpec(JitARMMixin, test_loop_unroll.LoopUnrollTest): # for the individual tests see # ====> ../../../metainterp/test/test_loop.py pass diff --git a/pypy/jit/codewriter/call.py b/pypy/jit/codewriter/call.py --- a/pypy/jit/codewriter/call.py +++ b/pypy/jit/codewriter/call.py @@ -134,9 +134,14 @@ if getattr(funcobj, 'graph', None) is None: return 'residual' targetgraph = funcobj.graph - if (hasattr(targetgraph, 'func') and - hasattr(targetgraph.func, 'oopspec')): - return 'builtin' + if hasattr(targetgraph, 'func'): + # must never produce JitCode for a function with + # _gctransformer_hint_close_stack_ set! + if getattr(targetgraph.func, + '_gctransformer_hint_close_stack_', False): + return 'residual' + if hasattr(targetgraph.func, 'oopspec'): + return 'builtin' elif op.opname == 'oosend': SELFTYPE, methname, opargs = support.decompose_oosend(op) if SELFTYPE.oopspec_name is not None: @@ -164,6 +169,13 @@ try: return self.jitcodes[graph] except KeyError: + # must never produce JitCode for a function with + # _gctransformer_hint_close_stack_ set! + if hasattr(graph, 'func') and getattr(graph.func, + '_gctransformer_hint_close_stack_', False): + raise AssertionError( + '%s has _gctransformer_hint_close_stack_' % (graph,)) + # fnaddr, calldescr = self.get_jitcode_calldescr(graph) jitcode = JitCode(graph.name, fnaddr, calldescr, called_from=called_from) diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py --- a/pypy/jit/metainterp/test/test_ajit.py +++ b/pypy/jit/metainterp/test/test_ajit.py @@ -3979,5 +3979,35 @@ rgc.add_memory_pressure(1234) return 3 - self.interp_operations(f, []) + def test_external_call(self): + from pypy.rlib.objectmodel import invoke_around_extcall + T = rffi.CArrayPtr(rffi.TIME_T) + external = rffi.llexternal("time", [T], rffi.TIME_T) + + class Oups(Exception): + pass + class State: + pass + state = State() + + def before(): + if we_are_jitted(): + raise Oups + state.l.append("before") + + def after(): + if we_are_jitted(): + raise Oups + state.l.append("after") + + def f(): + state.l = [] + invoke_around_extcall(before, after) + external(lltype.nullptr(T.TO)) + return len(state.l) + + res = self.interp_operations(f, []) + assert res == 2 + res = self.interp_operations(f, []) + assert res == 2 diff --git a/pypy/module/README.txt b/pypy/module/README.txt --- a/pypy/module/README.txt +++ b/pypy/module/README.txt @@ -3,7 +3,7 @@ that require access to interpreter level. See here for more information: - http://codespeak.net/pypy/dist/pypy/doc/coding-guide.html#modules-in-pypy + http://doc.pypy.org/en/latest/coding-guide.html#modules-in-pypy ATTENTION: don't put any '.py' files directly into pypy/module because you can easily get import mixups on e.g. "import sys" diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py --- a/pypy/module/_sre/interp_sre.py +++ b/pypy/module/_sre/interp_sre.py @@ -54,7 +54,7 @@ result = [-1] * (2 * num_groups) mark = ctx.match_marks while mark is not None: - index = mark.gid + index = jit.promote(mark.gid) if result[index] == -1: result[index] = mark.position mark = mark.prev @@ -90,7 +90,7 @@ # SRE_Pattern class class W_SRE_Pattern(Wrappable): - _immutable_fields_ = ["code", "flags", "num_groups"] + _immutable_fields_ = ["code", "flags", "num_groups", "w_groupindex"] def cannot_copy_w(self): space = self.space diff --git a/pypy/module/sys/app.py b/pypy/module/sys/app.py --- a/pypy/module/sys/app.py +++ b/pypy/module/sys/app.py @@ -66,11 +66,11 @@ return None copyright_str = """ -Copyright 2003-2012 PyPy development team. +Copyright 2003-2013 PyPy development team. All Rights Reserved. For further information, see -Portions Copyright (c) 2001-2012 Python Software Foundation. +Portions Copyright (c) 2001-2013 Python Software Foundation. All Rights Reserved. Portions Copyright (c) 2000 BeOpen.com. diff --git a/pypy/tool/option.py b/pypy/tool/option.py --- a/pypy/tool/option.py +++ b/pypy/tool/option.py @@ -6,7 +6,7 @@ import optparse extra_useage = """For detailed descriptions of all the options see -http://codespeak.net/pypy/dist/pypy/doc/config/commandline.html""" +http://doc.pypy.org/en/latest/config/commandline.html""" def get_standard_options(): config = get_pypy_config() diff --git a/pypy/tool/release/make_release.py b/pypy/tool/release/make_release.py deleted file mode 100755 --- a/pypy/tool/release/make_release.py +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env python - -""" A tool to download correct pypy-c's from nightly build run and package them -into release packages. Note: you must run apropriate buildbots first and -make sure there are no failures. Use force-builds.py from the same directory. - -Usage: make_release.py - e.g.: make_release.py release-1.4.1 1.4.1 -""" - -import autopath -import sys -import urllib2 -from xml.dom import minidom -import re -import py -from pypy.tool.udir import udir -from pypy.tool.release.package import package -import tarfile -import os -import shutil - -BASEURL = 'http://buildbot.pypy.org/nightly/' -PAUSE = False - -def browse_nightly(branch, - baseurl=BASEURL, - override_xml=None): - if override_xml is None: - url = baseurl + branch + '/' - xml = urllib2.urlopen(url).read() - else: - xml = override_xml - dom = minidom.parseString(xml) - refs = [node.getAttribute('href') for node in dom.getElementsByTagName('a') - if 'pypy' in node.getAttribute('href')] - # all refs are of form: pypy-c-{type}-{revnum}-{hghash}-{platform}.tar.bz2 - r = re.compile('pypy-c-([\w\d]+)-(\d+)-([0-9a-f]+)-([\w\d]+).tar.bz2$') - d = {} - for ref in refs: - kind, revnum, hghash, platform = r.match(ref).groups() - rev = int(revnum) - try: - lastrev, _ = d[(kind, platform)] - except KeyError: - lastrev = -1 - if rev > lastrev: - d[(kind, platform)] = rev, ref - return d - -def main(branch, release): - to_download = browse_nightly(branch) - tmpdir = udir.join('download') - tmpdir.ensure(dir=True) - alltars = [] - olddir = os.getcwd() - try: - os.chdir(str(tmpdir)) - print 'Using tmpdir', str(tmpdir) - for (kind, platform), (rev, name) in to_download.iteritems(): - if platform == 'win32': - print 'Ignoring %s, windows unsupported' % name - else: - print "Downloading %s at rev %d" % (name, rev) - url = BASEURL + branch + "/" + name - data = urllib2.urlopen(url).read() - tmpdir.join(name).write(data, mode="wb") - t = tarfile.open(str(tmpdir.join(name))) - dirname = t.getmembers()[0].name - t.extractall(path=str(tmpdir)) - if kind == 'jit': - kind = '' - else: - kind = '-' + kind - topdirname = 'pypy-%s-%s%s' % (release, platform, kind) - os.system('mv %s %s' % (str(tmpdir.join(dirname)), - str(tmpdir.join(topdirname)))) - if PAUSE: - print 'Pausing, press Enter...' - raw_input() - name = '%s.tar.bz2' % topdirname - print "Building %s" % name - t = tarfile.open(name, 'w:bz2') - t.add(topdirname) - alltars.append(name) - t.close() - shutil.rmtree(str(tmpdir.join(topdirname))) - for name in alltars: - print "Uploading %s" % name - os.system('scp %s codespeak.net:/www/pypy.org/htdocs/download' % name) - finally: - os.chdir(olddir) - -if __name__ == '__main__': - if len(sys.argv) != 3: - print __doc__ - sys.exit(1) - main(sys.argv[1], release=sys.argv[2]) - diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -122,7 +122,7 @@ try: os.chdir(str(builddir)) # - # 'strip' fun: see https://codespeak.net/issue/pypy-dev/issue587 + # 'strip' fun: see issue #587 for source, target in binaries: if sys.platform == 'win32': pass From noreply at buildbot.pypy.org Sat Jan 5 13:59:23 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 5 Jan 2013 13:59:23 +0100 (CET) Subject: [pypy-commit] pypy default: (fijal, arigo) Message-ID: <20130105125923.73B9E1C0DDA@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r59720:2da23f0737b0 Date: 2013-01-05 13:05 +0100 http://bitbucket.org/pypy/pypy/changeset/2da23f0737b0/ Log: (fijal, arigo) Give rffi's function call_external_function() an 'extrainfo' that will let pyjitpl know that it can directly generate a CALL_RELEASE_GIL. diff --git a/pypy/jit/codewriter/call.py b/pypy/jit/codewriter/call.py --- a/pypy/jit/codewriter/call.py +++ b/pypy/jit/codewriter/call.py @@ -220,6 +220,7 @@ # get the 'elidable' and 'loopinvariant' flags from the function object elidable = False loopinvariant = False + call_release_gil_target = llmemory.NULL if op.opname == "direct_call": funcobj = get_funcobj(op.args[0].value) assert getattr(funcobj, 'calling_conv', 'c') == 'c', ( @@ -230,6 +231,10 @@ if loopinvariant: assert not NON_VOID_ARGS, ("arguments not supported for " "loop-invariant function!") + if getattr(func, "_call_aroundstate_target_", None): + call_release_gil_target = func._call_aroundstate_target_ + call_release_gil_target = llmemory.cast_ptr_to_adr( + call_release_gil_target) # build the extraeffect random_effects = self.randomeffects_analyzer.analyze(op) if random_effects: @@ -253,7 +258,7 @@ # effectinfo = effectinfo_from_writeanalyze( self.readwrite_analyzer.analyze(op), self.cpu, extraeffect, - oopspecindex, can_invalidate) + oopspecindex, can_invalidate, call_release_gil_target) # assert effectinfo is not None if elidable or loopinvariant: diff --git a/pypy/jit/codewriter/effectinfo.py b/pypy/jit/codewriter/effectinfo.py --- a/pypy/jit/codewriter/effectinfo.py +++ b/pypy/jit/codewriter/effectinfo.py @@ -1,6 +1,6 @@ from pypy.jit.metainterp.typesystem import deref, fieldType, arrayItem from pypy.rpython.lltypesystem.rclass import OBJECT -from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.ootypesystem import ootype from pypy.translator.backendopt.graphanalyze import BoolGraphAnalyzer @@ -87,7 +87,8 @@ write_descrs_fields, write_descrs_arrays, extraeffect=EF_CAN_RAISE, oopspecindex=OS_NONE, - can_invalidate=False): + can_invalidate=False, + call_release_gil_target=llmemory.NULL): key = (frozenset_or_none(readonly_descrs_fields), frozenset_or_none(readonly_descrs_arrays), frozenset_or_none(write_descrs_fields), @@ -95,6 +96,8 @@ extraeffect, oopspecindex, can_invalidate) + if call_release_gil_target: + key += (object(),) # don't care about caching in this case if key in cls._cache: return cls._cache[key] if extraeffect == EffectInfo.EF_RANDOM_EFFECTS: @@ -121,6 +124,7 @@ result.extraeffect = extraeffect result.can_invalidate = can_invalidate result.oopspecindex = oopspecindex + result.call_release_gil_target = call_release_gil_target if result.check_can_raise(): assert oopspecindex in cls._OS_CANRAISE cls._cache[key] = result @@ -142,6 +146,9 @@ def has_random_effects(self): return self.extraeffect >= self.EF_RANDOM_EFFECTS + def is_call_release_gil(self): + return bool(self.call_release_gil_target) + def frozenset_or_none(x): if x is None: @@ -156,7 +163,8 @@ def effectinfo_from_writeanalyze(effects, cpu, extraeffect=EffectInfo.EF_CAN_RAISE, oopspecindex=EffectInfo.OS_NONE, - can_invalidate=False): + can_invalidate=False, + call_release_gil_target=llmemory.NULL): from pypy.translator.backendopt.writeanalyze import top_set if effects is top_set or extraeffect == EffectInfo.EF_RANDOM_EFFECTS: readonly_descrs_fields = None @@ -204,7 +212,8 @@ write_descrs_arrays, extraeffect, oopspecindex, - can_invalidate) + can_invalidate, + call_release_gil_target) def consider_struct(TYPE, fieldname): if fieldType(TYPE, fieldname) is lltype.Void: diff --git a/pypy/jit/codewriter/test/test_call.py b/pypy/jit/codewriter/test/test_call.py --- a/pypy/jit/codewriter/test/test_call.py +++ b/pypy/jit/codewriter/test/test_call.py @@ -1,6 +1,6 @@ import py from pypy.objspace.flow.model import SpaceOperation, Constant, Variable -from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.translator.unsimplify import varoftype from pypy.rlib import jit from pypy.jit.codewriter.call import CallControl @@ -192,6 +192,32 @@ [op] = block.operations call_descr = cc.getcalldescr(op) assert call_descr.extrainfo.has_random_effects() + assert call_descr.extrainfo.is_call_release_gil() is False + +def test_call_release_gil(): + from pypy.jit.backend.llgraph.runner import LLtypeCPU + + T = rffi.CArrayPtr(rffi.TIME_T) + external = rffi.llexternal("time", [T], rffi.TIME_T, threadsafe=True) + + # no jit.dont_look_inside in this test + def f(): + return external(lltype.nullptr(T.TO)) + + rtyper = support.annotate(f, []) + jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0]) + cc = CallControl(LLtypeCPU(rtyper), jitdrivers_sd=[jitdriver_sd]) + res = cc.find_all_graphs(FakePolicy()) + + [llext_graph] = [x for x in res if x.func is external] + [block, _] = list(llext_graph.iterblocks()) + [op] = block.operations + call_target = op.args[0].value._obj.graph.func._call_aroundstate_target_ + call_target = llmemory.cast_ptr_to_adr(call_target) + call_descr = cc.getcalldescr(op) + assert call_descr.extrainfo.has_random_effects() + assert call_descr.extrainfo.is_call_release_gil() is True + assert call_descr.extrainfo.call_release_gil_target == call_target def test_random_effects_on_stacklet_switch(): from pypy.jit.backend.llgraph.runner import LLtypeCPU diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py --- a/pypy/rpython/lltypesystem/rffi.py +++ b/pypy/rpython/lltypesystem/rffi.py @@ -171,6 +171,7 @@ call_external_function._dont_inline_ = True call_external_function._annspecialcase_ = 'specialize:ll' call_external_function._gctransformer_hint_close_stack_ = True + call_external_function._call_aroundstate_target_ = funcptr call_external_function = func_with_new_name(call_external_function, 'ccall_' + name) # don't inline, as a hack to guarantee that no GC pointer is alive From noreply at buildbot.pypy.org Sat Jan 5 13:59:24 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 5 Jan 2013 13:59:24 +0100 (CET) Subject: [pypy-commit] pypy remove-globals-in-jit: hg merge default Message-ID: <20130105125924.9D1931C0DDA@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: remove-globals-in-jit Changeset: r59721:db015e15e91a Date: 2013-01-05 13:52 +0100 http://bitbucket.org/pypy/pypy/changeset/db015e15e91a/ Log: hg merge default diff --git a/pypy/jit/codewriter/call.py b/pypy/jit/codewriter/call.py --- a/pypy/jit/codewriter/call.py +++ b/pypy/jit/codewriter/call.py @@ -220,6 +220,7 @@ # get the 'elidable' and 'loopinvariant' flags from the function object elidable = False loopinvariant = False + call_release_gil_target = llmemory.NULL if op.opname == "direct_call": funcobj = get_funcobj(op.args[0].value) assert getattr(funcobj, 'calling_conv', 'c') == 'c', ( @@ -230,6 +231,10 @@ if loopinvariant: assert not NON_VOID_ARGS, ("arguments not supported for " "loop-invariant function!") + if getattr(func, "_call_aroundstate_target_", None): + call_release_gil_target = func._call_aroundstate_target_ + call_release_gil_target = llmemory.cast_ptr_to_adr( + call_release_gil_target) # build the extraeffect random_effects = self.randomeffects_analyzer.analyze(op) if random_effects: @@ -253,7 +258,7 @@ # effectinfo = effectinfo_from_writeanalyze( self.readwrite_analyzer.analyze(op), self.cpu, extraeffect, - oopspecindex, can_invalidate) + oopspecindex, can_invalidate, call_release_gil_target) # assert effectinfo is not None if elidable or loopinvariant: diff --git a/pypy/jit/codewriter/effectinfo.py b/pypy/jit/codewriter/effectinfo.py --- a/pypy/jit/codewriter/effectinfo.py +++ b/pypy/jit/codewriter/effectinfo.py @@ -1,6 +1,6 @@ from pypy.jit.metainterp.typesystem import deref, fieldType, arrayItem from pypy.rpython.lltypesystem.rclass import OBJECT -from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.ootypesystem import ootype from pypy.translator.backendopt.graphanalyze import BoolGraphAnalyzer @@ -87,7 +87,8 @@ write_descrs_fields, write_descrs_arrays, extraeffect=EF_CAN_RAISE, oopspecindex=OS_NONE, - can_invalidate=False): + can_invalidate=False, + call_release_gil_target=llmemory.NULL): key = (frozenset_or_none(readonly_descrs_fields), frozenset_or_none(readonly_descrs_arrays), frozenset_or_none(write_descrs_fields), @@ -95,6 +96,8 @@ extraeffect, oopspecindex, can_invalidate) + if call_release_gil_target: + key += (object(),) # don't care about caching in this case if key in cls._cache: return cls._cache[key] if extraeffect == EffectInfo.EF_RANDOM_EFFECTS: @@ -121,6 +124,7 @@ result.extraeffect = extraeffect result.can_invalidate = can_invalidate result.oopspecindex = oopspecindex + result.call_release_gil_target = call_release_gil_target if result.check_can_raise(): assert oopspecindex in cls._OS_CANRAISE cls._cache[key] = result @@ -142,6 +146,9 @@ def has_random_effects(self): return self.extraeffect >= self.EF_RANDOM_EFFECTS + def is_call_release_gil(self): + return bool(self.call_release_gil_target) + def frozenset_or_none(x): if x is None: @@ -156,7 +163,8 @@ def effectinfo_from_writeanalyze(effects, cpu, extraeffect=EffectInfo.EF_CAN_RAISE, oopspecindex=EffectInfo.OS_NONE, - can_invalidate=False): + can_invalidate=False, + call_release_gil_target=llmemory.NULL): from pypy.translator.backendopt.writeanalyze import top_set if effects is top_set or extraeffect == EffectInfo.EF_RANDOM_EFFECTS: readonly_descrs_fields = None @@ -204,7 +212,8 @@ write_descrs_arrays, extraeffect, oopspecindex, - can_invalidate) + can_invalidate, + call_release_gil_target) def consider_struct(TYPE, fieldname): if fieldType(TYPE, fieldname) is lltype.Void: diff --git a/pypy/jit/codewriter/test/test_call.py b/pypy/jit/codewriter/test/test_call.py --- a/pypy/jit/codewriter/test/test_call.py +++ b/pypy/jit/codewriter/test/test_call.py @@ -1,6 +1,6 @@ import py from pypy.objspace.flow.model import SpaceOperation, Constant, Variable -from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.translator.unsimplify import varoftype from pypy.rlib import jit from pypy.jit.codewriter.call import CallControl @@ -192,6 +192,32 @@ [op] = block.operations call_descr = cc.getcalldescr(op) assert call_descr.extrainfo.has_random_effects() + assert call_descr.extrainfo.is_call_release_gil() is False + +def test_call_release_gil(): + from pypy.jit.backend.llgraph.runner import LLtypeCPU + + T = rffi.CArrayPtr(rffi.TIME_T) + external = rffi.llexternal("time", [T], rffi.TIME_T, threadsafe=True) + + # no jit.dont_look_inside in this test + def f(): + return external(lltype.nullptr(T.TO)) + + rtyper = support.annotate(f, []) + jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0]) + cc = CallControl(LLtypeCPU(rtyper), jitdrivers_sd=[jitdriver_sd]) + res = cc.find_all_graphs(FakePolicy()) + + [llext_graph] = [x for x in res if x.func is external] + [block, _] = list(llext_graph.iterblocks()) + [op] = block.operations + call_target = op.args[0].value._obj.graph.func._call_aroundstate_target_ + call_target = llmemory.cast_ptr_to_adr(call_target) + call_descr = cc.getcalldescr(op) + assert call_descr.extrainfo.has_random_effects() + assert call_descr.extrainfo.is_call_release_gil() is True + assert call_descr.extrainfo.call_release_gil_target == call_target def test_random_effects_on_stacklet_switch(): from pypy.jit.backend.llgraph.runner import LLtypeCPU diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py --- a/pypy/rpython/lltypesystem/rffi.py +++ b/pypy/rpython/lltypesystem/rffi.py @@ -171,6 +171,7 @@ call_external_function._dont_inline_ = True call_external_function._annspecialcase_ = 'specialize:ll' call_external_function._gctransformer_hint_close_stack_ = True + call_external_function._call_aroundstate_target_ = funcptr call_external_function = func_with_new_name(call_external_function, 'ccall_' + name) # don't inline, as a hack to guarantee that no GC pointer is alive From noreply at buildbot.pypy.org Sat Jan 5 13:59:25 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 5 Jan 2013 13:59:25 +0100 (CET) Subject: [pypy-commit] pypy remove-globals-in-jit: (arigo, fijal around) Message-ID: <20130105125925.CB4291C0DDA@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: remove-globals-in-jit Changeset: r59722:fd30ac31a2f2 Date: 2013-01-05 14:01 +0100 http://bitbucket.org/pypy/pypy/changeset/fd30ac31a2f2/ Log: (arigo, fijal around) Finally add the shortcut we were looking for: now calls to external functions that release the GIL should be turned into just a CALL_RELEASE_GIL. diff --git a/pypy/jit/backend/llgraph/runner.py b/pypy/jit/backend/llgraph/runner.py --- a/pypy/jit/backend/llgraph/runner.py +++ b/pypy/jit/backend/llgraph/runner.py @@ -814,6 +814,12 @@ return res def execute_call_release_gil(self, descr, func, *args): + if hasattr(descr, '_original_func_'): + func = descr._original_func_ # see pyjitpl.py + # we want to call the function that does the aroundstate + # manipulation here (as a hack, instead of really doing + # the aroundstate manipulation ourselves) + return self.execute_call_may_force(descr, func, *args) call_args = support.cast_call_args_in_order(descr.ARGS, args) FUNC = lltype.FuncType(descr.ARGS, descr.RESULT) func_to_call = rffi.cast(lltype.Ptr(FUNC), func) diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py --- a/pypy/jit/metainterp/pyjitpl.py +++ b/pypy/jit/metainterp/pyjitpl.py @@ -1348,6 +1348,8 @@ self.metainterp.vable_and_vrefs_before_residual_call() resbox = self.metainterp.execute_and_record_varargs( rop.CALL_MAY_FORCE, allboxes, descr=descr) + if effectinfo.is_call_release_gil(): + self.metainterp.direct_call_release_gil() self.metainterp.vrefs_after_residual_call() vablebox = None if assembler_call: @@ -1360,6 +1362,7 @@ if vablebox is not None: self.metainterp.history.record(rop.KEEPALIVE, [vablebox], None) self.metainterp.handle_possible_exception() + # XXX refactor: direct_libffi_call() is a hack if effectinfo.oopspecindex == effectinfo.OS_LIBFFI_CALL: self.metainterp.direct_libffi_call() return resbox @@ -2606,6 +2609,19 @@ [box_resultpos, ConstInt(0), box_result], None, descr) + def direct_call_release_gil(self): + op = self.history.operations.pop() + assert op.opnum == rop.CALL_MAY_FORCE + descr = op.getdescr() + effectinfo = descr.get_extra_info() + realfuncaddr = effectinfo.call_release_gil_target + funcbox = ConstInt(heaptracker.adr2int(realfuncaddr)) + self.history.record(rop.CALL_RELEASE_GIL, + [funcbox] + op.getarglist()[1:], + op.result, descr) + if not we_are_translated(): # for llgraph + descr._original_func_ = op.getarg(0).value + # ____________________________________________________________ class ChangeFrame(JitException): diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py --- a/pypy/jit/metainterp/test/test_ajit.py +++ b/pypy/jit/metainterp/test/test_ajit.py @@ -4011,3 +4011,4 @@ assert res == 2 res = self.interp_operations(f, []) assert res == 2 + self.check_operations_history(call_release_gil=1, call_may_force=0) From noreply at buildbot.pypy.org Sat Jan 5 14:19:05 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 5 Jan 2013 14:19:05 +0100 (CET) Subject: [pypy-commit] pypy remove-globals-in-jit: Fix for test_free_object Message-ID: <20130105131905.091841C01EE@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: remove-globals-in-jit Changeset: r59723:e93f864eda21 Date: 2013-01-05 14:18 +0100 http://bitbucket.org/pypy/pypy/changeset/e93f864eda21/ Log: Fix for test_free_object diff --git a/pypy/jit/backend/llgraph/runner.py b/pypy/jit/backend/llgraph/runner.py --- a/pypy/jit/backend/llgraph/runner.py +++ b/pypy/jit/backend/llgraph/runner.py @@ -28,7 +28,7 @@ try: newbox = _cache[box] except KeyError: - newbox = _cache[box] = box.clonebox() + newbox = _cache[box] = box.__class__() return newbox # self.inputargs = map(mapping, inputargs) From noreply at buildbot.pypy.org Sat Jan 5 14:24:54 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 5 Jan 2013 14:24:54 +0100 (CET) Subject: [pypy-commit] pypy remove-globals-in-jit: small cleanup here Message-ID: <20130105132454.23C4D1C01EE@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: remove-globals-in-jit Changeset: r59724:68c4c8de1f82 Date: 2013-01-05 14:58 +0200 http://bitbucket.org/pypy/pypy/changeset/68c4c8de1f82/ Log: small cleanup here diff --git a/pypy/rpython/annlowlevel.py b/pypy/rpython/annlowlevel.py --- a/pypy/rpython/annlowlevel.py +++ b/pypy/rpython/annlowlevel.py @@ -2,7 +2,6 @@ The code needed to flow and annotate low-level helpers -- the ll_*() functions """ -import types from pypy.tool.sourcetools import valid_identifier from pypy.annotation import model as annmodel from pypy.annotation.policy import AnnotatorPolicy, Sig @@ -13,6 +12,7 @@ from pypy.objspace.flow.model import Constant from pypy.translator.simplify import get_functype from pypy.rpython.rmodel import warning +from pypy.rlib.objectmodel import specialize class KeyComp(object): def __init__(self, val): @@ -504,13 +504,13 @@ else: raise NotImplementedError("cast_object_to_ptr(%r, ...)" % PTR) + at specialize.argtype(0) def cast_instance_to_base_ptr(instance): return cast_object_to_ptr(base_ptr_lltype(), instance) -cast_instance_to_base_ptr._annspecialcase_ = 'specialize:argtype(0)' + at specialize.argtype(0) def cast_instance_to_base_obj(instance): return cast_object_to_ptr(base_obj_ootype(), instance) -cast_instance_to_base_obj._annspecialcase_ = 'specialize:argtype(0)' def base_ptr_lltype(): from pypy.rpython.lltypesystem.rclass import OBJECTPTR From noreply at buildbot.pypy.org Sat Jan 5 14:24:55 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 5 Jan 2013 14:24:55 +0100 (CET) Subject: [pypy-commit] pypy remove-globals-in-jit: remove tabs Message-ID: <20130105132455.645D91C01EE@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: remove-globals-in-jit Changeset: r59725:7bd2b76db5cd Date: 2013-01-05 15:15 +0200 http://bitbucket.org/pypy/pypy/changeset/7bd2b76db5cd/ Log: remove tabs diff --git a/pypy/jit/backend/test/test_random.py b/pypy/jit/backend/test/test_random.py --- a/pypy/jit/backend/test/test_random.py +++ b/pypy/jit/backend/test/test_random.py @@ -675,8 +675,8 @@ cpu = self.builder.cpu self.clear_state() # disable check for now - # exc = cpu.grab_exc_value() - # assert not exc + # exc = cpu.grab_exc_value() + # assert not exc arguments = [box.value for box in self.loop.inputargs] deadframe = cpu.execute_token(self.runjitcelltoken(), *arguments) From noreply at buildbot.pypy.org Sat Jan 5 14:24:56 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 5 Jan 2013 14:24:56 +0100 (CET) Subject: [pypy-commit] pypy remove-globals-in-jit: fix tests Message-ID: <20130105132456.83C671C01EE@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: remove-globals-in-jit Changeset: r59726:584b196955de Date: 2013-01-05 15:23 +0200 http://bitbucket.org/pypy/pypy/changeset/584b196955de/ Log: fix tests diff --git a/pypy/jit/codewriter/test/test_call.py b/pypy/jit/codewriter/test/test_call.py --- a/pypy/jit/codewriter/test/test_call.py +++ b/pypy/jit/codewriter/test/test_call.py @@ -173,7 +173,7 @@ py.test.skip("XXX add a test for CallControl.getcalldescr() -> EF_xxx") def test_releases_gil_analyzer(): - from pypy.jit.backend.llgraph.runner import LLtypeCPU + from pypy.jit.backend.llgraph.runner import LLGraphCPU T = rffi.CArrayPtr(rffi.TIME_T) external = rffi.llexternal("time", [T], rffi.TIME_T, threadsafe=True) @@ -184,7 +184,7 @@ rtyper = support.annotate(f, []) jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0]) - cc = CallControl(LLtypeCPU(rtyper), jitdrivers_sd=[jitdriver_sd]) + cc = CallControl(LLGraphCPU(rtyper), jitdrivers_sd=[jitdriver_sd]) res = cc.find_all_graphs(FakePolicy()) [f_graph] = [x for x in res if x.func is f] @@ -194,7 +194,7 @@ assert call_descr.extrainfo.has_random_effects() def test_random_effects_on_stacklet_switch(): - from pypy.jit.backend.llgraph.runner import LLtypeCPU + from pypy.jit.backend.llgraph.runner import LLGraphCPU from pypy.translator.platform import CompilationError try: from pypy.rlib._rffi_stacklet import switch, thread_handle, handle @@ -209,7 +209,7 @@ rtyper = support.annotate(f, []) jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0]) - cc = CallControl(LLtypeCPU(rtyper), jitdrivers_sd=[jitdriver_sd]) + cc = CallControl(LLGraphCPU(rtyper), jitdrivers_sd=[jitdriver_sd]) res = cc.find_all_graphs(FakePolicy()) [f_graph] = [x for x in res if x.func is f] From noreply at buildbot.pypy.org Sat Jan 5 14:24:57 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 5 Jan 2013 14:24:57 +0100 (CET) Subject: [pypy-commit] pypy remove-globals-in-jit: merge Message-ID: <20130105132457.DF1D11C01EE@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: remove-globals-in-jit Changeset: r59727:a4cc62478e77 Date: 2013-01-05 15:24 +0200 http://bitbucket.org/pypy/pypy/changeset/a4cc62478e77/ Log: merge diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -28,7 +28,7 @@ DEALINGS IN THE SOFTWARE. -PyPy Copyright holders 2003-2012 +PyPy Copyright holders 2003-2013 ----------------------------------- Except when otherwise stated (look for LICENSE files or information at diff --git a/lib-python/2.7/sre_parse.py b/lib-python/2.7/sre_parse.py --- a/lib-python/2.7/sre_parse.py +++ b/lib-python/2.7/sre_parse.py @@ -16,6 +16,12 @@ from sre_constants import * +try: + from __pypy__ import newdict +except ImportError: + def newdict(tp): + return {} + SPECIAL_CHARS = ".\\[{()*+?^$|" REPEAT_CHARS = "*+?{" @@ -68,7 +74,7 @@ self.flags = 0 self.open = [] self.groups = 1 - self.groupdict = {} + self.groupdict = newdict("module") def opengroup(self, name=None): gid = self.groups self.groups = gid + 1 diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst --- a/pypy/doc/index.rst +++ b/pypy/doc/index.rst @@ -67,7 +67,7 @@ .. _Python: http://docs.python.org/index.html .. _`more...`: architecture.html#mission-statement .. _`PyPy blog`: http://morepypy.blogspot.com/ -.. _`development bug/feature tracker`: https://codespeak.net/issue/pypy-dev/ +.. _`development bug/feature tracker`: https://bugs.pypy.org .. _here: http://tismerysoft.de/pypy/irc-logs/pypy .. _`Mercurial commit mailing list`: http://python.org/mailman/listinfo/pypy-commit .. _`development mailing list`: http://python.org/mailman/listinfo/pypy-dev diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py --- a/pypy/jit/backend/arm/assembler.py +++ b/pypy/jit/backend/arm/assembler.py @@ -1,5 +1,6 @@ from __future__ import with_statement import os +from pypy.jit.backend.llsupport import jitframe from pypy.jit.backend.arm.helper.assembler import saved_registers from pypy.jit.backend.arm import conditions as c from pypy.jit.backend.arm import registers as r @@ -25,12 +26,12 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.jit.backend.arm.opassembler import ResOpAssembler from pypy.rlib.debug import (debug_print, debug_start, debug_stop, - have_debug_prints) + have_debug_prints, fatalerror) from pypy.rlib.jit import AsmInfo from pypy.rlib.objectmodel import compute_unique_id # XXX Move to llsupport -from pypy.jit.backend.x86.support import values_array, memcpy_fn +from pypy.jit.backend.x86.support import memcpy_fn DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', ('i', lltype.Signed), ('type', lltype.Char), # 'b'ridge, 'l'abel or @@ -44,14 +45,8 @@ debug = True - def __init__(self, cpu, failargs_limit=1000): + def __init__(self, cpu, translate_support_code=False): self.cpu = cpu - self.fail_boxes_int = values_array(lltype.Signed, failargs_limit) - self.fail_boxes_float = values_array(longlong.FLOATSTORAGE, - failargs_limit) - self.fail_boxes_ptr = values_array(llmemory.GCREF, failargs_limit) - self.fail_boxes_count = 0 - self.fail_force_index = 0 self.setup_failure_recovery() self.mc = None self.memcpy_addr = 0 @@ -68,6 +63,7 @@ self._debug = False self.loop_run_counters = [] self.debug_counter_descr = cpu.fielddescrof(DEBUG_COUNTER, 'i') + self.force_token_to_dead_frame = {} # XXX temporary hack def set_debug(self, v): r = self._debug @@ -112,9 +108,13 @@ gc_ll_descr.initialize() self._build_wb_slowpath(False) self._build_wb_slowpath(True) + self._build_failure_recovery(exc=True, withfloats=False) + self._build_failure_recovery(exc=False, withfloats=False) if self.cpu.supports_floats: self._build_wb_slowpath(False, withfloats=True) self._build_wb_slowpath(True, withfloats=True) + self._build_failure_recovery(exc=True, withfloats=True) + self._build_failure_recovery(exc=False, withfloats=True) self._build_propagate_exception_path() if gc_ll_descr.get_malloc_slowpath_addr is not None: self._build_malloc_slowpath() @@ -122,10 +122,7 @@ if gc_ll_descr.gcrootmap and gc_ll_descr.gcrootmap.is_shadow_stack: self._build_release_gil(gc_ll_descr.gcrootmap) self.memcpy_addr = self.cpu.cast_ptr_to_int(memcpy_fn) - self._exit_code_addr = self._gen_exit_path() - self._leave_jitted_hook_save_exc = \ - self._gen_leave_jitted_hook_code(True) - self._leave_jitted_hook = self._gen_leave_jitted_hook_code(False) + if not self._debug: # if self._debug is already set it means that someone called # set_debug by hand before initializing the assembler. Leave it @@ -220,36 +217,17 @@ self.releasegil_addr = rffi.cast(lltype.Signed, releasegil_func) self.reacqgil_addr = rffi.cast(lltype.Signed, reacqgil_func) - def _gen_leave_jitted_hook_code(self, save_exc): - mc = ARMv7Builder() - if self.cpu.supports_floats: - floats = r.caller_vfp_resp - else: - floats = [] - with saved_registers(mc, r.caller_resp + [r.lr], floats): - addr = self.cpu.get_on_leave_jitted_int(save_exception=save_exc) - mc.BL(addr) - assert self._exit_code_addr != 0 - mc.B(self._exit_code_addr) - return mc.materialize(self.cpu.asmmemmgr, [], - self.cpu.gc_ll_descr.gcrootmap) - def _build_propagate_exception_path(self): if self.cpu.propagate_exception_v < 0: return # not supported (for tests, or non-translated) # mc = ARMv7Builder() - # call on_leave_jitted_save_exc() - if self.cpu.supports_floats: - floats = r.caller_vfp_resp - else: - floats = [] - with saved_registers(mc, r.caller_resp + [r.lr], floats): - addr = self.cpu.get_on_leave_jitted_int(save_exception=True, - default_to_memoryerror=True) - mc.BL(addr) - mc.gen_load_int(r.ip.value, self.cpu.propagate_exception_v) - mc.MOV_rr(r.r0.value, r.ip.value) + # + # Call the helper, which will return a dead frame object with + # the correct exception set, or MemoryError by default + # XXX make sure we return the correct value here + addr = rffi.cast(lltype.Signed, self.cpu.get_propagate_exception()) + mc.BL(addr) self.gen_func_epilog(mc=mc) self.propagate_exception_path = mc.materialize(self.cpu.asmmemmgr, []) @@ -282,12 +260,12 @@ # restore registers and return # We check for c.EQ here, meaning all bits zero in this case mc.POP([reg.value for reg in r.argument_regs] + [r.pc.value], cond=c.EQ) - # call on_leave_jitted_save_exc() - addr = self.cpu.get_on_leave_jitted_int(save_exception=True) + # + # Call the helper, which will return a dead frame object with + # the correct exception set, or MemoryError by default + addr = rffi.cast(lltype.Signed, self.cpu.get_propagate_exception()) mc.BL(addr) # - mc.gen_load_int(r.r0.value, self.cpu.propagate_exception_v) - # # footer -- note the ADD, which skips the return address of this # function, and will instead return to the caller's caller. Note # also that we completely ignore the saved arguments, because we @@ -339,39 +317,82 @@ def setup_failure_recovery(self): - @rgc.no_collect + #@rgc.no_collect -- XXX still true, but hacked gc_set_extra_threshold def failure_recovery_func(mem_loc, frame_pointer, stack_pointer): """mem_loc is a structure in memory describing where the values for the failargs are stored. frame loc is the address of the frame pointer for the frame to be decoded frame """ - vfp_registers = rffi.cast(rffi.LONGLONGP, stack_pointer) - registers = rffi.ptradd(vfp_registers, len(r.all_vfp_regs)) + vfp_registers = rffi.cast(rffi.LONGP, stack_pointer) + registers = rffi.ptradd(vfp_registers, 2*len(r.all_vfp_regs)) registers = rffi.cast(rffi.LONGP, registers) - return self.decode_registers_and_descr(mem_loc, frame_pointer, + bytecode = rffi.cast(rffi.UCHARP, mem_loc) + return self.grab_frame_values(self.cpu, bytecode, frame_pointer, registers, vfp_registers) + self.failure_recovery_code = [0, 0, 0, 0] self.failure_recovery_func = failure_recovery_func - recovery_func_sign = lltype.Ptr(lltype.FuncType([lltype.Signed] * 3, - lltype.Signed)) + _FAILURE_RECOVERY_FUNC = lltype.Ptr(lltype.FuncType([rffi.LONGP] * 3, + llmemory.GCREF)) - @rgc.no_collect - def decode_registers_and_descr(self, mem_loc, frame_pointer, + @staticmethod + #@rgc.no_collect -- XXX still true, but hacked gc_set_extra_threshold + def grab_frame_values(cpu, bytecode, frame_pointer, registers, vfp_registers): - """Decode locations encoded in memory at mem_loc and write the values - to the failboxes. Values for spilled vars and registers are stored on - stack at frame_loc """ - assert frame_pointer & 1 == 0 - self.fail_force_index = frame_pointer - bytecode = rffi.cast(rffi.UCHARP, mem_loc) + # no malloc allowed here!! xxx apart from one, hacking a lot + force_index = rffi.cast(lltype.Signed, frame_pointer) num = 0 - value = 0 - fvalue = 0 + deadframe = lltype.nullptr(jitframe.DEADFRAME) + # step 1: lots of mess just to count the final value of 'num' + bytecode1 = bytecode + while 1: + code = rffi.cast(lltype.Signed, bytecode1[0]) + bytecode1 = rffi.ptradd(bytecode1, 1) + if code >= AssemblerARM.CODE_FROMSTACK: + while code > 0x7F: + code = rffi.cast(lltype.Signed, bytecode1[0]) + bytecode1 = rffi.ptradd(bytecode1, 1) + else: + kind = code & 3 + if kind == AssemblerARM.DESCR_SPECIAL: + if code == AssemblerARM.CODE_HOLE: + num += 1 + continue + if code == AssemblerARM.CODE_INPUTARG: + continue + if code == AssemblerARM.CODE_FORCED: + # resuming from a GUARD_NOT_FORCED + token = force_index + deadframe = ( + cpu.assembler.force_token_to_dead_frame.pop(token)) + deadframe = lltype.cast_opaque_ptr( + jitframe.DEADFRAMEPTR, deadframe) + continue + assert code == AssemblerARM.CODE_STOP + break + num += 1 + + # allocate the deadframe + if not deadframe: + # Remove the "reserve" at the end of the nursery. This means + # that it is guaranteed that the following malloc() works + # without requiring a collect(), but it needs to be re-added + # as soon as possible. + cpu.gc_clear_extra_threshold() + assert num <= cpu.get_failargs_limit() + try: + deadframe = lltype.malloc(jitframe.DEADFRAME, num) + except MemoryError: + fatalerror("memory usage error in grab_frame_values") + # fill it code_inputarg = False - while True: + num = 0 + value_hi = 0 + while 1: + # decode the next instruction from the bytecode code = rffi.cast(lltype.Signed, bytecode[0]) bytecode = rffi.ptradd(bytecode, 1) - if code >= self.CODE_FROMSTACK: + if code >= AssemblerARM.CODE_FROMSTACK: if code > 0x7F: shift = 7 code &= 0x7F @@ -384,54 +405,63 @@ break # load the value from the stack kind = code & 3 - code = int((code - self.CODE_FROMSTACK) >> 2) + code = (code - AssemblerARM.CODE_FROMSTACK) >> 2 if code_inputarg: code = ~code code_inputarg = False - if kind == self.DESCR_FLOAT: - # we use code + 1 to get the hi word of the double worded float - stackloc = frame_pointer - get_fp_offset(int(code) + 1) - assert stackloc & 3 == 0 - fvalue = rffi.cast(rffi.LONGLONGP, stackloc)[0] - else: - stackloc = frame_pointer - get_fp_offset(int(code)) - assert stackloc & 1 == 0 - value = rffi.cast(rffi.LONGP, stackloc)[0] + stackloc = force_index - get_fp_offset(int(code)) + value = rffi.cast(rffi.LONGP, stackloc)[0] + if kind == AssemblerARM.DESCR_FLOAT: + assert WORD == 4 + value_hi = value + value = rffi.cast(rffi.LONGP, stackloc - WORD)[0] else: - # 'code' identifies a register: load its value kind = code & 3 - if kind == self.DESCR_SPECIAL: - if code == self.CODE_HOLE: + if kind == AssemblerARM.DESCR_SPECIAL: + if code == AssemblerARM.CODE_HOLE: num += 1 continue - if code == self.CODE_INPUTARG: + if code == AssemblerARM.CODE_INPUTARG: code_inputarg = True continue - assert code == self.CODE_STOP + if code == AssemblerARM.CODE_FORCED: + continue + assert code == AssemblerARM.CODE_STOP break + # 'code' identifies a register: load its value code >>= 2 - if kind == self.DESCR_FLOAT: - fvalue = vfp_registers[code] + if kind == AssemblerARM.DESCR_FLOAT: + if WORD == 4: + value = vfp_registers[2*code] + value_hi = vfp_registers[2*code + 1] + else: + value = registers[code] else: value = registers[code] # store the loaded value into fail_boxes_ - if kind == self.DESCR_FLOAT: - tgt = self.fail_boxes_float.get_addr_for_num(num) - rffi.cast(rffi.LONGLONGP, tgt)[0] = fvalue + if kind == AssemblerARM.DESCR_INT: + deadframe.jf_values[num].int = value + elif kind == AssemblerARM.DESCR_REF: + deadframe.jf_values[num].ref = rffi.cast(llmemory.GCREF, value) + elif kind == AssemblerARM.DESCR_FLOAT: + assert WORD == 4 + assert not longlong.is_64_bit + floatvalue = rffi.cast(lltype.SignedLongLong, value_hi) + floatvalue <<= 32 + floatvalue |= rffi.cast(lltype.SignedLongLong, + rffi.cast(lltype.Unsigned, value)) + deadframe.jf_values[num].float = floatvalue else: - if kind == self.DESCR_INT: - tgt = self.fail_boxes_int.get_addr_for_num(num) - elif kind == self.DESCR_REF: - assert (value & 3) == 0, "misaligned pointer" - tgt = self.fail_boxes_ptr.get_addr_for_num(num) - else: - assert 0, "bogus kind" - rffi.cast(rffi.LONGP, tgt)[0] = value + assert 0, "bogus kind" num += 1 - self.fail_boxes_count = num + # + assert num == len(deadframe.jf_values) + if not we_are_translated(): + assert bytecode[4] == 0xCC fail_index = rffi.cast(rffi.INTP, bytecode)[0] - fail_index = rffi.cast(lltype.Signed, fail_index) - return fail_index + fail_descr = cpu.get_fail_descr_from_number(fail_index) + deadframe.jf_descr = fail_descr.hide(cpu) + return lltype.cast_opaque_ptr(llmemory.GCREF, deadframe) def decode_inputargs(self, code): descr_to_box_type = [REF, INT, FLOAT] @@ -514,12 +544,26 @@ self.mc.CMP_ri(r.r0.value, 0) self.mc.B(self.propagate_exception_path, c=c.EQ) - def _gen_exit_path(self): + def _build_failure_recovery(self, exc, withfloats=False): mc = ARMv7Builder() - decode_registers_addr = llhelper(self.recovery_func_sign, + failure_recovery = llhelper(self._FAILURE_RECOVERY_FUNC, self.failure_recovery_func) self._insert_checks(mc) - with saved_registers(mc, r.all_regs, r.all_vfp_regs): + if withfloats: + f = r.all_vfp_regs + else: + f = [] + with saved_registers(mc, r.all_regs, f): + if exc: + # We might have an exception pending. Load it into r4 + # (this is a register saved across calls) + mc.gen_load_int(r.r5.value, self.cpu.pos_exc_value()) + mc.LDR_ri(r.r4.value, self.cpu.pos_exc_value()) + # clear the exc flags + mc.gen_load_int(r.r6.value, 0) + mc.STR_ri(r.r6.value, r.r5.value) + mc.gen_load_int(r.r5.value, self.cpu.pos_exception()) + mc.STR_ri(r.r6.value, r.r5.value) # move mem block address, to r0 to pass as mc.MOV_rr(r.r0.value, r.lr.value) # pass the current frame pointer as second param @@ -527,23 +571,32 @@ # pass the current stack pointer as third param mc.MOV_rr(r.r2.value, r.sp.value) self._insert_checks(mc) - mc.BL(rffi.cast(lltype.Signed, decode_registers_addr)) + mc.BL(rffi.cast(lltype.Signed, failure_recovery)) + if exc: + # save ebx into 'jf_guard_exc' + from pypy.jit.backend.llsupport.descr import unpack_fielddescr + descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) + offset, size, _ = unpack_fielddescr(descrs.jf_guard_exc) + mc.STR_rr(r.r4.value, r.r0.value, offset, cond=c.AL) mc.MOV_rr(r.ip.value, r.r0.value) mc.MOV_rr(r.r0.value, r.ip.value) self.gen_func_epilog(mc=mc) - return mc.materialize(self.cpu.asmmemmgr, [], + rawstart = mc.materialize(self.cpu.asmmemmgr, [], self.cpu.gc_ll_descr.gcrootmap) + self.failure_recovery_code[exc + 2 * withfloats] = rawstart + self.mc = None DESCR_REF = 0x00 DESCR_INT = 0x01 DESCR_FLOAT = 0x02 DESCR_SPECIAL = 0x03 CODE_FROMSTACK = 64 - CODE_STOP = 0 | DESCR_SPECIAL - CODE_HOLE = 4 | DESCR_SPECIAL - CODE_INPUTARG = 8 | DESCR_SPECIAL + CODE_STOP = 0 | DESCR_SPECIAL + CODE_HOLE = 4 | DESCR_SPECIAL + CODE_INPUTARG = 8 | DESCR_SPECIAL + CODE_FORCED = 12 | DESCR_SPECIAL #XXX where should this be written? - def gen_descr_encoding(self, descr, failargs, locs): + def write_failure_recovery_description(self, descr, failargs, locs): assert self.mc is not None for i in range(len(failargs)): arg = failargs[i] @@ -575,26 +628,33 @@ self.mc.writechar(chr(n)) self.mc.writechar(chr(self.CODE_STOP)) - fdescr = self.cpu.get_fail_descr_number(descr) - self.mc.write32(fdescr) + + def generate_quick_failure(self, guardtok, fcond=c.AL): + assert isinstance(guardtok.save_exc, bool) + fail_index = self.cpu.get_fail_descr_number(guardtok.descr) + startpos = self.mc.currpos() + withfloats = False + for box in guardtok.failargs: + if box is not None and box.type == FLOAT: + withfloats = True + break + exc = guardtok.save_exc + target = self.failure_recovery_code[exc + 2 * withfloats] + assert target != 0 + self.mc.BL(target) + # write tight data that describes the failure recovery + if guardtok.is_guard_not_forced: + self.mc.writechar(chr(self.CODE_FORCED)) + self.write_failure_recovery_description(guardtok.descr, + guardtok.failargs, guardtok.faillocs[1:]) + self.mc.write32(fail_index) + # for testing the decoding, write a final byte 0xCC + if not we_are_translated(): + self.mc.writechar('\xCC') + faillocs = [loc for loc in guardtok.faillocs if loc is not None] + guardtok.descr._arm_debug_faillocs = faillocs self.align() - - # assert that the fail_boxes lists are big enough - assert len(failargs) <= self.fail_boxes_int.SIZE - - def _gen_path_to_exit_path(self, descr, args, arglocs, - save_exc, fcond=c.AL): - assert isinstance(save_exc, bool) - self.gen_exit_code(self.mc, save_exc, fcond) - self.gen_descr_encoding(descr, args, arglocs[1:]) - - def gen_exit_code(self, mc, save_exc, fcond=c.AL): - assert isinstance(save_exc, bool) - if save_exc: - path = self._leave_jitted_hook_save_exc - else: - path = self._leave_jitted_hook - mc.BL(path) + return startpos def align(self): while(self.mc.currpos() % FUNC_ALIGN != 0): @@ -841,15 +901,10 @@ def write_pending_failure_recoveries(self): for tok in self.pending_guards: - descr = tok.descr #generate the exit stub and the encoded representation - pos = self.mc.currpos() - tok.pos_recovery_stub = pos - - self._gen_path_to_exit_path(descr, tok.failargs, - tok.faillocs, save_exc=tok.save_exc) + tok.pos_recovery_stub = self.generate_quick_failure(tok) # store info on the descr - descr._arm_current_frame_depth = tok.faillocs[0].getint() + tok.descr._arm_current_frame_depth = tok.faillocs[0].getint() def process_pending_guards(self, block_start): clt = self.current_clt @@ -860,7 +915,7 @@ descr._arm_failure_recovery_block = failure_recovery_pos relative_offset = tok.pos_recovery_stub - tok.offset guard_pos = block_start + tok.offset - if not tok.is_invalidate: + if not tok.is_guard_not_invalidated: # patch the guard jumpt to the stub # overwrite the generate NOP with a B_offs to the pos of the # stub @@ -1255,11 +1310,6 @@ else: raise AssertionError('Trying to pop to an invalid location') - def leave_jitted_hook(self): - ptrs = self.fail_boxes_ptr.ar - llop.gc_assume_young_pointers(lltype.Void, - llmemory.cast_ptr_to_adr(ptrs)) - def malloc_cond(self, nursery_free_adr, nursery_top_adr, size): assert size & (WORD-1) == 0 # must be correctly aligned diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py --- a/pypy/jit/backend/arm/opassembler.py +++ b/pypy/jit/backend/arm/opassembler.py @@ -16,6 +16,7 @@ gen_emit_unary_float_op, saved_registers, count_reg_args) +from pypy.jit.backend.arm.helper.regalloc import check_imm_arg from pypy.jit.backend.arm.codebuilder import ARMv7Builder, OverwritingBuilder from pypy.jit.backend.arm.jump import remap_frame_layout from pypy.jit.backend.arm.regalloc import TempInt, TempPtr @@ -27,18 +28,20 @@ from pypy.jit.metainterp.history import JitCellToken, TargetToken from pypy.jit.metainterp.resoperation import rop from pypy.rlib.objectmodel import we_are_translated -from pypy.rpython.lltypesystem import rstr +from pypy.rlib import rgc +from pypy.rpython.lltypesystem import rstr, rffi, lltype, llmemory NO_FORCE_INDEX = -1 class GuardToken(object): - def __init__(self, descr, failargs, faillocs, offset, - save_exc, fcond=c.AL, is_invalidate=False): + def __init__(self, descr, failargs, faillocs, offset, save_exc, fcond=c.AL, + is_guard_not_invalidated=False, is_guard_not_forced=False): assert isinstance(save_exc, bool) self.descr = descr self.offset = offset - self.is_invalidate = is_invalidate + self.is_guard_not_invalidated = is_guard_not_invalidated + self.is_guard_not_forced = is_guard_not_forced self.failargs = failargs self.faillocs = faillocs self.save_exc = save_exc @@ -190,7 +193,8 @@ return fcond def _emit_guard(self, op, arglocs, fcond, save_exc, - is_guard_not_invalidated=False): + is_guard_not_invalidated=False, + is_guard_not_forced=False): assert isinstance(save_exc, bool) assert isinstance(fcond, int) descr = op.getdescr() @@ -210,7 +214,8 @@ faillocs=arglocs, offset=pos, save_exc=save_exc, - is_invalidate=is_guard_not_invalidated, + is_guard_not_invalidated=is_guard_not_invalidated, + is_guard_not_forced=is_guard_not_forced, fcond=fcond)) return c.AL @@ -312,49 +317,10 @@ return fcond def emit_op_finish(self, op, arglocs, regalloc, fcond): - for i in range(len(arglocs) - 1): - loc = arglocs[i] - box = op.getarg(i) - if loc is None: - continue - if loc.is_reg(): - if box.type == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(i) - elif box.type == INT: - adr = self.fail_boxes_int.get_addr_for_num(i) - else: - assert 0 - self.mc.gen_load_int(r.ip.value, adr) - self.mc.STR_ri(loc.value, r.ip.value) - elif loc.is_vfp_reg(): - assert box.type == FLOAT - adr = self.fail_boxes_float.get_addr_for_num(i) - self.mc.gen_load_int(r.ip.value, adr) - self.mc.VSTR(loc.value, r.ip.value) - elif loc.is_stack() or loc.is_imm() or loc.is_imm_float(): - if box.type == FLOAT: - adr = self.fail_boxes_float.get_addr_for_num(i) - self.mov_loc_loc(loc, r.vfp_ip) - self.mc.gen_load_int(r.ip.value, adr) - self.mc.VSTR(r.vfp_ip.value, r.ip.value) - elif box.type == REF or box.type == INT: - if box.type == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(i) - elif box.type == INT: - adr = self.fail_boxes_int.get_addr_for_num(i) - else: - assert 0 - self.mov_loc_loc(loc, r.ip) - self.mc.gen_load_int(r.lr.value, adr) - self.mc.STR_ri(r.ip.value, r.lr.value) - else: - assert 0 - # note: no exception should currently be set in llop.get_exception_addr - # even if this finish may be an exit_frame_with_exception (in this case - # the exception instance is in arglocs[0]). - addr = self.cpu.get_on_leave_jitted_int(save_exception=False) - self.mc.BL(addr) - self.mc.gen_load_int(r.r0.value, arglocs[-1].value) + [argloc] = arglocs + if argloc is not r.r0: #XXX verify this + self.mov_loc_loc(argloc, r.r0, fcond) + # exit function self.gen_func_epilog() return fcond @@ -1152,8 +1118,26 @@ value = self.cpu.done_with_this_frame_float_v else: raise AssertionError(kind) - self.mc.gen_load_int(r.ip.value, value) - self.mc.CMP_rr(tmploc.value, r.ip.value) + from pypy.jit.backend.llsupport.descr import unpack_fielddescr + from pypy.jit.backend.llsupport.descr import unpack_interiorfielddescr + descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) + _offset, _size, _ = unpack_fielddescr(descrs.jf_descr) + fail_descr = self.cpu.get_fail_descr_from_number(value) + value = fail_descr.hide(self.cpu) + rgc._make_sure_does_not_move(value) + value = rffi.cast(lltype.Signed, value) + + if check_imm_arg(_offset): + self.mc.LDR_ri(r.ip.value, tmploc.value, imm=_offset) + else: + self.mc.gen_load_int(r.ip.value, _offset) + self.mc.LDR_rr(r.ip.value, tmploc.value, r.ip.value) + if check_imm_arg(value): + self.mc.CMP_ri(r.ip.value, imm=value) + else: + self.mc.gen_load_int(r.lr.value, value) + self.mc.CMP_rr(r.lr.value, r.ip.value) + #if values are equal we take the fast path # Slow path, calling helper @@ -1180,19 +1164,31 @@ if op.result is not None: # load the return value from fail_boxes_xxx[0] kind = op.result.type - if kind == INT: - adr = self.fail_boxes_int.get_addr_for_num(0) - elif kind == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(0) - elif kind == FLOAT: - adr = self.fail_boxes_float.get_addr_for_num(0) + if kind == FLOAT: + t = unpack_interiorfielddescr(descrs.as_float)[0] + if not check_imm_arg(t): + self.mc.gen_load_int(r.ip.value, t, cond=fast_path_cond) + self.mc.ADD_rr(r.ip.value, r.r0.value, r.ip.value, + cond=fast_path_cond) + t = 0 + base = r.ip + else: + base = r.r0 + self.mc.VLDR(resloc.value, base.value, imm=t, + cond=fast_path_cond) else: - raise AssertionError(kind) - self.mc.gen_load_int(r.ip.value, adr, cond=fast_path_cond) - if op.result.type == FLOAT: - self.mc.VLDR(resloc.value, r.ip.value, cond=fast_path_cond) - else: - self.mc.LDR_ri(resloc.value, r.ip.value, cond=fast_path_cond) + assert resloc is r.r0 + if kind == INT: + t = unpack_interiorfielddescr(descrs.as_int)[0] + else: + t = unpack_interiorfielddescr(descrs.as_ref)[0] + if not check_imm_arg(t): + self.mc.gen_load_int(r.ip.value, t, cond=fast_path_cond) + self.mc.LDR_rr(resloc.value, resloc.value, r.ip.value, + cond=fast_path_cond) + else: + self.mc.LDR_ri(resloc.value, resloc.value, imm=t, + cond=fast_path_cond) # jump to merge point jmp_pos = self.mc.currpos() self.mc.BKPT() @@ -1266,7 +1262,8 @@ self.mc.LDR_ri(r.ip.value, r.fp.value) self.mc.CMP_ri(r.ip.value, 0) - self._emit_guard(guard_op, arglocs[1 + numargs:], c.GE, save_exc=True) + self._emit_guard(guard_op, arglocs[1 + numargs:], c.GE, + save_exc=True, is_guard_not_forced=True) return fcond def emit_guard_call_release_gil(self, op, guard_op, arglocs, regalloc, diff --git a/pypy/jit/backend/arm/regalloc.py b/pypy/jit/backend/arm/regalloc.py --- a/pypy/jit/backend/arm/regalloc.py +++ b/pypy/jit/backend/arm/regalloc.py @@ -644,15 +644,9 @@ return args def prepare_op_finish(self, op, fcond): - args = [None] * (op.numargs() + 1) - for i in range(op.numargs()): - arg = op.getarg(i) - if arg: - args[i] = self.loc(arg) - self.possibly_free_var(arg) - n = self.cpu.get_fail_descr_number(op.getdescr()) - args[-1] = imm(n) - return args + loc = self.loc(op.getarg(0)) + self.possibly_free_var(op.getarg(0)) + return [loc] def prepare_op_guard_true(self, op, fcond): l0 = self.make_sure_var_in_reg(op.getarg(0)) diff --git a/pypy/jit/backend/arm/runner.py b/pypy/jit/backend/arm/runner.py --- a/pypy/jit/backend/arm/runner.py +++ b/pypy/jit/backend/arm/runner.py @@ -23,15 +23,21 @@ AbstractLLCPU.__init__(self, rtyper, stats, opts, translate_support_code, gcdescr) + from pypy.jit.backend.llsupport import jitframe + self.deadframe_size_max = llmemory.sizeof(jitframe.DEADFRAME, + self.get_failargs_limit()) + def set_debug(self, flag): return self.assembler.set_debug(flag) + def get_failargs_limit(self): + if self.opts is not None: + return self.opts.failargs_limit + else: + return 1000 + def setup(self): - if self.opts is not None: - failargs_limit = self.opts.failargs_limit - else: - failargs_limit = 1000 - self.assembler = AssemblerARM(self, failargs_limit=failargs_limit) + self.assembler = AssemblerARM(self, self.translate_support_code) def setup_once(self): self.assembler.setup_once() @@ -51,24 +57,6 @@ return self.assembler.assemble_bridge(faildescr, inputargs, operations, original_loop_token, log=log) - def get_latest_value_float(self, index): - return self.assembler.fail_boxes_float.getitem(index) - - def get_latest_value_int(self, index): - return self.assembler.fail_boxes_int.getitem(index) - - def get_latest_value_ref(self, index): - return self.assembler.fail_boxes_ptr.getitem(index) - - def get_latest_value_count(self): - return self.assembler.fail_boxes_count - - def get_latest_force_token(self): - return self.assembler.fail_force_index - - def get_on_leave_jitted_hook(self): - return self.assembler.leave_jitted_hook - def clear_latest_values(self, count): setitem = self.assembler.fail_boxes_ptr.setitem null = lltype.nullptr(llmemory.GCREF.TO) @@ -76,7 +64,7 @@ setitem(index, null) def make_execute_token(self, *ARGS): - FUNCPTR = lltype.Ptr(lltype.FuncType(ARGS, lltype.Signed)) + FUNCPTR = lltype.Ptr(lltype.FuncType(ARGS, llmemory.GCREF)) def execute_token(executable_token, *args): clt = executable_token.compiled_loop_token @@ -91,12 +79,13 @@ prev_interpreter = LLInterpreter.current_interpreter LLInterpreter.current_interpreter = self.debug_ll_interpreter try: - fail_index = func(*args) + deadframe = func(*args) finally: if not self.translate_support_code: LLInterpreter.current_interpreter = prev_interpreter #llop.debug_print(lltype.Void, "<<<< Back") - return self.get_fail_descr_from_number(fail_index) + self.gc_set_extra_threshold() + return deadframe return execute_token def cast_ptr_to_int(x): @@ -115,17 +104,22 @@ assert fail_index >= 0, "already forced!" faildescr = self.get_fail_descr_from_number(fail_index) rffi.cast(TP, addr_of_force_index)[0] = ~fail_index - bytecode = self.assembler._find_failure_recovery_bytecode(faildescr) + frb = self.assembler._find_failure_recovery_bytecode(faildescr) + bytecode = rffi.cast(rffi.UCHARP, frb) addr_all_null_regsiters = rffi.cast(rffi.LONG, self.all_null_registers) - # start of "no gc operation!" block - fail_index_2 = self.assembler.failure_recovery_func( - bytecode, - addr_of_force_index, - addr_all_null_regsiters) - self.assembler.leave_jitted_hook() - # end of "no gc operation!" block - assert fail_index == fail_index_2 - return faildescr + # + assert (rffi.cast(lltype.Signed, bytecode[0]) == + self.assembler.CODE_FORCED) + bytecode = rffi.ptradd(bytecode, 1) + deadframe = self.assembler.grab_frame_values(self, + bytecode, addr_of_force_index, + self.all_null_registers, + self.all_null_registers) + # + assert self.get_latest_descr(deadframe) is faildescr + self.assembler.force_token_to_dead_frame[addr_of_force_index] = ( + deadframe) + return deadframe def redirect_call_assembler(self, oldlooptoken, newlooptoken): self.assembler.redirect_call_assembler(oldlooptoken, newlooptoken) diff --git a/pypy/jit/backend/arm/test/test_loop_unroll.py b/pypy/jit/backend/arm/test/test_loop_unroll.py --- a/pypy/jit/backend/arm/test/test_loop_unroll.py +++ b/pypy/jit/backend/arm/test/test_loop_unroll.py @@ -1,8 +1,8 @@ import py -from pypy.jit.backend.x86.test.test_basic import Jit386Mixin +from pypy.jit.backend.arm.test.support import JitARMMixin from pypy.jit.metainterp.test import test_loop_unroll -class TestLoopSpec(Jit386Mixin, test_loop_unroll.LoopUnrollTest): +class TestLoopSpec(JitARMMixin, test_loop_unroll.LoopUnrollTest): # for the individual tests see # ====> ../../../metainterp/test/test_loop.py pass diff --git a/pypy/jit/backend/arm/test/test_runner.py b/pypy/jit/backend/arm/test/test_runner.py --- a/pypy/jit/backend/arm/test/test_runner.py +++ b/pypy/jit/backend/arm/test/test_runner.py @@ -60,16 +60,18 @@ ] cpu.compile_loop(inp, operations, looptoken) args = [i for i in range(1, 15)] - self.cpu.execute_token(looptoken, *args) - output = [self.cpu.get_latest_value_int(i - 1) for i in range(1, 15)] + deadframe = self.cpu.execute_token(looptoken, *args) + output = [self.cpu.get_latest_value_int(deadframe, i - 1) for i in range(1, 15)] expected = [3, 7, 11, 15, 19, 23, 27, 3, 7, 11, 15, 19, 23, 27] assert output == expected - def test_redirect_call_assember2(self): - def assembler_helper(failindex, virtualizable): - return self.cpu.get_latest_value_int(0) + def test_redirect_call_assembler2(self): + def assembler_helper(deadframe, virtualizable): + x = self.cpu.get_latest_value_int(deadframe, 0) + assert x == 11 + return 7 - FUNCPTR = lltype.Ptr(lltype.FuncType([lltype.Signed, llmemory.GCREF], + FUNCPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF, llmemory.GCREF], lltype.Signed)) class FakeJitDriverSD: @@ -101,12 +103,12 @@ self.cpu.compile_loop(loop2.inputargs, loop2.operations, lt2) self.cpu.compile_loop(loop3.inputargs, loop3.operations, lt3) self.cpu.compile_loop(loop1.inputargs, loop1.operations, lt1) - self.cpu.execute_token(lt1, 11) - assert self.cpu.get_latest_value_int(0) == 12 + df = self.cpu.execute_token(lt1, 10) + assert self.cpu.get_latest_value_int(df, 0) == 7 self.cpu.redirect_call_assembler(lt2, lt3) - self.cpu.execute_token(lt1, 11) - assert self.cpu.get_latest_value_int(0) == 10 + df = self.cpu.execute_token(lt1, 12) + assert self.cpu.get_latest_value_int(df, 0) == 7 SFloat = lltype.GcForwardReference() SFloat.become(lltype.GcStruct('SFloat', ('parent', rclass.OBJECT), @@ -202,8 +204,8 @@ ARGS = [lltype.Signed] * numargs RES = lltype.Signed args = [i+1 for i in range(numargs)] - res = self.cpu.execute_token(looptoken, *args) - assert self.cpu.get_latest_value_int(0) == sum(args) + deadframe = self.cpu.execute_token(looptoken, *args) + assert self.cpu.get_latest_value_int(deadframe, 0) == sum(args) def test_debugger_on(self): from pypy.rlib import debug diff --git a/pypy/jit/backend/llgraph/runner.py b/pypy/jit/backend/llgraph/runner.py --- a/pypy/jit/backend/llgraph/runner.py +++ b/pypy/jit/backend/llgraph/runner.py @@ -28,7 +28,7 @@ try: newbox = _cache[box] except KeyError: - newbox = _cache[box] = box.clonebox() + newbox = _cache[box] = box.__class__() return newbox # self.inputargs = map(mapping, inputargs) @@ -814,6 +814,12 @@ return res def execute_call_release_gil(self, descr, func, *args): + if hasattr(descr, '_original_func_'): + func = descr._original_func_ # see pyjitpl.py + # we want to call the function that does the aroundstate + # manipulation here (as a hack, instead of really doing + # the aroundstate manipulation ourselves) + return self.execute_call_may_force(descr, func, *args) call_args = support.cast_call_args_in_order(descr.ARGS, args) FUNC = lltype.FuncType(descr.ARGS, descr.RESULT) func_to_call = rffi.cast(lltype.Ptr(FUNC), func) diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py --- a/pypy/jit/backend/llsupport/llmodel.py +++ b/pypy/jit/backend/llsupport/llmodel.py @@ -193,6 +193,12 @@ # ------------------- helpers and descriptions -------------------- + def gc_set_extra_threshold(self): + llop.gc_set_extra_threshold(lltype.Void, self.deadframe_size_max) + + def gc_clear_extra_threshold(self): + llop.gc_set_extra_threshold(lltype.Void, 0) + @staticmethod def _cast_int_to_gcref(x): # dangerous! only use if you are sure no collection could occur diff --git a/pypy/jit/backend/x86/runner.py b/pypy/jit/backend/x86/runner.py --- a/pypy/jit/backend/x86/runner.py +++ b/pypy/jit/backend/x86/runner.py @@ -58,12 +58,6 @@ else: return 1000 - def gc_set_extra_threshold(self): - llop.gc_set_extra_threshold(lltype.Void, self.deadframe_size_max) - - def gc_clear_extra_threshold(self): - llop.gc_set_extra_threshold(lltype.Void, 0) - def setup(self): self.assembler = Assembler386(self, self.translate_support_code) diff --git a/pypy/jit/codewriter/call.py b/pypy/jit/codewriter/call.py --- a/pypy/jit/codewriter/call.py +++ b/pypy/jit/codewriter/call.py @@ -134,9 +134,14 @@ if getattr(funcobj, 'graph', None) is None: return 'residual' targetgraph = funcobj.graph - if (hasattr(targetgraph, 'func') and - hasattr(targetgraph.func, 'oopspec')): - return 'builtin' + if hasattr(targetgraph, 'func'): + # must never produce JitCode for a function with + # _gctransformer_hint_close_stack_ set! + if getattr(targetgraph.func, + '_gctransformer_hint_close_stack_', False): + return 'residual' + if hasattr(targetgraph.func, 'oopspec'): + return 'builtin' elif op.opname == 'oosend': SELFTYPE, methname, opargs = support.decompose_oosend(op) if SELFTYPE.oopspec_name is not None: @@ -164,6 +169,13 @@ try: return self.jitcodes[graph] except KeyError: + # must never produce JitCode for a function with + # _gctransformer_hint_close_stack_ set! + if hasattr(graph, 'func') and getattr(graph.func, + '_gctransformer_hint_close_stack_', False): + raise AssertionError( + '%s has _gctransformer_hint_close_stack_' % (graph,)) + # fnaddr, calldescr = self.get_jitcode_calldescr(graph) jitcode = JitCode(graph.name, fnaddr, calldescr, called_from=called_from) @@ -208,6 +220,7 @@ # get the 'elidable' and 'loopinvariant' flags from the function object elidable = False loopinvariant = False + call_release_gil_target = llmemory.NULL if op.opname == "direct_call": funcobj = get_funcobj(op.args[0].value) assert getattr(funcobj, 'calling_conv', 'c') == 'c', ( @@ -218,6 +231,10 @@ if loopinvariant: assert not NON_VOID_ARGS, ("arguments not supported for " "loop-invariant function!") + if getattr(func, "_call_aroundstate_target_", None): + call_release_gil_target = func._call_aroundstate_target_ + call_release_gil_target = llmemory.cast_ptr_to_adr( + call_release_gil_target) # build the extraeffect random_effects = self.randomeffects_analyzer.analyze(op) if random_effects: @@ -241,7 +258,7 @@ # effectinfo = effectinfo_from_writeanalyze( self.readwrite_analyzer.analyze(op), self.cpu, extraeffect, - oopspecindex, can_invalidate) + oopspecindex, can_invalidate, call_release_gil_target) # assert effectinfo is not None if elidable or loopinvariant: diff --git a/pypy/jit/codewriter/effectinfo.py b/pypy/jit/codewriter/effectinfo.py --- a/pypy/jit/codewriter/effectinfo.py +++ b/pypy/jit/codewriter/effectinfo.py @@ -1,6 +1,6 @@ from pypy.jit.metainterp.typesystem import deref, fieldType, arrayItem from pypy.rpython.lltypesystem.rclass import OBJECT -from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.ootypesystem import ootype from pypy.translator.backendopt.graphanalyze import BoolGraphAnalyzer @@ -87,7 +87,8 @@ write_descrs_fields, write_descrs_arrays, extraeffect=EF_CAN_RAISE, oopspecindex=OS_NONE, - can_invalidate=False): + can_invalidate=False, + call_release_gil_target=llmemory.NULL): key = (frozenset_or_none(readonly_descrs_fields), frozenset_or_none(readonly_descrs_arrays), frozenset_or_none(write_descrs_fields), @@ -95,6 +96,8 @@ extraeffect, oopspecindex, can_invalidate) + if call_release_gil_target: + key += (object(),) # don't care about caching in this case if key in cls._cache: return cls._cache[key] if extraeffect == EffectInfo.EF_RANDOM_EFFECTS: @@ -121,6 +124,7 @@ result.extraeffect = extraeffect result.can_invalidate = can_invalidate result.oopspecindex = oopspecindex + result.call_release_gil_target = call_release_gil_target if result.check_can_raise(): assert oopspecindex in cls._OS_CANRAISE cls._cache[key] = result @@ -142,6 +146,9 @@ def has_random_effects(self): return self.extraeffect >= self.EF_RANDOM_EFFECTS + def is_call_release_gil(self): + return bool(self.call_release_gil_target) + def frozenset_or_none(x): if x is None: @@ -156,7 +163,8 @@ def effectinfo_from_writeanalyze(effects, cpu, extraeffect=EffectInfo.EF_CAN_RAISE, oopspecindex=EffectInfo.OS_NONE, - can_invalidate=False): + can_invalidate=False, + call_release_gil_target=llmemory.NULL): from pypy.translator.backendopt.writeanalyze import top_set if effects is top_set or extraeffect == EffectInfo.EF_RANDOM_EFFECTS: readonly_descrs_fields = None @@ -204,7 +212,8 @@ write_descrs_arrays, extraeffect, oopspecindex, - can_invalidate) + can_invalidate, + call_release_gil_target) def consider_struct(TYPE, fieldname): if fieldType(TYPE, fieldname) is lltype.Void: diff --git a/pypy/jit/codewriter/test/test_call.py b/pypy/jit/codewriter/test/test_call.py --- a/pypy/jit/codewriter/test/test_call.py +++ b/pypy/jit/codewriter/test/test_call.py @@ -1,6 +1,6 @@ import py from pypy.objspace.flow.model import SpaceOperation, Constant, Variable -from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.translator.unsimplify import varoftype from pypy.rlib import jit from pypy.jit.codewriter.call import CallControl @@ -192,6 +192,32 @@ [op] = block.operations call_descr = cc.getcalldescr(op) assert call_descr.extrainfo.has_random_effects() + assert call_descr.extrainfo.is_call_release_gil() is False + +def test_call_release_gil(): + from pypy.jit.backend.llgraph.runner import LLtypeCPU + + T = rffi.CArrayPtr(rffi.TIME_T) + external = rffi.llexternal("time", [T], rffi.TIME_T, threadsafe=True) + + # no jit.dont_look_inside in this test + def f(): + return external(lltype.nullptr(T.TO)) + + rtyper = support.annotate(f, []) + jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0]) + cc = CallControl(LLtypeCPU(rtyper), jitdrivers_sd=[jitdriver_sd]) + res = cc.find_all_graphs(FakePolicy()) + + [llext_graph] = [x for x in res if x.func is external] + [block, _] = list(llext_graph.iterblocks()) + [op] = block.operations + call_target = op.args[0].value._obj.graph.func._call_aroundstate_target_ + call_target = llmemory.cast_ptr_to_adr(call_target) + call_descr = cc.getcalldescr(op) + assert call_descr.extrainfo.has_random_effects() + assert call_descr.extrainfo.is_call_release_gil() is True + assert call_descr.extrainfo.call_release_gil_target == call_target def test_random_effects_on_stacklet_switch(): from pypy.jit.backend.llgraph.runner import LLGraphCPU diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py --- a/pypy/jit/metainterp/pyjitpl.py +++ b/pypy/jit/metainterp/pyjitpl.py @@ -1348,6 +1348,8 @@ self.metainterp.vable_and_vrefs_before_residual_call() resbox = self.metainterp.execute_and_record_varargs( rop.CALL_MAY_FORCE, allboxes, descr=descr) + if effectinfo.is_call_release_gil(): + self.metainterp.direct_call_release_gil() self.metainterp.vrefs_after_residual_call() vablebox = None if assembler_call: @@ -1360,6 +1362,7 @@ if vablebox is not None: self.metainterp.history.record(rop.KEEPALIVE, [vablebox], None) self.metainterp.handle_possible_exception() + # XXX refactor: direct_libffi_call() is a hack if effectinfo.oopspecindex == effectinfo.OS_LIBFFI_CALL: self.metainterp.direct_libffi_call() return resbox @@ -2606,6 +2609,19 @@ [box_resultpos, ConstInt(0), box_result], None, descr) + def direct_call_release_gil(self): + op = self.history.operations.pop() + assert op.opnum == rop.CALL_MAY_FORCE + descr = op.getdescr() + effectinfo = descr.get_extra_info() + realfuncaddr = effectinfo.call_release_gil_target + funcbox = ConstInt(heaptracker.adr2int(realfuncaddr)) + self.history.record(rop.CALL_RELEASE_GIL, + [funcbox] + op.getarglist()[1:], + op.result, descr) + if not we_are_translated(): # for llgraph + descr._original_func_ = op.getarg(0).value + # ____________________________________________________________ class ChangeFrame(JitException): diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py --- a/pypy/jit/metainterp/test/test_ajit.py +++ b/pypy/jit/metainterp/test/test_ajit.py @@ -3979,5 +3979,36 @@ rgc.add_memory_pressure(1234) return 3 - self.interp_operations(f, []) + def test_external_call(self): + from pypy.rlib.objectmodel import invoke_around_extcall + T = rffi.CArrayPtr(rffi.TIME_T) + external = rffi.llexternal("time", [T], rffi.TIME_T) + + class Oups(Exception): + pass + class State: + pass + state = State() + + def before(): + if we_are_jitted(): + raise Oups + state.l.append("before") + + def after(): + if we_are_jitted(): + raise Oups + state.l.append("after") + + def f(): + state.l = [] + invoke_around_extcall(before, after) + external(lltype.nullptr(T.TO)) + return len(state.l) + + res = self.interp_operations(f, []) + assert res == 2 + res = self.interp_operations(f, []) + assert res == 2 + self.check_operations_history(call_release_gil=1, call_may_force=0) diff --git a/pypy/module/README.txt b/pypy/module/README.txt --- a/pypy/module/README.txt +++ b/pypy/module/README.txt @@ -3,7 +3,7 @@ that require access to interpreter level. See here for more information: - http://codespeak.net/pypy/dist/pypy/doc/coding-guide.html#modules-in-pypy + http://doc.pypy.org/en/latest/coding-guide.html#modules-in-pypy ATTENTION: don't put any '.py' files directly into pypy/module because you can easily get import mixups on e.g. "import sys" diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py --- a/pypy/module/_sre/interp_sre.py +++ b/pypy/module/_sre/interp_sre.py @@ -54,7 +54,7 @@ result = [-1] * (2 * num_groups) mark = ctx.match_marks while mark is not None: - index = mark.gid + index = jit.promote(mark.gid) if result[index] == -1: result[index] = mark.position mark = mark.prev @@ -90,7 +90,7 @@ # SRE_Pattern class class W_SRE_Pattern(Wrappable): - _immutable_fields_ = ["code", "flags", "num_groups"] + _immutable_fields_ = ["code", "flags", "num_groups", "w_groupindex"] def cannot_copy_w(self): space = self.space diff --git a/pypy/module/sys/app.py b/pypy/module/sys/app.py --- a/pypy/module/sys/app.py +++ b/pypy/module/sys/app.py @@ -66,11 +66,11 @@ return None copyright_str = """ -Copyright 2003-2012 PyPy development team. +Copyright 2003-2013 PyPy development team. All Rights Reserved. For further information, see -Portions Copyright (c) 2001-2012 Python Software Foundation. +Portions Copyright (c) 2001-2013 Python Software Foundation. All Rights Reserved. Portions Copyright (c) 2000 BeOpen.com. diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py --- a/pypy/rpython/lltypesystem/rffi.py +++ b/pypy/rpython/lltypesystem/rffi.py @@ -171,6 +171,7 @@ call_external_function._dont_inline_ = True call_external_function._annspecialcase_ = 'specialize:ll' call_external_function._gctransformer_hint_close_stack_ = True + call_external_function._call_aroundstate_target_ = funcptr call_external_function = func_with_new_name(call_external_function, 'ccall_' + name) # don't inline, as a hack to guarantee that no GC pointer is alive diff --git a/pypy/tool/option.py b/pypy/tool/option.py --- a/pypy/tool/option.py +++ b/pypy/tool/option.py @@ -6,7 +6,7 @@ import optparse extra_useage = """For detailed descriptions of all the options see -http://codespeak.net/pypy/dist/pypy/doc/config/commandline.html""" +http://doc.pypy.org/en/latest/config/commandline.html""" def get_standard_options(): config = get_pypy_config() diff --git a/pypy/tool/release/make_release.py b/pypy/tool/release/make_release.py deleted file mode 100755 --- a/pypy/tool/release/make_release.py +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env python - -""" A tool to download correct pypy-c's from nightly build run and package them -into release packages. Note: you must run apropriate buildbots first and -make sure there are no failures. Use force-builds.py from the same directory. - -Usage: make_release.py - e.g.: make_release.py release-1.4.1 1.4.1 -""" - -import autopath -import sys -import urllib2 -from xml.dom import minidom -import re -import py -from pypy.tool.udir import udir -from pypy.tool.release.package import package -import tarfile -import os -import shutil - -BASEURL = 'http://buildbot.pypy.org/nightly/' -PAUSE = False - -def browse_nightly(branch, - baseurl=BASEURL, - override_xml=None): - if override_xml is None: - url = baseurl + branch + '/' - xml = urllib2.urlopen(url).read() - else: - xml = override_xml - dom = minidom.parseString(xml) - refs = [node.getAttribute('href') for node in dom.getElementsByTagName('a') - if 'pypy' in node.getAttribute('href')] - # all refs are of form: pypy-c-{type}-{revnum}-{hghash}-{platform}.tar.bz2 - r = re.compile('pypy-c-([\w\d]+)-(\d+)-([0-9a-f]+)-([\w\d]+).tar.bz2$') - d = {} - for ref in refs: - kind, revnum, hghash, platform = r.match(ref).groups() - rev = int(revnum) - try: - lastrev, _ = d[(kind, platform)] - except KeyError: - lastrev = -1 - if rev > lastrev: - d[(kind, platform)] = rev, ref - return d - -def main(branch, release): - to_download = browse_nightly(branch) - tmpdir = udir.join('download') - tmpdir.ensure(dir=True) - alltars = [] - olddir = os.getcwd() - try: - os.chdir(str(tmpdir)) - print 'Using tmpdir', str(tmpdir) - for (kind, platform), (rev, name) in to_download.iteritems(): - if platform == 'win32': - print 'Ignoring %s, windows unsupported' % name - else: - print "Downloading %s at rev %d" % (name, rev) - url = BASEURL + branch + "/" + name - data = urllib2.urlopen(url).read() - tmpdir.join(name).write(data, mode="wb") - t = tarfile.open(str(tmpdir.join(name))) - dirname = t.getmembers()[0].name - t.extractall(path=str(tmpdir)) - if kind == 'jit': - kind = '' - else: - kind = '-' + kind - topdirname = 'pypy-%s-%s%s' % (release, platform, kind) - os.system('mv %s %s' % (str(tmpdir.join(dirname)), - str(tmpdir.join(topdirname)))) - if PAUSE: - print 'Pausing, press Enter...' - raw_input() - name = '%s.tar.bz2' % topdirname - print "Building %s" % name - t = tarfile.open(name, 'w:bz2') - t.add(topdirname) - alltars.append(name) - t.close() - shutil.rmtree(str(tmpdir.join(topdirname))) - for name in alltars: - print "Uploading %s" % name - os.system('scp %s codespeak.net:/www/pypy.org/htdocs/download' % name) - finally: - os.chdir(olddir) - -if __name__ == '__main__': - if len(sys.argv) != 3: - print __doc__ - sys.exit(1) - main(sys.argv[1], release=sys.argv[2]) - diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -122,7 +122,7 @@ try: os.chdir(str(builddir)) # - # 'strip' fun: see https://codespeak.net/issue/pypy-dev/issue587 + # 'strip' fun: see issue #587 for source, target in binaries: if sys.platform == 'win32': pass From noreply at buildbot.pypy.org Sat Jan 5 16:45:12 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 5 Jan 2013 16:45:12 +0100 (CET) Subject: [pypy-commit] pypy remove-globals-in-jit: Pff pff pff. Test (in model.py) and fix for running all of Message-ID: <20130105154512.9B5531C01EE@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: remove-globals-in-jit Changeset: r59728:6f6aaf8a40cf Date: 2013-01-05 16:44 +0100 http://bitbucket.org/pypy/pypy/changeset/6f6aaf8a40cf/ Log: Pff pff pff. Test (in model.py) and fix for running all of x86/test/test_ztranslation.py in the same test session. diff --git a/pypy/jit/backend/model.py b/pypy/jit/backend/model.py --- a/pypy/jit/backend/model.py +++ b/pypy/jit/backend/model.py @@ -1,4 +1,5 @@ from pypy.rlib.debug import debug_start, debug_print, debug_stop +from pypy.rlib.objectmodel import we_are_translated from pypy.jit.metainterp import history from pypy.rpython.lltypesystem import lltype @@ -42,6 +43,12 @@ def get_fail_descr_number(self, descr): assert isinstance(descr, history.AbstractFailDescr) + if not we_are_translated(): + if not hasattr(descr, '_cpu'): + assert descr.index == -1, "descr.index is already >= 0??" + descr._cpu = self + assert descr._cpu is self,"another CPU has already seen the descr!" + # n = descr.index if n < 0: n = self.reserve_some_free_fail_descr_number() diff --git a/pypy/jit/metainterp/compile.py b/pypy/jit/metainterp/compile.py --- a/pypy/jit/metainterp/compile.py +++ b/pypy/jit/metainterp/compile.py @@ -858,8 +858,6 @@ assert exception, "PropagateExceptionDescr: no exception??" raise metainterp_sd.ExitFrameWithExceptionRef(cpu, exception) -propagate_exception_descr = PropagateExceptionDescr() - def compile_tmp_callback(cpu, jitdriver_sd, greenboxes, redargtypes, memory_manager=None): """Make a LoopToken that corresponds to assembler code that just @@ -897,7 +895,7 @@ finishargs = [] # jd = jitdriver_sd - faildescr = propagate_exception_descr + faildescr = PropagateExceptionDescr() operations = [ ResOperation(rop.CALL, callargs, result, descr=jd.portal_calldescr), ResOperation(rop.GUARD_NO_EXCEPTION, [], None, descr=faildescr), diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py --- a/pypy/jit/metainterp/pyjitpl.py +++ b/pypy/jit/metainterp/pyjitpl.py @@ -1473,7 +1473,7 @@ num = self.cpu.get_fail_descr_number(tokens[0].finishdescr) setattr(self.cpu, 'done_with_this_frame_%s_v' % name, num) # - exc_descr = compile.propagate_exception_descr + exc_descr = compile.PropagateExceptionDescr() num = self.cpu.get_fail_descr_number(exc_descr) self.cpu.propagate_exception_v = num # From noreply at buildbot.pypy.org Sat Jan 5 17:05:46 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 5 Jan 2013 17:05:46 +0100 (CET) Subject: [pypy-commit] pypy remove-globals-in-jit: kill tabs Message-ID: <20130105160546.4383C1C01EE@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: remove-globals-in-jit Changeset: r59729:43b334a4ba6c Date: 2013-01-05 18:05 +0200 http://bitbucket.org/pypy/pypy/changeset/43b334a4ba6c/ Log: kill tabs diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py --- a/pypy/jit/backend/arm/assembler.py +++ b/pypy/jit/backend/arm/assembler.py @@ -225,7 +225,7 @@ # # Call the helper, which will return a dead frame object with # the correct exception set, or MemoryError by default - # XXX make sure we return the correct value here + # XXX make sure we return the correct value here addr = rffi.cast(lltype.Signed, self.cpu.get_propagate_exception()) mc.BL(addr) self.gen_func_epilog(mc=mc) diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py --- a/pypy/jit/backend/arm/opassembler.py +++ b/pypy/jit/backend/arm/opassembler.py @@ -36,7 +36,7 @@ class GuardToken(object): def __init__(self, descr, failargs, faillocs, offset, save_exc, fcond=c.AL, - is_guard_not_invalidated=False, is_guard_not_forced=False): + is_guard_not_invalidated=False, is_guard_not_forced=False): assert isinstance(save_exc, bool) self.descr = descr self.offset = offset @@ -319,7 +319,7 @@ def emit_op_finish(self, op, arglocs, regalloc, fcond): [argloc] = arglocs if argloc is not r.r0: #XXX verify this - self.mov_loc_loc(argloc, r.r0, fcond) + self.mov_loc_loc(argloc, r.r0, fcond) # exit function self.gen_func_epilog() return fcond From noreply at buildbot.pypy.org Sat Jan 5 17:08:40 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 5 Jan 2013 17:08:40 +0100 (CET) Subject: [pypy-commit] pypy default: Add a test file for this module. Message-ID: <20130105160840.7B5221C01EE@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r59730:bccfcc04e275 Date: 2013-01-05 17:08 +0100 http://bitbucket.org/pypy/pypy/changeset/bccfcc04e275/ Log: Add a test file for this module. diff --git a/pypy/rlib/test/test_jit_libffi.py b/pypy/rlib/test/test_jit_libffi.py new file mode 100644 --- /dev/null +++ b/pypy/rlib/test/test_jit_libffi.py @@ -0,0 +1,39 @@ +import math +import ctypes +from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rlib import clibffi +from pypy.rlib.rarithmetic import intmask +from pypy.rlib.jit_libffi import CIF_DESCRIPTION +from pypy.rlib.jit_libffi import jit_ffi_prep_cif, jit_ffi_call + + +math_sin = intmask(ctypes.cast(ctypes.CDLL(None).sin, ctypes.c_void_p).value) +math_sin = rffi.cast(rffi.VOIDP, math_sin) + + +def test_jit_ffi_call(): + cd = lltype.malloc(CIF_DESCRIPTION, 1, flavor='raw') + cd.abi = clibffi.FFI_DEFAULT_ABI + cd.nargs = 1 + cd.rtype = clibffi.cast_type_to_ffitype(rffi.DOUBLE) + atypes = lltype.malloc(clibffi.FFI_TYPE_PP.TO, 1, flavor='raw') + atypes[0] = clibffi.cast_type_to_ffitype(rffi.DOUBLE) + cd.atypes = atypes + cd.exchange_size = 64 # 64 bytes of exchange data + cd.exchange_result = 24 + cd.exchange_result_libffi = 24 + cd.exchange_args[0] = 16 + # + jit_ffi_prep_cif(cd) + # + assert rffi.sizeof(rffi.DOUBLE) == 8 + exb = lltype.malloc(rffi.DOUBLEP.TO, 8, flavor='raw') + exb[2] = 1.23 + jit_ffi_call(cd, math_sin, rffi.cast(rffi.CCHARP, exb)) + res = exb[3] + lltype.free(exb, flavor='raw') + # + lltype.free(atypes, flavor='raw') + lltype.free(cd, flavor='raw') + # + assert res == math.sin(1.23) From noreply at buildbot.pypy.org Sat Jan 5 17:20:33 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 5 Jan 2013 17:20:33 +0100 (CET) Subject: [pypy-commit] pypy remove-globals-in-jit: Fix Message-ID: <20130105162033.900141C01A6@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: remove-globals-in-jit Changeset: r59731:8c85766d55f6 Date: 2013-01-05 17:20 +0100 http://bitbucket.org/pypy/pypy/changeset/8c85766d55f6/ Log: Fix diff --git a/pypy/jit/codewriter/test/test_call.py b/pypy/jit/codewriter/test/test_call.py --- a/pypy/jit/codewriter/test/test_call.py +++ b/pypy/jit/codewriter/test/test_call.py @@ -195,7 +195,7 @@ assert call_descr.extrainfo.is_call_release_gil() is False def test_call_release_gil(): - from pypy.jit.backend.llgraph.runner import LLtypeCPU + from pypy.jit.backend.llgraph.runner import LLGraphCPU T = rffi.CArrayPtr(rffi.TIME_T) external = rffi.llexternal("time", [T], rffi.TIME_T, threadsafe=True) @@ -206,7 +206,7 @@ rtyper = support.annotate(f, []) jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0]) - cc = CallControl(LLtypeCPU(rtyper), jitdrivers_sd=[jitdriver_sd]) + cc = CallControl(LLGraphCPU(rtyper), jitdrivers_sd=[jitdriver_sd]) res = cc.find_all_graphs(FakePolicy()) [llext_graph] = [x for x in res if x.func is external] From noreply at buildbot.pypy.org Sat Jan 5 17:21:21 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sat, 5 Jan 2013 17:21:21 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed path of pypy-c in test_package Message-ID: <20130105162121.375491C01A6@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59732:4f78e016a782 Date: 2013-01-05 17:15 +0100 http://bitbucket.org/pypy/pypy/changeset/4f78e016a782/ Log: Fixed path of pypy-c in test_package diff --git a/pypy/tool/release/test/test_package.py b/pypy/tool/release/test/test_package.py --- a/pypy/tool/release/test/test_package.py +++ b/pypy/tool/release/test/test_package.py @@ -15,7 +15,7 @@ basename = 'pypy-c' rename_pypy_c = 'pypy' exe_name_in_archive = 'bin/pypy' - pypy_c = py.path.local(pypydir).join('translator', 'goal', basename) + pypy_c = py.path.local(pypydir).join('..', basename) if not pypy_c.check(): os.system("echo faked_pypy_c> %s" % (pypy_c,)) pypy_c.chmod(0755) From noreply at buildbot.pypy.org Sat Jan 5 17:21:22 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sat, 5 Jan 2013 17:21:22 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed path of pypy-c in package.py Message-ID: <20130105162122.695151C01A6@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59733:7f1c3c39723f Date: 2013-01-05 17:17 +0100 http://bitbucket.org/pypy/pypy/changeset/7f1c3c39723f/ Log: Fixed path of pypy-c in package.py diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -48,7 +48,7 @@ basename = 'pypy-c' if sys.platform == 'win32': basename += '.exe' - pypy_c = basedir.join('pypy', 'translator', 'goal', basename) + pypy_c = basedir.join(basename) else: pypy_c = py.path.local(override_pypy_c) if not pypy_c.check(): From noreply at buildbot.pypy.org Sat Jan 5 17:21:23 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sat, 5 Jan 2013 17:21:23 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed pypy_dir in test_getdocstrings Message-ID: <20130105162123.870171C01A6@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59734:1e9c46f91f7e Date: 2013-01-05 17:21 +0100 http://bitbucket.org/pypy/pypy/changeset/1e9c46f91f7e/ Log: Fixed pypy_dir in test_getdocstrings diff --git a/pypy/tool/test/test_getdocstrings.py b/pypy/tool/test/test_getdocstrings.py --- a/pypy/tool/test/test_getdocstrings.py +++ b/pypy/tool/test/test_getdocstrings.py @@ -2,7 +2,7 @@ import glob, os.path, py, re this_dir = os.path.realpath(os.path.dirname(__file__)) -pypy_dir = os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..')) +pypy_dir = os.path.realpath(os.path.join(os.path.dirname(__file__), '..')) from pypy.tool.getdocstrings import quote, triplequotes from pypy.tool.getdocstrings import mk_std_filelist From noreply at buildbot.pypy.org Sat Jan 5 17:23:15 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 5 Jan 2013 17:23:15 +0100 (CET) Subject: [pypy-commit] pypy remove-globals-in-jit: fix Message-ID: <20130105162315.A79741C01A6@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: remove-globals-in-jit Changeset: r59735:b67d69cc8c0d Date: 2013-01-05 18:22 +0200 http://bitbucket.org/pypy/pypy/changeset/b67d69cc8c0d/ Log: fix diff --git a/pypy/jit/tool/test/test_jitoutput.py b/pypy/jit/tool/test/test_jitoutput.py --- a/pypy/jit/tool/test/test_jitoutput.py +++ b/pypy/jit/tool/test/test_jitoutput.py @@ -21,7 +21,7 @@ cap = py.io.StdCaptureFD() try: - ll_meta_interp(f, [10], CPUClass=runner.LLtypeCPU, type_system='lltype', + ll_meta_interp(f, [10], CPUClass=runner.LLGraphCPU, type_system='lltype', ProfilerClass=Profiler) finally: out, err = cap.reset() From noreply at buildbot.pypy.org Sat Jan 5 17:30:05 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 5 Jan 2013 17:30:05 +0100 (CET) Subject: [pypy-commit] pypy remove-globals-in-jit: Restore this fact: PropagateExceptionDescr() should only be Message-ID: <20130105163005.91DB71C01A6@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: remove-globals-in-jit Changeset: r59736:b4a06c2187c9 Date: 2013-01-05 17:29 +0100 http://bitbucket.org/pypy/pypy/changeset/b4a06c2187c9/ Log: Restore this fact: PropagateExceptionDescr() should only be instantiated once. It's now cached in the JitDriverSD's instead of globally. diff --git a/pypy/jit/metainterp/compile.py b/pypy/jit/metainterp/compile.py --- a/pypy/jit/metainterp/compile.py +++ b/pypy/jit/metainterp/compile.py @@ -895,7 +895,7 @@ finishargs = [] # jd = jitdriver_sd - faildescr = PropagateExceptionDescr() + faildescr = jitdriver_sd.propagate_exc_descr operations = [ ResOperation(rop.CALL, callargs, result, descr=jd.portal_calldescr), ResOperation(rop.GUARD_NO_EXCEPTION, [], None, descr=faildescr), diff --git a/pypy/jit/metainterp/jitdriver.py b/pypy/jit/metainterp/jitdriver.py --- a/pypy/jit/metainterp/jitdriver.py +++ b/pypy/jit/metainterp/jitdriver.py @@ -19,6 +19,7 @@ # self.handle_jitexc_from_bh pypy.jit.metainterp.warmspot # self.no_loop_header ... pypy.jit.metainterp.warmspot # self.portal_finishtoken... pypy.jit.metainterp.pyjitpl + # self.propagate_exc_descr.. pypy.jit.metainterp.pyjitpl # self.index ... pypy.jit.codewriter.call # self.mainjitcode ... pypy.jit.codewriter.call diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py --- a/pypy/jit/metainterp/pyjitpl.py +++ b/pypy/jit/metainterp/pyjitpl.py @@ -1463,6 +1463,7 @@ # # store this information for fastpath of call_assembler # (only the paths that can actually be taken) + exc_descr = compile.PropagateExceptionDescr() for jd in self.jitdrivers_sd: name = {history.INT: 'int', history.REF: 'ref', @@ -1472,8 +1473,8 @@ jd.portal_finishtoken = tokens[0].finishdescr num = self.cpu.get_fail_descr_number(tokens[0].finishdescr) setattr(self.cpu, 'done_with_this_frame_%s_v' % name, num) + jd.propagate_exc_descr = exc_descr # - exc_descr = compile.PropagateExceptionDescr() num = self.cpu.get_fail_descr_number(exc_descr) self.cpu.propagate_exception_v = num # diff --git a/pypy/jit/metainterp/test/test_compile.py b/pypy/jit/metainterp/test/test_compile.py --- a/pypy/jit/metainterp/test/test_compile.py +++ b/pypy/jit/metainterp/test/test_compile.py @@ -166,6 +166,7 @@ portal_runner_adr = llmemory.cast_ptr_to_adr(portal_runner_ptr) portal_calldescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, None) portal_finishtoken = compile.DoneWithThisFrameDescrInt() + propagate_exc_descr = compile.PropagateExceptionDescr() num_red_args = 2 result_type = INT # From noreply at buildbot.pypy.org Sat Jan 5 17:35:06 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 5 Jan 2013 17:35:06 +0100 (CET) Subject: [pypy-commit] pypy default: Document a minor branch merge Message-ID: <20130105163506.B91111C01A6@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r59737:72067c5281f4 Date: 2013-01-05 17:34 +0100 http://bitbucket.org/pypy/pypy/changeset/72067c5281f4/ Log: Document a minor branch merge diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -28,5 +28,7 @@ .. branch: remove-PYPY_NOT_MAIN_FILE +.. branch: missing-jit-operations + .. branch: fix-lookinside-iff-oopspec Fixed the interaction between two internal tools for controlling the JIT. From noreply at buildbot.pypy.org Sat Jan 5 18:10:02 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 5 Jan 2013 18:10:02 +0100 (CET) Subject: [pypy-commit] pypy remove-globals-in-jit: Phew. Fix: gc_set_extra_threshold occasionally causes a collection. Message-ID: <20130105171002.2F86A1C01A6@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: remove-globals-in-jit Changeset: r59738:d1878a4133c1 Date: 2013-01-05 18:09 +0100 http://bitbucket.org/pypy/pypy/changeset/d1878a4133c1/ Log: Phew. Fix: gc_set_extra_threshold occasionally causes a collection. diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py --- a/pypy/rpython/lltypesystem/lloperation.py +++ b/pypy/rpython/lltypesystem/lloperation.py @@ -499,7 +499,7 @@ 'gc_typeids_z' : LLOp(), 'gc_gcflag_extra' : LLOp(), 'gc_add_memory_pressure': LLOp(), - 'gc_set_extra_threshold': LLOp(canrun=True), + 'gc_set_extra_threshold': LLOp(canrun=True, canmallocgc=True), # ------- JIT & GC interaction, only for some GCs ---------- From noreply at buildbot.pypy.org Sat Jan 5 18:14:28 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 5 Jan 2013 18:14:28 +0100 (CET) Subject: [pypy-commit] pypy remove-globals-in-jit: A minimal test for the previous checkin. That's the effect that we Message-ID: <20130105171428.C2BC71C01A6@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: remove-globals-in-jit Changeset: r59739:89cbe267edc1 Date: 2013-01-05 18:14 +0100 http://bitbucket.org/pypy/pypy/changeset/89cbe267edc1/ Log: A minimal test for the previous checkin. That's the effect that we really want, but the test is really a unit test only. diff --git a/pypy/rpython/memory/gctransform/test/test_framework.py b/pypy/rpython/memory/gctransform/test/test_framework.py --- a/pypy/rpython/memory/gctransform/test/test_framework.py +++ b/pypy/rpython/memory/gctransform/test/test_framework.py @@ -2,6 +2,7 @@ from pypy.annotation.model import SomeInteger from pypy.objspace.flow.model import Constant, SpaceOperation from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.memory.gc.semispace import SemiSpaceGC from pypy.rpython.memory.gctransform.framework import (CollectAnalyzer, find_initializing_stores, find_clean_setarrayitems) @@ -96,6 +97,13 @@ gg = graphof(t, g) assert CollectAnalyzer(t).analyze_direct_call(gg) +def test_cancollect_some_operation(): + def g(): + llop.gc_set_extra_threshold(lltype.Void, 32) + t = rtype(g, []) + gg = graphof(t, g) + assert CollectAnalyzer(t).analyze_direct_call(gg) + def test_no_collect(): from pypy.rlib import rgc from pypy.translator.c.genc import CStandaloneBuilder From noreply at buildbot.pypy.org Sat Jan 5 18:22:29 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 5 Jan 2013 18:22:29 +0100 (CET) Subject: [pypy-commit] pypy remove-globals-in-jit: LLtypeCPU => LLGraphCPU Message-ID: <20130105172229.B9DC71C01A6@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: remove-globals-in-jit Changeset: r59740:ea28111fbb8a Date: 2013-01-05 18:22 +0100 http://bitbucket.org/pypy/pypy/changeset/ea28111fbb8a/ Log: LLtypeCPU => LLGraphCPU diff --git a/pypy/jit/tl/jittest.py b/pypy/jit/tl/jittest.py --- a/pypy/jit/tl/jittest.py +++ b/pypy/jit/tl/jittest.py @@ -26,8 +26,8 @@ get_policy = driver.extra['jitpolicy'] jitpolicy = get_policy(driver) - from pypy.jit.backend.llgraph.runner import LLtypeCPU - apply_jit(jitpolicy, interp, graph, LLtypeCPU) + from pypy.jit.backend.llgraph.runner import LLGraphCPU + apply_jit(jitpolicy, interp, graph, LLGraphCPU) def apply_jit(policy, interp, graph, CPUClass): diff --git a/pypy/jit/tl/pypyjit_child.py b/pypy/jit/tl/pypyjit_child.py --- a/pypy/jit/tl/pypyjit_child.py +++ b/pypy/jit/tl/pypyjit_child.py @@ -14,9 +14,9 @@ return lltype.nullptr(T) interp.heap.malloc_nonmovable = returns_null # XXX - from pypy.jit.backend.llgraph.runner import LLtypeCPU - #LLtypeCPU.supports_floats = False # for now - apply_jit(interp, graph, LLtypeCPU) + from pypy.jit.backend.llgraph.runner import LLGraphCPU + #LLGraphCPU.supports_floats = False # for now + apply_jit(interp, graph, LLGraphCPU) def run_child_ootype(glob, loc): diff --git a/pypy/jit/tl/spli/test/test_jit.py b/pypy/jit/tl/spli/test/test_jit.py --- a/pypy/jit/tl/spli/test/test_jit.py +++ b/pypy/jit/tl/spli/test/test_jit.py @@ -8,7 +8,7 @@ class TestSPLIJit(JitMixin): type_system = 'lltype' - CPUClass = runner.LLtypeCPU + CPUClass = runner.LLGraphCPU def interpret(self, f, args): coderepr = serializer.serialize(f.func_code) From noreply at buildbot.pypy.org Sat Jan 5 18:24:12 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 5 Jan 2013 18:24:12 +0100 (CET) Subject: [pypy-commit] pypy default: Oups, forgot to remove the test file of this too. Message-ID: <20130105172412.E96A61C01A6@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r59741:126c5c9e832a Date: 2013-01-05 18:24 +0100 http://bitbucket.org/pypy/pypy/changeset/126c5c9e832a/ Log: Oups, forgot to remove the test file of this too. diff --git a/pypy/tool/release/test/test_make_release.py b/pypy/tool/release/test/test_make_release.py deleted file mode 100644 --- a/pypy/tool/release/test/test_make_release.py +++ /dev/null @@ -1,11 +0,0 @@ - -import py -from pypy.tool.release.make_release import browse_nightly - -XML = py.path.local(__file__).join('..', 'nightly.xml').read() - -def test_browse_nightly(): - res = browse_nightly('branch', override_xml=XML) - assert res[('jit', 'linux')] == (40170, 'pypy-c-jit-40170-c407c9dc5382-linux.tar.bz2') - assert len(res) == 6 - assert res[('nojit', 'linux64')] == (40170, 'pypy-c-nojit-40170-c407c9dc5382-linux64.tar.bz2') From noreply at buildbot.pypy.org Sat Jan 5 18:52:16 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 5 Jan 2013 18:52:16 +0100 (CET) Subject: [pypy-commit] pypy remove-globals-in-jit: close to be merged branch Message-ID: <20130105175216.22B841C01EE@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: remove-globals-in-jit Changeset: r59742:5c579b93173b Date: 2013-01-05 19:44 +0200 http://bitbucket.org/pypy/pypy/changeset/5c579b93173b/ Log: close to be merged branch From noreply at buildbot.pypy.org Sat Jan 5 18:52:18 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 5 Jan 2013 18:52:18 +0100 (CET) Subject: [pypy-commit] pypy default: (arigo, bivab, fijal) merge remove-globals-in-jit Message-ID: <20130105175218.D42DF1C01EE@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r59743:712289be800d Date: 2013-01-05 19:46 +0200 http://bitbucket.org/pypy/pypy/changeset/712289be800d/ Log: (arigo, bivab, fijal) merge remove-globals-in-jit This branch removes global state in the JIT, by introducing deadframe, which is then used to fish values. It also includes a set of refactorings, like the llgraph backend simplification. diff too long, truncating to 2000 out of 9528 lines diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py --- a/pypy/jit/backend/arm/assembler.py +++ b/pypy/jit/backend/arm/assembler.py @@ -1,5 +1,6 @@ from __future__ import with_statement import os +from pypy.jit.backend.llsupport import jitframe from pypy.jit.backend.arm.helper.assembler import saved_registers from pypy.jit.backend.arm import conditions as c from pypy.jit.backend.arm import registers as r @@ -25,12 +26,12 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.jit.backend.arm.opassembler import ResOpAssembler from pypy.rlib.debug import (debug_print, debug_start, debug_stop, - have_debug_prints) + have_debug_prints, fatalerror) from pypy.rlib.jit import AsmInfo from pypy.rlib.objectmodel import compute_unique_id # XXX Move to llsupport -from pypy.jit.backend.x86.support import values_array, memcpy_fn +from pypy.jit.backend.x86.support import memcpy_fn DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', ('i', lltype.Signed), ('type', lltype.Char), # 'b'ridge, 'l'abel or @@ -44,14 +45,8 @@ debug = True - def __init__(self, cpu, failargs_limit=1000): + def __init__(self, cpu, translate_support_code=False): self.cpu = cpu - self.fail_boxes_int = values_array(lltype.Signed, failargs_limit) - self.fail_boxes_float = values_array(longlong.FLOATSTORAGE, - failargs_limit) - self.fail_boxes_ptr = values_array(llmemory.GCREF, failargs_limit) - self.fail_boxes_count = 0 - self.fail_force_index = 0 self.setup_failure_recovery() self.mc = None self.memcpy_addr = 0 @@ -68,6 +63,7 @@ self._debug = False self.loop_run_counters = [] self.debug_counter_descr = cpu.fielddescrof(DEBUG_COUNTER, 'i') + self.force_token_to_dead_frame = {} # XXX temporary hack def set_debug(self, v): r = self._debug @@ -112,9 +108,13 @@ gc_ll_descr.initialize() self._build_wb_slowpath(False) self._build_wb_slowpath(True) + self._build_failure_recovery(exc=True, withfloats=False) + self._build_failure_recovery(exc=False, withfloats=False) if self.cpu.supports_floats: self._build_wb_slowpath(False, withfloats=True) self._build_wb_slowpath(True, withfloats=True) + self._build_failure_recovery(exc=True, withfloats=True) + self._build_failure_recovery(exc=False, withfloats=True) self._build_propagate_exception_path() if gc_ll_descr.get_malloc_slowpath_addr is not None: self._build_malloc_slowpath() @@ -122,10 +122,7 @@ if gc_ll_descr.gcrootmap and gc_ll_descr.gcrootmap.is_shadow_stack: self._build_release_gil(gc_ll_descr.gcrootmap) self.memcpy_addr = self.cpu.cast_ptr_to_int(memcpy_fn) - self._exit_code_addr = self._gen_exit_path() - self._leave_jitted_hook_save_exc = \ - self._gen_leave_jitted_hook_code(True) - self._leave_jitted_hook = self._gen_leave_jitted_hook_code(False) + if not self._debug: # if self._debug is already set it means that someone called # set_debug by hand before initializing the assembler. Leave it @@ -220,36 +217,17 @@ self.releasegil_addr = rffi.cast(lltype.Signed, releasegil_func) self.reacqgil_addr = rffi.cast(lltype.Signed, reacqgil_func) - def _gen_leave_jitted_hook_code(self, save_exc): - mc = ARMv7Builder() - if self.cpu.supports_floats: - floats = r.caller_vfp_resp - else: - floats = [] - with saved_registers(mc, r.caller_resp + [r.lr], floats): - addr = self.cpu.get_on_leave_jitted_int(save_exception=save_exc) - mc.BL(addr) - assert self._exit_code_addr != 0 - mc.B(self._exit_code_addr) - return mc.materialize(self.cpu.asmmemmgr, [], - self.cpu.gc_ll_descr.gcrootmap) - def _build_propagate_exception_path(self): if self.cpu.propagate_exception_v < 0: return # not supported (for tests, or non-translated) # mc = ARMv7Builder() - # call on_leave_jitted_save_exc() - if self.cpu.supports_floats: - floats = r.caller_vfp_resp - else: - floats = [] - with saved_registers(mc, r.caller_resp + [r.lr], floats): - addr = self.cpu.get_on_leave_jitted_int(save_exception=True, - default_to_memoryerror=True) - mc.BL(addr) - mc.gen_load_int(r.ip.value, self.cpu.propagate_exception_v) - mc.MOV_rr(r.r0.value, r.ip.value) + # + # Call the helper, which will return a dead frame object with + # the correct exception set, or MemoryError by default + # XXX make sure we return the correct value here + addr = rffi.cast(lltype.Signed, self.cpu.get_propagate_exception()) + mc.BL(addr) self.gen_func_epilog(mc=mc) self.propagate_exception_path = mc.materialize(self.cpu.asmmemmgr, []) @@ -282,12 +260,12 @@ # restore registers and return # We check for c.EQ here, meaning all bits zero in this case mc.POP([reg.value for reg in r.argument_regs] + [r.pc.value], cond=c.EQ) - # call on_leave_jitted_save_exc() - addr = self.cpu.get_on_leave_jitted_int(save_exception=True) + # + # Call the helper, which will return a dead frame object with + # the correct exception set, or MemoryError by default + addr = rffi.cast(lltype.Signed, self.cpu.get_propagate_exception()) mc.BL(addr) # - mc.gen_load_int(r.r0.value, self.cpu.propagate_exception_v) - # # footer -- note the ADD, which skips the return address of this # function, and will instead return to the caller's caller. Note # also that we completely ignore the saved arguments, because we @@ -339,39 +317,82 @@ def setup_failure_recovery(self): - @rgc.no_collect + #@rgc.no_collect -- XXX still true, but hacked gc_set_extra_threshold def failure_recovery_func(mem_loc, frame_pointer, stack_pointer): """mem_loc is a structure in memory describing where the values for the failargs are stored. frame loc is the address of the frame pointer for the frame to be decoded frame """ - vfp_registers = rffi.cast(rffi.LONGLONGP, stack_pointer) - registers = rffi.ptradd(vfp_registers, len(r.all_vfp_regs)) + vfp_registers = rffi.cast(rffi.LONGP, stack_pointer) + registers = rffi.ptradd(vfp_registers, 2*len(r.all_vfp_regs)) registers = rffi.cast(rffi.LONGP, registers) - return self.decode_registers_and_descr(mem_loc, frame_pointer, + bytecode = rffi.cast(rffi.UCHARP, mem_loc) + return self.grab_frame_values(self.cpu, bytecode, frame_pointer, registers, vfp_registers) + self.failure_recovery_code = [0, 0, 0, 0] self.failure_recovery_func = failure_recovery_func - recovery_func_sign = lltype.Ptr(lltype.FuncType([lltype.Signed] * 3, - lltype.Signed)) + _FAILURE_RECOVERY_FUNC = lltype.Ptr(lltype.FuncType([rffi.LONGP] * 3, + llmemory.GCREF)) - @rgc.no_collect - def decode_registers_and_descr(self, mem_loc, frame_pointer, + @staticmethod + #@rgc.no_collect -- XXX still true, but hacked gc_set_extra_threshold + def grab_frame_values(cpu, bytecode, frame_pointer, registers, vfp_registers): - """Decode locations encoded in memory at mem_loc and write the values - to the failboxes. Values for spilled vars and registers are stored on - stack at frame_loc """ - assert frame_pointer & 1 == 0 - self.fail_force_index = frame_pointer - bytecode = rffi.cast(rffi.UCHARP, mem_loc) + # no malloc allowed here!! xxx apart from one, hacking a lot + force_index = rffi.cast(lltype.Signed, frame_pointer) num = 0 - value = 0 - fvalue = 0 + deadframe = lltype.nullptr(jitframe.DEADFRAME) + # step 1: lots of mess just to count the final value of 'num' + bytecode1 = bytecode + while 1: + code = rffi.cast(lltype.Signed, bytecode1[0]) + bytecode1 = rffi.ptradd(bytecode1, 1) + if code >= AssemblerARM.CODE_FROMSTACK: + while code > 0x7F: + code = rffi.cast(lltype.Signed, bytecode1[0]) + bytecode1 = rffi.ptradd(bytecode1, 1) + else: + kind = code & 3 + if kind == AssemblerARM.DESCR_SPECIAL: + if code == AssemblerARM.CODE_HOLE: + num += 1 + continue + if code == AssemblerARM.CODE_INPUTARG: + continue + if code == AssemblerARM.CODE_FORCED: + # resuming from a GUARD_NOT_FORCED + token = force_index + deadframe = ( + cpu.assembler.force_token_to_dead_frame.pop(token)) + deadframe = lltype.cast_opaque_ptr( + jitframe.DEADFRAMEPTR, deadframe) + continue + assert code == AssemblerARM.CODE_STOP + break + num += 1 + + # allocate the deadframe + if not deadframe: + # Remove the "reserve" at the end of the nursery. This means + # that it is guaranteed that the following malloc() works + # without requiring a collect(), but it needs to be re-added + # as soon as possible. + cpu.gc_clear_extra_threshold() + assert num <= cpu.get_failargs_limit() + try: + deadframe = lltype.malloc(jitframe.DEADFRAME, num) + except MemoryError: + fatalerror("memory usage error in grab_frame_values") + # fill it code_inputarg = False - while True: + num = 0 + value_hi = 0 + while 1: + # decode the next instruction from the bytecode code = rffi.cast(lltype.Signed, bytecode[0]) bytecode = rffi.ptradd(bytecode, 1) - if code >= self.CODE_FROMSTACK: + if code >= AssemblerARM.CODE_FROMSTACK: if code > 0x7F: shift = 7 code &= 0x7F @@ -384,54 +405,63 @@ break # load the value from the stack kind = code & 3 - code = int((code - self.CODE_FROMSTACK) >> 2) + code = (code - AssemblerARM.CODE_FROMSTACK) >> 2 if code_inputarg: code = ~code code_inputarg = False - if kind == self.DESCR_FLOAT: - # we use code + 1 to get the hi word of the double worded float - stackloc = frame_pointer - get_fp_offset(int(code) + 1) - assert stackloc & 3 == 0 - fvalue = rffi.cast(rffi.LONGLONGP, stackloc)[0] - else: - stackloc = frame_pointer - get_fp_offset(int(code)) - assert stackloc & 1 == 0 - value = rffi.cast(rffi.LONGP, stackloc)[0] + stackloc = force_index - get_fp_offset(int(code)) + value = rffi.cast(rffi.LONGP, stackloc)[0] + if kind == AssemblerARM.DESCR_FLOAT: + assert WORD == 4 + value_hi = value + value = rffi.cast(rffi.LONGP, stackloc - WORD)[0] else: - # 'code' identifies a register: load its value kind = code & 3 - if kind == self.DESCR_SPECIAL: - if code == self.CODE_HOLE: + if kind == AssemblerARM.DESCR_SPECIAL: + if code == AssemblerARM.CODE_HOLE: num += 1 continue - if code == self.CODE_INPUTARG: + if code == AssemblerARM.CODE_INPUTARG: code_inputarg = True continue - assert code == self.CODE_STOP + if code == AssemblerARM.CODE_FORCED: + continue + assert code == AssemblerARM.CODE_STOP break + # 'code' identifies a register: load its value code >>= 2 - if kind == self.DESCR_FLOAT: - fvalue = vfp_registers[code] + if kind == AssemblerARM.DESCR_FLOAT: + if WORD == 4: + value = vfp_registers[2*code] + value_hi = vfp_registers[2*code + 1] + else: + value = registers[code] else: value = registers[code] # store the loaded value into fail_boxes_ - if kind == self.DESCR_FLOAT: - tgt = self.fail_boxes_float.get_addr_for_num(num) - rffi.cast(rffi.LONGLONGP, tgt)[0] = fvalue + if kind == AssemblerARM.DESCR_INT: + deadframe.jf_values[num].int = value + elif kind == AssemblerARM.DESCR_REF: + deadframe.jf_values[num].ref = rffi.cast(llmemory.GCREF, value) + elif kind == AssemblerARM.DESCR_FLOAT: + assert WORD == 4 + assert not longlong.is_64_bit + floatvalue = rffi.cast(lltype.SignedLongLong, value_hi) + floatvalue <<= 32 + floatvalue |= rffi.cast(lltype.SignedLongLong, + rffi.cast(lltype.Unsigned, value)) + deadframe.jf_values[num].float = floatvalue else: - if kind == self.DESCR_INT: - tgt = self.fail_boxes_int.get_addr_for_num(num) - elif kind == self.DESCR_REF: - assert (value & 3) == 0, "misaligned pointer" - tgt = self.fail_boxes_ptr.get_addr_for_num(num) - else: - assert 0, "bogus kind" - rffi.cast(rffi.LONGP, tgt)[0] = value + assert 0, "bogus kind" num += 1 - self.fail_boxes_count = num + # + assert num == len(deadframe.jf_values) + if not we_are_translated(): + assert bytecode[4] == 0xCC fail_index = rffi.cast(rffi.INTP, bytecode)[0] - fail_index = rffi.cast(lltype.Signed, fail_index) - return fail_index + fail_descr = cpu.get_fail_descr_from_number(fail_index) + deadframe.jf_descr = fail_descr.hide(cpu) + return lltype.cast_opaque_ptr(llmemory.GCREF, deadframe) def decode_inputargs(self, code): descr_to_box_type = [REF, INT, FLOAT] @@ -514,12 +544,26 @@ self.mc.CMP_ri(r.r0.value, 0) self.mc.B(self.propagate_exception_path, c=c.EQ) - def _gen_exit_path(self): + def _build_failure_recovery(self, exc, withfloats=False): mc = ARMv7Builder() - decode_registers_addr = llhelper(self.recovery_func_sign, + failure_recovery = llhelper(self._FAILURE_RECOVERY_FUNC, self.failure_recovery_func) self._insert_checks(mc) - with saved_registers(mc, r.all_regs, r.all_vfp_regs): + if withfloats: + f = r.all_vfp_regs + else: + f = [] + with saved_registers(mc, r.all_regs, f): + if exc: + # We might have an exception pending. Load it into r4 + # (this is a register saved across calls) + mc.gen_load_int(r.r5.value, self.cpu.pos_exc_value()) + mc.LDR_ri(r.r4.value, self.cpu.pos_exc_value()) + # clear the exc flags + mc.gen_load_int(r.r6.value, 0) + mc.STR_ri(r.r6.value, r.r5.value) + mc.gen_load_int(r.r5.value, self.cpu.pos_exception()) + mc.STR_ri(r.r6.value, r.r5.value) # move mem block address, to r0 to pass as mc.MOV_rr(r.r0.value, r.lr.value) # pass the current frame pointer as second param @@ -527,23 +571,32 @@ # pass the current stack pointer as third param mc.MOV_rr(r.r2.value, r.sp.value) self._insert_checks(mc) - mc.BL(rffi.cast(lltype.Signed, decode_registers_addr)) + mc.BL(rffi.cast(lltype.Signed, failure_recovery)) + if exc: + # save ebx into 'jf_guard_exc' + from pypy.jit.backend.llsupport.descr import unpack_fielddescr + descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) + offset, size, _ = unpack_fielddescr(descrs.jf_guard_exc) + mc.STR_rr(r.r4.value, r.r0.value, offset, cond=c.AL) mc.MOV_rr(r.ip.value, r.r0.value) mc.MOV_rr(r.r0.value, r.ip.value) self.gen_func_epilog(mc=mc) - return mc.materialize(self.cpu.asmmemmgr, [], + rawstart = mc.materialize(self.cpu.asmmemmgr, [], self.cpu.gc_ll_descr.gcrootmap) + self.failure_recovery_code[exc + 2 * withfloats] = rawstart + self.mc = None DESCR_REF = 0x00 DESCR_INT = 0x01 DESCR_FLOAT = 0x02 DESCR_SPECIAL = 0x03 CODE_FROMSTACK = 64 - CODE_STOP = 0 | DESCR_SPECIAL - CODE_HOLE = 4 | DESCR_SPECIAL - CODE_INPUTARG = 8 | DESCR_SPECIAL + CODE_STOP = 0 | DESCR_SPECIAL + CODE_HOLE = 4 | DESCR_SPECIAL + CODE_INPUTARG = 8 | DESCR_SPECIAL + CODE_FORCED = 12 | DESCR_SPECIAL #XXX where should this be written? - def gen_descr_encoding(self, descr, failargs, locs): + def write_failure_recovery_description(self, descr, failargs, locs): assert self.mc is not None for i in range(len(failargs)): arg = failargs[i] @@ -575,26 +628,33 @@ self.mc.writechar(chr(n)) self.mc.writechar(chr(self.CODE_STOP)) - fdescr = self.cpu.get_fail_descr_number(descr) - self.mc.write32(fdescr) + + def generate_quick_failure(self, guardtok, fcond=c.AL): + assert isinstance(guardtok.save_exc, bool) + fail_index = self.cpu.get_fail_descr_number(guardtok.descr) + startpos = self.mc.currpos() + withfloats = False + for box in guardtok.failargs: + if box is not None and box.type == FLOAT: + withfloats = True + break + exc = guardtok.save_exc + target = self.failure_recovery_code[exc + 2 * withfloats] + assert target != 0 + self.mc.BL(target) + # write tight data that describes the failure recovery + if guardtok.is_guard_not_forced: + self.mc.writechar(chr(self.CODE_FORCED)) + self.write_failure_recovery_description(guardtok.descr, + guardtok.failargs, guardtok.faillocs[1:]) + self.mc.write32(fail_index) + # for testing the decoding, write a final byte 0xCC + if not we_are_translated(): + self.mc.writechar('\xCC') + faillocs = [loc for loc in guardtok.faillocs if loc is not None] + guardtok.descr._arm_debug_faillocs = faillocs self.align() - - # assert that the fail_boxes lists are big enough - assert len(failargs) <= self.fail_boxes_int.SIZE - - def _gen_path_to_exit_path(self, descr, args, arglocs, - save_exc, fcond=c.AL): - assert isinstance(save_exc, bool) - self.gen_exit_code(self.mc, save_exc, fcond) - self.gen_descr_encoding(descr, args, arglocs[1:]) - - def gen_exit_code(self, mc, save_exc, fcond=c.AL): - assert isinstance(save_exc, bool) - if save_exc: - path = self._leave_jitted_hook_save_exc - else: - path = self._leave_jitted_hook - mc.BL(path) + return startpos def align(self): while(self.mc.currpos() % FUNC_ALIGN != 0): @@ -841,15 +901,10 @@ def write_pending_failure_recoveries(self): for tok in self.pending_guards: - descr = tok.descr #generate the exit stub and the encoded representation - pos = self.mc.currpos() - tok.pos_recovery_stub = pos - - self._gen_path_to_exit_path(descr, tok.failargs, - tok.faillocs, save_exc=tok.save_exc) + tok.pos_recovery_stub = self.generate_quick_failure(tok) # store info on the descr - descr._arm_current_frame_depth = tok.faillocs[0].getint() + tok.descr._arm_current_frame_depth = tok.faillocs[0].getint() def process_pending_guards(self, block_start): clt = self.current_clt @@ -860,7 +915,7 @@ descr._arm_failure_recovery_block = failure_recovery_pos relative_offset = tok.pos_recovery_stub - tok.offset guard_pos = block_start + tok.offset - if not tok.is_invalidate: + if not tok.is_guard_not_invalidated: # patch the guard jumpt to the stub # overwrite the generate NOP with a B_offs to the pos of the # stub @@ -1255,11 +1310,6 @@ else: raise AssertionError('Trying to pop to an invalid location') - def leave_jitted_hook(self): - ptrs = self.fail_boxes_ptr.ar - llop.gc_assume_young_pointers(lltype.Void, - llmemory.cast_ptr_to_adr(ptrs)) - def malloc_cond(self, nursery_free_adr, nursery_top_adr, size): assert size & (WORD-1) == 0 # must be correctly aligned diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py --- a/pypy/jit/backend/arm/opassembler.py +++ b/pypy/jit/backend/arm/opassembler.py @@ -16,6 +16,7 @@ gen_emit_unary_float_op, saved_registers, count_reg_args) +from pypy.jit.backend.arm.helper.regalloc import check_imm_arg from pypy.jit.backend.arm.codebuilder import ARMv7Builder, OverwritingBuilder from pypy.jit.backend.arm.jump import remap_frame_layout from pypy.jit.backend.arm.regalloc import TempInt, TempPtr @@ -27,18 +28,20 @@ from pypy.jit.metainterp.history import JitCellToken, TargetToken from pypy.jit.metainterp.resoperation import rop from pypy.rlib.objectmodel import we_are_translated -from pypy.rpython.lltypesystem import rstr +from pypy.rlib import rgc +from pypy.rpython.lltypesystem import rstr, rffi, lltype, llmemory NO_FORCE_INDEX = -1 class GuardToken(object): - def __init__(self, descr, failargs, faillocs, offset, - save_exc, fcond=c.AL, is_invalidate=False): + def __init__(self, descr, failargs, faillocs, offset, save_exc, fcond=c.AL, + is_guard_not_invalidated=False, is_guard_not_forced=False): assert isinstance(save_exc, bool) self.descr = descr self.offset = offset - self.is_invalidate = is_invalidate + self.is_guard_not_invalidated = is_guard_not_invalidated + self.is_guard_not_forced = is_guard_not_forced self.failargs = failargs self.faillocs = faillocs self.save_exc = save_exc @@ -190,7 +193,8 @@ return fcond def _emit_guard(self, op, arglocs, fcond, save_exc, - is_guard_not_invalidated=False): + is_guard_not_invalidated=False, + is_guard_not_forced=False): assert isinstance(save_exc, bool) assert isinstance(fcond, int) descr = op.getdescr() @@ -210,7 +214,8 @@ faillocs=arglocs, offset=pos, save_exc=save_exc, - is_invalidate=is_guard_not_invalidated, + is_guard_not_invalidated=is_guard_not_invalidated, + is_guard_not_forced=is_guard_not_forced, fcond=fcond)) return c.AL @@ -312,49 +317,10 @@ return fcond def emit_op_finish(self, op, arglocs, regalloc, fcond): - for i in range(len(arglocs) - 1): - loc = arglocs[i] - box = op.getarg(i) - if loc is None: - continue - if loc.is_reg(): - if box.type == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(i) - elif box.type == INT: - adr = self.fail_boxes_int.get_addr_for_num(i) - else: - assert 0 - self.mc.gen_load_int(r.ip.value, adr) - self.mc.STR_ri(loc.value, r.ip.value) - elif loc.is_vfp_reg(): - assert box.type == FLOAT - adr = self.fail_boxes_float.get_addr_for_num(i) - self.mc.gen_load_int(r.ip.value, adr) - self.mc.VSTR(loc.value, r.ip.value) - elif loc.is_stack() or loc.is_imm() or loc.is_imm_float(): - if box.type == FLOAT: - adr = self.fail_boxes_float.get_addr_for_num(i) - self.mov_loc_loc(loc, r.vfp_ip) - self.mc.gen_load_int(r.ip.value, adr) - self.mc.VSTR(r.vfp_ip.value, r.ip.value) - elif box.type == REF or box.type == INT: - if box.type == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(i) - elif box.type == INT: - adr = self.fail_boxes_int.get_addr_for_num(i) - else: - assert 0 - self.mov_loc_loc(loc, r.ip) - self.mc.gen_load_int(r.lr.value, adr) - self.mc.STR_ri(r.ip.value, r.lr.value) - else: - assert 0 - # note: no exception should currently be set in llop.get_exception_addr - # even if this finish may be an exit_frame_with_exception (in this case - # the exception instance is in arglocs[0]). - addr = self.cpu.get_on_leave_jitted_int(save_exception=False) - self.mc.BL(addr) - self.mc.gen_load_int(r.r0.value, arglocs[-1].value) + [argloc] = arglocs + if argloc is not r.r0: #XXX verify this + self.mov_loc_loc(argloc, r.r0, fcond) + # exit function self.gen_func_epilog() return fcond @@ -1152,8 +1118,26 @@ value = self.cpu.done_with_this_frame_float_v else: raise AssertionError(kind) - self.mc.gen_load_int(r.ip.value, value) - self.mc.CMP_rr(tmploc.value, r.ip.value) + from pypy.jit.backend.llsupport.descr import unpack_fielddescr + from pypy.jit.backend.llsupport.descr import unpack_interiorfielddescr + descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) + _offset, _size, _ = unpack_fielddescr(descrs.jf_descr) + fail_descr = self.cpu.get_fail_descr_from_number(value) + value = fail_descr.hide(self.cpu) + rgc._make_sure_does_not_move(value) + value = rffi.cast(lltype.Signed, value) + + if check_imm_arg(_offset): + self.mc.LDR_ri(r.ip.value, tmploc.value, imm=_offset) + else: + self.mc.gen_load_int(r.ip.value, _offset) + self.mc.LDR_rr(r.ip.value, tmploc.value, r.ip.value) + if check_imm_arg(value): + self.mc.CMP_ri(r.ip.value, imm=value) + else: + self.mc.gen_load_int(r.lr.value, value) + self.mc.CMP_rr(r.lr.value, r.ip.value) + #if values are equal we take the fast path # Slow path, calling helper @@ -1180,19 +1164,31 @@ if op.result is not None: # load the return value from fail_boxes_xxx[0] kind = op.result.type - if kind == INT: - adr = self.fail_boxes_int.get_addr_for_num(0) - elif kind == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(0) - elif kind == FLOAT: - adr = self.fail_boxes_float.get_addr_for_num(0) + if kind == FLOAT: + t = unpack_interiorfielddescr(descrs.as_float)[0] + if not check_imm_arg(t): + self.mc.gen_load_int(r.ip.value, t, cond=fast_path_cond) + self.mc.ADD_rr(r.ip.value, r.r0.value, r.ip.value, + cond=fast_path_cond) + t = 0 + base = r.ip + else: + base = r.r0 + self.mc.VLDR(resloc.value, base.value, imm=t, + cond=fast_path_cond) else: - raise AssertionError(kind) - self.mc.gen_load_int(r.ip.value, adr, cond=fast_path_cond) - if op.result.type == FLOAT: - self.mc.VLDR(resloc.value, r.ip.value, cond=fast_path_cond) - else: - self.mc.LDR_ri(resloc.value, r.ip.value, cond=fast_path_cond) + assert resloc is r.r0 + if kind == INT: + t = unpack_interiorfielddescr(descrs.as_int)[0] + else: + t = unpack_interiorfielddescr(descrs.as_ref)[0] + if not check_imm_arg(t): + self.mc.gen_load_int(r.ip.value, t, cond=fast_path_cond) + self.mc.LDR_rr(resloc.value, resloc.value, r.ip.value, + cond=fast_path_cond) + else: + self.mc.LDR_ri(resloc.value, resloc.value, imm=t, + cond=fast_path_cond) # jump to merge point jmp_pos = self.mc.currpos() self.mc.BKPT() @@ -1266,7 +1262,8 @@ self.mc.LDR_ri(r.ip.value, r.fp.value) self.mc.CMP_ri(r.ip.value, 0) - self._emit_guard(guard_op, arglocs[1 + numargs:], c.GE, save_exc=True) + self._emit_guard(guard_op, arglocs[1 + numargs:], c.GE, + save_exc=True, is_guard_not_forced=True) return fcond def emit_guard_call_release_gil(self, op, guard_op, arglocs, regalloc, diff --git a/pypy/jit/backend/arm/regalloc.py b/pypy/jit/backend/arm/regalloc.py --- a/pypy/jit/backend/arm/regalloc.py +++ b/pypy/jit/backend/arm/regalloc.py @@ -644,15 +644,9 @@ return args def prepare_op_finish(self, op, fcond): - args = [None] * (op.numargs() + 1) - for i in range(op.numargs()): - arg = op.getarg(i) - if arg: - args[i] = self.loc(arg) - self.possibly_free_var(arg) - n = self.cpu.get_fail_descr_number(op.getdescr()) - args[-1] = imm(n) - return args + loc = self.loc(op.getarg(0)) + self.possibly_free_var(op.getarg(0)) + return [loc] def prepare_op_guard_true(self, op, fcond): l0 = self.make_sure_var_in_reg(op.getarg(0)) diff --git a/pypy/jit/backend/arm/runner.py b/pypy/jit/backend/arm/runner.py --- a/pypy/jit/backend/arm/runner.py +++ b/pypy/jit/backend/arm/runner.py @@ -23,15 +23,21 @@ AbstractLLCPU.__init__(self, rtyper, stats, opts, translate_support_code, gcdescr) + from pypy.jit.backend.llsupport import jitframe + self.deadframe_size_max = llmemory.sizeof(jitframe.DEADFRAME, + self.get_failargs_limit()) + def set_debug(self, flag): return self.assembler.set_debug(flag) + def get_failargs_limit(self): + if self.opts is not None: + return self.opts.failargs_limit + else: + return 1000 + def setup(self): - if self.opts is not None: - failargs_limit = self.opts.failargs_limit - else: - failargs_limit = 1000 - self.assembler = AssemblerARM(self, failargs_limit=failargs_limit) + self.assembler = AssemblerARM(self, self.translate_support_code) def setup_once(self): self.assembler.setup_once() @@ -51,24 +57,6 @@ return self.assembler.assemble_bridge(faildescr, inputargs, operations, original_loop_token, log=log) - def get_latest_value_float(self, index): - return self.assembler.fail_boxes_float.getitem(index) - - def get_latest_value_int(self, index): - return self.assembler.fail_boxes_int.getitem(index) - - def get_latest_value_ref(self, index): - return self.assembler.fail_boxes_ptr.getitem(index) - - def get_latest_value_count(self): - return self.assembler.fail_boxes_count - - def get_latest_force_token(self): - return self.assembler.fail_force_index - - def get_on_leave_jitted_hook(self): - return self.assembler.leave_jitted_hook - def clear_latest_values(self, count): setitem = self.assembler.fail_boxes_ptr.setitem null = lltype.nullptr(llmemory.GCREF.TO) @@ -76,7 +64,7 @@ setitem(index, null) def make_execute_token(self, *ARGS): - FUNCPTR = lltype.Ptr(lltype.FuncType(ARGS, lltype.Signed)) + FUNCPTR = lltype.Ptr(lltype.FuncType(ARGS, llmemory.GCREF)) def execute_token(executable_token, *args): clt = executable_token.compiled_loop_token @@ -91,12 +79,13 @@ prev_interpreter = LLInterpreter.current_interpreter LLInterpreter.current_interpreter = self.debug_ll_interpreter try: - fail_index = func(*args) + deadframe = func(*args) finally: if not self.translate_support_code: LLInterpreter.current_interpreter = prev_interpreter #llop.debug_print(lltype.Void, "<<<< Back") - return self.get_fail_descr_from_number(fail_index) + self.gc_set_extra_threshold() + return deadframe return execute_token def cast_ptr_to_int(x): @@ -115,17 +104,22 @@ assert fail_index >= 0, "already forced!" faildescr = self.get_fail_descr_from_number(fail_index) rffi.cast(TP, addr_of_force_index)[0] = ~fail_index - bytecode = self.assembler._find_failure_recovery_bytecode(faildescr) + frb = self.assembler._find_failure_recovery_bytecode(faildescr) + bytecode = rffi.cast(rffi.UCHARP, frb) addr_all_null_regsiters = rffi.cast(rffi.LONG, self.all_null_registers) - # start of "no gc operation!" block - fail_index_2 = self.assembler.failure_recovery_func( - bytecode, - addr_of_force_index, - addr_all_null_regsiters) - self.assembler.leave_jitted_hook() - # end of "no gc operation!" block - assert fail_index == fail_index_2 - return faildescr + # + assert (rffi.cast(lltype.Signed, bytecode[0]) == + self.assembler.CODE_FORCED) + bytecode = rffi.ptradd(bytecode, 1) + deadframe = self.assembler.grab_frame_values(self, + bytecode, addr_of_force_index, + self.all_null_registers, + self.all_null_registers) + # + assert self.get_latest_descr(deadframe) is faildescr + self.assembler.force_token_to_dead_frame[addr_of_force_index] = ( + deadframe) + return deadframe def redirect_call_assembler(self, oldlooptoken, newlooptoken): self.assembler.redirect_call_assembler(oldlooptoken, newlooptoken) diff --git a/pypy/jit/backend/arm/test/test_runner.py b/pypy/jit/backend/arm/test/test_runner.py --- a/pypy/jit/backend/arm/test/test_runner.py +++ b/pypy/jit/backend/arm/test/test_runner.py @@ -60,16 +60,18 @@ ] cpu.compile_loop(inp, operations, looptoken) args = [i for i in range(1, 15)] - self.cpu.execute_token(looptoken, *args) - output = [self.cpu.get_latest_value_int(i - 1) for i in range(1, 15)] + deadframe = self.cpu.execute_token(looptoken, *args) + output = [self.cpu.get_latest_value_int(deadframe, i - 1) for i in range(1, 15)] expected = [3, 7, 11, 15, 19, 23, 27, 3, 7, 11, 15, 19, 23, 27] assert output == expected - def test_redirect_call_assember2(self): - def assembler_helper(failindex, virtualizable): - return self.cpu.get_latest_value_int(0) + def test_redirect_call_assembler2(self): + def assembler_helper(deadframe, virtualizable): + x = self.cpu.get_latest_value_int(deadframe, 0) + assert x == 11 + return 7 - FUNCPTR = lltype.Ptr(lltype.FuncType([lltype.Signed, llmemory.GCREF], + FUNCPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF, llmemory.GCREF], lltype.Signed)) class FakeJitDriverSD: @@ -101,12 +103,12 @@ self.cpu.compile_loop(loop2.inputargs, loop2.operations, lt2) self.cpu.compile_loop(loop3.inputargs, loop3.operations, lt3) self.cpu.compile_loop(loop1.inputargs, loop1.operations, lt1) - self.cpu.execute_token(lt1, 11) - assert self.cpu.get_latest_value_int(0) == 12 + df = self.cpu.execute_token(lt1, 10) + assert self.cpu.get_latest_value_int(df, 0) == 7 self.cpu.redirect_call_assembler(lt2, lt3) - self.cpu.execute_token(lt1, 11) - assert self.cpu.get_latest_value_int(0) == 10 + df = self.cpu.execute_token(lt1, 12) + assert self.cpu.get_latest_value_int(df, 0) == 7 SFloat = lltype.GcForwardReference() SFloat.become(lltype.GcStruct('SFloat', ('parent', rclass.OBJECT), @@ -202,8 +204,8 @@ ARGS = [lltype.Signed] * numargs RES = lltype.Signed args = [i+1 for i in range(numargs)] - res = self.cpu.execute_token(looptoken, *args) - assert self.cpu.get_latest_value_int(0) == sum(args) + deadframe = self.cpu.execute_token(looptoken, *args) + assert self.cpu.get_latest_value_int(deadframe, 0) == sum(args) def test_debugger_on(self): from pypy.rlib import debug diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py deleted file mode 100644 --- a/pypy/jit/backend/llgraph/llimpl.py +++ /dev/null @@ -1,1968 +0,0 @@ -""" -The non-RPythonic part of the llgraph backend. -This contains all the code that is directly run -when executing on top of the llinterpreter. -""" - -import weakref -from pypy.objspace.flow.model import Variable, Constant -from pypy.annotation import model as annmodel -from pypy.jit.metainterp.history import REF, INT, FLOAT -from pypy.jit.metainterp import history -from pypy.jit.codewriter import heaptracker -from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr, rffi -from pypy.rpython.ootypesystem import ootype -from pypy.rpython.module.support import LLSupport, OOSupport -from pypy.rpython.llinterp import LLException -from pypy.rpython.extregistry import ExtRegistryEntry - -from pypy.jit.metainterp import resoperation -from pypy.jit.metainterp.resoperation import rop -from pypy.jit.backend.llgraph import symbolic -from pypy.jit.codewriter import longlong -from pypy.jit.codewriter.effectinfo import EffectInfo - -from pypy.rlib.objectmodel import ComputedIntSymbolic, we_are_translated -from pypy.rlib.rarithmetic import ovfcheck -from pypy.rlib.rarithmetic import r_longlong, r_ulonglong, r_uint -from pypy.rlib.rtimer import read_timestamp - -import py -from pypy.tool.ansi_print import ansi_log -log = py.log.Producer('runner') -py.log.setconsumer('runner', ansi_log) - -IS_32_BIT = r_ulonglong is not r_uint - - -def _from_opaque(opq): - return opq._obj.externalobj - -_TO_OPAQUE = {} - -def _to_opaque(value): - try: - return value._the_opaque_pointer - except AttributeError: - op = lltype.opaqueptr(_TO_OPAQUE[value.__class__], 'opaque', - externalobj=value) - value._the_opaque_pointer = op - return op - -def _normalize(value): - if isinstance(value, lltype._ptr): - value = lltype.top_container(value._obj) - return value - -def from_opaque_string(s): - if isinstance(s, str): - return s - elif isinstance(s, ootype._string): - return OOSupport.from_rstr(s) - else: - return LLSupport.from_rstr(s) - -FLOAT_ARRAY_TP = lltype.Ptr(lltype.Array(lltype.Float, hints={"nolength": True})) -def maybe_uncast(TP, array): - if array._TYPE.TO.OF != lltype.Float: - # array._TYPE.TO._hints.get("uncast_on_llgraph"): - array = rffi.cast(TP, array) - return array - -# a list of argtypes of all operations - couldn't find any and it's -# very useful. Note however that the table is half-broken here and -# there, in ways that are sometimes a bit hard to fix; that's why -# it is not "official". -TYPES = { - 'int_add' : (('int', 'int'), 'int'), - 'int_sub' : (('int', 'int'), 'int'), - 'int_mul' : (('int', 'int'), 'int'), - 'int_floordiv' : (('int', 'int'), 'int'), - 'int_mod' : (('int', 'int'), 'int'), - 'int_and' : (('int', 'int'), 'int'), - 'int_or' : (('int', 'int'), 'int'), - 'int_xor' : (('int', 'int'), 'int'), - 'int_lshift' : (('int', 'int'), 'int'), - 'int_rshift' : (('int', 'int'), 'int'), - 'int_lt' : (('int', 'int'), 'bool'), - 'int_gt' : (('int', 'int'), 'bool'), - 'int_ge' : (('int', 'int'), 'bool'), - 'int_le' : (('int', 'int'), 'bool'), - 'int_eq' : (('int', 'int'), 'bool'), - 'int_ne' : (('int', 'int'), 'bool'), - 'int_is_true' : (('int',), 'bool'), - 'int_is_zero' : (('int',), 'bool'), - 'int_neg' : (('int',), 'int'), - 'int_invert' : (('int',), 'int'), - 'int_add_ovf' : (('int', 'int'), 'int'), - 'int_sub_ovf' : (('int', 'int'), 'int'), - 'int_mul_ovf' : (('int', 'int'), 'int'), - 'int_force_ge_zero':(('int',), 'int'), - 'uint_add' : (('int', 'int'), 'int'), - 'uint_sub' : (('int', 'int'), 'int'), - 'uint_mul' : (('int', 'int'), 'int'), - 'uint_lt' : (('int', 'int'), 'bool'), - 'uint_le' : (('int', 'int'), 'bool'), - 'uint_eq' : (('int', 'int'), 'bool'), - 'uint_ne' : (('int', 'int'), 'bool'), - 'uint_gt' : (('int', 'int'), 'bool'), - 'uint_ge' : (('int', 'int'), 'bool'), - 'uint_xor' : (('int', 'int'), 'int'), - 'uint_rshift' : (('int', 'int'), 'int'), - 'uint_floordiv' : (('int', 'int'), 'int'), - 'float_add' : (('float', 'float'), 'float'), - 'float_sub' : (('float', 'float'), 'float'), - 'float_mul' : (('float', 'float'), 'float'), - 'float_truediv' : (('float', 'float'), 'float'), - 'float_lt' : (('float', 'float'), 'bool'), - 'float_le' : (('float', 'float'), 'bool'), - 'float_eq' : (('float', 'float'), 'bool'), - 'float_ne' : (('float', 'float'), 'bool'), - 'float_gt' : (('float', 'float'), 'bool'), - 'float_ge' : (('float', 'float'), 'bool'), - 'float_neg' : (('float',), 'float'), - 'float_abs' : (('float',), 'float'), - 'cast_float_to_int':(('float',), 'int'), - 'cast_int_to_float':(('int',), 'float'), - 'same_as' : (('int',), 'int'), # could also be ptr=>ptr - 'new_with_vtable' : (('ref',), 'ref'), - 'new' : ((), 'ref'), - 'new_array' : (('int',), 'ref'), - 'oois' : (('ref', 'ref'), 'bool'), - 'ooisnot' : (('ref', 'ref'), 'bool'), - 'instanceof' : (('ref',), 'bool'), - 'subclassof' : (('ref', 'ref'), 'bool'), - 'runtimenew' : (('ref',), 'ref'), - 'setfield_gc' : (('ref', 'intorptr'), None), - 'getfield_gc' : (('ref',), 'intorptr'), - 'getfield_gc_pure': (('ref',), 'intorptr'), - 'setfield_raw' : (('ref', 'intorptr'), None), - 'getfield_raw' : (('ref',), 'intorptr'), - 'getfield_raw_pure': (('ref',), 'intorptr'), - 'setarrayitem_gc' : (('ref', 'int', 'intorptr'), None), - 'getarrayitem_gc' : (('ref', 'int'), 'intorptr'), - 'getarrayitem_gc_pure' : (('ref', 'int'), 'intorptr'), - 'setarrayitem_raw' : (('ref', 'int', 'intorptr'), None), - 'getarrayitem_raw' : (('ref', 'int'), 'intorptr'), - 'getarrayitem_raw_pure' : (('ref', 'int'), 'intorptr'), - 'arraylen_gc' : (('ref',), 'int'), - 'call' : (('ref', 'varargs'), 'intorptr'), - 'call_assembler' : (('varargs',), 'intorptr'), - 'cond_call_gc_wb' : (('ptr', 'ptr'), None), - 'cond_call_gc_wb_array': (('ptr', 'int', 'ptr'), None), - 'oosend' : (('varargs',), 'intorptr'), - 'oosend_pure' : (('varargs',), 'intorptr'), - 'guard_true' : (('bool',), None), - 'guard_false' : (('bool',), None), - 'guard_value' : (('int', 'int'), None), - 'guard_class' : (('ref', 'ref'), None), - 'guard_no_exception' : ((), None), - 'guard_exception' : (('ref',), 'ref'), - 'guard_no_overflow' : ((), None), - 'guard_overflow' : ((), None), - 'guard_nonnull' : (('ref',), None), - 'guard_isnull' : (('ref',), None), - 'guard_nonnull_class' : (('ref', 'ref'), None), - 'newstr' : (('int',), 'ref'), - 'strlen' : (('ref',), 'int'), - 'strgetitem' : (('ref', 'int'), 'int'), - 'strsetitem' : (('ref', 'int', 'int'), None), - 'newunicode' : (('int',), 'ref'), - 'unicodelen' : (('ref',), 'int'), - 'unicodegetitem' : (('ref', 'int'), 'int'), - 'unicodesetitem' : (('ref', 'int', 'int'), 'int'), - 'cast_ptr_to_int' : (('ref',), 'int'), - 'cast_int_to_ptr' : (('int',), 'ref'), - 'debug_merge_point': (('ref', 'int', 'int'), None), - 'force_token' : ((), 'int'), - 'call_may_force' : (('int', 'varargs'), 'intorptr'), - 'guard_not_forced': ((), None), -} - -# ____________________________________________________________ - -class CompiledLoop(object): - has_been_freed = False - invalid = False - - def __init__(self): - self.inputargs = [] - self.operations = [] - - def getargtypes(self): - return [v.concretetype for v in self.inputargs] - - def __repr__(self): - lines = [] - self.as_text(lines, 1) - return 'CompiledLoop %s:\n%s' % (self.inputargs, '\n'.join(lines)) - - def as_text(self, lines, indent): - for op in self.operations: - lines.append('\t'*indent + repr(op)) - -class Operation(object): - result = None - descr = None - jump_target = None - fail_args = None - - def __init__(self, opnum): - self.opnum = opnum - self.args = [] - - def __repr__(self): - if self.result is not None: - sres = repr0(self.result) + ' = ' - else: - sres = '' - return '{%s%s(%s)}' % (sres, self.getopname(), - ', '.join(map(repr0, self.args))) - - def getopname(self): - try: - return resoperation.opname[self.opnum] - except KeyError: - return '<%d>' % self.opnum - - def is_guard(self): - return rop._GUARD_FIRST <= self.opnum <= rop._GUARD_LAST - - def is_final(self): - return rop._FINAL_FIRST <= self.opnum <= rop._FINAL_LAST - -def repr0(x): - if isinstance(x, list): - return '[' + ', '.join(repr0(y) for y in x) + ']' - elif isinstance(x, Constant): - return '(' + repr0(x.value) + ')' - elif isinstance(x, lltype._ptr): - x = llmemory.cast_ptr_to_adr(x) - if x.ptr: - try: - container = x.ptr._obj._normalizedcontainer() - return '* %s' % (container._TYPE._short_name(),) - except AttributeError: - return repr(x) - else: - return 'NULL' - else: - return repr(x) - -def repr_list(lst, types): - res_l = [] - if types and types[-1] == 'varargs': - types = types[:-1] + ('int',) * (len(lst) - len(types) + 1) - assert len(types) == len(lst) - for elem, tp in zip(lst, types): - if isinstance(elem, Constant): - res_l.append('(%s)' % repr1(elem, tp)) - else: - res_l.append(repr1(elem, tp)) - return '[%s]' % (', '.join(res_l)) - -def repr1(x, tp): - if tp == "intorptr": - TYPE = lltype.typeOf(x) - if isinstance(TYPE, lltype.Ptr) and TYPE.TO._gckind == 'gc': - tp = "ref" - else: - tp = "int" - if tp == 'int': - return str(x) - elif tp == 'void': - return '---' - elif tp == 'ref': - if not x: - return '(* None)' - if isinstance(x, int): - # XXX normalize? - ptr = str(llmemory.cast_int_to_adr(x)) - elif isinstance(ootype.typeOf(x), ootype.OOType): - return repr(x) - else: - if getattr(x, '_fake', None): - return repr(x) - if lltype.typeOf(x) == llmemory.GCREF: - TP = lltype.Ptr(lltype.typeOf(x._obj.container)) - ptr = lltype.cast_opaque_ptr(TP, x) - else: - ptr = x - try: - container = ptr._obj._normalizedcontainer() - return '(* %s)' % (container._TYPE._short_name(),) - except AttributeError: - return '(%r)' % (ptr,) - elif tp == 'bool': - assert x == 0 or x == 1 - return str(bool(x)) - #elif tp == 'fieldname': - # return str(symbolic.TokenToField[x...][1]) - elif tp == 'float': - return str(x) - else: - raise NotImplementedError("tp = %s" % tp) - -_variables = [] - -def compile_start(): - del _variables[:] - return _to_opaque(CompiledLoop()) - -def mark_as_free(loop): - loop = _from_opaque(loop) - assert not loop.has_been_freed - loop.has_been_freed = True - -def compile_start_int_var(loop): - return compile_start_ref_var(loop, lltype.Signed) - -def compile_start_float_var(loop): - return compile_start_ref_var(loop, longlong.FLOATSTORAGE) - -def compile_start_ref_var(loop, TYPE): - loop = _from_opaque(loop) - assert not loop.operations - v = Variable() - v.concretetype = TYPE - loop.inputargs.append(v) - r = len(_variables) - _variables.append(v) - return r - -def compile_started_vars(clt): - if not hasattr(clt, '_debug_argtypes'): # only when compiling the loop - argtypes = [v.concretetype for v in _variables] - try: - clt._debug_argtypes = argtypes - except AttributeError: # when 'clt' is actually a translated - pass # GcStruct - -def compile_add(loop, opnum): - loop = _from_opaque(loop) - loop.operations.append(Operation(opnum)) - -def compile_add_descr(loop, ofs, type, arg_types, extrainfo, width): - from pypy.jit.backend.llgraph.runner import Descr - loop = _from_opaque(loop) - op = loop.operations[-1] - assert isinstance(type, str) and len(type) == 1 - op.descr = Descr(ofs, type, arg_types=arg_types, extrainfo=extrainfo, width=width) - -def compile_add_descr_arg(loop, ofs, type, arg_types): - from pypy.jit.backend.llgraph.runner import Descr - loop = _from_opaque(loop) - op = loop.operations[-1] - assert isinstance(type, str) and len(type) == 1 - op.args.append(Descr(ofs, type, arg_types=arg_types)) - -def compile_add_loop_token(loop, descr): - if we_are_translated(): - raise ValueError("CALL_ASSEMBLER not supported") - loop = _from_opaque(loop) - op = loop.operations[-1] - op.descr = weakref.ref(descr) - -TARGET_TOKENS = weakref.WeakKeyDictionary() - -def compile_add_target_token(loop, descr, clt): - # here, 'clt' is the compiled_loop_token of the original loop that - # we are compiling - loop = _from_opaque(loop) - op = loop.operations[-1] - descrobj = _normalize(descr) - TARGET_TOKENS[descrobj] = loop, len(loop.operations), op.args, clt - -def compile_add_var(loop, intvar): - loop = _from_opaque(loop) - op = loop.operations[-1] - op.args.append(_variables[intvar]) - -def compile_add_int_const(loop, value): - compile_add_ref_const(loop, value, lltype.Signed) - -def compile_add_float_const(loop, value): - compile_add_ref_const(loop, value, longlong.FLOATSTORAGE) - -def compile_add_ref_const(loop, value, TYPE): - loop = _from_opaque(loop) - const = Constant(value) - const.concretetype = TYPE - op = loop.operations[-1] - op.args.append(const) - -def compile_add_int_result(loop): - return compile_add_ref_result(loop, lltype.Signed) - -def compile_add_float_result(loop): - return compile_add_ref_result(loop, longlong.FLOATSTORAGE) - -def compile_add_ref_result(loop, TYPE): - loop = _from_opaque(loop) - v = Variable() - v.concretetype = TYPE - op = loop.operations[-1] - op.result = v - r = len(_variables) - _variables.append(v) - return r - -def compile_add_jump_target(loop, targettoken, source_clt): - loop = _from_opaque(loop) - descrobj = _normalize(targettoken) - (loop_target, target_opindex, target_inputargs, target_clt - ) = TARGET_TOKENS[descrobj] - # - try: - assert source_clt._debug_argtypes == target_clt._debug_argtypes - except AttributeError: # when translated - pass - # - op = loop.operations[-1] - op.jump_target = loop_target - op.jump_target_opindex = target_opindex - op.jump_target_inputargs = target_inputargs - assert op.opnum == rop.JUMP - assert [v.concretetype for v in op.args] == ( - [v.concretetype for v in target_inputargs]) - # - if loop_target == loop: - log.info("compiling new loop") - else: - log.info("compiling new bridge") - -def compile_add_guard_jump_target(loop, loop_target): - loop = _from_opaque(loop) - loop_target = _from_opaque(loop_target) - op = loop.operations[-1] - assert op.is_guard() - op.jump_target = loop_target - -def compile_add_fail(loop, fail_index): - loop = _from_opaque(loop) - index = len(loop.operations)-1 - op = loop.operations[index] - op.fail_index = fail_index - return index - -def compile_add_fail_arg(loop, intvar): - loop = _from_opaque(loop) - op = loop.operations[-1] - if op.fail_args is None: - op.fail_args = [] - if intvar == -1: - op.fail_args.append(None) - else: - op.fail_args.append(_variables[intvar]) - -def compile_redirect_fail(old_loop, old_index, new_loop): - old_loop = _from_opaque(old_loop) - new_loop = _from_opaque(new_loop) - guard_op = old_loop.operations[old_index] - assert guard_op.is_guard() - guard_op.jump_target = new_loop - # check that the bridge's inputargs are of the correct number and - # kind for the guard - if guard_op.fail_args is not None: - argkinds = [v.concretetype for v in guard_op.fail_args if v] - else: - argkinds = [] - assert argkinds == [v.concretetype for v in new_loop.inputargs] - -# ------------------------------ - -class Frame(object): - OPHANDLERS = [None] * (rop._LAST+1) - - def __init__(self, cpu): - self.verbose = False - self.cpu = cpu - self.opindex = 1 - self._forced = False - self._may_force = -1 - - def getenv(self, v): - from pypy.jit.backend.llgraph.runner import Descr - if isinstance(v, Constant): - return v.value - elif isinstance(v, Descr): - return v - else: - return self.env[v] - - def _populate_fail_args(self, op, skip=None): - fail_args = [] - if op.fail_args: - for fail_arg in op.fail_args: - if fail_arg is None: - fail_args.append(None) - elif fail_arg is skip: - fail_args.append(fail_arg.concretetype._defl()) - else: - fail_args.append(self.getenv(fail_arg)) - self.fail_args = fail_args - self.fail_index = op.fail_index - - def execute(self): - """Execute all operations in a loop, - possibly following to other loops as well. - """ - global _last_exception - assert _last_exception is None, "exception left behind" - verbose = True - self.opindex = 0 - while True: - assert not self.loop.has_been_freed - op = self.loop.operations[self.opindex] - args = [self.getenv(v) for v in op.args] - if not op.is_final(): - try: - result = self.execute_operation(op.opnum, args, op.descr, - verbose) - except GuardFailed: - assert op.is_guard() - _stats.exec_conditional_jumps += 1 - if op.jump_target is not None: - # a patched guard, pointing to further code - if op.fail_args: - args = [self.getenv(v) for v in op.fail_args if v] - else: - args = [] - assert len(op.jump_target.inputargs) == len(args) - self.env = dict(zip(op.jump_target.inputargs, args)) - self.loop = op.jump_target - self.opindex = 0 - continue - else: - self._populate_fail_args(op) - # a non-patched guard - if self.verbose: - log.trace('failed: %s' % ( - ', '.join(map(str, fail_args)),)) - return op.fail_index - #verbose = self.verbose - assert (result is None) == (op.result is None) - if op.result is not None: - RESTYPE = op.result.concretetype - if RESTYPE is lltype.Signed: - x = self.as_int(result) - elif RESTYPE is llmemory.GCREF: - x = self.as_ptr(result) - elif RESTYPE is ootype.Object: - x = self.as_object(result) - elif RESTYPE is longlong.FLOATSTORAGE: - x = self.as_floatstorage(result) - else: - raise Exception("op.result.concretetype is %r" - % (RESTYPE,)) - self.env[op.result] = x - self.opindex += 1 - continue - if op.opnum == rop.JUMP: - inputargs = op.jump_target_inputargs - assert len(inputargs) == len(args) - self.env = dict(zip(inputargs, args)) - self.loop = op.jump_target - self.opindex = op.jump_target_opindex - _stats.exec_jumps += 1 - elif op.opnum == rop.FINISH: - if self.verbose: - log.trace('finished: %s' % ( - ', '.join(map(str, args)),)) - self.fail_args = args - return op.fail_index - - else: - assert 0, "unknown final operation %d" % (op.opnum,) - - def execute_operation(self, opnum, values, descr, verbose): - """Execute a single operation. - """ - ophandler = self.OPHANDLERS[opnum] - if ophandler is None: - self._define_impl(opnum) - ophandler = self.OPHANDLERS[opnum] - assert ophandler is not None, "missing impl for op %d" % opnum - opname = resoperation.opname[opnum].lower() - exec_counters = _stats.exec_counters - exec_counters[opname] = exec_counters.get(opname, 0) + 1 - for i in range(len(values)): - if isinstance(values[i], ComputedIntSymbolic): - values[i] = values[i].compute_fn() - res = NotImplemented - try: - res = ophandler(self, descr, *values) - finally: - if 0: # if verbose: - argtypes, restype = TYPES[opname] - if res is None: - resdata = '' - elif res is NotImplemented: - resdata = '*fail*' - else: - resdata = '-> ' + repr1(res, restype) - # fish the types - log.cpu('\t%s %s %s' % (opname, repr_list(values, argtypes), - resdata)) - return res - - def as_int(self, x): - return cast_to_int(x) - - def as_ptr(self, x): - return cast_to_ptr(x) - - def as_object(self, x): - return ootype.cast_to_object(x) - - def as_floatstorage(self, x): - return cast_to_floatstorage(x) - - def log_progress(self): - count = sum(_stats.exec_counters.values()) - count_jumps = _stats.exec_jumps - log.trace('ran %d operations, %d jumps' % (count, count_jumps)) - - # ---------- - - @classmethod - def _define_impl(cls, opnum): - opname = resoperation.opname[opnum] - try: - op = getattr(cls, 'op_' + opname.lower()) # op_guard_true etc. - except AttributeError: - try: - impl = globals()['do_' + opname.lower()] # do_arraylen_gc etc. - def op(self, descr, *args): - if descr is None: - return impl(*args) - else: - return impl(descr, *args) - except KeyError: - op = cls._make_impl_from_blackhole_interp(opname) - cls.OPHANDLERS[opnum] = op - - @classmethod - def _make_impl_from_blackhole_interp(cls, opname): - from pypy.jit.metainterp.blackhole import BlackholeInterpreter - name = 'bhimpl_' + opname.lower() - func = BlackholeInterpreter.__dict__[name] - for argtype in func.argtypes: - assert argtype in ('i', 'r', 'f') - # - def _op_default_implementation(self, descr, *args): - # for all operations implemented in the blackhole interpreter - return func(*args) - # - return _op_default_implementation - - def op_label(self, _, *args): - op = self.loop.operations[self.opindex] - assert op.opnum == rop.LABEL - assert len(op.args) == len(args) - newenv = {} - for v, value in zip(op.args, args): - newenv[v] = value - self.env = newenv - - def op_debug_merge_point(self, _, *args): - from pypy.jit.metainterp.warmspot import get_stats - try: - stats = get_stats() - except AttributeError: - pass - else: - stats.add_merge_point_location(args[1:]) - pass - - def op_guard_true(self, _, value): - if not value: - raise GuardFailed - - def op_guard_false(self, _, value): - if value: - raise GuardFailed - - op_guard_nonnull = op_guard_true - op_guard_isnull = op_guard_false - - def op_guard_class(self, _, value, expected_class): - value = lltype.cast_opaque_ptr(rclass.OBJECTPTR, value) - expected_class = llmemory.cast_adr_to_ptr( - llmemory.cast_int_to_adr(expected_class), - rclass.CLASSTYPE) - if value.typeptr != expected_class: - raise GuardFailed - - def op_guard_nonnull_class(self, _, value, expected_class): - if not value: - raise GuardFailed - self.op_guard_class(_, value, expected_class) - - def op_guard_value(self, _, value, expected_value): - if value != expected_value: - raise GuardFailed - - def op_guard_no_exception(self, _): - if _last_exception: - raise GuardFailed - - def _check_exception(self, expected_exception): - global _last_exception - expected_exception = self._cast_exception(expected_exception) - assert expected_exception - exc = _last_exception - if exc: - got = exc.args[0] - # exact match! - if got != expected_exception: - return False - return True - else: - return False - - def _cast_exception(self, exception): - return llmemory.cast_adr_to_ptr( - llmemory.cast_int_to_adr(exception), - rclass.CLASSTYPE) - - def _issubclass(self, cls1, cls2): - return rclass.ll_issubclass(cls1, cls2) - - def op_guard_exception(self, _, expected_exception): - global _last_exception - if not self._check_exception(expected_exception): - raise GuardFailed - res = _last_exception[1] - _last_exception = None - return res - - def op_guard_no_overflow(self, _): - flag = self.overflow_flag - del self.overflow_flag - if flag: - raise GuardFailed - - def op_guard_overflow(self, _): - flag = self.overflow_flag - del self.overflow_flag - if not flag: - raise GuardFailed - - def op_int_add_ovf(self, _, x, y): - try: - z = ovfcheck(x + y) - except OverflowError: - ovf = True - z = 0 - else: - ovf = False - self.overflow_flag = ovf - return z - - def op_int_sub_ovf(self, _, x, y): - try: - z = ovfcheck(x - y) - except OverflowError: - ovf = True - z = 0 - else: - ovf = False - self.overflow_flag = ovf - return z - - def op_int_mul_ovf(self, _, x, y): - try: - z = ovfcheck(x * y) - except OverflowError: - ovf = True - z = 0 - else: - ovf = False - self.overflow_flag = ovf - return z - - def op_keepalive(self, _, x): - pass - - # ---------- - # delegating to the builtins do_xxx() (done automatically for simple cases) - - def op_getarrayitem_gc(self, arraydescr, array, index): - if arraydescr.typeinfo == REF: - return do_getarrayitem_gc_ptr(array, index) - elif arraydescr.typeinfo == INT: - return do_getarrayitem_gc_int(array, index) - elif arraydescr.typeinfo == FLOAT: - return do_getarrayitem_gc_float(array, index) - else: - raise NotImplementedError - - op_getarrayitem_gc_pure = op_getarrayitem_gc - - def op_getarrayitem_raw(self, arraydescr, array, index): - if arraydescr.typeinfo == REF: - raise NotImplementedError("getarrayitem_raw -> gcref") - elif arraydescr.typeinfo == INT: - return do_getarrayitem_raw_int(array, index, arraydescr.ofs) - elif arraydescr.typeinfo == FLOAT: - return do_getarrayitem_raw_float(array, index) - else: - raise NotImplementedError - - op_getarrayitem_raw_pure = op_getarrayitem_raw - - def op_getfield_gc(self, fielddescr, struct): - if fielddescr.typeinfo == REF: - return do_getfield_gc_ptr(struct, fielddescr.ofs) - elif fielddescr.typeinfo == INT: - return do_getfield_gc_int(struct, fielddescr.ofs) - elif fielddescr.typeinfo == FLOAT: - return do_getfield_gc_float(struct, fielddescr.ofs) - else: - raise NotImplementedError - - op_getfield_gc_pure = op_getfield_gc - - def op_getfield_raw(self, fielddescr, struct): - if fielddescr.typeinfo == REF: - return do_getfield_raw_ptr(struct, fielddescr.ofs) - elif fielddescr.typeinfo == INT: - return do_getfield_raw_int(struct, fielddescr.ofs) - elif fielddescr.typeinfo == FLOAT: - return do_getfield_raw_float(struct, fielddescr.ofs) - else: - raise NotImplementedError - - op_getfield_raw_pure = op_getfield_raw - - def op_raw_store(self, arraydescr, addr, offset, value): - if arraydescr.typeinfo == REF: - raise AssertionError("cannot store GC pointer in raw storage") - elif arraydescr.typeinfo == INT: - do_raw_store_int(addr, offset, arraydescr.ofs, value) - elif arraydescr.typeinfo == FLOAT: - do_raw_store_float(addr, offset, value) - else: - raise NotImplementedError - - def op_raw_load(self, arraydescr, addr, offset): - if arraydescr.typeinfo == REF: - raise AssertionError("cannot store GC pointer in raw storage") - elif arraydescr.typeinfo == INT: - return do_raw_load_int(addr, offset, arraydescr.ofs) - elif arraydescr.typeinfo == FLOAT: - return do_raw_load_float(addr, offset) - else: - raise NotImplementedError - - def op_new(self, size): - return do_new(size.ofs) - - def op_new_with_vtable(self, descr, vtable): - assert descr is None - descr = heaptracker.vtable2descr(self.cpu, vtable) - result = do_new(descr.ofs) - value = lltype.cast_opaque_ptr(rclass.OBJECTPTR, result) - value.typeptr = cast_from_int(rclass.CLASSTYPE, vtable) - return result - - def op_setarrayitem_gc(self, arraydescr, array, index, newvalue): - if arraydescr.typeinfo == REF: - do_setarrayitem_gc_ptr(array, index, newvalue) - elif arraydescr.typeinfo == INT: - do_setarrayitem_gc_int(array, index, newvalue) - elif arraydescr.typeinfo == FLOAT: - do_setarrayitem_gc_float(array, index, newvalue) - else: - raise NotImplementedError - - def op_setarrayitem_raw(self, arraydescr, array, index, newvalue): - if arraydescr.typeinfo == REF: - raise NotImplementedError("setarrayitem_raw <- gcref") - elif arraydescr.typeinfo == INT: - do_setarrayitem_raw_int(array, index, newvalue, arraydescr.ofs) - elif arraydescr.typeinfo == FLOAT: - do_setarrayitem_raw_float(array, index, newvalue) - else: - raise NotImplementedError - - def op_getinteriorfield_gc(self, descr, array, index): - if descr.typeinfo == REF: - return do_getinteriorfield_gc_ptr(array, index, descr.ofs) - elif descr.typeinfo == INT: - return do_getinteriorfield_gc_int(array, index, descr.ofs) - elif descr.typeinfo == FLOAT: - return do_getinteriorfield_gc_float(array, index, descr.ofs) - else: - raise NotImplementedError - - def op_getinteriorfield_raw(self, descr, array, index): - if descr.typeinfo == REF: - return do_getinteriorfield_raw_ptr(array, index, descr.width, descr.ofs) - elif descr.typeinfo == INT: - return do_getinteriorfield_raw_int(array, index, descr.width, descr.ofs) - elif descr.typeinfo == FLOAT: - return do_getinteriorfield_raw_float(array, index, descr.width, descr.ofs) - else: - raise NotImplementedError - - def op_setinteriorfield_gc(self, descr, array, index, newvalue): - if descr.typeinfo == REF: - return do_setinteriorfield_gc_ptr(array, index, descr.ofs, - newvalue) - elif descr.typeinfo == INT: - return do_setinteriorfield_gc_int(array, index, descr.ofs, - newvalue) - elif descr.typeinfo == FLOAT: - return do_setinteriorfield_gc_float(array, index, descr.ofs, - newvalue) - else: - raise NotImplementedError - - def op_setinteriorfield_raw(self, descr, array, index, newvalue): - if descr.typeinfo == REF: - return do_setinteriorfield_raw_ptr(array, index, newvalue, descr.width, descr.ofs) - elif descr.typeinfo == INT: - return do_setinteriorfield_raw_int(array, index, newvalue, descr.width, descr.ofs) - elif descr.typeinfo == FLOAT: - return do_setinteriorfield_raw_float(array, index, newvalue, descr.width, descr.ofs) - else: - raise NotImplementedError - - def op_setfield_gc(self, fielddescr, struct, newvalue): - if fielddescr.typeinfo == REF: - do_setfield_gc_ptr(struct, fielddescr.ofs, newvalue) - elif fielddescr.typeinfo == INT: - do_setfield_gc_int(struct, fielddescr.ofs, newvalue) - elif fielddescr.typeinfo == FLOAT: - do_setfield_gc_float(struct, fielddescr.ofs, newvalue) - else: - raise NotImplementedError - - def op_setfield_raw(self, fielddescr, struct, newvalue): - if fielddescr.typeinfo == REF: - do_setfield_raw_ptr(struct, fielddescr.ofs, newvalue) - elif fielddescr.typeinfo == INT: - do_setfield_raw_int(struct, fielddescr.ofs, newvalue) - elif fielddescr.typeinfo == FLOAT: - do_setfield_raw_float(struct, fielddescr.ofs, newvalue) - else: - raise NotImplementedError - - def op_call(self, calldescr, func, *args): - effectinfo = calldescr.get_extra_info() - if effectinfo is not None and hasattr(effectinfo, 'oopspecindex'): - oopspecindex = effectinfo.oopspecindex - if oopspecindex == EffectInfo.OS_MATH_SQRT: - return do_math_sqrt(args[0]) - return self._do_call(calldescr, func, args, call_with_llptr=False) - - def op_call_release_gil(self, calldescr, func, *args): - return self._do_call(calldescr, func, args, call_with_llptr=True) - - def _do_call(self, calldescr, func, args, call_with_llptr): - global _last_exception - assert _last_exception is None, "exception left behind" - assert _call_args_i == _call_args_r == _call_args_f == [] - args_in_order = [] - for x in args: - T = lltype.typeOf(x) - if T is lltype.Signed: - args_in_order.append('i') - _call_args_i.append(x) - elif T == llmemory.GCREF: - args_in_order.append('r') - _call_args_r.append(x) - elif T is longlong.FLOATSTORAGE: - args_in_order.append('f') - _call_args_f.append(x) - else: - raise TypeError(x) - try: - return _do_call_common(func, args_in_order, calldescr, - call_with_llptr) - except LLException, lle: - _last_exception = lle - d = {'v': None, - REF: lltype.nullptr(llmemory.GCREF.TO), - INT: 0, - FLOAT: 0.0} - return d[calldescr.typeinfo] - - def op_cond_call_gc_wb(self, descr, a, b): - py.test.skip("cond_call_gc_wb not supported") - - def op_cond_call_gc_wb_array(self, descr, a, b, c): - py.test.skip("cond_call_gc_wb_array not supported") - - def op_oosend(self, descr, obj, *args): - raise NotImplementedError("oosend for lltype backend??") - - op_oosend_pure = op_oosend - - def op_new_array(self, arraydescr, count): - return do_new_array(arraydescr.ofs, count) - - def op_force_token(self, descr): - opaque_frame = _to_opaque(self) - return llmemory.cast_ptr_to_adr(opaque_frame) - - def op_read_timestamp(self, descr): - return read_timestamp() - - def op_call_may_force(self, calldescr, func, *args): - assert not self._forced - self._may_force = self.opindex - try: - return self.op_call(calldescr, func, *args) - finally: - self._may_force = -1 - - def op_call_assembler(self, wref_loop_token, *args): - if we_are_translated(): - raise ValueError("CALL_ASSEMBLER not supported") - return self._do_call_assembler(wref_loop_token, *args) - - def _do_call_assembler(self, wref_loop_token, *args): - global _last_exception - loop_token = wref_loop_token() - assert loop_token, "CALL_ASSEMBLER to a target that already died" - ctl = loop_token.compiled_loop_token - if hasattr(ctl, 'redirected'): - return self._do_call_assembler(ctl.redirected, *args) - assert not self._forced - self._may_force = self.opindex - try: - inpargs = _from_opaque(ctl.compiled_version).inputargs - assert len(inpargs) == len(args) - for i, inparg in enumerate(inpargs): - TYPE = inparg.concretetype - if TYPE is lltype.Signed: - set_future_value_int(i, args[i]) - elif isinstance(TYPE, lltype.Ptr): - set_future_value_ref(i, args[i]) - elif TYPE is longlong.FLOATSTORAGE: - set_future_value_float(i, args[i]) - else: - raise Exception("Nonsense type %s" % TYPE) - - failindex = self.cpu._execute_token(loop_token) - jd = loop_token.outermost_jitdriver_sd - assert jd is not None, ("call_assembler(): the loop_token needs " - "to have 'outermost_jitdriver_sd'") - if jd.index_of_virtualizable != -1: - vable = args[jd.index_of_virtualizable] - else: - vable = lltype.nullptr(llmemory.GCREF.TO) - # - # Emulate the fast path - if failindex == self.cpu.done_with_this_frame_int_v: - reset_vable(jd, vable) - return self.cpu.get_latest_value_int(0) - if failindex == self.cpu.done_with_this_frame_ref_v: - reset_vable(jd, vable) - return self.cpu.get_latest_value_ref(0) - if failindex == self.cpu.done_with_this_frame_float_v: - reset_vable(jd, vable) - return self.cpu.get_latest_value_float(0) - if failindex == self.cpu.done_with_this_frame_void_v: - reset_vable(jd, vable) - return None - # - assembler_helper_ptr = jd.assembler_helper_adr.ptr # fish - try: - return assembler_helper_ptr(failindex, vable) - except LLException, lle: - assert _last_exception is None, "exception left behind" - _last_exception = lle - # fish op - op = self.loop.operations[self.opindex] - if op.result is not None: - return 0 - finally: - self._may_force = -1 - - def op_guard_not_forced(self, descr): - forced = self._forced - self._forced = False - if forced: - raise GuardFailed - - def op_guard_not_invalidated(self, descr): - if self.loop.invalid: - raise GuardFailed - -class OOFrame(Frame): - - OPHANDLERS = [None] * (rop._LAST+1) - - def op_new_with_vtable(self, descr, vtable): - assert descr is None - typedescr = get_class_size(self.memocast, vtable) - return ootype.cast_to_object(ootype.new(typedescr.TYPE)) - - def op_new_array(self, typedescr, count): - res = ootype.oonewarray(typedescr.ARRAY, count) - return ootype.cast_to_object(res) - - def op_getfield_gc(self, fielddescr, obj): - TYPE = fielddescr.TYPE - fieldname = fielddescr.fieldname - _, T = TYPE._lookup_field(fieldname) - obj = ootype.cast_from_object(TYPE, obj) - res = getattr(obj, fieldname) - if isinstance(T, ootype.OOType): - return ootype.cast_to_object(res) - return res - - op_getfield_gc_pure = op_getfield_gc From noreply at buildbot.pypy.org Sat Jan 5 18:52:20 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 5 Jan 2013 18:52:20 +0100 (CET) Subject: [pypy-commit] pypy default: merge Message-ID: <20130105175220.05DD81C01EE@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r59744:c62951b8a169 Date: 2013-01-05 19:47 +0200 http://bitbucket.org/pypy/pypy/changeset/c62951b8a169/ Log: merge diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -28,5 +28,7 @@ .. branch: remove-PYPY_NOT_MAIN_FILE +.. branch: missing-jit-operations + .. branch: fix-lookinside-iff-oopspec Fixed the interaction between two internal tools for controlling the JIT. diff --git a/pypy/tool/release/test/test_make_release.py b/pypy/tool/release/test/test_make_release.py deleted file mode 100644 --- a/pypy/tool/release/test/test_make_release.py +++ /dev/null @@ -1,11 +0,0 @@ - -import py -from pypy.tool.release.make_release import browse_nightly - -XML = py.path.local(__file__).join('..', 'nightly.xml').read() - -def test_browse_nightly(): - res = browse_nightly('branch', override_xml=XML) - assert res[('jit', 'linux')] == (40170, 'pypy-c-jit-40170-c407c9dc5382-linux.tar.bz2') - assert len(res) == 6 - assert res[('nojit', 'linux64')] == (40170, 'pypy-c-nojit-40170-c407c9dc5382-linux64.tar.bz2') From noreply at buildbot.pypy.org Sat Jan 5 18:52:21 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 5 Jan 2013 18:52:21 +0100 (CET) Subject: [pypy-commit] pypy remove-globals-in-jit: reclose branch Message-ID: <20130105175221.1D09C1C01EE@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: remove-globals-in-jit Changeset: r59745:0e414d8e2d01 Date: 2013-01-05 19:49 +0200 http://bitbucket.org/pypy/pypy/changeset/0e414d8e2d01/ Log: reclose branch From noreply at buildbot.pypy.org Sat Jan 5 18:52:22 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 5 Jan 2013 18:52:22 +0100 (CET) Subject: [pypy-commit] pypy default: remerge remove-globals-in-jit Message-ID: <20130105175222.63E471C01EE@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r59746:86ee356f229e Date: 2013-01-05 19:49 +0200 http://bitbucket.org/pypy/pypy/changeset/86ee356f229e/ Log: remerge remove-globals-in-jit diff --git a/pypy/jit/metainterp/compile.py b/pypy/jit/metainterp/compile.py --- a/pypy/jit/metainterp/compile.py +++ b/pypy/jit/metainterp/compile.py @@ -895,7 +895,7 @@ finishargs = [] # jd = jitdriver_sd - faildescr = PropagateExceptionDescr() + faildescr = jitdriver_sd.propagate_exc_descr operations = [ ResOperation(rop.CALL, callargs, result, descr=jd.portal_calldescr), ResOperation(rop.GUARD_NO_EXCEPTION, [], None, descr=faildescr), diff --git a/pypy/jit/metainterp/jitdriver.py b/pypy/jit/metainterp/jitdriver.py --- a/pypy/jit/metainterp/jitdriver.py +++ b/pypy/jit/metainterp/jitdriver.py @@ -19,6 +19,7 @@ # self.handle_jitexc_from_bh pypy.jit.metainterp.warmspot # self.no_loop_header ... pypy.jit.metainterp.warmspot # self.portal_finishtoken... pypy.jit.metainterp.pyjitpl + # self.propagate_exc_descr.. pypy.jit.metainterp.pyjitpl # self.index ... pypy.jit.codewriter.call # self.mainjitcode ... pypy.jit.codewriter.call diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py --- a/pypy/jit/metainterp/pyjitpl.py +++ b/pypy/jit/metainterp/pyjitpl.py @@ -1463,6 +1463,7 @@ # # store this information for fastpath of call_assembler # (only the paths that can actually be taken) + exc_descr = compile.PropagateExceptionDescr() for jd in self.jitdrivers_sd: name = {history.INT: 'int', history.REF: 'ref', @@ -1472,8 +1473,8 @@ jd.portal_finishtoken = tokens[0].finishdescr num = self.cpu.get_fail_descr_number(tokens[0].finishdescr) setattr(self.cpu, 'done_with_this_frame_%s_v' % name, num) + jd.propagate_exc_descr = exc_descr # - exc_descr = compile.PropagateExceptionDescr() num = self.cpu.get_fail_descr_number(exc_descr) self.cpu.propagate_exception_v = num # diff --git a/pypy/jit/metainterp/test/test_compile.py b/pypy/jit/metainterp/test/test_compile.py --- a/pypy/jit/metainterp/test/test_compile.py +++ b/pypy/jit/metainterp/test/test_compile.py @@ -166,6 +166,7 @@ portal_runner_adr = llmemory.cast_ptr_to_adr(portal_runner_ptr) portal_calldescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, None) portal_finishtoken = compile.DoneWithThisFrameDescrInt() + propagate_exc_descr = compile.PropagateExceptionDescr() num_red_args = 2 result_type = INT # diff --git a/pypy/jit/tl/jittest.py b/pypy/jit/tl/jittest.py --- a/pypy/jit/tl/jittest.py +++ b/pypy/jit/tl/jittest.py @@ -26,8 +26,8 @@ get_policy = driver.extra['jitpolicy'] jitpolicy = get_policy(driver) - from pypy.jit.backend.llgraph.runner import LLtypeCPU - apply_jit(jitpolicy, interp, graph, LLtypeCPU) + from pypy.jit.backend.llgraph.runner import LLGraphCPU + apply_jit(jitpolicy, interp, graph, LLGraphCPU) def apply_jit(policy, interp, graph, CPUClass): diff --git a/pypy/jit/tl/pypyjit_child.py b/pypy/jit/tl/pypyjit_child.py --- a/pypy/jit/tl/pypyjit_child.py +++ b/pypy/jit/tl/pypyjit_child.py @@ -14,9 +14,9 @@ return lltype.nullptr(T) interp.heap.malloc_nonmovable = returns_null # XXX - from pypy.jit.backend.llgraph.runner import LLtypeCPU - #LLtypeCPU.supports_floats = False # for now - apply_jit(interp, graph, LLtypeCPU) + from pypy.jit.backend.llgraph.runner import LLGraphCPU + #LLGraphCPU.supports_floats = False # for now + apply_jit(interp, graph, LLGraphCPU) def run_child_ootype(glob, loc): diff --git a/pypy/jit/tl/spli/test/test_jit.py b/pypy/jit/tl/spli/test/test_jit.py --- a/pypy/jit/tl/spli/test/test_jit.py +++ b/pypy/jit/tl/spli/test/test_jit.py @@ -8,7 +8,7 @@ class TestSPLIJit(JitMixin): type_system = 'lltype' - CPUClass = runner.LLtypeCPU + CPUClass = runner.LLGraphCPU def interpret(self, f, args): coderepr = serializer.serialize(f.func_code) diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py --- a/pypy/rpython/lltypesystem/lloperation.py +++ b/pypy/rpython/lltypesystem/lloperation.py @@ -499,7 +499,7 @@ 'gc_typeids_z' : LLOp(), 'gc_gcflag_extra' : LLOp(), 'gc_add_memory_pressure': LLOp(), - 'gc_set_extra_threshold': LLOp(canrun=True), + 'gc_set_extra_threshold': LLOp(canrun=True, canmallocgc=True), # ------- JIT & GC interaction, only for some GCs ---------- diff --git a/pypy/rpython/memory/gctransform/test/test_framework.py b/pypy/rpython/memory/gctransform/test/test_framework.py --- a/pypy/rpython/memory/gctransform/test/test_framework.py +++ b/pypy/rpython/memory/gctransform/test/test_framework.py @@ -2,6 +2,7 @@ from pypy.annotation.model import SomeInteger from pypy.objspace.flow.model import Constant, SpaceOperation from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.memory.gc.semispace import SemiSpaceGC from pypy.rpython.memory.gctransform.framework import (CollectAnalyzer, find_initializing_stores, find_clean_setarrayitems) @@ -96,6 +97,13 @@ gg = graphof(t, g) assert CollectAnalyzer(t).analyze_direct_call(gg) +def test_cancollect_some_operation(): + def g(): + llop.gc_set_extra_threshold(lltype.Void, 32) + t = rtype(g, []) + gg = graphof(t, g) + assert CollectAnalyzer(t).analyze_direct_call(gg) + def test_no_collect(): from pypy.rlib import rgc from pypy.translator.c.genc import CStandaloneBuilder From noreply at buildbot.pypy.org Sat Jan 5 18:52:23 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 5 Jan 2013 18:52:23 +0100 (CET) Subject: [pypy-commit] pypy remove-globals-in-jit: merge Message-ID: <20130105175223.80C0F1C01EE@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: remove-globals-in-jit Changeset: r59747:3bc64e9c575d Date: 2013-01-05 19:50 +0200 http://bitbucket.org/pypy/pypy/changeset/3bc64e9c575d/ Log: merge From noreply at buildbot.pypy.org Sat Jan 5 18:52:24 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 5 Jan 2013 18:52:24 +0100 (CET) Subject: [pypy-commit] pypy remove-globals-in-jit: reclose Message-ID: <20130105175224.903C21C01EE@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: remove-globals-in-jit Changeset: r59748:e57ac24311b1 Date: 2013-01-05 19:51 +0200 http://bitbucket.org/pypy/pypy/changeset/e57ac24311b1/ Log: reclose From noreply at buildbot.pypy.org Sat Jan 5 18:52:25 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 5 Jan 2013 18:52:25 +0100 (CET) Subject: [pypy-commit] pypy default: remerge Message-ID: <20130105175225.A22741C01EE@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r59749:1d04d6dc7f24 Date: 2013-01-05 19:51 +0200 http://bitbucket.org/pypy/pypy/changeset/1d04d6dc7f24/ Log: remerge From noreply at buildbot.pypy.org Sat Jan 5 18:53:47 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Sat, 5 Jan 2013 18:53:47 +0100 (CET) Subject: [pypy-commit] pypy default: Mark two more fields in re as pure Message-ID: <20130105175347.C4D2C1C01EE@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: Changeset: r59750:4ba3a86f318e Date: 2013-01-05 09:53 -0800 http://bitbucket.org/pypy/pypy/changeset/4ba3a86f318e/ Log: Mark two more fields in re as pure diff --git a/pypy/rlib/rsre/rsre_core.py b/pypy/rlib/rsre/rsre_core.py --- a/pypy/rlib/rsre/rsre_core.py +++ b/pypy/rlib/rsre/rsre_core.py @@ -166,6 +166,8 @@ class StrMatchContext(AbstractMatchContext): """Concrete subclass for matching in a plain string.""" + _immutable_fields_ = ["_string"] + def __init__(self, pattern, string, match_start, end, flags): AbstractMatchContext.__init__(self, pattern, match_start, end, flags) self._string = string @@ -187,6 +189,8 @@ class UnicodeMatchContext(AbstractMatchContext): """Concrete subclass for matching in a unicode string.""" + _immutable_fields_ = ["_unicodestr"] + def __init__(self, pattern, unicodestr, match_start, end, flags): AbstractMatchContext.__init__(self, pattern, match_start, end, flags) self._unicodestr = unicodestr From noreply at buildbot.pypy.org Sat Jan 5 18:56:12 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 5 Jan 2013 18:56:12 +0100 (CET) Subject: [pypy-commit] pypy callback-jit: merge default Message-ID: <20130105175612.8B9C01C0C35@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: callback-jit Changeset: r59751:9edf594cac23 Date: 2013-01-05 19:55 +0200 http://bitbucket.org/pypy/pypy/changeset/9edf594cac23/ Log: merge default diff too long, truncating to 2000 out of 10135 lines diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -28,7 +28,7 @@ DEALINGS IN THE SOFTWARE. -PyPy Copyright holders 2003-2012 +PyPy Copyright holders 2003-2013 ----------------------------------- Except when otherwise stated (look for LICENSE files or information at diff --git a/lib-python/2.7/sre_parse.py b/lib-python/2.7/sre_parse.py --- a/lib-python/2.7/sre_parse.py +++ b/lib-python/2.7/sre_parse.py @@ -16,6 +16,12 @@ from sre_constants import * +try: + from __pypy__ import newdict +except ImportError: + def newdict(tp): + return {} + SPECIAL_CHARS = ".\\[{()*+?^$|" REPEAT_CHARS = "*+?{" @@ -68,7 +74,7 @@ self.flags = 0 self.open = [] self.groups = 1 - self.groupdict = {} + self.groupdict = newdict("module") def opengroup(self, name=None): gid = self.groups self.groups = gid + 1 diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -28,5 +28,7 @@ .. branch: remove-PYPY_NOT_MAIN_FILE +.. branch: missing-jit-operations + .. branch: fix-lookinside-iff-oopspec Fixed the interaction between two internal tools for controlling the JIT. diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py --- a/pypy/jit/backend/arm/assembler.py +++ b/pypy/jit/backend/arm/assembler.py @@ -1,5 +1,6 @@ from __future__ import with_statement import os +from pypy.jit.backend.llsupport import jitframe from pypy.jit.backend.arm.helper.assembler import saved_registers from pypy.jit.backend.arm import conditions as c from pypy.jit.backend.arm import registers as r @@ -25,12 +26,12 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.jit.backend.arm.opassembler import ResOpAssembler from pypy.rlib.debug import (debug_print, debug_start, debug_stop, - have_debug_prints) + have_debug_prints, fatalerror) from pypy.rlib.jit import AsmInfo from pypy.rlib.objectmodel import compute_unique_id # XXX Move to llsupport -from pypy.jit.backend.x86.support import values_array, memcpy_fn +from pypy.jit.backend.x86.support import memcpy_fn DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', ('i', lltype.Signed), ('type', lltype.Char), # 'b'ridge, 'l'abel or @@ -44,14 +45,8 @@ debug = True - def __init__(self, cpu, failargs_limit=1000): + def __init__(self, cpu, translate_support_code=False): self.cpu = cpu - self.fail_boxes_int = values_array(lltype.Signed, failargs_limit) - self.fail_boxes_float = values_array(longlong.FLOATSTORAGE, - failargs_limit) - self.fail_boxes_ptr = values_array(llmemory.GCREF, failargs_limit) - self.fail_boxes_count = 0 - self.fail_force_index = 0 self.setup_failure_recovery() self.mc = None self.memcpy_addr = 0 @@ -68,6 +63,7 @@ self._debug = False self.loop_run_counters = [] self.debug_counter_descr = cpu.fielddescrof(DEBUG_COUNTER, 'i') + self.force_token_to_dead_frame = {} # XXX temporary hack def set_debug(self, v): r = self._debug @@ -112,9 +108,13 @@ gc_ll_descr.initialize() self._build_wb_slowpath(False) self._build_wb_slowpath(True) + self._build_failure_recovery(exc=True, withfloats=False) + self._build_failure_recovery(exc=False, withfloats=False) if self.cpu.supports_floats: self._build_wb_slowpath(False, withfloats=True) self._build_wb_slowpath(True, withfloats=True) + self._build_failure_recovery(exc=True, withfloats=True) + self._build_failure_recovery(exc=False, withfloats=True) self._build_propagate_exception_path() if gc_ll_descr.get_malloc_slowpath_addr is not None: self._build_malloc_slowpath() @@ -122,10 +122,7 @@ if gc_ll_descr.gcrootmap and gc_ll_descr.gcrootmap.is_shadow_stack: self._build_release_gil(gc_ll_descr.gcrootmap) self.memcpy_addr = self.cpu.cast_ptr_to_int(memcpy_fn) - self._exit_code_addr = self._gen_exit_path() - self._leave_jitted_hook_save_exc = \ - self._gen_leave_jitted_hook_code(True) - self._leave_jitted_hook = self._gen_leave_jitted_hook_code(False) + if not self._debug: # if self._debug is already set it means that someone called # set_debug by hand before initializing the assembler. Leave it @@ -220,36 +217,17 @@ self.releasegil_addr = rffi.cast(lltype.Signed, releasegil_func) self.reacqgil_addr = rffi.cast(lltype.Signed, reacqgil_func) - def _gen_leave_jitted_hook_code(self, save_exc): - mc = ARMv7Builder() - if self.cpu.supports_floats: - floats = r.caller_vfp_resp - else: - floats = [] - with saved_registers(mc, r.caller_resp + [r.lr], floats): - addr = self.cpu.get_on_leave_jitted_int(save_exception=save_exc) - mc.BL(addr) - assert self._exit_code_addr != 0 - mc.B(self._exit_code_addr) - return mc.materialize(self.cpu.asmmemmgr, [], - self.cpu.gc_ll_descr.gcrootmap) - def _build_propagate_exception_path(self): if self.cpu.propagate_exception_v < 0: return # not supported (for tests, or non-translated) # mc = ARMv7Builder() - # call on_leave_jitted_save_exc() - if self.cpu.supports_floats: - floats = r.caller_vfp_resp - else: - floats = [] - with saved_registers(mc, r.caller_resp + [r.lr], floats): - addr = self.cpu.get_on_leave_jitted_int(save_exception=True, - default_to_memoryerror=True) - mc.BL(addr) - mc.gen_load_int(r.ip.value, self.cpu.propagate_exception_v) - mc.MOV_rr(r.r0.value, r.ip.value) + # + # Call the helper, which will return a dead frame object with + # the correct exception set, or MemoryError by default + # XXX make sure we return the correct value here + addr = rffi.cast(lltype.Signed, self.cpu.get_propagate_exception()) + mc.BL(addr) self.gen_func_epilog(mc=mc) self.propagate_exception_path = mc.materialize(self.cpu.asmmemmgr, []) @@ -282,12 +260,12 @@ # restore registers and return # We check for c.EQ here, meaning all bits zero in this case mc.POP([reg.value for reg in r.argument_regs] + [r.pc.value], cond=c.EQ) - # call on_leave_jitted_save_exc() - addr = self.cpu.get_on_leave_jitted_int(save_exception=True) + # + # Call the helper, which will return a dead frame object with + # the correct exception set, or MemoryError by default + addr = rffi.cast(lltype.Signed, self.cpu.get_propagate_exception()) mc.BL(addr) # - mc.gen_load_int(r.r0.value, self.cpu.propagate_exception_v) - # # footer -- note the ADD, which skips the return address of this # function, and will instead return to the caller's caller. Note # also that we completely ignore the saved arguments, because we @@ -339,39 +317,82 @@ def setup_failure_recovery(self): - @rgc.no_collect + #@rgc.no_collect -- XXX still true, but hacked gc_set_extra_threshold def failure_recovery_func(mem_loc, frame_pointer, stack_pointer): """mem_loc is a structure in memory describing where the values for the failargs are stored. frame loc is the address of the frame pointer for the frame to be decoded frame """ - vfp_registers = rffi.cast(rffi.LONGLONGP, stack_pointer) - registers = rffi.ptradd(vfp_registers, len(r.all_vfp_regs)) + vfp_registers = rffi.cast(rffi.LONGP, stack_pointer) + registers = rffi.ptradd(vfp_registers, 2*len(r.all_vfp_regs)) registers = rffi.cast(rffi.LONGP, registers) - return self.decode_registers_and_descr(mem_loc, frame_pointer, + bytecode = rffi.cast(rffi.UCHARP, mem_loc) + return self.grab_frame_values(self.cpu, bytecode, frame_pointer, registers, vfp_registers) + self.failure_recovery_code = [0, 0, 0, 0] self.failure_recovery_func = failure_recovery_func - recovery_func_sign = lltype.Ptr(lltype.FuncType([lltype.Signed] * 3, - lltype.Signed)) + _FAILURE_RECOVERY_FUNC = lltype.Ptr(lltype.FuncType([rffi.LONGP] * 3, + llmemory.GCREF)) - @rgc.no_collect - def decode_registers_and_descr(self, mem_loc, frame_pointer, + @staticmethod + #@rgc.no_collect -- XXX still true, but hacked gc_set_extra_threshold + def grab_frame_values(cpu, bytecode, frame_pointer, registers, vfp_registers): - """Decode locations encoded in memory at mem_loc and write the values - to the failboxes. Values for spilled vars and registers are stored on - stack at frame_loc """ - assert frame_pointer & 1 == 0 - self.fail_force_index = frame_pointer - bytecode = rffi.cast(rffi.UCHARP, mem_loc) + # no malloc allowed here!! xxx apart from one, hacking a lot + force_index = rffi.cast(lltype.Signed, frame_pointer) num = 0 - value = 0 - fvalue = 0 + deadframe = lltype.nullptr(jitframe.DEADFRAME) + # step 1: lots of mess just to count the final value of 'num' + bytecode1 = bytecode + while 1: + code = rffi.cast(lltype.Signed, bytecode1[0]) + bytecode1 = rffi.ptradd(bytecode1, 1) + if code >= AssemblerARM.CODE_FROMSTACK: + while code > 0x7F: + code = rffi.cast(lltype.Signed, bytecode1[0]) + bytecode1 = rffi.ptradd(bytecode1, 1) + else: + kind = code & 3 + if kind == AssemblerARM.DESCR_SPECIAL: + if code == AssemblerARM.CODE_HOLE: + num += 1 + continue + if code == AssemblerARM.CODE_INPUTARG: + continue + if code == AssemblerARM.CODE_FORCED: + # resuming from a GUARD_NOT_FORCED + token = force_index + deadframe = ( + cpu.assembler.force_token_to_dead_frame.pop(token)) + deadframe = lltype.cast_opaque_ptr( + jitframe.DEADFRAMEPTR, deadframe) + continue + assert code == AssemblerARM.CODE_STOP + break + num += 1 + + # allocate the deadframe + if not deadframe: + # Remove the "reserve" at the end of the nursery. This means + # that it is guaranteed that the following malloc() works + # without requiring a collect(), but it needs to be re-added + # as soon as possible. + cpu.gc_clear_extra_threshold() + assert num <= cpu.get_failargs_limit() + try: + deadframe = lltype.malloc(jitframe.DEADFRAME, num) + except MemoryError: + fatalerror("memory usage error in grab_frame_values") + # fill it code_inputarg = False - while True: + num = 0 + value_hi = 0 + while 1: + # decode the next instruction from the bytecode code = rffi.cast(lltype.Signed, bytecode[0]) bytecode = rffi.ptradd(bytecode, 1) - if code >= self.CODE_FROMSTACK: + if code >= AssemblerARM.CODE_FROMSTACK: if code > 0x7F: shift = 7 code &= 0x7F @@ -384,54 +405,63 @@ break # load the value from the stack kind = code & 3 - code = int((code - self.CODE_FROMSTACK) >> 2) + code = (code - AssemblerARM.CODE_FROMSTACK) >> 2 if code_inputarg: code = ~code code_inputarg = False - if kind == self.DESCR_FLOAT: - # we use code + 1 to get the hi word of the double worded float - stackloc = frame_pointer - get_fp_offset(int(code) + 1) - assert stackloc & 3 == 0 - fvalue = rffi.cast(rffi.LONGLONGP, stackloc)[0] - else: - stackloc = frame_pointer - get_fp_offset(int(code)) - assert stackloc & 1 == 0 - value = rffi.cast(rffi.LONGP, stackloc)[0] + stackloc = force_index - get_fp_offset(int(code)) + value = rffi.cast(rffi.LONGP, stackloc)[0] + if kind == AssemblerARM.DESCR_FLOAT: + assert WORD == 4 + value_hi = value + value = rffi.cast(rffi.LONGP, stackloc - WORD)[0] else: - # 'code' identifies a register: load its value kind = code & 3 - if kind == self.DESCR_SPECIAL: - if code == self.CODE_HOLE: + if kind == AssemblerARM.DESCR_SPECIAL: + if code == AssemblerARM.CODE_HOLE: num += 1 continue - if code == self.CODE_INPUTARG: + if code == AssemblerARM.CODE_INPUTARG: code_inputarg = True continue - assert code == self.CODE_STOP + if code == AssemblerARM.CODE_FORCED: + continue + assert code == AssemblerARM.CODE_STOP break + # 'code' identifies a register: load its value code >>= 2 - if kind == self.DESCR_FLOAT: - fvalue = vfp_registers[code] + if kind == AssemblerARM.DESCR_FLOAT: + if WORD == 4: + value = vfp_registers[2*code] + value_hi = vfp_registers[2*code + 1] + else: + value = registers[code] else: value = registers[code] # store the loaded value into fail_boxes_ - if kind == self.DESCR_FLOAT: - tgt = self.fail_boxes_float.get_addr_for_num(num) - rffi.cast(rffi.LONGLONGP, tgt)[0] = fvalue + if kind == AssemblerARM.DESCR_INT: + deadframe.jf_values[num].int = value + elif kind == AssemblerARM.DESCR_REF: + deadframe.jf_values[num].ref = rffi.cast(llmemory.GCREF, value) + elif kind == AssemblerARM.DESCR_FLOAT: + assert WORD == 4 + assert not longlong.is_64_bit + floatvalue = rffi.cast(lltype.SignedLongLong, value_hi) + floatvalue <<= 32 + floatvalue |= rffi.cast(lltype.SignedLongLong, + rffi.cast(lltype.Unsigned, value)) + deadframe.jf_values[num].float = floatvalue else: - if kind == self.DESCR_INT: - tgt = self.fail_boxes_int.get_addr_for_num(num) - elif kind == self.DESCR_REF: - assert (value & 3) == 0, "misaligned pointer" - tgt = self.fail_boxes_ptr.get_addr_for_num(num) - else: - assert 0, "bogus kind" - rffi.cast(rffi.LONGP, tgt)[0] = value + assert 0, "bogus kind" num += 1 - self.fail_boxes_count = num + # + assert num == len(deadframe.jf_values) + if not we_are_translated(): + assert bytecode[4] == 0xCC fail_index = rffi.cast(rffi.INTP, bytecode)[0] - fail_index = rffi.cast(lltype.Signed, fail_index) - return fail_index + fail_descr = cpu.get_fail_descr_from_number(fail_index) + deadframe.jf_descr = fail_descr.hide(cpu) + return lltype.cast_opaque_ptr(llmemory.GCREF, deadframe) def decode_inputargs(self, code): descr_to_box_type = [REF, INT, FLOAT] @@ -514,12 +544,26 @@ self.mc.CMP_ri(r.r0.value, 0) self.mc.B(self.propagate_exception_path, c=c.EQ) - def _gen_exit_path(self): + def _build_failure_recovery(self, exc, withfloats=False): mc = ARMv7Builder() - decode_registers_addr = llhelper(self.recovery_func_sign, + failure_recovery = llhelper(self._FAILURE_RECOVERY_FUNC, self.failure_recovery_func) self._insert_checks(mc) - with saved_registers(mc, r.all_regs, r.all_vfp_regs): + if withfloats: + f = r.all_vfp_regs + else: + f = [] + with saved_registers(mc, r.all_regs, f): + if exc: + # We might have an exception pending. Load it into r4 + # (this is a register saved across calls) + mc.gen_load_int(r.r5.value, self.cpu.pos_exc_value()) + mc.LDR_ri(r.r4.value, self.cpu.pos_exc_value()) + # clear the exc flags + mc.gen_load_int(r.r6.value, 0) + mc.STR_ri(r.r6.value, r.r5.value) + mc.gen_load_int(r.r5.value, self.cpu.pos_exception()) + mc.STR_ri(r.r6.value, r.r5.value) # move mem block address, to r0 to pass as mc.MOV_rr(r.r0.value, r.lr.value) # pass the current frame pointer as second param @@ -527,23 +571,32 @@ # pass the current stack pointer as third param mc.MOV_rr(r.r2.value, r.sp.value) self._insert_checks(mc) - mc.BL(rffi.cast(lltype.Signed, decode_registers_addr)) + mc.BL(rffi.cast(lltype.Signed, failure_recovery)) + if exc: + # save ebx into 'jf_guard_exc' + from pypy.jit.backend.llsupport.descr import unpack_fielddescr + descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) + offset, size, _ = unpack_fielddescr(descrs.jf_guard_exc) + mc.STR_rr(r.r4.value, r.r0.value, offset, cond=c.AL) mc.MOV_rr(r.ip.value, r.r0.value) mc.MOV_rr(r.r0.value, r.ip.value) self.gen_func_epilog(mc=mc) - return mc.materialize(self.cpu.asmmemmgr, [], + rawstart = mc.materialize(self.cpu.asmmemmgr, [], self.cpu.gc_ll_descr.gcrootmap) + self.failure_recovery_code[exc + 2 * withfloats] = rawstart + self.mc = None DESCR_REF = 0x00 DESCR_INT = 0x01 DESCR_FLOAT = 0x02 DESCR_SPECIAL = 0x03 CODE_FROMSTACK = 64 - CODE_STOP = 0 | DESCR_SPECIAL - CODE_HOLE = 4 | DESCR_SPECIAL - CODE_INPUTARG = 8 | DESCR_SPECIAL + CODE_STOP = 0 | DESCR_SPECIAL + CODE_HOLE = 4 | DESCR_SPECIAL + CODE_INPUTARG = 8 | DESCR_SPECIAL + CODE_FORCED = 12 | DESCR_SPECIAL #XXX where should this be written? - def gen_descr_encoding(self, descr, failargs, locs): + def write_failure_recovery_description(self, descr, failargs, locs): assert self.mc is not None for i in range(len(failargs)): arg = failargs[i] @@ -575,26 +628,33 @@ self.mc.writechar(chr(n)) self.mc.writechar(chr(self.CODE_STOP)) - fdescr = self.cpu.get_fail_descr_number(descr) - self.mc.write32(fdescr) + + def generate_quick_failure(self, guardtok, fcond=c.AL): + assert isinstance(guardtok.save_exc, bool) + fail_index = self.cpu.get_fail_descr_number(guardtok.descr) + startpos = self.mc.currpos() + withfloats = False + for box in guardtok.failargs: + if box is not None and box.type == FLOAT: + withfloats = True + break + exc = guardtok.save_exc + target = self.failure_recovery_code[exc + 2 * withfloats] + assert target != 0 + self.mc.BL(target) + # write tight data that describes the failure recovery + if guardtok.is_guard_not_forced: + self.mc.writechar(chr(self.CODE_FORCED)) + self.write_failure_recovery_description(guardtok.descr, + guardtok.failargs, guardtok.faillocs[1:]) + self.mc.write32(fail_index) + # for testing the decoding, write a final byte 0xCC + if not we_are_translated(): + self.mc.writechar('\xCC') + faillocs = [loc for loc in guardtok.faillocs if loc is not None] + guardtok.descr._arm_debug_faillocs = faillocs self.align() - - # assert that the fail_boxes lists are big enough - assert len(failargs) <= self.fail_boxes_int.SIZE - - def _gen_path_to_exit_path(self, descr, args, arglocs, - save_exc, fcond=c.AL): - assert isinstance(save_exc, bool) - self.gen_exit_code(self.mc, save_exc, fcond) - self.gen_descr_encoding(descr, args, arglocs[1:]) - - def gen_exit_code(self, mc, save_exc, fcond=c.AL): - assert isinstance(save_exc, bool) - if save_exc: - path = self._leave_jitted_hook_save_exc - else: - path = self._leave_jitted_hook - mc.BL(path) + return startpos def align(self): while(self.mc.currpos() % FUNC_ALIGN != 0): @@ -841,15 +901,10 @@ def write_pending_failure_recoveries(self): for tok in self.pending_guards: - descr = tok.descr #generate the exit stub and the encoded representation - pos = self.mc.currpos() - tok.pos_recovery_stub = pos - - self._gen_path_to_exit_path(descr, tok.failargs, - tok.faillocs, save_exc=tok.save_exc) + tok.pos_recovery_stub = self.generate_quick_failure(tok) # store info on the descr - descr._arm_current_frame_depth = tok.faillocs[0].getint() + tok.descr._arm_current_frame_depth = tok.faillocs[0].getint() def process_pending_guards(self, block_start): clt = self.current_clt @@ -860,7 +915,7 @@ descr._arm_failure_recovery_block = failure_recovery_pos relative_offset = tok.pos_recovery_stub - tok.offset guard_pos = block_start + tok.offset - if not tok.is_invalidate: + if not tok.is_guard_not_invalidated: # patch the guard jumpt to the stub # overwrite the generate NOP with a B_offs to the pos of the # stub @@ -1255,11 +1310,6 @@ else: raise AssertionError('Trying to pop to an invalid location') - def leave_jitted_hook(self): - ptrs = self.fail_boxes_ptr.ar - llop.gc_assume_young_pointers(lltype.Void, - llmemory.cast_ptr_to_adr(ptrs)) - def malloc_cond(self, nursery_free_adr, nursery_top_adr, size): assert size & (WORD-1) == 0 # must be correctly aligned diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py --- a/pypy/jit/backend/arm/opassembler.py +++ b/pypy/jit/backend/arm/opassembler.py @@ -16,6 +16,7 @@ gen_emit_unary_float_op, saved_registers, count_reg_args) +from pypy.jit.backend.arm.helper.regalloc import check_imm_arg from pypy.jit.backend.arm.codebuilder import ARMv7Builder, OverwritingBuilder from pypy.jit.backend.arm.jump import remap_frame_layout from pypy.jit.backend.arm.regalloc import TempInt, TempPtr @@ -27,18 +28,20 @@ from pypy.jit.metainterp.history import JitCellToken, TargetToken from pypy.jit.metainterp.resoperation import rop from pypy.rlib.objectmodel import we_are_translated -from pypy.rpython.lltypesystem import rstr +from pypy.rlib import rgc +from pypy.rpython.lltypesystem import rstr, rffi, lltype, llmemory NO_FORCE_INDEX = -1 class GuardToken(object): - def __init__(self, descr, failargs, faillocs, offset, - save_exc, fcond=c.AL, is_invalidate=False): + def __init__(self, descr, failargs, faillocs, offset, save_exc, fcond=c.AL, + is_guard_not_invalidated=False, is_guard_not_forced=False): assert isinstance(save_exc, bool) self.descr = descr self.offset = offset - self.is_invalidate = is_invalidate + self.is_guard_not_invalidated = is_guard_not_invalidated + self.is_guard_not_forced = is_guard_not_forced self.failargs = failargs self.faillocs = faillocs self.save_exc = save_exc @@ -190,7 +193,8 @@ return fcond def _emit_guard(self, op, arglocs, fcond, save_exc, - is_guard_not_invalidated=False): + is_guard_not_invalidated=False, + is_guard_not_forced=False): assert isinstance(save_exc, bool) assert isinstance(fcond, int) descr = op.getdescr() @@ -210,7 +214,8 @@ faillocs=arglocs, offset=pos, save_exc=save_exc, - is_invalidate=is_guard_not_invalidated, + is_guard_not_invalidated=is_guard_not_invalidated, + is_guard_not_forced=is_guard_not_forced, fcond=fcond)) return c.AL @@ -312,49 +317,10 @@ return fcond def emit_op_finish(self, op, arglocs, regalloc, fcond): - for i in range(len(arglocs) - 1): - loc = arglocs[i] - box = op.getarg(i) - if loc is None: - continue - if loc.is_reg(): - if box.type == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(i) - elif box.type == INT: - adr = self.fail_boxes_int.get_addr_for_num(i) - else: - assert 0 - self.mc.gen_load_int(r.ip.value, adr) - self.mc.STR_ri(loc.value, r.ip.value) - elif loc.is_vfp_reg(): - assert box.type == FLOAT - adr = self.fail_boxes_float.get_addr_for_num(i) - self.mc.gen_load_int(r.ip.value, adr) - self.mc.VSTR(loc.value, r.ip.value) - elif loc.is_stack() or loc.is_imm() or loc.is_imm_float(): - if box.type == FLOAT: - adr = self.fail_boxes_float.get_addr_for_num(i) - self.mov_loc_loc(loc, r.vfp_ip) - self.mc.gen_load_int(r.ip.value, adr) - self.mc.VSTR(r.vfp_ip.value, r.ip.value) - elif box.type == REF or box.type == INT: - if box.type == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(i) - elif box.type == INT: - adr = self.fail_boxes_int.get_addr_for_num(i) - else: - assert 0 - self.mov_loc_loc(loc, r.ip) - self.mc.gen_load_int(r.lr.value, adr) - self.mc.STR_ri(r.ip.value, r.lr.value) - else: - assert 0 - # note: no exception should currently be set in llop.get_exception_addr - # even if this finish may be an exit_frame_with_exception (in this case - # the exception instance is in arglocs[0]). - addr = self.cpu.get_on_leave_jitted_int(save_exception=False) - self.mc.BL(addr) - self.mc.gen_load_int(r.r0.value, arglocs[-1].value) + [argloc] = arglocs + if argloc is not r.r0: #XXX verify this + self.mov_loc_loc(argloc, r.r0, fcond) + # exit function self.gen_func_epilog() return fcond @@ -1152,8 +1118,26 @@ value = self.cpu.done_with_this_frame_float_v else: raise AssertionError(kind) - self.mc.gen_load_int(r.ip.value, value) - self.mc.CMP_rr(tmploc.value, r.ip.value) + from pypy.jit.backend.llsupport.descr import unpack_fielddescr + from pypy.jit.backend.llsupport.descr import unpack_interiorfielddescr + descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) + _offset, _size, _ = unpack_fielddescr(descrs.jf_descr) + fail_descr = self.cpu.get_fail_descr_from_number(value) + value = fail_descr.hide(self.cpu) + rgc._make_sure_does_not_move(value) + value = rffi.cast(lltype.Signed, value) + + if check_imm_arg(_offset): + self.mc.LDR_ri(r.ip.value, tmploc.value, imm=_offset) + else: + self.mc.gen_load_int(r.ip.value, _offset) + self.mc.LDR_rr(r.ip.value, tmploc.value, r.ip.value) + if check_imm_arg(value): + self.mc.CMP_ri(r.ip.value, imm=value) + else: + self.mc.gen_load_int(r.lr.value, value) + self.mc.CMP_rr(r.lr.value, r.ip.value) + #if values are equal we take the fast path # Slow path, calling helper @@ -1180,19 +1164,31 @@ if op.result is not None: # load the return value from fail_boxes_xxx[0] kind = op.result.type - if kind == INT: - adr = self.fail_boxes_int.get_addr_for_num(0) - elif kind == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(0) - elif kind == FLOAT: - adr = self.fail_boxes_float.get_addr_for_num(0) + if kind == FLOAT: + t = unpack_interiorfielddescr(descrs.as_float)[0] + if not check_imm_arg(t): + self.mc.gen_load_int(r.ip.value, t, cond=fast_path_cond) + self.mc.ADD_rr(r.ip.value, r.r0.value, r.ip.value, + cond=fast_path_cond) + t = 0 + base = r.ip + else: + base = r.r0 + self.mc.VLDR(resloc.value, base.value, imm=t, + cond=fast_path_cond) else: - raise AssertionError(kind) - self.mc.gen_load_int(r.ip.value, adr, cond=fast_path_cond) - if op.result.type == FLOAT: - self.mc.VLDR(resloc.value, r.ip.value, cond=fast_path_cond) - else: - self.mc.LDR_ri(resloc.value, r.ip.value, cond=fast_path_cond) + assert resloc is r.r0 + if kind == INT: + t = unpack_interiorfielddescr(descrs.as_int)[0] + else: + t = unpack_interiorfielddescr(descrs.as_ref)[0] + if not check_imm_arg(t): + self.mc.gen_load_int(r.ip.value, t, cond=fast_path_cond) + self.mc.LDR_rr(resloc.value, resloc.value, r.ip.value, + cond=fast_path_cond) + else: + self.mc.LDR_ri(resloc.value, resloc.value, imm=t, + cond=fast_path_cond) # jump to merge point jmp_pos = self.mc.currpos() self.mc.BKPT() @@ -1266,7 +1262,8 @@ self.mc.LDR_ri(r.ip.value, r.fp.value) self.mc.CMP_ri(r.ip.value, 0) - self._emit_guard(guard_op, arglocs[1 + numargs:], c.GE, save_exc=True) + self._emit_guard(guard_op, arglocs[1 + numargs:], c.GE, + save_exc=True, is_guard_not_forced=True) return fcond def emit_guard_call_release_gil(self, op, guard_op, arglocs, regalloc, diff --git a/pypy/jit/backend/arm/regalloc.py b/pypy/jit/backend/arm/regalloc.py --- a/pypy/jit/backend/arm/regalloc.py +++ b/pypy/jit/backend/arm/regalloc.py @@ -644,15 +644,9 @@ return args def prepare_op_finish(self, op, fcond): - args = [None] * (op.numargs() + 1) - for i in range(op.numargs()): - arg = op.getarg(i) - if arg: - args[i] = self.loc(arg) - self.possibly_free_var(arg) - n = self.cpu.get_fail_descr_number(op.getdescr()) - args[-1] = imm(n) - return args + loc = self.loc(op.getarg(0)) + self.possibly_free_var(op.getarg(0)) + return [loc] def prepare_op_guard_true(self, op, fcond): l0 = self.make_sure_var_in_reg(op.getarg(0)) diff --git a/pypy/jit/backend/arm/runner.py b/pypy/jit/backend/arm/runner.py --- a/pypy/jit/backend/arm/runner.py +++ b/pypy/jit/backend/arm/runner.py @@ -23,15 +23,21 @@ AbstractLLCPU.__init__(self, rtyper, stats, opts, translate_support_code, gcdescr) + from pypy.jit.backend.llsupport import jitframe + self.deadframe_size_max = llmemory.sizeof(jitframe.DEADFRAME, + self.get_failargs_limit()) + def set_debug(self, flag): return self.assembler.set_debug(flag) + def get_failargs_limit(self): + if self.opts is not None: + return self.opts.failargs_limit + else: + return 1000 + def setup(self): - if self.opts is not None: - failargs_limit = self.opts.failargs_limit - else: - failargs_limit = 1000 - self.assembler = AssemblerARM(self, failargs_limit=failargs_limit) + self.assembler = AssemblerARM(self, self.translate_support_code) def setup_once(self): self.assembler.setup_once() @@ -51,24 +57,6 @@ return self.assembler.assemble_bridge(faildescr, inputargs, operations, original_loop_token, log=log) - def get_latest_value_float(self, index): - return self.assembler.fail_boxes_float.getitem(index) - - def get_latest_value_int(self, index): - return self.assembler.fail_boxes_int.getitem(index) - - def get_latest_value_ref(self, index): - return self.assembler.fail_boxes_ptr.getitem(index) - - def get_latest_value_count(self): - return self.assembler.fail_boxes_count - - def get_latest_force_token(self): - return self.assembler.fail_force_index - - def get_on_leave_jitted_hook(self): - return self.assembler.leave_jitted_hook - def clear_latest_values(self, count): setitem = self.assembler.fail_boxes_ptr.setitem null = lltype.nullptr(llmemory.GCREF.TO) @@ -76,7 +64,7 @@ setitem(index, null) def make_execute_token(self, *ARGS): - FUNCPTR = lltype.Ptr(lltype.FuncType(ARGS, lltype.Signed)) + FUNCPTR = lltype.Ptr(lltype.FuncType(ARGS, llmemory.GCREF)) def execute_token(executable_token, *args): clt = executable_token.compiled_loop_token @@ -91,12 +79,13 @@ prev_interpreter = LLInterpreter.current_interpreter LLInterpreter.current_interpreter = self.debug_ll_interpreter try: - fail_index = func(*args) + deadframe = func(*args) finally: if not self.translate_support_code: LLInterpreter.current_interpreter = prev_interpreter #llop.debug_print(lltype.Void, "<<<< Back") - return self.get_fail_descr_from_number(fail_index) + self.gc_set_extra_threshold() + return deadframe return execute_token def cast_ptr_to_int(x): @@ -115,17 +104,22 @@ assert fail_index >= 0, "already forced!" faildescr = self.get_fail_descr_from_number(fail_index) rffi.cast(TP, addr_of_force_index)[0] = ~fail_index - bytecode = self.assembler._find_failure_recovery_bytecode(faildescr) + frb = self.assembler._find_failure_recovery_bytecode(faildescr) + bytecode = rffi.cast(rffi.UCHARP, frb) addr_all_null_regsiters = rffi.cast(rffi.LONG, self.all_null_registers) - # start of "no gc operation!" block - fail_index_2 = self.assembler.failure_recovery_func( - bytecode, - addr_of_force_index, - addr_all_null_regsiters) - self.assembler.leave_jitted_hook() - # end of "no gc operation!" block - assert fail_index == fail_index_2 - return faildescr + # + assert (rffi.cast(lltype.Signed, bytecode[0]) == + self.assembler.CODE_FORCED) + bytecode = rffi.ptradd(bytecode, 1) + deadframe = self.assembler.grab_frame_values(self, + bytecode, addr_of_force_index, + self.all_null_registers, + self.all_null_registers) + # + assert self.get_latest_descr(deadframe) is faildescr + self.assembler.force_token_to_dead_frame[addr_of_force_index] = ( + deadframe) + return deadframe def redirect_call_assembler(self, oldlooptoken, newlooptoken): self.assembler.redirect_call_assembler(oldlooptoken, newlooptoken) diff --git a/pypy/jit/backend/arm/test/test_loop_unroll.py b/pypy/jit/backend/arm/test/test_loop_unroll.py --- a/pypy/jit/backend/arm/test/test_loop_unroll.py +++ b/pypy/jit/backend/arm/test/test_loop_unroll.py @@ -1,8 +1,8 @@ import py -from pypy.jit.backend.x86.test.test_basic import Jit386Mixin +from pypy.jit.backend.arm.test.support import JitARMMixin from pypy.jit.metainterp.test import test_loop_unroll -class TestLoopSpec(Jit386Mixin, test_loop_unroll.LoopUnrollTest): +class TestLoopSpec(JitARMMixin, test_loop_unroll.LoopUnrollTest): # for the individual tests see # ====> ../../../metainterp/test/test_loop.py pass diff --git a/pypy/jit/backend/arm/test/test_runner.py b/pypy/jit/backend/arm/test/test_runner.py --- a/pypy/jit/backend/arm/test/test_runner.py +++ b/pypy/jit/backend/arm/test/test_runner.py @@ -60,16 +60,18 @@ ] cpu.compile_loop(inp, operations, looptoken) args = [i for i in range(1, 15)] - self.cpu.execute_token(looptoken, *args) - output = [self.cpu.get_latest_value_int(i - 1) for i in range(1, 15)] + deadframe = self.cpu.execute_token(looptoken, *args) + output = [self.cpu.get_latest_value_int(deadframe, i - 1) for i in range(1, 15)] expected = [3, 7, 11, 15, 19, 23, 27, 3, 7, 11, 15, 19, 23, 27] assert output == expected - def test_redirect_call_assember2(self): - def assembler_helper(failindex, virtualizable): - return self.cpu.get_latest_value_int(0) + def test_redirect_call_assembler2(self): + def assembler_helper(deadframe, virtualizable): + x = self.cpu.get_latest_value_int(deadframe, 0) + assert x == 11 + return 7 - FUNCPTR = lltype.Ptr(lltype.FuncType([lltype.Signed, llmemory.GCREF], + FUNCPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF, llmemory.GCREF], lltype.Signed)) class FakeJitDriverSD: @@ -101,12 +103,12 @@ self.cpu.compile_loop(loop2.inputargs, loop2.operations, lt2) self.cpu.compile_loop(loop3.inputargs, loop3.operations, lt3) self.cpu.compile_loop(loop1.inputargs, loop1.operations, lt1) - self.cpu.execute_token(lt1, 11) - assert self.cpu.get_latest_value_int(0) == 12 + df = self.cpu.execute_token(lt1, 10) + assert self.cpu.get_latest_value_int(df, 0) == 7 self.cpu.redirect_call_assembler(lt2, lt3) - self.cpu.execute_token(lt1, 11) - assert self.cpu.get_latest_value_int(0) == 10 + df = self.cpu.execute_token(lt1, 12) + assert self.cpu.get_latest_value_int(df, 0) == 7 SFloat = lltype.GcForwardReference() SFloat.become(lltype.GcStruct('SFloat', ('parent', rclass.OBJECT), @@ -202,8 +204,8 @@ ARGS = [lltype.Signed] * numargs RES = lltype.Signed args = [i+1 for i in range(numargs)] - res = self.cpu.execute_token(looptoken, *args) - assert self.cpu.get_latest_value_int(0) == sum(args) + deadframe = self.cpu.execute_token(looptoken, *args) + assert self.cpu.get_latest_value_int(deadframe, 0) == sum(args) def test_debugger_on(self): from pypy.rlib import debug diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py deleted file mode 100644 --- a/pypy/jit/backend/llgraph/llimpl.py +++ /dev/null @@ -1,1968 +0,0 @@ -""" -The non-RPythonic part of the llgraph backend. -This contains all the code that is directly run -when executing on top of the llinterpreter. -""" - -import weakref -from pypy.objspace.flow.model import Variable, Constant -from pypy.annotation import model as annmodel -from pypy.jit.metainterp.history import REF, INT, FLOAT -from pypy.jit.metainterp import history -from pypy.jit.codewriter import heaptracker -from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr, rffi -from pypy.rpython.ootypesystem import ootype -from pypy.rpython.module.support import LLSupport, OOSupport -from pypy.rpython.llinterp import LLException -from pypy.rpython.extregistry import ExtRegistryEntry - -from pypy.jit.metainterp import resoperation -from pypy.jit.metainterp.resoperation import rop -from pypy.jit.backend.llgraph import symbolic -from pypy.jit.codewriter import longlong -from pypy.jit.codewriter.effectinfo import EffectInfo - -from pypy.rlib.objectmodel import ComputedIntSymbolic, we_are_translated -from pypy.rlib.rarithmetic import ovfcheck -from pypy.rlib.rarithmetic import r_longlong, r_ulonglong, r_uint -from pypy.rlib.rtimer import read_timestamp - -import py -from pypy.tool.ansi_print import ansi_log -log = py.log.Producer('runner') -py.log.setconsumer('runner', ansi_log) - -IS_32_BIT = r_ulonglong is not r_uint - - -def _from_opaque(opq): - return opq._obj.externalobj - -_TO_OPAQUE = {} - -def _to_opaque(value): - try: - return value._the_opaque_pointer - except AttributeError: - op = lltype.opaqueptr(_TO_OPAQUE[value.__class__], 'opaque', - externalobj=value) - value._the_opaque_pointer = op - return op - -def _normalize(value): - if isinstance(value, lltype._ptr): - value = lltype.top_container(value._obj) - return value - -def from_opaque_string(s): - if isinstance(s, str): - return s - elif isinstance(s, ootype._string): - return OOSupport.from_rstr(s) - else: - return LLSupport.from_rstr(s) - -FLOAT_ARRAY_TP = lltype.Ptr(lltype.Array(lltype.Float, hints={"nolength": True})) -def maybe_uncast(TP, array): - if array._TYPE.TO.OF != lltype.Float: - # array._TYPE.TO._hints.get("uncast_on_llgraph"): - array = rffi.cast(TP, array) - return array - -# a list of argtypes of all operations - couldn't find any and it's -# very useful. Note however that the table is half-broken here and -# there, in ways that are sometimes a bit hard to fix; that's why -# it is not "official". -TYPES = { - 'int_add' : (('int', 'int'), 'int'), - 'int_sub' : (('int', 'int'), 'int'), - 'int_mul' : (('int', 'int'), 'int'), - 'int_floordiv' : (('int', 'int'), 'int'), - 'int_mod' : (('int', 'int'), 'int'), - 'int_and' : (('int', 'int'), 'int'), - 'int_or' : (('int', 'int'), 'int'), - 'int_xor' : (('int', 'int'), 'int'), - 'int_lshift' : (('int', 'int'), 'int'), - 'int_rshift' : (('int', 'int'), 'int'), - 'int_lt' : (('int', 'int'), 'bool'), - 'int_gt' : (('int', 'int'), 'bool'), - 'int_ge' : (('int', 'int'), 'bool'), - 'int_le' : (('int', 'int'), 'bool'), - 'int_eq' : (('int', 'int'), 'bool'), - 'int_ne' : (('int', 'int'), 'bool'), - 'int_is_true' : (('int',), 'bool'), - 'int_is_zero' : (('int',), 'bool'), - 'int_neg' : (('int',), 'int'), - 'int_invert' : (('int',), 'int'), - 'int_add_ovf' : (('int', 'int'), 'int'), - 'int_sub_ovf' : (('int', 'int'), 'int'), - 'int_mul_ovf' : (('int', 'int'), 'int'), - 'int_force_ge_zero':(('int',), 'int'), - 'uint_add' : (('int', 'int'), 'int'), - 'uint_sub' : (('int', 'int'), 'int'), - 'uint_mul' : (('int', 'int'), 'int'), - 'uint_lt' : (('int', 'int'), 'bool'), - 'uint_le' : (('int', 'int'), 'bool'), - 'uint_eq' : (('int', 'int'), 'bool'), - 'uint_ne' : (('int', 'int'), 'bool'), - 'uint_gt' : (('int', 'int'), 'bool'), - 'uint_ge' : (('int', 'int'), 'bool'), - 'uint_xor' : (('int', 'int'), 'int'), - 'uint_rshift' : (('int', 'int'), 'int'), - 'uint_floordiv' : (('int', 'int'), 'int'), - 'float_add' : (('float', 'float'), 'float'), - 'float_sub' : (('float', 'float'), 'float'), - 'float_mul' : (('float', 'float'), 'float'), - 'float_truediv' : (('float', 'float'), 'float'), - 'float_lt' : (('float', 'float'), 'bool'), - 'float_le' : (('float', 'float'), 'bool'), - 'float_eq' : (('float', 'float'), 'bool'), - 'float_ne' : (('float', 'float'), 'bool'), - 'float_gt' : (('float', 'float'), 'bool'), - 'float_ge' : (('float', 'float'), 'bool'), - 'float_neg' : (('float',), 'float'), - 'float_abs' : (('float',), 'float'), - 'cast_float_to_int':(('float',), 'int'), - 'cast_int_to_float':(('int',), 'float'), - 'same_as' : (('int',), 'int'), # could also be ptr=>ptr - 'new_with_vtable' : (('ref',), 'ref'), - 'new' : ((), 'ref'), - 'new_array' : (('int',), 'ref'), - 'oois' : (('ref', 'ref'), 'bool'), - 'ooisnot' : (('ref', 'ref'), 'bool'), - 'instanceof' : (('ref',), 'bool'), - 'subclassof' : (('ref', 'ref'), 'bool'), - 'runtimenew' : (('ref',), 'ref'), - 'setfield_gc' : (('ref', 'intorptr'), None), - 'getfield_gc' : (('ref',), 'intorptr'), - 'getfield_gc_pure': (('ref',), 'intorptr'), - 'setfield_raw' : (('ref', 'intorptr'), None), - 'getfield_raw' : (('ref',), 'intorptr'), - 'getfield_raw_pure': (('ref',), 'intorptr'), - 'setarrayitem_gc' : (('ref', 'int', 'intorptr'), None), - 'getarrayitem_gc' : (('ref', 'int'), 'intorptr'), - 'getarrayitem_gc_pure' : (('ref', 'int'), 'intorptr'), - 'setarrayitem_raw' : (('ref', 'int', 'intorptr'), None), - 'getarrayitem_raw' : (('ref', 'int'), 'intorptr'), - 'getarrayitem_raw_pure' : (('ref', 'int'), 'intorptr'), - 'arraylen_gc' : (('ref',), 'int'), - 'call' : (('ref', 'varargs'), 'intorptr'), - 'call_assembler' : (('varargs',), 'intorptr'), - 'cond_call_gc_wb' : (('ptr', 'ptr'), None), - 'cond_call_gc_wb_array': (('ptr', 'int', 'ptr'), None), - 'oosend' : (('varargs',), 'intorptr'), - 'oosend_pure' : (('varargs',), 'intorptr'), - 'guard_true' : (('bool',), None), - 'guard_false' : (('bool',), None), - 'guard_value' : (('int', 'int'), None), - 'guard_class' : (('ref', 'ref'), None), - 'guard_no_exception' : ((), None), - 'guard_exception' : (('ref',), 'ref'), - 'guard_no_overflow' : ((), None), - 'guard_overflow' : ((), None), - 'guard_nonnull' : (('ref',), None), - 'guard_isnull' : (('ref',), None), - 'guard_nonnull_class' : (('ref', 'ref'), None), - 'newstr' : (('int',), 'ref'), - 'strlen' : (('ref',), 'int'), - 'strgetitem' : (('ref', 'int'), 'int'), - 'strsetitem' : (('ref', 'int', 'int'), None), - 'newunicode' : (('int',), 'ref'), - 'unicodelen' : (('ref',), 'int'), - 'unicodegetitem' : (('ref', 'int'), 'int'), - 'unicodesetitem' : (('ref', 'int', 'int'), 'int'), - 'cast_ptr_to_int' : (('ref',), 'int'), - 'cast_int_to_ptr' : (('int',), 'ref'), - 'debug_merge_point': (('ref', 'int', 'int'), None), - 'force_token' : ((), 'int'), - 'call_may_force' : (('int', 'varargs'), 'intorptr'), - 'guard_not_forced': ((), None), -} - -# ____________________________________________________________ - -class CompiledLoop(object): - has_been_freed = False - invalid = False - - def __init__(self): - self.inputargs = [] - self.operations = [] - - def getargtypes(self): - return [v.concretetype for v in self.inputargs] - - def __repr__(self): - lines = [] - self.as_text(lines, 1) - return 'CompiledLoop %s:\n%s' % (self.inputargs, '\n'.join(lines)) - - def as_text(self, lines, indent): - for op in self.operations: - lines.append('\t'*indent + repr(op)) - -class Operation(object): - result = None - descr = None - jump_target = None - fail_args = None - - def __init__(self, opnum): - self.opnum = opnum - self.args = [] - - def __repr__(self): - if self.result is not None: - sres = repr0(self.result) + ' = ' - else: - sres = '' - return '{%s%s(%s)}' % (sres, self.getopname(), - ', '.join(map(repr0, self.args))) - - def getopname(self): - try: - return resoperation.opname[self.opnum] - except KeyError: - return '<%d>' % self.opnum - - def is_guard(self): - return rop._GUARD_FIRST <= self.opnum <= rop._GUARD_LAST - - def is_final(self): - return rop._FINAL_FIRST <= self.opnum <= rop._FINAL_LAST - -def repr0(x): - if isinstance(x, list): - return '[' + ', '.join(repr0(y) for y in x) + ']' - elif isinstance(x, Constant): - return '(' + repr0(x.value) + ')' - elif isinstance(x, lltype._ptr): - x = llmemory.cast_ptr_to_adr(x) - if x.ptr: - try: - container = x.ptr._obj._normalizedcontainer() - return '* %s' % (container._TYPE._short_name(),) - except AttributeError: - return repr(x) - else: - return 'NULL' - else: - return repr(x) - -def repr_list(lst, types): - res_l = [] - if types and types[-1] == 'varargs': - types = types[:-1] + ('int',) * (len(lst) - len(types) + 1) - assert len(types) == len(lst) - for elem, tp in zip(lst, types): - if isinstance(elem, Constant): - res_l.append('(%s)' % repr1(elem, tp)) - else: - res_l.append(repr1(elem, tp)) - return '[%s]' % (', '.join(res_l)) - -def repr1(x, tp): - if tp == "intorptr": - TYPE = lltype.typeOf(x) - if isinstance(TYPE, lltype.Ptr) and TYPE.TO._gckind == 'gc': - tp = "ref" - else: - tp = "int" - if tp == 'int': - return str(x) - elif tp == 'void': - return '---' - elif tp == 'ref': - if not x: - return '(* None)' - if isinstance(x, int): - # XXX normalize? - ptr = str(llmemory.cast_int_to_adr(x)) - elif isinstance(ootype.typeOf(x), ootype.OOType): - return repr(x) - else: - if getattr(x, '_fake', None): - return repr(x) - if lltype.typeOf(x) == llmemory.GCREF: - TP = lltype.Ptr(lltype.typeOf(x._obj.container)) - ptr = lltype.cast_opaque_ptr(TP, x) - else: - ptr = x - try: - container = ptr._obj._normalizedcontainer() - return '(* %s)' % (container._TYPE._short_name(),) - except AttributeError: - return '(%r)' % (ptr,) - elif tp == 'bool': - assert x == 0 or x == 1 - return str(bool(x)) - #elif tp == 'fieldname': - # return str(symbolic.TokenToField[x...][1]) - elif tp == 'float': - return str(x) - else: - raise NotImplementedError("tp = %s" % tp) - -_variables = [] - -def compile_start(): - del _variables[:] - return _to_opaque(CompiledLoop()) - -def mark_as_free(loop): - loop = _from_opaque(loop) - assert not loop.has_been_freed - loop.has_been_freed = True - -def compile_start_int_var(loop): - return compile_start_ref_var(loop, lltype.Signed) - -def compile_start_float_var(loop): - return compile_start_ref_var(loop, longlong.FLOATSTORAGE) - -def compile_start_ref_var(loop, TYPE): - loop = _from_opaque(loop) - assert not loop.operations - v = Variable() - v.concretetype = TYPE - loop.inputargs.append(v) - r = len(_variables) - _variables.append(v) - return r - -def compile_started_vars(clt): - if not hasattr(clt, '_debug_argtypes'): # only when compiling the loop - argtypes = [v.concretetype for v in _variables] - try: - clt._debug_argtypes = argtypes - except AttributeError: # when 'clt' is actually a translated - pass # GcStruct - -def compile_add(loop, opnum): - loop = _from_opaque(loop) - loop.operations.append(Operation(opnum)) - -def compile_add_descr(loop, ofs, type, arg_types, extrainfo, width): - from pypy.jit.backend.llgraph.runner import Descr - loop = _from_opaque(loop) - op = loop.operations[-1] - assert isinstance(type, str) and len(type) == 1 - op.descr = Descr(ofs, type, arg_types=arg_types, extrainfo=extrainfo, width=width) - -def compile_add_descr_arg(loop, ofs, type, arg_types): - from pypy.jit.backend.llgraph.runner import Descr - loop = _from_opaque(loop) - op = loop.operations[-1] - assert isinstance(type, str) and len(type) == 1 - op.args.append(Descr(ofs, type, arg_types=arg_types)) - -def compile_add_loop_token(loop, descr): - if we_are_translated(): - raise ValueError("CALL_ASSEMBLER not supported") - loop = _from_opaque(loop) - op = loop.operations[-1] - op.descr = weakref.ref(descr) - -TARGET_TOKENS = weakref.WeakKeyDictionary() - -def compile_add_target_token(loop, descr, clt): - # here, 'clt' is the compiled_loop_token of the original loop that - # we are compiling - loop = _from_opaque(loop) - op = loop.operations[-1] - descrobj = _normalize(descr) - TARGET_TOKENS[descrobj] = loop, len(loop.operations), op.args, clt - -def compile_add_var(loop, intvar): - loop = _from_opaque(loop) - op = loop.operations[-1] - op.args.append(_variables[intvar]) - -def compile_add_int_const(loop, value): - compile_add_ref_const(loop, value, lltype.Signed) - -def compile_add_float_const(loop, value): - compile_add_ref_const(loop, value, longlong.FLOATSTORAGE) - -def compile_add_ref_const(loop, value, TYPE): - loop = _from_opaque(loop) - const = Constant(value) - const.concretetype = TYPE - op = loop.operations[-1] - op.args.append(const) - -def compile_add_int_result(loop): - return compile_add_ref_result(loop, lltype.Signed) - -def compile_add_float_result(loop): - return compile_add_ref_result(loop, longlong.FLOATSTORAGE) - -def compile_add_ref_result(loop, TYPE): - loop = _from_opaque(loop) - v = Variable() - v.concretetype = TYPE - op = loop.operations[-1] - op.result = v - r = len(_variables) - _variables.append(v) - return r - -def compile_add_jump_target(loop, targettoken, source_clt): - loop = _from_opaque(loop) - descrobj = _normalize(targettoken) - (loop_target, target_opindex, target_inputargs, target_clt - ) = TARGET_TOKENS[descrobj] - # - try: - assert source_clt._debug_argtypes == target_clt._debug_argtypes - except AttributeError: # when translated - pass - # - op = loop.operations[-1] - op.jump_target = loop_target - op.jump_target_opindex = target_opindex - op.jump_target_inputargs = target_inputargs - assert op.opnum == rop.JUMP - assert [v.concretetype for v in op.args] == ( - [v.concretetype for v in target_inputargs]) - # - if loop_target == loop: - log.info("compiling new loop") - else: - log.info("compiling new bridge") - -def compile_add_guard_jump_target(loop, loop_target): - loop = _from_opaque(loop) - loop_target = _from_opaque(loop_target) - op = loop.operations[-1] - assert op.is_guard() - op.jump_target = loop_target - -def compile_add_fail(loop, fail_index): - loop = _from_opaque(loop) - index = len(loop.operations)-1 - op = loop.operations[index] - op.fail_index = fail_index - return index - -def compile_add_fail_arg(loop, intvar): - loop = _from_opaque(loop) - op = loop.operations[-1] - if op.fail_args is None: - op.fail_args = [] - if intvar == -1: - op.fail_args.append(None) - else: - op.fail_args.append(_variables[intvar]) - -def compile_redirect_fail(old_loop, old_index, new_loop): - old_loop = _from_opaque(old_loop) - new_loop = _from_opaque(new_loop) - guard_op = old_loop.operations[old_index] - assert guard_op.is_guard() - guard_op.jump_target = new_loop - # check that the bridge's inputargs are of the correct number and - # kind for the guard - if guard_op.fail_args is not None: - argkinds = [v.concretetype for v in guard_op.fail_args if v] - else: - argkinds = [] - assert argkinds == [v.concretetype for v in new_loop.inputargs] - -# ------------------------------ - -class Frame(object): - OPHANDLERS = [None] * (rop._LAST+1) - - def __init__(self, cpu): - self.verbose = False - self.cpu = cpu - self.opindex = 1 - self._forced = False - self._may_force = -1 - - def getenv(self, v): - from pypy.jit.backend.llgraph.runner import Descr - if isinstance(v, Constant): - return v.value - elif isinstance(v, Descr): - return v - else: - return self.env[v] - - def _populate_fail_args(self, op, skip=None): - fail_args = [] - if op.fail_args: - for fail_arg in op.fail_args: - if fail_arg is None: - fail_args.append(None) - elif fail_arg is skip: - fail_args.append(fail_arg.concretetype._defl()) - else: - fail_args.append(self.getenv(fail_arg)) - self.fail_args = fail_args - self.fail_index = op.fail_index - - def execute(self): - """Execute all operations in a loop, - possibly following to other loops as well. - """ - global _last_exception - assert _last_exception is None, "exception left behind" - verbose = True - self.opindex = 0 - while True: - assert not self.loop.has_been_freed - op = self.loop.operations[self.opindex] - args = [self.getenv(v) for v in op.args] - if not op.is_final(): - try: - result = self.execute_operation(op.opnum, args, op.descr, - verbose) - except GuardFailed: - assert op.is_guard() - _stats.exec_conditional_jumps += 1 - if op.jump_target is not None: - # a patched guard, pointing to further code - if op.fail_args: - args = [self.getenv(v) for v in op.fail_args if v] - else: - args = [] - assert len(op.jump_target.inputargs) == len(args) - self.env = dict(zip(op.jump_target.inputargs, args)) - self.loop = op.jump_target - self.opindex = 0 - continue - else: - self._populate_fail_args(op) - # a non-patched guard - if self.verbose: - log.trace('failed: %s' % ( - ', '.join(map(str, fail_args)),)) - return op.fail_index - #verbose = self.verbose - assert (result is None) == (op.result is None) - if op.result is not None: - RESTYPE = op.result.concretetype - if RESTYPE is lltype.Signed: - x = self.as_int(result) - elif RESTYPE is llmemory.GCREF: - x = self.as_ptr(result) - elif RESTYPE is ootype.Object: - x = self.as_object(result) - elif RESTYPE is longlong.FLOATSTORAGE: - x = self.as_floatstorage(result) - else: - raise Exception("op.result.concretetype is %r" - % (RESTYPE,)) - self.env[op.result] = x - self.opindex += 1 - continue - if op.opnum == rop.JUMP: - inputargs = op.jump_target_inputargs - assert len(inputargs) == len(args) - self.env = dict(zip(inputargs, args)) - self.loop = op.jump_target - self.opindex = op.jump_target_opindex - _stats.exec_jumps += 1 - elif op.opnum == rop.FINISH: - if self.verbose: - log.trace('finished: %s' % ( - ', '.join(map(str, args)),)) - self.fail_args = args - return op.fail_index - - else: - assert 0, "unknown final operation %d" % (op.opnum,) - - def execute_operation(self, opnum, values, descr, verbose): - """Execute a single operation. - """ - ophandler = self.OPHANDLERS[opnum] - if ophandler is None: - self._define_impl(opnum) - ophandler = self.OPHANDLERS[opnum] - assert ophandler is not None, "missing impl for op %d" % opnum - opname = resoperation.opname[opnum].lower() - exec_counters = _stats.exec_counters - exec_counters[opname] = exec_counters.get(opname, 0) + 1 - for i in range(len(values)): - if isinstance(values[i], ComputedIntSymbolic): - values[i] = values[i].compute_fn() - res = NotImplemented - try: - res = ophandler(self, descr, *values) - finally: - if 0: # if verbose: - argtypes, restype = TYPES[opname] - if res is None: - resdata = '' - elif res is NotImplemented: - resdata = '*fail*' - else: - resdata = '-> ' + repr1(res, restype) - # fish the types - log.cpu('\t%s %s %s' % (opname, repr_list(values, argtypes), - resdata)) - return res - - def as_int(self, x): - return cast_to_int(x) - - def as_ptr(self, x): - return cast_to_ptr(x) - - def as_object(self, x): - return ootype.cast_to_object(x) - - def as_floatstorage(self, x): - return cast_to_floatstorage(x) - - def log_progress(self): - count = sum(_stats.exec_counters.values()) - count_jumps = _stats.exec_jumps - log.trace('ran %d operations, %d jumps' % (count, count_jumps)) - - # ---------- - - @classmethod - def _define_impl(cls, opnum): - opname = resoperation.opname[opnum] - try: - op = getattr(cls, 'op_' + opname.lower()) # op_guard_true etc. - except AttributeError: - try: - impl = globals()['do_' + opname.lower()] # do_arraylen_gc etc. - def op(self, descr, *args): - if descr is None: - return impl(*args) - else: - return impl(descr, *args) - except KeyError: - op = cls._make_impl_from_blackhole_interp(opname) - cls.OPHANDLERS[opnum] = op - - @classmethod - def _make_impl_from_blackhole_interp(cls, opname): - from pypy.jit.metainterp.blackhole import BlackholeInterpreter - name = 'bhimpl_' + opname.lower() - func = BlackholeInterpreter.__dict__[name] - for argtype in func.argtypes: - assert argtype in ('i', 'r', 'f') - # - def _op_default_implementation(self, descr, *args): - # for all operations implemented in the blackhole interpreter - return func(*args) - # - return _op_default_implementation - - def op_label(self, _, *args): - op = self.loop.operations[self.opindex] - assert op.opnum == rop.LABEL - assert len(op.args) == len(args) - newenv = {} - for v, value in zip(op.args, args): - newenv[v] = value - self.env = newenv - - def op_debug_merge_point(self, _, *args): - from pypy.jit.metainterp.warmspot import get_stats - try: - stats = get_stats() - except AttributeError: - pass - else: - stats.add_merge_point_location(args[1:]) - pass - - def op_guard_true(self, _, value): - if not value: - raise GuardFailed - - def op_guard_false(self, _, value): - if value: - raise GuardFailed - - op_guard_nonnull = op_guard_true - op_guard_isnull = op_guard_false - - def op_guard_class(self, _, value, expected_class): - value = lltype.cast_opaque_ptr(rclass.OBJECTPTR, value) - expected_class = llmemory.cast_adr_to_ptr( - llmemory.cast_int_to_adr(expected_class), - rclass.CLASSTYPE) - if value.typeptr != expected_class: - raise GuardFailed - - def op_guard_nonnull_class(self, _, value, expected_class): - if not value: - raise GuardFailed - self.op_guard_class(_, value, expected_class) - - def op_guard_value(self, _, value, expected_value): - if value != expected_value: - raise GuardFailed - - def op_guard_no_exception(self, _): - if _last_exception: - raise GuardFailed - - def _check_exception(self, expected_exception): - global _last_exception - expected_exception = self._cast_exception(expected_exception) - assert expected_exception - exc = _last_exception - if exc: - got = exc.args[0] - # exact match! - if got != expected_exception: - return False - return True - else: - return False - - def _cast_exception(self, exception): - return llmemory.cast_adr_to_ptr( - llmemory.cast_int_to_adr(exception), - rclass.CLASSTYPE) - - def _issubclass(self, cls1, cls2): - return rclass.ll_issubclass(cls1, cls2) - - def op_guard_exception(self, _, expected_exception): - global _last_exception - if not self._check_exception(expected_exception): - raise GuardFailed - res = _last_exception[1] - _last_exception = None - return res - - def op_guard_no_overflow(self, _): - flag = self.overflow_flag - del self.overflow_flag - if flag: - raise GuardFailed - - def op_guard_overflow(self, _): - flag = self.overflow_flag - del self.overflow_flag - if not flag: - raise GuardFailed - - def op_int_add_ovf(self, _, x, y): - try: - z = ovfcheck(x + y) - except OverflowError: - ovf = True - z = 0 - else: - ovf = False - self.overflow_flag = ovf - return z - - def op_int_sub_ovf(self, _, x, y): - try: - z = ovfcheck(x - y) - except OverflowError: - ovf = True - z = 0 - else: - ovf = False - self.overflow_flag = ovf - return z - - def op_int_mul_ovf(self, _, x, y): - try: - z = ovfcheck(x * y) - except OverflowError: - ovf = True - z = 0 - else: - ovf = False - self.overflow_flag = ovf - return z - - def op_keepalive(self, _, x): - pass - - # ---------- - # delegating to the builtins do_xxx() (done automatically for simple cases) - - def op_getarrayitem_gc(self, arraydescr, array, index): - if arraydescr.typeinfo == REF: - return do_getarrayitem_gc_ptr(array, index) - elif arraydescr.typeinfo == INT: - return do_getarrayitem_gc_int(array, index) - elif arraydescr.typeinfo == FLOAT: - return do_getarrayitem_gc_float(array, index) - else: - raise NotImplementedError - - op_getarrayitem_gc_pure = op_getarrayitem_gc - - def op_getarrayitem_raw(self, arraydescr, array, index): - if arraydescr.typeinfo == REF: - raise NotImplementedError("getarrayitem_raw -> gcref") - elif arraydescr.typeinfo == INT: - return do_getarrayitem_raw_int(array, index, arraydescr.ofs) - elif arraydescr.typeinfo == FLOAT: - return do_getarrayitem_raw_float(array, index) - else: - raise NotImplementedError - - op_getarrayitem_raw_pure = op_getarrayitem_raw - - def op_getfield_gc(self, fielddescr, struct): - if fielddescr.typeinfo == REF: - return do_getfield_gc_ptr(struct, fielddescr.ofs) - elif fielddescr.typeinfo == INT: - return do_getfield_gc_int(struct, fielddescr.ofs) - elif fielddescr.typeinfo == FLOAT: - return do_getfield_gc_float(struct, fielddescr.ofs) - else: - raise NotImplementedError - - op_getfield_gc_pure = op_getfield_gc - - def op_getfield_raw(self, fielddescr, struct): - if fielddescr.typeinfo == REF: - return do_getfield_raw_ptr(struct, fielddescr.ofs) - elif fielddescr.typeinfo == INT: - return do_getfield_raw_int(struct, fielddescr.ofs) - elif fielddescr.typeinfo == FLOAT: - return do_getfield_raw_float(struct, fielddescr.ofs) - else: - raise NotImplementedError - - op_getfield_raw_pure = op_getfield_raw - - def op_raw_store(self, arraydescr, addr, offset, value): - if arraydescr.typeinfo == REF: - raise AssertionError("cannot store GC pointer in raw storage") - elif arraydescr.typeinfo == INT: - do_raw_store_int(addr, offset, arraydescr.ofs, value) - elif arraydescr.typeinfo == FLOAT: - do_raw_store_float(addr, offset, value) - else: - raise NotImplementedError - - def op_raw_load(self, arraydescr, addr, offset): - if arraydescr.typeinfo == REF: - raise AssertionError("cannot store GC pointer in raw storage") - elif arraydescr.typeinfo == INT: - return do_raw_load_int(addr, offset, arraydescr.ofs) - elif arraydescr.typeinfo == FLOAT: - return do_raw_load_float(addr, offset) - else: - raise NotImplementedError - - def op_new(self, size): - return do_new(size.ofs) - - def op_new_with_vtable(self, descr, vtable): - assert descr is None - descr = heaptracker.vtable2descr(self.cpu, vtable) - result = do_new(descr.ofs) - value = lltype.cast_opaque_ptr(rclass.OBJECTPTR, result) - value.typeptr = cast_from_int(rclass.CLASSTYPE, vtable) - return result - - def op_setarrayitem_gc(self, arraydescr, array, index, newvalue): - if arraydescr.typeinfo == REF: - do_setarrayitem_gc_ptr(array, index, newvalue) - elif arraydescr.typeinfo == INT: - do_setarrayitem_gc_int(array, index, newvalue) - elif arraydescr.typeinfo == FLOAT: - do_setarrayitem_gc_float(array, index, newvalue) - else: - raise NotImplementedError - - def op_setarrayitem_raw(self, arraydescr, array, index, newvalue): - if arraydescr.typeinfo == REF: - raise NotImplementedError("setarrayitem_raw <- gcref") - elif arraydescr.typeinfo == INT: - do_setarrayitem_raw_int(array, index, newvalue, arraydescr.ofs) - elif arraydescr.typeinfo == FLOAT: - do_setarrayitem_raw_float(array, index, newvalue) - else: - raise NotImplementedError - - def op_getinteriorfield_gc(self, descr, array, index): - if descr.typeinfo == REF: - return do_getinteriorfield_gc_ptr(array, index, descr.ofs) - elif descr.typeinfo == INT: - return do_getinteriorfield_gc_int(array, index, descr.ofs) - elif descr.typeinfo == FLOAT: - return do_getinteriorfield_gc_float(array, index, descr.ofs) - else: - raise NotImplementedError - - def op_getinteriorfield_raw(self, descr, array, index): - if descr.typeinfo == REF: - return do_getinteriorfield_raw_ptr(array, index, descr.width, descr.ofs) - elif descr.typeinfo == INT: - return do_getinteriorfield_raw_int(array, index, descr.width, descr.ofs) - elif descr.typeinfo == FLOAT: - return do_getinteriorfield_raw_float(array, index, descr.width, descr.ofs) - else: - raise NotImplementedError - - def op_setinteriorfield_gc(self, descr, array, index, newvalue): - if descr.typeinfo == REF: - return do_setinteriorfield_gc_ptr(array, index, descr.ofs, - newvalue) - elif descr.typeinfo == INT: - return do_setinteriorfield_gc_int(array, index, descr.ofs, - newvalue) - elif descr.typeinfo == FLOAT: - return do_setinteriorfield_gc_float(array, index, descr.ofs, - newvalue) - else: - raise NotImplementedError - - def op_setinteriorfield_raw(self, descr, array, index, newvalue): - if descr.typeinfo == REF: - return do_setinteriorfield_raw_ptr(array, index, newvalue, descr.width, descr.ofs) - elif descr.typeinfo == INT: - return do_setinteriorfield_raw_int(array, index, newvalue, descr.width, descr.ofs) - elif descr.typeinfo == FLOAT: - return do_setinteriorfield_raw_float(array, index, newvalue, descr.width, descr.ofs) - else: - raise NotImplementedError - - def op_setfield_gc(self, fielddescr, struct, newvalue): - if fielddescr.typeinfo == REF: - do_setfield_gc_ptr(struct, fielddescr.ofs, newvalue) - elif fielddescr.typeinfo == INT: - do_setfield_gc_int(struct, fielddescr.ofs, newvalue) - elif fielddescr.typeinfo == FLOAT: - do_setfield_gc_float(struct, fielddescr.ofs, newvalue) - else: - raise NotImplementedError - - def op_setfield_raw(self, fielddescr, struct, newvalue): - if fielddescr.typeinfo == REF: - do_setfield_raw_ptr(struct, fielddescr.ofs, newvalue) - elif fielddescr.typeinfo == INT: - do_setfield_raw_int(struct, fielddescr.ofs, newvalue) - elif fielddescr.typeinfo == FLOAT: - do_setfield_raw_float(struct, fielddescr.ofs, newvalue) - else: - raise NotImplementedError - - def op_call(self, calldescr, func, *args): - effectinfo = calldescr.get_extra_info() - if effectinfo is not None and hasattr(effectinfo, 'oopspecindex'): - oopspecindex = effectinfo.oopspecindex - if oopspecindex == EffectInfo.OS_MATH_SQRT: - return do_math_sqrt(args[0]) - return self._do_call(calldescr, func, args, call_with_llptr=False) - - def op_call_release_gil(self, calldescr, func, *args): - return self._do_call(calldescr, func, args, call_with_llptr=True) - - def _do_call(self, calldescr, func, args, call_with_llptr): - global _last_exception - assert _last_exception is None, "exception left behind" - assert _call_args_i == _call_args_r == _call_args_f == [] - args_in_order = [] - for x in args: - T = lltype.typeOf(x) - if T is lltype.Signed: - args_in_order.append('i') - _call_args_i.append(x) - elif T == llmemory.GCREF: - args_in_order.append('r') - _call_args_r.append(x) - elif T is longlong.FLOATSTORAGE: - args_in_order.append('f') - _call_args_f.append(x) - else: - raise TypeError(x) - try: - return _do_call_common(func, args_in_order, calldescr, - call_with_llptr) - except LLException, lle: - _last_exception = lle - d = {'v': None, - REF: lltype.nullptr(llmemory.GCREF.TO), - INT: 0, - FLOAT: 0.0} - return d[calldescr.typeinfo] - - def op_cond_call_gc_wb(self, descr, a, b): - py.test.skip("cond_call_gc_wb not supported") - - def op_cond_call_gc_wb_array(self, descr, a, b, c): - py.test.skip("cond_call_gc_wb_array not supported") - - def op_oosend(self, descr, obj, *args): - raise NotImplementedError("oosend for lltype backend??") - - op_oosend_pure = op_oosend - - def op_new_array(self, arraydescr, count): - return do_new_array(arraydescr.ofs, count) - - def op_force_token(self, descr): - opaque_frame = _to_opaque(self) - return llmemory.cast_ptr_to_adr(opaque_frame) - - def op_read_timestamp(self, descr): - return read_timestamp() - - def op_call_may_force(self, calldescr, func, *args): - assert not self._forced - self._may_force = self.opindex - try: - return self.op_call(calldescr, func, *args) - finally: - self._may_force = -1 - - def op_call_assembler(self, wref_loop_token, *args): - if we_are_translated(): - raise ValueError("CALL_ASSEMBLER not supported") - return self._do_call_assembler(wref_loop_token, *args) - - def _do_call_assembler(self, wref_loop_token, *args): - global _last_exception - loop_token = wref_loop_token() - assert loop_token, "CALL_ASSEMBLER to a target that already died" - ctl = loop_token.compiled_loop_token - if hasattr(ctl, 'redirected'): - return self._do_call_assembler(ctl.redirected, *args) - assert not self._forced - self._may_force = self.opindex - try: - inpargs = _from_opaque(ctl.compiled_version).inputargs - assert len(inpargs) == len(args) - for i, inparg in enumerate(inpargs): - TYPE = inparg.concretetype - if TYPE is lltype.Signed: - set_future_value_int(i, args[i]) - elif isinstance(TYPE, lltype.Ptr): - set_future_value_ref(i, args[i]) - elif TYPE is longlong.FLOATSTORAGE: - set_future_value_float(i, args[i]) - else: - raise Exception("Nonsense type %s" % TYPE) - - failindex = self.cpu._execute_token(loop_token) - jd = loop_token.outermost_jitdriver_sd - assert jd is not None, ("call_assembler(): the loop_token needs " - "to have 'outermost_jitdriver_sd'") - if jd.index_of_virtualizable != -1: - vable = args[jd.index_of_virtualizable] - else: - vable = lltype.nullptr(llmemory.GCREF.TO) From noreply at buildbot.pypy.org Sat Jan 5 19:12:27 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 5 Jan 2013 19:12:27 +0100 (CET) Subject: [pypy-commit] pypy default: try to jit this test Message-ID: <20130105181227.AAE9E1C0DDA@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r59752:da42f8f19188 Date: 2013-01-05 20:11 +0200 http://bitbucket.org/pypy/pypy/changeset/da42f8f19188/ Log: try to jit this test diff --git a/pypy/jit/metainterp/test/test_fficall.py b/pypy/jit/metainterp/test/test_fficall.py --- a/pypy/jit/metainterp/test/test_fficall.py +++ b/pypy/jit/metainterp/test/test_fficall.py @@ -1,9 +1,11 @@ -import py + +import ctypes, math from pypy.rpython.lltypesystem import lltype, rffi from pypy.jit.metainterp.test.support import LLJitMixin from pypy.rlib import jit from pypy.rlib.jit_libffi import types, CIF_DESCRIPTION, FFI_TYPE_PP from pypy.rlib.unroll import unrolling_iterable +from pypy.rlib.rarithmetic import intmask def get_description(atypes, rtype): @@ -103,4 +105,39 @@ class TestFfiCall(FfiCallTests, LLJitMixin): - pass + def test_jit_fii_vref(self): + from pypy.rlib import clibffi + from pypy.rlib.jit_libffi import jit_ffi_prep_cif, jit_ffi_call + + math_sin = intmask(ctypes.cast(ctypes.CDLL(None).sin, + ctypes.c_void_p).value) + math_sin = rffi.cast(rffi.VOIDP, math_sin) + + def f(): + cd = lltype.malloc(CIF_DESCRIPTION, 1, flavor='raw') + cd.abi = clibffi.FFI_DEFAULT_ABI + cd.nargs = 1 + cd.rtype = clibffi.cast_type_to_ffitype(rffi.DOUBLE) + atypes = lltype.malloc(clibffi.FFI_TYPE_PP.TO, 1, flavor='raw') + atypes[0] = clibffi.cast_type_to_ffitype(rffi.DOUBLE) + cd.atypes = atypes + cd.exchange_size = 64 # 64 bytes of exchange data + cd.exchange_result = 24 + cd.exchange_result_libffi = 24 + cd.exchange_args[0] = 16 + # + jit_ffi_prep_cif(cd) + # + assert rffi.sizeof(rffi.DOUBLE) == 8 + exb = lltype.malloc(rffi.DOUBLEP.TO, 8, flavor='raw') + exb[2] = 1.23 + jit_ffi_call(cd, math_sin, rffi.cast(rffi.CCHARP, exb)) + res = exb[3] + lltype.free(exb, flavor='raw') + # + lltype.free(atypes, flavor='raw') + lltype.free(cd, flavor='raw') + return res + # + res = self.interp_operations(f, []) + assert res == math.sin(1.23) diff --git a/pypy/rlib/jit_libffi.py b/pypy/rlib/jit_libffi.py --- a/pypy/rlib/jit_libffi.py +++ b/pypy/rlib/jit_libffi.py @@ -1,4 +1,4 @@ -import sys + from pypy.rpython.lltypesystem import lltype, rffi from pypy.rlib import clibffi, jit From noreply at buildbot.pypy.org Sat Jan 5 19:12:28 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 5 Jan 2013 19:12:28 +0100 (CET) Subject: [pypy-commit] pypy default: merge Message-ID: <20130105181228.C55C91C0DDA@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r59753:adb6b7eecfa3 Date: 2013-01-05 20:11 +0200 http://bitbucket.org/pypy/pypy/changeset/adb6b7eecfa3/ Log: merge diff --git a/pypy/rlib/rsre/rsre_core.py b/pypy/rlib/rsre/rsre_core.py --- a/pypy/rlib/rsre/rsre_core.py +++ b/pypy/rlib/rsre/rsre_core.py @@ -166,6 +166,8 @@ class StrMatchContext(AbstractMatchContext): """Concrete subclass for matching in a plain string.""" + _immutable_fields_ = ["_string"] + def __init__(self, pattern, string, match_start, end, flags): AbstractMatchContext.__init__(self, pattern, match_start, end, flags) self._string = string @@ -187,6 +189,8 @@ class UnicodeMatchContext(AbstractMatchContext): """Concrete subclass for matching in a unicode string.""" + _immutable_fields_ = ["_unicodestr"] + def __init__(self, pattern, unicodestr, match_start, end, flags): AbstractMatchContext.__init__(self, pattern, match_start, end, flags) self._unicodestr = unicodestr From noreply at buildbot.pypy.org Sat Jan 5 19:23:46 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 5 Jan 2013 19:23:46 +0100 (CET) Subject: [pypy-commit] pypy default: progress, but still failing Message-ID: <20130105182346.420121C0DDA@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r59754:08e2e4cd48c6 Date: 2013-01-05 20:23 +0200 http://bitbucket.org/pypy/pypy/changeset/08e2e4cd48c6/ Log: progress, but still failing diff --git a/pypy/jit/metainterp/test/test_fficall.py b/pypy/jit/metainterp/test/test_fficall.py --- a/pypy/jit/metainterp/test/test_fficall.py +++ b/pypy/jit/metainterp/test/test_fficall.py @@ -113,18 +113,19 @@ ctypes.c_void_p).value) math_sin = rffi.cast(rffi.VOIDP, math_sin) + cd = lltype.malloc(CIF_DESCRIPTION, 1, flavor='raw') + cd.abi = clibffi.FFI_DEFAULT_ABI + cd.nargs = 1 + cd.rtype = clibffi.cast_type_to_ffitype(rffi.DOUBLE) + atypes = lltype.malloc(clibffi.FFI_TYPE_PP.TO, 1, flavor='raw') + atypes[0] = clibffi.cast_type_to_ffitype(rffi.DOUBLE) + cd.atypes = atypes + cd.exchange_size = 64 # 64 bytes of exchange data + cd.exchange_result = 24 + cd.exchange_result_libffi = 24 + cd.exchange_args[0] = 16 + def f(): - cd = lltype.malloc(CIF_DESCRIPTION, 1, flavor='raw') - cd.abi = clibffi.FFI_DEFAULT_ABI - cd.nargs = 1 - cd.rtype = clibffi.cast_type_to_ffitype(rffi.DOUBLE) - atypes = lltype.malloc(clibffi.FFI_TYPE_PP.TO, 1, flavor='raw') - atypes[0] = clibffi.cast_type_to_ffitype(rffi.DOUBLE) - cd.atypes = atypes - cd.exchange_size = 64 # 64 bytes of exchange data - cd.exchange_result = 24 - cd.exchange_result_libffi = 24 - cd.exchange_args[0] = 16 # jit_ffi_prep_cif(cd) # @@ -136,8 +137,8 @@ lltype.free(exb, flavor='raw') # lltype.free(atypes, flavor='raw') - lltype.free(cd, flavor='raw') return res # res = self.interp_operations(f, []) + lltype.free(cd, flavor='raw') assert res == math.sin(1.23) From noreply at buildbot.pypy.org Sat Jan 5 19:33:08 2013 From: noreply at buildbot.pypy.org (wlav) Date: Sat, 5 Jan 2013 19:33:08 +0100 (CET) Subject: [pypy-commit] pypy reflex-support: improve simple benchmarking (and shows problem with new objects) Message-ID: <20130105183308.196641C0DDA@cobra.cs.uni-duesseldorf.de> Author: Wim Lavrijsen Branch: reflex-support Changeset: r59755:b8153a9f359c Date: 2012-12-19 09:45 -0800 http://bitbucket.org/pypy/pypy/changeset/b8153a9f359c/ Log: improve simple benchmarking (and shows problem with new objects) diff --git a/pypy/module/cppyy/test/bench1.py b/pypy/module/cppyy/test/bench1.py --- a/pypy/module/cppyy/test/bench1.py +++ b/pypy/module/cppyy/test/bench1.py @@ -4,14 +4,14 @@ def run_bench(bench): - global t_loop_offset + global t_loop_offset, NNN t1 = time.time() - bench() + bench(NNN) t2 = time.time() - t_bench = (t2-t1)-t_loop_offset - return bench.scale*t_bench + t_bench = (t2-t1) + return bench.scale*t_bench-t_loop_offset def print_bench(name, t_bench): global t_cppref @@ -23,7 +23,7 @@ return i class PyCintexBench1(object): - scale = 10 + scale = 5 def __init__(self): import PyCintex self.lib = PyCintex.gbl.gSystem.Load("./example01Dict.so") @@ -31,11 +31,11 @@ self.cls = PyCintex.gbl.example01 self.inst = self.cls(0) - def __call__(self): + def __call__(self, repeat): # note that PyCintex calls don't actually scale linearly, but worse # than linear (leak or wrong filling of a cache??) instance = self.inst - niter = NNN/self.scale + niter = repeat/self.scale for i in range(niter): instance.addDataToInt(i) return i @@ -43,12 +43,13 @@ class PyROOTBench1(PyCintexBench1): def __init__(self): import ROOT - self.lib = ROOT.gSystem.Load("./example01Dict_cint.so") + self.lib = ROOT.gSystem.Load("./example01Dict.so") self.cls = ROOT.example01 self.inst = self.cls(0) class CppyyInterpBench1(object): + title = "cppyy interp" scale = 1 def __init__(self): import cppyy @@ -57,30 +58,33 @@ self.cls = cppyy._scope_byname("example01") self.inst = self.cls.get_overload(self.cls.type_name).call(None, 0) - def __call__(self): + def __call__(self, repeat): addDataToInt = self.cls.get_overload("addDataToInt") instance = self.inst - for i in range(NNN): + for i in range(repeat): addDataToInt.call(instance, i) return i class CppyyInterpBench2(CppyyInterpBench1): - def __call__(self): + title = "... overload" + def __call__(self, repeat): addDataToInt = self.cls.get_overload("overloadedAddDataToInt") instance = self.inst - for i in range(NNN): + for i in range(repeat): addDataToInt.call(instance, i) return i class CppyyInterpBench3(CppyyInterpBench1): - def __call__(self): + title = "... constref" + def __call__(self, repeat): addDataToInt = self.cls.get_overload("addDataToIntConstRef") instance = self.inst - for i in range(NNN): + for i in range(repeat): addDataToInt.call(instance, i) return i class CppyyPythonBench1(object): + title = "cppyy python" scale = 1 def __init__(self): import cppyy @@ -89,12 +93,34 @@ self.cls = cppyy.gbl.example01 self.inst = self.cls(0) - def __call__(self): + def __call__(self, repeat): instance = self.inst - for i in range(NNN): + for i in range(repeat): instance.addDataToInt(i) return i +class CppyyPythonBench2(CppyyPythonBench1): + title = "... objbyval" + def __call__(self, repeat): + import cppyy + pl = cppyy.gbl.payload(3.14) + + instance = self.inst + for i in range(repeat): + instance.copyCyclePayload(pl) + return i + +class CppyyPythonBench3(CppyyPythonBench1): + title = "... objbyptr" + def __call__(self, repeat): + import cppyy + pl = cppyy.gbl.payload(3.14) + + instance = self.inst + for i in range(repeat): + instance.cyclePayload(pl) + return i + if __name__ == '__main__': python_loop_offset(); @@ -111,7 +137,7 @@ print run_bench(cintex_bench1) sys.exit(0) - # special case for PyCintex (run under python, not pypy-c) + # special case for PyROOT (run under python, not pypy-c) if '--pyroot' in sys.argv: pyroot_bench1 = PyROOTBench1() print run_bench(pyroot_bench1) @@ -125,22 +151,22 @@ stat, cppref = commands.getstatusoutput("./bench1.exe") t_cppref = float(cppref) + # created object + benches = [ + CppyyInterpBench1(), CppyyInterpBench2(), CppyyInterpBench3(), + CppyyPythonBench1(), CppyyPythonBench2(), CppyyPythonBench3() ] + # warm-up print "warming up ... " - interp_bench1 = CppyyInterpBench1() - interp_bench2 = CppyyInterpBench2() - interp_bench3 = CppyyInterpBench3() - python_bench1 = CppyyPythonBench1() - interp_bench1(); interp_bench2(); python_bench1() + for bench in benches: + bench(2000) # to allow some consistency checking print "C++ reference uses %.3fs" % t_cppref # test runs ... - print_bench("cppyy interp", run_bench(interp_bench1)) - print_bench("... overload", run_bench(interp_bench2)) - print_bench("... constref", run_bench(interp_bench3)) - print_bench("cppyy python", run_bench(python_bench1)) + for bench in benches: + print_bench(bench.title, run_bench(bench)) stat, t_cintex = commands.getstatusoutput("python bench1.py --pycintex") print_bench("pycintex ", float(t_cintex)) #stat, t_pyroot = commands.getstatusoutput("python bench1.py --pyroot") From noreply at buildbot.pypy.org Sat Jan 5 19:33:09 2013 From: noreply at buildbot.pypy.org (wlav) Date: Sat, 5 Jan 2013 19:33:09 +0100 (CET) Subject: [pypy-commit] pypy reflex-support: simplifications Message-ID: <20130105183309.3FAFA1C0DDA@cobra.cs.uni-duesseldorf.de> Author: Wim Lavrijsen Branch: reflex-support Changeset: r59756:56c486ea1203 Date: 2012-12-21 16:18 -0800 http://bitbucket.org/pypy/pypy/changeset/56c486ea1203/ Log: simplifications diff --git a/pypy/module/cppyy/interp_cppyy.py b/pypy/module/cppyy/interp_cppyy.py --- a/pypy/module/cppyy/interp_cppyy.py +++ b/pypy/module/cppyy/interp_cppyy.py @@ -88,7 +88,6 @@ state = space.fromcache(State) state.w_clgen_callback = w_callback - at unwrap_spec(w_pycppclass=W_Root) def register_class(space, w_pycppclass): w_cppclass = space.findattr(w_pycppclass, space.wrap("_cpp_proxy")) cppclass = space.interp_w(W_CPPClass, w_cppclass, can_be_None=False) @@ -1026,17 +1025,17 @@ final_name = capi.c_scoped_final_name(handle) # the callback will cache the class by calling register_class w_pycppclass = space.call_function(state.w_clgen_callback, space.wrap(final_name)) + assert w_pycppclass is state.cppclass_registry[handle] return w_pycppclass def wrap_new_cppobject_nocast(space, w_pycppclass, cppclass, rawobject, isref, python_owns): rawobject = rffi.cast(capi.C_OBJECT, rawobject) if space.is_w(w_pycppclass, space.w_None): w_pycppclass = get_pythonized_cppclass(space, cppclass.handle) - w_cppinstance = space.allocate_instance(W_CPPInstance, w_pycppclass) - cppinstance = space.interp_w(W_CPPInstance, w_cppinstance, can_be_None=False) + cppinstance = space.allocate_instance(W_CPPInstance, w_pycppclass) W_CPPInstance.__init__(cppinstance, space, cppclass, rawobject, isref, python_owns) memory_regulator.register(cppinstance) - return w_cppinstance + return space.wrap(cppinstance) def wrap_cppobject_nocast(space, w_pycppclass, cppclass, rawobject, isref, python_owns): rawobject = rffi.cast(capi.C_OBJECT, rawobject) diff --git a/pypy/module/cppyy/pythonify.py b/pypy/module/cppyy/pythonify.py --- a/pypy/module/cppyy/pythonify.py +++ b/pypy/module/cppyy/pythonify.py @@ -18,11 +18,14 @@ def __dir__(cls): return cls._cpp_proxy.__dir__() -class CppyyClass(CppyyScopeMeta): +class CppyyClassMeta(CppyyScopeMeta): pass -class CPPObject(cppyy.CPPInstance): - __metaclass__ = CppyyClass +class CppyyClass(cppyy.CPPInstance): + __metaclass__ = CppyyClassMeta + + def __init__(self, *args, **kwds): + pass # ignored, for the C++ backend, ctor == __new__ + __init__ class CppyyTemplateType(object): @@ -155,7 +158,7 @@ # get a list of base classes for class creation bases = [get_pycppclass(base) for base in cppclass.get_base_names()] if not bases: - bases = [CPPObject,] + bases = [CppyyClass,] else: # it's technically possible that the required class now has been built # if one of the base classes uses it in e.g. a function interface @@ -166,7 +169,7 @@ # create a meta class to allow properties (for static data write access) metabases = [type(base) for base in bases] - metacpp = type(CppyyClass)(class_name+'_meta', _drop_cycles(metabases), {}) + metacpp = type(CppyyClassMeta)(class_name+'_meta', _drop_cycles(metabases), {}) # create the python-side C++ class representation def dispatch(self, name, signature): From noreply at buildbot.pypy.org Sat Jan 5 19:33:10 2013 From: noreply at buildbot.pypy.org (wlav) Date: Sat, 5 Jan 2013 19:33:10 +0100 (CET) Subject: [pypy-commit] pypy reflex-support: never release GIL with CINT backend Message-ID: <20130105183310.5E4BE1C0DDA@cobra.cs.uni-duesseldorf.de> Author: Wim Lavrijsen Branch: reflex-support Changeset: r59757:a3856c2e0535 Date: 2013-01-04 15:55 -0800 http://bitbucket.org/pypy/pypy/changeset/a3856c2e0535/ Log: never release GIL with CINT backend diff --git a/pypy/module/cppyy/capi/cint_capi.py b/pypy/module/cppyy/capi/cint_capi.py --- a/pypy/module/cppyy/capi/cint_capi.py +++ b/pypy/module/cppyy/capi/cint_capi.py @@ -36,8 +36,8 @@ ts_reflect = False ts_call = False -ts_memory = 'auto' -ts_helper = 'auto' +ts_memory = False +ts_helper = False # force loading in global mode of core libraries, rather than linking with # them as PyPy uses various version of dlopen in various places; note that From noreply at buildbot.pypy.org Sat Jan 5 19:33:11 2013 From: noreply at buildbot.pypy.org (wlav) Date: Sat, 5 Jan 2013 19:33:11 +0100 (CET) Subject: [pypy-commit] pypy reflex-support: o) tests and fix for shadowed data members Message-ID: <20130105183311.76A271C0DDA@cobra.cs.uni-duesseldorf.de> Author: Wim Lavrijsen Branch: reflex-support Changeset: r59758:4024dbe01c52 Date: 2013-01-04 19:10 -0800 http://bitbucket.org/pypy/pypy/changeset/4024dbe01c52/ Log: o) tests and fix for shadowed data members o) some cleanup diff --git a/pypy/module/cppyy/interp_cppyy.py b/pypy/module/cppyy/interp_cppyy.py --- a/pypy/module/cppyy/interp_cppyy.py +++ b/pypy/module/cppyy/interp_cppyy.py @@ -9,7 +9,6 @@ from pypy.rlib import jit, rdynload, rweakref from pypy.rlib import jit_libffi, clibffi -from pypy.rlib.objectmodel import we_are_translated from pypy.module.cppyy import converter, executor, helper @@ -1025,17 +1024,17 @@ final_name = capi.c_scoped_final_name(handle) # the callback will cache the class by calling register_class w_pycppclass = space.call_function(state.w_clgen_callback, space.wrap(final_name)) - assert w_pycppclass is state.cppclass_registry[handle] return w_pycppclass def wrap_new_cppobject_nocast(space, w_pycppclass, cppclass, rawobject, isref, python_owns): rawobject = rffi.cast(capi.C_OBJECT, rawobject) if space.is_w(w_pycppclass, space.w_None): w_pycppclass = get_pythonized_cppclass(space, cppclass.handle) - cppinstance = space.allocate_instance(W_CPPInstance, w_pycppclass) - W_CPPInstance.__init__(cppinstance, space, cppclass, rawobject, isref, python_owns) + w_cppinstance = space.allocate_instance(W_CPPInstance, w_pycppclass) + cppinstance = space.interp_w(W_CPPInstance, w_cppinstance) + cppinstance.__init__(space, cppclass, rawobject, isref, python_owns) memory_regulator.register(cppinstance) - return space.wrap(cppinstance) + return w_cppinstance def wrap_cppobject_nocast(space, w_pycppclass, cppclass, rawobject, isref, python_owns): rawobject = rffi.cast(capi.C_OBJECT, rawobject) @@ -1049,11 +1048,17 @@ if rawobject: actual = capi.c_actual_class(cppclass, rawobject) if actual != cppclass.handle: - offset = capi._c_base_offset(actual, cppclass.handle, rawobject, -1) - rawobject = capi.direct_ptradd(rawobject, offset) - w_pycppclass = get_pythonized_cppclass(space, actual) - w_cppclass = space.findattr(w_pycppclass, space.wrap("_cpp_proxy")) - cppclass = space.interp_w(W_CPPClass, w_cppclass, can_be_None=False) + try: + w_pycppclass = get_pythonized_cppclass(space, actual) + offset = capi._c_base_offset(actual, cppclass.handle, rawobject, -1) + rawobject = capi.direct_ptradd(rawobject, offset) + w_cppclass = space.findattr(w_pycppclass, space.wrap("_cpp_proxy")) + cppclass = space.interp_w(W_CPPClass, w_cppclass, can_be_None=False) + except Exception: + # failed to locate/build the derived class, so stick to the base (note + # that only get_pythonized_cppclass is expected to raise, so none of + # the variables are re-assigned yet) + pass return wrap_cppobject_nocast(space, w_pycppclass, cppclass, rawobject, isref, python_owns) @unwrap_spec(cppinstance=W_CPPInstance) diff --git a/pypy/module/cppyy/pythonify.py b/pypy/module/cppyy/pythonify.py --- a/pypy/module/cppyy/pythonify.py +++ b/pypy/module/cppyy/pythonify.py @@ -11,8 +11,9 @@ def __getattr__(self, name): try: return get_pycppitem(self, name) # will cache on self - except TypeError, t: - raise AttributeError("%s object has no attribute '%s'" % (self, name)) + except Exception, e: + raise AttributeError("%s object has no attribute '%s' (details: %s)" % + (self, name, str(e))) class CppyyNamespaceMeta(CppyyScopeMeta): def __dir__(cls): @@ -198,9 +199,12 @@ cppdm = cppclass.get_datamember(dm_name) pydm = make_datamember(cppdm) - setattr(pycppclass, dm_name, pydm) + # here, setattr() can not be used, because a data member can shadow one in + # its base class, resulting in the __set__() of its base class being called + # by setattr(); so, store directly on the dictionary + pycppclass.__dict__[dm_name] = pydm if cppdm.is_static(): - setattr(metacpp, dm_name, pydm) + metacpp.__dict__[dm_name] = pydm # the call to register will add back-end specific pythonizations and thus # needs to run first, so that the generic pythonizations can use them diff --git a/pypy/module/cppyy/test/fragile.cxx b/pypy/module/cppyy/test/fragile.cxx --- a/pypy/module/cppyy/test/fragile.cxx +++ b/pypy/module/cppyy/test/fragile.cxx @@ -9,3 +9,31 @@ void fragile::fglobal(int, double, char) { /* empty; only used for doc-string testing */ } + +namespace fragile { + + class Kderived : public K { + public: + virtual ~Kderived(); + }; + +} // namespace fragile + +fragile::Kderived::~Kderived() {} + +fragile::K::~K() {} + +fragile::K* fragile::K::GimeK(bool derived) { + if (!derived) return this; + else { + static Kderived kd; + return &kd; + } +}; + +fragile::K* fragile::K::GimeL() { + static L l; + return &l; +} + +fragile::L::~L() {} diff --git a/pypy/module/cppyy/test/fragile.h b/pypy/module/cppyy/test/fragile.h --- a/pypy/module/cppyy/test/fragile.h +++ b/pypy/module/cppyy/test/fragile.h @@ -87,4 +87,28 @@ } // namespace nested2 } // namespace nested1 +class K { +public: + virtual ~K(); + K* GimeK(bool derived); + K* GimeL(); +}; + +class L : public K { +public: + virtual ~L(); + no_such_class* m_no_such; +}; + +class M { +public: + enum E1 { kOnce=42 }; + enum E2 { kTwice=12 }; +}; + +class N : public M { +public: + enum E2 { kTwice=12 }; +}; + } // namespace fragile diff --git a/pypy/module/cppyy/test/fragile_LinkDef.h b/pypy/module/cppyy/test/fragile_LinkDef.h --- a/pypy/module/cppyy/test/fragile_LinkDef.h +++ b/pypy/module/cppyy/test/fragile_LinkDef.h @@ -19,6 +19,10 @@ #pragma link C++ class fragile::H; #pragma link C++ class fragile::I; #pragma link C++ class fragile::J; +#pragma link C++ class fragile::K; +#pragma link C++ class fragile::L; +#pragma link C++ class fragile::M; +#pragma link C++ class fragile::N; #pragma link C++ class fragile::nested1::A; #pragma link C++ class fragile::nested1::nested2::A; #pragma link C++ class fragile::nested1::nested2::nested3::A; diff --git a/pypy/module/cppyy/test/test_cint.py b/pypy/module/cppyy/test/test_cint.py --- a/pypy/module/cppyy/test/test_cint.py +++ b/pypy/module/cppyy/test/test_cint.py @@ -254,7 +254,7 @@ def test05_branch_activation(self): """Test of automatic branch activation""" - from cppyy import gbl # bootstraps, only needed for tests + from cppyy import gbl from cppyy.gbl import TFile, TTree from cppyy.gbl.std import vector @@ -306,3 +306,19 @@ i += 1 assert i == self.N + +class AppTestRegression: + spaceconfig = dict(usemodules=['cppyy']) + + def test01_regression(self): + """TPaveText::AddText() used to result in KeyError""" + + # This is where the original problem was discovered, and the test is + # left in. However, the detailed underlying problem, as well as the + # solution to it, is tested in test_fragile.py + + from cppyy import gbl + from cppyy.gbl import TPaveText + + hello = TPaveText( .1, .8, .9, .97 ) + hello.AddText( 'Hello, World!' ) diff --git a/pypy/module/cppyy/test/test_fragile.py b/pypy/module/cppyy/test/test_fragile.py --- a/pypy/module/cppyy/test/test_fragile.py +++ b/pypy/module/cppyy/test/test_fragile.py @@ -255,3 +255,32 @@ from cppyy.gbl.fragile.nested1.nested2.nested3 import A assert cppyy.gbl.fragile.nested1.nested2.nested3.A is nested3.A + + def test13_missing_casts(self): + """Test proper handling when a hierarchy is not fully available""" + + import cppyy + + k = cppyy.gbl.fragile.K() + + assert k is k.GimeK(False) + assert k is not k.GimeK(True) + + kd = k.GimeK(True) + assert kd is k.GimeK(True) + assert kd is not k.GimeK(False) + + l = k.GimeL() + assert l is k.GimeL() + + def test14_double_enum_trouble(self): + """Test a redefinition of enum in a derived class""" + + import cppyy + + M = cppyy.gbl.fragile.M + N = cppyy.gbl.fragile.N + + assert M.kOnce == N.kOnce + assert M.kTwice == N.kTwice + assert M.__dict__['kTwice'] is not N.__dict__['kTwice'] From noreply at buildbot.pypy.org Sat Jan 5 19:33:12 2013 From: noreply at buildbot.pypy.org (wlav) Date: Sat, 5 Jan 2013 19:33:12 +0100 (CET) Subject: [pypy-commit] pypy reflex-support: simplify and improve consistency Message-ID: <20130105183312.9DE431C0DDA@cobra.cs.uni-duesseldorf.de> Author: Wim Lavrijsen Branch: reflex-support Changeset: r59759:b8823f459324 Date: 2013-01-04 23:44 -0800 http://bitbucket.org/pypy/pypy/changeset/b8823f459324/ Log: simplify and improve consistency diff --git a/pypy/module/cppyy/__init__.py b/pypy/module/cppyy/__init__.py --- a/pypy/module/cppyy/__init__.py +++ b/pypy/module/cppyy/__init__.py @@ -12,6 +12,7 @@ '_template_byname' : 'interp_cppyy.template_byname', '_set_class_generator' : 'interp_cppyy.set_class_generator', '_register_class' : 'interp_cppyy.register_class', + '_is_static' : 'interp_cppyy.is_static', 'CPPInstance' : 'interp_cppyy.W_CPPInstance', 'addressof' : 'interp_cppyy.addressof', 'bind_object' : 'interp_cppyy.bind_object', diff --git a/pypy/module/cppyy/converter.py b/pypy/module/cppyy/converter.py --- a/pypy/module/cppyy/converter.py +++ b/pypy/module/cppyy/converter.py @@ -450,7 +450,7 @@ address = rffi.cast(capi.C_OBJECT, self._get_raw_address(space, w_obj, offset)) from pypy.module.cppyy import interp_cppyy return interp_cppyy.wrap_cppobject_nocast( - space, w_pycppclass, self.cppclass, address, isref=True, python_owns=False) + space, space.w_None, self.cppclass, address, isref=True, python_owns=False) def to_memory(self, space, w_obj, w_value, offset): address = rffi.cast(rffi.VOIDPP, self._get_raw_address(space, w_obj, offset)) @@ -466,7 +466,7 @@ address = rffi.cast(capi.C_OBJECT, self._get_raw_address(space, w_obj, offset)) from pypy.module.cppyy import interp_cppyy return interp_cppyy.wrap_cppobject_nocast( - space, w_pycppclass, self.cppclass, address, isref=False, python_owns=False) + space, space.w_None, self.cppclass, address, isref=False, python_owns=False) def to_memory(self, space, w_obj, w_value, offset): self._is_abstract(space) diff --git a/pypy/module/cppyy/interp_cppyy.py b/pypy/module/cppyy/interp_cppyy.py --- a/pypy/module/cppyy/interp_cppyy.py +++ b/pypy/module/cppyy/interp_cppyy.py @@ -493,22 +493,18 @@ class W_CPPDataMember(Wrappable): - _attrs_ = ['space', 'scope', 'converter', 'offset', '_is_static'] - _immutable_fields = ['scope', 'converter', 'offset', '_is_static'] + _attrs_ = ['space', 'scope', 'converter', 'offset'] + _immutable_fields = ['scope', 'converter', 'offset'] - def __init__(self, space, containing_scope, type_name, offset, is_static): + def __init__(self, space, containing_scope, type_name, offset): self.space = space self.scope = containing_scope self.converter = converter.get_converter(self.space, type_name, '') self.offset = offset - self._is_static = is_static def get_returntype(self): return self.space.wrap(self.converter.name) - def is_static(self): - return self.space.newbool(self._is_static) - @jit.elidable_promote() def _get_offset(self, cppinstance): if cppinstance: @@ -532,13 +528,37 @@ W_CPPDataMember.typedef = TypeDef( 'CPPDataMember', - is_static = interp2app(W_CPPDataMember.is_static), get_returntype = interp2app(W_CPPDataMember.get_returntype), - get = interp2app(W_CPPDataMember.get), - set = interp2app(W_CPPDataMember.set), + __get__ = interp2app(W_CPPDataMember.get), + __set__ = interp2app(W_CPPDataMember.set), ) W_CPPDataMember.typedef.acceptable_as_base_class = False +class W_CPPStaticData(W_CPPDataMember): + def _get_offset(self): + return self.offset + + def get(self, w_cppinstance, w_pycppclass): + return self.converter.from_memory(self.space, self.space.w_None, w_pycppclass, self.offset) + + def set(self, w_cppinstance, w_value): + self.converter.to_memory(self.space, self.space.w_None, w_value, self.offset) + return self.space.w_None + +W_CPPStaticData.typedef = TypeDef( + 'CPPStaticData', + get_returntype = interp2app(W_CPPStaticData.get_returntype), + __get__ = interp2app(W_CPPStaticData.get), + __set__ = interp2app(W_CPPStaticData.set), +) +W_CPPStaticData.typedef.acceptable_as_base_class = False + +def is_static(space, w_obj): + try: + space.interp_w(W_CPPStaticData, w_obj, can_be_None=False) + return space.w_True + except Exception: + return space.w_False class W_CPPScope(Wrappable): _attrs_ = ['space', 'name', 'handle', 'methods', 'datamembers'] @@ -659,7 +679,7 @@ def _make_datamember(self, dm_name, dm_idx): type_name = capi.c_datamember_type(self, dm_idx) offset = capi.c_datamember_offset(self, dm_idx) - datamember = W_CPPDataMember(self.space, self, type_name, offset, True) + datamember = W_CPPStaticData(self.space, self, type_name, offset) self.datamembers[dm_name] = datamember return datamember @@ -769,7 +789,10 @@ type_name = capi.c_datamember_type(self, i) offset = capi.c_datamember_offset(self, i) is_static = bool(capi.c_is_staticdata(self, i)) - datamember = W_CPPDataMember(self.space, self, type_name, offset, is_static) + if is_static: + datamember = W_CPPStaticData(self.space, self, type_name, offset) + else: + datamember = W_CPPDataMember(self.space, self, type_name, offset) self.datamembers[datamember_name] = datamember def construct(self): diff --git a/pypy/module/cppyy/pythonify.py b/pypy/module/cppyy/pythonify.py --- a/pypy/module/cppyy/pythonify.py +++ b/pypy/module/cppyy/pythonify.py @@ -68,30 +68,6 @@ return method -def make_datamember(cppdm): - rettype = cppdm.get_returntype() - if not rettype: # return builtin type - cppclass = None - else: # return instance - try: - cppclass = get_pycppclass(rettype) - except AttributeError: - import warnings - warnings.warn("class %s unknown: no data member access" % rettype, - RuntimeWarning) - cppclass = None - if cppdm.is_static(): - def binder(obj): - return cppdm.get(None, cppclass) - def setter(obj, value): - return cppdm.set(None, value) - else: - def binder(obj): - return cppdm.get(obj, cppclass) - setter = cppdm.set - return property(binder, setter) - - def make_cppnamespace(scope, namespace_name, cppns, build_in_full=True): # build up a representation of a C++ namespace (namespaces are classes) @@ -123,11 +99,10 @@ # add all data members to the dictionary of the class to be created, and # static ones also to the meta class (needed for property setters) - for dm in cppns.get_datamember_names(): - cppdm = cppns.get_datamember(dm) - pydm = make_datamember(cppdm) - setattr(pycppns, dm, pydm) - setattr(metans, dm, pydm) + for dm_name in cppns.get_datamember_names(): + cppdm = cppns.get_datamember(dm_name) + setattr(pycppns, dm_name, cppdm) + setattr(metans, dm_name, cppdm) modname = pycppns.__name__.replace('::', '.') sys.modules['cppyy.gbl.'+modname] = pycppns @@ -197,14 +172,13 @@ # static ones also to the meta class (needed for property setters) for dm_name in cppclass.get_datamember_names(): cppdm = cppclass.get_datamember(dm_name) - pydm = make_datamember(cppdm) # here, setattr() can not be used, because a data member can shadow one in # its base class, resulting in the __set__() of its base class being called # by setattr(); so, store directly on the dictionary - pycppclass.__dict__[dm_name] = pydm - if cppdm.is_static(): - metacpp.__dict__[dm_name] = pydm + pycppclass.__dict__[dm_name] = cppdm + if cppyy._is_static(cppdm): + metacpp.__dict__[dm_name] = cppdm # the call to register will add back-end specific pythonizations and thus # needs to run first, so that the generic pythonizations can use them @@ -254,11 +228,10 @@ # data if not cppitem: try: - cppitem = scope._cpp_proxy.get_datamember(name) - pycppitem = make_datamember(cppitem) - setattr(scope, name, pycppitem) - if cppitem.is_static(): - setattr(scope.__class__, name, pycppitem) + cppdm = scope._cpp_proxy.get_datamember(name) + setattr(scope, name, cppdm) + if cppyy._is_static(cppdm): + setattr(scope.__class__, name, cppdm) pycppitem = getattr(scope, name) # gets actual property value except AttributeError: pass diff --git a/pypy/module/cppyy/test/advancedcpp.h b/pypy/module/cppyy/test/advancedcpp.h --- a/pypy/module/cppyy/test/advancedcpp.h +++ b/pypy/module/cppyy/test/advancedcpp.h @@ -155,7 +155,7 @@ class T1 { public: T1(T t = T(1)) : m_t1(t) {} - T value() { return m_t1; } + T get_value() { return m_t1; } public: T m_t1; @@ -165,7 +165,7 @@ class T2 { public: T2(T t = T(2)) : m_t2(t) {} - T value() { return m_t2; } + T get_value() { return m_t2; } public: T m_t2; @@ -175,8 +175,8 @@ class T3 { public: T3(T t = T(3), U u = U(33)) : m_t3(t), m_u3(u) {} - T value_t() { return m_t3; } - U value_u() { return m_u3; } + T get_value_t() { return m_t3; } + U get_value_u() { return m_u3; } public: T m_t3; @@ -189,7 +189,7 @@ class T4 { public: T4(T t = T(4)) : m_t4(t) {} - T value() { return m_t4; } + T get_value() { return m_t4; } public: T m_t4; diff --git a/pypy/module/cppyy/test/test_advancedcpp.py b/pypy/module/cppyy/test/test_advancedcpp.py --- a/pypy/module/cppyy/test/test_advancedcpp.py +++ b/pypy/module/cppyy/test/test_advancedcpp.py @@ -204,24 +204,24 @@ #----- t1 = gbl.T1(int)() - assert t1.m_t1 == 1 - assert t1.value() == 1 + assert t1.m_t1 == 1 + assert t1.get_value() == 1 t1.destruct() #----- t1 = gbl.T1(int)(11) - assert t1.m_t1 == 11 - assert t1.value() == 11 + assert t1.m_t1 == 11 + assert t1.get_value() == 11 t1.m_t1 = 111 - assert t1.value() == 111 - assert t1.m_t1 == 111 + assert t1.get_value() == 111 + assert t1.m_t1 == 111 t1.destruct() #----- t2 = gbl.T2(gbl.T1(int))(gbl.T1(int)(32)) t2.m_t2.m_t1 = 32 - assert t2.m_t2.value() == 32 - assert t2.m_t2.m_t1 == 32 + assert t2.m_t2.get_value() == 32 + assert t2.m_t2.m_t1 == 32 t2.destruct() def test05_abstract_classes(self): diff --git a/pypy/module/cppyy/test/test_pythonify.py b/pypy/module/cppyy/test/test_pythonify.py --- a/pypy/module/cppyy/test/test_pythonify.py +++ b/pypy/module/cppyy/test/test_pythonify.py @@ -365,8 +365,8 @@ oldval = cppyy.gbl.ns_example01.gMyGlobalInt assert oldval == 99 - proxy = cppyy.gbl.ns_example01.__class__.gMyGlobalInt + proxy = cppyy.gbl.ns_example01.__class__.__dict__['gMyGlobalInt'] cppyy.gbl.ns_example01.gMyGlobalInt = 3 - assert proxy.__get__(proxy) == 3 + assert proxy.__get__(proxy, None) == 3 cppyy.gbl.ns_example01.gMyGlobalInt = oldval From noreply at buildbot.pypy.org Sat Jan 5 19:33:13 2013 From: noreply at buildbot.pypy.org (wlav) Date: Sat, 5 Jan 2013 19:33:13 +0100 (CET) Subject: [pypy-commit] pypy reflex-support: fixups for CINT backend after last cleanup Message-ID: <20130105183313.B8D621C0DDA@cobra.cs.uni-duesseldorf.de> Author: Wim Lavrijsen Branch: reflex-support Changeset: r59760:3129f9b33e37 Date: 2013-01-05 00:15 -0800 http://bitbucket.org/pypy/pypy/changeset/3129f9b33e37/ Log: fixups for CINT backend after last cleanup diff --git a/pypy/module/cppyy/capi/cint_capi.py b/pypy/module/cppyy/capi/cint_capi.py --- a/pypy/module/cppyy/capi/cint_capi.py +++ b/pypy/module/cppyy/capi/cint_capi.py @@ -166,6 +166,8 @@ # setup branch as a data member and enable it for reading space = tree.space # holds the class cache in State w_branch = space.call_method(w_self, "GetBranch", args_w[0]) + if not space.is_true(w_branch): + raise OperationError(space.w_AttributeError, args_w[0]) w_klassname = space.call_method(w_branch, "GetClassName") klass = interp_cppyy.scope_byname(space, space.str_w(w_klassname)) w_obj = klass.construct() @@ -178,7 +180,6 @@ class W_TTreeIter(Wrappable): def __init__(self, space, w_tree): - from pypy.module.cppyy import interp_cppyy tree = space.interp_w(interp_cppyy.W_CPPInstance, w_tree) self.tree = tree.get_cppthis(tree.cppclass) diff --git a/pypy/module/cppyy/test/test_cint.py b/pypy/module/cppyy/test/test_cint.py --- a/pypy/module/cppyy/test/test_cint.py +++ b/pypy/module/cppyy/test/test_cint.py @@ -41,9 +41,9 @@ oldval = cppyy.gbl.gDebug assert oldval != 3 - proxy = cppyy.gbl.__class__.gDebug + proxy = cppyy.gbl.__class__.__dict__['gDebug'] cppyy.gbl.gDebug = 3 - assert proxy.__get__(proxy) == 3 + assert proxy.__get__(proxy, None) == 3 # this is where this test differs from test03_write_access_to_globals # in test_pythonify.py @@ -60,8 +60,8 @@ cppyy.gbl.gROOT.ProcessLine('double gMyOwnGlobal = 3.1415') assert cppyy.gbl.gMyOwnGlobal == 3.1415 - proxy = cppyy.gbl.__class__.gMyOwnGlobal - assert proxy.__get__(proxy) == 3.1415 + proxy = cppyy.gbl.__class__.__dict__['gMyOwnGlobal'] + assert proxy.__get__(proxy, None) == 3.1415 def test04_auto_loading(self): """Test auto-loading by retrieving a non-preloaded class""" From noreply at buildbot.pypy.org Sat Jan 5 19:33:14 2013 From: noreply at buildbot.pypy.org (wlav) Date: Sat, 5 Jan 2013 19:33:14 +0100 (CET) Subject: [pypy-commit] pypy reflex-support: simplification of interface for bringing cpp objects from interpreter to app Message-ID: <20130105183314.D4AB71C0DDA@cobra.cs.uni-duesseldorf.de> Author: Wim Lavrijsen Branch: reflex-support Changeset: r59761:1ce8a3559f72 Date: 2013-01-05 00:52 -0800 http://bitbucket.org/pypy/pypy/changeset/1ce8a3559f72/ Log: simplification of interface for bringing cpp objects from interpreter to app diff --git a/pypy/module/cppyy/converter.py b/pypy/module/cppyy/converter.py --- a/pypy/module/cppyy/converter.py +++ b/pypy/module/cppyy/converter.py @@ -449,8 +449,8 @@ def from_memory(self, space, w_obj, w_pycppclass, offset): address = rffi.cast(capi.C_OBJECT, self._get_raw_address(space, w_obj, offset)) from pypy.module.cppyy import interp_cppyy - return interp_cppyy.wrap_cppobject_nocast( - space, space.w_None, self.cppclass, address, isref=True, python_owns=False) + return interp_cppyy.wrap_cppobject(space, address, self.cppclass, + do_cast=False, is_ref=True) def to_memory(self, space, w_obj, w_value, offset): address = rffi.cast(rffi.VOIDPP, self._get_raw_address(space, w_obj, offset)) @@ -465,8 +465,7 @@ def from_memory(self, space, w_obj, w_pycppclass, offset): address = rffi.cast(capi.C_OBJECT, self._get_raw_address(space, w_obj, offset)) from pypy.module.cppyy import interp_cppyy - return interp_cppyy.wrap_cppobject_nocast( - space, space.w_None, self.cppclass, address, isref=False, python_owns=False) + return interp_cppyy.wrap_cppobject(space, address, self.cppclass, do_cast=False) def to_memory(self, space, w_obj, w_value, offset): self._is_abstract(space) diff --git a/pypy/module/cppyy/executor.py b/pypy/module/cppyy/executor.py --- a/pypy/module/cppyy/executor.py +++ b/pypy/module/cppyy/executor.py @@ -159,16 +159,14 @@ from pypy.module.cppyy import interp_cppyy long_result = capi.c_call_l(cppmethod, cppthis, num_args, args) ptr_result = rffi.cast(capi.C_OBJECT, long_result) - return interp_cppyy.wrap_cppobject( - space, space.w_None, self.cppclass, ptr_result, isref=False, python_owns=False) + return interp_cppyy.wrap_cppobject(space, ptr_result, self.cppclass) def execute_libffi(self, space, cif_descr, funcaddr, buffer): jit_libffi.jit_ffi_call(cif_descr, funcaddr, buffer) result = rffi.ptradd(buffer, cif_descr.exchange_result) from pypy.module.cppyy import interp_cppyy ptr_result = rffi.cast(capi.C_OBJECT, rffi.cast(rffi.VOIDPP, result)[0]) - return interp_cppyy.wrap_cppobject( - space, space.w_None, self.cppclass, ptr_result, isref=False, python_owns=False) + return interp_cppyy.wrap_cppobject(space, ptr_result, self.cppclass) class InstancePtrPtrExecutor(InstancePtrExecutor): @@ -177,8 +175,7 @@ voidp_result = capi.c_call_r(cppmethod, cppthis, num_args, args) ref_address = rffi.cast(rffi.VOIDPP, voidp_result) ptr_result = rffi.cast(capi.C_OBJECT, ref_address[0]) - return interp_cppyy.wrap_cppobject( - space, space.w_None, self.cppclass, ptr_result, isref=False, python_owns=False) + return interp_cppyy.wrap_cppobject(space, ptr_result, self.cppclass) def execute_libffi(self, space, cif_descr, funcaddr, buffer): from pypy.module.cppyy.interp_cppyy import FastCallNotPossible @@ -190,8 +187,8 @@ from pypy.module.cppyy import interp_cppyy long_result = capi.c_call_o(cppmethod, cppthis, num_args, args, self.cppclass) ptr_result = rffi.cast(capi.C_OBJECT, long_result) - return interp_cppyy.wrap_cppobject_nocast( - space, space.w_None, self.cppclass, ptr_result, isref=False, python_owns=True) + return interp_cppyy.wrap_cppobject(space, ptr_result, self.cppclass, + do_cast=False, python_owns=True) def execute_libffi(self, space, cif_descr, funcaddr, buffer): from pypy.module.cppyy.interp_cppyy import FastCallNotPossible diff --git a/pypy/module/cppyy/interp_cppyy.py b/pypy/module/cppyy/interp_cppyy.py --- a/pypy/module/cppyy/interp_cppyy.py +++ b/pypy/module/cppyy/interp_cppyy.py @@ -384,8 +384,8 @@ vscope = rffi.cast(capi.C_OBJECT, self.scope.handle) w_result = CPPMethod.call(self, vscope, args_w) newthis = rffi.cast(capi.C_OBJECT, self.space.int_w(w_result)) - return wrap_new_cppobject_nocast( - self.space, self.space.w_None, self.scope, newthis, isref=False, python_owns=True) + return wrap_cppobject(self.space, newthis, self.scope, + do_cast=False, python_owns=True, fresh=True) def __repr__(self): return "CPPConstructor: %s" % self.signature() @@ -1049,26 +1049,13 @@ w_pycppclass = space.call_function(state.w_clgen_callback, space.wrap(final_name)) return w_pycppclass -def wrap_new_cppobject_nocast(space, w_pycppclass, cppclass, rawobject, isref, python_owns): +def wrap_cppobject(space, rawobject, cppclass, + do_cast=True, python_owns=False, is_ref=False, fresh=False): rawobject = rffi.cast(capi.C_OBJECT, rawobject) - if space.is_w(w_pycppclass, space.w_None): - w_pycppclass = get_pythonized_cppclass(space, cppclass.handle) - w_cppinstance = space.allocate_instance(W_CPPInstance, w_pycppclass) - cppinstance = space.interp_w(W_CPPInstance, w_cppinstance) - cppinstance.__init__(space, cppclass, rawobject, isref, python_owns) - memory_regulator.register(cppinstance) - return w_cppinstance -def wrap_cppobject_nocast(space, w_pycppclass, cppclass, rawobject, isref, python_owns): - rawobject = rffi.cast(capi.C_OBJECT, rawobject) - obj = memory_regulator.retrieve(rawobject) - if obj is not None and obj.cppclass is cppclass: - return obj - return wrap_new_cppobject_nocast(space, w_pycppclass, cppclass, rawobject, isref, python_owns) - -def wrap_cppobject(space, w_pycppclass, cppclass, rawobject, isref, python_owns): - rawobject = rffi.cast(capi.C_OBJECT, rawobject) - if rawobject: + # cast to actual cast if requested and possible + w_pycppclass = space.w_None + if do_cast and rawobject: actual = capi.c_actual_class(cppclass, rawobject) if actual != cppclass.handle: try: @@ -1082,7 +1069,22 @@ # that only get_pythonized_cppclass is expected to raise, so none of # the variables are re-assigned yet) pass - return wrap_cppobject_nocast(space, w_pycppclass, cppclass, rawobject, isref, python_owns) + + if space.is_w(w_pycppclass, space.w_None): + w_pycppclass = get_pythonized_cppclass(space, cppclass.handle) + + # try to recycle existing object if this one is not newly created + if not fresh: + obj = memory_regulator.retrieve(rawobject) + if obj is not None and obj.cppclass is cppclass: + return obj + + # fresh creation + w_cppinstance = space.allocate_instance(W_CPPInstance, w_pycppclass) + cppinstance = space.interp_w(W_CPPInstance, w_cppinstance) + cppinstance.__init__(space, cppclass, rawobject, is_ref, python_owns) + memory_regulator.register(cppinstance) + return w_cppinstance @unwrap_spec(cppinstance=W_CPPInstance) def addressof(space, cppinstance): @@ -1096,4 +1098,4 @@ rawobject = rffi.cast(capi.C_OBJECT, address) w_cppclass = space.findattr(w_pycppclass, space.wrap("_cpp_proxy")) cppclass = space.interp_w(W_CPPClass, w_cppclass, can_be_None=False) - return wrap_cppobject_nocast(space, w_pycppclass, cppclass, rawobject, False, owns) + return wrap_cppobject(space, rawobject, cppclass, do_cast=False, python_owns=owns) From noreply at buildbot.pypy.org Sat Jan 5 19:33:16 2013 From: noreply at buildbot.pypy.org (wlav) Date: Sat, 5 Jan 2013 19:33:16 +0100 (CET) Subject: [pypy-commit] pypy reflex-support: fixup for CINT backend Message-ID: <20130105183316.00B3A1C0DDA@cobra.cs.uni-duesseldorf.de> Author: Wim Lavrijsen Branch: reflex-support Changeset: r59762:4cf5a9c3d177 Date: 2013-01-05 01:02 -0800 http://bitbucket.org/pypy/pypy/changeset/4cf5a9c3d177/ Log: fixup for CINT backend diff --git a/pypy/module/cppyy/capi/cint_capi.py b/pypy/module/cppyy/capi/cint_capi.py --- a/pypy/module/cppyy/capi/cint_capi.py +++ b/pypy/module/cppyy/capi/cint_capi.py @@ -138,8 +138,7 @@ # call the helper stub to by-pass CINT vbranch = _ttree_Branch(vtree, branchname, klassname, address, bufsize, splitlevel) branch_class = interp_cppyy.scope_byname(space, "TBranch") - w_branch = interp_cppyy.wrap_cppobject( - space, space.w_None, branch_class, vbranch, isref=False, python_owns=False) + w_branch = interp_cppyy.wrap_cppobject(space, vbranch, branch_class) return w_branch except (OperationError, TypeError, IndexError), e: pass From noreply at buildbot.pypy.org Sat Jan 5 19:33:17 2013 From: noreply at buildbot.pypy.org (wlav) Date: Sat, 5 Jan 2013 19:33:17 +0100 (CET) Subject: [pypy-commit] pypy reflex-support: rtyper fix Message-ID: <20130105183317.1A3EC1C0DDA@cobra.cs.uni-duesseldorf.de> Author: Wim Lavrijsen Branch: reflex-support Changeset: r59763:77c78edcc384 Date: 2013-01-05 10:32 -0800 http://bitbucket.org/pypy/pypy/changeset/77c78edcc384/ Log: rtyper fix diff --git a/pypy/module/cppyy/interp_cppyy.py b/pypy/module/cppyy/interp_cppyy.py --- a/pypy/module/cppyy/interp_cppyy.py +++ b/pypy/module/cppyy/interp_cppyy.py @@ -535,7 +535,8 @@ W_CPPDataMember.typedef.acceptable_as_base_class = False class W_CPPStaticData(W_CPPDataMember): - def _get_offset(self): + @jit.elidable_promote() + def _get_offset(self, cppinstance): return self.offset def get(self, w_cppinstance, w_pycppclass): From noreply at buildbot.pypy.org Sat Jan 5 19:36:44 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sat, 5 Jan 2013 19:36:44 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: I'm sorry :( py.py -> pyinteractive.py Message-ID: <20130105183644.843F01C0DDA@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59764:69a40a56749f Date: 2013-01-05 19:10 +0100 http://bitbucket.org/pypy/pypy/changeset/69a40a56749f/ Log: I'm sorry :( py.py -> pyinteractive.py diff --git a/pypy/bin/py.py b/pypy/bin/pyinteractive.py rename from pypy/bin/py.py rename to pypy/bin/pyinteractive.py --- a/pypy/bin/py.py +++ b/pypy/bin/pyinteractive.py @@ -9,7 +9,8 @@ import os, sys import time -sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..'))) +if os.path.exists(os.path.join(os.path.dirname(__file__), '..', '..', 'pypy', '__init__.py')): + sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..'))) import pypy from pypy.tool import option From noreply at buildbot.pypy.org Sat Jan 5 19:36:45 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sat, 5 Jan 2013 19:36:45 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed pypydir Message-ID: <20130105183645.AD7451C0DDA@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59765:5df7187493f2 Date: 2013-01-05 19:36 +0100 http://bitbucket.org/pypy/pypy/changeset/5df7187493f2/ Log: Fixed pypydir diff --git a/lib_pypy/ctypes_config_cache/autopath.py b/lib_pypy/ctypes_config_cache/autopath.py deleted file mode 100644 --- a/lib_pypy/ctypes_config_cache/autopath.py +++ /dev/null @@ -1,131 +0,0 @@ -""" -self cloning, automatic path configuration - -copy this into any subdirectory of pypy from which scripts need -to be run, typically all of the test subdirs. -The idea is that any such script simply issues - - import autopath - -and this will make sure that the parent directory containing "pypy" -is in sys.path. - -If you modify the master "autopath.py" version (in pypy/tool/autopath.py) -you can directly run it which will copy itself on all autopath.py files -it finds under the pypy root directory. - -This module always provides these attributes: - - pypydir pypy root directory path - this_dir directory where this autopath.py resides - -""" - -def __dirinfo(part): - """ return (partdir, this_dir) and insert parent of partdir - into sys.path. If the parent directories don't have the part - an EnvironmentError is raised.""" - - import sys, os - try: - head = this_dir = os.path.realpath(os.path.dirname(__file__)) - except NameError: - head = this_dir = os.path.realpath(os.path.dirname(sys.argv[0])) - - error = None - while head: - partdir = head - head, tail = os.path.split(head) - if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') - if not os.path.exists(checkfile): - error = "Cannot find %r" % (os.path.normpath(checkfile),) - break - else: - error = "Cannot find the parent directory %r of the path %r" % ( - partdir, this_dir) - if not error: - # check for bogus end-of-line style (e.g. files checked out on - # Windows and moved to Unix) - f = open(__file__.replace('.pyc', '.py'), 'r') - data = f.read() - f.close() - if data.endswith('\r\n') or data.endswith('\r'): - error = ("Bad end-of-line style in the .py files. Typically " - "caused by a zip file or a checkout done on Windows and " - "moved to Unix or vice-versa.") - if error: - raise EnvironmentError("Invalid source tree - bogus checkout! " + - error) - - pypy_root = os.path.join(head, '') - try: - sys.path.remove(head) - except ValueError: - pass - sys.path.insert(0, head) - - munged = {} - for name, mod in sys.modules.items(): - if '.' in name: - continue - fn = getattr(mod, '__file__', None) - if not isinstance(fn, str): - continue - newname = os.path.splitext(os.path.basename(fn))[0] - if not newname.startswith(part + '.'): - continue - path = os.path.join(os.path.dirname(os.path.realpath(fn)), '') - if path.startswith(pypy_root) and newname != part: - modpaths = os.path.normpath(path[len(pypy_root):]).split(os.sep) - if newname != '__init__': - modpaths.append(newname) - modpath = '.'.join(modpaths) - if modpath not in sys.modules: - munged[modpath] = mod - - for name, mod in munged.iteritems(): - if name not in sys.modules: - sys.modules[name] = mod - if '.' in name: - prename = name[:name.rfind('.')] - postname = name[len(prename)+1:] - if prename not in sys.modules: - __import__(prename) - if not hasattr(sys.modules[prename], postname): - setattr(sys.modules[prename], postname, mod) - - return partdir, this_dir - -def __clone(): - """ clone master version of autopath.py into all subdirs """ - from os.path import join, walk - if not this_dir.endswith(join('pypy','tool')): - raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) - - - def sync_walker(arg, dirname, fnames): - if _myname in fnames: - fn = join(dirname, _myname) - f = open(fn, 'rwb+') - try: - if f.read() == arg: - print "checkok", fn - else: - print "syncing", fn - f = open(fn, 'w') - f.write(arg) - finally: - f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) - -_myname = 'autopath.py' - -# set guaranteed attributes - -pypydir, this_dir = __dirinfo('pypy') - -if __name__ == '__main__': - __clone() diff --git a/lib_pypy/ctypes_config_cache/rebuild.py b/lib_pypy/ctypes_config_cache/rebuild.py --- a/lib_pypy/ctypes_config_cache/rebuild.py +++ b/lib_pypy/ctypes_config_cache/rebuild.py @@ -1,16 +1,10 @@ #! /usr/bin/env python # Run this script to rebuild all caches from the *.ctc.py files. -# hack: we cannot directly import autopath, as we are outside the pypy -# package. However, we pretend to be inside pypy/tool and manually run it, to -# get the correct path -import os.path -this_dir = os.path.dirname(__file__) -autopath_py = os.path.join(this_dir, '../../pypy/tool/autopath.py') -autopath_py = os.path.abspath(autopath_py) -execfile(autopath_py, dict(__name__='autopath', __file__=autopath_py)) +import os, sys -import os, sys +sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..'))) + import py _dirpath = os.path.dirname(__file__) or os.curdir diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -10,7 +10,7 @@ # option = None -pypydir = os.path.realpath(os.path.join(os.path.dirname(__file__), '..')) +pypydir = os.path.realpath(os.path.dirname(__file__)) def braindead_deindent(self): """monkeypatch that wont end up doing stupid in the python tokenizer""" diff --git a/pypy/tool/test/test_getdocstrings.py b/pypy/tool/test/test_getdocstrings.py --- a/pypy/tool/test/test_getdocstrings.py +++ b/pypy/tool/test/test_getdocstrings.py @@ -2,7 +2,7 @@ import glob, os.path, py, re this_dir = os.path.realpath(os.path.dirname(__file__)) -pypy_dir = os.path.realpath(os.path.join(os.path.dirname(__file__), '..')) +pypy_dir = os.path.realpath(os.path.join(this_dir, '..', '..')) from pypy.tool.getdocstrings import quote, triplequotes from pypy.tool.getdocstrings import mk_std_filelist From noreply at buildbot.pypy.org Sat Jan 5 19:58:02 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 5 Jan 2013 19:58:02 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: hg merge default Message-ID: <20130105185802.2DA961C01EE@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r59766:759e48aef604 Date: 2013-01-05 19:47 +0100 http://bitbucket.org/pypy/pypy/changeset/759e48aef604/ Log: hg merge default diff too long, truncating to 2000 out of 6159 lines diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -28,7 +28,7 @@ DEALINGS IN THE SOFTWARE. -PyPy Copyright holders 2003-2012 +PyPy Copyright holders 2003-2013 ----------------------------------- Except when otherwise stated (look for LICENSE files or information at diff --git a/lib-python/2.7/ctypes/test/test_internals.py b/lib-python/2.7/ctypes/test/test_internals.py --- a/lib-python/2.7/ctypes/test/test_internals.py +++ b/lib-python/2.7/ctypes/test/test_internals.py @@ -1,7 +1,10 @@ # This tests the internal _objects attribute import unittest from ctypes import * -from sys import getrefcount as grc +try: + from sys import getrefcount as grc +except ImportError: + grc = None # e.g. PyPy # XXX This test must be reviewed for correctness!!! @@ -22,6 +25,8 @@ self.assertEqual(id(a), id(b)) def test_ints(self): + if grc is None: + return unittest.skip("no sys.getrefcount()") i = 42000123 refcnt = grc(i) ci = c_int(i) @@ -29,6 +34,8 @@ self.assertEqual(ci._objects, None) def test_c_char_p(self): + if grc is None: + return unittest.skip("no sys.getrefcount()") s = "Hello, World" refcnt = grc(s) cs = c_char_p(s) diff --git a/lib-python/2.7/ctypes/test/test_memfunctions.py b/lib-python/2.7/ctypes/test/test_memfunctions.py --- a/lib-python/2.7/ctypes/test/test_memfunctions.py +++ b/lib-python/2.7/ctypes/test/test_memfunctions.py @@ -53,7 +53,8 @@ s = string_at("foo bar") # XXX The following may be wrong, depending on how Python # manages string instances - self.assertEqual(2, sys.getrefcount(s)) + if hasattr(sys, 'getrefcount'): + self.assertEqual(2, sys.getrefcount(s)) self.assertTrue(s, "foo bar") self.assertEqual(string_at("foo bar", 8), "foo bar\0") diff --git a/lib-python/2.7/ctypes/test/test_python_api.py b/lib-python/2.7/ctypes/test/test_python_api.py --- a/lib-python/2.7/ctypes/test/test_python_api.py +++ b/lib-python/2.7/ctypes/test/test_python_api.py @@ -9,7 +9,10 @@ ################################################################ -from sys import getrefcount as grc +try: + from sys import getrefcount as grc +except ImportError: + grc = None # e.g. PyPy if sys.version_info > (2, 4): c_py_ssize_t = c_size_t else: diff --git a/lib-python/2.7/ctypes/test/test_refcounts.py b/lib-python/2.7/ctypes/test/test_refcounts.py --- a/lib-python/2.7/ctypes/test/test_refcounts.py +++ b/lib-python/2.7/ctypes/test/test_refcounts.py @@ -11,7 +11,10 @@ class RefcountTestCase(unittest.TestCase): def test_1(self): - from sys import getrefcount as grc + try: + from sys import getrefcount as grc + except ImportError: + return unittest.skip("no sys.getrefcount()") f = dll._testfunc_callback_i_if f.restype = ctypes.c_int @@ -35,7 +38,10 @@ def test_refcount(self): - from sys import getrefcount as grc + try: + from sys import getrefcount as grc + except ImportError: + return unittest.skip("no sys.getrefcount()") def func(*args): pass # this is the standard refcount for func @@ -84,6 +90,10 @@ class AnotherLeak(unittest.TestCase): def test_callback(self): import sys + try: + from sys import getrefcount + except ImportError: + return unittest.skip("no sys.getrefcount()") proto = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int, ctypes.c_int) def func(a, b): diff --git a/lib-python/2.7/sre_parse.py b/lib-python/2.7/sre_parse.py --- a/lib-python/2.7/sre_parse.py +++ b/lib-python/2.7/sre_parse.py @@ -16,6 +16,12 @@ from sre_constants import * +try: + from __pypy__ import newdict +except ImportError: + def newdict(tp): + return {} + SPECIAL_CHARS = ".\\[{()*+?^$|" REPEAT_CHARS = "*+?{" @@ -68,7 +74,7 @@ self.flags = 0 self.open = [] self.groups = 1 - self.groupdict = {} + self.groupdict = newdict("module") def opengroup(self, name=None): gid = self.groups self.groups = gid + 1 diff --git a/lib-python/2.7/timeit.py b/lib-python/2.7/timeit.py --- a/lib-python/2.7/timeit.py +++ b/lib-python/2.7/timeit.py @@ -190,7 +190,8 @@ else: it = [None] * number gcold = gc.isenabled() - gc.disable() + if '__pypy__' not in sys.builtin_module_names: + gc.disable() # only do that on CPython try: timing = self.inner(it, self.timer) finally: diff --git a/lib_pypy/_ctypes_test.py b/lib_pypy/_ctypes_test.py --- a/lib_pypy/_ctypes_test.py +++ b/lib_pypy/_ctypes_test.py @@ -2,10 +2,6 @@ import tempfile import gc -# Monkeypatch & hacks to let ctypes.tests import. -# This should be removed at some point. -sys.getrefcount = lambda x: len(gc.get_referrers(x)) - 1 - def compile_shared(): """Compile '_ctypes_test.c' into an extension module, and import it """ diff --git a/lib_pypy/numpypy/core/arrayprint.py b/lib_pypy/numpypy/core/arrayprint.py --- a/lib_pypy/numpypy/core/arrayprint.py +++ b/lib_pypy/numpypy/core/arrayprint.py @@ -248,9 +248,9 @@ 'int' : IntegerFormat(data), 'float' : FloatFormat(data, precision, suppress_small), 'longfloat' : LongFloatFormat(precision), - #'complexfloat' : ComplexFormat(data, precision, - # suppress_small), - #'longcomplexfloat' : LongComplexFormat(precision), + 'complexfloat' : ComplexFormat(data, precision, + suppress_small), + 'longcomplexfloat' : LongComplexFormat(precision), 'datetime' : DatetimeFormat(data), 'timedelta' : TimedeltaFormat(data), 'numpystr' : repr_format, @@ -294,19 +294,19 @@ #else: format_function = formatdict['int'] elif issubclass(dtypeobj, _nt.floating): - #if issubclass(dtypeobj, _nt.longfloat): - # format_function = formatdict['longfloat'] - #else: - format_function = formatdict['float'] - #elif issubclass(dtypeobj, _nt.complexfloating): - # if issubclass(dtypeobj, _nt.clongfloat): - # format_function = formatdict['longcomplexfloat'] - # else: - # format_function = formatdict['complexfloat'] + if issubclass(dtypeobj, _nt.longfloat): + format_function = formatdict['longfloat'] + else: + format_function = formatdict['float'] + elif issubclass(dtypeobj, _nt.complexfloating): + if issubclass(dtypeobj, _nt.clongfloat): + format_function = formatdict['longcomplexfloat'] + else: + format_function = formatdict['complexfloat'] elif issubclass(dtypeobj, (_nt.unicode_, _nt.string_)): format_function = formatdict['numpystr'] - elif issubclass(dtypeobj, _nt.datetime64): - format_function = formatdict['datetime'] + #elif issubclass(dtypeobj, _nt.datetime64): + # format_function = formatdict['datetime'] else: format_function = formatdict['str'] diff --git a/pypy/annotation/annrpython.py b/pypy/annotation/annrpython.py --- a/pypy/annotation/annrpython.py +++ b/pypy/annotation/annrpython.py @@ -8,7 +8,7 @@ from pypy.objspace.flow.model import (Variable, Constant, FunctionGraph, c_last_exception, checkgraph) from pypy.translator import simplify, transform -from pypy.annotation import model as annmodel, signature +from pypy.annotation import model as annmodel, signature, unaryop, binaryop from pypy.annotation.bookkeeper import Bookkeeper import py log = py.log.Producer("annrpython") @@ -453,12 +453,12 @@ # occour for this specific, typed operation. if block.exitswitch == c_last_exception: op = block.operations[-1] - if op.opname in annmodel.BINARY_OPERATIONS: + if op.opname in binaryop.BINARY_OPERATIONS: arg1 = self.binding(op.args[0]) arg2 = self.binding(op.args[1]) binop = getattr(pair(arg1, arg2), op.opname, None) can_only_throw = annmodel.read_can_only_throw(binop, arg1, arg2) - elif op.opname in annmodel.UNARY_OPERATIONS: + elif op.opname in unaryop.UNARY_OPERATIONS: arg1 = self.binding(op.args[0]) opname = op.opname if opname == 'contains': opname = 'op_contains' @@ -629,10 +629,10 @@ return self.bookkeeper.newdict() - def _registeroperations(cls, model): + def _registeroperations(cls, unary_ops, binary_ops): # All unary operations d = {} - for opname in model.UNARY_OPERATIONS: + for opname in unary_ops: fnname = 'consider_op_' + opname exec py.code.Source(""" def consider_op_%s(self, arg, *args): @@ -640,7 +640,7 @@ """ % (opname, opname)).compile() in globals(), d setattr(cls, fnname, d[fnname]) # All binary operations - for opname in model.BINARY_OPERATIONS: + for opname in binary_ops: fnname = 'consider_op_' + opname exec py.code.Source(""" def consider_op_%s(self, arg1, arg2, *args): @@ -650,7 +650,7 @@ _registeroperations = classmethod(_registeroperations) # register simple operations handling -RPythonAnnotator._registeroperations(annmodel) +RPythonAnnotator._registeroperations(unaryop.UNARY_OPERATIONS, binaryop.BINARY_OPERATIONS) class BlockedInference(Exception): diff --git a/pypy/annotation/binaryop.py b/pypy/annotation/binaryop.py --- a/pypy/annotation/binaryop.py +++ b/pypy/annotation/binaryop.py @@ -7,10 +7,10 @@ from pypy.tool.pairtype import pair, pairtype from pypy.annotation.model import SomeObject, SomeInteger, SomeBool, s_Bool from pypy.annotation.model import SomeString, SomeChar, SomeList, SomeDict -from pypy.annotation.model import SomeUnicodeCodePoint, SomeStringOrUnicode +from pypy.annotation.model import SomeUnicodeCodePoint, SomeUnicodeString from pypy.annotation.model import SomeTuple, SomeImpossibleValue, s_ImpossibleValue from pypy.annotation.model import SomeInstance, SomeBuiltin, SomeIterator -from pypy.annotation.model import SomePBC, SomeFloat, s_None +from pypy.annotation.model import SomePBC, SomeFloat, s_None, SomeByteArray from pypy.annotation.model import SomeExternalObject, SomeWeakRef from pypy.annotation.model import SomeAddress, SomeTypedAddressAccess from pypy.annotation.model import SomeSingleFloat, SomeLongFloat, SomeType @@ -19,7 +19,6 @@ from pypy.annotation.model import read_can_only_throw from pypy.annotation.model import add_knowntypedata, merge_knowntypedata from pypy.annotation.model import SomeGenericCallable -from pypy.annotation.model import SomeUnicodeString from pypy.annotation.bookkeeper import getbookkeeper from pypy.objspace.flow.model import Variable, Constant from pypy.rlib import rarithmetic @@ -416,6 +415,34 @@ result.const = str1.const + str2.const return result +class __extend__(pairtype(SomeByteArray, SomeByteArray)): + def union((b1, b2)): + can_be_None = b1.can_be_None or b2.can_be_None + return SomeByteArray(can_be_None=can_be_None) + + def add((b1, b2)): + result = SomeByteArray() + if b1.is_immutable_constant() and b2.is_immutable_constant(): + result.const = b1.const + b2.const + return result + +class __extend__(pairtype(SomeByteArray, SomeInteger)): + def getitem((s_b, s_i)): + return SomeInteger() + + def setitem((s_b, s_i), s_i2): + assert isinstance(s_i2, SomeInteger) + +class __extend__(pairtype(SomeString, SomeByteArray), + pairtype(SomeByteArray, SomeString), + pairtype(SomeChar, SomeByteArray), + pairtype(SomeByteArray, SomeChar)): + def add((b1, b2)): + result = SomeByteArray() + if b1.is_immutable_constant() and b2.is_immutable_constant(): + result.const = b1.const + b2.const + return result + class __extend__(pairtype(SomeChar, SomeChar)): def union((chr1, chr2)): @@ -458,7 +485,8 @@ for s_item in s_tuple.items: if isinstance(s_item, SomeFloat): pass # or s_item is a subclass, like SomeInteger - elif isinstance(s_item, SomeStringOrUnicode) and s_item.no_nul: + elif (isinstance(s_item, SomeString) or + isinstance(s_item, SomeUnicodeString)) and s_item.no_nul: pass else: no_nul = False diff --git a/pypy/annotation/bookkeeper.py b/pypy/annotation/bookkeeper.py --- a/pypy/annotation/bookkeeper.py +++ b/pypy/annotation/bookkeeper.py @@ -13,7 +13,7 @@ SomeUnicodeCodePoint, SomeOOStaticMeth, s_None, s_ImpossibleValue, \ SomeLLADTMeth, SomeBool, SomeTuple, SomeOOClass, SomeImpossibleValue, \ SomeUnicodeString, SomeList, SomeObject, HarmlesslyBlocked, \ - SomeWeakRef, lltype_to_annotation, SomeType + SomeWeakRef, lltype_to_annotation, SomeType, SomeByteArray from pypy.annotation.classdef import InstanceSource, ClassDef from pypy.annotation.listdef import ListDef, ListItem from pypy.annotation.dictdef import DictDef @@ -349,6 +349,8 @@ result = SomeUnicodeCodePoint() else: result = SomeUnicodeString() + elif tp is bytearray: + result = SomeByteArray() elif tp is tuple: result = SomeTuple(items = [self.immutablevalue(e, need_const) for e in x]) elif tp is float: diff --git a/pypy/annotation/builtin.py b/pypy/annotation/builtin.py --- a/pypy/annotation/builtin.py +++ b/pypy/annotation/builtin.py @@ -9,7 +9,7 @@ from pypy.annotation.model import SomeFloat, unionof, SomeUnicodeString from pypy.annotation.model import SomePBC, SomeInstance, SomeDict, SomeList from pypy.annotation.model import SomeWeakRef, SomeIterator -from pypy.annotation.model import SomeOOObject +from pypy.annotation.model import SomeOOObject, SomeByteArray from pypy.annotation.model import annotation_to_lltype, lltype_to_annotation, ll_to_annotation from pypy.annotation.model import add_knowntypedata from pypy.annotation.model import s_ImpossibleValue @@ -119,6 +119,9 @@ def builtin_unicode(s_unicode): return constpropagate(unicode, [s_unicode], SomeUnicodeString()) +def builtin_bytearray(s_str): + return constpropagate(bytearray, [s_str], SomeByteArray()) + def our_issubclass(cls1, cls2): """ we're going to try to be less silly in the face of old-style classes""" from pypy.annotation.classdef import ClassDef @@ -253,24 +256,6 @@ s = SomeInteger(nonneg=True, knowntype=s.knowntype) return s -def builtin_apply(*stuff): - getbookkeeper().warning("ignoring apply%r" % (stuff,)) - return SomeObject() - -##def builtin_slice(*args): -## bk = getbookkeeper() -## if len(args) == 1: -## return SomeSlice( -## bk.immutablevalue(None), args[0], bk.immutablevalue(None)) -## elif len(args) == 2: -## return SomeSlice( -## args[0], args[1], bk.immutablevalue(None)) -## elif len(args) == 3: -## return SomeSlice( -## args[0], args[1], args[2]) -## else: -## raise Exception, "bogus call to slice()" - def OSError_init(s_self, *args): pass diff --git a/pypy/annotation/model.py b/pypy/annotation/model.py --- a/pypy/annotation/model.py +++ b/pypy/annotation/model.py @@ -201,11 +201,16 @@ self.knowntypedata = knowntypedata class SomeStringOrUnicode(SomeObject): + """Base class for shared implementation of SomeString and SomeUnicodeString. + + Cannot be an annotation.""" + immutable = True can_be_None=False no_nul = False # No NUL character in the string. def __init__(self, can_be_None=False, no_nul=False): + assert type(self) is not SomeStringOrUnicode if can_be_None: self.can_be_None = True if no_nul: @@ -224,19 +229,19 @@ d2 = d2.copy(); d2['no_nul'] = 0 # ignored return d1 == d2 + def nonnoneify(self): + return self.__class__(can_be_None=False, no_nul=self.no_nul) + class SomeString(SomeStringOrUnicode): "Stands for an object which is known to be a string." knowntype = str - def nonnoneify(self): - return SomeString(can_be_None=False, no_nul=self.no_nul) - class SomeUnicodeString(SomeStringOrUnicode): "Stands for an object which is known to be an unicode string" knowntype = unicode - def nonnoneify(self): - return SomeUnicodeString(can_be_None=False, no_nul=self.no_nul) +class SomeByteArray(SomeStringOrUnicode): + knowntype = bytearray class SomeChar(SomeString): "Stands for an object known to be a string of length 1." @@ -772,7 +777,3 @@ else: raise RuntimeError("The annotator relies on 'assert' statements from the\n" "\tannotated program: you cannot run it with 'python -O'.") - -# this has the side-effect of registering the unary and binary operations -from pypy.annotation.unaryop import UNARY_OPERATIONS -from pypy.annotation.binaryop import BINARY_OPERATIONS diff --git a/pypy/annotation/signature.py b/pypy/annotation/signature.py --- a/pypy/annotation/signature.py +++ b/pypy/annotation/signature.py @@ -151,4 +151,9 @@ actualtypes[:] = params_s def enforce_signature_return(funcdesc, sigtype, inferredtype): - return finish_type(sigtype, funcdesc.bookkeeper, funcdesc.pyobj) + s_sigret = finish_type(sigtype, funcdesc.bookkeeper, funcdesc.pyobj) + if not s_sigret.contains(inferredtype): + raise Exception("%r return value:\n" + "expected %s,\n" + " got %s" % (funcdesc, s_sigret, inferredtype)) + return s_sigret diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py --- a/pypy/annotation/test/test_annrpython.py +++ b/pypy/annotation/test/test_annrpython.py @@ -3819,6 +3819,37 @@ a = self.RPythonAnnotator() a.build_types(f, []) # assert did not explode + def test_bytearray(self): + def f(): + return bytearray("xyz") + + a = self.RPythonAnnotator() + assert isinstance(a.build_types(f, []), annmodel.SomeByteArray) + + def test_bytearray_add(self): + def f(a): + return a + bytearray("xyz") + + a = self.RPythonAnnotator() + assert isinstance(a.build_types(f, [annmodel.SomeByteArray()]), + annmodel.SomeByteArray) + a = self.RPythonAnnotator() + assert isinstance(a.build_types(f, [str]), + annmodel.SomeByteArray) + a = self.RPythonAnnotator() + assert isinstance(a.build_types(f, [annmodel.SomeChar()]), + annmodel.SomeByteArray) + + def test_bytearray_setitem_getitem(self): + def f(b, i, c): + b[i] = c + return b[i + 1] + + a = self.RPythonAnnotator() + assert isinstance(a.build_types(f, [annmodel.SomeByteArray(), + int, int]), + annmodel.SomeInteger) + def g(n): return [0,1,2,n] diff --git a/pypy/doc/config/objspace.usemodules.time.txt b/pypy/doc/config/objspace.usemodules.time.txt --- a/pypy/doc/config/objspace.usemodules.time.txt +++ b/pypy/doc/config/objspace.usemodules.time.txt @@ -1,4 +1,5 @@ Use the 'time' module. Obsolete; use :config:`objspace.usemodules.rctime` for our up-to-date version -of the application-level 'time' module. +of the application-level 'time' module, at least for C-like targets (the C +and LLVM backends). diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst --- a/pypy/doc/index.rst +++ b/pypy/doc/index.rst @@ -67,7 +67,7 @@ .. _Python: http://docs.python.org/index.html .. _`more...`: architecture.html#mission-statement .. _`PyPy blog`: http://morepypy.blogspot.com/ -.. _`development bug/feature tracker`: https://codespeak.net/issue/pypy-dev/ +.. _`development bug/feature tracker`: https://bugs.pypy.org .. _here: http://tismerysoft.de/pypy/irc-logs/pypy .. _`Mercurial commit mailing list`: http://python.org/mailman/listinfo/pypy-commit .. _`development mailing list`: http://python.org/mailman/listinfo/pypy-dev diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -8,12 +8,27 @@ .. branch: length-hint Implement __lenght_hint__ according to PEP 424 +.. branch: numpypy-longdouble +Long double support for numpypy + +.. branch: signatures +Improved RPython typing + +.. branch: rpython-bytearray +Rudimentary support for bytearray in RPython + .. branches we don't care about .. branch: autoreds +.. branch: reflex-support +.. branch: kill-faking +.. branch: improved_ebnfparse_error +.. branch: task-decorator .. branch: release-2.0-beta1 .. branch: remove-PYPY_NOT_MAIN_FILE +.. branch: missing-jit-operations + .. branch: fix-lookinside-iff-oopspec Fixed the interaction between two internal tools for controlling the JIT. diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py --- a/pypy/jit/backend/arm/assembler.py +++ b/pypy/jit/backend/arm/assembler.py @@ -1,5 +1,6 @@ from __future__ import with_statement import os +from pypy.jit.backend.llsupport import jitframe from pypy.jit.backend.arm.helper.assembler import saved_registers from pypy.jit.backend.arm import conditions as c from pypy.jit.backend.arm import registers as r @@ -8,7 +9,7 @@ from pypy.jit.backend.arm.codebuilder import ARMv7Builder, OverwritingBuilder from pypy.jit.backend.arm.locations import get_fp_offset from pypy.jit.backend.arm.regalloc import (Regalloc, ARMFrameManager, - ARMv7RegisterManager, check_imm_arg, + CoreRegisterManager, check_imm_arg, operations as regalloc_operations, operations_with_guard as regalloc_operations_with_guard) from pypy.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper @@ -25,12 +26,12 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.jit.backend.arm.opassembler import ResOpAssembler from pypy.rlib.debug import (debug_print, debug_start, debug_stop, - have_debug_prints) + have_debug_prints, fatalerror) from pypy.rlib.jit import AsmInfo from pypy.rlib.objectmodel import compute_unique_id # XXX Move to llsupport -from pypy.jit.backend.x86.support import values_array, memcpy_fn +from pypy.jit.backend.x86.support import memcpy_fn DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', ('i', lltype.Signed), ('type', lltype.Char), # 'b'ridge, 'l'abel or @@ -44,14 +45,8 @@ debug = True - def __init__(self, cpu, failargs_limit=1000): + def __init__(self, cpu, translate_support_code=False): self.cpu = cpu - self.fail_boxes_int = values_array(lltype.Signed, failargs_limit) - self.fail_boxes_float = values_array(longlong.FLOATSTORAGE, - failargs_limit) - self.fail_boxes_ptr = values_array(llmemory.GCREF, failargs_limit) - self.fail_boxes_count = 0 - self.fail_force_index = 0 self.setup_failure_recovery() self.mc = None self.memcpy_addr = 0 @@ -68,6 +63,7 @@ self._debug = False self.loop_run_counters = [] self.debug_counter_descr = cpu.fielddescrof(DEBUG_COUNTER, 'i') + self.force_token_to_dead_frame = {} # XXX temporary hack def set_debug(self, v): r = self._debug @@ -112,9 +108,13 @@ gc_ll_descr.initialize() self._build_wb_slowpath(False) self._build_wb_slowpath(True) + self._build_failure_recovery(exc=True, withfloats=False) + self._build_failure_recovery(exc=False, withfloats=False) if self.cpu.supports_floats: self._build_wb_slowpath(False, withfloats=True) self._build_wb_slowpath(True, withfloats=True) + self._build_failure_recovery(exc=True, withfloats=True) + self._build_failure_recovery(exc=False, withfloats=True) self._build_propagate_exception_path() if gc_ll_descr.get_malloc_slowpath_addr is not None: self._build_malloc_slowpath() @@ -122,10 +122,7 @@ if gc_ll_descr.gcrootmap and gc_ll_descr.gcrootmap.is_shadow_stack: self._build_release_gil(gc_ll_descr.gcrootmap) self.memcpy_addr = self.cpu.cast_ptr_to_int(memcpy_fn) - self._exit_code_addr = self._gen_exit_path() - self._leave_jitted_hook_save_exc = \ - self._gen_leave_jitted_hook_code(True) - self._leave_jitted_hook = self._gen_leave_jitted_hook_code(False) + if not self._debug: # if self._debug is already set it means that someone called # set_debug by hand before initializing the assembler. Leave it @@ -220,36 +217,17 @@ self.releasegil_addr = rffi.cast(lltype.Signed, releasegil_func) self.reacqgil_addr = rffi.cast(lltype.Signed, reacqgil_func) - def _gen_leave_jitted_hook_code(self, save_exc): - mc = ARMv7Builder() - if self.cpu.supports_floats: - floats = r.caller_vfp_resp - else: - floats = [] - with saved_registers(mc, r.caller_resp + [r.lr], floats): - addr = self.cpu.get_on_leave_jitted_int(save_exception=save_exc) - mc.BL(addr) - assert self._exit_code_addr != 0 - mc.B(self._exit_code_addr) - return mc.materialize(self.cpu.asmmemmgr, [], - self.cpu.gc_ll_descr.gcrootmap) - def _build_propagate_exception_path(self): if self.cpu.propagate_exception_v < 0: return # not supported (for tests, or non-translated) # mc = ARMv7Builder() - # call on_leave_jitted_save_exc() - if self.cpu.supports_floats: - floats = r.caller_vfp_resp - else: - floats = [] - with saved_registers(mc, r.caller_resp + [r.lr], floats): - addr = self.cpu.get_on_leave_jitted_int(save_exception=True, - default_to_memoryerror=True) - mc.BL(addr) - mc.gen_load_int(r.ip.value, self.cpu.propagate_exception_v) - mc.MOV_rr(r.r0.value, r.ip.value) + # + # Call the helper, which will return a dead frame object with + # the correct exception set, or MemoryError by default + # XXX make sure we return the correct value here + addr = rffi.cast(lltype.Signed, self.cpu.get_propagate_exception()) + mc.BL(addr) self.gen_func_epilog(mc=mc) self.propagate_exception_path = mc.materialize(self.cpu.asmmemmgr, []) @@ -282,12 +260,12 @@ # restore registers and return # We check for c.EQ here, meaning all bits zero in this case mc.POP([reg.value for reg in r.argument_regs] + [r.pc.value], cond=c.EQ) - # call on_leave_jitted_save_exc() - addr = self.cpu.get_on_leave_jitted_int(save_exception=True) + # + # Call the helper, which will return a dead frame object with + # the correct exception set, or MemoryError by default + addr = rffi.cast(lltype.Signed, self.cpu.get_propagate_exception()) mc.BL(addr) # - mc.gen_load_int(r.r0.value, self.cpu.propagate_exception_v) - # # footer -- note the ADD, which skips the return address of this # function, and will instead return to the caller's caller. Note # also that we completely ignore the saved arguments, because we @@ -339,39 +317,82 @@ def setup_failure_recovery(self): - @rgc.no_collect + #@rgc.no_collect -- XXX still true, but hacked gc_set_extra_threshold def failure_recovery_func(mem_loc, frame_pointer, stack_pointer): """mem_loc is a structure in memory describing where the values for the failargs are stored. frame loc is the address of the frame pointer for the frame to be decoded frame """ - vfp_registers = rffi.cast(rffi.LONGLONGP, stack_pointer) - registers = rffi.ptradd(vfp_registers, len(r.all_vfp_regs)) + vfp_registers = rffi.cast(rffi.LONGP, stack_pointer) + registers = rffi.ptradd(vfp_registers, 2*len(r.all_vfp_regs)) registers = rffi.cast(rffi.LONGP, registers) - return self.decode_registers_and_descr(mem_loc, frame_pointer, + bytecode = rffi.cast(rffi.UCHARP, mem_loc) + return self.grab_frame_values(self.cpu, bytecode, frame_pointer, registers, vfp_registers) + self.failure_recovery_code = [0, 0, 0, 0] self.failure_recovery_func = failure_recovery_func - recovery_func_sign = lltype.Ptr(lltype.FuncType([lltype.Signed] * 3, - lltype.Signed)) + _FAILURE_RECOVERY_FUNC = lltype.Ptr(lltype.FuncType([rffi.LONGP] * 3, + llmemory.GCREF)) - @rgc.no_collect - def decode_registers_and_descr(self, mem_loc, frame_pointer, + @staticmethod + #@rgc.no_collect -- XXX still true, but hacked gc_set_extra_threshold + def grab_frame_values(cpu, bytecode, frame_pointer, registers, vfp_registers): - """Decode locations encoded in memory at mem_loc and write the values - to the failboxes. Values for spilled vars and registers are stored on - stack at frame_loc """ - assert frame_pointer & 1 == 0 - self.fail_force_index = frame_pointer - bytecode = rffi.cast(rffi.UCHARP, mem_loc) + # no malloc allowed here!! xxx apart from one, hacking a lot + force_index = rffi.cast(lltype.Signed, frame_pointer) num = 0 - value = 0 - fvalue = 0 + deadframe = lltype.nullptr(jitframe.DEADFRAME) + # step 1: lots of mess just to count the final value of 'num' + bytecode1 = bytecode + while 1: + code = rffi.cast(lltype.Signed, bytecode1[0]) + bytecode1 = rffi.ptradd(bytecode1, 1) + if code >= AssemblerARM.CODE_FROMSTACK: + while code > 0x7F: + code = rffi.cast(lltype.Signed, bytecode1[0]) + bytecode1 = rffi.ptradd(bytecode1, 1) + else: + kind = code & 3 + if kind == AssemblerARM.DESCR_SPECIAL: + if code == AssemblerARM.CODE_HOLE: + num += 1 + continue + if code == AssemblerARM.CODE_INPUTARG: + continue + if code == AssemblerARM.CODE_FORCED: + # resuming from a GUARD_NOT_FORCED + token = force_index + deadframe = ( + cpu.assembler.force_token_to_dead_frame.pop(token)) + deadframe = lltype.cast_opaque_ptr( + jitframe.DEADFRAMEPTR, deadframe) + continue + assert code == AssemblerARM.CODE_STOP + break + num += 1 + + # allocate the deadframe + if not deadframe: + # Remove the "reserve" at the end of the nursery. This means + # that it is guaranteed that the following malloc() works + # without requiring a collect(), but it needs to be re-added + # as soon as possible. + cpu.gc_clear_extra_threshold() + assert num <= cpu.get_failargs_limit() + try: + deadframe = lltype.malloc(jitframe.DEADFRAME, num) + except MemoryError: + fatalerror("memory usage error in grab_frame_values") + # fill it code_inputarg = False - while True: + num = 0 + value_hi = 0 + while 1: + # decode the next instruction from the bytecode code = rffi.cast(lltype.Signed, bytecode[0]) bytecode = rffi.ptradd(bytecode, 1) - if code >= self.CODE_FROMSTACK: + if code >= AssemblerARM.CODE_FROMSTACK: if code > 0x7F: shift = 7 code &= 0x7F @@ -384,54 +405,63 @@ break # load the value from the stack kind = code & 3 - code = int((code - self.CODE_FROMSTACK) >> 2) + code = (code - AssemblerARM.CODE_FROMSTACK) >> 2 if code_inputarg: code = ~code code_inputarg = False - if kind == self.DESCR_FLOAT: - # we use code + 1 to get the hi word of the double worded float - stackloc = frame_pointer - get_fp_offset(int(code) + 1) - assert stackloc & 3 == 0 - fvalue = rffi.cast(rffi.LONGLONGP, stackloc)[0] - else: - stackloc = frame_pointer - get_fp_offset(int(code)) - assert stackloc & 1 == 0 - value = rffi.cast(rffi.LONGP, stackloc)[0] + stackloc = force_index - get_fp_offset(int(code)) + value = rffi.cast(rffi.LONGP, stackloc)[0] + if kind == AssemblerARM.DESCR_FLOAT: + assert WORD == 4 + value_hi = value + value = rffi.cast(rffi.LONGP, stackloc - WORD)[0] else: - # 'code' identifies a register: load its value kind = code & 3 - if kind == self.DESCR_SPECIAL: - if code == self.CODE_HOLE: + if kind == AssemblerARM.DESCR_SPECIAL: + if code == AssemblerARM.CODE_HOLE: num += 1 continue - if code == self.CODE_INPUTARG: + if code == AssemblerARM.CODE_INPUTARG: code_inputarg = True continue - assert code == self.CODE_STOP + if code == AssemblerARM.CODE_FORCED: + continue + assert code == AssemblerARM.CODE_STOP break + # 'code' identifies a register: load its value code >>= 2 - if kind == self.DESCR_FLOAT: - fvalue = vfp_registers[code] + if kind == AssemblerARM.DESCR_FLOAT: + if WORD == 4: + value = vfp_registers[2*code] + value_hi = vfp_registers[2*code + 1] + else: + value = registers[code] else: value = registers[code] # store the loaded value into fail_boxes_ - if kind == self.DESCR_FLOAT: - tgt = self.fail_boxes_float.get_addr_for_num(num) - rffi.cast(rffi.LONGLONGP, tgt)[0] = fvalue + if kind == AssemblerARM.DESCR_INT: + deadframe.jf_values[num].int = value + elif kind == AssemblerARM.DESCR_REF: + deadframe.jf_values[num].ref = rffi.cast(llmemory.GCREF, value) + elif kind == AssemblerARM.DESCR_FLOAT: + assert WORD == 4 + assert not longlong.is_64_bit + floatvalue = rffi.cast(lltype.SignedLongLong, value_hi) + floatvalue <<= 32 + floatvalue |= rffi.cast(lltype.SignedLongLong, + rffi.cast(lltype.Unsigned, value)) + deadframe.jf_values[num].float = floatvalue else: - if kind == self.DESCR_INT: - tgt = self.fail_boxes_int.get_addr_for_num(num) - elif kind == self.DESCR_REF: - assert (value & 3) == 0, "misaligned pointer" - tgt = self.fail_boxes_ptr.get_addr_for_num(num) - else: - assert 0, "bogus kind" - rffi.cast(rffi.LONGP, tgt)[0] = value + assert 0, "bogus kind" num += 1 - self.fail_boxes_count = num + # + assert num == len(deadframe.jf_values) + if not we_are_translated(): + assert bytecode[4] == 0xCC fail_index = rffi.cast(rffi.INTP, bytecode)[0] - fail_index = rffi.cast(lltype.Signed, fail_index) - return fail_index + fail_descr = cpu.get_fail_descr_from_number(fail_index) + deadframe.jf_descr = fail_descr.hide(cpu) + return lltype.cast_opaque_ptr(llmemory.GCREF, deadframe) def decode_inputargs(self, code): descr_to_box_type = [REF, INT, FLOAT] @@ -492,10 +522,10 @@ # are stored in r0 and r1. mc.SUB_rr(r.r0.value, r.r1.value, r.r0.value) addr = self.cpu.gc_ll_descr.get_malloc_slowpath_addr() - for reg, ofs in ARMv7RegisterManager.REGLOC_TO_COPY_AREA_OFS.items(): + for reg, ofs in CoreRegisterManager.REGLOC_TO_COPY_AREA_OFS.items(): mc.STR_ri(reg.value, r.fp.value, imm=ofs) mc.BL(addr) - for reg, ofs in ARMv7RegisterManager.REGLOC_TO_COPY_AREA_OFS.items(): + for reg, ofs in CoreRegisterManager.REGLOC_TO_COPY_AREA_OFS.items(): mc.LDR_ri(reg.value, r.fp.value, imm=ofs) mc.CMP_ri(r.r0.value, 0) @@ -514,12 +544,26 @@ self.mc.CMP_ri(r.r0.value, 0) self.mc.B(self.propagate_exception_path, c=c.EQ) - def _gen_exit_path(self): + def _build_failure_recovery(self, exc, withfloats=False): mc = ARMv7Builder() - decode_registers_addr = llhelper(self.recovery_func_sign, + failure_recovery = llhelper(self._FAILURE_RECOVERY_FUNC, self.failure_recovery_func) self._insert_checks(mc) - with saved_registers(mc, r.all_regs, r.all_vfp_regs): + if withfloats: + f = r.all_vfp_regs + else: + f = [] + with saved_registers(mc, r.all_regs, f): + if exc: + # We might have an exception pending. Load it into r4 + # (this is a register saved across calls) + mc.gen_load_int(r.r5.value, self.cpu.pos_exc_value()) + mc.LDR_ri(r.r4.value, self.cpu.pos_exc_value()) + # clear the exc flags + mc.gen_load_int(r.r6.value, 0) + mc.STR_ri(r.r6.value, r.r5.value) + mc.gen_load_int(r.r5.value, self.cpu.pos_exception()) + mc.STR_ri(r.r6.value, r.r5.value) # move mem block address, to r0 to pass as mc.MOV_rr(r.r0.value, r.lr.value) # pass the current frame pointer as second param @@ -527,23 +571,32 @@ # pass the current stack pointer as third param mc.MOV_rr(r.r2.value, r.sp.value) self._insert_checks(mc) - mc.BL(rffi.cast(lltype.Signed, decode_registers_addr)) + mc.BL(rffi.cast(lltype.Signed, failure_recovery)) + if exc: + # save ebx into 'jf_guard_exc' + from pypy.jit.backend.llsupport.descr import unpack_fielddescr + descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) + offset, size, _ = unpack_fielddescr(descrs.jf_guard_exc) + mc.STR_rr(r.r4.value, r.r0.value, offset, cond=c.AL) mc.MOV_rr(r.ip.value, r.r0.value) mc.MOV_rr(r.r0.value, r.ip.value) self.gen_func_epilog(mc=mc) - return mc.materialize(self.cpu.asmmemmgr, [], + rawstart = mc.materialize(self.cpu.asmmemmgr, [], self.cpu.gc_ll_descr.gcrootmap) + self.failure_recovery_code[exc + 2 * withfloats] = rawstart + self.mc = None DESCR_REF = 0x00 DESCR_INT = 0x01 DESCR_FLOAT = 0x02 DESCR_SPECIAL = 0x03 CODE_FROMSTACK = 64 - CODE_STOP = 0 | DESCR_SPECIAL - CODE_HOLE = 4 | DESCR_SPECIAL - CODE_INPUTARG = 8 | DESCR_SPECIAL + CODE_STOP = 0 | DESCR_SPECIAL + CODE_HOLE = 4 | DESCR_SPECIAL + CODE_INPUTARG = 8 | DESCR_SPECIAL + CODE_FORCED = 12 | DESCR_SPECIAL #XXX where should this be written? - def gen_descr_encoding(self, descr, failargs, locs): + def write_failure_recovery_description(self, descr, failargs, locs): assert self.mc is not None for i in range(len(failargs)): arg = failargs[i] @@ -575,26 +628,33 @@ self.mc.writechar(chr(n)) self.mc.writechar(chr(self.CODE_STOP)) - fdescr = self.cpu.get_fail_descr_number(descr) - self.mc.write32(fdescr) + + def generate_quick_failure(self, guardtok, fcond=c.AL): + assert isinstance(guardtok.save_exc, bool) + fail_index = self.cpu.get_fail_descr_number(guardtok.descr) + startpos = self.mc.currpos() + withfloats = False + for box in guardtok.failargs: + if box is not None and box.type == FLOAT: + withfloats = True + break + exc = guardtok.save_exc + target = self.failure_recovery_code[exc + 2 * withfloats] + assert target != 0 + self.mc.BL(target) + # write tight data that describes the failure recovery + if guardtok.is_guard_not_forced: + self.mc.writechar(chr(self.CODE_FORCED)) + self.write_failure_recovery_description(guardtok.descr, + guardtok.failargs, guardtok.faillocs[1:]) + self.mc.write32(fail_index) + # for testing the decoding, write a final byte 0xCC + if not we_are_translated(): + self.mc.writechar('\xCC') + faillocs = [loc for loc in guardtok.faillocs if loc is not None] + guardtok.descr._arm_debug_faillocs = faillocs self.align() - - # assert that the fail_boxes lists are big enough - assert len(failargs) <= self.fail_boxes_int.SIZE - - def _gen_path_to_exit_path(self, descr, args, arglocs, - save_exc, fcond=c.AL): - assert isinstance(save_exc, bool) - self.gen_exit_code(self.mc, save_exc, fcond) - self.gen_descr_encoding(descr, args, arglocs[1:]) - - def gen_exit_code(self, mc, save_exc, fcond=c.AL): - assert isinstance(save_exc, bool) - if save_exc: - path = self._leave_jitted_hook_save_exc - else: - path = self._leave_jitted_hook - mc.BL(path) + return startpos def align(self): while(self.mc.currpos() % FUNC_ALIGN != 0): @@ -841,15 +901,10 @@ def write_pending_failure_recoveries(self): for tok in self.pending_guards: - descr = tok.descr #generate the exit stub and the encoded representation - pos = self.mc.currpos() - tok.pos_recovery_stub = pos - - self._gen_path_to_exit_path(descr, tok.failargs, - tok.faillocs, save_exc=tok.save_exc) + tok.pos_recovery_stub = self.generate_quick_failure(tok) # store info on the descr - descr._arm_current_frame_depth = tok.faillocs[0].getint() + tok.descr._arm_current_frame_depth = tok.faillocs[0].getint() def process_pending_guards(self, block_start): clt = self.current_clt @@ -860,7 +915,7 @@ descr._arm_failure_recovery_block = failure_recovery_pos relative_offset = tok.pos_recovery_stub - tok.offset guard_pos = block_start + tok.offset - if not tok.is_invalidate: + if not tok.is_guard_not_invalidated: # patch the guard jumpt to the stub # overwrite the generate NOP with a B_offs to the pos of the # stub @@ -1255,11 +1310,6 @@ else: raise AssertionError('Trying to pop to an invalid location') - def leave_jitted_hook(self): - ptrs = self.fail_boxes_ptr.ar - llop.gc_assume_young_pointers(lltype.Void, - llmemory.cast_ptr_to_adr(ptrs)) - def malloc_cond(self, nursery_free_adr, nursery_top_adr, size): assert size & (WORD-1) == 0 # must be correctly aligned diff --git a/pypy/jit/backend/arm/helper/regalloc.py b/pypy/jit/backend/arm/helper/regalloc.py --- a/pypy/jit/backend/arm/helper/regalloc.py +++ b/pypy/jit/backend/arm/helper/regalloc.py @@ -32,14 +32,14 @@ imm_a0 = check_imm_box(a0, imm_size, allow_zero=allow_zero) imm_a1 = check_imm_box(a1, imm_size, allow_zero=allow_zero) if not imm_a0 and imm_a1: - l0 = self._ensure_value_is_boxed(a0) + l0 = self.make_sure_var_in_reg(a0) l1 = self.convert_to_imm(a1) elif commutative and imm_a0 and not imm_a1: l1 = self.convert_to_imm(a0) - l0 = self._ensure_value_is_boxed(a1, boxes) + l0 = self.make_sure_var_in_reg(a1, boxes) else: - l0 = self._ensure_value_is_boxed(a0, boxes) - l1 = self._ensure_value_is_boxed(a1, boxes) + l0 = self.make_sure_var_in_reg(a0, boxes) + l1 = self.make_sure_var_in_reg(a1, boxes) self.possibly_free_vars_for_op(op) self.free_temp_vars() res = self.force_allocate_reg(op.result, boxes) @@ -52,10 +52,10 @@ if guard: def f(self, op, guard_op, fcond): locs = [] - loc1 = self._ensure_value_is_boxed(op.getarg(0)) + loc1 = self.make_sure_var_in_reg(op.getarg(0)) locs.append(loc1) if base: - loc2 = self._ensure_value_is_boxed(op.getarg(1)) + loc2 = self.make_sure_var_in_reg(op.getarg(1)) locs.append(loc2) self.possibly_free_vars_for_op(op) self.free_temp_vars() @@ -70,10 +70,10 @@ else: def f(self, op, fcond): locs = [] - loc1 = self._ensure_value_is_boxed(op.getarg(0)) + loc1 = self.make_sure_var_in_reg(op.getarg(0)) locs.append(loc1) if base: - loc2 = self._ensure_value_is_boxed(op.getarg(1)) + loc2 = self.make_sure_var_in_reg(op.getarg(1)) locs.append(loc2) self.possibly_free_vars_for_op(op) self.free_temp_vars() @@ -111,11 +111,11 @@ arg0, arg1 = boxes imm_a1 = check_imm_box(arg1) - l0 = self._ensure_value_is_boxed(arg0, forbidden_vars=boxes) + l0 = self.make_sure_var_in_reg(arg0, forbidden_vars=boxes) if imm_a1: l1 = self.convert_to_imm(arg1) else: - l1 = self._ensure_value_is_boxed(arg1, forbidden_vars=boxes) + l1 = self.make_sure_var_in_reg(arg1, forbidden_vars=boxes) self.possibly_free_vars_for_op(op) self.free_temp_vars() @@ -134,7 +134,7 @@ assert fcond is not None a0 = op.getarg(0) assert isinstance(a0, Box) - reg = self._ensure_value_is_boxed(a0) + reg = self.make_sure_var_in_reg(a0) self.possibly_free_vars_for_op(op) if guard_op is None: res = self.force_allocate_reg(op.result, [a0]) diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py --- a/pypy/jit/backend/arm/opassembler.py +++ b/pypy/jit/backend/arm/opassembler.py @@ -16,6 +16,7 @@ gen_emit_unary_float_op, saved_registers, count_reg_args) +from pypy.jit.backend.arm.helper.regalloc import check_imm_arg from pypy.jit.backend.arm.codebuilder import ARMv7Builder, OverwritingBuilder from pypy.jit.backend.arm.jump import remap_frame_layout from pypy.jit.backend.arm.regalloc import TempInt, TempPtr @@ -27,18 +28,20 @@ from pypy.jit.metainterp.history import JitCellToken, TargetToken from pypy.jit.metainterp.resoperation import rop from pypy.rlib.objectmodel import we_are_translated -from pypy.rpython.lltypesystem import rstr +from pypy.rlib import rgc +from pypy.rpython.lltypesystem import rstr, rffi, lltype, llmemory NO_FORCE_INDEX = -1 class GuardToken(object): - def __init__(self, descr, failargs, faillocs, offset, - save_exc, fcond=c.AL, is_invalidate=False): + def __init__(self, descr, failargs, faillocs, offset, save_exc, fcond=c.AL, + is_guard_not_invalidated=False, is_guard_not_forced=False): assert isinstance(save_exc, bool) self.descr = descr self.offset = offset - self.is_invalidate = is_invalidate + self.is_guard_not_invalidated = is_guard_not_invalidated + self.is_guard_not_forced = is_guard_not_forced self.failargs = failargs self.faillocs = faillocs self.save_exc = save_exc @@ -190,7 +193,8 @@ return fcond def _emit_guard(self, op, arglocs, fcond, save_exc, - is_guard_not_invalidated=False): + is_guard_not_invalidated=False, + is_guard_not_forced=False): assert isinstance(save_exc, bool) assert isinstance(fcond, int) descr = op.getdescr() @@ -210,7 +214,8 @@ faillocs=arglocs, offset=pos, save_exc=save_exc, - is_invalidate=is_guard_not_invalidated, + is_guard_not_invalidated=is_guard_not_invalidated, + is_guard_not_forced=is_guard_not_forced, fcond=fcond)) return c.AL @@ -312,49 +317,10 @@ return fcond def emit_op_finish(self, op, arglocs, regalloc, fcond): - for i in range(len(arglocs) - 1): - loc = arglocs[i] - box = op.getarg(i) - if loc is None: - continue - if loc.is_reg(): - if box.type == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(i) - elif box.type == INT: - adr = self.fail_boxes_int.get_addr_for_num(i) - else: - assert 0 - self.mc.gen_load_int(r.ip.value, adr) - self.mc.STR_ri(loc.value, r.ip.value) - elif loc.is_vfp_reg(): - assert box.type == FLOAT - adr = self.fail_boxes_float.get_addr_for_num(i) - self.mc.gen_load_int(r.ip.value, adr) - self.mc.VSTR(loc.value, r.ip.value) - elif loc.is_stack() or loc.is_imm() or loc.is_imm_float(): - if box.type == FLOAT: - adr = self.fail_boxes_float.get_addr_for_num(i) - self.mov_loc_loc(loc, r.vfp_ip) - self.mc.gen_load_int(r.ip.value, adr) - self.mc.VSTR(r.vfp_ip.value, r.ip.value) - elif box.type == REF or box.type == INT: - if box.type == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(i) - elif box.type == INT: - adr = self.fail_boxes_int.get_addr_for_num(i) - else: - assert 0 - self.mov_loc_loc(loc, r.ip) - self.mc.gen_load_int(r.lr.value, adr) - self.mc.STR_ri(r.ip.value, r.lr.value) - else: - assert 0 - # note: no exception should currently be set in llop.get_exception_addr - # even if this finish may be an exit_frame_with_exception (in this case - # the exception instance is in arglocs[0]). - addr = self.cpu.get_on_leave_jitted_int(save_exception=False) - self.mc.BL(addr) - self.mc.gen_load_int(r.r0.value, arglocs[-1].value) + [argloc] = arglocs + if argloc is not r.r0: #XXX verify this + self.mov_loc_loc(argloc, r.r0, fcond) + # exit function self.gen_func_epilog() return fcond @@ -988,8 +954,8 @@ def _emit_copystrcontent(self, op, regalloc, fcond, is_unicode): # compute the source address args = op.getarglist() - base_loc = regalloc._ensure_value_is_boxed(args[0], args) - ofs_loc = regalloc._ensure_value_is_boxed(args[2], args) + base_loc = regalloc.make_sure_var_in_reg(args[0], args) + ofs_loc = regalloc.make_sure_var_in_reg(args[2], args) assert args[0] is not args[1] # forbidden case of aliasing regalloc.possibly_free_var(args[0]) regalloc.free_temp_vars() @@ -1009,8 +975,8 @@ dstaddr_loc = regalloc.force_allocate_reg(dstaddr_box, selected_reg=r.r0) forbidden_vars.append(dstaddr_box) - base_loc = regalloc._ensure_value_is_boxed(args[1], forbidden_vars) - ofs_loc = regalloc._ensure_value_is_boxed(args[3], forbidden_vars) + base_loc = regalloc.make_sure_var_in_reg(args[1], forbidden_vars) + ofs_loc = regalloc.make_sure_var_in_reg(args[3], forbidden_vars) assert base_loc.is_reg() assert ofs_loc.is_reg() regalloc.possibly_free_var(args[1]) @@ -1026,7 +992,7 @@ # need the box here if isinstance(args[4], Box): length_box = args[4] - length_loc = regalloc._ensure_value_is_boxed(args[4], + length_loc = regalloc.make_sure_var_in_reg(args[4], forbidden_vars) else: length_box = TempInt() @@ -1152,8 +1118,26 @@ value = self.cpu.done_with_this_frame_float_v else: raise AssertionError(kind) - self.mc.gen_load_int(r.ip.value, value) - self.mc.CMP_rr(tmploc.value, r.ip.value) + from pypy.jit.backend.llsupport.descr import unpack_fielddescr + from pypy.jit.backend.llsupport.descr import unpack_interiorfielddescr + descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) + _offset, _size, _ = unpack_fielddescr(descrs.jf_descr) + fail_descr = self.cpu.get_fail_descr_from_number(value) + value = fail_descr.hide(self.cpu) + rgc._make_sure_does_not_move(value) + value = rffi.cast(lltype.Signed, value) + + if check_imm_arg(_offset): + self.mc.LDR_ri(r.ip.value, tmploc.value, imm=_offset) + else: + self.mc.gen_load_int(r.ip.value, _offset) + self.mc.LDR_rr(r.ip.value, tmploc.value, r.ip.value) + if check_imm_arg(value): + self.mc.CMP_ri(r.ip.value, imm=value) + else: + self.mc.gen_load_int(r.lr.value, value) + self.mc.CMP_rr(r.lr.value, r.ip.value) + #if values are equal we take the fast path # Slow path, calling helper @@ -1180,19 +1164,31 @@ if op.result is not None: # load the return value from fail_boxes_xxx[0] kind = op.result.type - if kind == INT: - adr = self.fail_boxes_int.get_addr_for_num(0) - elif kind == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(0) - elif kind == FLOAT: - adr = self.fail_boxes_float.get_addr_for_num(0) + if kind == FLOAT: + t = unpack_interiorfielddescr(descrs.as_float)[0] + if not check_imm_arg(t): + self.mc.gen_load_int(r.ip.value, t, cond=fast_path_cond) + self.mc.ADD_rr(r.ip.value, r.r0.value, r.ip.value, + cond=fast_path_cond) + t = 0 + base = r.ip + else: + base = r.r0 + self.mc.VLDR(resloc.value, base.value, imm=t, + cond=fast_path_cond) else: - raise AssertionError(kind) - self.mc.gen_load_int(r.ip.value, adr, cond=fast_path_cond) - if op.result.type == FLOAT: - self.mc.VLDR(resloc.value, r.ip.value, cond=fast_path_cond) - else: - self.mc.LDR_ri(resloc.value, r.ip.value, cond=fast_path_cond) + assert resloc is r.r0 + if kind == INT: + t = unpack_interiorfielddescr(descrs.as_int)[0] + else: + t = unpack_interiorfielddescr(descrs.as_ref)[0] + if not check_imm_arg(t): + self.mc.gen_load_int(r.ip.value, t, cond=fast_path_cond) + self.mc.LDR_rr(resloc.value, resloc.value, r.ip.value, + cond=fast_path_cond) + else: + self.mc.LDR_ri(resloc.value, resloc.value, imm=t, + cond=fast_path_cond) # jump to merge point jmp_pos = self.mc.currpos() self.mc.BKPT() @@ -1266,7 +1262,8 @@ self.mc.LDR_ri(r.ip.value, r.fp.value) self.mc.CMP_ri(r.ip.value, 0) - self._emit_guard(guard_op, arglocs[1 + numargs:], c.GE, save_exc=True) + self._emit_guard(guard_op, arglocs[1 + numargs:], c.GE, + save_exc=True, is_guard_not_forced=True) return fcond def emit_guard_call_release_gil(self, op, guard_op, arglocs, regalloc, diff --git a/pypy/jit/backend/arm/regalloc.py b/pypy/jit/backend/arm/regalloc.py --- a/pypy/jit/backend/arm/regalloc.py +++ b/pypy/jit/backend/arm/regalloc.py @@ -84,8 +84,28 @@ def void(self, op, fcond): return [] +class ARMRegisterManager(RegisterManager): + def return_constant(self, v, forbidden_vars=[], selected_reg=None): + self._check_type(v) + if isinstance(v, Const): + if isinstance(v, ConstPtr): + tp = REF + elif isinstance(v, ConstFloat): + tp = FLOAT + else: + tp = INT + loc = self.get_scratch_reg(tp, + self.temp_boxes + forbidden_vars, + selected_reg=selected_reg) + immvalue = self.convert_to_imm(v) + self.assembler.load(loc, immvalue) + return loc + else: + return RegisterManager.return_constant(self, v, + forbidden_vars, selected_reg) -class VFPRegisterManager(RegisterManager): + +class VFPRegisterManager(ARMRegisterManager): all_regs = r.all_vfp_regs box_types = [FLOAT] save_around_call_regs = r.all_vfp_regs @@ -107,20 +127,7 @@ reg = self.force_allocate_reg(v, selected_reg=r.d0) return reg - def ensure_value_is_boxed(self, thing, forbidden_vars=[]): - loc = None - if isinstance(thing, Const): - assert isinstance(thing, ConstFloat) - loc = self.get_scratch_reg(FLOAT, self.temp_boxes + forbidden_vars) - immvalue = self.convert_to_imm(thing) - self.assembler.load(loc, immvalue) - else: - loc = self.make_sure_var_in_reg(thing, - forbidden_vars=self.temp_boxes + forbidden_vars) - return loc - - def get_scratch_reg(self, type=FLOAT, forbidden_vars=[], - selected_reg=None): + def get_scratch_reg(self, type=FLOAT, forbidden_vars=[], selected_reg=None): assert type == FLOAT # for now box = TempFloat() self.temp_boxes.append(box) @@ -129,7 +136,7 @@ return reg -class ARMv7RegisterManager(RegisterManager): +class CoreRegisterManager(ARMRegisterManager): all_regs = r.all_regs box_types = None # or a list of acceptable types no_lower_byte_regs = all_regs @@ -162,22 +169,6 @@ return locations.ImmLocation(rffi.cast(lltype.Signed, c.value)) assert 0 - def ensure_value_is_boxed(self, thing, forbidden_vars=None): - loc = None - if isinstance(thing, Const): - if isinstance(thing, ConstPtr): - tp = REF - else: - tp = INT - loc = self.get_scratch_reg(tp, forbidden_vars=self.temp_boxes - + forbidden_vars) - immvalue = self.convert_to_imm(thing) - self.assembler.load(loc, immvalue) - else: - loc = self.make_sure_var_in_reg(thing, - forbidden_vars=self.temp_boxes + forbidden_vars) - return loc - def get_scratch_reg(self, type=INT, forbidden_vars=[], selected_reg=None): assert type == INT or type == REF box = TempBox() @@ -277,7 +268,12 @@ def make_sure_var_in_reg(self, var, forbidden_vars=[], selected_reg=None, need_lower_byte=False): - assert 0, 'should not be called directly' + if var.type == FLOAT: + return self.vfprm.make_sure_var_in_reg(var, forbidden_vars, + selected_reg, need_lower_byte) + else: + return self.rm.make_sure_var_in_reg(var, forbidden_vars, + selected_reg, need_lower_byte) def convert_to_imm(self, value): if isinstance(value, ConstInt): @@ -294,7 +290,7 @@ fm = self.frame_manager asm = self.assembler self.vfprm = VFPRegisterManager(longevity, fm, asm) - self.rm = ARMv7RegisterManager(longevity, fm, asm) + self.rm = CoreRegisterManager(longevity, fm, asm) def prepare_loop(self, inputargs, operations): self._prepare(inputargs, operations) @@ -426,12 +422,6 @@ self.rm.before_call(force_store, save_all_regs) self.vfprm.before_call(force_store, save_all_regs) - def _ensure_value_is_boxed(self, thing, forbidden_vars=[]): - if thing.type == FLOAT: - return self.vfprm.ensure_value_is_boxed(thing, forbidden_vars) - else: - return self.rm.ensure_value_is_boxed(thing, forbidden_vars) - def _sync_var(self, v): if v.type == FLOAT: self.vfprm._sync_var(v) @@ -444,14 +434,14 @@ imm_a0 = check_imm_box(a0) imm_a1 = check_imm_box(a1) if not imm_a0 and imm_a1: - l0 = self._ensure_value_is_boxed(a0, boxes) + l0 = self.make_sure_var_in_reg(a0, boxes) l1 = self.convert_to_imm(a1) elif imm_a0 and not imm_a1: l0 = self.convert_to_imm(a0) - l1 = self._ensure_value_is_boxed(a1, boxes) + l1 = self.make_sure_var_in_reg(a1, boxes) else: - l0 = self._ensure_value_is_boxed(a0, boxes) - l1 = self._ensure_value_is_boxed(a1, boxes) + l0 = self.make_sure_var_in_reg(a0, boxes) + l1 = self.make_sure_var_in_reg(a1, boxes) return [l0, l1] def prepare_op_int_add(self, op, fcond): @@ -466,14 +456,14 @@ imm_a0 = check_imm_box(a0) imm_a1 = check_imm_box(a1) if not imm_a0 and imm_a1: - l0 = self._ensure_value_is_boxed(a0, boxes) + l0 = self.make_sure_var_in_reg(a0, boxes) l1 = self.convert_to_imm(a1) elif imm_a0 and not imm_a1: l0 = self.convert_to_imm(a0) - l1 = self._ensure_value_is_boxed(a1, boxes) + l1 = self.make_sure_var_in_reg(a1, boxes) else: - l0 = self._ensure_value_is_boxed(a0, boxes) - l1 = self._ensure_value_is_boxed(a1, boxes) + l0 = self.make_sure_var_in_reg(a0, boxes) + l1 = self.make_sure_var_in_reg(a1, boxes) return [l0, l1] def prepare_op_int_sub(self, op, fcond): @@ -487,8 +477,8 @@ boxes = op.getarglist() a0, a1 = boxes - reg1 = self._ensure_value_is_boxed(a0, forbidden_vars=boxes) - reg2 = self._ensure_value_is_boxed(a1, forbidden_vars=boxes) + reg1 = self.make_sure_var_in_reg(a0, forbidden_vars=boxes) + reg2 = self.make_sure_var_in_reg(a1, forbidden_vars=boxes) self.possibly_free_vars(boxes) self.possibly_free_vars_for_op(op) @@ -497,14 +487,14 @@ return [reg1, reg2, res] def prepare_op_int_force_ge_zero(self, op, fcond): - argloc = self._ensure_value_is_boxed(op.getarg(0)) + argloc = self.make_sure_var_in_reg(op.getarg(0)) resloc = self.force_allocate_reg(op.result, [op.getarg(0)]) return [argloc, resloc] def prepare_guard_int_mul_ovf(self, op, guard, fcond): boxes = op.getarglist() - reg1 = self._ensure_value_is_boxed(boxes[0], forbidden_vars=boxes) - reg2 = self._ensure_value_is_boxed(boxes[1], forbidden_vars=boxes) + reg1 = self.make_sure_var_in_reg(boxes[0], forbidden_vars=boxes) + reg2 = self.make_sure_var_in_reg(boxes[1], forbidden_vars=boxes) res = self.force_allocate_reg(op.result) return self._prepare_guard(guard, [reg1, reg2, res]) @@ -576,7 +566,7 @@ prepare_guard_int_is_zero = prepare_op_unary_cmp('int_is_zero') def prepare_op_int_neg(self, op, fcond): - l0 = self._ensure_value_is_boxed(op.getarg(0)) + l0 = self.make_sure_var_in_reg(op.getarg(0)) self.possibly_free_vars_for_op(op) self.free_temp_vars() resloc = self.force_allocate_reg(op.result) @@ -629,15 +619,15 @@ def _prepare_llong_binop_xx(self, op, fcond): # arg 0 is the address of the function - loc0 = self._ensure_value_is_boxed(op.getarg(1)) - loc1 = self._ensure_value_is_boxed(op.getarg(2)) + loc0 = self.make_sure_var_in_reg(op.getarg(1)) + loc1 = self.make_sure_var_in_reg(op.getarg(2)) self.possibly_free_vars_for_op(op) self.free_temp_vars() res = self.vfprm.force_allocate_reg(op.result) return [loc0, loc1, res] def _prepare_llong_to_int(self, op, fcond): - loc0 = self._ensure_value_is_boxed(op.getarg(1)) + loc0 = self.make_sure_var_in_reg(op.getarg(1)) res = self.force_allocate_reg(op.result) return [loc0, res] @@ -654,18 +644,12 @@ return args def prepare_op_finish(self, op, fcond): - args = [None] * (op.numargs() + 1) - for i in range(op.numargs()): - arg = op.getarg(i) - if arg: - args[i] = self.loc(arg) - self.possibly_free_var(arg) - n = self.cpu.get_fail_descr_number(op.getdescr()) - args[-1] = imm(n) - return args + loc = self.loc(op.getarg(0)) + self.possibly_free_var(op.getarg(0)) + return [loc] def prepare_op_guard_true(self, op, fcond): - l0 = self._ensure_value_is_boxed(op.getarg(0)) + l0 = self.make_sure_var_in_reg(op.getarg(0)) args = self._prepare_guard(op, [l0]) return args @@ -677,9 +661,9 @@ boxes = op.getarglist() a0, a1 = boxes imm_a1 = check_imm_box(a1) - l0 = self._ensure_value_is_boxed(a0, boxes) + l0 = self.make_sure_var_in_reg(a0, boxes) if not imm_a1: - l1 = self._ensure_value_is_boxed(a1, boxes) + l1 = self.make_sure_var_in_reg(a1, boxes) else: l1 = self.convert_to_imm(a1) assert op.result is None @@ -699,7 +683,7 @@ def prepare_op_guard_exception(self, op, fcond): boxes = op.getarglist() arg0 = ConstInt(rffi.cast(lltype.Signed, op.getarg(0).getint())) - loc = self._ensure_value_is_boxed(arg0) + loc = self.make_sure_var_in_reg(arg0) loc1 = self.get_scratch_reg(INT, boxes) if op.result in self.longevity: resloc = self.force_allocate_reg(op.result, boxes) @@ -713,7 +697,7 @@ return arglocs def prepare_op_guard_no_exception(self, op, fcond): - loc = self._ensure_value_is_boxed( + loc = self.make_sure_var_in_reg( ConstInt(self.cpu.pos_exception())) arglocs = self._prepare_guard(op, [loc]) return arglocs @@ -727,7 +711,7 @@ assert isinstance(op.getarg(0), Box) boxes = op.getarglist() - x = self._ensure_value_is_boxed(boxes[0], boxes) + x = self.make_sure_var_in_reg(boxes[0], boxes) y_val = rffi.cast(lltype.Signed, op.getarg(1).getint()) arglocs = [x, None, None] @@ -837,8 +821,8 @@ boxes = op.getarglist() a0, a1 = boxes ofs, size, sign = unpack_fielddescr(op.getdescr()) - base_loc = self._ensure_value_is_boxed(a0, boxes) - value_loc = self._ensure_value_is_boxed(a1, boxes) + base_loc = self.make_sure_var_in_reg(a0, boxes) + value_loc = self.make_sure_var_in_reg(a1, boxes) if check_imm_arg(ofs): ofs_loc = imm(ofs) else: @@ -851,7 +835,7 @@ def prepare_op_getfield_gc(self, op, fcond): a0 = op.getarg(0) ofs, size, sign = unpack_fielddescr(op.getdescr()) - base_loc = self._ensure_value_is_boxed(a0) + base_loc = self.make_sure_var_in_reg(a0) immofs = imm(ofs) if check_imm_arg(ofs): ofs_loc = immofs @@ -871,8 +855,8 @@ t = unpack_interiorfielddescr(op.getdescr()) ofs, itemsize, fieldsize, sign = t args = op.getarglist() - base_loc = self._ensure_value_is_boxed(op.getarg(0), args) - index_loc = self._ensure_value_is_boxed(op.getarg(1), args) + base_loc = self.make_sure_var_in_reg(op.getarg(0), args) + index_loc = self.make_sure_var_in_reg(op.getarg(1), args) immofs = imm(ofs) if check_imm_arg(ofs): ofs_loc = immofs @@ -889,9 +873,9 @@ t = unpack_interiorfielddescr(op.getdescr()) ofs, itemsize, fieldsize, sign = t args = op.getarglist() - base_loc = self._ensure_value_is_boxed(op.getarg(0), args) - index_loc = self._ensure_value_is_boxed(op.getarg(1), args) - value_loc = self._ensure_value_is_boxed(op.getarg(2), args) + base_loc = self.make_sure_var_in_reg(op.getarg(0), args) + index_loc = self.make_sure_var_in_reg(op.getarg(1), args) + value_loc = self.make_sure_var_in_reg(op.getarg(2), args) immofs = imm(ofs) if check_imm_arg(ofs): ofs_loc = immofs @@ -907,7 +891,7 @@ assert isinstance(arraydescr, ArrayDescr) ofs = arraydescr.lendescr.offset arg = op.getarg(0) - base_loc = self._ensure_value_is_boxed(arg) + base_loc = self.make_sure_var_in_reg(arg) self.possibly_free_vars_for_op(op) self.free_temp_vars() res = self.force_allocate_reg(op.result) @@ -917,9 +901,9 @@ size, ofs, _ = unpack_arraydescr(op.getdescr()) scale = get_scale(size) args = op.getarglist() - base_loc = self._ensure_value_is_boxed(args[0], args) - ofs_loc = self._ensure_value_is_boxed(args[1], args) - value_loc = self._ensure_value_is_boxed(args[2], args) + base_loc = self.make_sure_var_in_reg(args[0], args) + ofs_loc = self.make_sure_var_in_reg(args[1], args) + value_loc = self.make_sure_var_in_reg(args[2], args) assert check_imm_arg(ofs) return [value_loc, base_loc, ofs_loc, imm(scale), imm(ofs)] prepare_op_setarrayitem_raw = prepare_op_setarrayitem_gc @@ -929,8 +913,8 @@ boxes = op.getarglist() size, ofs, _ = unpack_arraydescr(op.getdescr()) scale = get_scale(size) - base_loc = self._ensure_value_is_boxed(boxes[0], boxes) - ofs_loc = self._ensure_value_is_boxed(boxes[1], boxes) + base_loc = self.make_sure_var_in_reg(boxes[0], boxes) + ofs_loc = self.make_sure_var_in_reg(boxes[1], boxes) self.possibly_free_vars_for_op(op) self.free_temp_vars() res = self.force_allocate_reg(op.result) @@ -944,7 +928,7 @@ def prepare_op_strlen(self, op, fcond): args = op.getarglist() - l0 = self._ensure_value_is_boxed(op.getarg(0)) + l0 = self.make_sure_var_in_reg(op.getarg(0)) basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, self.cpu.translate_support_code) immofs = imm(ofs_length) @@ -963,14 +947,14 @@ def prepare_op_strgetitem(self, op, fcond): boxes = op.getarglist() - base_loc = self._ensure_value_is_boxed(boxes[0]) + base_loc = self.make_sure_var_in_reg(boxes[0]) a1 = boxes[1] imm_a1 = check_imm_box(a1) if imm_a1: ofs_loc = self.convert_to_imm(a1) else: - ofs_loc = self._ensure_value_is_boxed(a1, boxes) + ofs_loc = self.make_sure_var_in_reg(a1, boxes) self.possibly_free_vars_for_op(op) self.free_temp_vars() @@ -983,9 +967,9 @@ def prepare_op_strsetitem(self, op, fcond): boxes = op.getarglist() - base_loc = self._ensure_value_is_boxed(boxes[0], boxes) - ofs_loc = self._ensure_value_is_boxed(boxes[1], boxes) - value_loc = self._ensure_value_is_boxed(boxes[2], boxes) + base_loc = self.make_sure_var_in_reg(boxes[0], boxes) + ofs_loc = self.make_sure_var_in_reg(boxes[1], boxes) + value_loc = self.make_sure_var_in_reg(boxes[2], boxes) basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, self.cpu.translate_support_code) assert itemsize == 1 @@ -995,7 +979,7 @@ prepare_op_copyunicodecontent = void def prepare_op_unicodelen(self, op, fcond): - l0 = self._ensure_value_is_boxed(op.getarg(0)) + l0 = self.make_sure_var_in_reg(op.getarg(0)) basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, self.cpu.translate_support_code) immofs = imm(ofs_length) @@ -1012,8 +996,8 @@ def prepare_op_unicodegetitem(self, op, fcond): boxes = op.getarglist() - base_loc = self._ensure_value_is_boxed(boxes[0], boxes) - ofs_loc = self._ensure_value_is_boxed(boxes[1], boxes) + base_loc = self.make_sure_var_in_reg(boxes[0], boxes) + ofs_loc = self.make_sure_var_in_reg(boxes[1], boxes) self.possibly_free_vars_for_op(op) self.free_temp_vars() @@ -1027,9 +1011,9 @@ def prepare_op_unicodesetitem(self, op, fcond): boxes = op.getarglist() - base_loc = self._ensure_value_is_boxed(boxes[0], boxes) - ofs_loc = self._ensure_value_is_boxed(boxes[1], boxes) - value_loc = self._ensure_value_is_boxed(boxes[2], boxes) + base_loc = self.make_sure_var_in_reg(boxes[0], boxes) + ofs_loc = self.make_sure_var_in_reg(boxes[1], boxes) + value_loc = self.make_sure_var_in_reg(boxes[2], boxes) basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, self.cpu.translate_support_code) scale = itemsize / 2 @@ -1042,7 +1026,7 @@ if imm_arg: argloc = self.convert_to_imm(arg) else: - argloc = self._ensure_value_is_boxed(arg) + argloc = self.make_sure_var_in_reg(arg) self.possibly_free_vars_for_op(op) self.free_temp_vars() resloc = self.force_allocate_reg(op.result) @@ -1093,7 +1077,7 @@ # twice from the memory. N = op.numargs() args = op.getarglist() - arglocs = [self._ensure_value_is_boxed(op.getarg(i), args) + arglocs = [self.make_sure_var_in_reg(op.getarg(i), args) for i in range(N)] tmp = self.get_scratch_reg(INT, args) assert tmp not in arglocs @@ -1215,7 +1199,7 @@ float_result=False, name='prepare_guard_float_ge') def prepare_op_math_sqrt(self, op, fcond): - loc = self._ensure_value_is_boxed(op.getarg(1)) + loc = self.make_sure_var_in_reg(op.getarg(1)) self.possibly_free_vars_for_op(op) self.free_temp_vars() res = self.vfprm.force_allocate_reg(op.result) @@ -1223,12 +1207,12 @@ return [loc, res] def prepare_op_cast_float_to_int(self, op, fcond): - loc1 = self._ensure_value_is_boxed(op.getarg(0)) + loc1 = self.make_sure_var_in_reg(op.getarg(0)) res = self.rm.force_allocate_reg(op.result) return [loc1, res] def prepare_op_cast_int_to_float(self, op, fcond): - loc1 = self._ensure_value_is_boxed(op.getarg(0)) + loc1 = self.make_sure_var_in_reg(op.getarg(0)) res = self.vfprm.force_allocate_reg(op.result) return [loc1, res] @@ -1247,12 +1231,12 @@ return [loc, res] def prepare_op_cast_float_to_singlefloat(self, op, fcond): - loc1 = self._ensure_value_is_boxed(op.getarg(0)) + loc1 = self.make_sure_var_in_reg(op.getarg(0)) res = self.force_allocate_reg(op.result) return [loc1, res] - + def prepare_op_cast_singlefloat_to_float(self, op, fcond): - loc1 = self._ensure_value_is_boxed(op.getarg(0)) + loc1 = self.make_sure_var_in_reg(op.getarg(0)) res = self.force_allocate_reg(op.result) return [loc1, res] diff --git a/pypy/jit/backend/arm/runner.py b/pypy/jit/backend/arm/runner.py --- a/pypy/jit/backend/arm/runner.py +++ b/pypy/jit/backend/arm/runner.py @@ -23,15 +23,21 @@ AbstractLLCPU.__init__(self, rtyper, stats, opts, translate_support_code, gcdescr) + from pypy.jit.backend.llsupport import jitframe + self.deadframe_size_max = llmemory.sizeof(jitframe.DEADFRAME, + self.get_failargs_limit()) + def set_debug(self, flag): return self.assembler.set_debug(flag) + def get_failargs_limit(self): + if self.opts is not None: + return self.opts.failargs_limit + else: + return 1000 + def setup(self): - if self.opts is not None: - failargs_limit = self.opts.failargs_limit - else: - failargs_limit = 1000 - self.assembler = AssemblerARM(self, failargs_limit=failargs_limit) + self.assembler = AssemblerARM(self, self.translate_support_code) def setup_once(self): self.assembler.setup_once() @@ -51,24 +57,6 @@ return self.assembler.assemble_bridge(faildescr, inputargs, operations, original_loop_token, log=log) - def get_latest_value_float(self, index): - return self.assembler.fail_boxes_float.getitem(index) - - def get_latest_value_int(self, index): - return self.assembler.fail_boxes_int.getitem(index) - - def get_latest_value_ref(self, index): - return self.assembler.fail_boxes_ptr.getitem(index) - - def get_latest_value_count(self): - return self.assembler.fail_boxes_count - - def get_latest_force_token(self): - return self.assembler.fail_force_index - - def get_on_leave_jitted_hook(self): - return self.assembler.leave_jitted_hook - def clear_latest_values(self, count): setitem = self.assembler.fail_boxes_ptr.setitem null = lltype.nullptr(llmemory.GCREF.TO) @@ -76,7 +64,7 @@ setitem(index, null) def make_execute_token(self, *ARGS): - FUNCPTR = lltype.Ptr(lltype.FuncType(ARGS, lltype.Signed)) + FUNCPTR = lltype.Ptr(lltype.FuncType(ARGS, llmemory.GCREF)) def execute_token(executable_token, *args): clt = executable_token.compiled_loop_token @@ -91,12 +79,13 @@ prev_interpreter = LLInterpreter.current_interpreter LLInterpreter.current_interpreter = self.debug_ll_interpreter try: - fail_index = func(*args) + deadframe = func(*args) finally: if not self.translate_support_code: LLInterpreter.current_interpreter = prev_interpreter #llop.debug_print(lltype.Void, "<<<< Back") - return self.get_fail_descr_from_number(fail_index) + self.gc_set_extra_threshold() + return deadframe return execute_token def cast_ptr_to_int(x): @@ -115,17 +104,22 @@ assert fail_index >= 0, "already forced!" faildescr = self.get_fail_descr_from_number(fail_index) rffi.cast(TP, addr_of_force_index)[0] = ~fail_index - bytecode = self.assembler._find_failure_recovery_bytecode(faildescr) + frb = self.assembler._find_failure_recovery_bytecode(faildescr) + bytecode = rffi.cast(rffi.UCHARP, frb) addr_all_null_regsiters = rffi.cast(rffi.LONG, self.all_null_registers) - # start of "no gc operation!" block - fail_index_2 = self.assembler.failure_recovery_func( - bytecode, - addr_of_force_index, - addr_all_null_regsiters) - self.assembler.leave_jitted_hook() - # end of "no gc operation!" block - assert fail_index == fail_index_2 - return faildescr + # + assert (rffi.cast(lltype.Signed, bytecode[0]) == + self.assembler.CODE_FORCED) + bytecode = rffi.ptradd(bytecode, 1) + deadframe = self.assembler.grab_frame_values(self, + bytecode, addr_of_force_index, + self.all_null_registers, + self.all_null_registers) + # + assert self.get_latest_descr(deadframe) is faildescr + self.assembler.force_token_to_dead_frame[addr_of_force_index] = ( + deadframe) + return deadframe def redirect_call_assembler(self, oldlooptoken, newlooptoken): self.assembler.redirect_call_assembler(oldlooptoken, newlooptoken) diff --git a/pypy/jit/backend/arm/test/test_gc_integration.py b/pypy/jit/backend/arm/test/test_gc_integration.py --- a/pypy/jit/backend/arm/test/test_gc_integration.py +++ b/pypy/jit/backend/arm/test/test_gc_integration.py @@ -20,7 +20,7 @@ from pypy.jit.backend.arm.test.test_regalloc import BaseTestRegalloc from pypy.jit.backend.arm.regalloc import ARMFrameManager, VFPRegisterManager from pypy.jit.codewriter.effectinfo import EffectInfo -from pypy.jit.backend.arm.regalloc import Regalloc, ARMv7RegisterManager +from pypy.jit.backend.arm.regalloc import Regalloc CPU = getcpuclass() diff --git a/pypy/jit/backend/arm/test/test_loop_unroll.py b/pypy/jit/backend/arm/test/test_loop_unroll.py --- a/pypy/jit/backend/arm/test/test_loop_unroll.py +++ b/pypy/jit/backend/arm/test/test_loop_unroll.py @@ -1,8 +1,8 @@ import py -from pypy.jit.backend.x86.test.test_basic import Jit386Mixin +from pypy.jit.backend.arm.test.support import JitARMMixin from pypy.jit.metainterp.test import test_loop_unroll -class TestLoopSpec(Jit386Mixin, test_loop_unroll.LoopUnrollTest): +class TestLoopSpec(JitARMMixin, test_loop_unroll.LoopUnrollTest): # for the individual tests see # ====> ../../../metainterp/test/test_loop.py From noreply at buildbot.pypy.org Sat Jan 5 19:58:03 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 5 Jan 2013 19:58:03 +0100 (CET) Subject: [pypy-commit] pypy default: An additional sanity check. Message-ID: <20130105185803.849AC1C01EE@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r59767:16cf9cdeaae2 Date: 2013-01-05 19:54 +0100 http://bitbucket.org/pypy/pypy/changeset/16cf9cdeaae2/ Log: An additional sanity check. diff --git a/pypy/jit/metainterp/resume.py b/pypy/jit/metainterp/resume.py --- a/pypy/jit/metainterp/resume.py +++ b/pypy/jit/metainterp/resume.py @@ -1085,6 +1085,9 @@ def consume_virtualref_info(self, vrefinfo, numb, end): # we have to decode a list of references containing pairs # [..., virtual, vref, ...] stopping at 'end' + if vrefinfo is None: + assert end == 0 + return assert (end & 1) == 0 for i in range(0, end, 2): virtual = self.decode_ref(numb.nums[i]) From noreply at buildbot.pypy.org Sat Jan 5 19:58:04 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 5 Jan 2013 19:58:04 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: Left over from the merge. Message-ID: <20130105185804.B82B11C01EE@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r59768:0ee7313cbf43 Date: 2013-01-05 19:55 +0100 http://bitbucket.org/pypy/pypy/changeset/0ee7313cbf43/ Log: Left over from the merge. diff --git a/pypy/jit/metainterp/compile.py b/pypy/jit/metainterp/compile.py --- a/pypy/jit/metainterp/compile.py +++ b/pypy/jit/metainterp/compile.py @@ -858,8 +858,6 @@ assert exception, "PropagateExceptionDescr: no exception??" raise metainterp_sd.ExitFrameWithExceptionRef(cpu, exception) -propagate_exception_descr = PropagateExceptionDescr() - def compile_tmp_callback(cpu, jitdriver_sd, greenboxes, redargtypes, memory_manager=None): """Make a LoopToken that corresponds to assembler code that just From noreply at buildbot.pypy.org Sat Jan 5 19:58:05 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 5 Jan 2013 19:58:05 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: hg merge default Message-ID: <20130105185805.DC3041C01EE@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r59769:8eae75666f4a Date: 2013-01-05 19:56 +0100 http://bitbucket.org/pypy/pypy/changeset/8eae75666f4a/ Log: hg merge default diff --git a/pypy/jit/metainterp/resume.py b/pypy/jit/metainterp/resume.py --- a/pypy/jit/metainterp/resume.py +++ b/pypy/jit/metainterp/resume.py @@ -1086,6 +1086,9 @@ def consume_virtualref_info(self, vrefinfo, numb, end): # we have to decode a list of references containing pairs # [..., virtual, vref, ...] stopping at 'end' + if vrefinfo is None: + assert end == 0 + return assert (end & 1) == 0 for i in range(0, end, 2): virtual = self.decode_ref(numb.nums[i]) From noreply at buildbot.pypy.org Sat Jan 5 19:58:07 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 5 Jan 2013 19:58:07 +0100 (CET) Subject: [pypy-commit] pypy default: merge heads Message-ID: <20130105185807.061AA1C01EE@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r59770:9b45d96d5a09 Date: 2013-01-05 19:57 +0100 http://bitbucket.org/pypy/pypy/changeset/9b45d96d5a09/ Log: merge heads diff --git a/pypy/jit/metainterp/test/test_fficall.py b/pypy/jit/metainterp/test/test_fficall.py --- a/pypy/jit/metainterp/test/test_fficall.py +++ b/pypy/jit/metainterp/test/test_fficall.py @@ -1,9 +1,11 @@ -import py + +import ctypes, math from pypy.rpython.lltypesystem import lltype, rffi from pypy.jit.metainterp.test.support import LLJitMixin from pypy.rlib import jit from pypy.rlib.jit_libffi import types, CIF_DESCRIPTION, FFI_TYPE_PP from pypy.rlib.unroll import unrolling_iterable +from pypy.rlib.rarithmetic import intmask def get_description(atypes, rtype): @@ -103,4 +105,40 @@ class TestFfiCall(FfiCallTests, LLJitMixin): - pass + def test_jit_fii_vref(self): + from pypy.rlib import clibffi + from pypy.rlib.jit_libffi import jit_ffi_prep_cif, jit_ffi_call + + math_sin = intmask(ctypes.cast(ctypes.CDLL(None).sin, + ctypes.c_void_p).value) + math_sin = rffi.cast(rffi.VOIDP, math_sin) + + cd = lltype.malloc(CIF_DESCRIPTION, 1, flavor='raw') + cd.abi = clibffi.FFI_DEFAULT_ABI + cd.nargs = 1 + cd.rtype = clibffi.cast_type_to_ffitype(rffi.DOUBLE) + atypes = lltype.malloc(clibffi.FFI_TYPE_PP.TO, 1, flavor='raw') + atypes[0] = clibffi.cast_type_to_ffitype(rffi.DOUBLE) + cd.atypes = atypes + cd.exchange_size = 64 # 64 bytes of exchange data + cd.exchange_result = 24 + cd.exchange_result_libffi = 24 + cd.exchange_args[0] = 16 + + def f(): + # + jit_ffi_prep_cif(cd) + # + assert rffi.sizeof(rffi.DOUBLE) == 8 + exb = lltype.malloc(rffi.DOUBLEP.TO, 8, flavor='raw') + exb[2] = 1.23 + jit_ffi_call(cd, math_sin, rffi.cast(rffi.CCHARP, exb)) + res = exb[3] + lltype.free(exb, flavor='raw') + # + lltype.free(atypes, flavor='raw') + return res + # + res = self.interp_operations(f, []) + lltype.free(cd, flavor='raw') + assert res == math.sin(1.23) diff --git a/pypy/rlib/jit_libffi.py b/pypy/rlib/jit_libffi.py --- a/pypy/rlib/jit_libffi.py +++ b/pypy/rlib/jit_libffi.py @@ -1,4 +1,4 @@ -import sys + from pypy.rpython.lltypesystem import lltype, rffi from pypy.rlib import clibffi, jit From noreply at buildbot.pypy.org Sat Jan 5 20:04:53 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sat, 5 Jan 2013 20:04:53 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed a few missing py.py renames Message-ID: <20130105190453.95E791C0C35@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59771:5770ebef6742 Date: 2013-01-05 20:04 +0100 http://bitbucket.org/pypy/pypy/changeset/5770ebef6742/ Log: Fixed a few missing py.py renames diff --git a/pypy/bin/pyinteractive.py b/pypy/bin/pyinteractive.py --- a/pypy/bin/pyinteractive.py +++ b/pypy/bin/pyinteractive.py @@ -2,7 +2,7 @@ """Main entry point into the PyPy interpreter. For a list of options, type - py.py --help + pyinteractive.py --help """ @@ -21,7 +21,7 @@ from pypy.config import pypyoption -cmdline_optiondescr = OptionDescription("interactive", "the options of py.py", [ +cmdline_optiondescr = OptionDescription("interactive", "the options of pyinteractive.py", [ BoolOption("verbose", "show verbose interpreter-level traceback", default=os.getenv("PYPY_TB"), cmdline="-v"), BoolOption("interactive", "inspect interactively after running script", @@ -189,6 +189,6 @@ if __name__ == '__main__': if hasattr(sys, 'setrecursionlimit'): - # for running "python -i py.py -Si -- py.py -Si" + # for running "python -i pyinteractive.py -Si -- py.py -Si" sys.setrecursionlimit(3000) sys.exit(main_(sys.argv)) diff --git a/pypy/doc/getting-started-dev.rst b/pypy/doc/getting-started-dev.rst --- a/pypy/doc/getting-started-dev.rst +++ b/pypy/doc/getting-started-dev.rst @@ -256,7 +256,7 @@ If you start an untranslated Python interpreter via:: - python pypy/bin/py.py + python pypy/bin/pyinteractive.py If you press on the console you enter the interpreter-level console, a diff --git a/pypy/doc/getting-started-python.rst b/pypy/doc/getting-started-python.rst --- a/pypy/doc/getting-started-python.rst +++ b/pypy/doc/getting-started-python.rst @@ -241,19 +241,19 @@ most code will be fine. However, the ``sys.prefix`` will be unset and some existing libraries assume that this is never the case. -.. _`py.py interpreter`: +.. _`pyinteractive.py interpreter`: Running the Python Interpreter Without Translation --------------------------------------------------- -The py.py interpreter +The pyinteractive.py interpreter +++++++++++++++++++++ To start interpreting Python with PyPy, install a C compiler that is supported by distutils and use Python 2.5 or greater to run PyPy:: cd pypy - python bin/py.py + python bin/pyinteractive.py After a few seconds (remember: this is running on top of CPython), you should be at the PyPy prompt, which is the same as the Python @@ -273,24 +273,24 @@ PyPy version (i.e. when PyPy's interpreter itself is being interpreted by CPython). -py.py options +pyinteractive.py options +++++++++++++ To list the PyPy interpreter command line options, type:: cd pypy - python bin/py.py --help + python bin/pyinteractive.py --help -py.py supports most of the options that CPython supports too (in addition to a -large amount of options that can be used to customize py.py). +pyinteractive.py supports most of the options that CPython supports too (in addition to a +large amount of options that can be used to customize pyinteractive.py). As an example of using PyPy from the command line, you could type:: - python py.py -c "from test import pystone; pystone.main(10)" + python pyinteractive.py -c "from test import pystone; pystone.main(10)" Alternatively, as with regular Python, you can simply give a script name on the command line:: - python py.py ../../lib-python/2.7/test/pystone.py 10 + python pyinteractive.py ../../lib-python/2.7/test/pystone.py 10 See our `configuration sections`_ for details about what all the commandline options do. diff --git a/pypy/interpreter/test/test_zpy.py b/pypy/interpreter/test/test_zpy.py --- a/pypy/interpreter/test/test_zpy.py +++ b/pypy/interpreter/test/test_zpy.py @@ -5,7 +5,7 @@ import pypy import subprocess -pypypath = py.path.local(pypy.__file__).dirpath("bin", "py.py") +pypypath = py.path.local(pypy.__file__).dirpath("bin", "pyinteractive.py") def run(*args): argslist = map(str, args) diff --git a/pypy/module/_minimal_curses/test/test_curses.py b/pypy/module/_minimal_curses/test/test_curses.py --- a/pypy/module/_minimal_curses/test/test_curses.py +++ b/pypy/module/_minimal_curses/test/test_curses.py @@ -25,7 +25,7 @@ return child def spawn(self, argv): - py_py = py.path.local(pypydir).join('bin', 'py.py') + py_py = py.path.local(pypydir).join('bin', 'pyinteractive.py') return self._spawn(sys.executable, [str(py_py)] + argv) def setup_class(self): diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -1127,7 +1127,7 @@ return child def spawn(self, argv): - py_py = py.path.local(pypydir).join('bin', 'py.py') + py_py = py.path.local(pypydir).join('bin', 'pyinteractive.py') return self._spawn(sys.executable, [str(py_py)] + argv) def test_ttyname(self): diff --git a/pypy/module/termios/test/test_termios.py b/pypy/module/termios/test/test_termios.py --- a/pypy/module/termios/test/test_termios.py +++ b/pypy/module/termios/test/test_termios.py @@ -14,7 +14,7 @@ import termios except ImportError: py.test.skip("termios not found") - py_py = py.path.local(pypydir).join('bin', 'py.py') + py_py = py.path.local(pypydir).join('bin', 'pyinteractive.py') assert py_py.check() cls.py_py = py_py cls.termios = termios diff --git a/pypy/objspace/std/test/test_setobject.py b/pypy/objspace/std/test/test_setobject.py --- a/pypy/objspace/std/test/test_setobject.py +++ b/pypy/objspace/std/test/test_setobject.py @@ -3,7 +3,7 @@ in the stdlibs test/test_set.py which is located in lib-python go there and invoke:: - ../../../pypy/bin/py.py test_set.py + ../../../pypy/bin/pyinteractive.py test_set.py This file just contains some basic tests that make sure, the implementation is not too wrong. From noreply at buildbot.pypy.org Sat Jan 5 20:11:29 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 5 Jan 2013 20:11:29 +0100 (CET) Subject: [pypy-commit] pypy default: a silly speedup Message-ID: <20130105191129.66B9E1C0C35@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r59772:558ae4717951 Date: 2013-01-05 21:10 +0200 http://bitbucket.org/pypy/pypy/changeset/558ae4717951/ Log: a silly speedup diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py --- a/pypy/module/pyexpat/interp_pyexpat.py +++ b/pypy/module/pyexpat/interp_pyexpat.py @@ -182,6 +182,8 @@ def clear(self): self.next_id = 0 + self._last_object_id = -1 + self._last_object = None self.storage = {} @staticmethod @@ -194,10 +196,15 @@ @staticmethod def get_object(id): + if id == global_storage._last_object_id: + return global_storage._last_object return global_storage.storage[id] @staticmethod def free_nonmoving_id(id): + if id == global_storage._last_object_id: + global_storage._last_object = None + global_storage._last_object_id = -1 del global_storage.storage[id] global_storage = Storage() From noreply at buildbot.pypy.org Sat Jan 5 20:14:27 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 5 Jan 2013 20:14:27 +0100 (CET) Subject: [pypy-commit] pypy default: (arigo, agaynor) actually do a speedup Message-ID: <20130105191427.DD7FE1C0C35@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r59773:f955a74e6c70 Date: 2013-01-05 21:13 +0200 http://bitbucket.org/pypy/pypy/changeset/f955a74e6c70/ Log: (arigo, agaynor) actually do a speedup diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py --- a/pypy/module/pyexpat/interp_pyexpat.py +++ b/pypy/module/pyexpat/interp_pyexpat.py @@ -198,7 +198,10 @@ def get_object(id): if id == global_storage._last_object_id: return global_storage._last_object - return global_storage.storage[id] + result = global_storage.storage[id] + global_storage._last_object_id = id + global_storage._last_object = result + return result @staticmethod def free_nonmoving_id(id): From noreply at buildbot.pypy.org Sat Jan 5 20:57:29 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 5 Jan 2013 20:57:29 +0100 (CET) Subject: [pypy-commit] pypy callback-jit: close ready-to-be-merged branch Message-ID: <20130105195729.F35571C01EE@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: callback-jit Changeset: r59774:0c90acafea5d Date: 2013-01-05 21:55 +0200 http://bitbucket.org/pypy/pypy/changeset/0c90acafea5d/ Log: close ready-to-be-merged branch From noreply at buildbot.pypy.org Sat Jan 5 20:57:31 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 5 Jan 2013 20:57:31 +0100 (CET) Subject: [pypy-commit] pypy default: merge callback-jit - this adds jitting to each callback that calls Python Message-ID: <20130105195731.5D0781C01EE@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r59775:df9336b282bb Date: 2013-01-05 21:56 +0200 http://bitbucket.org/pypy/pypy/changeset/df9336b282bb/ Log: merge callback-jit - this adds jitting to each callback that calls Python from C. Speeds up XML parsing using pyexpat quite a lot. Additionally adds some missing operations to the JIT and looks into pyexpat module diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py --- a/pypy/jit/metainterp/pyjitpl.py +++ b/pypy/jit/metainterp/pyjitpl.py @@ -1379,16 +1379,15 @@ """The 'residual_call' operation is emitted in two cases: when we have to generate a residual CALL operation, but also to handle an indirect_call that may need to be inlined.""" - assert isinstance(funcbox, Const) - sd = self.metainterp.staticdata - key = sd.cpu.ts.getaddr_for_box(funcbox) - jitcode = sd.bytecode_for_address(key) - if jitcode is not None: - # we should follow calls to this graph - return self.metainterp.perform_call(jitcode, argboxes) - else: - # but we should not follow calls to that graph - return self.do_residual_call(funcbox, argboxes, calldescr) + if isinstance(funcbox, Const): + sd = self.metainterp.staticdata + key = sd.cpu.ts.getaddr_for_box(funcbox) + jitcode = sd.bytecode_for_address(key) + if jitcode is not None: + # we should follow calls to this graph + return self.metainterp.perform_call(jitcode, argboxes) + # but we should not follow calls to that graph + return self.do_residual_call(funcbox, argboxes, calldescr) # ____________________________________________________________ diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py --- a/pypy/jit/metainterp/test/test_ajit.py +++ b/pypy/jit/metainterp/test/test_ajit.py @@ -3979,6 +3979,8 @@ rgc.add_memory_pressure(1234) return 3 + self.interp_operations(f, []) + def test_external_call(self): from pypy.rlib.objectmodel import invoke_around_extcall diff --git a/pypy/jit/metainterp/test/test_call.py b/pypy/jit/metainterp/test/test_call.py new file mode 100644 --- /dev/null +++ b/pypy/jit/metainterp/test/test_call.py @@ -0,0 +1,27 @@ + +from pypy.jit.metainterp.test.support import LLJitMixin +from pypy.rlib import jit + +class TestCall(LLJitMixin): + def test_indirect_call(self): + @jit.dont_look_inside + def f1(x): + return x + 1 + + @jit.dont_look_inside + def f2(x): + return x + 2 + + @jit.dont_look_inside + def choice(i): + if i: + return f1 + return f2 + + def f(i): + func = choice(i) + return func(i) + + res = self.interp_operations(f, [3]) + assert res == f(3) + diff --git a/pypy/jit/metainterp/test/test_warmspot.py b/pypy/jit/metainterp/test/test_warmspot.py --- a/pypy/jit/metainterp/test/test_warmspot.py +++ b/pypy/jit/metainterp/test/test_warmspot.py @@ -519,6 +519,44 @@ self.check_trace_count(1) + def test_callback_jit_merge_point(self): + from pypy.rlib.objectmodel import register_around_callback_hook + from pypy.rpython.lltypesystem import lltype, rffi + from pypy.translator.tool.cbuild import ExternalCompilationInfo + + callback_jit_driver = JitDriver(greens = ['name'], reds = 'auto') + + def callback_merge_point(name): + callback_jit_driver.jit_merge_point(name=name) + + @callback_jit_driver.inline(callback_merge_point) + def callback_hook(name): + pass + + def callback(a, b): + if a > b: + return 1 + return -1 + + CB_TP = rffi.CCallback([lltype.Signed, lltype.Signed], lltype.Signed) + eci = ExternalCompilationInfo(includes=['stdlib.h']) + qsort = rffi.llexternal('qsort', + [rffi.VOIDP, lltype.Signed, lltype.Signed, + CB_TP], lltype.Void, compilation_info=eci) + ARR = rffi.CArray(lltype.Signed) + + def main(): + register_around_callback_hook(callback_hook) + raw = lltype.malloc(ARR, 10, flavor='raw') + for i in range(10): + raw[i] = 10 - i + qsort(raw, 10, rffi.sizeof(lltype.Signed), callback) + lltype.free(raw, flavor='raw') + + self.meta_interp(main, []) + self.check_trace_count(1) + + class TestLLWarmspot(WarmspotTests, LLJitMixin): CPUClass = runner.LLGraphCPU type_system = 'lltype' diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py --- a/pypy/jit/metainterp/warmspot.py +++ b/pypy/jit/metainterp/warmspot.py @@ -290,11 +290,13 @@ callgraph = inlinable_static_callers(self.translator.graphs, store_calls=True) new_callgraph = [] new_portals = set() + inlined_jit_merge_points = set() for caller, block, op_call, callee in callgraph: func = getattr(callee, 'func', None) _inline_jit_merge_point_ = getattr(func, '_inline_jit_merge_point_', None) if _inline_jit_merge_point_: _inline_jit_merge_point_._always_inline_ = True + inlined_jit_merge_points.add(_inline_jit_merge_point_) op_jmp_call, jmp_graph = get_jmp_call(callee, _inline_jit_merge_point_) # # now we move the op_jmp_call from callee to caller, just @@ -315,6 +317,9 @@ # inline them! inline_threshold = 0.1 # we rely on the _always_inline_ set above auto_inlining(self.translator, inline_threshold, new_callgraph) + # clean up _always_inline_ = True, it can explode later + for item in inlined_jit_merge_points: + del item._always_inline_ # make a fresh copy of the JitDriver in all newly created # jit_merge_points @@ -1011,6 +1016,9 @@ origblock.operations.append(newop) origblock.exitswitch = None origblock.recloseblock(Link([v_result], origportalgraph.returnblock)) + # the origportal now can raise (even if it did not raise before), + # which means that we cannot inline it anywhere any more, but that's + # fine since any forced inlining has been done before # checkgraph(origportalgraph) diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py --- a/pypy/module/_cffi_backend/ccallback.py +++ b/pypy/module/_cffi_backend/ccallback.py @@ -3,10 +3,10 @@ """ import os from pypy.interpreter.error import OperationError, operationerrfmt -from pypy.rpython.lltypesystem import lltype, llmemory, rffi +from pypy.rpython.lltypesystem import lltype, rffi from pypy.rlib.objectmodel import compute_unique_id, keepalive_until_here -from pypy.rlib import clibffi, rweakref, rgc -from pypy.rlib.rarithmetic import r_ulonglong +from pypy.rlib import clibffi, rweakref +from pypy.rlib import jit from pypy.module._cffi_backend.cdataobj import W_CData from pypy.module._cffi_backend.ctypefunc import SIZE_OF_FFI_ARG, BIG_ENDIAN @@ -77,6 +77,7 @@ space.wrap("expected a function ctype")) return ctype + @jit.unroll_safe def invoke(self, ll_args): space = self.space ctype = self.getfunctype() diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py --- a/pypy/module/pypyjit/interp_jit.py +++ b/pypy/module/pypyjit/interp_jit.py @@ -6,7 +6,7 @@ from pypy.tool.pairtype import extendabletype from pypy.rlib.rarithmetic import r_uint, intmask from pypy.rlib.jit import JitDriver, hint, we_are_jitted, dont_look_inside -from pypy.rlib import jit +from pypy.rlib import jit, objectmodel from pypy.rlib.jit import current_trace_length, unroll_parameters import pypy.interpreter.pyopcode # for side-effects from pypy.interpreter.error import OperationError, operationerrfmt @@ -97,6 +97,15 @@ is_being_profiled=self.is_being_profiled) return jumpto +callback_jit_driver = JitDriver(greens = ['name'], reds = 'auto') + +def callback_merge_point(name): + callback_jit_driver.jit_merge_point(name=name) + + at callback_jit_driver.inline(callback_merge_point) +def callback_hook(name): + pass + def _get_adapted_tick_counter(): # Normally, the tick counter is decremented by 100 for every # Python opcode. Here, to better support JIT compilation of diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py --- a/pypy/module/pypyjit/policy.py +++ b/pypy/module/pypyjit/policy.py @@ -106,7 +106,7 @@ 'posix', '_socket', '_sre', '_lsprof', '_weakref', '__pypy__', 'cStringIO', '_collections', 'struct', 'mmap', 'marshal', '_codecs', 'rctime', 'cppyy', - '_cffi_backend']: + '_cffi_backend', 'pyexpat']: if modname == 'pypyjit' and 'interp_resop' in rest: return False return True diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py --- a/pypy/rlib/objectmodel.py +++ b/pypy/rlib/objectmodel.py @@ -595,6 +595,16 @@ llhelper(rffi.AroundFnPtr, before) llhelper(rffi.AroundFnPtr, after) +def register_around_callback_hook(hook): + """ Register a hook that's called before a callback from C calls RPython. + Primary usage is for JIT to have 'started from' hook. + """ + from pypy.rpython.lltypesystem import rffi + from pypy.rpython.annlowlevel import llhelper + + rffi.aroundstate.callback_hook = hook + llhelper(rffi.CallbackHookPtr, hook) + def is_in_callback(): from pypy.rpython.lltypesystem import rffi return rffi.stackcounter.stacks_counter > 1 diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py --- a/pypy/rpython/lltypesystem/rffi.py +++ b/pypy/rpython/lltypesystem/rffi.py @@ -1,6 +1,6 @@ import py from pypy.annotation import model as annmodel -from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem import lltype, rstr from pypy.rpython.lltypesystem import ll2ctypes from pypy.rpython.lltypesystem.llmemory import cast_adr_to_ptr, cast_ptr_to_adr from pypy.rpython.lltypesystem.llmemory import itemoffsetof, raw_memcopy @@ -13,7 +13,7 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.rpython.tool.rfficache import platform, sizeof_c_type from pypy.translator.tool.cbuild import ExternalCompilationInfo -from pypy.rpython.annlowlevel import llhelper +from pypy.rpython.annlowlevel import llhelper, llstr from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.rstring import StringBuilder, UnicodeBuilder, assert_str0 from pypy.rlib import jit @@ -279,9 +279,17 @@ callable_name = getattr(callable, '__name__', '?') if callbackholder is not None: callbackholder.callbacks[callable] = True + callable_name_descr = str(callable).replace('"', '\\"') args = ', '.join(['a%d' % i for i in range(len(TP.TO.ARGS))]) source = py.code.Source(r""" - def wrapper(%s): # no *args - no GIL for mallocing the tuple + def inner_wrapper(%(args)s): + callback_hook = aroundstate.callback_hook + if callback_hook: + callback_hook(llstr("%(callable_name_descr)s")) + return callable(%(args)s) + inner_wrapper._never_inline_ = True + + def wrapper(%(args)s): # no *args - no GIL for mallocing the tuple llop.gc_stack_bottom(lltype.Void) # marker for trackgcroot.py if aroundstate is not None: after = aroundstate.after @@ -290,7 +298,7 @@ # from now on we hold the GIL stackcounter.stacks_counter += 1 try: - result = callable(%s) + result = inner_wrapper(%(args)s) except Exception, e: os.write(2, "Warning: uncaught exception in callback: %%s %%s\n" %% @@ -308,10 +316,11 @@ # by llexternal, it is essential that no exception checking occurs # after the call to before(). return result - """ % (args, args)) + """ % locals()) miniglobals = locals().copy() miniglobals['Exception'] = Exception miniglobals['os'] = os + miniglobals['llstr'] = llstr miniglobals['we_are_translated'] = we_are_translated miniglobals['stackcounter'] = stackcounter exec source.compile() in miniglobals @@ -319,10 +328,14 @@ _make_wrapper_for._annspecialcase_ = 'specialize:memo' AroundFnPtr = lltype.Ptr(lltype.FuncType([], lltype.Void)) +CallbackHookPtr = lltype.Ptr(lltype.FuncType([lltype.Ptr(rstr.STR)], lltype.Void)) + class AroundState: + callback_hook = None + def _cleanup_(self): - self.before = None # or a regular RPython function - self.after = None # or a regular RPython function + self.before = None # or a regular RPython function + self.after = None # or a regular RPython function aroundstate = AroundState() aroundstate._cleanup_() diff --git a/pypy/translator/goal/targetpypystandalone.py b/pypy/translator/goal/targetpypystandalone.py --- a/pypy/translator/goal/targetpypystandalone.py +++ b/pypy/translator/goal/targetpypystandalone.py @@ -28,6 +28,11 @@ w_call_startup_gateway = space.wrap(gateway.interp2app(call_startup)) withjit = space.config.objspace.usemodules.pypyjit + if withjit: + from pypy.module.pypyjit.interp_jit import callback_hook + from pypy.rlib import objectmodel + objectmodel.register_around_callback_hook(callback_hook) + def entry_point(argv): if withjit: from pypy.jit.backend.hlinfo import highleveljitinfo From noreply at buildbot.pypy.org Sat Jan 5 21:21:51 2013 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 5 Jan 2013 21:21:51 +0100 (CET) Subject: [pypy-commit] pypy default: document availability of build dependencies for windows Message-ID: <20130105202151.7F82C1C01A6@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r59776:9446c0b876d5 Date: 2013-01-05 22:21 +0200 http://bitbucket.org/pypy/pypy/changeset/9446c0b876d5/ Log: document availability of build dependencies for windows diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -56,11 +56,24 @@ ---------------------------- On Windows, there is no standard place where to download, build and -install third-party libraries. We chose to install them in the parent +install third-party libraries. We recommend installing them in the parent directory of the pypy checkout. For example, if you installed pypy in ``d:\pypy\trunk\`` (This directory contains a README file), the base -directory is ``d:\pypy``. You may choose different values by setting the -INCLUDE, LIB and PATH (for DLLs) +directory is ``d:\pypy``. You must then set the +INCLUDE, LIB and PATH (for DLLs) environment variables appropriately. + +Abridged method (for -Ojit builds using Visual Studio 2008) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Download the versions of all the external packages +from +https://bitbucket.org/pypy/pypy/downloads/local.zip +Then expand it into the base directory (base_dir) and modify your environment to reflect this:: + + set PATH=\bin;%PATH% + set INCLUDE=\include;%INCLUDE% + set LIB=\lib;%LIB% + +Now you should be good to go. Read on for more information. The Boehm garbage collector ~~~~~~~~~~~~~~~~~~~~~~~~~~~ From noreply at buildbot.pypy.org Sat Jan 5 21:26:06 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 5 Jan 2013 21:26:06 +0100 (CET) Subject: [pypy-commit] pypy default: ll_arraycopy for array of structs Message-ID: <20130105202606.56D521C01A6@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r59777:5c826cf7eb7f Date: 2013-01-05 22:23 +0200 http://bitbucket.org/pypy/pypy/changeset/5c826cf7eb7f/ Log: ll_arraycopy for array of structs diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py --- a/pypy/rlib/rgc.py +++ b/pypy/rlib/rgc.py @@ -139,6 +139,40 @@ hop.exception_cannot_occur() return hop.genop(opname, vlist, resulttype = hop.r_result.lowleveltype) +def copy_struct_item(source, dest, si, di): + TP = lltype.typeOf(source) + i = 0 + while i < len(TP._names): + setattr(dest[di], TP._names[i], getattr(source[si], TP._names[i])) + +class CopyStructEntry(ExtRegistryEntry): + _about_ = copy_struct_item + + def compute_result_annotation(self, s_source, s_dest, si, di): + pass + + def specialize_call(self, hop): + v_source, v_dest, v_si, v_di = hop.inputargs(hop.args_r[0], + hop.args_r[1], + lltype.Signed, + lltype.Signed) + hop.exception_cannot_occur() + TP = v_source.concretetype.TO.OF + for name, TP in TP._flds.iteritems(): + c_name = hop.inputconst(lltype.Void, name) + v_fld = hop.genop('getinteriorfield', [v_source, v_si, c_name], + resulttype=TP) + hop.genop('setinteriorfield', [v_dest, v_di, c_name, v_fld]) + + + at specialize.ll() +def copy_item(source, dest, si, di): + TP = lltype.typeOf(source) + if isinstance(TP.TO.OF, lltype.Struct): + copy_struct_item(source, dest, si, di) + else: + dest[di] = source[si] + @jit.oopspec('list.ll_arraycopy(source, dest, source_start, dest_start, length)') @enforceargs(None, None, int, int, int) @specialize.ll() @@ -150,7 +184,7 @@ # and also, maybe, speed up very small cases if length <= 1: if length == 1: - dest[dest_start] = source[source_start] + copy_item(source, dest, source_start, dest_start) return # supports non-overlapping copies only @@ -170,7 +204,7 @@ # if the write barrier is not supported, copy by hand i = 0 while i < length: - dest[i + dest_start] = source[i + source_start] + copy_item(source, dest, i + source_start, i + dest_start) i += 1 return source_addr = llmemory.cast_ptr_to_adr(source) diff --git a/pypy/rlib/test/test_rgc.py b/pypy/rlib/test/test_rgc.py --- a/pypy/rlib/test/test_rgc.py +++ b/pypy/rlib/test/test_rgc.py @@ -134,6 +134,24 @@ assert check.called +def test_ll_arraycopy_array_of_structs(): + TP = lltype.GcArray(lltype.Struct('x', ('x', lltype.Signed), + ('y', lltype.Signed))) + def f(): + a1 = lltype.malloc(TP, 3) + a2 = lltype.malloc(TP, 3) + for i in range(3): + a1[i].x = 2 * i + a1[i].y = 2 * i + 1 + rgc.ll_arraycopy(a1, a2, 0, 0, 3) + for i in range(3): + assert a2[i].x == 2 * i + assert a2[i].y == 2 * i + 1 + + + interpret(f, []) + f() + def test_ll_arraycopy_small(): TYPE = lltype.GcArray(lltype.Signed) for length in range(5): diff --git a/pypy/rpython/lltypesystem/test/test_lltype.py b/pypy/rpython/lltypesystem/test/test_lltype.py --- a/pypy/rpython/lltypesystem/test/test_lltype.py +++ b/pypy/rpython/lltypesystem/test/test_lltype.py @@ -808,7 +808,6 @@ assert F.RESULT == Signed assert F.ARGS == (Signed,) - class TestTrackAllocation: def test_automatic_tracking(self): # calls to start_tracking_allocations/stop_tracking_allocations From noreply at buildbot.pypy.org Sat Jan 5 21:26:09 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 5 Jan 2013 21:26:09 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments: merge default Message-ID: <20130105202609.4A8A61C01A6@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments Changeset: r59778:195cfa2bf9bb Date: 2013-01-05 22:24 +0200 http://bitbucket.org/pypy/pypy/changeset/195cfa2bf9bb/ Log: merge default diff too long, truncating to 2000 out of 10572 lines diff --git a/lib-python/2.7/sre_parse.py b/lib-python/2.7/sre_parse.py --- a/lib-python/2.7/sre_parse.py +++ b/lib-python/2.7/sre_parse.py @@ -16,6 +16,12 @@ from sre_constants import * +try: + from __pypy__ import newdict +except ImportError: + def newdict(tp): + return {} + SPECIAL_CHARS = ".\\[{()*+?^$|" REPEAT_CHARS = "*+?{" @@ -68,7 +74,7 @@ self.flags = 0 self.open = [] self.groups = 1 - self.groupdict = {} + self.groupdict = newdict("module") def opengroup(self, name=None): gid = self.groups self.groups = gid + 1 diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -28,5 +28,7 @@ .. branch: remove-PYPY_NOT_MAIN_FILE +.. branch: missing-jit-operations + .. branch: fix-lookinside-iff-oopspec Fixed the interaction between two internal tools for controlling the JIT. diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -56,11 +56,24 @@ ---------------------------- On Windows, there is no standard place where to download, build and -install third-party libraries. We chose to install them in the parent +install third-party libraries. We recommend installing them in the parent directory of the pypy checkout. For example, if you installed pypy in ``d:\pypy\trunk\`` (This directory contains a README file), the base -directory is ``d:\pypy``. You may choose different values by setting the -INCLUDE, LIB and PATH (for DLLs) +directory is ``d:\pypy``. You must then set the +INCLUDE, LIB and PATH (for DLLs) environment variables appropriately. + +Abridged method (for -Ojit builds using Visual Studio 2008) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Download the versions of all the external packages +from +https://bitbucket.org/pypy/pypy/downloads/local.zip +Then expand it into the base directory (base_dir) and modify your environment to reflect this:: + + set PATH=\bin;%PATH% + set INCLUDE=\include;%INCLUDE% + set LIB=\lib;%LIB% + +Now you should be good to go. Read on for more information. The Boehm garbage collector ~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py --- a/pypy/jit/backend/arm/assembler.py +++ b/pypy/jit/backend/arm/assembler.py @@ -1,5 +1,6 @@ from __future__ import with_statement import os +from pypy.jit.backend.llsupport import jitframe from pypy.jit.backend.arm.helper.assembler import saved_registers from pypy.jit.backend.arm import conditions as c from pypy.jit.backend.arm import registers as r @@ -25,12 +26,12 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.jit.backend.arm.opassembler import ResOpAssembler from pypy.rlib.debug import (debug_print, debug_start, debug_stop, - have_debug_prints) + have_debug_prints, fatalerror) from pypy.rlib.jit import AsmInfo from pypy.rlib.objectmodel import compute_unique_id # XXX Move to llsupport -from pypy.jit.backend.x86.support import values_array, memcpy_fn +from pypy.jit.backend.x86.support import memcpy_fn DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', ('i', lltype.Signed), ('type', lltype.Char), # 'b'ridge, 'l'abel or @@ -44,14 +45,8 @@ debug = True - def __init__(self, cpu, failargs_limit=1000): + def __init__(self, cpu, translate_support_code=False): self.cpu = cpu - self.fail_boxes_int = values_array(lltype.Signed, failargs_limit) - self.fail_boxes_float = values_array(longlong.FLOATSTORAGE, - failargs_limit) - self.fail_boxes_ptr = values_array(llmemory.GCREF, failargs_limit) - self.fail_boxes_count = 0 - self.fail_force_index = 0 self.setup_failure_recovery() self.mc = None self.memcpy_addr = 0 @@ -68,6 +63,7 @@ self._debug = False self.loop_run_counters = [] self.debug_counter_descr = cpu.fielddescrof(DEBUG_COUNTER, 'i') + self.force_token_to_dead_frame = {} # XXX temporary hack def set_debug(self, v): r = self._debug @@ -112,9 +108,13 @@ gc_ll_descr.initialize() self._build_wb_slowpath(False) self._build_wb_slowpath(True) + self._build_failure_recovery(exc=True, withfloats=False) + self._build_failure_recovery(exc=False, withfloats=False) if self.cpu.supports_floats: self._build_wb_slowpath(False, withfloats=True) self._build_wb_slowpath(True, withfloats=True) + self._build_failure_recovery(exc=True, withfloats=True) + self._build_failure_recovery(exc=False, withfloats=True) self._build_propagate_exception_path() if gc_ll_descr.get_malloc_slowpath_addr is not None: self._build_malloc_slowpath() @@ -122,10 +122,7 @@ if gc_ll_descr.gcrootmap and gc_ll_descr.gcrootmap.is_shadow_stack: self._build_release_gil(gc_ll_descr.gcrootmap) self.memcpy_addr = self.cpu.cast_ptr_to_int(memcpy_fn) - self._exit_code_addr = self._gen_exit_path() - self._leave_jitted_hook_save_exc = \ - self._gen_leave_jitted_hook_code(True) - self._leave_jitted_hook = self._gen_leave_jitted_hook_code(False) + if not self._debug: # if self._debug is already set it means that someone called # set_debug by hand before initializing the assembler. Leave it @@ -220,36 +217,17 @@ self.releasegil_addr = rffi.cast(lltype.Signed, releasegil_func) self.reacqgil_addr = rffi.cast(lltype.Signed, reacqgil_func) - def _gen_leave_jitted_hook_code(self, save_exc): - mc = ARMv7Builder() - if self.cpu.supports_floats: - floats = r.caller_vfp_resp - else: - floats = [] - with saved_registers(mc, r.caller_resp + [r.lr], floats): - addr = self.cpu.get_on_leave_jitted_int(save_exception=save_exc) - mc.BL(addr) - assert self._exit_code_addr != 0 - mc.B(self._exit_code_addr) - return mc.materialize(self.cpu.asmmemmgr, [], - self.cpu.gc_ll_descr.gcrootmap) - def _build_propagate_exception_path(self): if self.cpu.propagate_exception_v < 0: return # not supported (for tests, or non-translated) # mc = ARMv7Builder() - # call on_leave_jitted_save_exc() - if self.cpu.supports_floats: - floats = r.caller_vfp_resp - else: - floats = [] - with saved_registers(mc, r.caller_resp + [r.lr], floats): - addr = self.cpu.get_on_leave_jitted_int(save_exception=True, - default_to_memoryerror=True) - mc.BL(addr) - mc.gen_load_int(r.ip.value, self.cpu.propagate_exception_v) - mc.MOV_rr(r.r0.value, r.ip.value) + # + # Call the helper, which will return a dead frame object with + # the correct exception set, or MemoryError by default + # XXX make sure we return the correct value here + addr = rffi.cast(lltype.Signed, self.cpu.get_propagate_exception()) + mc.BL(addr) self.gen_func_epilog(mc=mc) self.propagate_exception_path = mc.materialize(self.cpu.asmmemmgr, []) @@ -282,12 +260,12 @@ # restore registers and return # We check for c.EQ here, meaning all bits zero in this case mc.POP([reg.value for reg in r.argument_regs] + [r.pc.value], cond=c.EQ) - # call on_leave_jitted_save_exc() - addr = self.cpu.get_on_leave_jitted_int(save_exception=True) + # + # Call the helper, which will return a dead frame object with + # the correct exception set, or MemoryError by default + addr = rffi.cast(lltype.Signed, self.cpu.get_propagate_exception()) mc.BL(addr) # - mc.gen_load_int(r.r0.value, self.cpu.propagate_exception_v) - # # footer -- note the ADD, which skips the return address of this # function, and will instead return to the caller's caller. Note # also that we completely ignore the saved arguments, because we @@ -339,39 +317,82 @@ def setup_failure_recovery(self): - @rgc.no_collect + #@rgc.no_collect -- XXX still true, but hacked gc_set_extra_threshold def failure_recovery_func(mem_loc, frame_pointer, stack_pointer): """mem_loc is a structure in memory describing where the values for the failargs are stored. frame loc is the address of the frame pointer for the frame to be decoded frame """ - vfp_registers = rffi.cast(rffi.LONGLONGP, stack_pointer) - registers = rffi.ptradd(vfp_registers, len(r.all_vfp_regs)) + vfp_registers = rffi.cast(rffi.LONGP, stack_pointer) + registers = rffi.ptradd(vfp_registers, 2*len(r.all_vfp_regs)) registers = rffi.cast(rffi.LONGP, registers) - return self.decode_registers_and_descr(mem_loc, frame_pointer, + bytecode = rffi.cast(rffi.UCHARP, mem_loc) + return self.grab_frame_values(self.cpu, bytecode, frame_pointer, registers, vfp_registers) + self.failure_recovery_code = [0, 0, 0, 0] self.failure_recovery_func = failure_recovery_func - recovery_func_sign = lltype.Ptr(lltype.FuncType([lltype.Signed] * 3, - lltype.Signed)) + _FAILURE_RECOVERY_FUNC = lltype.Ptr(lltype.FuncType([rffi.LONGP] * 3, + llmemory.GCREF)) - @rgc.no_collect - def decode_registers_and_descr(self, mem_loc, frame_pointer, + @staticmethod + #@rgc.no_collect -- XXX still true, but hacked gc_set_extra_threshold + def grab_frame_values(cpu, bytecode, frame_pointer, registers, vfp_registers): - """Decode locations encoded in memory at mem_loc and write the values - to the failboxes. Values for spilled vars and registers are stored on - stack at frame_loc """ - assert frame_pointer & 1 == 0 - self.fail_force_index = frame_pointer - bytecode = rffi.cast(rffi.UCHARP, mem_loc) + # no malloc allowed here!! xxx apart from one, hacking a lot + force_index = rffi.cast(lltype.Signed, frame_pointer) num = 0 - value = 0 - fvalue = 0 + deadframe = lltype.nullptr(jitframe.DEADFRAME) + # step 1: lots of mess just to count the final value of 'num' + bytecode1 = bytecode + while 1: + code = rffi.cast(lltype.Signed, bytecode1[0]) + bytecode1 = rffi.ptradd(bytecode1, 1) + if code >= AssemblerARM.CODE_FROMSTACK: + while code > 0x7F: + code = rffi.cast(lltype.Signed, bytecode1[0]) + bytecode1 = rffi.ptradd(bytecode1, 1) + else: + kind = code & 3 + if kind == AssemblerARM.DESCR_SPECIAL: + if code == AssemblerARM.CODE_HOLE: + num += 1 + continue + if code == AssemblerARM.CODE_INPUTARG: + continue + if code == AssemblerARM.CODE_FORCED: + # resuming from a GUARD_NOT_FORCED + token = force_index + deadframe = ( + cpu.assembler.force_token_to_dead_frame.pop(token)) + deadframe = lltype.cast_opaque_ptr( + jitframe.DEADFRAMEPTR, deadframe) + continue + assert code == AssemblerARM.CODE_STOP + break + num += 1 + + # allocate the deadframe + if not deadframe: + # Remove the "reserve" at the end of the nursery. This means + # that it is guaranteed that the following malloc() works + # without requiring a collect(), but it needs to be re-added + # as soon as possible. + cpu.gc_clear_extra_threshold() + assert num <= cpu.get_failargs_limit() + try: + deadframe = lltype.malloc(jitframe.DEADFRAME, num) + except MemoryError: + fatalerror("memory usage error in grab_frame_values") + # fill it code_inputarg = False - while True: + num = 0 + value_hi = 0 + while 1: + # decode the next instruction from the bytecode code = rffi.cast(lltype.Signed, bytecode[0]) bytecode = rffi.ptradd(bytecode, 1) - if code >= self.CODE_FROMSTACK: + if code >= AssemblerARM.CODE_FROMSTACK: if code > 0x7F: shift = 7 code &= 0x7F @@ -384,54 +405,63 @@ break # load the value from the stack kind = code & 3 - code = int((code - self.CODE_FROMSTACK) >> 2) + code = (code - AssemblerARM.CODE_FROMSTACK) >> 2 if code_inputarg: code = ~code code_inputarg = False - if kind == self.DESCR_FLOAT: - # we use code + 1 to get the hi word of the double worded float - stackloc = frame_pointer - get_fp_offset(int(code) + 1) - assert stackloc & 3 == 0 - fvalue = rffi.cast(rffi.LONGLONGP, stackloc)[0] - else: - stackloc = frame_pointer - get_fp_offset(int(code)) - assert stackloc & 1 == 0 - value = rffi.cast(rffi.LONGP, stackloc)[0] + stackloc = force_index - get_fp_offset(int(code)) + value = rffi.cast(rffi.LONGP, stackloc)[0] + if kind == AssemblerARM.DESCR_FLOAT: + assert WORD == 4 + value_hi = value + value = rffi.cast(rffi.LONGP, stackloc - WORD)[0] else: - # 'code' identifies a register: load its value kind = code & 3 - if kind == self.DESCR_SPECIAL: - if code == self.CODE_HOLE: + if kind == AssemblerARM.DESCR_SPECIAL: + if code == AssemblerARM.CODE_HOLE: num += 1 continue - if code == self.CODE_INPUTARG: + if code == AssemblerARM.CODE_INPUTARG: code_inputarg = True continue - assert code == self.CODE_STOP + if code == AssemblerARM.CODE_FORCED: + continue + assert code == AssemblerARM.CODE_STOP break + # 'code' identifies a register: load its value code >>= 2 - if kind == self.DESCR_FLOAT: - fvalue = vfp_registers[code] + if kind == AssemblerARM.DESCR_FLOAT: + if WORD == 4: + value = vfp_registers[2*code] + value_hi = vfp_registers[2*code + 1] + else: + value = registers[code] else: value = registers[code] # store the loaded value into fail_boxes_ - if kind == self.DESCR_FLOAT: - tgt = self.fail_boxes_float.get_addr_for_num(num) - rffi.cast(rffi.LONGLONGP, tgt)[0] = fvalue + if kind == AssemblerARM.DESCR_INT: + deadframe.jf_values[num].int = value + elif kind == AssemblerARM.DESCR_REF: + deadframe.jf_values[num].ref = rffi.cast(llmemory.GCREF, value) + elif kind == AssemblerARM.DESCR_FLOAT: + assert WORD == 4 + assert not longlong.is_64_bit + floatvalue = rffi.cast(lltype.SignedLongLong, value_hi) + floatvalue <<= 32 + floatvalue |= rffi.cast(lltype.SignedLongLong, + rffi.cast(lltype.Unsigned, value)) + deadframe.jf_values[num].float = floatvalue else: - if kind == self.DESCR_INT: - tgt = self.fail_boxes_int.get_addr_for_num(num) - elif kind == self.DESCR_REF: - assert (value & 3) == 0, "misaligned pointer" - tgt = self.fail_boxes_ptr.get_addr_for_num(num) - else: - assert 0, "bogus kind" - rffi.cast(rffi.LONGP, tgt)[0] = value + assert 0, "bogus kind" num += 1 - self.fail_boxes_count = num + # + assert num == len(deadframe.jf_values) + if not we_are_translated(): + assert bytecode[4] == 0xCC fail_index = rffi.cast(rffi.INTP, bytecode)[0] - fail_index = rffi.cast(lltype.Signed, fail_index) - return fail_index + fail_descr = cpu.get_fail_descr_from_number(fail_index) + deadframe.jf_descr = fail_descr.hide(cpu) + return lltype.cast_opaque_ptr(llmemory.GCREF, deadframe) def decode_inputargs(self, code): descr_to_box_type = [REF, INT, FLOAT] @@ -514,12 +544,26 @@ self.mc.CMP_ri(r.r0.value, 0) self.mc.B(self.propagate_exception_path, c=c.EQ) - def _gen_exit_path(self): + def _build_failure_recovery(self, exc, withfloats=False): mc = ARMv7Builder() - decode_registers_addr = llhelper(self.recovery_func_sign, + failure_recovery = llhelper(self._FAILURE_RECOVERY_FUNC, self.failure_recovery_func) self._insert_checks(mc) - with saved_registers(mc, r.all_regs, r.all_vfp_regs): + if withfloats: + f = r.all_vfp_regs + else: + f = [] + with saved_registers(mc, r.all_regs, f): + if exc: + # We might have an exception pending. Load it into r4 + # (this is a register saved across calls) + mc.gen_load_int(r.r5.value, self.cpu.pos_exc_value()) + mc.LDR_ri(r.r4.value, self.cpu.pos_exc_value()) + # clear the exc flags + mc.gen_load_int(r.r6.value, 0) + mc.STR_ri(r.r6.value, r.r5.value) + mc.gen_load_int(r.r5.value, self.cpu.pos_exception()) + mc.STR_ri(r.r6.value, r.r5.value) # move mem block address, to r0 to pass as mc.MOV_rr(r.r0.value, r.lr.value) # pass the current frame pointer as second param @@ -527,23 +571,32 @@ # pass the current stack pointer as third param mc.MOV_rr(r.r2.value, r.sp.value) self._insert_checks(mc) - mc.BL(rffi.cast(lltype.Signed, decode_registers_addr)) + mc.BL(rffi.cast(lltype.Signed, failure_recovery)) + if exc: + # save ebx into 'jf_guard_exc' + from pypy.jit.backend.llsupport.descr import unpack_fielddescr + descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) + offset, size, _ = unpack_fielddescr(descrs.jf_guard_exc) + mc.STR_rr(r.r4.value, r.r0.value, offset, cond=c.AL) mc.MOV_rr(r.ip.value, r.r0.value) mc.MOV_rr(r.r0.value, r.ip.value) self.gen_func_epilog(mc=mc) - return mc.materialize(self.cpu.asmmemmgr, [], + rawstart = mc.materialize(self.cpu.asmmemmgr, [], self.cpu.gc_ll_descr.gcrootmap) + self.failure_recovery_code[exc + 2 * withfloats] = rawstart + self.mc = None DESCR_REF = 0x00 DESCR_INT = 0x01 DESCR_FLOAT = 0x02 DESCR_SPECIAL = 0x03 CODE_FROMSTACK = 64 - CODE_STOP = 0 | DESCR_SPECIAL - CODE_HOLE = 4 | DESCR_SPECIAL - CODE_INPUTARG = 8 | DESCR_SPECIAL + CODE_STOP = 0 | DESCR_SPECIAL + CODE_HOLE = 4 | DESCR_SPECIAL + CODE_INPUTARG = 8 | DESCR_SPECIAL + CODE_FORCED = 12 | DESCR_SPECIAL #XXX where should this be written? - def gen_descr_encoding(self, descr, failargs, locs): + def write_failure_recovery_description(self, descr, failargs, locs): assert self.mc is not None for i in range(len(failargs)): arg = failargs[i] @@ -575,26 +628,33 @@ self.mc.writechar(chr(n)) self.mc.writechar(chr(self.CODE_STOP)) - fdescr = self.cpu.get_fail_descr_number(descr) - self.mc.write32(fdescr) + + def generate_quick_failure(self, guardtok, fcond=c.AL): + assert isinstance(guardtok.save_exc, bool) + fail_index = self.cpu.get_fail_descr_number(guardtok.descr) + startpos = self.mc.currpos() + withfloats = False + for box in guardtok.failargs: + if box is not None and box.type == FLOAT: + withfloats = True + break + exc = guardtok.save_exc + target = self.failure_recovery_code[exc + 2 * withfloats] + assert target != 0 + self.mc.BL(target) + # write tight data that describes the failure recovery + if guardtok.is_guard_not_forced: + self.mc.writechar(chr(self.CODE_FORCED)) + self.write_failure_recovery_description(guardtok.descr, + guardtok.failargs, guardtok.faillocs[1:]) + self.mc.write32(fail_index) + # for testing the decoding, write a final byte 0xCC + if not we_are_translated(): + self.mc.writechar('\xCC') + faillocs = [loc for loc in guardtok.faillocs if loc is not None] + guardtok.descr._arm_debug_faillocs = faillocs self.align() - - # assert that the fail_boxes lists are big enough - assert len(failargs) <= self.fail_boxes_int.SIZE - - def _gen_path_to_exit_path(self, descr, args, arglocs, - save_exc, fcond=c.AL): - assert isinstance(save_exc, bool) - self.gen_exit_code(self.mc, save_exc, fcond) - self.gen_descr_encoding(descr, args, arglocs[1:]) - - def gen_exit_code(self, mc, save_exc, fcond=c.AL): - assert isinstance(save_exc, bool) - if save_exc: - path = self._leave_jitted_hook_save_exc - else: - path = self._leave_jitted_hook - mc.BL(path) + return startpos def align(self): while(self.mc.currpos() % FUNC_ALIGN != 0): @@ -841,15 +901,10 @@ def write_pending_failure_recoveries(self): for tok in self.pending_guards: - descr = tok.descr #generate the exit stub and the encoded representation - pos = self.mc.currpos() - tok.pos_recovery_stub = pos - - self._gen_path_to_exit_path(descr, tok.failargs, - tok.faillocs, save_exc=tok.save_exc) + tok.pos_recovery_stub = self.generate_quick_failure(tok) # store info on the descr - descr._arm_current_frame_depth = tok.faillocs[0].getint() + tok.descr._arm_current_frame_depth = tok.faillocs[0].getint() def process_pending_guards(self, block_start): clt = self.current_clt @@ -860,7 +915,7 @@ descr._arm_failure_recovery_block = failure_recovery_pos relative_offset = tok.pos_recovery_stub - tok.offset guard_pos = block_start + tok.offset - if not tok.is_invalidate: + if not tok.is_guard_not_invalidated: # patch the guard jumpt to the stub # overwrite the generate NOP with a B_offs to the pos of the # stub @@ -1255,11 +1310,6 @@ else: raise AssertionError('Trying to pop to an invalid location') - def leave_jitted_hook(self): - ptrs = self.fail_boxes_ptr.ar - llop.gc_assume_young_pointers(lltype.Void, - llmemory.cast_ptr_to_adr(ptrs)) - def malloc_cond(self, nursery_free_adr, nursery_top_adr, size): assert size & (WORD-1) == 0 # must be correctly aligned diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py --- a/pypy/jit/backend/arm/opassembler.py +++ b/pypy/jit/backend/arm/opassembler.py @@ -16,6 +16,7 @@ gen_emit_unary_float_op, saved_registers, count_reg_args) +from pypy.jit.backend.arm.helper.regalloc import check_imm_arg from pypy.jit.backend.arm.codebuilder import ARMv7Builder, OverwritingBuilder from pypy.jit.backend.arm.jump import remap_frame_layout from pypy.jit.backend.arm.regalloc import TempInt, TempPtr @@ -27,18 +28,20 @@ from pypy.jit.metainterp.history import JitCellToken, TargetToken from pypy.jit.metainterp.resoperation import rop from pypy.rlib.objectmodel import we_are_translated -from pypy.rpython.lltypesystem import rstr +from pypy.rlib import rgc +from pypy.rpython.lltypesystem import rstr, rffi, lltype, llmemory NO_FORCE_INDEX = -1 class GuardToken(object): - def __init__(self, descr, failargs, faillocs, offset, - save_exc, fcond=c.AL, is_invalidate=False): + def __init__(self, descr, failargs, faillocs, offset, save_exc, fcond=c.AL, + is_guard_not_invalidated=False, is_guard_not_forced=False): assert isinstance(save_exc, bool) self.descr = descr self.offset = offset - self.is_invalidate = is_invalidate + self.is_guard_not_invalidated = is_guard_not_invalidated + self.is_guard_not_forced = is_guard_not_forced self.failargs = failargs self.faillocs = faillocs self.save_exc = save_exc @@ -190,7 +193,8 @@ return fcond def _emit_guard(self, op, arglocs, fcond, save_exc, - is_guard_not_invalidated=False): + is_guard_not_invalidated=False, + is_guard_not_forced=False): assert isinstance(save_exc, bool) assert isinstance(fcond, int) descr = op.getdescr() @@ -210,7 +214,8 @@ faillocs=arglocs, offset=pos, save_exc=save_exc, - is_invalidate=is_guard_not_invalidated, + is_guard_not_invalidated=is_guard_not_invalidated, + is_guard_not_forced=is_guard_not_forced, fcond=fcond)) return c.AL @@ -312,49 +317,10 @@ return fcond def emit_op_finish(self, op, arglocs, regalloc, fcond): - for i in range(len(arglocs) - 1): - loc = arglocs[i] - box = op.getarg(i) - if loc is None: - continue - if loc.is_reg(): - if box.type == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(i) - elif box.type == INT: - adr = self.fail_boxes_int.get_addr_for_num(i) - else: - assert 0 - self.mc.gen_load_int(r.ip.value, adr) - self.mc.STR_ri(loc.value, r.ip.value) - elif loc.is_vfp_reg(): - assert box.type == FLOAT - adr = self.fail_boxes_float.get_addr_for_num(i) - self.mc.gen_load_int(r.ip.value, adr) - self.mc.VSTR(loc.value, r.ip.value) - elif loc.is_stack() or loc.is_imm() or loc.is_imm_float(): - if box.type == FLOAT: - adr = self.fail_boxes_float.get_addr_for_num(i) - self.mov_loc_loc(loc, r.vfp_ip) - self.mc.gen_load_int(r.ip.value, adr) - self.mc.VSTR(r.vfp_ip.value, r.ip.value) - elif box.type == REF or box.type == INT: - if box.type == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(i) - elif box.type == INT: - adr = self.fail_boxes_int.get_addr_for_num(i) - else: - assert 0 - self.mov_loc_loc(loc, r.ip) - self.mc.gen_load_int(r.lr.value, adr) - self.mc.STR_ri(r.ip.value, r.lr.value) - else: - assert 0 - # note: no exception should currently be set in llop.get_exception_addr - # even if this finish may be an exit_frame_with_exception (in this case - # the exception instance is in arglocs[0]). - addr = self.cpu.get_on_leave_jitted_int(save_exception=False) - self.mc.BL(addr) - self.mc.gen_load_int(r.r0.value, arglocs[-1].value) + [argloc] = arglocs + if argloc is not r.r0: #XXX verify this + self.mov_loc_loc(argloc, r.r0, fcond) + # exit function self.gen_func_epilog() return fcond @@ -1152,8 +1118,26 @@ value = self.cpu.done_with_this_frame_float_v else: raise AssertionError(kind) - self.mc.gen_load_int(r.ip.value, value) - self.mc.CMP_rr(tmploc.value, r.ip.value) + from pypy.jit.backend.llsupport.descr import unpack_fielddescr + from pypy.jit.backend.llsupport.descr import unpack_interiorfielddescr + descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) + _offset, _size, _ = unpack_fielddescr(descrs.jf_descr) + fail_descr = self.cpu.get_fail_descr_from_number(value) + value = fail_descr.hide(self.cpu) + rgc._make_sure_does_not_move(value) + value = rffi.cast(lltype.Signed, value) + + if check_imm_arg(_offset): + self.mc.LDR_ri(r.ip.value, tmploc.value, imm=_offset) + else: + self.mc.gen_load_int(r.ip.value, _offset) + self.mc.LDR_rr(r.ip.value, tmploc.value, r.ip.value) + if check_imm_arg(value): + self.mc.CMP_ri(r.ip.value, imm=value) + else: + self.mc.gen_load_int(r.lr.value, value) + self.mc.CMP_rr(r.lr.value, r.ip.value) + #if values are equal we take the fast path # Slow path, calling helper @@ -1180,19 +1164,31 @@ if op.result is not None: # load the return value from fail_boxes_xxx[0] kind = op.result.type - if kind == INT: - adr = self.fail_boxes_int.get_addr_for_num(0) - elif kind == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(0) - elif kind == FLOAT: - adr = self.fail_boxes_float.get_addr_for_num(0) + if kind == FLOAT: + t = unpack_interiorfielddescr(descrs.as_float)[0] + if not check_imm_arg(t): + self.mc.gen_load_int(r.ip.value, t, cond=fast_path_cond) + self.mc.ADD_rr(r.ip.value, r.r0.value, r.ip.value, + cond=fast_path_cond) + t = 0 + base = r.ip + else: + base = r.r0 + self.mc.VLDR(resloc.value, base.value, imm=t, + cond=fast_path_cond) else: - raise AssertionError(kind) - self.mc.gen_load_int(r.ip.value, adr, cond=fast_path_cond) - if op.result.type == FLOAT: - self.mc.VLDR(resloc.value, r.ip.value, cond=fast_path_cond) - else: - self.mc.LDR_ri(resloc.value, r.ip.value, cond=fast_path_cond) + assert resloc is r.r0 + if kind == INT: + t = unpack_interiorfielddescr(descrs.as_int)[0] + else: + t = unpack_interiorfielddescr(descrs.as_ref)[0] + if not check_imm_arg(t): + self.mc.gen_load_int(r.ip.value, t, cond=fast_path_cond) + self.mc.LDR_rr(resloc.value, resloc.value, r.ip.value, + cond=fast_path_cond) + else: + self.mc.LDR_ri(resloc.value, resloc.value, imm=t, + cond=fast_path_cond) # jump to merge point jmp_pos = self.mc.currpos() self.mc.BKPT() @@ -1266,7 +1262,8 @@ self.mc.LDR_ri(r.ip.value, r.fp.value) self.mc.CMP_ri(r.ip.value, 0) - self._emit_guard(guard_op, arglocs[1 + numargs:], c.GE, save_exc=True) + self._emit_guard(guard_op, arglocs[1 + numargs:], c.GE, + save_exc=True, is_guard_not_forced=True) return fcond def emit_guard_call_release_gil(self, op, guard_op, arglocs, regalloc, diff --git a/pypy/jit/backend/arm/regalloc.py b/pypy/jit/backend/arm/regalloc.py --- a/pypy/jit/backend/arm/regalloc.py +++ b/pypy/jit/backend/arm/regalloc.py @@ -644,15 +644,9 @@ return args def prepare_op_finish(self, op, fcond): - args = [None] * (op.numargs() + 1) - for i in range(op.numargs()): - arg = op.getarg(i) - if arg: - args[i] = self.loc(arg) - self.possibly_free_var(arg) - n = self.cpu.get_fail_descr_number(op.getdescr()) - args[-1] = imm(n) - return args + loc = self.loc(op.getarg(0)) + self.possibly_free_var(op.getarg(0)) + return [loc] def prepare_op_guard_true(self, op, fcond): l0 = self.make_sure_var_in_reg(op.getarg(0)) diff --git a/pypy/jit/backend/arm/runner.py b/pypy/jit/backend/arm/runner.py --- a/pypy/jit/backend/arm/runner.py +++ b/pypy/jit/backend/arm/runner.py @@ -23,15 +23,21 @@ AbstractLLCPU.__init__(self, rtyper, stats, opts, translate_support_code, gcdescr) + from pypy.jit.backend.llsupport import jitframe + self.deadframe_size_max = llmemory.sizeof(jitframe.DEADFRAME, + self.get_failargs_limit()) + def set_debug(self, flag): return self.assembler.set_debug(flag) + def get_failargs_limit(self): + if self.opts is not None: + return self.opts.failargs_limit + else: + return 1000 + def setup(self): - if self.opts is not None: - failargs_limit = self.opts.failargs_limit - else: - failargs_limit = 1000 - self.assembler = AssemblerARM(self, failargs_limit=failargs_limit) + self.assembler = AssemblerARM(self, self.translate_support_code) def setup_once(self): self.assembler.setup_once() @@ -51,24 +57,6 @@ return self.assembler.assemble_bridge(faildescr, inputargs, operations, original_loop_token, log=log) - def get_latest_value_float(self, index): - return self.assembler.fail_boxes_float.getitem(index) - - def get_latest_value_int(self, index): - return self.assembler.fail_boxes_int.getitem(index) - - def get_latest_value_ref(self, index): - return self.assembler.fail_boxes_ptr.getitem(index) - - def get_latest_value_count(self): - return self.assembler.fail_boxes_count - - def get_latest_force_token(self): - return self.assembler.fail_force_index - - def get_on_leave_jitted_hook(self): - return self.assembler.leave_jitted_hook - def clear_latest_values(self, count): setitem = self.assembler.fail_boxes_ptr.setitem null = lltype.nullptr(llmemory.GCREF.TO) @@ -76,7 +64,7 @@ setitem(index, null) def make_execute_token(self, *ARGS): - FUNCPTR = lltype.Ptr(lltype.FuncType(ARGS, lltype.Signed)) + FUNCPTR = lltype.Ptr(lltype.FuncType(ARGS, llmemory.GCREF)) def execute_token(executable_token, *args): clt = executable_token.compiled_loop_token @@ -91,12 +79,13 @@ prev_interpreter = LLInterpreter.current_interpreter LLInterpreter.current_interpreter = self.debug_ll_interpreter try: - fail_index = func(*args) + deadframe = func(*args) finally: if not self.translate_support_code: LLInterpreter.current_interpreter = prev_interpreter #llop.debug_print(lltype.Void, "<<<< Back") - return self.get_fail_descr_from_number(fail_index) + self.gc_set_extra_threshold() + return deadframe return execute_token def cast_ptr_to_int(x): @@ -115,17 +104,22 @@ assert fail_index >= 0, "already forced!" faildescr = self.get_fail_descr_from_number(fail_index) rffi.cast(TP, addr_of_force_index)[0] = ~fail_index - bytecode = self.assembler._find_failure_recovery_bytecode(faildescr) + frb = self.assembler._find_failure_recovery_bytecode(faildescr) + bytecode = rffi.cast(rffi.UCHARP, frb) addr_all_null_regsiters = rffi.cast(rffi.LONG, self.all_null_registers) - # start of "no gc operation!" block - fail_index_2 = self.assembler.failure_recovery_func( - bytecode, - addr_of_force_index, - addr_all_null_regsiters) - self.assembler.leave_jitted_hook() - # end of "no gc operation!" block - assert fail_index == fail_index_2 - return faildescr + # + assert (rffi.cast(lltype.Signed, bytecode[0]) == + self.assembler.CODE_FORCED) + bytecode = rffi.ptradd(bytecode, 1) + deadframe = self.assembler.grab_frame_values(self, + bytecode, addr_of_force_index, + self.all_null_registers, + self.all_null_registers) + # + assert self.get_latest_descr(deadframe) is faildescr + self.assembler.force_token_to_dead_frame[addr_of_force_index] = ( + deadframe) + return deadframe def redirect_call_assembler(self, oldlooptoken, newlooptoken): self.assembler.redirect_call_assembler(oldlooptoken, newlooptoken) diff --git a/pypy/jit/backend/arm/test/test_loop_unroll.py b/pypy/jit/backend/arm/test/test_loop_unroll.py --- a/pypy/jit/backend/arm/test/test_loop_unroll.py +++ b/pypy/jit/backend/arm/test/test_loop_unroll.py @@ -1,8 +1,8 @@ import py -from pypy.jit.backend.x86.test.test_basic import Jit386Mixin +from pypy.jit.backend.arm.test.support import JitARMMixin from pypy.jit.metainterp.test import test_loop_unroll -class TestLoopSpec(Jit386Mixin, test_loop_unroll.LoopUnrollTest): +class TestLoopSpec(JitARMMixin, test_loop_unroll.LoopUnrollTest): # for the individual tests see # ====> ../../../metainterp/test/test_loop.py pass diff --git a/pypy/jit/backend/arm/test/test_runner.py b/pypy/jit/backend/arm/test/test_runner.py --- a/pypy/jit/backend/arm/test/test_runner.py +++ b/pypy/jit/backend/arm/test/test_runner.py @@ -60,16 +60,18 @@ ] cpu.compile_loop(inp, operations, looptoken) args = [i for i in range(1, 15)] - self.cpu.execute_token(looptoken, *args) - output = [self.cpu.get_latest_value_int(i - 1) for i in range(1, 15)] + deadframe = self.cpu.execute_token(looptoken, *args) + output = [self.cpu.get_latest_value_int(deadframe, i - 1) for i in range(1, 15)] expected = [3, 7, 11, 15, 19, 23, 27, 3, 7, 11, 15, 19, 23, 27] assert output == expected - def test_redirect_call_assember2(self): - def assembler_helper(failindex, virtualizable): - return self.cpu.get_latest_value_int(0) + def test_redirect_call_assembler2(self): + def assembler_helper(deadframe, virtualizable): + x = self.cpu.get_latest_value_int(deadframe, 0) + assert x == 11 + return 7 - FUNCPTR = lltype.Ptr(lltype.FuncType([lltype.Signed, llmemory.GCREF], + FUNCPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF, llmemory.GCREF], lltype.Signed)) class FakeJitDriverSD: @@ -101,12 +103,12 @@ self.cpu.compile_loop(loop2.inputargs, loop2.operations, lt2) self.cpu.compile_loop(loop3.inputargs, loop3.operations, lt3) self.cpu.compile_loop(loop1.inputargs, loop1.operations, lt1) - self.cpu.execute_token(lt1, 11) - assert self.cpu.get_latest_value_int(0) == 12 + df = self.cpu.execute_token(lt1, 10) + assert self.cpu.get_latest_value_int(df, 0) == 7 self.cpu.redirect_call_assembler(lt2, lt3) - self.cpu.execute_token(lt1, 11) - assert self.cpu.get_latest_value_int(0) == 10 + df = self.cpu.execute_token(lt1, 12) + assert self.cpu.get_latest_value_int(df, 0) == 7 SFloat = lltype.GcForwardReference() SFloat.become(lltype.GcStruct('SFloat', ('parent', rclass.OBJECT), @@ -202,8 +204,8 @@ ARGS = [lltype.Signed] * numargs RES = lltype.Signed args = [i+1 for i in range(numargs)] - res = self.cpu.execute_token(looptoken, *args) - assert self.cpu.get_latest_value_int(0) == sum(args) + deadframe = self.cpu.execute_token(looptoken, *args) + assert self.cpu.get_latest_value_int(deadframe, 0) == sum(args) def test_debugger_on(self): from pypy.rlib import debug diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py deleted file mode 100644 --- a/pypy/jit/backend/llgraph/llimpl.py +++ /dev/null @@ -1,1968 +0,0 @@ -""" -The non-RPythonic part of the llgraph backend. -This contains all the code that is directly run -when executing on top of the llinterpreter. -""" - -import weakref -from pypy.objspace.flow.model import Variable, Constant -from pypy.annotation import model as annmodel -from pypy.jit.metainterp.history import REF, INT, FLOAT -from pypy.jit.metainterp import history -from pypy.jit.codewriter import heaptracker -from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr, rffi -from pypy.rpython.ootypesystem import ootype -from pypy.rpython.module.support import LLSupport, OOSupport -from pypy.rpython.llinterp import LLException -from pypy.rpython.extregistry import ExtRegistryEntry - -from pypy.jit.metainterp import resoperation -from pypy.jit.metainterp.resoperation import rop -from pypy.jit.backend.llgraph import symbolic -from pypy.jit.codewriter import longlong -from pypy.jit.codewriter.effectinfo import EffectInfo - -from pypy.rlib.objectmodel import ComputedIntSymbolic, we_are_translated -from pypy.rlib.rarithmetic import ovfcheck -from pypy.rlib.rarithmetic import r_longlong, r_ulonglong, r_uint -from pypy.rlib.rtimer import read_timestamp - -import py -from pypy.tool.ansi_print import ansi_log -log = py.log.Producer('runner') -py.log.setconsumer('runner', ansi_log) - -IS_32_BIT = r_ulonglong is not r_uint - - -def _from_opaque(opq): - return opq._obj.externalobj - -_TO_OPAQUE = {} - -def _to_opaque(value): - try: - return value._the_opaque_pointer - except AttributeError: - op = lltype.opaqueptr(_TO_OPAQUE[value.__class__], 'opaque', - externalobj=value) - value._the_opaque_pointer = op - return op - -def _normalize(value): - if isinstance(value, lltype._ptr): - value = lltype.top_container(value._obj) - return value - -def from_opaque_string(s): - if isinstance(s, str): - return s - elif isinstance(s, ootype._string): - return OOSupport.from_rstr(s) - else: - return LLSupport.from_rstr(s) - -FLOAT_ARRAY_TP = lltype.Ptr(lltype.Array(lltype.Float, hints={"nolength": True})) -def maybe_uncast(TP, array): - if array._TYPE.TO.OF != lltype.Float: - # array._TYPE.TO._hints.get("uncast_on_llgraph"): - array = rffi.cast(TP, array) - return array - -# a list of argtypes of all operations - couldn't find any and it's -# very useful. Note however that the table is half-broken here and -# there, in ways that are sometimes a bit hard to fix; that's why -# it is not "official". -TYPES = { - 'int_add' : (('int', 'int'), 'int'), - 'int_sub' : (('int', 'int'), 'int'), - 'int_mul' : (('int', 'int'), 'int'), - 'int_floordiv' : (('int', 'int'), 'int'), - 'int_mod' : (('int', 'int'), 'int'), - 'int_and' : (('int', 'int'), 'int'), - 'int_or' : (('int', 'int'), 'int'), - 'int_xor' : (('int', 'int'), 'int'), - 'int_lshift' : (('int', 'int'), 'int'), - 'int_rshift' : (('int', 'int'), 'int'), - 'int_lt' : (('int', 'int'), 'bool'), - 'int_gt' : (('int', 'int'), 'bool'), - 'int_ge' : (('int', 'int'), 'bool'), - 'int_le' : (('int', 'int'), 'bool'), - 'int_eq' : (('int', 'int'), 'bool'), - 'int_ne' : (('int', 'int'), 'bool'), - 'int_is_true' : (('int',), 'bool'), - 'int_is_zero' : (('int',), 'bool'), - 'int_neg' : (('int',), 'int'), - 'int_invert' : (('int',), 'int'), - 'int_add_ovf' : (('int', 'int'), 'int'), - 'int_sub_ovf' : (('int', 'int'), 'int'), - 'int_mul_ovf' : (('int', 'int'), 'int'), - 'int_force_ge_zero':(('int',), 'int'), - 'uint_add' : (('int', 'int'), 'int'), - 'uint_sub' : (('int', 'int'), 'int'), - 'uint_mul' : (('int', 'int'), 'int'), - 'uint_lt' : (('int', 'int'), 'bool'), - 'uint_le' : (('int', 'int'), 'bool'), - 'uint_eq' : (('int', 'int'), 'bool'), - 'uint_ne' : (('int', 'int'), 'bool'), - 'uint_gt' : (('int', 'int'), 'bool'), - 'uint_ge' : (('int', 'int'), 'bool'), - 'uint_xor' : (('int', 'int'), 'int'), - 'uint_rshift' : (('int', 'int'), 'int'), - 'uint_floordiv' : (('int', 'int'), 'int'), - 'float_add' : (('float', 'float'), 'float'), - 'float_sub' : (('float', 'float'), 'float'), - 'float_mul' : (('float', 'float'), 'float'), - 'float_truediv' : (('float', 'float'), 'float'), - 'float_lt' : (('float', 'float'), 'bool'), - 'float_le' : (('float', 'float'), 'bool'), - 'float_eq' : (('float', 'float'), 'bool'), - 'float_ne' : (('float', 'float'), 'bool'), - 'float_gt' : (('float', 'float'), 'bool'), - 'float_ge' : (('float', 'float'), 'bool'), - 'float_neg' : (('float',), 'float'), - 'float_abs' : (('float',), 'float'), - 'cast_float_to_int':(('float',), 'int'), - 'cast_int_to_float':(('int',), 'float'), - 'same_as' : (('int',), 'int'), # could also be ptr=>ptr - 'new_with_vtable' : (('ref',), 'ref'), - 'new' : ((), 'ref'), - 'new_array' : (('int',), 'ref'), - 'oois' : (('ref', 'ref'), 'bool'), - 'ooisnot' : (('ref', 'ref'), 'bool'), - 'instanceof' : (('ref',), 'bool'), - 'subclassof' : (('ref', 'ref'), 'bool'), - 'runtimenew' : (('ref',), 'ref'), - 'setfield_gc' : (('ref', 'intorptr'), None), - 'getfield_gc' : (('ref',), 'intorptr'), - 'getfield_gc_pure': (('ref',), 'intorptr'), - 'setfield_raw' : (('ref', 'intorptr'), None), - 'getfield_raw' : (('ref',), 'intorptr'), - 'getfield_raw_pure': (('ref',), 'intorptr'), - 'setarrayitem_gc' : (('ref', 'int', 'intorptr'), None), - 'getarrayitem_gc' : (('ref', 'int'), 'intorptr'), - 'getarrayitem_gc_pure' : (('ref', 'int'), 'intorptr'), - 'setarrayitem_raw' : (('ref', 'int', 'intorptr'), None), - 'getarrayitem_raw' : (('ref', 'int'), 'intorptr'), - 'getarrayitem_raw_pure' : (('ref', 'int'), 'intorptr'), - 'arraylen_gc' : (('ref',), 'int'), - 'call' : (('ref', 'varargs'), 'intorptr'), - 'call_assembler' : (('varargs',), 'intorptr'), - 'cond_call_gc_wb' : (('ptr', 'ptr'), None), - 'cond_call_gc_wb_array': (('ptr', 'int', 'ptr'), None), - 'oosend' : (('varargs',), 'intorptr'), - 'oosend_pure' : (('varargs',), 'intorptr'), - 'guard_true' : (('bool',), None), - 'guard_false' : (('bool',), None), - 'guard_value' : (('int', 'int'), None), - 'guard_class' : (('ref', 'ref'), None), - 'guard_no_exception' : ((), None), - 'guard_exception' : (('ref',), 'ref'), - 'guard_no_overflow' : ((), None), - 'guard_overflow' : ((), None), - 'guard_nonnull' : (('ref',), None), - 'guard_isnull' : (('ref',), None), - 'guard_nonnull_class' : (('ref', 'ref'), None), - 'newstr' : (('int',), 'ref'), - 'strlen' : (('ref',), 'int'), - 'strgetitem' : (('ref', 'int'), 'int'), - 'strsetitem' : (('ref', 'int', 'int'), None), - 'newunicode' : (('int',), 'ref'), - 'unicodelen' : (('ref',), 'int'), - 'unicodegetitem' : (('ref', 'int'), 'int'), - 'unicodesetitem' : (('ref', 'int', 'int'), 'int'), - 'cast_ptr_to_int' : (('ref',), 'int'), - 'cast_int_to_ptr' : (('int',), 'ref'), - 'debug_merge_point': (('ref', 'int', 'int'), None), - 'force_token' : ((), 'int'), - 'call_may_force' : (('int', 'varargs'), 'intorptr'), - 'guard_not_forced': ((), None), -} - -# ____________________________________________________________ - -class CompiledLoop(object): - has_been_freed = False - invalid = False - - def __init__(self): - self.inputargs = [] - self.operations = [] - - def getargtypes(self): - return [v.concretetype for v in self.inputargs] - - def __repr__(self): - lines = [] - self.as_text(lines, 1) - return 'CompiledLoop %s:\n%s' % (self.inputargs, '\n'.join(lines)) - - def as_text(self, lines, indent): - for op in self.operations: - lines.append('\t'*indent + repr(op)) - -class Operation(object): - result = None - descr = None - jump_target = None - fail_args = None - - def __init__(self, opnum): - self.opnum = opnum - self.args = [] - - def __repr__(self): - if self.result is not None: - sres = repr0(self.result) + ' = ' - else: - sres = '' - return '{%s%s(%s)}' % (sres, self.getopname(), - ', '.join(map(repr0, self.args))) - - def getopname(self): - try: - return resoperation.opname[self.opnum] - except KeyError: - return '<%d>' % self.opnum - - def is_guard(self): - return rop._GUARD_FIRST <= self.opnum <= rop._GUARD_LAST - - def is_final(self): - return rop._FINAL_FIRST <= self.opnum <= rop._FINAL_LAST - -def repr0(x): - if isinstance(x, list): - return '[' + ', '.join(repr0(y) for y in x) + ']' - elif isinstance(x, Constant): - return '(' + repr0(x.value) + ')' - elif isinstance(x, lltype._ptr): - x = llmemory.cast_ptr_to_adr(x) - if x.ptr: - try: - container = x.ptr._obj._normalizedcontainer() - return '* %s' % (container._TYPE._short_name(),) - except AttributeError: - return repr(x) - else: - return 'NULL' - else: - return repr(x) - -def repr_list(lst, types): - res_l = [] - if types and types[-1] == 'varargs': - types = types[:-1] + ('int',) * (len(lst) - len(types) + 1) - assert len(types) == len(lst) - for elem, tp in zip(lst, types): - if isinstance(elem, Constant): - res_l.append('(%s)' % repr1(elem, tp)) - else: - res_l.append(repr1(elem, tp)) - return '[%s]' % (', '.join(res_l)) - -def repr1(x, tp): - if tp == "intorptr": - TYPE = lltype.typeOf(x) - if isinstance(TYPE, lltype.Ptr) and TYPE.TO._gckind == 'gc': - tp = "ref" - else: - tp = "int" - if tp == 'int': - return str(x) - elif tp == 'void': - return '---' - elif tp == 'ref': - if not x: - return '(* None)' - if isinstance(x, int): - # XXX normalize? - ptr = str(llmemory.cast_int_to_adr(x)) - elif isinstance(ootype.typeOf(x), ootype.OOType): - return repr(x) - else: - if getattr(x, '_fake', None): - return repr(x) - if lltype.typeOf(x) == llmemory.GCREF: - TP = lltype.Ptr(lltype.typeOf(x._obj.container)) - ptr = lltype.cast_opaque_ptr(TP, x) - else: - ptr = x - try: - container = ptr._obj._normalizedcontainer() - return '(* %s)' % (container._TYPE._short_name(),) - except AttributeError: - return '(%r)' % (ptr,) - elif tp == 'bool': - assert x == 0 or x == 1 - return str(bool(x)) - #elif tp == 'fieldname': - # return str(symbolic.TokenToField[x...][1]) - elif tp == 'float': - return str(x) - else: - raise NotImplementedError("tp = %s" % tp) - -_variables = [] - -def compile_start(): - del _variables[:] - return _to_opaque(CompiledLoop()) - -def mark_as_free(loop): - loop = _from_opaque(loop) - assert not loop.has_been_freed - loop.has_been_freed = True - -def compile_start_int_var(loop): - return compile_start_ref_var(loop, lltype.Signed) - -def compile_start_float_var(loop): - return compile_start_ref_var(loop, longlong.FLOATSTORAGE) - -def compile_start_ref_var(loop, TYPE): - loop = _from_opaque(loop) - assert not loop.operations - v = Variable() - v.concretetype = TYPE - loop.inputargs.append(v) - r = len(_variables) - _variables.append(v) - return r - -def compile_started_vars(clt): - if not hasattr(clt, '_debug_argtypes'): # only when compiling the loop - argtypes = [v.concretetype for v in _variables] - try: - clt._debug_argtypes = argtypes - except AttributeError: # when 'clt' is actually a translated - pass # GcStruct - -def compile_add(loop, opnum): - loop = _from_opaque(loop) - loop.operations.append(Operation(opnum)) - -def compile_add_descr(loop, ofs, type, arg_types, extrainfo, width): - from pypy.jit.backend.llgraph.runner import Descr - loop = _from_opaque(loop) - op = loop.operations[-1] - assert isinstance(type, str) and len(type) == 1 - op.descr = Descr(ofs, type, arg_types=arg_types, extrainfo=extrainfo, width=width) - -def compile_add_descr_arg(loop, ofs, type, arg_types): - from pypy.jit.backend.llgraph.runner import Descr - loop = _from_opaque(loop) - op = loop.operations[-1] - assert isinstance(type, str) and len(type) == 1 - op.args.append(Descr(ofs, type, arg_types=arg_types)) - -def compile_add_loop_token(loop, descr): - if we_are_translated(): - raise ValueError("CALL_ASSEMBLER not supported") - loop = _from_opaque(loop) - op = loop.operations[-1] - op.descr = weakref.ref(descr) - -TARGET_TOKENS = weakref.WeakKeyDictionary() - -def compile_add_target_token(loop, descr, clt): - # here, 'clt' is the compiled_loop_token of the original loop that - # we are compiling - loop = _from_opaque(loop) - op = loop.operations[-1] - descrobj = _normalize(descr) - TARGET_TOKENS[descrobj] = loop, len(loop.operations), op.args, clt - -def compile_add_var(loop, intvar): - loop = _from_opaque(loop) - op = loop.operations[-1] - op.args.append(_variables[intvar]) - -def compile_add_int_const(loop, value): - compile_add_ref_const(loop, value, lltype.Signed) - -def compile_add_float_const(loop, value): - compile_add_ref_const(loop, value, longlong.FLOATSTORAGE) - -def compile_add_ref_const(loop, value, TYPE): - loop = _from_opaque(loop) - const = Constant(value) - const.concretetype = TYPE - op = loop.operations[-1] - op.args.append(const) - -def compile_add_int_result(loop): - return compile_add_ref_result(loop, lltype.Signed) - -def compile_add_float_result(loop): - return compile_add_ref_result(loop, longlong.FLOATSTORAGE) - -def compile_add_ref_result(loop, TYPE): - loop = _from_opaque(loop) - v = Variable() - v.concretetype = TYPE - op = loop.operations[-1] - op.result = v - r = len(_variables) - _variables.append(v) - return r - -def compile_add_jump_target(loop, targettoken, source_clt): - loop = _from_opaque(loop) - descrobj = _normalize(targettoken) - (loop_target, target_opindex, target_inputargs, target_clt - ) = TARGET_TOKENS[descrobj] - # - try: - assert source_clt._debug_argtypes == target_clt._debug_argtypes - except AttributeError: # when translated - pass - # - op = loop.operations[-1] - op.jump_target = loop_target - op.jump_target_opindex = target_opindex - op.jump_target_inputargs = target_inputargs - assert op.opnum == rop.JUMP - assert [v.concretetype for v in op.args] == ( - [v.concretetype for v in target_inputargs]) - # - if loop_target == loop: - log.info("compiling new loop") - else: - log.info("compiling new bridge") - -def compile_add_guard_jump_target(loop, loop_target): - loop = _from_opaque(loop) - loop_target = _from_opaque(loop_target) - op = loop.operations[-1] - assert op.is_guard() - op.jump_target = loop_target - -def compile_add_fail(loop, fail_index): - loop = _from_opaque(loop) - index = len(loop.operations)-1 - op = loop.operations[index] - op.fail_index = fail_index - return index - -def compile_add_fail_arg(loop, intvar): - loop = _from_opaque(loop) - op = loop.operations[-1] - if op.fail_args is None: - op.fail_args = [] - if intvar == -1: - op.fail_args.append(None) - else: - op.fail_args.append(_variables[intvar]) - -def compile_redirect_fail(old_loop, old_index, new_loop): - old_loop = _from_opaque(old_loop) - new_loop = _from_opaque(new_loop) - guard_op = old_loop.operations[old_index] - assert guard_op.is_guard() - guard_op.jump_target = new_loop - # check that the bridge's inputargs are of the correct number and - # kind for the guard - if guard_op.fail_args is not None: - argkinds = [v.concretetype for v in guard_op.fail_args if v] - else: - argkinds = [] - assert argkinds == [v.concretetype for v in new_loop.inputargs] - -# ------------------------------ - -class Frame(object): - OPHANDLERS = [None] * (rop._LAST+1) - - def __init__(self, cpu): - self.verbose = False - self.cpu = cpu - self.opindex = 1 - self._forced = False - self._may_force = -1 - - def getenv(self, v): - from pypy.jit.backend.llgraph.runner import Descr - if isinstance(v, Constant): - return v.value - elif isinstance(v, Descr): - return v - else: - return self.env[v] - - def _populate_fail_args(self, op, skip=None): - fail_args = [] - if op.fail_args: - for fail_arg in op.fail_args: - if fail_arg is None: - fail_args.append(None) - elif fail_arg is skip: - fail_args.append(fail_arg.concretetype._defl()) - else: - fail_args.append(self.getenv(fail_arg)) - self.fail_args = fail_args - self.fail_index = op.fail_index - - def execute(self): - """Execute all operations in a loop, - possibly following to other loops as well. - """ - global _last_exception - assert _last_exception is None, "exception left behind" - verbose = True - self.opindex = 0 - while True: - assert not self.loop.has_been_freed - op = self.loop.operations[self.opindex] - args = [self.getenv(v) for v in op.args] - if not op.is_final(): - try: - result = self.execute_operation(op.opnum, args, op.descr, - verbose) - except GuardFailed: - assert op.is_guard() - _stats.exec_conditional_jumps += 1 - if op.jump_target is not None: - # a patched guard, pointing to further code - if op.fail_args: - args = [self.getenv(v) for v in op.fail_args if v] - else: - args = [] - assert len(op.jump_target.inputargs) == len(args) - self.env = dict(zip(op.jump_target.inputargs, args)) - self.loop = op.jump_target - self.opindex = 0 - continue - else: - self._populate_fail_args(op) - # a non-patched guard - if self.verbose: - log.trace('failed: %s' % ( - ', '.join(map(str, fail_args)),)) - return op.fail_index - #verbose = self.verbose - assert (result is None) == (op.result is None) - if op.result is not None: - RESTYPE = op.result.concretetype - if RESTYPE is lltype.Signed: - x = self.as_int(result) - elif RESTYPE is llmemory.GCREF: - x = self.as_ptr(result) - elif RESTYPE is ootype.Object: - x = self.as_object(result) - elif RESTYPE is longlong.FLOATSTORAGE: - x = self.as_floatstorage(result) - else: - raise Exception("op.result.concretetype is %r" - % (RESTYPE,)) - self.env[op.result] = x - self.opindex += 1 - continue - if op.opnum == rop.JUMP: - inputargs = op.jump_target_inputargs - assert len(inputargs) == len(args) - self.env = dict(zip(inputargs, args)) - self.loop = op.jump_target - self.opindex = op.jump_target_opindex - _stats.exec_jumps += 1 - elif op.opnum == rop.FINISH: - if self.verbose: - log.trace('finished: %s' % ( - ', '.join(map(str, args)),)) - self.fail_args = args - return op.fail_index - - else: - assert 0, "unknown final operation %d" % (op.opnum,) - - def execute_operation(self, opnum, values, descr, verbose): - """Execute a single operation. - """ - ophandler = self.OPHANDLERS[opnum] - if ophandler is None: - self._define_impl(opnum) - ophandler = self.OPHANDLERS[opnum] - assert ophandler is not None, "missing impl for op %d" % opnum - opname = resoperation.opname[opnum].lower() - exec_counters = _stats.exec_counters - exec_counters[opname] = exec_counters.get(opname, 0) + 1 - for i in range(len(values)): - if isinstance(values[i], ComputedIntSymbolic): - values[i] = values[i].compute_fn() - res = NotImplemented - try: - res = ophandler(self, descr, *values) - finally: - if 0: # if verbose: - argtypes, restype = TYPES[opname] - if res is None: - resdata = '' - elif res is NotImplemented: - resdata = '*fail*' - else: - resdata = '-> ' + repr1(res, restype) - # fish the types - log.cpu('\t%s %s %s' % (opname, repr_list(values, argtypes), - resdata)) - return res - - def as_int(self, x): - return cast_to_int(x) - - def as_ptr(self, x): - return cast_to_ptr(x) - - def as_object(self, x): - return ootype.cast_to_object(x) - - def as_floatstorage(self, x): - return cast_to_floatstorage(x) - - def log_progress(self): - count = sum(_stats.exec_counters.values()) - count_jumps = _stats.exec_jumps - log.trace('ran %d operations, %d jumps' % (count, count_jumps)) - - # ---------- - - @classmethod - def _define_impl(cls, opnum): - opname = resoperation.opname[opnum] - try: - op = getattr(cls, 'op_' + opname.lower()) # op_guard_true etc. - except AttributeError: - try: - impl = globals()['do_' + opname.lower()] # do_arraylen_gc etc. - def op(self, descr, *args): - if descr is None: - return impl(*args) - else: - return impl(descr, *args) - except KeyError: - op = cls._make_impl_from_blackhole_interp(opname) - cls.OPHANDLERS[opnum] = op - - @classmethod - def _make_impl_from_blackhole_interp(cls, opname): - from pypy.jit.metainterp.blackhole import BlackholeInterpreter - name = 'bhimpl_' + opname.lower() - func = BlackholeInterpreter.__dict__[name] - for argtype in func.argtypes: - assert argtype in ('i', 'r', 'f') - # - def _op_default_implementation(self, descr, *args): - # for all operations implemented in the blackhole interpreter - return func(*args) - # - return _op_default_implementation - - def op_label(self, _, *args): - op = self.loop.operations[self.opindex] - assert op.opnum == rop.LABEL - assert len(op.args) == len(args) - newenv = {} - for v, value in zip(op.args, args): - newenv[v] = value - self.env = newenv - - def op_debug_merge_point(self, _, *args): - from pypy.jit.metainterp.warmspot import get_stats - try: - stats = get_stats() - except AttributeError: - pass - else: - stats.add_merge_point_location(args[1:]) - pass - - def op_guard_true(self, _, value): - if not value: - raise GuardFailed - - def op_guard_false(self, _, value): - if value: - raise GuardFailed - - op_guard_nonnull = op_guard_true - op_guard_isnull = op_guard_false - - def op_guard_class(self, _, value, expected_class): - value = lltype.cast_opaque_ptr(rclass.OBJECTPTR, value) - expected_class = llmemory.cast_adr_to_ptr( - llmemory.cast_int_to_adr(expected_class), - rclass.CLASSTYPE) - if value.typeptr != expected_class: - raise GuardFailed - - def op_guard_nonnull_class(self, _, value, expected_class): - if not value: - raise GuardFailed - self.op_guard_class(_, value, expected_class) - - def op_guard_value(self, _, value, expected_value): - if value != expected_value: - raise GuardFailed - - def op_guard_no_exception(self, _): - if _last_exception: - raise GuardFailed - - def _check_exception(self, expected_exception): - global _last_exception - expected_exception = self._cast_exception(expected_exception) - assert expected_exception - exc = _last_exception - if exc: - got = exc.args[0] - # exact match! - if got != expected_exception: - return False - return True - else: - return False - - def _cast_exception(self, exception): - return llmemory.cast_adr_to_ptr( - llmemory.cast_int_to_adr(exception), - rclass.CLASSTYPE) - - def _issubclass(self, cls1, cls2): - return rclass.ll_issubclass(cls1, cls2) - - def op_guard_exception(self, _, expected_exception): - global _last_exception - if not self._check_exception(expected_exception): - raise GuardFailed - res = _last_exception[1] - _last_exception = None - return res - - def op_guard_no_overflow(self, _): - flag = self.overflow_flag - del self.overflow_flag - if flag: - raise GuardFailed - - def op_guard_overflow(self, _): - flag = self.overflow_flag - del self.overflow_flag - if not flag: - raise GuardFailed - - def op_int_add_ovf(self, _, x, y): - try: - z = ovfcheck(x + y) - except OverflowError: - ovf = True - z = 0 - else: - ovf = False - self.overflow_flag = ovf - return z - - def op_int_sub_ovf(self, _, x, y): - try: - z = ovfcheck(x - y) - except OverflowError: - ovf = True - z = 0 - else: - ovf = False - self.overflow_flag = ovf - return z - - def op_int_mul_ovf(self, _, x, y): - try: - z = ovfcheck(x * y) - except OverflowError: - ovf = True - z = 0 - else: - ovf = False - self.overflow_flag = ovf - return z - - def op_keepalive(self, _, x): - pass - - # ---------- - # delegating to the builtins do_xxx() (done automatically for simple cases) - - def op_getarrayitem_gc(self, arraydescr, array, index): - if arraydescr.typeinfo == REF: - return do_getarrayitem_gc_ptr(array, index) - elif arraydescr.typeinfo == INT: - return do_getarrayitem_gc_int(array, index) - elif arraydescr.typeinfo == FLOAT: - return do_getarrayitem_gc_float(array, index) - else: - raise NotImplementedError - - op_getarrayitem_gc_pure = op_getarrayitem_gc - - def op_getarrayitem_raw(self, arraydescr, array, index): - if arraydescr.typeinfo == REF: - raise NotImplementedError("getarrayitem_raw -> gcref") - elif arraydescr.typeinfo == INT: - return do_getarrayitem_raw_int(array, index, arraydescr.ofs) - elif arraydescr.typeinfo == FLOAT: - return do_getarrayitem_raw_float(array, index) - else: - raise NotImplementedError - - op_getarrayitem_raw_pure = op_getarrayitem_raw - - def op_getfield_gc(self, fielddescr, struct): - if fielddescr.typeinfo == REF: - return do_getfield_gc_ptr(struct, fielddescr.ofs) - elif fielddescr.typeinfo == INT: - return do_getfield_gc_int(struct, fielddescr.ofs) - elif fielddescr.typeinfo == FLOAT: - return do_getfield_gc_float(struct, fielddescr.ofs) - else: - raise NotImplementedError - - op_getfield_gc_pure = op_getfield_gc - - def op_getfield_raw(self, fielddescr, struct): - if fielddescr.typeinfo == REF: - return do_getfield_raw_ptr(struct, fielddescr.ofs) - elif fielddescr.typeinfo == INT: - return do_getfield_raw_int(struct, fielddescr.ofs) - elif fielddescr.typeinfo == FLOAT: - return do_getfield_raw_float(struct, fielddescr.ofs) - else: - raise NotImplementedError - - op_getfield_raw_pure = op_getfield_raw - - def op_raw_store(self, arraydescr, addr, offset, value): - if arraydescr.typeinfo == REF: - raise AssertionError("cannot store GC pointer in raw storage") - elif arraydescr.typeinfo == INT: - do_raw_store_int(addr, offset, arraydescr.ofs, value) - elif arraydescr.typeinfo == FLOAT: - do_raw_store_float(addr, offset, value) - else: - raise NotImplementedError - - def op_raw_load(self, arraydescr, addr, offset): - if arraydescr.typeinfo == REF: - raise AssertionError("cannot store GC pointer in raw storage") - elif arraydescr.typeinfo == INT: - return do_raw_load_int(addr, offset, arraydescr.ofs) - elif arraydescr.typeinfo == FLOAT: - return do_raw_load_float(addr, offset) - else: - raise NotImplementedError - - def op_new(self, size): - return do_new(size.ofs) - - def op_new_with_vtable(self, descr, vtable): - assert descr is None - descr = heaptracker.vtable2descr(self.cpu, vtable) - result = do_new(descr.ofs) - value = lltype.cast_opaque_ptr(rclass.OBJECTPTR, result) - value.typeptr = cast_from_int(rclass.CLASSTYPE, vtable) - return result - - def op_setarrayitem_gc(self, arraydescr, array, index, newvalue): - if arraydescr.typeinfo == REF: - do_setarrayitem_gc_ptr(array, index, newvalue) - elif arraydescr.typeinfo == INT: - do_setarrayitem_gc_int(array, index, newvalue) - elif arraydescr.typeinfo == FLOAT: - do_setarrayitem_gc_float(array, index, newvalue) - else: - raise NotImplementedError - - def op_setarrayitem_raw(self, arraydescr, array, index, newvalue): - if arraydescr.typeinfo == REF: - raise NotImplementedError("setarrayitem_raw <- gcref") - elif arraydescr.typeinfo == INT: - do_setarrayitem_raw_int(array, index, newvalue, arraydescr.ofs) - elif arraydescr.typeinfo == FLOAT: - do_setarrayitem_raw_float(array, index, newvalue) - else: - raise NotImplementedError - - def op_getinteriorfield_gc(self, descr, array, index): - if descr.typeinfo == REF: - return do_getinteriorfield_gc_ptr(array, index, descr.ofs) - elif descr.typeinfo == INT: - return do_getinteriorfield_gc_int(array, index, descr.ofs) - elif descr.typeinfo == FLOAT: - return do_getinteriorfield_gc_float(array, index, descr.ofs) - else: - raise NotImplementedError - - def op_getinteriorfield_raw(self, descr, array, index): - if descr.typeinfo == REF: - return do_getinteriorfield_raw_ptr(array, index, descr.width, descr.ofs) - elif descr.typeinfo == INT: - return do_getinteriorfield_raw_int(array, index, descr.width, descr.ofs) - elif descr.typeinfo == FLOAT: - return do_getinteriorfield_raw_float(array, index, descr.width, descr.ofs) - else: - raise NotImplementedError - - def op_setinteriorfield_gc(self, descr, array, index, newvalue): - if descr.typeinfo == REF: - return do_setinteriorfield_gc_ptr(array, index, descr.ofs, - newvalue) - elif descr.typeinfo == INT: - return do_setinteriorfield_gc_int(array, index, descr.ofs, - newvalue) - elif descr.typeinfo == FLOAT: - return do_setinteriorfield_gc_float(array, index, descr.ofs, - newvalue) - else: - raise NotImplementedError - - def op_setinteriorfield_raw(self, descr, array, index, newvalue): - if descr.typeinfo == REF: - return do_setinteriorfield_raw_ptr(array, index, newvalue, descr.width, descr.ofs) - elif descr.typeinfo == INT: - return do_setinteriorfield_raw_int(array, index, newvalue, descr.width, descr.ofs) - elif descr.typeinfo == FLOAT: - return do_setinteriorfield_raw_float(array, index, newvalue, descr.width, descr.ofs) - else: - raise NotImplementedError - - def op_setfield_gc(self, fielddescr, struct, newvalue): - if fielddescr.typeinfo == REF: - do_setfield_gc_ptr(struct, fielddescr.ofs, newvalue) - elif fielddescr.typeinfo == INT: - do_setfield_gc_int(struct, fielddescr.ofs, newvalue) - elif fielddescr.typeinfo == FLOAT: - do_setfield_gc_float(struct, fielddescr.ofs, newvalue) - else: - raise NotImplementedError - - def op_setfield_raw(self, fielddescr, struct, newvalue): - if fielddescr.typeinfo == REF: - do_setfield_raw_ptr(struct, fielddescr.ofs, newvalue) - elif fielddescr.typeinfo == INT: - do_setfield_raw_int(struct, fielddescr.ofs, newvalue) - elif fielddescr.typeinfo == FLOAT: - do_setfield_raw_float(struct, fielddescr.ofs, newvalue) - else: - raise NotImplementedError - - def op_call(self, calldescr, func, *args): - effectinfo = calldescr.get_extra_info() - if effectinfo is not None and hasattr(effectinfo, 'oopspecindex'): - oopspecindex = effectinfo.oopspecindex - if oopspecindex == EffectInfo.OS_MATH_SQRT: - return do_math_sqrt(args[0]) - return self._do_call(calldescr, func, args, call_with_llptr=False) - - def op_call_release_gil(self, calldescr, func, *args): - return self._do_call(calldescr, func, args, call_with_llptr=True) - - def _do_call(self, calldescr, func, args, call_with_llptr): - global _last_exception - assert _last_exception is None, "exception left behind" - assert _call_args_i == _call_args_r == _call_args_f == [] - args_in_order = [] - for x in args: - T = lltype.typeOf(x) - if T is lltype.Signed: - args_in_order.append('i') - _call_args_i.append(x) - elif T == llmemory.GCREF: - args_in_order.append('r') - _call_args_r.append(x) - elif T is longlong.FLOATSTORAGE: - args_in_order.append('f') - _call_args_f.append(x) - else: - raise TypeError(x) - try: - return _do_call_common(func, args_in_order, calldescr, - call_with_llptr) - except LLException, lle: - _last_exception = lle - d = {'v': None, - REF: lltype.nullptr(llmemory.GCREF.TO), - INT: 0, - FLOAT: 0.0} - return d[calldescr.typeinfo] - - def op_cond_call_gc_wb(self, descr, a, b): - py.test.skip("cond_call_gc_wb not supported") - - def op_cond_call_gc_wb_array(self, descr, a, b, c): - py.test.skip("cond_call_gc_wb_array not supported") - - def op_oosend(self, descr, obj, *args): - raise NotImplementedError("oosend for lltype backend??") - - op_oosend_pure = op_oosend - - def op_new_array(self, arraydescr, count): - return do_new_array(arraydescr.ofs, count) - - def op_force_token(self, descr): - opaque_frame = _to_opaque(self) - return llmemory.cast_ptr_to_adr(opaque_frame) - - def op_read_timestamp(self, descr): - return read_timestamp() - - def op_call_may_force(self, calldescr, func, *args): - assert not self._forced - self._may_force = self.opindex - try: - return self.op_call(calldescr, func, *args) - finally: - self._may_force = -1 - - def op_call_assembler(self, wref_loop_token, *args): - if we_are_translated(): - raise ValueError("CALL_ASSEMBLER not supported") - return self._do_call_assembler(wref_loop_token, *args) - - def _do_call_assembler(self, wref_loop_token, *args): - global _last_exception - loop_token = wref_loop_token() - assert loop_token, "CALL_ASSEMBLER to a target that already died" - ctl = loop_token.compiled_loop_token - if hasattr(ctl, 'redirected'): - return self._do_call_assembler(ctl.redirected, *args) - assert not self._forced - self._may_force = self.opindex - try: - inpargs = _from_opaque(ctl.compiled_version).inputargs - assert len(inpargs) == len(args) From noreply at buildbot.pypy.org Sat Jan 5 21:26:10 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 5 Jan 2013 21:26:10 +0100 (CET) Subject: [pypy-commit] pypy default: merge Message-ID: <20130105202610.8790B1C01A6@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r59779:99da6b4f5a3f Date: 2013-01-05 22:25 +0200 http://bitbucket.org/pypy/pypy/changeset/99da6b4f5a3f/ Log: merge diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py --- a/pypy/rlib/rgc.py +++ b/pypy/rlib/rgc.py @@ -139,6 +139,40 @@ hop.exception_cannot_occur() return hop.genop(opname, vlist, resulttype = hop.r_result.lowleveltype) +def copy_struct_item(source, dest, si, di): + TP = lltype.typeOf(source) + i = 0 + while i < len(TP._names): + setattr(dest[di], TP._names[i], getattr(source[si], TP._names[i])) + +class CopyStructEntry(ExtRegistryEntry): + _about_ = copy_struct_item + + def compute_result_annotation(self, s_source, s_dest, si, di): + pass + + def specialize_call(self, hop): + v_source, v_dest, v_si, v_di = hop.inputargs(hop.args_r[0], + hop.args_r[1], + lltype.Signed, + lltype.Signed) + hop.exception_cannot_occur() + TP = v_source.concretetype.TO.OF + for name, TP in TP._flds.iteritems(): + c_name = hop.inputconst(lltype.Void, name) + v_fld = hop.genop('getinteriorfield', [v_source, v_si, c_name], + resulttype=TP) + hop.genop('setinteriorfield', [v_dest, v_di, c_name, v_fld]) + + + at specialize.ll() +def copy_item(source, dest, si, di): + TP = lltype.typeOf(source) + if isinstance(TP.TO.OF, lltype.Struct): + copy_struct_item(source, dest, si, di) + else: + dest[di] = source[si] + @jit.oopspec('list.ll_arraycopy(source, dest, source_start, dest_start, length)') @enforceargs(None, None, int, int, int) @specialize.ll() @@ -150,7 +184,7 @@ # and also, maybe, speed up very small cases if length <= 1: if length == 1: - dest[dest_start] = source[source_start] + copy_item(source, dest, source_start, dest_start) return # supports non-overlapping copies only @@ -170,7 +204,7 @@ # if the write barrier is not supported, copy by hand i = 0 while i < length: - dest[i + dest_start] = source[i + source_start] + copy_item(source, dest, i + source_start, i + dest_start) i += 1 return source_addr = llmemory.cast_ptr_to_adr(source) diff --git a/pypy/rlib/test/test_rgc.py b/pypy/rlib/test/test_rgc.py --- a/pypy/rlib/test/test_rgc.py +++ b/pypy/rlib/test/test_rgc.py @@ -134,6 +134,24 @@ assert check.called +def test_ll_arraycopy_array_of_structs(): + TP = lltype.GcArray(lltype.Struct('x', ('x', lltype.Signed), + ('y', lltype.Signed))) + def f(): + a1 = lltype.malloc(TP, 3) + a2 = lltype.malloc(TP, 3) + for i in range(3): + a1[i].x = 2 * i + a1[i].y = 2 * i + 1 + rgc.ll_arraycopy(a1, a2, 0, 0, 3) + for i in range(3): + assert a2[i].x == 2 * i + assert a2[i].y == 2 * i + 1 + + + interpret(f, []) + f() + def test_ll_arraycopy_small(): TYPE = lltype.GcArray(lltype.Signed) for length in range(5): diff --git a/pypy/rpython/lltypesystem/test/test_lltype.py b/pypy/rpython/lltypesystem/test/test_lltype.py --- a/pypy/rpython/lltypesystem/test/test_lltype.py +++ b/pypy/rpython/lltypesystem/test/test_lltype.py @@ -808,7 +808,6 @@ assert F.RESULT == Signed assert F.ARGS == (Signed,) - class TestTrackAllocation: def test_automatic_tracking(self): # calls to start_tracking_allocations/stop_tracking_allocations From noreply at buildbot.pypy.org Sat Jan 5 21:27:37 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 5 Jan 2013 21:27:37 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments: merge default Message-ID: <20130105202737.AA8961C01A6@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments Changeset: r59780:45bee24a2c4b Date: 2013-01-05 22:27 +0200 http://bitbucket.org/pypy/pypy/changeset/45bee24a2c4b/ Log: merge default diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py --- a/pypy/rlib/rgc.py +++ b/pypy/rlib/rgc.py @@ -139,6 +139,40 @@ hop.exception_cannot_occur() return hop.genop(opname, vlist, resulttype = hop.r_result.lowleveltype) +def copy_struct_item(source, dest, si, di): + TP = lltype.typeOf(source) + i = 0 + while i < len(TP._names): + setattr(dest[di], TP._names[i], getattr(source[si], TP._names[i])) + +class CopyStructEntry(ExtRegistryEntry): + _about_ = copy_struct_item + + def compute_result_annotation(self, s_source, s_dest, si, di): + pass + + def specialize_call(self, hop): + v_source, v_dest, v_si, v_di = hop.inputargs(hop.args_r[0], + hop.args_r[1], + lltype.Signed, + lltype.Signed) + hop.exception_cannot_occur() + TP = v_source.concretetype.TO.OF + for name, TP in TP._flds.iteritems(): + c_name = hop.inputconst(lltype.Void, name) + v_fld = hop.genop('getinteriorfield', [v_source, v_si, c_name], + resulttype=TP) + hop.genop('setinteriorfield', [v_dest, v_di, c_name, v_fld]) + + + at specialize.ll() +def copy_item(source, dest, si, di): + TP = lltype.typeOf(source) + if isinstance(TP.TO.OF, lltype.Struct): + copy_struct_item(source, dest, si, di) + else: + dest[di] = source[si] + @jit.oopspec('list.ll_arraycopy(source, dest, source_start, dest_start, length)') @enforceargs(None, None, int, int, int) @specialize.ll() @@ -150,7 +184,7 @@ # and also, maybe, speed up very small cases if length <= 1: if length == 1: - dest[dest_start] = source[source_start] + copy_item(source, dest, source_start, dest_start) return # supports non-overlapping copies only @@ -170,7 +204,7 @@ # if the write barrier is not supported, copy by hand i = 0 while i < length: - dest[i + dest_start] = source[i + source_start] + copy_item(source, dest, i + source_start, i + dest_start) i += 1 return source_addr = llmemory.cast_ptr_to_adr(source) diff --git a/pypy/rlib/test/test_rgc.py b/pypy/rlib/test/test_rgc.py --- a/pypy/rlib/test/test_rgc.py +++ b/pypy/rlib/test/test_rgc.py @@ -134,6 +134,24 @@ assert check.called +def test_ll_arraycopy_array_of_structs(): + TP = lltype.GcArray(lltype.Struct('x', ('x', lltype.Signed), + ('y', lltype.Signed))) + def f(): + a1 = lltype.malloc(TP, 3) + a2 = lltype.malloc(TP, 3) + for i in range(3): + a1[i].x = 2 * i + a1[i].y = 2 * i + 1 + rgc.ll_arraycopy(a1, a2, 0, 0, 3) + for i in range(3): + assert a2[i].x == 2 * i + assert a2[i].y == 2 * i + 1 + + + interpret(f, []) + f() + def test_ll_arraycopy_small(): TYPE = lltype.GcArray(lltype.Signed) for length in range(5): diff --git a/pypy/rpython/lltypesystem/test/test_lltype.py b/pypy/rpython/lltypesystem/test/test_lltype.py --- a/pypy/rpython/lltypesystem/test/test_lltype.py +++ b/pypy/rpython/lltypesystem/test/test_lltype.py @@ -808,7 +808,6 @@ assert F.RESULT == Signed assert F.ARGS == (Signed,) - class TestTrackAllocation: def test_automatic_tracking(self): # calls to start_tracking_allocations/stop_tracking_allocations From noreply at buildbot.pypy.org Sat Jan 5 21:31:58 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 5 Jan 2013 21:31:58 +0100 (CET) Subject: [pypy-commit] pypy default: a test and a fix Message-ID: <20130105203158.876011C01EE@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r59781:dc75dbc8a7dd Date: 2013-01-05 22:31 +0200 http://bitbucket.org/pypy/pypy/changeset/dc75dbc8a7dd/ Log: a test and a fix diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py --- a/pypy/rlib/rgc.py +++ b/pypy/rlib/rgc.py @@ -140,10 +140,11 @@ return hop.genop(opname, vlist, resulttype = hop.r_result.lowleveltype) def copy_struct_item(source, dest, si, di): - TP = lltype.typeOf(source) + TP = lltype.typeOf(source).TO.OF i = 0 while i < len(TP._names): setattr(dest[di], TP._names[i], getattr(source[si], TP._names[i])) + i += 1 class CopyStructEntry(ExtRegistryEntry): _about_ = copy_struct_item diff --git a/pypy/rlib/test/test_rgc.py b/pypy/rlib/test/test_rgc.py --- a/pypy/rlib/test/test_rgc.py +++ b/pypy/rlib/test/test_rgc.py @@ -150,7 +150,13 @@ interpret(f, []) - f() + a1 = lltype.malloc(TP, 3) + a2 = lltype.malloc(TP, 3) + a1[1].x = 3 + a1[1].y = 15 + rgc.copy_struct_item(a1, a2, 1, 2) + assert a2[2].x == 3 + assert a2[2].y == 15 def test_ll_arraycopy_small(): TYPE = lltype.GcArray(lltype.Signed) From noreply at buildbot.pypy.org Sat Jan 5 21:36:51 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 5 Jan 2013 21:36:51 +0100 (CET) Subject: [pypy-commit] pypy default: a fix for writebarrier (a bit hard to test otherwise) and a direct test Message-ID: <20130105203651.CD3FD1C01EE@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r59782:3705ab3e421a Date: 2013-01-05 22:36 +0200 http://bitbucket.org/pypy/pypy/changeset/3705ab3e421a/ Log: a fix for writebarrier (a bit hard to test otherwise) and a direct test diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py --- a/pypy/rlib/rgc.py +++ b/pypy/rlib/rgc.py @@ -174,6 +174,15 @@ else: dest[di] = source[si] + at specialize.memo() +def _contains_gcptr(TP): + if not isinstance(TP, lltype.Struct): + return False + for TP in TP._flds.itervalues(): + if isinstance(TP, lltype.Ptr) and TP.TO._gckind == 'gc': + return True + return False + @jit.oopspec('list.ll_arraycopy(source, dest, source_start, dest_start, length)') @enforceargs(None, None, int, int, int) @specialize.ll() @@ -196,7 +205,8 @@ TP = lltype.typeOf(source).TO assert TP == lltype.typeOf(dest).TO - if isinstance(TP.OF, lltype.Ptr) and TP.OF.TO._gckind == 'gc': + if isinstance(TP.OF, lltype.Ptr) and (TP.OF.TO._gckind == 'gc' + or _contains_gcptr(TP.OF.TO)): # perform a write barrier that copies necessary flags from # source to dest if not llop.gc_writebarrier_before_copy(lltype.Bool, source, dest, diff --git a/pypy/rlib/test/test_rgc.py b/pypy/rlib/test/test_rgc.py --- a/pypy/rlib/test/test_rgc.py +++ b/pypy/rlib/test/test_rgc.py @@ -158,6 +158,14 @@ assert a2[2].x == 3 assert a2[2].y == 15 +def test__contains_gcptr(): + assert not rgc._contains_gcptr(lltype.Signed) + assert not rgc._contains_gcptr(lltype.Struct('x', ('x', lltype.Signed))) + assert rgc._contains_gcptr(lltype.Struct('x', ('x', lltype.Signed), + ('y', lltype.Ptr(lltype.GcArray(lltype.Signed))))) + assert rgc._contains_gcptr(lltype.Struct('x', ('x', lltype.Signed), + ('y', llmemory.GCREF))) + def test_ll_arraycopy_small(): TYPE = lltype.GcArray(lltype.Signed) for length in range(5): From noreply at buildbot.pypy.org Sat Jan 5 21:36:53 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 5 Jan 2013 21:36:53 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments: merge default Message-ID: <20130105203653.20EA71C01EE@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments Changeset: r59783:c6a93cd01585 Date: 2013-01-05 22:36 +0200 http://bitbucket.org/pypy/pypy/changeset/c6a93cd01585/ Log: merge default diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py --- a/pypy/rlib/rgc.py +++ b/pypy/rlib/rgc.py @@ -140,10 +140,11 @@ return hop.genop(opname, vlist, resulttype = hop.r_result.lowleveltype) def copy_struct_item(source, dest, si, di): - TP = lltype.typeOf(source) + TP = lltype.typeOf(source).TO.OF i = 0 while i < len(TP._names): setattr(dest[di], TP._names[i], getattr(source[si], TP._names[i])) + i += 1 class CopyStructEntry(ExtRegistryEntry): _about_ = copy_struct_item @@ -173,6 +174,15 @@ else: dest[di] = source[si] + at specialize.memo() +def _contains_gcptr(TP): + if not isinstance(TP, lltype.Struct): + return False + for TP in TP._flds.itervalues(): + if isinstance(TP, lltype.Ptr) and TP.TO._gckind == 'gc': + return True + return False + @jit.oopspec('list.ll_arraycopy(source, dest, source_start, dest_start, length)') @enforceargs(None, None, int, int, int) @specialize.ll() @@ -195,7 +205,8 @@ TP = lltype.typeOf(source).TO assert TP == lltype.typeOf(dest).TO - if isinstance(TP.OF, lltype.Ptr) and TP.OF.TO._gckind == 'gc': + if isinstance(TP.OF, lltype.Ptr) and (TP.OF.TO._gckind == 'gc' + or _contains_gcptr(TP.OF.TO)): # perform a write barrier that copies necessary flags from # source to dest if not llop.gc_writebarrier_before_copy(lltype.Bool, source, dest, diff --git a/pypy/rlib/test/test_rgc.py b/pypy/rlib/test/test_rgc.py --- a/pypy/rlib/test/test_rgc.py +++ b/pypy/rlib/test/test_rgc.py @@ -150,7 +150,21 @@ interpret(f, []) - f() + a1 = lltype.malloc(TP, 3) + a2 = lltype.malloc(TP, 3) + a1[1].x = 3 + a1[1].y = 15 + rgc.copy_struct_item(a1, a2, 1, 2) + assert a2[2].x == 3 + assert a2[2].y == 15 + +def test__contains_gcptr(): + assert not rgc._contains_gcptr(lltype.Signed) + assert not rgc._contains_gcptr(lltype.Struct('x', ('x', lltype.Signed))) + assert rgc._contains_gcptr(lltype.Struct('x', ('x', lltype.Signed), + ('y', lltype.Ptr(lltype.GcArray(lltype.Signed))))) + assert rgc._contains_gcptr(lltype.Struct('x', ('x', lltype.Signed), + ('y', llmemory.GCREF))) def test_ll_arraycopy_small(): TYPE = lltype.GcArray(lltype.Signed) From noreply at buildbot.pypy.org Sat Jan 5 21:42:59 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 5 Jan 2013 21:42:59 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments: fix fix fix Message-ID: <20130105204259.C468B1C01EE@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments Changeset: r59784:c1833b664452 Date: 2013-01-05 22:42 +0200 http://bitbucket.org/pypy/pypy/changeset/c1833b664452/ Log: fix fix fix diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py --- a/pypy/rlib/rgc.py +++ b/pypy/rlib/rgc.py @@ -176,7 +176,10 @@ @specialize.memo() def _contains_gcptr(TP): + TP = TP.OF if not isinstance(TP, lltype.Struct): + if isinstance(TP, lltype.Ptr) and TP.TO._gckind == 'gc': + return True return False for TP in TP._flds.itervalues(): if isinstance(TP, lltype.Ptr) and TP.TO._gckind == 'gc': @@ -205,8 +208,7 @@ TP = lltype.typeOf(source).TO assert TP == lltype.typeOf(dest).TO - if isinstance(TP.OF, lltype.Ptr) and (TP.OF.TO._gckind == 'gc' - or _contains_gcptr(TP.OF.TO)): + if _contains_gcptr(TP): # perform a write barrier that copies necessary flags from # source to dest if not llop.gc_writebarrier_before_copy(lltype.Bool, source, dest, diff --git a/pypy/rlib/test/test_rgc.py b/pypy/rlib/test/test_rgc.py --- a/pypy/rlib/test/test_rgc.py +++ b/pypy/rlib/test/test_rgc.py @@ -159,12 +159,17 @@ assert a2[2].y == 15 def test__contains_gcptr(): - assert not rgc._contains_gcptr(lltype.Signed) - assert not rgc._contains_gcptr(lltype.Struct('x', ('x', lltype.Signed))) - assert rgc._contains_gcptr(lltype.Struct('x', ('x', lltype.Signed), - ('y', lltype.Ptr(lltype.GcArray(lltype.Signed))))) - assert rgc._contains_gcptr(lltype.Struct('x', ('x', lltype.Signed), - ('y', llmemory.GCREF))) + assert not rgc._contains_gcptr(lltype.GcArray(lltype.Signed)) + assert not rgc._contains_gcptr(lltype.GcArray( + lltype.Struct('x', ('x', lltype.Signed)))) + assert rgc._contains_gcptr(lltype.GcArray( + lltype.Struct('x', ('x', lltype.Signed), + ('y', lltype.Ptr(lltype.GcArray(lltype.Signed)))))) + assert rgc._contains_gcptr(lltype.GcArray( + lltype.Struct('x', ('x', lltype.Signed), + ('y', llmemory.GCREF)))) + assert rgc._contains_gcptr(lltype.GcArray(lltype.Ptr(lltype.GcStruct('x')))) + assert not rgc._contains_gcptr(lltype.GcArray(lltype.Ptr(lltype.Struct('x')))) def test_ll_arraycopy_small(): TYPE = lltype.GcArray(lltype.Signed) From noreply at buildbot.pypy.org Sat Jan 5 21:48:35 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sat, 5 Jan 2013 21:48:35 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Moved cdir to rpython.conftest, moved app_main to pypy/interpreter Message-ID: <20130105204835.1DBE41C0C35@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59785:62d8bb3280a1 Date: 2013-01-05 21:48 +0100 http://bitbucket.org/pypy/pypy/changeset/62d8bb3280a1/ Log: Moved cdir to rpython.conftest, moved app_main to pypy/interpreter diff --git a/rpython/translator/goal/app_main.py b/pypy/interpreter/app_main.py rename from rpython/translator/goal/app_main.py rename to pypy/interpreter/app_main.py diff --git a/pypy/module/_lsprof/interp_lsprof.py b/pypy/module/_lsprof/interp_lsprof.py --- a/pypy/module/_lsprof/interp_lsprof.py +++ b/pypy/module/_lsprof/interp_lsprof.py @@ -11,7 +11,7 @@ from rpython.rlib.rtimer import read_timestamp, _is_64_bit from rpython.rtyper.lltypesystem import rffi, lltype from rpython.translator.tool.cbuild import ExternalCompilationInfo -from rpython.translator.translator import cdir +from rpython.conftest import cdir from rpython.rlib.rarithmetic import r_longlong import time, sys diff --git a/pypy/module/_multibytecodec/c_codecs.py b/pypy/module/_multibytecodec/c_codecs.py --- a/pypy/module/_multibytecodec/c_codecs.py +++ b/pypy/module/_multibytecodec/c_codecs.py @@ -1,7 +1,7 @@ import py from rpython.rtyper.lltypesystem import lltype, rffi from rpython.translator.tool.cbuild import ExternalCompilationInfo -from rpython.translator.translator import cdir +from rpython.conftest import cdir UNICODE_REPLACEMENT_CHARACTER = u'\uFFFD' diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -10,7 +10,7 @@ from rpython.rtyper.lltypesystem import ll2ctypes from rpython.rtyper.annlowlevel import llhelper from rpython.rlib.objectmodel import we_are_translated -from rpython.translator.translator import cdir +from rpython.conftest import cdir from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.translator.gensupp import NameManager from rpython.tool.udir import udir diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py --- a/pypy/module/signal/interp_signal.py +++ b/pypy/module/signal/interp_signal.py @@ -7,7 +7,7 @@ from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rtyper.tool import rffi_platform from rpython.translator.tool.cbuild import ExternalCompilationInfo -from rpython.translator.translator import cdir +from rpython.conftest import cdir import py import sys from rpython.rlib import jit, rposix diff --git a/pypy/module/thread/ll_thread.py b/pypy/module/thread/ll_thread.py --- a/pypy/module/thread/ll_thread.py +++ b/pypy/module/thread/ll_thread.py @@ -1,7 +1,7 @@ from rpython.rtyper.lltypesystem import rffi, lltype, llmemory from rpython.translator.tool.cbuild import ExternalCompilationInfo -from rpython.translator.translator import cdir +from rpython.conftest import cdir import py from rpython.rlib import jit, rgc from rpython.rlib.debug import ll_assert diff --git a/rpython/rlib/_rffi_stacklet.py b/rpython/rlib/_rffi_stacklet.py --- a/rpython/rlib/_rffi_stacklet.py +++ b/rpython/rlib/_rffi_stacklet.py @@ -3,7 +3,7 @@ from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.rtyper.tool import rffi_platform from rpython.rlib.rarithmetic import is_emulated_long -from rpython.translator.translator import cdir +from rpython.conftest import cdir import sys diff --git a/rpython/rlib/clibffi.py b/rpython/rlib/clibffi.py --- a/rpython/rlib/clibffi.py +++ b/rpython/rlib/clibffi.py @@ -14,7 +14,7 @@ from rpython.rlib.objectmodel import specialize from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.translator.platform import platform -from rpython.translator.translator import cdir +from rpython.conftest import cdir import py import os import sys diff --git a/rpython/rlib/rdtoa.py b/rpython/rlib/rdtoa.py --- a/rpython/rlib/rdtoa.py +++ b/rpython/rlib/rdtoa.py @@ -1,7 +1,7 @@ from __future__ import with_statement from rpython.rlib import rfloat from rpython.translator.tool.cbuild import ExternalCompilationInfo -from rpython.translator.translator import cdir +from rpython.conftest import cdir from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rlib import jit from rpython.rlib.rstring import StringBuilder diff --git a/rpython/rlib/rstack.py b/rpython/rlib/rstack.py --- a/rpython/rlib/rstack.py +++ b/rpython/rlib/rstack.py @@ -14,7 +14,7 @@ from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rtyper.lltypesystem.lloperation import llop from rpython.rtyper.controllerentry import Controller, SomeControlledInstance -from rpython.translator.translator import cdir +from rpython.conftest import cdir from rpython.translator.tool.cbuild import ExternalCompilationInfo # ____________________________________________________________ diff --git a/rpython/rtyper/lltypesystem/module/ll_math.py b/rpython/rtyper/lltypesystem/module/ll_math.py --- a/rpython/rtyper/lltypesystem/module/ll_math.py +++ b/rpython/rtyper/lltypesystem/module/ll_math.py @@ -5,7 +5,7 @@ from rpython.rtyper.lltypesystem import lltype, rffi from rpython.tool.sourcetools import func_with_new_name -from rpython.translator.translator import cdir +from rpython.conftest import cdir from rpython.rlib import jit, rposix from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.translator.platform import platform diff --git a/rpython/rtyper/module/ll_strtod.py b/rpython/rtyper/module/ll_strtod.py --- a/rpython/rtyper/module/ll_strtod.py +++ b/rpython/rtyper/module/ll_strtod.py @@ -6,7 +6,7 @@ from rpython.rtyper.ootypesystem import ootype from rpython.rlib import rposix from rpython.translator.tool.cbuild import ExternalCompilationInfo -from rpython.translator.translator import cdir +from rpython.conftest import cdir from rpython.annotator.model import SomeString class CConfig: diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py --- a/rpython/translator/c/test/test_standalone.py +++ b/rpython/translator/c/test/test_standalone.py @@ -5,11 +5,12 @@ from rpython.rlib.rarithmetic import r_longlong from rpython.rlib.debug import ll_assert, have_debug_prints, debug_flush from rpython.rlib.debug import debug_print, debug_start, debug_stop, debug_offset -from rpython.translator.translator import TranslationContext, cdir +from rpython.translator.translator import TranslationContext from rpython.translator.backendopt import all from rpython.translator.c.genc import CStandaloneBuilder, ExternalCompilationInfo from rpython.annotator.listdef import s_list_of_strings from rpython.tool.udir import udir +from rpython.conftest import cdir from pypy.conftest import option diff --git a/rpython/translator/translator.py b/rpython/translator/translator.py --- a/rpython/translator/translator.py +++ b/rpython/translator/translator.py @@ -6,8 +6,6 @@ """ import os, sys, types, copy -cdir = os.path.join(os.path.dirname(__file__), 'c') - from rpython.translator import simplify from rpython.flowspace.model import FunctionGraph, checkgraph, Block from rpython.flowspace.objspace import FlowObjSpace diff --git a/targetpypystandalone.py b/targetpypystandalone.py --- a/targetpypystandalone.py +++ b/targetpypystandalone.py @@ -8,6 +8,7 @@ from pypy.config.config import Config, to_optparse, make_dict, SUPPRESS_USAGE from pypy.config.config import ConflictConfigError from pypy.tool.option import make_objspace +from pypy.conftest import pypydir thisdir = py.path.local(__file__).dirpath() @@ -227,7 +228,7 @@ space = make_objspace(config) # manually imports app_main.py - filename = os.path.join(this_dir, 'app_main.py') + filename = os.path.join(pypydir, 'interpreter', 'app_main.py') app = gateway.applevel(open(filename).read(), 'app_main.py', 'app_main') app.hidden_applevel = False w_dict = app.getwdict(space) From noreply at buildbot.pypy.org Sat Jan 5 21:50:31 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 5 Jan 2013 21:50:31 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments: use smaller int, for fun Message-ID: <20130105205031.4FB2C1C0C35@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments Changeset: r59786:562571a76e0b Date: 2013-01-05 22:49 +0200 http://bitbucket.org/pypy/pypy/changeset/562571a76e0b/ Log: use smaller int, for fun diff --git a/pypy/rlib/dict.py b/pypy/rlib/dict.py --- a/pypy/rlib/dict.py +++ b/pypy/rlib/dict.py @@ -1,18 +1,18 @@ from pypy.rlib.jit_hooks import _cast_to_gcref from pypy.rlib import rgc -from pypy.rpython.lltypesystem import lltype, llmemory, rclass +from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rffi from pypy.rpython.annlowlevel import cast_base_ptr_to_instance # Placeholder constants -FREE = -1 -DUMMY = -2 TP = lltype.GcArray(lltype.Struct('dictentry', ('key', lltype.Signed), ('value', llmemory.GCREF))) -MAIN_TP = lltype.GcArray(lltype.Signed) +MAIN_TP = lltype.GcArray(rffi.INT_real) +FREE = rffi.cast(rffi.INT_real, -1) +DUMMY = rffi.cast(rffi.INT_real, -2) class Dict(object): 'Space efficient dictionary with fast iteration and cheap resizes.' @@ -29,7 +29,7 @@ n = len(self.indices) i = perturb & (n - 1) while True: - index = self.indices[i] + index = rffi.cast(lltype.Signed, self.indices[i]) if index == FREE: return (FREE, i) if freeslot == -1 else (DUMMY, freeslot) elif index == DUMMY: @@ -76,7 +76,7 @@ i = 5 * i + perturb + 1 i = i & (n - 1) perturb >>= PERTURB_SHIFT - self.indices[i] = index + self.indices[i] = rffi.cast(rffi.INT_real, index) self.filled = self.used old_values = self.values self.values = lltype.malloc(TP, new_size * 2 / 3 + 1) @@ -103,7 +103,7 @@ hashvalue = key # hash index, i = self._lookup(key, hashvalue) if index < 0: - self.indices[i] = self.used + self.indices[i] = rffi.cast(rffi.INT_real, self.used) self.values[self.used].key = key self.values[self.used].value = _cast_to_gcref(value) self.used += 1 diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py --- a/pypy/rpython/lltypesystem/opimpl.py +++ b/pypy/rpython/lltypesystem/opimpl.py @@ -525,11 +525,11 @@ A = lltype.typeOf(source) assert A == lltype.typeOf(dest) assert isinstance(A.TO, lltype.GcArray) - assert isinstance(A.TO.OF, lltype.Ptr) - assert A.TO.OF.TO._gckind == 'gc' - assert type(source_start) is int - assert type(dest_start) is int - assert type(length) is int + if isinstance(A.TO.OF, lltype.Ptr): + assert A.TO.OF.TO._gckind == 'gc' + assert type(source_start) is int + assert type(dest_start) is int + assert type(length) is int return True def op_getfield(p, name): From noreply at buildbot.pypy.org Sat Jan 5 21:53:12 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 5 Jan 2013 21:53:12 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments: eh, comparison too? Message-ID: <20130105205312.AA2A41C0C35@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments Changeset: r59787:f5d98ccd462f Date: 2013-01-05 22:52 +0200 http://bitbucket.org/pypy/pypy/changeset/f5d98ccd462f/ Log: eh, comparison too? diff --git a/pypy/rlib/dict.py b/pypy/rlib/dict.py --- a/pypy/rlib/dict.py +++ b/pypy/rlib/dict.py @@ -11,8 +11,8 @@ ('value', llmemory.GCREF))) MAIN_TP = lltype.GcArray(rffi.INT_real) -FREE = rffi.cast(rffi.INT_real, -1) -DUMMY = rffi.cast(rffi.INT_real, -2) +FREE = -1 +DUMMY = -2 class Dict(object): 'Space efficient dictionary with fast iteration and cheap resizes.' @@ -48,7 +48,7 @@ #if n <= 2**31: return array.array('l', [FREE]) * n # signed long v = lltype.malloc(MAIN_TP, n) for i in range(n): - v[i] = FREE + v[i] = rffi.cast(rffi.INT_real, FREE) return v def _resize(self, n): @@ -71,7 +71,7 @@ perturb = hashvalue i = hashvalue & (n - 1) while True: - if self.indices[i] == FREE: + if rffi.cast(lltype.Signed, self.indices[i]) == FREE: break i = 5 * i + perturb + 1 i = i & (n - 1) From noreply at buildbot.pypy.org Sat Jan 5 21:56:17 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sat, 5 Jan 2013 21:56:17 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Oops. Added rpython/conftest.py Message-ID: <20130105205617.BC5621C0C35@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59788:50650aa47c37 Date: 2013-01-05 21:55 +0100 http://bitbucket.org/pypy/pypy/changeset/50650aa47c37/ Log: Oops. Added rpython/conftest.py diff --git a/rpython/conftest.py b/rpython/conftest.py new file mode 100644 --- /dev/null +++ b/rpython/conftest.py @@ -0,0 +1,3 @@ +import os + +cdir = os.path.join(os.path.dirname(__file__), 'translator', 'c') From noreply at buildbot.pypy.org Sat Jan 5 22:00:53 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 5 Jan 2013 22:00:53 +0100 (CET) Subject: [pypy-commit] pypy default: fix fix fix Message-ID: <20130105210053.7A9251C0C35@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r59789:90fcb923c94d Date: 2013-01-05 22:42 +0200 http://bitbucket.org/pypy/pypy/changeset/90fcb923c94d/ Log: fix fix fix diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py --- a/pypy/rlib/rgc.py +++ b/pypy/rlib/rgc.py @@ -176,7 +176,10 @@ @specialize.memo() def _contains_gcptr(TP): + TP = TP.OF if not isinstance(TP, lltype.Struct): + if isinstance(TP, lltype.Ptr) and TP.TO._gckind == 'gc': + return True return False for TP in TP._flds.itervalues(): if isinstance(TP, lltype.Ptr) and TP.TO._gckind == 'gc': @@ -205,8 +208,7 @@ TP = lltype.typeOf(source).TO assert TP == lltype.typeOf(dest).TO - if isinstance(TP.OF, lltype.Ptr) and (TP.OF.TO._gckind == 'gc' - or _contains_gcptr(TP.OF.TO)): + if _contains_gcptr(TP): # perform a write barrier that copies necessary flags from # source to dest if not llop.gc_writebarrier_before_copy(lltype.Bool, source, dest, diff --git a/pypy/rlib/test/test_rgc.py b/pypy/rlib/test/test_rgc.py --- a/pypy/rlib/test/test_rgc.py +++ b/pypy/rlib/test/test_rgc.py @@ -159,12 +159,17 @@ assert a2[2].y == 15 def test__contains_gcptr(): - assert not rgc._contains_gcptr(lltype.Signed) - assert not rgc._contains_gcptr(lltype.Struct('x', ('x', lltype.Signed))) - assert rgc._contains_gcptr(lltype.Struct('x', ('x', lltype.Signed), - ('y', lltype.Ptr(lltype.GcArray(lltype.Signed))))) - assert rgc._contains_gcptr(lltype.Struct('x', ('x', lltype.Signed), - ('y', llmemory.GCREF))) + assert not rgc._contains_gcptr(lltype.GcArray(lltype.Signed)) + assert not rgc._contains_gcptr(lltype.GcArray( + lltype.Struct('x', ('x', lltype.Signed)))) + assert rgc._contains_gcptr(lltype.GcArray( + lltype.Struct('x', ('x', lltype.Signed), + ('y', lltype.Ptr(lltype.GcArray(lltype.Signed)))))) + assert rgc._contains_gcptr(lltype.GcArray( + lltype.Struct('x', ('x', lltype.Signed), + ('y', llmemory.GCREF)))) + assert rgc._contains_gcptr(lltype.GcArray(lltype.Ptr(lltype.GcStruct('x')))) + assert not rgc._contains_gcptr(lltype.GcArray(lltype.Ptr(lltype.Struct('x')))) def test_ll_arraycopy_small(): TYPE = lltype.GcArray(lltype.Signed) From noreply at buildbot.pypy.org Sat Jan 5 22:00:54 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 5 Jan 2013 22:00:54 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments: I'm closing this experimental branch. It seems that it's interesting enough Message-ID: <20130105210054.8CF631C0C35@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments Changeset: r59790:85c14a1a3239 Date: 2013-01-05 23:00 +0200 http://bitbucket.org/pypy/pypy/changeset/85c14a1a3239/ Log: I'm closing this experimental branch. It seems that it's interesting enough to try the full rdict conversion. From noreply at buildbot.pypy.org Sat Jan 5 22:54:57 2013 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 5 Jan 2013 22:54:57 +0100 (CET) Subject: [pypy-commit] pypy default: rebuilding website on readthedocs was broken, maybe this will fix it? Message-ID: <20130105215457.D92F61C01A6@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r59791:b36997015a40 Date: 2013-01-05 23:54 +0200 http://bitbucket.org/pypy/pypy/changeset/b36997015a40/ Log: rebuilding website on readthedocs was broken, maybe this will fix it? diff --git a/pypy/config/makerestdoc.py b/pypy/config/makerestdoc.py --- a/pypy/config/makerestdoc.py +++ b/pypy/config/makerestdoc.py @@ -198,7 +198,7 @@ from docutils import nodes from pypy.config.pypyoption import get_pypy_config from pypy.config.makerestdoc import get_cmdline - txt = docdir.join("config", text + ".rst") + txt = docdir.join("config", text + ".txt") html = docdir.join("config", text + ".html") assert txt.check() assert name == "config" From noreply at buildbot.pypy.org Sat Jan 5 22:56:37 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 5 Jan 2013 22:56:37 +0100 (CET) Subject: [pypy-commit] pypy default: Test and fix: must call _contains_gcptr recursively Message-ID: <20130105215637.17C0E1C01A6@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r59792:0603719c6c31 Date: 2013-01-05 22:56 +0100 http://bitbucket.org/pypy/pypy/changeset/0603719c6c31/ Log: Test and fix: must call _contains_gcptr recursively diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py --- a/pypy/rlib/rgc.py +++ b/pypy/rlib/rgc.py @@ -176,13 +176,12 @@ @specialize.memo() def _contains_gcptr(TP): - TP = TP.OF if not isinstance(TP, lltype.Struct): if isinstance(TP, lltype.Ptr) and TP.TO._gckind == 'gc': return True return False for TP in TP._flds.itervalues(): - if isinstance(TP, lltype.Ptr) and TP.TO._gckind == 'gc': + if _contains_gcptr(TP): return True return False @@ -208,7 +207,7 @@ TP = lltype.typeOf(source).TO assert TP == lltype.typeOf(dest).TO - if _contains_gcptr(TP): + if _contains_gcptr(TP.OF): # perform a write barrier that copies necessary flags from # source to dest if not llop.gc_writebarrier_before_copy(lltype.Bool, source, dest, diff --git a/pypy/rlib/test/test_rgc.py b/pypy/rlib/test/test_rgc.py --- a/pypy/rlib/test/test_rgc.py +++ b/pypy/rlib/test/test_rgc.py @@ -159,17 +159,20 @@ assert a2[2].y == 15 def test__contains_gcptr(): - assert not rgc._contains_gcptr(lltype.GcArray(lltype.Signed)) - assert not rgc._contains_gcptr(lltype.GcArray( - lltype.Struct('x', ('x', lltype.Signed)))) - assert rgc._contains_gcptr(lltype.GcArray( + assert not rgc._contains_gcptr(lltype.Signed) + assert not rgc._contains_gcptr( + lltype.Struct('x', ('x', lltype.Signed))) + assert rgc._contains_gcptr( lltype.Struct('x', ('x', lltype.Signed), - ('y', lltype.Ptr(lltype.GcArray(lltype.Signed)))))) - assert rgc._contains_gcptr(lltype.GcArray( + ('y', lltype.Ptr(lltype.GcArray(lltype.Signed))))) + assert rgc._contains_gcptr( lltype.Struct('x', ('x', lltype.Signed), - ('y', llmemory.GCREF)))) - assert rgc._contains_gcptr(lltype.GcArray(lltype.Ptr(lltype.GcStruct('x')))) - assert not rgc._contains_gcptr(lltype.GcArray(lltype.Ptr(lltype.Struct('x')))) + ('y', llmemory.GCREF))) + assert rgc._contains_gcptr(lltype.Ptr(lltype.GcStruct('x'))) + assert not rgc._contains_gcptr(lltype.Ptr(lltype.Struct('x'))) + GCPTR = lltype.Ptr(lltype.GcStruct('x')) + assert rgc._contains_gcptr( + lltype.Struct('FOO', ('s', lltype.Struct('BAR', ('y', GCPTR))))) def test_ll_arraycopy_small(): TYPE = lltype.GcArray(lltype.Signed) From noreply at buildbot.pypy.org Sat Jan 5 23:09:12 2013 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 5 Jan 2013 23:09:12 +0100 (CET) Subject: [pypy-commit] pypy default: Backed out changeset: b36997015a40 Message-ID: <20130105220912.BA5E81C01A6@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r59793:f07f59245fc3 Date: 2013-01-05 23:57 +0200 http://bitbucket.org/pypy/pypy/changeset/f07f59245fc3/ Log: Backed out changeset: b36997015a40 diff --git a/pypy/config/makerestdoc.py b/pypy/config/makerestdoc.py --- a/pypy/config/makerestdoc.py +++ b/pypy/config/makerestdoc.py @@ -198,7 +198,7 @@ from docutils import nodes from pypy.config.pypyoption import get_pypy_config from pypy.config.makerestdoc import get_cmdline - txt = docdir.join("config", text + ".txt") + txt = docdir.join("config", text + ".rst") html = docdir.join("config", text + ".html") assert txt.check() assert name == "config" From noreply at buildbot.pypy.org Sat Jan 5 23:09:13 2013 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 5 Jan 2013 23:09:13 +0100 (CET) Subject: [pypy-commit] pypy default: merge heads Message-ID: <20130105220913.F2E421C01A6@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r59794:f84c2c99dbe3 Date: 2013-01-06 00:08 +0200 http://bitbucket.org/pypy/pypy/changeset/f84c2c99dbe3/ Log: merge heads diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py --- a/pypy/rlib/rgc.py +++ b/pypy/rlib/rgc.py @@ -176,13 +176,12 @@ @specialize.memo() def _contains_gcptr(TP): - TP = TP.OF if not isinstance(TP, lltype.Struct): if isinstance(TP, lltype.Ptr) and TP.TO._gckind == 'gc': return True return False for TP in TP._flds.itervalues(): - if isinstance(TP, lltype.Ptr) and TP.TO._gckind == 'gc': + if _contains_gcptr(TP): return True return False @@ -208,7 +207,7 @@ TP = lltype.typeOf(source).TO assert TP == lltype.typeOf(dest).TO - if _contains_gcptr(TP): + if _contains_gcptr(TP.OF): # perform a write barrier that copies necessary flags from # source to dest if not llop.gc_writebarrier_before_copy(lltype.Bool, source, dest, diff --git a/pypy/rlib/test/test_rgc.py b/pypy/rlib/test/test_rgc.py --- a/pypy/rlib/test/test_rgc.py +++ b/pypy/rlib/test/test_rgc.py @@ -159,17 +159,20 @@ assert a2[2].y == 15 def test__contains_gcptr(): - assert not rgc._contains_gcptr(lltype.GcArray(lltype.Signed)) - assert not rgc._contains_gcptr(lltype.GcArray( - lltype.Struct('x', ('x', lltype.Signed)))) - assert rgc._contains_gcptr(lltype.GcArray( + assert not rgc._contains_gcptr(lltype.Signed) + assert not rgc._contains_gcptr( + lltype.Struct('x', ('x', lltype.Signed))) + assert rgc._contains_gcptr( lltype.Struct('x', ('x', lltype.Signed), - ('y', lltype.Ptr(lltype.GcArray(lltype.Signed)))))) - assert rgc._contains_gcptr(lltype.GcArray( + ('y', lltype.Ptr(lltype.GcArray(lltype.Signed))))) + assert rgc._contains_gcptr( lltype.Struct('x', ('x', lltype.Signed), - ('y', llmemory.GCREF)))) - assert rgc._contains_gcptr(lltype.GcArray(lltype.Ptr(lltype.GcStruct('x')))) - assert not rgc._contains_gcptr(lltype.GcArray(lltype.Ptr(lltype.Struct('x')))) + ('y', llmemory.GCREF))) + assert rgc._contains_gcptr(lltype.Ptr(lltype.GcStruct('x'))) + assert not rgc._contains_gcptr(lltype.Ptr(lltype.Struct('x'))) + GCPTR = lltype.Ptr(lltype.GcStruct('x')) + assert rgc._contains_gcptr( + lltype.Struct('FOO', ('s', lltype.Struct('BAR', ('y', GCPTR))))) def test_ll_arraycopy_small(): TYPE = lltype.GcArray(lltype.Signed) From noreply at buildbot.pypy.org Sat Jan 5 23:16:56 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sat, 5 Jan 2013 23:16:56 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: pypy.module.thread.ll_thread -> rpython.rlib.rthread Message-ID: <20130105221656.332151C01A6@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59795:c192bbff8225 Date: 2013-01-05 22:53 +0100 http://bitbucket.org/pypy/pypy/changeset/c192bbff8225/ Log: pypy.module.thread.ll_thread -> rpython.rlib.rthread diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -627,7 +627,7 @@ return dummy_lock def __allocate_lock(self): - from pypy.module.thread.ll_thread import allocate_lock, error + from rpython.rlib.rthread import allocate_lock, error try: return allocate_lock() except error: diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py --- a/pypy/module/_io/interp_bufferedio.py +++ b/pypy/module/_io/interp_bufferedio.py @@ -12,7 +12,7 @@ W_IOBase, DEFAULT_BUFFER_SIZE, convert_size, check_readable_w, check_writable_w, check_seekable_w) from pypy.module._io.interp_io import W_BlockingIOError -from pypy.module.thread import ll_thread +from rpython.rlib import rthread import errno STATE_ZERO, STATE_OK, STATE_DETACHED = range(3) diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py --- a/pypy/module/_multiprocessing/interp_semaphore.py +++ b/pypy/module/_multiprocessing/interp_semaphore.py @@ -8,7 +8,7 @@ from rpython.rlib.rarithmetic import r_uint from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.rtyper.tool import rffi_platform as platform -from pypy.module.thread import ll_thread +from rpython.rlib import rthread from pypy.module._multiprocessing.interp_connection import w_handle import sys, os, time, errno diff --git a/pypy/module/_ssl/thread_lock.py b/pypy/module/_ssl/thread_lock.py --- a/pypy/module/_ssl/thread_lock.py +++ b/pypy/module/_ssl/thread_lock.py @@ -62,7 +62,7 @@ } """ -from pypy.module.thread import ll_thread +from rpython.rlib import rthread eci = ll_thread.eci.merge(ExternalCompilationInfo( separate_module_sources=[separate_module_source], diff --git a/pypy/module/cpyext/pystate.py b/pypy/module/cpyext/pystate.py --- a/pypy/module/cpyext/pystate.py +++ b/pypy/module/cpyext/pystate.py @@ -2,7 +2,8 @@ cpython_api, generic_cpy_call, CANNOT_FAIL, CConfig, cpython_struct) from pypy.module.cpyext.pyobject import PyObject, Py_DecRef, make_ref, from_ref from rpython.rtyper.lltypesystem import rffi, lltype -from pypy.module.thread import ll_thread, os_thread +from rpython.rlib import rthread +from pypy.module.thread import os_thread PyInterpreterStateStruct = lltype.ForwardReference() PyInterpreterState = lltype.Ptr(PyInterpreterStateStruct) diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py --- a/pypy/module/rctime/interp_time.py +++ b/pypy/module/rctime/interp_time.py @@ -30,7 +30,7 @@ # and the wait function exits. from rpython.rlib import rwin32 from pypy.interpreter.error import wrap_windowserror, wrap_oserror - from pypy.module.thread import ll_thread as thread + from rpython.rlib import rthread as thread eci = ExternalCompilationInfo( includes = ['windows.h'], diff --git a/pypy/module/thread/gil.py b/pypy/module/thread/gil.py --- a/pypy/module/thread/gil.py +++ b/pypy/module/thread/gil.py @@ -7,7 +7,7 @@ # all but one will be blocked. The other threads get a chance to run # from time to time, using the periodic action GILReleaseAction. -from pypy.module.thread import ll_thread as thread +from rpython.rlib import rthread as thread from pypy.module.thread.error import wrap_thread_error from pypy.interpreter.executioncontext import PeriodicAsyncAction from pypy.module.thread.threadlocals import OSThreadLocals diff --git a/pypy/module/thread/os_lock.py b/pypy/module/thread/os_lock.py --- a/pypy/module/thread/os_lock.py +++ b/pypy/module/thread/os_lock.py @@ -2,7 +2,7 @@ Python locks, based on true threading locks provided by the OS. """ -from pypy.module.thread import ll_thread as thread +from rpython.rlib import rthread as thread from pypy.module.thread.error import wrap_thread_error from pypy.interpreter.baseobjspace import Wrappable from pypy.interpreter.gateway import interp2app, unwrap_spec diff --git a/pypy/module/thread/os_thread.py b/pypy/module/thread/os_thread.py --- a/pypy/module/thread/os_thread.py +++ b/pypy/module/thread/os_thread.py @@ -2,7 +2,7 @@ Thread support based on OS-level threads. """ -from pypy.module.thread import ll_thread as thread +from rpython.rlib import rthread as thread from pypy.module.thread.error import wrap_thread_error from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.gateway import unwrap_spec, Arguments diff --git a/pypy/module/thread/test/test_gil.py b/pypy/module/thread/test/test_gil.py --- a/pypy/module/thread/test/test_gil.py +++ b/pypy/module/thread/test/test_gil.py @@ -1,7 +1,7 @@ import time from pypy.module.thread import gil from pypy.module.thread.test import test_ll_thread -from pypy.module.thread import ll_thread as thread +from rpython.rlib import rthread as thread from rpython.rlib.objectmodel import we_are_translated class FakeEC(object): diff --git a/pypy/module/thread/test/test_ll_thread.py b/pypy/module/thread/test/test_ll_thread.py --- a/pypy/module/thread/test/test_ll_thread.py +++ b/pypy/module/thread/test/test_ll_thread.py @@ -1,5 +1,5 @@ import gc -from pypy.module.thread.ll_thread import * +from rpython.rlib.rthread import * from rpython.translator.c.test.test_boehm import AbstractGCTestClass from rpython.rtyper.lltypesystem import lltype, rffi import py diff --git a/pypy/module/thread/test/test_lock.py b/pypy/module/thread/test/test_lock.py --- a/pypy/module/thread/test/test_lock.py +++ b/pypy/module/thread/test/test_lock.py @@ -48,7 +48,7 @@ def test_compile_lock(): from rpython.rlib import rgc - from pypy.module.thread.ll_thread import allocate_lock + from rpython.rlib.rthread import allocate_lock def g(): l = allocate_lock() ok1 = l.acquire(True) diff --git a/pypy/module/thread/threadlocals.py b/pypy/module/thread/threadlocals.py --- a/pypy/module/thread/threadlocals.py +++ b/pypy/module/thread/threadlocals.py @@ -1,4 +1,4 @@ -from pypy.module.thread import ll_thread as thread +from rpython.rlib import rthread as thread class OSThreadLocals: diff --git a/pypy/module/thread/ll_thread.py b/rpython/rlib/rthread.py rename from pypy/module/thread/ll_thread.py rename to rpython/rlib/rthread.py --- a/pypy/module/thread/ll_thread.py +++ b/rpython/rlib/rthread.py @@ -8,12 +8,10 @@ from rpython.rlib.objectmodel import we_are_translated, specialize from rpython.rtyper.lltypesystem.lloperation import llop from rpython.rtyper.tool import rffi_platform -from pypy.conftest import pypydir class error(Exception): pass -pypydir = py.path.local(pypydir) translator_c_dir = py.path.local(cdir) eci = ExternalCompilationInfo( diff --git a/rpython/rtyper/memory/gctransform/asmgcroot.py b/rpython/rtyper/memory/gctransform/asmgcroot.py --- a/rpython/rtyper/memory/gctransform/asmgcroot.py +++ b/rpython/rtyper/memory/gctransform/asmgcroot.py @@ -159,7 +159,7 @@ # gc_thread_before_fork and gc_thread_after_fork to get rid of # all ASM_FRAMEDATA structures that do no belong to the current # thread after a fork(). - from pypy.module.thread import ll_thread + from rpython.rlib import rthread from rpython.rtyper.memory.support import AddressDict from rpython.rtyper.memory.support import copy_without_null_values from rpython.annotator import model as annmodel diff --git a/rpython/rtyper/memory/gctransform/shadowstack.py b/rpython/rtyper/memory/gctransform/shadowstack.py --- a/rpython/rtyper/memory/gctransform/shadowstack.py +++ b/rpython/rtyper/memory/gctransform/shadowstack.py @@ -105,7 +105,7 @@ gcdata.root_stack_base, gcdata.root_stack_top) def need_thread_support(self, gctransformer, getfn): - from pypy.module.thread import ll_thread # xxx fish + from rpython.rlib import rthread # xxx fish gcdata = self.gcdata # the interfacing between the threads and the GC is done via # two completely ad-hoc operations at the moment: diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py --- a/rpython/rtyper/module/ll_os.py +++ b/rpython/rtyper/module/ll_os.py @@ -1635,7 +1635,7 @@ @registering_if(os, 'fork') def register_os_fork(self): - from pypy.module.thread import ll_thread + from rpython.rlib import rthread eci = self.gcc_profiling_bug_workaround('pid_t _noprof_fork(void)', 'return fork();') os_fork = self.llexternal('_noprof_fork', [], rffi.PID_T, diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py --- a/rpython/translator/c/test/test_standalone.py +++ b/rpython/translator/c/test/test_standalone.py @@ -852,7 +852,7 @@ def test_stack_size(self): import time - from pypy.module.thread import ll_thread + from rpython.rlib import rthread from rpython.rtyper.lltypesystem import lltype from rpython.rlib.objectmodel import invoke_around_extcall @@ -951,7 +951,7 @@ def test_thread_and_gc(self): import time, gc - from pypy.module.thread import ll_thread + from rpython.rlib import rthread from rpython.rtyper.lltypesystem import lltype from rpython.rlib.objectmodel import invoke_around_extcall @@ -1056,7 +1056,7 @@ # least that the object referenced from stacks that are no longer # alive are really freed. import time, gc, os - from pypy.module.thread import ll_thread + from rpython.rlib import rthread from rpython.rlib.objectmodel import invoke_around_extcall if not hasattr(os, 'fork'): py.test.skip("requires fork()") From noreply at buildbot.pypy.org Sat Jan 5 23:16:57 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sat, 5 Jan 2013 23:16:57 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: pypy.tool.staticmethods -> rpython.tool.staticmethods Message-ID: <20130105221657.8E0E81C01A6@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59796:3eacb0c147c1 Date: 2013-01-05 22:59 +0100 http://bitbucket.org/pypy/pypy/changeset/3eacb0c147c1/ Log: pypy.tool.staticmethods -> rpython.tool.staticmethods diff --git a/rpython/rlib/_stacklet_n_a.py b/rpython/rlib/_stacklet_n_a.py --- a/rpython/rlib/_stacklet_n_a.py +++ b/rpython/rlib/_stacklet_n_a.py @@ -1,7 +1,7 @@ from rpython.rlib import _rffi_stacklet as _c from rpython.rlib import objectmodel, debug from rpython.rtyper.annlowlevel import llhelper -from pypy.tool.staticmethods import StaticMethods +from rpython.tool.staticmethods import StaticMethods class StackletGcRootFinder: diff --git a/rpython/rlib/_stacklet_shadowstack.py b/rpython/rlib/_stacklet_shadowstack.py --- a/rpython/rlib/_stacklet_shadowstack.py +++ b/rpython/rlib/_stacklet_shadowstack.py @@ -3,7 +3,7 @@ from rpython.rtyper.annlowlevel import llhelper from rpython.rtyper.lltypesystem import lltype, llmemory from rpython.rtyper.lltypesystem.lloperation import llop -from pypy.tool.staticmethods import StaticMethods +from rpython.tool.staticmethods import StaticMethods NULL_SUSPSTACK = lltype.nullptr(llmemory.GCREF.TO) diff --git a/rpython/rtyper/module/ll_os_path.py b/rpython/rtyper/module/ll_os_path.py --- a/rpython/rtyper/module/ll_os_path.py +++ b/rpython/rtyper/module/ll_os_path.py @@ -6,7 +6,7 @@ import stat import os -from pypy.tool.staticmethods import ClassMethods +from rpython.tool.staticmethods import ClassMethods # Does a path exist? # This is false for dangling symbolic links. diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py --- a/rpython/rtyper/rstr.py +++ b/rpython/rtyper/rstr.py @@ -1,4 +1,4 @@ -from pypy.tool.staticmethods import StaticMethods +from rpython.tool.staticmethods import StaticMethods from rpython.tool.pairtype import pairtype, pair from rpython.tool.sourcetools import func_with_new_name from rpython.annotator import model as annmodel diff --git a/pypy/tool/staticmethods.py b/rpython/tool/staticmethods.py rename from pypy/tool/staticmethods.py rename to rpython/tool/staticmethods.py From noreply at buildbot.pypy.org Sat Jan 5 23:16:58 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sat, 5 Jan 2013 23:16:58 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Moved watchdog to rpython Message-ID: <20130105221658.B5D951C01A6@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59797:ec6e57ae881e Date: 2013-01-05 23:11 +0100 http://bitbucket.org/pypy/pypy/changeset/ec6e57ae881e/ Log: Moved watchdog to rpython diff --git a/pypy/tool/watchdog_nt.py b/pypy/tool/watchdog_nt.py deleted file mode 100644 --- a/pypy/tool/watchdog_nt.py +++ /dev/null @@ -1,33 +0,0 @@ -import sys, os -import threading -import ctypes - -def childkill(pid): - global timedout - timedout = True - sys.stderr.write("==== test running for %d seconds ====\n" % timeout) - sys.stderr.write("="*26 + "timedout" + "="*26 + "\n") - ctypes.windll.kernel32.TerminateProcess(pid, 1) - -if __name__ == '__main__': - PROCESS_TERMINATE = 0x1 - - timeout = float(sys.argv[1]) - timedout = False - - pid = os.spawnv(os.P_NOWAIT, sys.argv[2], sys.argv[2:]) - - t = threading.Timer(timeout, childkill, (pid,)) - t.start() - while True: - try: - pid, status = os.waitpid(pid, 0) - except KeyboardInterrupt: - continue - else: - t.cancel() - break - - #print 'status ', status >> 8 - sys.exit(status >> 8) - diff --git a/pypy/tool/watchdog.py b/rpython/tool/watchdog.py rename from pypy/tool/watchdog.py rename to rpython/tool/watchdog.py diff --git a/rpython/translator/cli/gencli.py b/rpython/translator/cli/gencli.py --- a/rpython/translator/cli/gencli.py +++ b/rpython/translator/cli/gencli.py @@ -91,8 +91,7 @@ args = [helper] + args if timeout and not sys.platform.startswith('win'): import os - from pypy.conftest import pypydir - watchdog = os.path.join(pypydir, 'tool', 'watchdog.py') + watchdog = os.path.join(os.path.dirname(__file__), '..', '..', 'tool', 'watchdog.py') args[:0] = [sys.executable, watchdog, str(float(timeout))] proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = proc.communicate() From noreply at buildbot.pypy.org Sat Jan 5 23:36:51 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sat, 5 Jan 2013 23:36:51 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: pypy.tool.cache -> rpython.rlib.cache Message-ID: <20130105223651.E8DE61C01EE@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59798:1ed63b0b4127 Date: 2013-01-05 23:29 +0100 http://bitbucket.org/pypy/pypy/changeset/1ed63b0b4127/ Log: pypy.tool.cache -> rpython.rlib.cache diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -6,7 +6,7 @@ new_exception_class, typed_unwrap_error_msg) from pypy.interpreter.argument import Arguments from pypy.interpreter.miscutils import ThreadLocals -from pypy.tool.cache import Cache +from rpython.rlib.cache import Cache from rpython.tool.uid import HUGEVAL_BYTES from rpython.rlib import jit from rpython.rlib.debug import make_sure_not_resized diff --git a/pypy/tool/test/test_cache.py b/pypy/tool/test/test_cache.py --- a/pypy/tool/test/test_cache.py +++ b/pypy/tool/test/test_cache.py @@ -1,4 +1,4 @@ -from pypy.tool.cache import Cache +from rpython.rlib.cache import Cache class MyCache(Cache): counter = 0 diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py --- a/rpython/annotator/test/test_annrpython.py +++ b/rpython/annotator/test/test_annrpython.py @@ -2214,7 +2214,7 @@ assert not s.is_constant() def test_getorbuild_as_attr(self): - from pypy.tool.cache import Cache + from rpython.rlib.cache import Cache class SpaceCache(Cache): def _build(self, callable): return callable() diff --git a/pypy/tool/cache.py b/rpython/rlib/cache.py rename from pypy/tool/cache.py rename to rpython/rlib/cache.py diff --git a/rpython/rtyper/test/test_rpbc.py b/rpython/rtyper/test/test_rpbc.py --- a/rpython/rtyper/test/test_rpbc.py +++ b/rpython/rtyper/test/test_rpbc.py @@ -359,9 +359,9 @@ # this test checks that we add a separate field # per specialization and also it uses a subclass of - # the standard pypy.tool.cache.Cache + # the standard rpython.rlib.cache.Cache - from pypy.tool.cache import Cache + from rpython.rlib.cache import Cache fr1 = Freezing() fr2 = Freezing() From noreply at buildbot.pypy.org Sat Jan 5 23:36:53 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sat, 5 Jan 2013 23:36:53 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed missing rename of ll_thread Message-ID: <20130105223653.240EA1C01EE@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59799:021e4dfcf6c8 Date: 2013-01-05 23:33 +0100 http://bitbucket.org/pypy/pypy/changeset/021e4dfcf6c8/ Log: Fixed missing rename of ll_thread diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py --- a/pypy/module/_io/interp_bufferedio.py +++ b/pypy/module/_io/interp_bufferedio.py @@ -53,10 +53,10 @@ def __enter__(self): if not self.lock.acquire(False): - if self.owner == ll_thread.get_ident(): + if self.owner == rthread.get_ident(): raise self.operr self.lock.acquire(True) - self.owner = ll_thread.get_ident() + self.owner = rthread.get_ident() def __exit__(self,*args): self.owner = 0 diff --git a/pypy/module/_multiprocessing/interp_semaphore.py b/pypy/module/_multiprocessing/interp_semaphore.py --- a/pypy/module/_multiprocessing/interp_semaphore.py +++ b/pypy/module/_multiprocessing/interp_semaphore.py @@ -434,7 +434,7 @@ return space.wrap(self.count) def _ismine(self): - return self.count > 0 and ll_thread.get_ident() == self.last_tid + return self.count > 0 and rthread.get_ident() == self.last_tid def is_mine(self, space): return space.wrap(self._ismine()) @@ -466,7 +466,7 @@ raise wrap_oserror(space, e) if got: - self.last_tid = ll_thread.get_ident() + self.last_tid = rthread.get_ident() self.count += 1 return space.w_True else: diff --git a/pypy/module/_ssl/thread_lock.py b/pypy/module/_ssl/thread_lock.py --- a/pypy/module/_ssl/thread_lock.py +++ b/pypy/module/_ssl/thread_lock.py @@ -64,7 +64,7 @@ from rpython.rlib import rthread -eci = ll_thread.eci.merge(ExternalCompilationInfo( +eci = rthread.eci.merge(ExternalCompilationInfo( separate_module_sources=[separate_module_source], post_include_bits=[ "int _PyPy_SSL_SetupThreads(void);"], diff --git a/pypy/module/cpyext/pystate.py b/pypy/module/cpyext/pystate.py --- a/pypy/module/cpyext/pystate.py +++ b/pypy/module/cpyext/pystate.py @@ -232,12 +232,12 @@ """Create a new thread state object belonging to the given interpreter object. The global interpreter lock need not be held, but may be held if it is necessary to serialize calls to this function.""" - ll_thread.gc_thread_prepare() + rthread.gc_thread_prepare() # PyThreadState_Get will allocate a new execution context, # we need to protect gc and other globals with the GIL. rffi.aroundstate.after() try: - ll_thread.gc_thread_start() + rthread.gc_thread_start() return PyThreadState_Get(space) finally: rffi.aroundstate.before() @@ -250,7 +250,7 @@ tstate.c_dict = lltype.nullptr(PyObject.TO) space.threadlocals.leave_thread(space) space.getexecutioncontext().cleanup_cpyext_state() - ll_thread.gc_thread_die() + rthread.gc_thread_die() @cpython_api([PyThreadState], lltype.Void) def PyThreadState_Delete(space, tstate): diff --git a/pypy/module/thread/os_thread.py b/pypy/module/thread/os_thread.py --- a/pypy/module/thread/os_thread.py +++ b/pypy/module/thread/os_thread.py @@ -48,7 +48,7 @@ class Bootstrapper(object): "A global container used to pass information to newly starting threads." - # Passing a closure argument to ll_thread.start_new_thread() would be + # Passing a closure argument to rthread.start_new_thread() would be # theoretically nicer, but comes with messy memory management issues. # This is much more straightforward. diff --git a/pypy/module/thread/test/test_gil.py b/pypy/module/thread/test/test_gil.py --- a/pypy/module/thread/test/test_gil.py +++ b/pypy/module/thread/test/test_gil.py @@ -1,6 +1,6 @@ import time from pypy.module.thread import gil -from pypy.module.thread.test import test_ll_thread +from pypy.module.thread.test import test_rthread from rpython.rlib import rthread as thread from rpython.rlib.objectmodel import we_are_translated @@ -26,7 +26,7 @@ raise NotImplementedError -class GILTests(test_ll_thread.AbstractGCTestClass): +class GILTests(test_rthread.AbstractGCTestClass): use_threads = True bigtest = False diff --git a/pypy/module/thread/test/test_ll_thread.py b/pypy/module/thread/test/test_ll_thread.py --- a/pypy/module/thread/test/test_ll_thread.py +++ b/pypy/module/thread/test/test_ll_thread.py @@ -6,7 +6,7 @@ def setup_module(mod): # Hack to avoid a deadlock if the module is run after other test files :-( - # In this module, we assume that ll_thread.start_new_thread() is not + # In this module, we assume that rthread.start_new_thread() is not # providing us with a GIL equivalent, except in test_gc_locking # which installs its own aroundstate. rffi.aroundstate._cleanup_() diff --git a/rpython/rtyper/memory/gctransform/asmgcroot.py b/rpython/rtyper/memory/gctransform/asmgcroot.py --- a/rpython/rtyper/memory/gctransform/asmgcroot.py +++ b/rpython/rtyper/memory/gctransform/asmgcroot.py @@ -167,7 +167,7 @@ def get_aid(): """Return the thread identifier, cast to an (opaque) address.""" - return llmemory.cast_int_to_adr(ll_thread.get_ident()) + return llmemory.cast_int_to_adr(rthread.get_ident()) def thread_start(): value = llop.stack_current(llmemory.Address) diff --git a/rpython/rtyper/memory/gctransform/shadowstack.py b/rpython/rtyper/memory/gctransform/shadowstack.py --- a/rpython/rtyper/memory/gctransform/shadowstack.py +++ b/rpython/rtyper/memory/gctransform/shadowstack.py @@ -119,7 +119,7 @@ gcdata.thread_stacks = None # Return the thread identifier, as an integer. - get_tid = ll_thread.get_ident + get_tid = rthread.get_ident def thread_setup(): tid = get_tid() diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py --- a/rpython/rtyper/module/ll_os.py +++ b/rpython/rtyper/module/ll_os.py @@ -1643,9 +1643,9 @@ _nowrapper = True) def fork_llimpl(): - opaqueaddr = ll_thread.gc_thread_before_fork() + opaqueaddr = rthread.gc_thread_before_fork() childpid = rffi.cast(lltype.Signed, os_fork()) - ll_thread.gc_thread_after_fork(childpid, opaqueaddr) + rthread.gc_thread_after_fork(childpid, opaqueaddr) if childpid == -1: raise OSError(rposix.get_errno(), "os_fork failed") return rffi.cast(lltype.Signed, childpid) diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py --- a/rpython/translator/c/gc.py +++ b/rpython/translator/c/gc.py @@ -230,7 +230,7 @@ eci = eci.merge(ExternalCompilationInfo( pre_include_bits=pre_include_bits, # The following define is required by the thread module, - # See module/thread/test/test_ll_thread.py + # See module/thread/test/test_rthread.py compile_extra=['-DPYPY_USING_BOEHM_GC'], )) diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py --- a/rpython/translator/c/test/test_standalone.py +++ b/rpython/translator/c/test/test_standalone.py @@ -862,13 +862,13 @@ def before(): debug_print("releasing...") - ll_assert(not ll_thread.acquire_NOAUTO(state.ll_lock, False), + ll_assert(not rthread.acquire_NOAUTO(state.ll_lock, False), "lock not held!") - ll_thread.release_NOAUTO(state.ll_lock) + rthread.release_NOAUTO(state.ll_lock) debug_print("released") def after(): debug_print("waiting...") - ll_thread.acquire_NOAUTO(state.ll_lock, True) + rthread.acquire_NOAUTO(state.ll_lock, True) debug_print("acquired") def recurse(n): @@ -897,7 +897,7 @@ def entry_point(argv): os.write(1, "hello world\n") - error = ll_thread.set_stacksize(int(argv[1])) + error = rthread.set_stacksize(int(argv[1])) if error != 0: os.write(2, "set_stacksize(%d) returned %d\n" % ( int(argv[1]), error)) @@ -906,13 +906,13 @@ s1 = State(); s2 = State(); s3 = State() s1.x = 0x11111111; s2.x = 0x22222222; s3.x = 0x33333333 # start 3 new threads - state.ll_lock = ll_thread.allocate_ll_lock() + state.ll_lock = rthread.allocate_ll_lock() after() state.count = 0 invoke_around_extcall(before, after) - ident1 = ll_thread.start_new_thread(bootstrap, ()) - ident2 = ll_thread.start_new_thread(bootstrap, ()) - ident3 = ll_thread.start_new_thread(bootstrap, ()) + ident1 = rthread.start_new_thread(bootstrap, ()) + ident2 = rthread.start_new_thread(bootstrap, ()) + ident3 = rthread.start_new_thread(bootstrap, ()) # wait for the 3 threads to finish while True: if state.count == 3: @@ -960,12 +960,12 @@ state = State() def before(): - ll_assert(not ll_thread.acquire_NOAUTO(state.ll_lock, False), + ll_assert(not rthread.acquire_NOAUTO(state.ll_lock, False), "lock not held!") - ll_thread.release_NOAUTO(state.ll_lock) + rthread.release_NOAUTO(state.ll_lock) def after(): - ll_thread.acquire_NOAUTO(state.ll_lock, True) - ll_thread.gc_thread_run() + rthread.acquire_NOAUTO(state.ll_lock, True) + rthread.gc_thread_run() class Cons: def __init__(self, head, tail): @@ -973,14 +973,14 @@ self.tail = tail def bootstrap(): - ll_thread.gc_thread_start() + rthread.gc_thread_start() state.xlist.append(Cons(123, Cons(456, None))) gc.collect() - ll_thread.gc_thread_die() + rthread.gc_thread_die() def new_thread(): - ll_thread.gc_thread_prepare() - ident = ll_thread.start_new_thread(bootstrap, ()) + rthread.gc_thread_prepare() + ident = rthread.start_new_thread(bootstrap, ()) time.sleep(0.5) # enough time to start, hopefully return ident @@ -989,7 +989,7 @@ state.xlist = [] x2 = Cons(51, Cons(62, Cons(74, None))) # start 5 new threads - state.ll_lock = ll_thread.allocate_ll_lock() + state.ll_lock = rthread.allocate_ll_lock() after() invoke_around_extcall(before, after) ident1 = new_thread() @@ -1066,12 +1066,12 @@ state = State() def before(): - ll_assert(not ll_thread.acquire_NOAUTO(state.ll_lock, False), + ll_assert(not rthread.acquire_NOAUTO(state.ll_lock, False), "lock not held!") - ll_thread.release_NOAUTO(state.ll_lock) + rthread.release_NOAUTO(state.ll_lock) def after(): - ll_thread.acquire_NOAUTO(state.ll_lock, True) - ll_thread.gc_thread_run() + rthread.acquire_NOAUTO(state.ll_lock, True) + rthread.gc_thread_run() class Cons: def __init__(self, head, tail): @@ -1095,7 +1095,7 @@ return childpid def bootstrap(): - ll_thread.gc_thread_start() + rthread.gc_thread_start() childpid = run_in_thread() gc.collect() # collect both in the child and in the parent gc.collect() @@ -1104,15 +1104,15 @@ os.write(state.write_end, 'c') # "I did not die!" from child else: os.write(state.write_end, 'p') # "I did not die!" from parent - ll_thread.gc_thread_die() + rthread.gc_thread_die() def new_thread(): - ll_thread.gc_thread_prepare() - ident = ll_thread.start_new_thread(bootstrap, ()) + rthread.gc_thread_prepare() + ident = rthread.start_new_thread(bootstrap, ()) time.sleep(0.5) # enough time to start, hopefully return ident - def start_all_threads(): + def start_arthreads(): s = allocate_stuff() ident1 = new_thread() ident2 = new_thread() @@ -1130,10 +1130,10 @@ state.read_end, state.write_end = os.pipe() x2 = Cons(51, Cons(62, Cons(74, None))) # start 5 new threads - state.ll_lock = ll_thread.allocate_ll_lock() + state.ll_lock = rthread.allocate_ll_lock() after() invoke_around_extcall(before, after) - start_all_threads() + start_arthreads() # force freeing gc.collect() gc.collect() From noreply at buildbot.pypy.org Sun Jan 6 05:54:49 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sun, 6 Jan 2013 05:54:49 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Moved descriptor to rpython Message-ID: <20130106045449.746D21C01A6@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59800:03c299d3e6d3 Date: 2013-01-06 00:36 +0100 http://bitbucket.org/pypy/pypy/changeset/03c299d3e6d3/ Log: Moved descriptor to rpython diff --git a/pypy/module/thread/test/test_ll_thread.py b/pypy/module/thread/test/test_rthread.py rename from pypy/module/thread/test/test_ll_thread.py rename to pypy/module/thread/test/test_rthread.py diff --git a/pypy/tool/test/test_descriptor.py b/pypy/tool/test/test_descriptor.py --- a/pypy/tool/test/test_descriptor.py +++ b/pypy/tool/test/test_descriptor.py @@ -1,4 +1,4 @@ -from pypy.tool.descriptor import InstanceMethod +rpython.tool.descriptor import InstanceMethod class X(object): def f(self, *args, **kwds): diff --git a/rpython/annotator/model.py b/rpython/annotator/model.py --- a/rpython/annotator/model.py +++ b/rpython/annotator/model.py @@ -31,7 +31,7 @@ from types import BuiltinFunctionType, MethodType, FunctionType import rpython -from pypy.tool import descriptor +from rpython.tool import descriptor from rpython.tool.pairtype import pair, extendabletype from rpython.rlib.rarithmetic import r_uint, r_ulonglong, base_int from rpython.rlib.rarithmetic import r_singlefloat, r_longfloat diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py --- a/rpython/flowspace/test/test_objspace.py +++ b/rpython/flowspace/test/test_objspace.py @@ -706,7 +706,7 @@ def test_catch_importerror_1(self): def f(): try: - import pypy.this_does_not_exist + import rpython.this_does_not_exist except ImportError: return 1 graph = self.codetest(f) @@ -719,7 +719,7 @@ def test_catch_importerror_2(self): def f(): try: - from pypy import this_does_not_exist + from rpython import this_does_not_exist except ImportError: return 1 graph = self.codetest(f) @@ -731,12 +731,12 @@ def test_importerror_1(self): def f(): - import pypy.this_does_not_exist + import rpython.this_does_not_exist py.test.raises(ImportError, 'self.codetest(f)') def test_importerror_2(self): def f(): - from pypy import this_does_not_exist + from rpython import this_does_not_exist py.test.raises(ImportError, 'self.codetest(f)') def test_relative_import(self): diff --git a/pypy/tool/descriptor.py b/rpython/tool/descriptor.py rename from pypy/tool/descriptor.py rename to rpython/tool/descriptor.py From noreply at buildbot.pypy.org Sun Jan 6 05:54:50 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sun, 6 Jan 2013 05:54:50 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Moved frozenlist, killsubprocess, nullpath and runsubprocess to rpython Message-ID: <20130106045450.A999E1C01A6@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59801:7cae0958a8bb Date: 2013-01-06 05:54 +0100 http://bitbucket.org/pypy/pypy/changeset/7cae0958a8bb/ Log: Moved frozenlist, killsubprocess, nullpath and runsubprocess to rpython diff --git a/pypy/tool/test/test_frozenlist.py b/pypy/tool/test/test_frozenlist.py --- a/pypy/tool/test/test_frozenlist.py +++ b/pypy/tool/test/test_frozenlist.py @@ -1,5 +1,5 @@ import py -from pypy.tool.frozenlist import frozenlist +from rpython.tool.frozenlist import frozenlist def test_frozenlist(): l = frozenlist([1, 2, 3]) diff --git a/pypy/tool/test/test_killsubprocess.py b/pypy/tool/test/test_killsubprocess.py --- a/pypy/tool/test/test_killsubprocess.py +++ b/pypy/tool/test/test_killsubprocess.py @@ -1,6 +1,6 @@ import sys, time import subprocess -from pypy.tool.killsubprocess import killsubprocess +from rpython.tool.killsubprocess import killsubprocess def waitdead(process): for i in range(50): diff --git a/pypy/tool/test/test_nullpath.py b/pypy/tool/test/test_nullpath.py --- a/pypy/tool/test/test_nullpath.py +++ b/pypy/tool/test/test_nullpath.py @@ -1,6 +1,6 @@ import sys, os import py -from pypy.tool.nullpath import NullPyPathLocal +from rpython.tool.nullpath import NullPyPathLocal def test_nullpath(tmpdir): path = NullPyPathLocal(tmpdir) diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -666,7 +666,7 @@ When we_are_translated(), do nothing and just return the old list. """ - from pypy.tool.frozenlist import frozenlist + from rpython.tool.frozenlist import frozenlist if we_are_translated(): return operations # diff --git a/pypy/tool/frozenlist.py b/rpython/tool/frozenlist.py rename from pypy/tool/frozenlist.py rename to rpython/tool/frozenlist.py diff --git a/pypy/tool/killsubprocess.py b/rpython/tool/killsubprocess.py rename from pypy/tool/killsubprocess.py rename to rpython/tool/killsubprocess.py diff --git a/pypy/tool/nullpath.py b/rpython/tool/nullpath.py rename from pypy/tool/nullpath.py rename to rpython/tool/nullpath.py diff --git a/pypy/tool/runsubprocess.py b/rpython/tool/runsubprocess.py rename from pypy/tool/runsubprocess.py rename to rpython/tool/runsubprocess.py diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py --- a/rpython/translator/c/genc.py +++ b/rpython/translator/c/genc.py @@ -3,8 +3,8 @@ import sys, os from rpython.rlib import exports from rpython.rtyper.typesystem import getfunctionptr -from pypy.tool import runsubprocess -from pypy.tool.nullpath import NullPyPathLocal +from rpython.tool import runsubprocess +from rpython.tool.nullpath import NullPyPathLocal from rpython.tool.udir import udir from rpython.translator.c import gc from rpython.translator.c.database import LowLevelDatabase diff --git a/rpython/translator/c/test/test_genc.py b/rpython/translator/c/test/test_genc.py --- a/rpython/translator/c/test/test_genc.py +++ b/rpython/translator/c/test/test_genc.py @@ -10,7 +10,7 @@ from rpython.rtyper.lltypesystem import lltype from rpython.rtyper.lltypesystem.lltype import * from rpython.rtyper.lltypesystem.rstr import STR -from pypy.tool.nullpath import NullPyPathLocal +from rpython.tool.nullpath import NullPyPathLocal from rpython.translator.c import genc from rpython.translator.interactive import Translation from rpython.translator.translator import TranslationContext, graphof diff --git a/rpython/translator/platform/__init__.py b/rpython/translator/platform/__init__.py --- a/rpython/translator/platform/__init__.py +++ b/rpython/translator/platform/__init__.py @@ -3,7 +3,7 @@ import py, os, sys from rpython.tool.ansi_print import ansi_log -from pypy.tool.runsubprocess import run_subprocess as _run_subprocess +from rpython.tool.runsubprocess import run_subprocess as _run_subprocess from rpython.tool.udir import udir log = py.log.Producer("platform") diff --git a/rpython/translator/sandbox/sandlib.py b/rpython/translator/sandbox/sandlib.py --- a/rpython/translator/sandbox/sandlib.py +++ b/rpython/translator/sandbox/sandlib.py @@ -6,7 +6,7 @@ import sys, os, posixpath, errno, stat, time import subprocess -from pypy.tool.killsubprocess import killsubprocess +from rpython.tool.killsubprocess import killsubprocess from rpython.translator.sandbox.vfs import UID, GID import py From noreply at buildbot.pypy.org Sun Jan 6 06:49:25 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sun, 6 Jan 2013 06:49:25 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Added /goals for pypy. Moved pypy.tool.progressbar to rpython Message-ID: <20130106054925.273C51C0312@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59802:dd8815a42e4f Date: 2013-01-06 06:38 +0100 http://bitbucket.org/pypy/pypy/changeset/dd8815a42e4f/ Log: Added /goals for pypy. Moved pypy.tool.progressbar to rpython diff --git a/rpython/translator/goal/ann_override.py b/goal/ann_override.py rename from rpython/translator/goal/ann_override.py rename to goal/ann_override.py diff --git a/rpython/translator/goal/multibuild.py b/goal/multibuild.py rename from rpython/translator/goal/multibuild.py rename to goal/multibuild.py diff --git a/targetpypystandalone.py b/goal/targetpypystandalone.py rename from targetpypystandalone.py rename to goal/targetpypystandalone.py diff --git a/rpython/tool/progressbar.py b/rpython/tool/progressbar.py --- a/rpython/tool/progressbar.py +++ b/rpython/tool/progressbar.py @@ -7,7 +7,7 @@ p.render(percentage, message) """ -from pypy.tool import terminal +from rpython.tool import terminal import sys class ProgressBar(object): diff --git a/pypy/tool/terminal.py b/rpython/tool/terminal.py rename from pypy/tool/terminal.py rename to rpython/tool/terminal.py diff --git a/rpython/translator/goal/test2/test_targetpypy.py b/rpython/translator/goal/test2/test_targetpypy.py --- a/rpython/translator/goal/test2/test_targetpypy.py +++ b/rpython/translator/goal/test2/test_targetpypy.py @@ -1,6 +1,6 @@ import py -from rpython.translator.goal.targetpypystandalone import get_entry_point +from goal.targetpypystandalone import get_entry_point from pypy.config.pypyoption import get_pypy_config class TestTargetPyPy(object): From noreply at buildbot.pypy.org Sun Jan 6 06:49:26 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sun, 6 Jan 2013 06:49:26 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Moved rpython tests to rpython Message-ID: <20130106054926.706CF1C0312@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59803:a3ebad4b3e13 Date: 2013-01-06 06:48 +0100 http://bitbucket.org/pypy/pypy/changeset/a3ebad4b3e13/ Log: Moved rpython tests to rpython diff --git a/pypy/tool/test/test_cache.py b/rpython/rlib/test/test_cache.py rename from pypy/tool/test/test_cache.py rename to rpython/rlib/test/test_cache.py diff --git a/pypy/module/thread/test/test_rthread.py b/rpython/rlib/test/test_rthread.py rename from pypy/module/thread/test/test_rthread.py rename to rpython/rlib/test/test_rthread.py diff --git a/pypy/tool/test/test_descriptor.py b/rpython/tool/test/test_descriptor.py rename from pypy/tool/test/test_descriptor.py rename to rpython/tool/test/test_descriptor.py diff --git a/pypy/tool/test/test_error.py b/rpython/tool/test/test_error.py rename from pypy/tool/test/test_error.py rename to rpython/tool/test/test_error.py diff --git a/pypy/tool/test/test_frozenlist.py b/rpython/tool/test/test_frozenlist.py rename from pypy/tool/test/test_frozenlist.py rename to rpython/tool/test/test_frozenlist.py diff --git a/pypy/tool/test/test_identitydict.py b/rpython/tool/test/test_identitydict.py rename from pypy/tool/test/test_identitydict.py rename to rpython/tool/test/test_identitydict.py diff --git a/pypy/tool/test/test_killsubprocess.py b/rpython/tool/test/test_killsubprocess.py rename from pypy/tool/test/test_killsubprocess.py rename to rpython/tool/test/test_killsubprocess.py diff --git a/pypy/tool/test/test_leakfinder.py b/rpython/tool/test/test_leakfinder.py rename from pypy/tool/test/test_leakfinder.py rename to rpython/tool/test/test_leakfinder.py diff --git a/pypy/tool/test/test_logparser.py b/rpython/tool/test/test_logparser.py rename from pypy/tool/test/test_logparser.py rename to rpython/tool/test/test_logparser.py diff --git a/pypy/tool/test/test_nullpath.py b/rpython/tool/test/test_nullpath.py rename from pypy/tool/test/test_nullpath.py rename to rpython/tool/test/test_nullpath.py diff --git a/pypy/tool/test/test_pairtype.py b/rpython/tool/test/test_pairtype.py rename from pypy/tool/test/test_pairtype.py rename to rpython/tool/test/test_pairtype.py diff --git a/pypy/tool/test/test_runsubprocess.py b/rpython/tool/test/test_runsubprocess.py rename from pypy/tool/test/test_runsubprocess.py rename to rpython/tool/test/test_runsubprocess.py diff --git a/pypy/tool/test/test_sourcetools.py b/rpython/tool/test/test_sourcetools.py rename from pypy/tool/test/test_sourcetools.py rename to rpython/tool/test/test_sourcetools.py diff --git a/pypy/tool/test/test_udir.py b/rpython/tool/test/test_udir.py rename from pypy/tool/test/test_udir.py rename to rpython/tool/test/test_udir.py From noreply at buildbot.pypy.org Sun Jan 6 07:15:49 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sun, 6 Jan 2013 07:15:49 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed test_gil Message-ID: <20130106061549.11FBC1C0AD6@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59804:8f81efec234f Date: 2013-01-06 07:04 +0100 http://bitbucket.org/pypy/pypy/changeset/8f81efec234f/ Log: Fixed test_gil diff --git a/pypy/module/thread/test/test_gil.py b/pypy/module/thread/test/test_gil.py --- a/pypy/module/thread/test/test_gil.py +++ b/pypy/module/thread/test/test_gil.py @@ -1,6 +1,6 @@ import time from pypy.module.thread import gil -from pypy.module.thread.test import test_rthread +from rpython.rlib.test import test_rthread from rpython.rlib import rthread as thread from rpython.rlib.objectmodel import we_are_translated From noreply at buildbot.pypy.org Sun Jan 6 07:15:50 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sun, 6 Jan 2013 07:15:50 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed test_cmath Message-ID: <20130106061550.4F8E61C0AD6@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59805:69a5cc72693b Date: 2013-01-06 07:12 +0100 http://bitbucket.org/pypy/pypy/changeset/69a5cc72693b/ Log: Fixed test_cmath diff --git a/pypy/module/cmath/test/test_cmath.py b/pypy/module/cmath/test/test_cmath.py --- a/pypy/module/cmath/test/test_cmath.py +++ b/pypy/module/cmath/test/test_cmath.py @@ -184,8 +184,9 @@ #if not float.__getformat__("double").startswith("IEEE"): # return + import rpython # too fragile... - fname = os.path.join(os.path.dirname(__file__), '../../../rlib/test', 'rcomplex_testcases.txt') + fname = os.path.join(os.path.dirname(rpython.rlib.test.test_rcomplex.__file__), 'rcomplex_testcases.txt') for id, fn, ar, ai, er, ei, flags in parse_testfile(fname): arg = (ar, ai) expected = (er, ei) From noreply at buildbot.pypy.org Sun Jan 6 07:15:51 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sun, 6 Jan 2013 07:15:51 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed test_runsubprocess Message-ID: <20130106061551.7404E1C0AD6@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59806:6b7fd402726d Date: 2013-01-06 07:14 +0100 http://bitbucket.org/pypy/pypy/changeset/6b7fd402726d/ Log: Fixed test_runsubprocess diff --git a/rpython/tool/test/test_runsubprocess.py b/rpython/tool/test/test_runsubprocess.py --- a/rpython/tool/test/test_runsubprocess.py +++ b/rpython/tool/test/test_runsubprocess.py @@ -1,5 +1,5 @@ import py, os -from pypy.tool.runsubprocess import run_subprocess +from rpython.tool.runsubprocess import run_subprocess def test_no_such_command(): py.test.raises(EnvironmentError, run_subprocess, @@ -34,7 +34,7 @@ def test_recover_lost_process(): if not hasattr(os, 'fork'): py.test.skip("there is no os.fork()") - from pypy.tool import runsubprocess + from rpython.tool import runsubprocess import signal os.kill(runsubprocess._child.pid, signal.SIGTERM) runsubprocess._child.wait() From noreply at buildbot.pypy.org Sun Jan 6 10:27:26 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 6 Jan 2013 10:27:26 +0100 (CET) Subject: [pypy-commit] cffi default: Document intptr_t Message-ID: <20130106092726.285A11C0312@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1114:372ff4956b15 Date: 2013-01-06 10:27 +0100 http://bitbucket.org/cffi/cffi/changeset/372ff4956b15/ Log: Document intptr_t diff --git a/doc/source/index.rst b/doc/source/index.rst --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -721,6 +721,12 @@ >>> int(x) 42 +To cast a pointer to an int, cast it to ``intptr_t``, which is defined +by C to be a large enough integer type:: + + >>> int(ffi.cast("intptr_t", pointer_cdata)) + 135812708 + The initializer given as the optional second argument to ``ffi.new()`` can be mostly anything that you would use as an initializer for C code, with lists or tuples instead of using the C syntax ``{ .., .., .. }``. From noreply at buildbot.pypy.org Sun Jan 6 10:34:15 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 6 Jan 2013 10:34:15 +0100 (CET) Subject: [pypy-commit] cffi default: Mention uintptr_t too Message-ID: <20130106093415.415631C0312@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1115:c5c24774010b Date: 2013-01-06 10:34 +0100 http://bitbucket.org/cffi/cffi/changeset/c5c24774010b/ Log: Mention uintptr_t too diff --git a/doc/source/index.rst b/doc/source/index.rst --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -721,11 +721,14 @@ >>> int(x) 42 -To cast a pointer to an int, cast it to ``intptr_t``, which is defined -by C to be a large enough integer type:: +To cast a pointer to an int, cast it to ``intptr_t`` or ``uintptr_t``, +which are defined by C to be large enough integer types (example on 32 +bits):: - >>> int(ffi.cast("intptr_t", pointer_cdata)) - 135812708 + >>> int(ffi.cast("intptr_t", pointer_cdata)) # signed + -1340782304 + >>> int(ffi.cast("uintptr_t", pointer_cdata)) # unsigned + 2954184992L The initializer given as the optional second argument to ``ffi.new()`` can be mostly anything that you would use as an initializer for C code, From noreply at buildbot.pypy.org Sun Jan 6 10:42:36 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 6 Jan 2013 10:42:36 +0100 (CET) Subject: [pypy-commit] pypy default: fix test_whatsnew Message-ID: <20130106094236.7D58C1C0312@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r59807:00d40f125c12 Date: 2013-01-06 11:42 +0200 http://bitbucket.org/pypy/pypy/changeset/00d40f125c12/ Log: fix test_whatsnew diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -5,6 +5,11 @@ .. this is a revision shortly after release-2.0-beta1 .. startrev: 0e6161a009c6 +.. branch: callback-jit +Callbacks from C are now better JITted + +.. branch: remove-globals-in-jit + .. branch: length-hint Implement __lenght_hint__ according to PEP 424 From noreply at buildbot.pypy.org Sun Jan 6 10:44:39 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 6 Jan 2013 10:44:39 +0100 (CET) Subject: [pypy-commit] pypy default: aroundstate can be None Message-ID: <20130106094439.79FE41C0312@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r59808:2852f69d5cbd Date: 2013-01-06 11:44 +0200 http://bitbucket.org/pypy/pypy/changeset/2852f69d5cbd/ Log: aroundstate can be None diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py --- a/pypy/rpython/lltypesystem/rffi.py +++ b/pypy/rpython/lltypesystem/rffi.py @@ -283,9 +283,10 @@ args = ', '.join(['a%d' % i for i in range(len(TP.TO.ARGS))]) source = py.code.Source(r""" def inner_wrapper(%(args)s): - callback_hook = aroundstate.callback_hook - if callback_hook: - callback_hook(llstr("%(callable_name_descr)s")) + if aroundstate is not None: + callback_hook = aroundstate.callback_hook + if callback_hook: + callback_hook(llstr("%(callable_name_descr)s")) return callable(%(args)s) inner_wrapper._never_inline_ = True From noreply at buildbot.pypy.org Sun Jan 6 10:50:21 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 6 Jan 2013 10:50:21 +0100 (CET) Subject: [pypy-commit] pypy default: Fix test_assembler_call_float on 32-bit Message-ID: <20130106095021.897781C0312@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r59809:a299cd0a893f Date: 2013-01-06 10:50 +0100 http://bitbucket.org/pypy/pypy/changeset/a299cd0a893f/ Log: Fix test_assembler_call_float on 32-bit diff --git a/pypy/jit/backend/llgraph/runner.py b/pypy/jit/backend/llgraph/runner.py --- a/pypy/jit/backend/llgraph/runner.py +++ b/pypy/jit/backend/llgraph/runner.py @@ -877,13 +877,16 @@ # assembler_helper_ptr = jd.assembler_helper_adr.ptr # fish try: - return assembler_helper_ptr(pframe, vable) + result = assembler_helper_ptr(pframe, vable) except LLException, lle: assert self.last_exception is None, "exception left behind" self.last_exception = lle # fish op op = self.current_op return op.result and op.result.value + if isinstance(result, float): + result = support.cast_to_floatstorage(result) + return result def execute_same_as(self, _, x): return x From noreply at buildbot.pypy.org Sun Jan 6 16:54:13 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 6 Jan 2013 16:54:13 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: Branch to try rhettingers idea on dicts. So far fight with rtyper and try to Message-ID: <20130106155413.B84A11C0312@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59810:29353cb470d2 Date: 2013-01-06 17:51 +0200 http://bitbucket.org/pypy/pypy/changeset/29353cb470d2/ Log: Branch to try rhettingers idea on dicts. So far fight with rtyper and try to factor out just enough to write tests. diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py --- a/pypy/rpython/lltypesystem/rdict.py +++ b/pypy/rpython/lltypesystem/rdict.py @@ -1,9 +1,9 @@ from pypy.tool.pairtype import pairtype from pypy.objspace.flow.model import Constant from pypy.rpython.rdict import (AbstractDictRepr, AbstractDictIteratorRepr, - rtype_newdict) + rtype_newdict) from pypy.rpython.lltypesystem import lltype -from pypy.rlib import objectmodel, jit +from pypy.rlib import objectmodel, jit, rgc from pypy.rlib.debug import ll_assert from pypy.rlib.rarithmetic import r_uint, intmask, LONG_BIT from pypy.rpython import rmodel @@ -13,18 +13,17 @@ HIGHEST_BIT = intmask(1 << (LONG_BIT - 1)) MASK = intmask(HIGHEST_BIT - 1) +FREE = -2 +DELETED = -1 + # ____________________________________________________________ # # generic implementation of RPython dictionary, with parametric DICTKEY and # DICTVALUE types. # -# XXX for immutable dicts, the array should be inlined and -# resize_counter and everused are not needed. # # struct dictentry { # DICTKEY key; -# bool f_valid; # (optional) the entry is filled -# bool f_everused; # (optional) the entry is or has ever been filled # DICTVALUE value; # int f_hash; # (optional) key hash, if hard to recompute # } @@ -32,6 +31,7 @@ # struct dicttable { # int num_items; # int resize_counter; +# int *indexes; # note that this can be different int # Array *entries; # (Function DICTKEY, DICTKEY -> bool) *fnkeyeq; # (Function DICTKEY -> int) *fnkeyhash; @@ -39,14 +39,87 @@ # # +def get_ll_dict(DICTKEY, DICTVALUE, get_custom_eq_hash=None, DICT=None, + ll_fasthash_function=None, ll_hash_function=None, + ll_eq_function=None): + # get the actual DICT type. if DICT is None, it's created, otherwise + # forward reference is becoming DICT + if DICT is None: + DICT = lltype.GcForwardReference() + # compute the shape of the DICTENTRY structure + entryfields = [] + entrymeths = { + 'allocate': lltype.typeMethod(_ll_malloc_entries), + 'must_clear_key': (isinstance(DICTKEY, lltype.Ptr) + and DICTKEY._needsgc()), + 'must_clear_value': (isinstance(DICTVALUE, lltype.Ptr) + and DICTVALUE._needsgc()), + } + + # * the key + entryfields.append(("key", DICTKEY)) + + # * the value + entryfields.append(("value", DICTVALUE)) + + # * the hash, if needed + if get_custom_eq_hash is not None: + fasthashfn = None + else: + fasthashfn = ll_fasthash_function + if fasthashfn is None: + entryfields.append(("f_hash", lltype.Signed)) + entrymeths['hash'] = ll_hash_from_cache + else: + entrymeths['hash'] = ll_hash_recomputed + entrymeths['fasthashfn'] = fasthashfn + + # Build the lltype data structures + DICTENTRY = lltype.Struct("dictentry", *entryfields) + DICTENTRYARRAY = lltype.GcArray(DICTENTRY, + adtmeths=entrymeths) + fields = [("num_items", lltype.Signed), + ("resize_counter", lltype.Signed), + ("entries", lltype.Ptr(DICTENTRYARRAY)), + ("indexes", lltype.Ptr(lltype.GcArray(lltype.Signed)))] + if get_custom_eq_hash is not None: + r_rdict_eqfn, r_rdict_hashfn = get_custom_eq_hash() + fields.extend([ ("fnkeyeq", r_rdict_eqfn.lowleveltype), + ("fnkeyhash", r_rdict_hashfn.lowleveltype) ]) + adtmeths = { + 'keyhash': ll_keyhash_custom, + 'keyeq': ll_keyeq_custom, + 'r_rdict_eqfn': r_rdict_eqfn, + 'r_rdict_hashfn': r_rdict_hashfn, + 'paranoia': True, + } + else: + # figure out which functions must be used to hash and compare + ll_keyhash = ll_hash_function + ll_keyeq = ll_eq_function # can be None + ll_keyhash = lltype.staticAdtMethod(ll_keyhash) + if ll_keyeq is not None: + ll_keyeq = lltype.staticAdtMethod(ll_keyeq) + adtmeths = { + 'keyhash': ll_keyhash, + 'keyeq': ll_keyeq, + 'paranoia': False, + } + adtmeths['KEY'] = DICTKEY + adtmeths['VALUE'] = DICTVALUE + adtmeths['allocate'] = lltype.typeMethod(_ll_malloc_dict) + DICT.become(lltype.GcStruct("dicttable", adtmeths=adtmeths, + *fields)) + return DICT + class DictRepr(AbstractDictRepr): def __init__(self, rtyper, key_repr, value_repr, dictkey, dictvalue, - custom_eq_hash=None, force_non_null=False): + custom_eq_hash=None): self.rtyper = rtyper self.DICT = lltype.GcForwardReference() self.lowleveltype = lltype.Ptr(self.DICT) - self.custom_eq_hash = custom_eq_hash is not None + self.custom_eq_hash = custom_eq_hash if not isinstance(key_repr, rmodel.Repr): # not computed yet, done by setup() assert callable(key_repr) self._key_repr_computer = key_repr @@ -60,8 +133,6 @@ self.dictkey = dictkey self.dictvalue = dictvalue self.dict_cache = {} - self._custom_eq_hash_repr = custom_eq_hash - self.force_non_null = force_non_null # setup() needs to be called to finish this initialization def _externalvsinternal(self, rtyper, item_repr): @@ -74,138 +145,9 @@ if 'value_repr' not in self.__dict__: self.external_value_repr, self.value_repr = self.pickrepr(self._value_repr_computer()) if isinstance(self.DICT, lltype.GcForwardReference): - self.DICTKEY = self.key_repr.lowleveltype - self.DICTVALUE = self.value_repr.lowleveltype - - # compute the shape of the DICTENTRY structure - entryfields = [] - entrymeths = { - 'allocate': lltype.typeMethod(_ll_malloc_entries), - 'delete': _ll_free_entries, - 'must_clear_key': (isinstance(self.DICTKEY, lltype.Ptr) - and self.DICTKEY._needsgc()), - 'must_clear_value': (isinstance(self.DICTVALUE, lltype.Ptr) - and self.DICTVALUE._needsgc()), - } - - # * the key - entryfields.append(("key", self.DICTKEY)) - - # * if NULL is not a valid ll value for the key or the value - # field of the entry, it can be used as a marker for - # never-used entries. Otherwise, we need an explicit flag. - s_key = self.dictkey.s_value - s_value = self.dictvalue.s_value - nullkeymarker = not self.key_repr.can_ll_be_null(s_key) - nullvaluemarker = not self.value_repr.can_ll_be_null(s_value) - if self.force_non_null: - if not nullkeymarker: - rmodel.warning("%s can be null, but forcing non-null in dict key" % s_key) - nullkeymarker = True - if not nullvaluemarker: - rmodel.warning("%s can be null, but forcing non-null in dict value" % s_value) - nullvaluemarker = True - dummykeyobj = self.key_repr.get_ll_dummyval_obj(self.rtyper, - s_key) - dummyvalueobj = self.value_repr.get_ll_dummyval_obj(self.rtyper, - s_value) - - # * the state of the entry - trying to encode it as dummy objects - if nullkeymarker and dummykeyobj: - # all the state can be encoded in the key - entrymeths['everused'] = ll_everused_from_key - entrymeths['dummy_obj'] = dummykeyobj - entrymeths['valid'] = ll_valid_from_key - entrymeths['mark_deleted'] = ll_mark_deleted_in_key - # the key is overwritten by 'dummy' when the entry is deleted - entrymeths['must_clear_key'] = False - - elif nullvaluemarker and dummyvalueobj: - # all the state can be encoded in the value - entrymeths['everused'] = ll_everused_from_value - entrymeths['dummy_obj'] = dummyvalueobj - entrymeths['valid'] = ll_valid_from_value - entrymeths['mark_deleted'] = ll_mark_deleted_in_value - # value is overwritten by 'dummy' when entry is deleted - entrymeths['must_clear_value'] = False - - else: - # we need a flag to know if the entry was ever used - # (we cannot use a NULL as a marker for this, because - # the key and value will be reset to NULL to clear their - # reference) - entryfields.append(("f_everused", lltype.Bool)) - entrymeths['everused'] = ll_everused_from_flag - - # can we still rely on a dummy obj to mark deleted entries? - if dummykeyobj: - entrymeths['dummy_obj'] = dummykeyobj - entrymeths['valid'] = ll_valid_from_key - entrymeths['mark_deleted'] = ll_mark_deleted_in_key - # key is overwritten by 'dummy' when entry is deleted - entrymeths['must_clear_key'] = False - elif dummyvalueobj: - entrymeths['dummy_obj'] = dummyvalueobj - entrymeths['valid'] = ll_valid_from_value - entrymeths['mark_deleted'] = ll_mark_deleted_in_value - # value is overwritten by 'dummy' when entry is deleted - entrymeths['must_clear_value'] = False - else: - entryfields.append(("f_valid", lltype.Bool)) - entrymeths['valid'] = ll_valid_from_flag - entrymeths['mark_deleted'] = ll_mark_deleted_in_flag - - # * the value - entryfields.append(("value", self.DICTVALUE)) - - # * the hash, if needed - if self.custom_eq_hash: - fasthashfn = None - else: - fasthashfn = self.key_repr.get_ll_fasthash_function() - if fasthashfn is None: - entryfields.append(("f_hash", lltype.Signed)) - entrymeths['hash'] = ll_hash_from_cache - else: - entrymeths['hash'] = ll_hash_recomputed - entrymeths['fasthashfn'] = fasthashfn - - # Build the lltype data structures - self.DICTENTRY = lltype.Struct("dictentry", *entryfields) - self.DICTENTRYARRAY = lltype.GcArray(self.DICTENTRY, - adtmeths=entrymeths) - fields = [ ("num_items", lltype.Signed), - ("resize_counter", lltype.Signed), - ("entries", lltype.Ptr(self.DICTENTRYARRAY)) ] - if self.custom_eq_hash: - self.r_rdict_eqfn, self.r_rdict_hashfn = self._custom_eq_hash_repr() - fields.extend([ ("fnkeyeq", self.r_rdict_eqfn.lowleveltype), - ("fnkeyhash", self.r_rdict_hashfn.lowleveltype) ]) - adtmeths = { - 'keyhash': ll_keyhash_custom, - 'keyeq': ll_keyeq_custom, - 'r_rdict_eqfn': self.r_rdict_eqfn, - 'r_rdict_hashfn': self.r_rdict_hashfn, - 'paranoia': True, - } - else: - # figure out which functions must be used to hash and compare - ll_keyhash = self.key_repr.get_ll_hash_function() - ll_keyeq = self.key_repr.get_ll_eq_function() # can be None - ll_keyhash = lltype.staticAdtMethod(ll_keyhash) - if ll_keyeq is not None: - ll_keyeq = lltype.staticAdtMethod(ll_keyeq) - adtmeths = { - 'keyhash': ll_keyhash, - 'keyeq': ll_keyeq, - 'paranoia': False, - } - adtmeths['KEY'] = self.DICTKEY - adtmeths['VALUE'] = self.DICTVALUE - adtmeths['allocate'] = lltype.typeMethod(_ll_malloc_dict) - self.DICT.become(lltype.GcStruct("dicttable", adtmeths=adtmeths, - *fields)) - + DICTKEY = self.key_repr.lowleveltype + DICTVALUE = self.value_repr.lowleveltype + xxx def convert_const(self, dictobj): from pypy.rpython.lltypesystem import llmemory @@ -385,41 +327,6 @@ # be direct_call'ed from rtyped flow graphs, which means that they will # get flowed and annotated, mostly with SomePtr. -def ll_everused_from_flag(entries, i): - return entries[i].f_everused - -def ll_everused_from_key(entries, i): - return bool(entries[i].key) - -def ll_everused_from_value(entries, i): - return bool(entries[i].value) - -def ll_valid_from_flag(entries, i): - return entries[i].f_valid - -def ll_mark_deleted_in_flag(entries, i): - entries[i].f_valid = False - -def ll_valid_from_key(entries, i): - ENTRIES = lltype.typeOf(entries).TO - dummy = ENTRIES.dummy_obj.ll_dummy_value - return entries.everused(i) and entries[i].key != dummy - -def ll_mark_deleted_in_key(entries, i): - ENTRIES = lltype.typeOf(entries).TO - dummy = ENTRIES.dummy_obj.ll_dummy_value - entries[i].key = dummy - -def ll_valid_from_value(entries, i): - ENTRIES = lltype.typeOf(entries).TO - dummy = ENTRIES.dummy_obj.ll_dummy_value - return entries.everused(i) and entries[i].value != dummy - -def ll_mark_deleted_in_value(entries, i): - ENTRIES = lltype.typeOf(entries).TO - dummy = ENTRIES.dummy_obj.ll_dummy_value - entries[i].value = dummy - def ll_hash_from_cache(entries, i): return entries[i].f_hash @@ -464,15 +371,17 @@ valid = (i & HIGHEST_BIT) == 0 i = i & MASK ENTRY = lltype.typeOf(d.entries).TO.OF - entry = d.entries[i] - if not d.entries.everused(i): + index = d.indexes[i] + entry = d.entries[index] + if index == FREE: # a new entry that was never used before ll_assert(not valid, "valid but not everused") rc = d.resize_counter - 3 if rc <= 0: # if needed, resize the dict -- before the insertion ll_dict_resize(d) - i = ll_dict_lookup_clean(d, hash) # then redo the lookup for 'key' - entry = d.entries[i] + index = ll_dict_lookup_clean(d, hash) + # then redo the lookup for 'key' + entry = d.entries[index] rc = d.resize_counter - 3 ll_assert(rc > 0, "ll_dict_resize failed?") d.resize_counter = rc @@ -496,12 +405,11 @@ # to user code. ll_dict_insertclean() doesn't resize the dict, either. i = ll_dict_lookup_clean(d, hash) ENTRY = lltype.typeOf(d.entries).TO.OF - entry = d.entries[i] + index = d.indexes[i] + entry = d.entries[index] entry.value = value entry.key = key if hasattr(ENTRY, 'f_hash'): entry.f_hash = hash - if hasattr(ENTRY, 'f_valid'): entry.f_valid = True - if hasattr(ENTRY, 'f_everused'): entry.f_everused = True d.num_items += 1 d.resize_counter -= 3 @@ -513,6 +421,7 @@ @jit.look_inside_iff(lambda d, i: jit.isvirtual(d) and jit.isconstant(i)) def _ll_dict_del(d, i): + XXX d.entries.mark_deleted(i) d.num_items -= 1 # clear the key and the value if they are GC pointers @@ -539,7 +448,8 @@ def ll_dict_resize(d): old_entries = d.entries - old_size = len(old_entries) + old_indexes = d.indexes + old_size = len(old_indexes) # make a 'new_size' estimate and shrink it if there are many # deleted entry markers. See CPython for why it is a good idea to # quadruple the dictionary size as long as it's not too big. @@ -550,17 +460,19 @@ while new_size <= new_estimate: new_size *= 2 # - d.entries = lltype.typeOf(old_entries).TO.allocate(new_size) - d.num_items = 0 + new_item_size = new_size // 3 * 2 + 1 + d.entries = lltype.typeOf(old_entries).TO.allocate(new_item_size) + d.indexes = lltype.malloc(lltype.typeOf(d).TO.indexes.TO, new_size) + d.num_items = len(old_entries) d.resize_counter = new_size * 2 i = 0 + indexes = d.indexes while i < old_size: - if old_entries.valid(i): - hash = old_entries.hash(i) - entry = old_entries[i] - ll_dict_insertclean(d, entry.key, entry.value, hash) + index = old_indexes[i] + if index >= 0: + indexes[ll_dict_lookup_clean(d, old_entries.hash(i))] = index i += 1 - old_entries.delete() + rgc.ll_arraycopy(old_entries, d.entries, 0, 0, len(old_entries)) ll_dict_resize.oopspec = 'dict.resize(d)' # ------- a port of CPython's dictobject.c's lookdict implementation ------- @@ -569,28 +481,30 @@ @jit.look_inside_iff(lambda d, key, hash: jit.isvirtual(d) and jit.isconstant(key)) def ll_dict_lookup(d, key, hash): entries = d.entries + indexes = d.indexes ENTRIES = lltype.typeOf(entries).TO direct_compare = not hasattr(ENTRIES, 'no_direct_compare') mask = len(entries) - 1 i = hash & mask # do the first try before any looping - if entries.valid(i): - checkingkey = entries[i].key + index = indexes[i] + if index >= 0: + checkingkey = entries[index].key if direct_compare and checkingkey == key: - return i # found the entry - if d.keyeq is not None and entries.hash(i) == hash: + return index # found the entry + if d.keyeq is not None and entries.hash(index) == hash: # correct hash, maybe the key is e.g. a different pointer to # an equal object found = d.keyeq(checkingkey, key) if d.paranoia: if (entries != d.entries or - not entries.valid(i) or entries[i].key != checkingkey): + not indexes[i] >= 0 or entries[index].key != checkingkey): # the compare did major nasty stuff to the dict: start over return ll_dict_lookup(d, key, hash) if found: - return i # found the entry + return index # found the entry freeslot = -1 - elif entries.everused(i): + elif index == DELETED: freeslot = i else: return i | HIGHEST_BIT # pristine entry -- lookup failed @@ -603,28 +517,30 @@ i = r_uint(i) i = (i << 2) + i + perturb + 1 i = intmask(i) & mask + index = indexes[i] # keep 'i' as a signed number here, to consistently pass signed # arguments to the small helper methods. - if not entries.everused(i): + if index == FREE: if freeslot == -1: freeslot = i return freeslot | HIGHEST_BIT - elif entries.valid(i): - checkingkey = entries[i].key + elif index >= 0: + checkingkey = entries[index].key if direct_compare and checkingkey == key: - return i - if d.keyeq is not None and entries.hash(i) == hash: + return index + if d.keyeq is not None and entries.hash(index) == hash: # correct hash, maybe the key is e.g. a different pointer to # an equal object found = d.keyeq(checkingkey, key) if d.paranoia: if (entries != d.entries or - not entries.valid(i) or entries[i].key != checkingkey): + not indexes[i] >= 0 or + entries[index].key != checkingkey): # the compare did major nasty stuff to the dict: # start over return ll_dict_lookup(d, key, hash) if found: - return i # found the entry + return index # found the entry elif freeslot == -1: freeslot = i perturb >>= PERTURB_SHIFT @@ -633,11 +549,11 @@ # a simplified version of ll_dict_lookup() which assumes that the # key is new, and the dictionary doesn't contain deleted entries. # It only finds the next free slot for the given hash. - entries = d.entries - mask = len(entries) - 1 + indexes = d.indexes + mask = len(indexes) - 1 i = hash & mask perturb = r_uint(hash) - while entries.everused(i): + while i != FREE: i = r_uint(i) i = (i << 2) + i + perturb + 1 i = intmask(i) & mask @@ -649,10 +565,15 @@ # Irregular operations. DICT_INITSIZE = 8 +DICT_ITEMS_INITSIZE = 5 + at jit.unroll_safe # we always unroll the small allocation def ll_newdict(DICT): d = DICT.allocate() - d.entries = DICT.entries.TO.allocate(DICT_INITSIZE) + d.indexes = lltype.malloc(DICT.indexes.TO, DICT_INITSIZE) + for i in range(DICT_INITSIZE): + d.indexes[i] = FREE + d.entries = DICT.entries.TO.allocate(DICT_ITEMS_INITSIZE) d.num_items = 0 d.resize_counter = DICT_INITSIZE * 2 return d @@ -662,8 +583,10 @@ n = DICT_INITSIZE while n < length_estimate: n *= 2 + items_size = n // 3 * 2 + 1 d = DICT.allocate() - d.entries = DICT.entries.TO.allocate(n) + d.entries = DICT.entries.TO.allocate(items_size) + d.indexes = lltype.malloc(DICT.indexes.TO, n) d.num_items = 0 d.resize_counter = n * 2 return d @@ -675,19 +598,14 @@ return lltype.malloc(DICT) def _ll_malloc_entries(ENTRIES, n): return lltype.malloc(ENTRIES, n, zero=True) -def _ll_free_entries(entries): - pass -def rtype_r_dict(hop, i_force_non_null=None): +def rtype_r_dict(hop): r_dict = hop.r_result if not r_dict.custom_eq_hash: raise TyperError("r_dict() call does not return an r_dict instance") v_eqfn = hop.inputarg(r_dict.r_rdict_eqfn, arg=0) v_hashfn = hop.inputarg(r_dict.r_rdict_hashfn, arg=1) - if i_force_non_null is not None: - assert i_force_non_null == 2 - hop.inputarg(lltype.Void, arg=2) cDICT = hop.inputconst(lltype.Void, r_dict.DICT) hop.exception_cannot_occur() v_result = hop.gendirectcall(ll_newdict, cDICT) @@ -780,6 +698,7 @@ return default def ll_copy(dict): + xxx DICT = lltype.typeOf(dict).TO dictsize = len(dict.entries) d = DICT.allocate() @@ -803,6 +722,7 @@ ll_copy.oopspec = 'dict.copy(dict)' def ll_clear(d): + xxx if (len(d.entries) == DICT_INITSIZE and d.resize_counter == DICT_INITSIZE * 2): return @@ -810,10 +730,10 @@ d.entries = lltype.typeOf(old_entries).TO.allocate(DICT_INITSIZE) d.num_items = 0 d.resize_counter = DICT_INITSIZE * 2 - old_entries.delete() ll_clear.oopspec = 'dict.clear(d)' def ll_update(dic1, dic2): + xxx entries = dic2.entries d2len = len(entries) i = 0 diff --git a/pypy/rpython/test/test_rdict.py b/pypy/rpython/test/test_rdict.py --- a/pypy/rpython/test/test_rdict.py +++ b/pypy/rpython/test/test_rdict.py @@ -4,7 +4,9 @@ from pypy.rpython.lltypesystem import rdict, rstr from pypy.rpython.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin from pypy.rlib.objectmodel import r_dict -from pypy.rlib.rarithmetic import r_int, r_uint, r_longlong, r_ulonglong +from pypy.rlib.rarithmetic import r_int, r_uint, r_longlong, r_ulonglong,\ + intmask +from pypy.rpython.annlowlevel import llstr, hlstr import py py.log.setconsumer("rtyper", py.log.STDOUT) @@ -22,6 +24,15 @@ yield x +class TestRDictDirect(object): + def test_dict_creation(self): + DICT = rdict.get_ll_dict(lltype.Ptr(rstr.STR), lltype.Signed, + ll_fasthash_function=rstr.LLHelpers.ll_strhash, + ll_hash_function=rstr.LLHelpers.ll_strhash, + ll_eq_function=rstr.LLHelpers.ll_streq) + ll_d = rdict.ll_newdict(DICT) + rdict.ll_dict_setitem(ll_d, llstr("abc"), 13) + class BaseTestRdict(BaseRtypingTest): def test_dict_creation(self): From noreply at buildbot.pypy.org Sun Jan 6 17:48:31 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 6 Jan 2013 17:48:31 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: hack enough to start passing tests Message-ID: <20130106164831.AB9D71C0AD6@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59811:bae0ef64aac4 Date: 2013-01-06 18:47 +0200 http://bitbucket.org/pypy/pypy/changeset/bae0ef64aac4/ Log: hack enough to start passing tests diff --git a/pypy/annotation/bookkeeper.py b/pypy/annotation/bookkeeper.py --- a/pypy/annotation/bookkeeper.py +++ b/pypy/annotation/bookkeeper.py @@ -298,13 +298,12 @@ listdef.generalize_range_step(flags['range_step']) return SomeList(listdef) - def getdictdef(self, is_r_dict=False, force_non_null=False): + def getdictdef(self, is_r_dict=False): """Get the DictDef associated with the current position.""" try: dictdef = self.dictdefs[self.position_key] except KeyError: - dictdef = DictDef(self, is_r_dict=is_r_dict, - force_non_null=force_non_null) + dictdef = DictDef(self, is_r_dict=is_r_dict) self.dictdefs[self.position_key] = dictdef return dictdef diff --git a/pypy/annotation/builtin.py b/pypy/annotation/builtin.py --- a/pypy/annotation/builtin.py +++ b/pypy/annotation/builtin.py @@ -294,14 +294,8 @@ clsdef = clsdef.commonbase(cdef) return SomeInstance(clsdef) -def robjmodel_r_dict(s_eqfn, s_hashfn, s_force_non_null=None): - if s_force_non_null is None: - force_non_null = False - else: - assert s_force_non_null.is_constant() - force_non_null = s_force_non_null.const - dictdef = getbookkeeper().getdictdef(is_r_dict=True, - force_non_null=force_non_null) +def robjmodel_r_dict(s_eqfn, s_hashfn): + dictdef = getbookkeeper().getdictdef(is_r_dict=True) dictdef.dictkey.update_rdict_annotations(s_eqfn, s_hashfn) return SomeDict(dictdef) diff --git a/pypy/annotation/dictdef.py b/pypy/annotation/dictdef.py --- a/pypy/annotation/dictdef.py +++ b/pypy/annotation/dictdef.py @@ -85,14 +85,12 @@ def __init__(self, bookkeeper, s_key = s_ImpossibleValue, s_value = s_ImpossibleValue, - is_r_dict = False, - force_non_null = False): + is_r_dict = False): self.dictkey = DictKey(bookkeeper, s_key, is_r_dict) self.dictkey.itemof[self] = True self.dictvalue = DictValue(bookkeeper, s_value) self.dictvalue.itemof[self] = True self.bookkeeper = bookkeeper - self.force_non_null = force_non_null def read_key(self, position_key=None): if position_key is None: diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py --- a/pypy/objspace/std/dictmultiobject.py +++ b/pypy/objspace/std/dictmultiobject.py @@ -9,7 +9,6 @@ from pypy.rlib.objectmodel import r_dict, we_are_translated, specialize,\ newlist_hint -from pypy.rlib.debug import mark_dict_non_null from pypy.tool.sourcetools import func_with_new_name from pypy.rlib import rerased @@ -559,8 +558,7 @@ return True def get_empty_storage(self): - new_dict = r_dict(self.space.eq_w, self.space.hash_w, - force_non_null=True) + new_dict = r_dict(self.space.eq_w, self.space.hash_w) return self.erase(new_dict) def _never_equal_to(self, w_lookup_type): @@ -589,7 +587,6 @@ def get_empty_storage(self): res = {} - mark_dict_non_null(res) return self.erase(res) def _never_equal_to(self, w_lookup_type): @@ -654,7 +651,6 @@ def get_empty_storage(self): res = {} - mark_dict_non_null(res) return self.erase(res) def _never_equal_to(self, w_lookup_type): diff --git a/pypy/objspace/std/identitydict.py b/pypy/objspace/std/identitydict.py --- a/pypy/objspace/std/identitydict.py +++ b/pypy/objspace/std/identitydict.py @@ -2,7 +2,6 @@ ## dict strategy (see dictmultiobject.py) from pypy.rlib import rerased -from pypy.rlib.debug import mark_dict_non_null from pypy.objspace.std.dictmultiobject import (AbstractTypedStrategy, DictStrategy, create_iterator_classes) @@ -66,7 +65,6 @@ def get_empty_storage(self): d = {} - mark_dict_non_null(d) return self.erase(d) def is_correct_type(self, w_obj): diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py --- a/pypy/objspace/std/setobject.py +++ b/pypy/objspace/std/setobject.py @@ -982,7 +982,7 @@ # some helper functions def newset(space): - return r_dict(space.eq_w, space.hash_w, force_non_null=True) + return r_dict(space.eq_w, space.hash_w) def set_strategy_and_setdata(space, w_set, w_iterable): from pypy.objspace.std.intobject import W_IntObject diff --git a/pypy/rlib/debug.py b/pypy/rlib/debug.py --- a/pypy/rlib/debug.py +++ b/pypy/rlib/debug.py @@ -284,28 +284,6 @@ return hop.inputarg(hop.args_r[0], arg=0) -def mark_dict_non_null(d): - """ Mark dictionary as having non-null keys and values. A warning would - be emitted (not an error!) in case annotation disagrees. - """ - assert isinstance(d, dict) - return d - - -class DictMarkEntry(ExtRegistryEntry): - _about_ = mark_dict_non_null - - def compute_result_annotation(self, s_dict): - from pypy.annotation.model import SomeDict, s_None - - assert isinstance(s_dict, SomeDict) - s_dict.dictdef.force_non_null = True - return s_dict - - def specialize_call(self, hop): - hop.exception_cannot_occur() - return hop.inputarg(hop.args_r[0], arg=0) - class IntegerCanBeNegative(Exception): pass diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py --- a/pypy/rlib/objectmodel.py +++ b/pypy/rlib/objectmodel.py @@ -650,11 +650,10 @@ The functions key_eq() and key_hash() are used by the key comparison algorithm.""" - def __init__(self, key_eq, key_hash, force_non_null=False): + def __init__(self, key_eq, key_hash): self._dict = {} self.key_eq = key_eq self.key_hash = key_hash - self.force_non_null = force_non_null def __getitem__(self, key): return self._dict[_r_dictkey(self, key)] diff --git a/pypy/rlib/test/test_debug.py b/pypy/rlib/test/test_debug.py --- a/pypy/rlib/test/test_debug.py +++ b/pypy/rlib/test/test_debug.py @@ -3,8 +3,8 @@ from pypy.rlib.debug import (check_annotation, make_sure_not_resized, debug_print, debug_start, debug_stop, have_debug_prints, debug_offset, debug_flush, - check_nonneg, IntegerCanBeNegative, - mark_dict_non_null) + check_nonneg, IntegerCanBeNegative) + from pypy.rlib import debug from pypy.rpython.test.test_llinterp import interpret, gengraph @@ -53,15 +53,6 @@ py.test.raises(ListChangeUnallowed, interpret, f, [], list_comprehension_operations=True) -def test_mark_dict_non_null(): - def f(): - d = {"ac": "bx"} - mark_dict_non_null(d) - return d - - t, typer, graph = gengraph(f, []) - assert sorted(graph.returnblock.inputargs[0].concretetype.TO.entries.TO.OF._flds.keys()) == ['key', 'value'] - class DebugTests(object): diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py --- a/pypy/rpython/lltypesystem/rdict.py +++ b/pypy/rpython/lltypesystem/rdict.py @@ -147,7 +147,11 @@ if isinstance(self.DICT, lltype.GcForwardReference): DICTKEY = self.key_repr.lowleveltype DICTVALUE = self.value_repr.lowleveltype - xxx + get_ll_dict(DICTKEY, DICTVALUE, DICT=self.DICT, + ll_fasthash_function=self.key_repr.get_ll_fasthash_function(), + ll_hash_function=self.key_repr.get_ll_hash_function(), + ll_eq_function=self.key_repr.get_ll_eq_function(), + get_custom_eq_hash=self.custom_eq_hash) def convert_const(self, dictobj): from pypy.rpython.lltypesystem import llmemory @@ -372,8 +376,9 @@ i = i & MASK ENTRY = lltype.typeOf(d.entries).TO.OF index = d.indexes[i] - entry = d.entries[index] if index == FREE: + index = d.num_items + entry = d.entries[index] # a new entry that was never used before ll_assert(not valid, "valid but not everused") rc = d.resize_counter - 3 @@ -385,16 +390,21 @@ rc = d.resize_counter - 3 ll_assert(rc > 0, "ll_dict_resize failed?") d.resize_counter = rc - if hasattr(ENTRY, 'f_everused'): entry.f_everused = True + d.indexes[i] = index + entry.value = value + elif index == DELETED: + index = d.num_items + entry = d.entries[index] + d.indexes[i] = index entry.value = value else: # override an existing or deleted entry + entry = d.entries[index] entry.value = value if valid: return entry.key = key if hasattr(ENTRY, 'f_hash'): entry.f_hash = hash - if hasattr(ENTRY, 'f_valid'): entry.f_valid = True d.num_items += 1 def ll_dict_insertclean(d, key, value, hash): @@ -482,15 +492,13 @@ def ll_dict_lookup(d, key, hash): entries = d.entries indexes = d.indexes - ENTRIES = lltype.typeOf(entries).TO - direct_compare = not hasattr(ENTRIES, 'no_direct_compare') mask = len(entries) - 1 i = hash & mask # do the first try before any looping index = indexes[i] if index >= 0: checkingkey = entries[index].key - if direct_compare and checkingkey == key: + if checkingkey == key: return index # found the entry if d.keyeq is not None and entries.hash(index) == hash: # correct hash, maybe the key is e.g. a different pointer to @@ -526,7 +534,7 @@ return freeslot | HIGHEST_BIT elif index >= 0: checkingkey = entries[index].key - if direct_compare and checkingkey == key: + if checkingkey == key: return index if d.keyeq is not None and entries.hash(index) == hash: # correct hash, maybe the key is e.g. a different pointer to @@ -713,8 +721,6 @@ entry = dict.entries[i] ENTRY = lltype.typeOf(d.entries).TO.OF d_entry.key = entry.key - if hasattr(ENTRY, 'f_valid'): d_entry.f_valid = entry.f_valid - if hasattr(ENTRY, 'f_everused'): d_entry.f_everused = entry.f_everused d_entry.value = entry.value if hasattr(ENTRY, 'f_hash'): d_entry.f_hash = entry.f_hash i += 1 diff --git a/pypy/rpython/ootypesystem/rdict.py b/pypy/rpython/ootypesystem/rdict.py --- a/pypy/rpython/ootypesystem/rdict.py +++ b/pypy/rpython/ootypesystem/rdict.py @@ -13,7 +13,7 @@ class DictRepr(AbstractDictRepr): def __init__(self, rtyper, key_repr, value_repr, dictkey, dictvalue, - custom_eq_hash=None, force_non_null=False): + custom_eq_hash=None): self.rtyper = rtyper self.custom_eq_hash = custom_eq_hash is not None @@ -260,7 +260,7 @@ methodname = None return fn, v_obj, methodname -def rtype_r_dict(hop, i_force_non_null=None): +def rtype_r_dict(hop): from pypy.rlib import jit r_dict = hop.r_result diff --git a/pypy/rpython/rdict.py b/pypy/rpython/rdict.py --- a/pypy/rpython/rdict.py +++ b/pypy/rpython/rdict.py @@ -9,7 +9,6 @@ dictvalue = self.dictdef.dictvalue s_key = dictkey .s_value s_value = dictvalue.s_value - force_non_null = self.dictdef.force_non_null if dictkey.custom_eq_hash: custom_eq_hash = lambda: (rtyper.getrepr(dictkey.s_rdict_eqfn), rtyper.getrepr(dictkey.s_rdict_hashfn)) @@ -20,8 +19,7 @@ lambda: rtyper.getrepr(s_value), dictkey, dictvalue, - custom_eq_hash, - force_non_null) + custom_eq_hash) def rtyper_makekey(self): self.dictdef.dictkey .dont_change_any_more = True diff --git a/pypy/rpython/test/test_rdict.py b/pypy/rpython/test/test_rdict.py --- a/pypy/rpython/test/test_rdict.py +++ b/pypy/rpython/test/test_rdict.py @@ -32,6 +32,10 @@ ll_eq_function=rstr.LLHelpers.ll_streq) ll_d = rdict.ll_newdict(DICT) rdict.ll_dict_setitem(ll_d, llstr("abc"), 13) + assert (len([i for i in ll_d.indexes if i == rdict.FREE]) == + rdict.DICT_INITSIZE - 1) + assert rdict.ll_dict_getitem(ll_d, llstr("abc")) == 13 + class BaseTestRdict(BaseRtypingTest): @@ -969,7 +973,7 @@ return 3 def func(i): - d = r_dict(eq, rhash, force_non_null=True) + d = r_dict(eq, rhash) if not i: d[None] = i else: From noreply at buildbot.pypy.org Sun Jan 6 18:30:33 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 6 Jan 2013 18:30:33 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: hack hack and test test Message-ID: <20130106173033.637DA1C0AD6@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59812:b07d28a54bea Date: 2013-01-06 19:29 +0200 http://bitbucket.org/pypy/pypy/changeset/b07d28a54bea/ Log: hack hack and test test diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py --- a/pypy/rpython/lltypesystem/rdict.py +++ b/pypy/rpython/lltypesystem/rdict.py @@ -339,7 +339,7 @@ return ENTRIES.fasthashfn(entries[i].key) def ll_get_value(d, i): - return d.entries[i].value + return d.entries[d.indexes[i]].value def ll_keyhash_custom(d, key): DICT = lltype.typeOf(d).TO @@ -384,7 +384,7 @@ rc = d.resize_counter - 3 if rc <= 0: # if needed, resize the dict -- before the insertion ll_dict_resize(d) - index = ll_dict_lookup_clean(d, hash) + index = d.indexes[ll_dict_lookup_clean(d, hash)] # then redo the lookup for 'key' entry = d.entries[index] rc = d.resize_counter - 3 @@ -431,13 +431,24 @@ @jit.look_inside_iff(lambda d, i: jit.isvirtual(d) and jit.isconstant(i)) def _ll_dict_del(d, i): - XXX - d.entries.mark_deleted(i) + index = d.indexes[i] + d.indexes[i] = DELETED d.num_items -= 1 - # clear the key and the value if they are GC pointers ENTRIES = lltype.typeOf(d.entries).TO ENTRY = ENTRIES.OF - entry = d.entries[i] + if index != d.num_items: + old_entry = d.entries[d.num_items] + key = old_entry.key + to_insert_i = ll_dict_lookup(d, key, d.keyhash(key)) + d.indexes[to_insert_i] = index + # copy the value + new_entry = d.entries[index] + new_entry.key = key + new_entry.value = old_entry.value + if hasattr(ENTRY, 'f_hash'): + new_entry.f_hash = old_entry.f_hash + # clear the key and the value if they are GC pointers + entry = d.entries[d.num_items] if ENTRIES.must_clear_key: entry.key = lltype.nullptr(ENTRY.key.TO) if ENTRIES.must_clear_value: @@ -499,7 +510,7 @@ if index >= 0: checkingkey = entries[index].key if checkingkey == key: - return index # found the entry + return i # found the entry if d.keyeq is not None and entries.hash(index) == hash: # correct hash, maybe the key is e.g. a different pointer to # an equal object @@ -510,7 +521,7 @@ # the compare did major nasty stuff to the dict: start over return ll_dict_lookup(d, key, hash) if found: - return index # found the entry + return i # found the entry freeslot = -1 elif index == DELETED: freeslot = i @@ -548,7 +559,7 @@ # start over return ll_dict_lookup(d, key, hash) if found: - return index # found the entry + return i # found the entry elif freeslot == -1: freeslot = i perturb >>= PERTURB_SHIFT diff --git a/pypy/rpython/test/test_rdict.py b/pypy/rpython/test/test_rdict.py --- a/pypy/rpython/test/test_rdict.py +++ b/pypy/rpython/test/test_rdict.py @@ -25,17 +25,43 @@ class TestRDictDirect(object): - def test_dict_creation(self): + def _get_str_dict(self): + # STR -> lltype.Signed DICT = rdict.get_ll_dict(lltype.Ptr(rstr.STR), lltype.Signed, ll_fasthash_function=rstr.LLHelpers.ll_strhash, ll_hash_function=rstr.LLHelpers.ll_strhash, ll_eq_function=rstr.LLHelpers.ll_streq) + return DICT + + def test_dict_creation(self): + DICT = self._get_str_dict() ll_d = rdict.ll_newdict(DICT) rdict.ll_dict_setitem(ll_d, llstr("abc"), 13) assert (len([i for i in ll_d.indexes if i == rdict.FREE]) == rdict.DICT_INITSIZE - 1) assert rdict.ll_dict_getitem(ll_d, llstr("abc")) == 13 + def test_dict_del_lastitem(self): + DICT = self._get_str_dict() + ll_d = rdict.ll_newdict(DICT) + py.test.raises(KeyError, rdict.ll_dict_delitem, ll_d, llstr("abc")) + rdict.ll_dict_setitem(ll_d, llstr("abc"), 13) + py.test.raises(KeyError, rdict.ll_dict_delitem, ll_d, llstr("def")) + rdict.ll_dict_delitem(ll_d, llstr("abc")) + assert (len([i for i in ll_d.indexes if i == rdict.FREE]) == + rdict.DICT_INITSIZE - 1) + assert (len([i for i in ll_d.indexes if i == rdict.DELETED]) == 1) + py.test.raises(KeyError, rdict.ll_dict_getitem, ll_d, llstr("abc")) + + def test_dict_del_not_lastitem(self): + DICT = self._get_str_dict() + ll_d = rdict.ll_newdict(DICT) + rdict.ll_dict_setitem(ll_d, llstr("abc"), 13) + rdict.ll_dict_setitem(ll_d, llstr("def"), 15) + rdict.ll_dict_delitem(ll_d, llstr("abc")) + assert (len([i for i in ll_d.indexes if i == rdict.FREE]) == + rdict.DICT_INITSIZE - 2) + assert (len([i for i in ll_d.indexes if i == rdict.DELETED]) == 1) class BaseTestRdict(BaseRtypingTest): From noreply at buildbot.pypy.org Sun Jan 6 19:28:56 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 6 Jan 2013 19:28:56 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: hack hack fix fix test test Message-ID: <20130106182856.33D391C0AD6@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59813:0d9d5d21c955 Date: 2013-01-06 20:28 +0200 http://bitbucket.org/pypy/pypy/changeset/0d9d5d21c955/ Log: hack hack fix fix test test diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py --- a/pypy/rpython/lltypesystem/opimpl.py +++ b/pypy/rpython/lltypesystem/opimpl.py @@ -525,6 +525,8 @@ A = lltype.typeOf(source) assert A == lltype.typeOf(dest) assert isinstance(A.TO, lltype.GcArray) + if isinstance(A.TO.OF, lltype.Struct): + return True assert isinstance(A.TO.OF, lltype.Ptr) assert A.TO.OF.TO._gckind == 'gc' assert type(source_start) is int diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py --- a/pypy/rpython/lltypesystem/rdict.py +++ b/pypy/rpython/lltypesystem/rdict.py @@ -78,10 +78,12 @@ DICTENTRY = lltype.Struct("dictentry", *entryfields) DICTENTRYARRAY = lltype.GcArray(DICTENTRY, adtmeths=entrymeths) + array_adtmeths = {'allocate': lltype.typeMethod(_ll_malloc_items)} fields = [("num_items", lltype.Signed), ("resize_counter", lltype.Signed), ("entries", lltype.Ptr(DICTENTRYARRAY)), - ("indexes", lltype.Ptr(lltype.GcArray(lltype.Signed)))] + ("indexes", lltype.Ptr(lltype.GcArray(lltype.Signed, + adtmeths=array_adtmeths)))] if get_custom_eq_hash is not None: r_rdict_eqfn, r_rdict_hashfn = get_custom_eq_hash() fields.extend([ ("fnkeyeq", r_rdict_eqfn.lowleveltype), @@ -368,9 +370,12 @@ i = ll_dict_lookup(d, key, hash) return _ll_dict_setitem_lookup_done(d, key, value, hash, i) +def _look_inside_setitem(d, key, value, hash, i): + return jit.isvirtual(d) and jit.isconstant(key) + # It may be safe to look inside always, it has a few branches though, and their # frequencies needs to be investigated. - at jit.look_inside_iff(lambda d, key, value, hash, i: jit.isvirtual(d) and jit.isconstant(key)) + at jit.look_inside_iff(_look_inside_setitem) def _ll_dict_setitem_lookup_done(d, key, value, hash, i): valid = (i & HIGHEST_BIT) == 0 i = i & MASK @@ -384,7 +389,7 @@ rc = d.resize_counter - 3 if rc <= 0: # if needed, resize the dict -- before the insertion ll_dict_resize(d) - index = d.indexes[ll_dict_lookup_clean(d, hash)] + i = ll_dict_lookup_clean(d, hash) # then redo the lookup for 'key' entry = d.entries[index] rc = d.resize_counter - 3 @@ -483,7 +488,7 @@ # new_item_size = new_size // 3 * 2 + 1 d.entries = lltype.typeOf(old_entries).TO.allocate(new_item_size) - d.indexes = lltype.malloc(lltype.typeOf(d).TO.indexes.TO, new_size) + d.indexes = lltype.typeOf(d).TO.indexes.TO.allocate(new_size) d.num_items = len(old_entries) d.resize_counter = new_size * 2 i = 0 @@ -499,11 +504,13 @@ # ------- a port of CPython's dictobject.c's lookdict implementation ------- PERTURB_SHIFT = 5 - at jit.look_inside_iff(lambda d, key, hash: jit.isvirtual(d) and jit.isconstant(key)) +_look_inside_lookup = lambda d, key, hash: jit.isvirtual(d) and jit.isconstant(key) + + at jit.look_inside_iff(_look_inside_lookup) def ll_dict_lookup(d, key, hash): entries = d.entries indexes = d.indexes - mask = len(entries) - 1 + mask = len(indexes) - 1 i = hash & mask # do the first try before any looping index = indexes[i] @@ -572,7 +579,7 @@ mask = len(indexes) - 1 i = hash & mask perturb = r_uint(hash) - while i != FREE: + while d.indexes[i] != FREE: i = r_uint(i) i = (i << 2) + i + perturb + 1 i = intmask(i) & mask @@ -584,11 +591,13 @@ # Irregular operations. DICT_INITSIZE = 8 -DICT_ITEMS_INITSIZE = 5 +DICT_ITEMS_INITSIZE = 6 @jit.unroll_safe # we always unroll the small allocation def ll_newdict(DICT): d = DICT.allocate() + # XXX don't use _ll_items_allocate because of jit.unroll_safe, + # should be *really* jit_unroll_iff d.indexes = lltype.malloc(DICT.indexes.TO, DICT_INITSIZE) for i in range(DICT_INITSIZE): d.indexes[i] = FREE @@ -605,7 +614,7 @@ items_size = n // 3 * 2 + 1 d = DICT.allocate() d.entries = DICT.entries.TO.allocate(items_size) - d.indexes = lltype.malloc(DICT.indexes.TO, n) + d.indexes = DICT.indexes.TO.allocate(n) d.num_items = 0 d.resize_counter = n * 2 return d @@ -617,7 +626,13 @@ return lltype.malloc(DICT) def _ll_malloc_entries(ENTRIES, n): return lltype.malloc(ENTRIES, n, zero=True) - +def _ll_malloc_items(ITEMS, n): + res = lltype.malloc(ITEMS, n) + i = 0 + while i < n: + res[i] = FREE + i += 1 + return res def rtype_r_dict(hop): r_dict = hop.r_result @@ -739,12 +754,12 @@ ll_copy.oopspec = 'dict.copy(dict)' def ll_clear(d): - xxx - if (len(d.entries) == DICT_INITSIZE and + if (len(d.indexes) == DICT_INITSIZE and d.resize_counter == DICT_INITSIZE * 2): return old_entries = d.entries - d.entries = lltype.typeOf(old_entries).TO.allocate(DICT_INITSIZE) + d.entries = lltype.typeOf(old_entries).TO.allocate(DICT_ITEMS_INITSIZE) + d.indexes = lltype.typeOf(d).TO.indexes.TO.allocate(DICT_INITSIZE) d.num_items = 0 d.resize_counter = DICT_INITSIZE * 2 ll_clear.oopspec = 'dict.clear(d)' diff --git a/pypy/rpython/test/test_rdict.py b/pypy/rpython/test/test_rdict.py --- a/pypy/rpython/test/test_rdict.py +++ b/pypy/rpython/test/test_rdict.py @@ -63,6 +63,18 @@ rdict.DICT_INITSIZE - 2) assert (len([i for i in ll_d.indexes if i == rdict.DELETED]) == 1) + def test_dict_resize(self): + DICT = self._get_str_dict() + ll_d = rdict.ll_newdict(DICT) + rdict.ll_dict_setitem(ll_d, llstr("a"), 1) + rdict.ll_dict_setitem(ll_d, llstr("b"), 2) + rdict.ll_dict_setitem(ll_d, llstr("c"), 3) + rdict.ll_dict_setitem(ll_d, llstr("d"), 4) + rdict.ll_dict_setitem(ll_d, llstr("e"), 5) + assert len(ll_d.indexes) == 8 + rdict.ll_dict_setitem(ll_d, llstr("f"), 6) + assert len(ll_d.indexes) == 32 + class BaseTestRdict(BaseRtypingTest): def test_dict_creation(self): From noreply at buildbot.pypy.org Sun Jan 6 19:42:07 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 6 Jan 2013 19:42:07 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: fixes and tests Message-ID: <20130106184207.605611C0312@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59814:aa62c8953637 Date: 2013-01-06 20:41 +0200 http://bitbucket.org/pypy/pypy/changeset/aa62c8953637/ Log: fixes and tests diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py --- a/pypy/rpython/lltypesystem/rdict.py +++ b/pypy/rpython/lltypesystem/rdict.py @@ -185,16 +185,14 @@ for dictkeycontainer, dictvalue in dictobj._dict.items(): llkey = r_key.convert_const(dictkeycontainer.key) llvalue = r_value.convert_const(dictvalue) - ll_dict_insertclean(l_dict, llkey, llvalue, - dictkeycontainer.hash) + ll_dict_setitem(l_dict, llkey, llvalue) return l_dict else: for dictkey, dictvalue in dictobj.items(): llkey = r_key.convert_const(dictkey) llvalue = r_value.convert_const(dictvalue) - ll_dict_insertclean(l_dict, llkey, llvalue, - l_dict.keyhash(llkey)) + ll_dict_setitem(l_dict, llkey, llvalue) return l_dict def rtype_len(self, hop): @@ -386,13 +384,13 @@ entry = d.entries[index] # a new entry that was never used before ll_assert(not valid, "valid but not everused") - rc = d.resize_counter - 3 + rc = d.resize_counter - 1 if rc <= 0: # if needed, resize the dict -- before the insertion ll_dict_resize(d) i = ll_dict_lookup_clean(d, hash) # then redo the lookup for 'key' entry = d.entries[index] - rc = d.resize_counter - 3 + rc = d.resize_counter - 1 ll_assert(rc > 0, "ll_dict_resize failed?") d.resize_counter = rc d.indexes[i] = index @@ -412,22 +410,6 @@ if hasattr(ENTRY, 'f_hash'): entry.f_hash = hash d.num_items += 1 -def ll_dict_insertclean(d, key, value, hash): - # Internal routine used by ll_dict_resize() to insert an item which is - # known to be absent from the dict. This routine also assumes that - # the dict contains no deleted entries. This routine has the advantage - # of never calling d.keyhash() and d.keyeq(), so it cannot call back - # to user code. ll_dict_insertclean() doesn't resize the dict, either. - i = ll_dict_lookup_clean(d, hash) - ENTRY = lltype.typeOf(d.entries).TO.OF - index = d.indexes[i] - entry = d.entries[index] - entry.value = value - entry.key = key - if hasattr(ENTRY, 'f_hash'): entry.f_hash = hash - d.num_items += 1 - d.resize_counter -= 3 - def ll_dict_delitem(d, key): i = ll_dict_lookup(d, key, d.keyhash(key)) if i & HIGHEST_BIT: @@ -489,14 +471,14 @@ new_item_size = new_size // 3 * 2 + 1 d.entries = lltype.typeOf(old_entries).TO.allocate(new_item_size) d.indexes = lltype.typeOf(d).TO.indexes.TO.allocate(new_size) - d.num_items = len(old_entries) - d.resize_counter = new_size * 2 + d.num_items = len(old_entries) - 1 + d.resize_counter = new_item_size i = 0 indexes = d.indexes while i < old_size: index = old_indexes[i] if index >= 0: - indexes[ll_dict_lookup_clean(d, old_entries.hash(i))] = index + indexes[ll_dict_lookup_clean(d, old_entries.hash(index))] = index i += 1 rgc.ll_arraycopy(old_entries, d.entries, 0, 0, len(old_entries)) ll_dict_resize.oopspec = 'dict.resize(d)' @@ -591,7 +573,7 @@ # Irregular operations. DICT_INITSIZE = 8 -DICT_ITEMS_INITSIZE = 6 +DICT_ITEMS_INITSIZE = 5 @jit.unroll_safe # we always unroll the small allocation def ll_newdict(DICT): @@ -603,7 +585,7 @@ d.indexes[i] = FREE d.entries = DICT.entries.TO.allocate(DICT_ITEMS_INITSIZE) d.num_items = 0 - d.resize_counter = DICT_INITSIZE * 2 + d.resize_counter = DICT_ITEMS_INITSIZE return d def ll_newdict_size(DICT, length_estimate): @@ -616,7 +598,7 @@ d.entries = DICT.entries.TO.allocate(items_size) d.indexes = DICT.indexes.TO.allocate(n) d.num_items = 0 - d.resize_counter = n * 2 + d.resize_counter = items_size return d # pypy.rpython.memory.lldict uses a dict based on Struct and Array @@ -755,13 +737,13 @@ def ll_clear(d): if (len(d.indexes) == DICT_INITSIZE and - d.resize_counter == DICT_INITSIZE * 2): + d.resize_counter == DICT_ITEMS_INITSIZE): return old_entries = d.entries d.entries = lltype.typeOf(old_entries).TO.allocate(DICT_ITEMS_INITSIZE) d.indexes = lltype.typeOf(d).TO.indexes.TO.allocate(DICT_INITSIZE) d.num_items = 0 - d.resize_counter = DICT_INITSIZE * 2 + d.resize_counter = DICT_ITEMS_INITSIZE ll_clear.oopspec = 'dict.clear(d)' def ll_update(dic1, dic2): diff --git a/pypy/rpython/test/test_rdict.py b/pypy/rpython/test/test_rdict.py --- a/pypy/rpython/test/test_rdict.py +++ b/pypy/rpython/test/test_rdict.py @@ -69,11 +69,13 @@ rdict.ll_dict_setitem(ll_d, llstr("a"), 1) rdict.ll_dict_setitem(ll_d, llstr("b"), 2) rdict.ll_dict_setitem(ll_d, llstr("c"), 3) - rdict.ll_dict_setitem(ll_d, llstr("d"), 4) + rdict.ll_dict_setitem(ll_d, llstr("d"), 4) + assert len(ll_d.indexes) == 8 rdict.ll_dict_setitem(ll_d, llstr("e"), 5) - assert len(ll_d.indexes) == 8 rdict.ll_dict_setitem(ll_d, llstr("f"), 6) assert len(ll_d.indexes) == 32 + for item in ['a', 'b', 'c', 'd', 'e', 'f']: + assert rdict.ll_dict_getitem(ll_d, llstr(item)) == ord(item) - ord('a') + 1 class BaseTestRdict(BaseRtypingTest): From noreply at buildbot.pypy.org Sun Jan 6 21:15:35 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 6 Jan 2013 21:15:35 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: fix iterators Message-ID: <20130106201535.EA1CF1C0312@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59815:1b1d4733a1fd Date: 2013-01-06 20:51 +0200 http://bitbucket.org/pypy/pypy/changeset/1b1d4733a1fd/ Log: fix iterators diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py --- a/pypy/rpython/lltypesystem/rdict.py +++ b/pypy/rpython/lltypesystem/rdict.py @@ -637,14 +637,17 @@ # # Iteration. +def get_ll_dictiter(DICT): + return lltype.Ptr(lltype.GcStruct('dictiter', + ('dict', DICT), + ('index', lltype.Signed))) + class DictIteratorRepr(AbstractDictIteratorRepr): def __init__(self, r_dict, variant="keys"): self.r_dict = r_dict self.variant = variant - self.lowleveltype = lltype.Ptr(lltype.GcStruct('dictiter', - ('dict', r_dict.lowleveltype), - ('index', lltype.Signed))) + self.lowleveltype = get_ll_dictiter(r_dict.lowleveltype) self.ll_dictiter = ll_dictiter self.ll_dictnext = ll_dictnext_group[variant] @@ -664,30 +667,23 @@ def ll_dictnext(RETURNTYPE, iter): # note that RETURNTYPE is None for keys and values dict = iter.dict - if dict: - entries = dict.entries - index = iter.index - entries_len = len(entries) - while index < entries_len: - entry = entries[index] - is_valid = entries.valid(index) - index = index + 1 - if is_valid: - iter.index = index - if RETURNTYPE is lltype.Void: - return None - elif kind == 'items': - r = lltype.malloc(RETURNTYPE.TO) - r.item0 = recast(RETURNTYPE.TO.item0, entry.key) - r.item1 = recast(RETURNTYPE.TO.item1, entry.value) - return r - elif kind == 'keys': - return entry.key - elif kind == 'values': - return entry.value + if not dict or iter.index >= dict.num_items: # clear the reference to the dict and prevent restarts iter.dict = lltype.nullptr(lltype.typeOf(iter).TO.dict.TO) - raise StopIteration + raise StopIteration + entry = dict.entries[iter.index] + iter.index += 1 + if RETURNTYPE is lltype.Void: + return None + elif kind == 'items': + r = lltype.malloc(RETURNTYPE.TO) + r.item0 = recast(RETURNTYPE.TO.item0, entry.key) + r.item1 = recast(RETURNTYPE.TO.item1, entry.value) + return r + elif kind == 'keys': + return entry.key + elif kind == 'values': + return entry.value return ll_dictnext ll_dictnext_group = {'keys' : _make_ll_dictnext('keys'), diff --git a/pypy/rpython/test/test_rdict.py b/pypy/rpython/test/test_rdict.py --- a/pypy/rpython/test/test_rdict.py +++ b/pypy/rpython/test/test_rdict.py @@ -77,6 +77,20 @@ for item in ['a', 'b', 'c', 'd', 'e', 'f']: assert rdict.ll_dict_getitem(ll_d, llstr(item)) == ord(item) - ord('a') + 1 + def test_dict_iteration(self): + DICT = self._get_str_dict() + ll_d = rdict.ll_newdict(DICT) + rdict.ll_dict_setitem(ll_d, llstr("k"), 1) + rdict.ll_dict_setitem(ll_d, llstr("j"), 2) + ITER = rdict.get_ll_dictiter(lltype.Ptr(DICT)) + ll_iter = rdict.ll_dictiter(ITER, ll_d) + ll_iterkeys = rdict.ll_dictnext_group['keys'] + next = ll_iterkeys(lltype.Signed, ll_iter) + assert hlstr(next) == "k" + next = ll_iterkeys(lltype.Signed, ll_iter) + assert hlstr(next) == "j" + py.test.raises(StopIteration, ll_iterkeys, lltype.Signed, ll_iter) + class BaseTestRdict(BaseRtypingTest): def test_dict_creation(self): From noreply at buildbot.pypy.org Sun Jan 6 21:15:37 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 6 Jan 2013 21:15:37 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: finish tests Message-ID: <20130106201537.2A9FE1C0312@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59816:290adf4d1f44 Date: 2013-01-06 22:14 +0200 http://bitbucket.org/pypy/pypy/changeset/290adf4d1f44/ Log: finish tests diff --git a/pypy/rpython/lltypesystem/llmemory.py b/pypy/rpython/lltypesystem/llmemory.py --- a/pypy/rpython/lltypesystem/llmemory.py +++ b/pypy/rpython/lltypesystem/llmemory.py @@ -90,6 +90,7 @@ try: endmarker = _end_markers[parent] except KeyError: + xxx endmarker = _endmarker_struct(A, parent=parent, parentindex=index) _end_markers[parent] = endmarker diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py --- a/pypy/rpython/lltypesystem/rdict.py +++ b/pypy/rpython/lltypesystem/rdict.py @@ -3,6 +3,7 @@ from pypy.rpython.rdict import (AbstractDictRepr, AbstractDictIteratorRepr, rtype_newdict) from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib import objectmodel, jit, rgc from pypy.rlib.debug import ll_assert from pypy.rlib.rarithmetic import r_uint, intmask, LONG_BIT @@ -154,6 +155,8 @@ ll_hash_function=self.key_repr.get_ll_hash_function(), ll_eq_function=self.key_repr.get_ll_eq_function(), get_custom_eq_hash=self.custom_eq_hash) + if self.custom_eq_hash is not None: + self.r_rdict_eqfn, self.r_rdict_hashfn = self.custom_eq_hash() def convert_const(self, dictobj): from pypy.rpython.lltypesystem import llmemory @@ -366,7 +369,8 @@ def ll_dict_setitem(d, key, value): hash = d.keyhash(key) i = ll_dict_lookup(d, key, hash) - return _ll_dict_setitem_lookup_done(d, key, value, hash, i) + res = _ll_dict_setitem_lookup_done(d, key, value, hash, i) + return res def _look_inside_setitem(d, key, value, hash, i): return jit.isvirtual(d) and jit.isconstant(key) @@ -392,6 +396,7 @@ entry = d.entries[index] rc = d.resize_counter - 1 ll_assert(rc > 0, "ll_dict_resize failed?") + ll_assert(index < len(d.entries), "invalid insert") d.resize_counter = rc d.indexes[i] = index entry.value = value @@ -419,14 +424,13 @@ @jit.look_inside_iff(lambda d, i: jit.isvirtual(d) and jit.isconstant(i)) def _ll_dict_del(d, i): index = d.indexes[i] - d.indexes[i] = DELETED - d.num_items -= 1 ENTRIES = lltype.typeOf(d.entries).TO ENTRY = ENTRIES.OF - if index != d.num_items: - old_entry = d.entries[d.num_items] + if index != d.num_items - 1: + old_entry = d.entries[d.num_items - 1] key = old_entry.key to_insert_i = ll_dict_lookup(d, key, d.keyhash(key)) + ll_assert(not to_insert_i & HIGHEST_BIT, "invalid entry") d.indexes[to_insert_i] = index # copy the value new_entry = d.entries[index] @@ -435,6 +439,8 @@ if hasattr(ENTRY, 'f_hash'): new_entry.f_hash = old_entry.f_hash # clear the key and the value if they are GC pointers + d.indexes[i] = DELETED + d.num_items -= 1 entry = d.entries[d.num_items] if ENTRIES.must_clear_key: entry.key = lltype.nullptr(ENTRY.key.TO) @@ -471,8 +477,7 @@ new_item_size = new_size // 3 * 2 + 1 d.entries = lltype.typeOf(old_entries).TO.allocate(new_item_size) d.indexes = lltype.typeOf(d).TO.indexes.TO.allocate(new_size) - d.num_items = len(old_entries) - 1 - d.resize_counter = new_item_size + d.resize_counter = new_item_size - d.num_items i = 0 indexes = d.indexes while i < old_size: @@ -480,7 +485,8 @@ if index >= 0: indexes[ll_dict_lookup_clean(d, old_entries.hash(index))] = index i += 1 - rgc.ll_arraycopy(old_entries, d.entries, 0, 0, len(old_entries)) + rgc.ll_arraycopy(old_entries, d.entries, 0, 0, min(len(old_entries), + len(d.entries))) ll_dict_resize.oopspec = 'dict.resize(d)' # ------- a port of CPython's dictobject.c's lookdict implementation ------- @@ -535,7 +541,7 @@ elif index >= 0: checkingkey = entries[index].key if checkingkey == key: - return index + return i if d.keyeq is not None and entries.hash(index) == hash: # correct hash, maybe the key is e.g. a different pointer to # an equal object @@ -710,24 +716,17 @@ return default def ll_copy(dict): - xxx DICT = lltype.typeOf(dict).TO dictsize = len(dict.entries) d = DICT.allocate() - d.entries = DICT.entries.TO.allocate(dictsize) + d.entries = lltype.malloc(DICT.entries.TO, dictsize) + d.indexes = DICT.indexes.TO.allocate(len(dict.indexes)) d.num_items = dict.num_items d.resize_counter = dict.resize_counter if hasattr(DICT, 'fnkeyeq'): d.fnkeyeq = dict.fnkeyeq if hasattr(DICT, 'fnkeyhash'): d.fnkeyhash = dict.fnkeyhash - i = 0 - while i < dictsize: - d_entry = d.entries[i] - entry = dict.entries[i] - ENTRY = lltype.typeOf(d.entries).TO.OF - d_entry.key = entry.key - d_entry.value = entry.value - if hasattr(ENTRY, 'f_hash'): d_entry.f_hash = entry.f_hash - i += 1 + rgc.ll_arraycopy(dict.indexes, d.indexes, 0, 0, len(dict.indexes)) + rgc.ll_arraycopy(dict.entries, d.entries, 0, 0, len(dict.entries)) return d ll_copy.oopspec = 'dict.copy(dict)' @@ -743,17 +742,15 @@ ll_clear.oopspec = 'dict.clear(d)' def ll_update(dic1, dic2): - xxx entries = dic2.entries - d2len = len(entries) + d2len = dic2.num_items i = 0 while i < d2len: - if entries.valid(i): - entry = entries[i] - hash = entries.hash(i) - key = entry.key - j = ll_dict_lookup(dic1, key, hash) - _ll_dict_setitem_lookup_done(dic1, key, entry.value, hash, j) + entry = entries[i] + hash = entries.hash(i) + key = entry.key + j = ll_dict_lookup(dic1, key, hash) + _ll_dict_setitem_lookup_done(dic1, key, entry.value, hash, j) i += 1 ll_update.oopspec = 'dict.update(dic1, dic2)' @@ -772,27 +769,23 @@ def ll_kvi(LIST, dic): res = LIST.ll_newlist(dic.num_items) entries = dic.entries - dlen = len(entries) + dlen = dic.num_items items = res.ll_items() i = 0 - p = 0 while i < dlen: - if entries.valid(i): - ELEM = lltype.typeOf(items).TO.OF - if ELEM is not lltype.Void: - entry = entries[i] - if kind == 'items': - r = lltype.malloc(ELEM.TO) - r.item0 = recast(ELEM.TO.item0, entry.key) - r.item1 = recast(ELEM.TO.item1, entry.value) - items[p] = r - elif kind == 'keys': - items[p] = recast(ELEM, entry.key) - elif kind == 'values': - items[p] = recast(ELEM, entry.value) - p += 1 + ELEM = lltype.typeOf(items).TO.OF + if ELEM is not lltype.Void: + entry = entries[i] + if kind == 'items': + r = lltype.malloc(ELEM.TO) + r.item0 = recast(ELEM.TO.item0, entry.key) + r.item1 = recast(ELEM.TO.item1, entry.value) + items[i] = r + elif kind == 'keys': + items[i] = recast(ELEM, entry.key) + elif kind == 'values': + items[i] = recast(ELEM, entry.value) i += 1 - assert p == res.ll_length() return res ll_kvi.oopspec = 'dict.%s(dic)' % kind return ll_kvi @@ -805,40 +798,14 @@ i = ll_dict_lookup(d, key, d.keyhash(key)) return not i & HIGHEST_BIT -POPITEMINDEX = lltype.Struct('PopItemIndex', ('nextindex', lltype.Signed)) -global_popitem_index = lltype.malloc(POPITEMINDEX, zero=True, immortal=True) - -def _ll_getnextitem(dic): - entries = dic.entries - ENTRY = lltype.typeOf(entries).TO.OF - dmask = len(entries) - 1 - if hasattr(ENTRY, 'f_hash'): - if entries.valid(0): - return 0 - base = entries[0].f_hash - else: - base = global_popitem_index.nextindex - counter = 0 - while counter <= dmask: - i = (base + counter) & dmask - counter += 1 - if entries.valid(i): - break - else: +def ll_popitem(ELEM, dic): + if dic.num_items == 0: raise KeyError - if hasattr(ENTRY, 'f_hash'): - entries[0].f_hash = base + counter - else: - global_popitem_index.nextindex = base + counter - return i - -def ll_popitem(ELEM, dic): - i = _ll_getnextitem(dic) - entry = dic.entries[i] + entry = dic.entries[dic.num_items - 1] r = lltype.malloc(ELEM.TO) r.item0 = recast(ELEM.TO.item0, entry.key) r.item1 = recast(ELEM.TO.item1, entry.value) - _ll_dict_del(dic, i) + ll_dict_delitem(dic, entry.key) return r def ll_pop(dic, key): diff --git a/pypy/rpython/test/test_rdict.py b/pypy/rpython/test/test_rdict.py --- a/pypy/rpython/test/test_rdict.py +++ b/pypy/rpython/test/test_rdict.py @@ -91,6 +91,33 @@ assert hlstr(next) == "j" py.test.raises(StopIteration, ll_iterkeys, lltype.Signed, ll_iter) + def test_popitem(self): + DICT = self._get_str_dict() + ll_d = rdict.ll_newdict(DICT) + rdict.ll_dict_setitem(ll_d, llstr("k"), 1) + rdict.ll_dict_setitem(ll_d, llstr("j"), 2) + ll_elem = rdict.ll_popitem(lltype.Ptr( + lltype.GcStruct('x', ('item0', lltype.Ptr(rstr.STR)), + ('item1', lltype.Signed))), ll_d) + assert hlstr(ll_elem.item0) == "j" + assert ll_elem.item1 == 2 + + def test_direct_enter_and_del(self): + def eq(a, b): + return a == b + + DICT = rdict.get_ll_dict(lltype.Signed, lltype.Signed, + ll_fasthash_function=intmask, + ll_hash_function=intmask, + ll_eq_function=eq) + ll_d = rdict.ll_newdict(DICT) + numbers = [i * rdict.DICT_INITSIZE + 1 for i in range(8)] + for num in numbers: + rdict.ll_dict_setitem(ll_d, num, 1) + rdict.ll_dict_delitem(ll_d, num) + for k in ll_d.indexes: + assert k < 0 + class BaseTestRdict(BaseRtypingTest): def test_dict_creation(self): @@ -258,12 +285,12 @@ res = self.interpret(f, ()) assert res == 2 - def f(): + def f2(): d = {} d.setdefault('a', 2) x = d.setdefault('a', -3) return x - res = self.interpret(f, ()) + res = self.interpret(f2, ()) assert res == 2 def test_dict_copy(self): @@ -802,8 +829,7 @@ return d res = self.interpret(func2, [ord(x), ord(y)]) - for i in range(len(res.entries)): - assert not (res.entries.everused(i) and not res.entries.valid(i)) + assert len([i for i in res.indexes if i >= 0]) == 2 def func3(c0, c1, c2, c3, c4, c5, c6, c7): d = {} @@ -822,8 +848,8 @@ res = self.interpret(func3, [ord(char_by_hash[i][0]) for i in range(rdict.DICT_INITSIZE)]) count_frees = 0 - for i in range(len(res.entries)): - if not res.entries.everused(i): + for i in res.indexes: + if i == -1: count_frees += 1 assert count_frees >= 3 @@ -844,9 +870,9 @@ del d[chr(ord('A') - i)] return d res = self.interpret(func, [0]) - assert len(res.entries) > rdict.DICT_INITSIZE + assert len(res.indexes) > rdict.DICT_INITSIZE res = self.interpret(func, [1]) - assert len(res.entries) == rdict.DICT_INITSIZE + assert len(res.indexes) == rdict.DICT_INITSIZE def test_dict_valid_resize(self): # see if we find our keys after resize @@ -864,87 +890,6 @@ # ____________________________________________________________ - def test_opt_nullkeymarker(self): - def f(): - d = {"hello": None} - d["world"] = None - return "hello" in d, d - res = self.interpret(f, []) - assert res.item0 == True - DICT = lltype.typeOf(res.item1).TO - assert not hasattr(DICT.entries.TO.OF, 'f_everused')# non-None string keys - assert not hasattr(DICT.entries.TO.OF, 'f_valid') # strings have a dummy - - def test_opt_nullvaluemarker(self): - def f(n): - d = {-5: "abcd"} - d[123] = "def" - return len(d[n]), d - res = self.interpret(f, [-5]) - assert res.item0 == 4 - DICT = lltype.typeOf(res.item1).TO - assert not hasattr(DICT.entries.TO.OF, 'f_everused')# non-None str values - assert not hasattr(DICT.entries.TO.OF, 'f_valid') # strs have a dummy - - def test_opt_nonullmarker(self): - class A: - pass - def f(n): - if n > 5: - a = A() - else: - a = None - d = {a: -5441} - d[A()] = n+9872 - return d[a], d - res = self.interpret(f, [-5]) - assert res.item0 == -5441 - DICT = lltype.typeOf(res.item1).TO - assert hasattr(DICT.entries.TO.OF, 'f_everused') # can-be-None A instances - assert not hasattr(DICT.entries.TO.OF, 'f_valid')# with a dummy A instance - - res = self.interpret(f, [6]) - assert res.item0 == -5441 - - def test_opt_nonnegint_dummy(self): - def f(n): - d = {n: 12} - d[-87] = 24 - del d[n] - return len(d.copy()), d[-87], d - res = self.interpret(f, [5]) - assert res.item0 == 1 - assert res.item1 == 24 - DICT = lltype.typeOf(res.item2).TO - assert hasattr(DICT.entries.TO.OF, 'f_everused') # all ints can be zero - assert not hasattr(DICT.entries.TO.OF, 'f_valid')# nonneg int: dummy -1 - - def test_opt_no_dummy(self): - def f(n): - d = {n: 12} - d[-87] = -24 - del d[n] - return len(d.copy()), d[-87], d - res = self.interpret(f, [5]) - assert res.item0 == 1 - assert res.item1 == -24 - DICT = lltype.typeOf(res.item2).TO - assert hasattr(DICT.entries.TO.OF, 'f_everused') # all ints can be zero - assert hasattr(DICT.entries.TO.OF, 'f_valid') # no dummy available - - def test_opt_boolean_has_no_dummy(self): - def f(n): - d = {n: True} - d[-87] = True - del d[n] - return len(d.copy()), d[-87], d - res = self.interpret(f, [5]) - assert res.item0 == 1 - assert res.item1 is True - DICT = lltype.typeOf(res.item2).TO - assert hasattr(DICT.entries.TO.OF, 'f_everused') # all ints can be zero - assert hasattr(DICT.entries.TO.OF, 'f_valid') # no dummy available - def test_opt_multiple_identical_dicts(self): def f(n): s = "x" * n @@ -1159,7 +1104,7 @@ DictValue(None, annmodel.SomeString(value_can_be_none))) dictrepr.setup() print dictrepr.lowleveltype - for key, value in dictrepr.DICTENTRY._adtmeths.items(): + for key, value in dictrepr.DICT.entries.TO._adtmeths.items(): print ' %s = %s' % (key, value) l_dict = rdict.ll_newdict(dictrepr.DICT) referencetable = [None] * 400 From noreply at buildbot.pypy.org Sun Jan 6 21:23:27 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 6 Jan 2013 21:23:27 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: fix? Message-ID: <20130106202327.B36431C0312@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59817:33f815dfb12b Date: 2013-01-06 22:22 +0200 http://bitbucket.org/pypy/pypy/changeset/33f815dfb12b/ Log: fix? diff --git a/pypy/module/cStringIO/interp_stringio.py b/pypy/module/cStringIO/interp_stringio.py --- a/pypy/module/cStringIO/interp_stringio.py +++ b/pypy/module/cStringIO/interp_stringio.py @@ -1,5 +1,5 @@ from pypy.interpreter.error import OperationError -from pypy.interpreter.baseobjspace import Wrappable +from pypy.interpreter.baseobjspace import Wrappable, W_Root from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.rlib.rStringIO import RStringIO @@ -165,6 +165,7 @@ self.seek(i) return ''.join(bigbuffer[p:i]) + @unwrap_spec(w_size=W_Root) def descr_truncate(self, w_size=None): self.check_closed() space = self.space From noreply at buildbot.pypy.org Sun Jan 6 21:31:33 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 6 Jan 2013 21:31:33 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: another fix Message-ID: <20130106203133.033EF1C0312@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59818:8f5b33f94dad Date: 2013-01-06 22:27 +0200 http://bitbucket.org/pypy/pypy/changeset/8f5b33f94dad/ Log: another fix diff --git a/pypy/module/cStringIO/interp_stringio.py b/pypy/module/cStringIO/interp_stringio.py --- a/pypy/module/cStringIO/interp_stringio.py +++ b/pypy/module/cStringIO/interp_stringio.py @@ -249,6 +249,7 @@ # ____________________________________________________________ + at unwrap_spec(w_string=W_Root) def StringIO(space, w_string=None): if space.is_none(w_string): return space.wrap(W_OutputType(space)) From noreply at buildbot.pypy.org Sun Jan 6 21:31:34 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 6 Jan 2013 21:31:34 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: no wait it was nonsense Message-ID: <20130106203134.470A11C0312@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59819:46a19e804d89 Date: 2013-01-06 22:31 +0200 http://bitbucket.org/pypy/pypy/changeset/46a19e804d89/ Log: no wait it was nonsense diff --git a/pypy/module/cStringIO/interp_stringio.py b/pypy/module/cStringIO/interp_stringio.py --- a/pypy/module/cStringIO/interp_stringio.py +++ b/pypy/module/cStringIO/interp_stringio.py @@ -165,7 +165,6 @@ self.seek(i) return ''.join(bigbuffer[p:i]) - @unwrap_spec(w_size=W_Root) def descr_truncate(self, w_size=None): self.check_closed() space = self.space @@ -249,7 +248,6 @@ # ____________________________________________________________ - at unwrap_spec(w_string=W_Root) def StringIO(space, w_string=None): if space.is_none(w_string): return space.wrap(W_OutputType(space)) From noreply at buildbot.pypy.org Sun Jan 6 21:32:17 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sun, 6 Jan 2013 21:32:17 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed test_cmath again Message-ID: <20130106203217.6CC141C0312@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59820:d717246d2abd Date: 2013-01-06 07:30 +0100 http://bitbucket.org/pypy/pypy/changeset/d717246d2abd/ Log: Fixed test_cmath again diff --git a/pypy/module/cmath/test/test_cmath.py b/pypy/module/cmath/test/test_cmath.py --- a/pypy/module/cmath/test/test_cmath.py +++ b/pypy/module/cmath/test/test_cmath.py @@ -186,7 +186,7 @@ import rpython # too fragile... - fname = os.path.join(os.path.dirname(rpython.rlib.test.test_rcomplex.__file__), 'rcomplex_testcases.txt') + fname = os.path.join(os.path.dirname(rpython.rlib.__file__), 'test', 'rcomplex_testcases.txt') for id, fn, ar, ai, er, ei, flags in parse_testfile(fname): arg = (ar, ai) expected = (er, ei) From noreply at buildbot.pypy.org Sun Jan 6 21:32:18 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sun, 6 Jan 2013 21:32:18 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Moved offset2lineno from pypy.interpreter.pytraceback to rpython.tool.error Message-ID: <20130106203218.D1FE31C0312@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59821:f54abb775eb1 Date: 2013-01-06 19:57 +0100 http://bitbucket.org/pypy/pypy/changeset/f54abb775eb1/ Log: Moved offset2lineno from pypy.interpreter.pytraceback to rpython.tool.error diff --git a/pypy/interpreter/pytraceback.py b/pypy/interpreter/pytraceback.py --- a/pypy/interpreter/pytraceback.py +++ b/pypy/interpreter/pytraceback.py @@ -1,5 +1,6 @@ from pypy.interpreter import baseobjspace from pypy.interpreter.error import OperationError +from rpython.tool.error import offset2lineno class PyTraceback(baseobjspace.Wrappable): @@ -55,17 +56,6 @@ tb = PyTraceback(space, frame, last_instruction, tb) operror.set_traceback(tb) -def offset2lineno(c, stopat): - tab = c.co_lnotab - line = c.co_firstlineno - addr = 0 - for i in range(0, len(tab), 2): - addr = addr + ord(tab[i]) - if addr > stopat: - break - line = line + ord(tab[i+1]) - return line - def check_traceback(space, w_tb, msg): from pypy.interpreter.typedef import PyTraceback tb = space.interpclass_w(w_tb) diff --git a/rpython/tool/error.py b/rpython/tool/error.py --- a/rpython/tool/error.py +++ b/rpython/tool/error.py @@ -14,8 +14,6 @@ SHOW_ANNOTATIONS = True SHOW_DEFAULT_LINES_OF_CODE = 0 -from pypy.interpreter.pytraceback import offset2lineno - def source_lines1(graph, block, operindex=None, offset=None, long=False, \ show_lines_of_code=SHOW_DEFAULT_LINES_OF_CODE): if block is not None: @@ -161,3 +159,15 @@ if use_pdb: pdb_plus_show = PdbPlusShow(t) pdb_plus_show.start(tb) + + +def offset2lineno(c, stopat): + tab = c.co_lnotab + line = c.co_firstlineno + addr = 0 + for i in range(0, len(tab), 2): + addr = addr + ord(tab[i]) + if addr > stopat: + break + line = line + ord(tab[i+1]) + return line \ No newline at end of file diff --git a/rpython/translator/tool/make_dot.py b/rpython/translator/tool/make_dot.py --- a/rpython/translator/tool/make_dot.py +++ b/rpython/translator/tool/make_dot.py @@ -4,7 +4,7 @@ from rpython.flowspace.objspace import FlowObjSpace as Space from rpython.tool.udir import udir from py.process import cmdexec -from pypy.interpreter.pytraceback import offset2lineno +from rpython.tool.error import offset2lineno class DotGen: From noreply at buildbot.pypy.org Sun Jan 6 21:32:20 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sun, 6 Jan 2013 21:32:20 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Moved pypy.tool.gcc_cache to rpython Message-ID: <20130106203220.248241C0312@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59822:02243944e04a Date: 2013-01-06 20:02 +0100 http://bitbucket.org/pypy/pypy/changeset/02243944e04a/ Log: Moved pypy.tool.gcc_cache to rpython diff --git a/rpython/rtyper/tool/rffi_platform.py b/rpython/rtyper/tool/rffi_platform.py --- a/rpython/rtyper/tool/rffi_platform.py +++ b/rpython/rtyper/tool/rffi_platform.py @@ -7,7 +7,7 @@ from rpython.rtyper.lltypesystem import lltype from rpython.rtyper.lltypesystem import rffi from rpython.rtyper.lltypesystem import llmemory -from pypy.tool.gcc_cache import build_executable_cache, try_compile_cache +from rpython.tool.gcc_cache import build_executable_cache, try_compile_cache from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.translator.platform import CompilationError from rpython.tool.udir import udir diff --git a/rpython/rtyper/tool/rfficache.py b/rpython/rtyper/tool/rfficache.py --- a/rpython/rtyper/tool/rfficache.py +++ b/rpython/rtyper/tool/rfficache.py @@ -8,7 +8,7 @@ from rpython.tool.udir import udir from rpython.rlib import rarithmetic from rpython.rtyper.lltypesystem import lltype -from pypy.tool.gcc_cache import build_executable_cache +from rpython.tool.gcc_cache import build_executable_cache def ask_gcc(question, add_source="", ignore_errors=False): from rpython.translator.platform import platform diff --git a/pypy/tool/gcc_cache.py b/rpython/tool/gcc_cache.py rename from pypy/tool/gcc_cache.py rename to rpython/tool/gcc_cache.py diff --git a/pypy/tool/test/test_gcc_cache.py b/rpython/tool/test/test_gcc_cache.py rename from pypy/tool/test/test_gcc_cache.py rename to rpython/tool/test/test_gcc_cache.py --- a/pypy/tool/test/test_gcc_cache.py +++ b/rpython/tool/test/test_gcc_cache.py @@ -1,5 +1,5 @@ import sys -from pypy.tool.gcc_cache import * +from rpython.tool.gcc_cache import * from rpython.tool.udir import udir import md5, cStringIO from rpython.translator.tool.cbuild import ExternalCompilationInfo From noreply at buildbot.pypy.org Sun Jan 6 21:32:21 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sun, 6 Jan 2013 21:32:21 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Moved numpy and pypy target tests to pypy Message-ID: <20130106203221.49C9E1C0312@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59823:cc7885d4993a Date: 2013-01-06 20:46 +0100 http://bitbucket.org/pypy/pypy/changeset/cc7885d4993a/ Log: Moved numpy and pypy target tests to pypy diff --git a/rpython/translator/goal/targetnumpystandalone.py b/goal/targetnumpystandalone.py rename from rpython/translator/goal/targetnumpystandalone.py rename to goal/targetnumpystandalone.py diff --git a/rpython/translator/goal/test2/__init__.py b/goal/test/__init__.py rename from rpython/translator/goal/test2/__init__.py rename to goal/test/__init__.py diff --git a/rpython/translator/goal/test2/mymodule.py b/goal/test/mymodule.py rename from rpython/translator/goal/test2/mymodule.py rename to goal/test/mymodule.py diff --git a/rpython/translator/goal/test2/test_app_main.py b/goal/test/test_app_main.py rename from rpython/translator/goal/test2/test_app_main.py rename to goal/test/test_app_main.py diff --git a/rpython/translator/goal/test2/test_targetpypy.py b/goal/test/test_targetpypy.py rename from rpython/translator/goal/test2/test_targetpypy.py rename to goal/test/test_targetpypy.py From noreply at buildbot.pypy.org Sun Jan 6 21:32:22 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sun, 6 Jan 2013 21:32:22 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Moved pypy.tool.compat to rpython Message-ID: <20130106203222.6A1451C0312@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59824:4fb5e0824537 Date: 2013-01-06 20:51 +0100 http://bitbucket.org/pypy/pypy/changeset/4fb5e0824537/ Log: Moved pypy.tool.compat to rpython diff --git a/pypy/tool/compat.py b/rpython/tool/compat.py rename from pypy/tool/compat.py rename to rpython/tool/compat.py diff --git a/rpython/tool/gcc_cache.py b/rpython/tool/gcc_cache.py --- a/rpython/tool/gcc_cache.py +++ b/rpython/tool/gcc_cache.py @@ -1,7 +1,7 @@ from pypy.conftest import pypydir from rpython.translator.platform import CompilationError from rpython.translator.tool.cbuild import ExternalCompilationInfo -from pypy.tool.compat import md5 +from rpython.tool.compat import md5 import py cache_dir_root = py.path.local(pypydir).join('_cache').ensure(dir=1) diff --git a/rpython/translator/backendopt/stat.py b/rpython/translator/backendopt/stat.py --- a/rpython/translator/backendopt/stat.py +++ b/rpython/translator/backendopt/stat.py @@ -1,5 +1,5 @@ from rpython.translator.simplify import get_graph -from pypy.tool.compat import md5 +from rpython.tool.compat import md5 def get_statistics(graph, translator, save_per_graph_details=None, ignore_stack_checks=False): seen_graphs = {} diff --git a/rpython/translator/backendopt/support.py b/rpython/translator/backendopt/support.py --- a/rpython/translator/backendopt/support.py +++ b/rpython/translator/backendopt/support.py @@ -116,7 +116,7 @@ return loop def md5digest(translator): - from pypy.tool.compat import md5 + from rpython.tool.compat import md5 graph2digest = {} for graph in translator.graphs: m = md5() From noreply at buildbot.pypy.org Sun Jan 6 21:32:23 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sun, 6 Jan 2013 21:32:23 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: goal/test -> goal/test2 Message-ID: <20130106203223.8C6591C0312@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59825:4c3b77b41d62 Date: 2013-01-06 20:52 +0100 http://bitbucket.org/pypy/pypy/changeset/4c3b77b41d62/ Log: goal/test -> goal/test2 diff --git a/goal/test/__init__.py b/goal/test2/__init__.py rename from goal/test/__init__.py rename to goal/test2/__init__.py diff --git a/goal/test/mymodule.py b/goal/test2/mymodule.py rename from goal/test/mymodule.py rename to goal/test2/mymodule.py diff --git a/goal/test/test_app_main.py b/goal/test2/test_app_main.py rename from goal/test/test_app_main.py rename to goal/test2/test_app_main.py diff --git a/goal/test/test_targetpypy.py b/goal/test2/test_targetpypy.py rename from goal/test/test_targetpypy.py rename to goal/test2/test_targetpypy.py From noreply at buildbot.pypy.org Sun Jan 6 21:32:24 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sun, 6 Jan 2013 21:32:24 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Renamed pypy_interact to pypy.sandbox.pypysandbox. Removed pypydir import from cbuild Message-ID: <20130106203224.C45211C0312@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59826:dbc4e7b736ec Date: 2013-01-06 21:30 +0100 http://bitbucket.org/pypy/pypy/changeset/dbc4e7b736ec/ Log: Renamed pypy_interact to pypy.sandbox.pypysandbox. Removed pypydir import from cbuild diff --git a/pypy/bin/pypysandbox b/pypy/bin/pypysandbox new file mode 100644 --- /dev/null +++ b/pypy/bin/pypysandbox @@ -0,0 +1,5 @@ +#!/usr/bin/env pypy + +from pypy.sandbox import pypysandbox + +pypysandbox.main() \ No newline at end of file diff --git a/pypy/sandbox/__init__.py b/pypy/sandbox/__init__.py new file mode 100644 --- /dev/null +++ b/pypy/sandbox/__init__.py @@ -0,0 +1,1 @@ +# empty \ No newline at end of file diff --git a/rpython/translator/sandbox/pypy_interact.py b/pypy/sandbox/pypysandbox.py rename from rpython/translator/sandbox/pypy_interact.py rename to pypy/sandbox/pypysandbox.py --- a/rpython/translator/sandbox/pypy_interact.py +++ b/pypy/sandbox/pypysandbox.py @@ -64,8 +64,7 @@ 'tmp': tmpdirnode, }) - -if __name__ == '__main__': +def main(): from getopt import getopt # and not gnu_getopt! options, arguments = getopt(sys.argv[1:], 't:hv', ['tmp=', 'heapsize=', 'timeout=', 'log=', @@ -125,3 +124,6 @@ sandproc.interact() finally: sandproc.kill() + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/rpython/translator/sandbox/test/test_pypy_interact.py b/pypy/sandbox/test/test_pypysandbox.py rename from rpython/translator/sandbox/test/test_pypy_interact.py rename to pypy/sandbox/test/test_pypysandbox.py --- a/rpython/translator/sandbox/test/test_pypy_interact.py +++ b/pypy/sandbox/test/test_pypysandbox.py @@ -1,5 +1,5 @@ import os, sys, stat, errno -from rpython.translator.sandbox.pypy_interact import PyPySandboxedProc +from pypy.sandbox.pypysandbox import PyPySandboxedProc from rpython.translator.interactive import Translation from pypy.module.sys.version import CPYTHON_VERSION diff --git a/rpython/translator/tool/cbuild.py b/rpython/translator/tool/cbuild.py --- a/rpython/translator/tool/cbuild.py +++ b/rpython/translator/tool/cbuild.py @@ -1,7 +1,6 @@ import py import sys -from pypy.conftest import pypydir from rpython.translator.platform import host from rpython.tool.udir import udir From noreply at buildbot.pypy.org Mon Jan 7 00:19:52 2013 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 7 Jan 2013 00:19:52 +0100 (CET) Subject: [pypy-commit] cffi default: Code simplification: instead of allocating & freeing some strings around Message-ID: <20130106231952.CE1CD1C0F8A@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1116:ec0d6d9b042a Date: 2013-01-06 23:38 +0100 http://bitbucket.org/cffi/cffi/changeset/ec0d6d9b042a/ Log: Code simplification: instead of allocating & freeing some strings around the calls in some cases, just use alloca(). diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -1914,17 +1914,20 @@ return ct_int; } -static int +static Py_ssize_t _prepare_pointer_call_argument(CTypeDescrObject *ctptr, PyObject *init, char **output_data) { /* 'ctptr' is here a pointer type 'ITEM *'. Accept as argument an initializer for an array 'ITEM[]'. This includes the case of - passing a Python byte string to a 'char *' argument. */ + passing a Python byte string to a 'char *' argument. + + This function returns -1 if an error occurred, + 0 if conversion succeeded (into *output_data), + or N > 0 if conversion would require N bytes of storage. + */ Py_ssize_t length, datasize; CTypeDescrObject *ctitem = ctptr->ct_itemdescr; - PyObject *result; - char *data; /* XXX some code duplication, how to avoid it? */ if (PyBytes_Check(init)) { @@ -1933,11 +1936,11 @@ if ((ctptr->ct_flags & CT_CAST_ANYTHING) || ((ctitem->ct_flags & (CT_PRIMITIVE_SIGNED|CT_PRIMITIVE_UNSIGNED)) && (ctitem->ct_size == sizeof(char)))) { - output_data[0] = PyBytes_AS_STRING(init); - return 1; + *output_data = PyBytes_AS_STRING(init); + return 0; } else - return 0; + goto convert_default; } else if (PyList_Check(init) || PyTuple_Check(init)) { length = PySequence_Fast_GET_SIZE(init); @@ -1947,39 +1950,31 @@ length = _my_PyUnicode_SizeAsWideChar(init) + 1; } else if ((ctitem->ct_flags & CT_IS_FILE) && PyFile_Check(init)) { - output_data[0] = (char *)PyFile_AsFile(init); - if (output_data[0] == NULL && PyErr_Occurred()) + *output_data = (char *)PyFile_AsFile(init); + if (*output_data == NULL && PyErr_Occurred()) return -1; - return 1; + return 0; } else { /* refuse to receive just an integer (and interpret it as the array size) */ - return 0; + goto convert_default; } if (ctitem->ct_size <= 0) - return 0; + goto convert_default; datasize = length * ctitem->ct_size; if ((datasize / ctitem->ct_size) != length) { PyErr_SetString(PyExc_OverflowError, "array size would overflow a Py_ssize_t"); return -1; } - - result = PyBytes_FromStringAndSize(NULL, datasize); - if (result == NULL) - return -1; - - data = PyBytes_AS_STRING(result); - memset(data, 0, datasize); - if (convert_array_from_object(data, ctptr, init) < 0) { - Py_DECREF(result); - return -1; - } - output_data[0] = data; - output_data[1] = (char *)result; - return 1; + if (datasize <= 0) + datasize = 1; + return datasize; + + convert_default: + return convert_from_object((char *)output_data, ctptr, init); } static PyObject* @@ -1988,7 +1983,7 @@ char *buffer; void** buffer_array; cif_description_t *cif_descr; - Py_ssize_t i, nargs, nargs_declared, free_me_until = 0; + Py_ssize_t i, nargs, nargs_declared; PyObject *signature, *res = NULL, *fvarargs; CTypeDescrObject *fresult; char *resultdata; @@ -2098,22 +2093,23 @@ else argtype = (CTypeDescrObject *)PyTuple_GET_ITEM(fvarargs, i); - if (argtype->ct_flags & CT_POINTER) { - ((char **)data)[1] = NULL; - - if (!CData_Check(obj)) { - int res = _prepare_pointer_call_argument(argtype, obj, - (char **)data); - if (res != 0) { - if (res < 0) - goto error; - assert(i < nargs_declared); /* otherwise, obj is a CData */ - free_me_until = i + 1; - continue; - } + if ((argtype->ct_flags & CT_POINTER) && !CData_Check(obj)) { + char *tmpbuf; + Py_ssize_t datasize = _prepare_pointer_call_argument( + argtype, obj, (char **)data); + if (datasize == 0) + ; /* successfully filled '*data' */ + else if (datasize < 0) + goto error; + else { + tmpbuf = alloca(datasize); + memset(tmpbuf, 0, datasize); + *(char **)data = tmpbuf; + if (convert_array_from_object(tmpbuf, argtype, obj) < 0) + goto error; } } - if (convert_from_object(data, argtype, obj) < 0) + else if (convert_from_object(data, argtype, obj) < 0) goto error; } @@ -2151,15 +2147,6 @@ /* fall-through */ error: - for (i=0; ict_flags & CT_POINTER) { - char *data = buffer + cif_descr->exchange_offset_arg[1 + i]; - PyObject *tmpobj_or_null = (PyObject *)(((char **)data)[1]); - Py_XDECREF(tmpobj_or_null); - } - } if (buffer) PyObject_Free(buffer); if (fvarargs != NULL) { @@ -3667,15 +3654,6 @@ exchange_offset = ALIGN_ARG(exchange_offset); cif_descr->exchange_offset_arg[1 + i] = exchange_offset; exchange_offset += atype->size; - /* if 'farg' is a pointer type 'ITEM *', then we might receive - as argument to the function call what is an initializer - for an array 'ITEM[]'. This includes the case of passing a - Python string to a 'char *' argument. In this case, we - convert the initializer to a cdata 'ITEM[]' that gets - temporarily stored here: */ - if (farg->ct_flags & CT_POINTER) { - exchange_offset += sizeof(PyObject *); - } } } From noreply at buildbot.pypy.org Mon Jan 7 00:19:53 2013 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 7 Jan 2013 00:19:53 +0100 (CET) Subject: [pypy-commit] cffi default: Test and fix for issue #51: unlike what is documented, we can't pass a Message-ID: <20130106231953.F00861C10DC@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1117:05e4ecdd2943 Date: 2013-01-07 00:19 +0100 http://bitbucket.org/cffi/cffi/changeset/05e4ecdd2943/ Log: Test and fix for issue #51: unlike what is documented, we can't pass a list argument to a function expecting a "foo *" argument. It doesn't work on CPython (but works with vengine_gen or on PyPy). diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -1927,8 +1927,12 @@ or N > 0 if conversion would require N bytes of storage. */ Py_ssize_t length, datasize; - CTypeDescrObject *ctitem = ctptr->ct_itemdescr; - + CTypeDescrObject *ctitem; + + if (CData_Check(init)) + goto convert_default; + + ctitem = ctptr->ct_itemdescr; /* XXX some code duplication, how to avoid it? */ if (PyBytes_Check(init)) { /* from a string: just returning the string here is fine. @@ -2093,7 +2097,7 @@ else argtype = (CTypeDescrObject *)PyTuple_GET_ITEM(fvarargs, i); - if ((argtype->ct_flags & CT_POINTER) && !CData_Check(obj)) { + if (argtype->ct_flags & CT_POINTER) { char *tmpbuf; Py_ssize_t datasize = _prepare_pointer_call_argument( argtype, obj, (char **)data); @@ -4753,18 +4757,6 @@ /************************************************************/ /* Functions used by '_cffi_N.so', the generated modules */ -static char *_cffi_to_c_char_p(PyObject *obj) -{ - if (PyBytes_Check(obj)) { - return PyBytes_AS_STRING(obj); - } - if (CData_Check(obj)) { - return ((CDataObject *)obj)->c_data; - } - _convert_error(obj, "char * or void *", "compatible pointer"); - return NULL; -} - #define _cffi_to_c_SIGNED_FN(RETURNTYPE, SIZE) \ static RETURNTYPE _cffi_to_c_i##SIZE(PyObject *obj) { \ PY_LONG_LONG tmp = _my_PyLong_AsLongLong(obj); \ @@ -4877,7 +4869,7 @@ #endif static void *cffi_exports[] = { - _cffi_to_c_char_p, + 0, _cffi_to_c_i8, _cffi_to_c_u8, _cffi_to_c_i16, @@ -4905,6 +4897,8 @@ #endif _cffi_to_c_long_double, _cffi_to_c__Bool, + _prepare_pointer_call_argument, + convert_array_from_object, }; /************************************************************/ diff --git a/cffi/vengine_cpy.py b/cffi/vengine_cpy.py --- a/cffi/vengine_cpy.py +++ b/cffi/vengine_cpy.py @@ -201,14 +201,9 @@ errvalue = '-1' # elif isinstance(tp, model.PointerType): - if ((isinstance(tp.totype, model.PrimitiveType) and - tp.totype.name == 'char') or - isinstance(tp.totype, model.VoidType)): - converter = '_cffi_to_c_char_p' - else: - converter = '(%s)_cffi_to_c_pointer' % tp.get_c_name('') - extraarg = ', _cffi_type(%d)' % self._gettypenum(tp) - errvalue = 'NULL' + self._convert_funcarg_to_c_ptr_or_array(tp, fromvar, + tovar, errcode) + return # elif isinstance(tp, (model.StructOrUnion, model.EnumType)): # a struct (not a struct pointer) as a function argument @@ -230,6 +225,25 @@ tovar, tp.get_c_name(''), errvalue)) self._prnt(' %s;' % errcode) + def _extra_local_variables(self, tp, localvars): + if isinstance(tp, model.PointerType): + localvars.add('Py_ssize_t datasize') + + def _convert_funcarg_to_c_ptr_or_array(self, tp, fromvar, tovar, errcode): + self._prnt(' datasize = _cffi_prepare_pointer_call_argument(') + self._prnt(' _cffi_type(%d), %s, (char **)&%s);' % ( + self._gettypenum(tp), fromvar, tovar)) + self._prnt(' if (datasize != 0) {') + self._prnt(' if (datasize < 0)') + self._prnt(' %s;' % errcode) + self._prnt(' %s = alloca(datasize);' % (tovar,)) + self._prnt(' memset((void *)%s, 0, datasize);' % (tovar,)) + self._prnt(' if (_cffi_convert_array_from_object(' + '(char *)%s, _cffi_type(%d), %s) < 0)' % ( + tovar, self._gettypenum(tp), fromvar)) + self._prnt(' %s;' % errcode) + self._prnt(' }') + def _convert_expr_from_c(self, tp, var, context): if isinstance(tp, model.PrimitiveType): if tp.is_integer_type(): @@ -304,6 +318,13 @@ context = 'argument of %s' % name for i, type in enumerate(tp.args): prnt(' %s;' % type.get_c_name(' x%d' % i, context)) + # + localvars = set() + for type in tp.args: + self._extra_local_variables(type, localvars) + for decl in localvars: + prnt(' %s;' % (decl,)) + # if not isinstance(tp.result, model.VoidType): result_code = 'result = ' context = 'result of %s' % name @@ -734,8 +755,6 @@ sizeof(type) == 8 ? _cffi_to_c_u64(o) : \ (Py_FatalError("unsupported size for type " #type), 0)) -#define _cffi_to_c_char_p \ - ((char *(*)(PyObject *))_cffi_exports[0]) #define _cffi_to_c_i8 \ ((int(*)(PyObject *))_cffi_exports[1]) #define _cffi_to_c_u8 \ @@ -780,7 +799,11 @@ ((long double(*)(PyObject *))_cffi_exports[21]) #define _cffi_to_c__Bool \ ((_Bool(*)(PyObject *))_cffi_exports[22]) -#define _CFFI_NUM_EXPORTS 23 +#define _cffi_prepare_pointer_call_argument \ + ((Py_ssize_t(*)(CTypeDescrObject *, PyObject *, char **))_cffi_exports[23]) +#define _cffi_convert_array_from_object \ + ((int(*)(char *, CTypeDescrObject *, PyObject *))_cffi_exports[24]) +#define _CFFI_NUM_EXPORTS 25 typedef struct _ctypedescr CTypeDescrObject; diff --git a/testing/test_verify.py b/testing/test_verify.py --- a/testing/test_verify.py +++ b/testing/test_verify.py @@ -1481,3 +1481,23 @@ res = lib.some_c_function(lib.c_callback) assert res == 84 assert seen == [[10, 20], [30, 40, 50]] + +def test_floatstar_argument(): + ffi = FFI() + ffi.cdef("float sum3floats(float *);") + lib = ffi.verify(""" + float sum3floats(float *f) { + return f[0] + f[1] + f[2]; + } + """) + assert lib.sum3floats((1.5, 2.5, 3.5)) == 7.5 + +def test_charstar_argument(): + ffi = FFI() + ffi.cdef("char sum3chars(char *);") + lib = ffi.verify(""" + char sum3chars(char *f) { + return f[0] + f[1] + f[2]; + } + """) + assert lib.sum3chars(('\x10', '\x20', '\x30')) == '\x60' From noreply at buildbot.pypy.org Mon Jan 7 00:22:02 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 7 Jan 2013 00:22:02 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: in-progress of rweakkeydict Message-ID: <20130106232202.C94851C0F8A@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59827:1fb600db1932 Date: 2013-01-07 01:20 +0200 http://bitbucket.org/pypy/pypy/changeset/1fb600db1932/ Log: in-progress of rweakkeydict diff --git a/pypy/rlib/_rweakkeydict.py b/pypy/rlib/_rweakkeydict.py --- a/pypy/rlib/_rweakkeydict.py +++ b/pypy/rlib/_rweakkeydict.py @@ -1,13 +1,13 @@ from pypy.objspace.flow.model import Constant from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rdict +from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.lltypesystem.llmemory import weakref_create, weakref_deref -from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.rclass import getinstancerepr from pypy.rpython.rmodel import Repr from pypy.rlib.rweakref import RWeakKeyDictionary from pypy.rlib import jit +from pypy.rlib.debug import ll_assert from pypy.rlib.objectmodel import compute_identity_hash -from pypy.rlib.objectmodel import we_are_translated # Warning: this implementation of RWeakKeyDictionary is not exactly @@ -25,7 +25,7 @@ def convert_const(self, weakdict): if not isinstance(weakdict, RWeakKeyDictionary): - raise TyperError("expected an RWeakKeyDictionary: %r" % ( + raise TypeError("expected an RWeakKeyDictionary: %r" % ( weakdict,)) try: key = Constant(weakdict) @@ -33,7 +33,7 @@ except KeyError: self.setup() if weakdict.length() != 0: - raise TyperError("got a non-empty prebuilt RWeakKeyDictionary") + raise TypeError("got a non-empty prebuilt RWeakKeyDictionary") l_dict = ll_new_weakdict() self.dict_cache[key] = l_dict return l_dict @@ -83,8 +83,10 @@ h = 0 return '<%x>' % (h,) -def ll_valid(entries, i): - key = entries[i].key +def ll_valid(entries, index): + if index < 0: + return False + key = entries[index].key if not key: return False elif weakref_deref(rclass.OBJECTPTR, key): @@ -93,19 +95,16 @@ # The entry might be a dead weakref still holding a strong # reference to the value; for this case, we clear the old # value from the entry, if any. - entries[i].value = NULLVALUE + entries[index].value = NULLVALUE return False -def ll_everused(entries, i): - return bool(entries[i].key) - entrymeths = { 'allocate': lltype.typeMethod(rdict._ll_malloc_entries), - 'delete': rdict._ll_free_entries, - 'valid': ll_valid, - 'everused': ll_everused, 'hash': rdict.ll_hash_from_cache, 'no_direct_compare': True, + 'clear_key': lambda : llmemory.dead_wref, + 'clear_value': lambda : lltype.nullptr(rclass.OBJECTPTR.TO), + 'valid': ll_valid, } WEAKDICTENTRYARRAY = lltype.GcArray(WEAKDICTENTRY, adtmeths=entrymeths, @@ -115,22 +114,30 @@ @jit.dont_look_inside def ll_new_weakdict(): d = lltype.malloc(WEAKDICT) - d.entries = WEAKDICT.entries.TO.allocate(rdict.DICT_INITSIZE) + d.entries = WEAKDICT.entries.TO.allocate(rdict.DICT_ITEMS_INITSIZE) + d.indexes = WEAKDICT.indexes.TO.allocate(rdict.DICT_INITSIZE) d.num_items = 0 - d.resize_counter = rdict.DICT_INITSIZE * 2 + d.resize_counter = rdict.DICT_ITEMS_INITSIZE return d @jit.dont_look_inside def ll_get(d, llkey): hash = compute_identity_hash(llkey) + llop.debug_print(lltype.Void, "computed key", ll_debugrepr(llkey), + hex(hash)) i = rdict.ll_dict_lookup(d, llkey, hash) & rdict.MASK - #llop.debug_print(lltype.Void, i, 'get', hex(hash), - # ll_debugrepr(d.entries[i].key), - # ll_debugrepr(d.entries[i].value)) + index = d.indexes[i] + if index < 0: + llop.debug_print(lltype.Void, i, 'get', hex(hash), "null") + return NULLVALUE + llop.debug_print(lltype.Void, i, "getting", index) + llop.debug_print(lltype.Void, i, 'get', hex(hash), + ll_debugrepr(d.entries[index].key), + ll_debugrepr(d.entries[index].value)) # NB. ll_valid() above was just called at least on entry i, so if # it is an invalid entry with a dead weakref, the value was reset # to NULLVALUE. - return d.entries[i].value + return d.entries[index].value @jit.dont_look_inside def ll_set(d, llkey, llvalue): @@ -144,43 +151,77 @@ hash = compute_identity_hash(llkey) keyref = weakref_create(llkey) # GC effects here, before the rest i = rdict.ll_dict_lookup(d, llkey, hash) & rdict.MASK - everused = d.entries.everused(i) - d.entries[i].key = keyref - d.entries[i].value = llvalue - d.entries[i].f_hash = hash - #llop.debug_print(lltype.Void, i, 'stored', hex(hash), - # ll_debugrepr(llkey), - # ll_debugrepr(llvalue)) + index = d.indexes[i] + everused = index != rdict.FREE + if index < 0: + index = d.num_items + d.indexes[i] = index + d.num_items += 1 + d.entries[index].key = keyref + d.entries[index].value = llvalue + d.entries[index].f_hash = hash + llop.debug_print(lltype.Void, i, 'stored', index, d.num_items, hex(hash), + ll_debugrepr(llkey), + ll_debugrepr(llvalue)) if not everused: - d.resize_counter -= 3 + d.resize_counter -= 1 if d.resize_counter <= 0: - #llop.debug_print(lltype.Void, 'RESIZE') + llop.debug_print(lltype.Void, 'RESIZE') ll_weakdict_resize(d) @jit.dont_look_inside def ll_set_null(d, llkey): hash = compute_identity_hash(llkey) i = rdict.ll_dict_lookup(d, llkey, hash) & rdict.MASK - if d.entries.everused(i): - # If the entry was ever used, clean up its key and value. - # We don't store a NULL value, but a dead weakref, because - # the entry must still be marked as everused(). - d.entries[i].key = llmemory.dead_wref - d.entries[i].value = NULLVALUE - #llop.debug_print(lltype.Void, i, 'zero') - -def ll_update_num_items(d): - entries = d.entries - num_items = 0 - for i in range(len(entries)): - if entries.valid(i): - num_items += 1 - d.num_items = num_items + index = d.indexes[i] + if d.entries.valid(index): + d.entries[index].key = llmemory.dead_wref + d.entries[index].value = NULLVALUE + llop.debug_print(lltype.Void, i, index, 'zero') def ll_weakdict_resize(d): - # first set num_items to its correct, up-to-date value - ll_update_num_items(d) - rdict.ll_dict_resize(d) + llop.debug_print(lltype.Void, "weakdict resize") + old_entries = d.entries + old_indexes = d.indexes + old_size = len(old_indexes) + # make a 'new_size' estimate and shrink it if there are many + # deleted entry markers. See CPython for why it is a good idea to + # quadruple the dictionary size as long as it's not too big. + # count the number of valid entries + i = 0 + num_items = 0 + while i < d.num_items: + if old_entries.valid(i): + num_items += 1 + i += 1 + if num_items > 50000: new_estimate = (num_items + 1) * 2 + else: new_estimate = (num_items + 1) * 4 + new_size = rdict.DICT_INITSIZE + while new_size <= new_estimate: + new_size *= 2 + # + new_item_size = new_size // 3 * 2 + 1 + d.entries = lltype.typeOf(old_entries).TO.allocate(new_item_size) + d.indexes = lltype.typeOf(d).TO.indexes.TO.allocate(new_size) + d.resize_counter = new_item_size - num_items + i = 0 + indexes = d.indexes + j = 0 + while i < old_size: + index = old_indexes[i] + if old_entries.valid(index): + hash = old_entries.hash(index) + lookup_i = rdict.ll_dict_lookup_clean(d, hash) + indexes[lookup_i] = j + llop.debug_print(lltype.Void, "inserting", hex(hash), i, + "to", lookup_i, index, "=>", j) + llop.debug_print(lltype.Void, hex(old_entries[index].f_hash)) + d.entries[j].key = old_entries[index].key + d.entries[j].value = old_entries[index].value + d.entries[j].f_hash = old_entries[index].f_hash + j += 1 + i += 1 + d.num_items = j def ll_keyeq(d, weakkey1, realkey2): # only called by ll_dict_lookup() with the first arg coming from an @@ -188,13 +229,15 @@ if not weakkey1: assert bool(realkey2) return False - return weakref_deref(rclass.OBJECTPTR, weakkey1) == realkey2 + realkey1 = weakref_deref(rclass.OBJECTPTR, weakkey1) + llop.debug_print(lltype.Void, "comparison", realkey1, realkey2) + return realkey1 == realkey2 @jit.dont_look_inside def ll_length(d): # xxx slow, but it's only for debugging - ll_update_num_items(d) - #llop.debug_print(lltype.Void, 'length:', d.num_items) + d.resize() + llop.debug_print(lltype.Void, 'length:', d.num_items) return d.num_items dictmeths = { @@ -202,10 +245,15 @@ 'll_set': ll_set, 'keyeq': ll_keyeq, 'paranoia': False, + 'resize': ll_weakdict_resize, } +INDEXESARRAY = lltype.GcArray(lltype.Signed, + adtmeths={'allocate' : lltype.typeMethod(rdict._ll_malloc_indexes)}) + WEAKDICT = lltype.GcStruct("weakkeydict", ("num_items", lltype.Signed), ("resize_counter", lltype.Signed), + ("indexes", lltype.Ptr(INDEXESARRAY)), ("entries", lltype.Ptr(WEAKDICTENTRYARRAY)), adtmeths=dictmeths) diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py --- a/pypy/rlib/objectmodel.py +++ b/pypy/rlib/objectmodel.py @@ -410,6 +410,8 @@ lltypesystem. """ assert x is not None + if hasattr(x, '_obj'): + return compute_identity_hash(x._obj) # hack hack hack result = object.__hash__(x) try: x.__dict__['__precomputed_identity_hash'] = result diff --git a/pypy/rlib/rweakref.py b/pypy/rlib/rweakref.py --- a/pypy/rlib/rweakref.py +++ b/pypy/rlib/rweakref.py @@ -68,7 +68,6 @@ from pypy.rpython import extregistry from pypy.annotation import model as annmodel -from pypy.annotation.bookkeeper import getbookkeeper from pypy.tool.pairtype import pairtype class Entry(extregistry.ExtRegistryEntry): @@ -175,9 +174,9 @@ class __extend__(pairtype(SomeWeakKeyDict, SomeWeakKeyDict)): def union((s_wkd1, s_wkd2)): if s_wkd1.keyclassdef is not s_wkd2.keyclassdef: - return SomeObject() # not the same key class! complain... + return annmodel.SomeObject() # not the same key class! complain... if s_wkd1.valueclassdef is not s_wkd2.valueclassdef: - return SomeObject() # not the same value class! complain... + return annmodel.SomeObject() # not the same value class! complain... return SomeWeakKeyDict(s_wkd1.keyclassdef, s_wkd1.valueclassdef) class Entry(extregistry.ExtRegistryEntry): diff --git a/pypy/rlib/test/test_rweakkeydict.py b/pypy/rlib/test/test_rweakkeydict.py --- a/pypy/rlib/test/test_rweakkeydict.py +++ b/pypy/rlib/test/test_rweakkeydict.py @@ -2,6 +2,8 @@ from pypy.rlib import rgc from pypy.rlib.rweakref import RWeakKeyDictionary from pypy.rpython.test.test_llinterp import interpret +from pypy.rpython.lltypesystem.lloperation import llop +from pypy.rpython.lltypesystem import lltype class KX(object): pass @@ -15,7 +17,6 @@ class VY(VX): pass - def make_test(loop=100, prebuilt=None): def g(d): assert d.get(KX()) is None @@ -35,19 +36,25 @@ d = prebuilt if d is None: d = RWeakKeyDictionary(KX, VX) + llop.debug_print(lltype.Void, "XXX 1") k1, k3, v1, v2, v3 = g(d) rgc.collect(); rgc.collect() + llop.debug_print(lltype.Void, "XXX 2") assert d.get(k1) is v1 assert d.get(k3) is v3 assert d.get(k1) is not v2 assert d.get(k3) is not v2 + llop.debug_print(lltype.Void, "XXX 3") assert d.length() == 2 + llop.debug_print(lltype.Void, "XXX 4") d.set(k1, None) assert d.get(k1) is None assert d.get(k3) is v3 assert d.length() == 1 + llop.debug_print(lltype.Void, "XXX 5") # resizing should also work lots_of_keys = [KX() for i in range(loop)] + llop.debug_print(lltype.Void, "XXX 6") for k in lots_of_keys: d.set(k, v1) for k in lots_of_keys: @@ -56,19 +63,26 @@ assert d.get(k3) is v3 assert d.length() == loop + 1 # a subclass + llop.debug_print(lltype.Void, "XXX 7") ky = KY() vy = VY() d.set(ky, vy) assert d.get(ky) is vy assert d.length() == loop + 2 # deleting by storing Nones + llop.debug_print(lltype.Void, "XXX 8") for k in lots_of_keys: d.set(k, None) + llop.debug_print(lltype.Void, "XXX 9") for k in lots_of_keys: assert d.get(k) is None + llop.debug_print(lltype.Void, "XXX 10") assert d.get(k1) is None + llop.debug_print(lltype.Void, "XXX 11") assert d.get(k3) is v3 + llop.debug_print(lltype.Void, "XXX 12") assert d.get(ky) is vy + llop.debug_print(lltype.Void, "XXX 13") assert d.length() == 2 return f diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py --- a/pypy/rpython/lltypesystem/rdict.py +++ b/pypy/rpython/lltypesystem/rdict.py @@ -42,20 +42,19 @@ def get_ll_dict(DICTKEY, DICTVALUE, get_custom_eq_hash=None, DICT=None, ll_fasthash_function=None, ll_hash_function=None, - ll_eq_function=None): + ll_eq_function=None, method_cache={}): # get the actual DICT type. if DICT is None, it's created, otherwise # forward reference is becoming DICT if DICT is None: DICT = lltype.GcForwardReference() # compute the shape of the DICTENTRY structure entryfields = [] - entrymeths = { - 'allocate': lltype.typeMethod(_ll_malloc_entries), - 'must_clear_key': (isinstance(DICTKEY, lltype.Ptr) - and DICTKEY._needsgc()), - 'must_clear_value': (isinstance(DICTVALUE, lltype.Ptr) - and DICTVALUE._needsgc()), - } + entrymeths = {'allocate': lltype.typeMethod(_ll_malloc_entries), + 'valid': ll_valid_entry, + 'clear_key': (isinstance(DICTKEY, lltype.Ptr) and + DICTKEY._needsgc()), + 'clear_value': (isinstance(DICTVALUE, lltype.Ptr) and + DICTVALUE._needsgc())} # * the key entryfields.append(("key", DICTKEY)) @@ -79,7 +78,7 @@ DICTENTRY = lltype.Struct("dictentry", *entryfields) DICTENTRYARRAY = lltype.GcArray(DICTENTRY, adtmeths=entrymeths) - array_adtmeths = {'allocate': lltype.typeMethod(_ll_malloc_items)} + array_adtmeths = {'allocate': lltype.typeMethod(_ll_malloc_indexes)} fields = [("num_items", lltype.Signed), ("resize_counter", lltype.Signed), ("entries", lltype.Ptr(DICTENTRYARRAY)), @@ -111,6 +110,7 @@ adtmeths['KEY'] = DICTKEY adtmeths['VALUE'] = DICTVALUE adtmeths['allocate'] = lltype.typeMethod(_ll_malloc_dict) + adtmeths['resize'] = ll_dict_resize DICT.become(lltype.GcStruct("dicttable", adtmeths=adtmeths, *fields)) return DICT @@ -337,6 +337,9 @@ def ll_hash_from_cache(entries, i): return entries[i].f_hash +def ll_valid_entry(entires, index): + return index >= 0 + def ll_hash_recomputed(entries, i): ENTRIES = lltype.typeOf(entries).TO return ENTRIES.fasthashfn(entries[i].key) @@ -390,7 +393,7 @@ ll_assert(not valid, "valid but not everused") rc = d.resize_counter - 1 if rc <= 0: # if needed, resize the dict -- before the insertion - ll_dict_resize(d) + d.resize() i = ll_dict_lookup_clean(d, hash) # then redo the lookup for 'key' entry = d.entries[index] @@ -442,9 +445,9 @@ d.indexes[i] = DELETED d.num_items -= 1 entry = d.entries[d.num_items] - if ENTRIES.must_clear_key: + if ENTRIES.clear_key: entry.key = lltype.nullptr(ENTRY.key.TO) - if ENTRIES.must_clear_value: + if ENTRIES.clear_value: entry.value = lltype.nullptr(ENTRY.value.TO) # # The rest is commented out: like CPython we no longer shrink the @@ -482,7 +485,7 @@ indexes = d.indexes while i < old_size: index = old_indexes[i] - if index >= 0: + if old_entries.valid(index): indexes[ll_dict_lookup_clean(d, old_entries.hash(index))] = index i += 1 rgc.ll_arraycopy(old_entries, d.entries, 0, 0, min(len(old_entries), @@ -493,6 +496,14 @@ PERTURB_SHIFT = 5 _look_inside_lookup = lambda d, key, hash: jit.isvirtual(d) and jit.isconstant(key) +from pypy.rlib.objectmodel import compute_identity_hash + +def ll_debugrepr(x): + if x: + h = compute_identity_hash(x) + else: + h = 0 + return '<%x>' % (h,) @jit.look_inside_iff(_look_inside_lookup) def ll_dict_lookup(d, key, hash): @@ -501,18 +512,21 @@ mask = len(indexes) - 1 i = hash & mask # do the first try before any looping + ENTRIES = lltype.typeOf(entries).TO + direct_compare = not hasattr(ENTRIES, 'no_direct_compare') index = indexes[i] - if index >= 0: + if entries.valid(index): checkingkey = entries[index].key - if checkingkey == key: + if direct_compare and checkingkey == key: return i # found the entry if d.keyeq is not None and entries.hash(index) == hash: # correct hash, maybe the key is e.g. a different pointer to # an equal object found = d.keyeq(checkingkey, key) + llop.debug_print(lltype.Void, "comparing keys", ll_debugrepr(checkingkey), ll_debugrepr(key), found) if d.paranoia: if (entries != d.entries or - not indexes[i] >= 0 or entries[index].key != checkingkey): + not entries.valid(indexes[i]) or entries[index].key != checkingkey): # the compare did major nasty stuff to the dict: start over return ll_dict_lookup(d, key, hash) if found: @@ -538,9 +552,9 @@ if freeslot == -1: freeslot = i return freeslot | HIGHEST_BIT - elif index >= 0: + elif entries.valid(index): checkingkey = entries[index].key - if checkingkey == key: + if direct_compare and checkingkey == key: return i if d.keyeq is not None and entries.hash(index) == hash: # correct hash, maybe the key is e.g. a different pointer to @@ -548,7 +562,7 @@ found = d.keyeq(checkingkey, key) if d.paranoia: if (entries != d.entries or - not indexes[i] >= 0 or + not entries.valid(indexes[i]) or entries[index].key != checkingkey): # the compare did major nasty stuff to the dict: # start over @@ -614,7 +628,7 @@ return lltype.malloc(DICT) def _ll_malloc_entries(ENTRIES, n): return lltype.malloc(ENTRIES, n, zero=True) -def _ll_malloc_items(ITEMS, n): +def _ll_malloc_indexes(ITEMS, n): res = lltype.malloc(ITEMS, n) i = 0 while i < n: From noreply at buildbot.pypy.org Mon Jan 7 00:28:41 2013 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 7 Jan 2013 00:28:41 +0100 (CET) Subject: [pypy-commit] cffi default: Win32: needs this include now. Message-ID: <20130106232841.E4BCA1C0F8A@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1118:382e3612620e Date: 2013-01-07 00:28 +0100 http://bitbucket.org/cffi/cffi/changeset/382e3612620e/ Log: Win32: needs this include now. diff --git a/cffi/vengine_cpy.py b/cffi/vengine_cpy.py --- a/cffi/vengine_cpy.py +++ b/cffi/vengine_cpy.py @@ -701,6 +701,7 @@ #include #ifdef MS_WIN32 +#include /* for alloca() */ typedef __int8 int8_t; typedef __int16 int16_t; typedef __int32 int32_t; From noreply at buildbot.pypy.org Mon Jan 7 00:33:26 2013 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 7 Jan 2013 00:33:26 +0100 (CET) Subject: [pypy-commit] cffi default: Extra tests Message-ID: <20130106233326.D18A61C0F8A@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1119:fc21aeb1e44e Date: 2013-01-07 00:33 +0100 http://bitbucket.org/cffi/cffi/changeset/fc21aeb1e44e/ Log: Extra tests diff --git a/testing/test_verify.py b/testing/test_verify.py --- a/testing/test_verify.py +++ b/testing/test_verify.py @@ -1491,6 +1491,8 @@ } """) assert lib.sum3floats((1.5, 2.5, 3.5)) == 7.5 + p = ffi.new("float[]", (1.5, 2.5, 3.5)) + assert lib.sum3floats(p) == 7.5 def test_charstar_argument(): ffi = FFI() @@ -1501,3 +1503,5 @@ } """) assert lib.sum3chars(('\x10', '\x20', '\x30')) == '\x60' + p = ffi.new("char[]", '\x10\x20\x30') + assert lib.sum3chars(p) == '\x60' From noreply at buildbot.pypy.org Mon Jan 7 00:51:39 2013 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 7 Jan 2013 00:51:39 +0100 (CET) Subject: [pypy-commit] pypy default: Write a warning in this big block of text Message-ID: <20130106235139.5D8C51C0AD6@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r59828:4d436d0b7748 Date: 2013-01-07 00:51 +0100 http://bitbucket.org/pypy/pypy/changeset/4d436d0b7748/ Log: Write a warning in this big block of text diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py --- a/pypy/interpreter/typedef.py +++ b/pypy/interpreter/typedef.py @@ -105,6 +105,13 @@ # typical usage (case 1 is the most common kind of app-level subclasses; # case 2 is the memory-saving kind defined with __slots__). # +# +----------------------------------------------------------------+ +# | NOTE: if withmapdict is enabled, the following doesn't apply! | +# | Map dicts can flexibly allow any slots/__dict__/__weakref__ to | +# | show up only when needed. In particular there is no way with | +# | mapdict to prevent some objects from being weakrefable. | +# +----------------------------------------------------------------+ +# # dict slots del weakrefable # # 1. Y N N Y UserDictWeakref From noreply at buildbot.pypy.org Mon Jan 7 01:06:02 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 7 Jan 2013 01:06:02 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: port weakvalueref Message-ID: <20130107000602.EE1F41C0AD6@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59829:802afeec68e5 Date: 2013-01-07 02:05 +0200 http://bitbucket.org/pypy/pypy/changeset/802afeec68e5/ Log: port weakvalueref diff --git a/pypy/rlib/_rweakkeydict.py b/pypy/rlib/_rweakkeydict.py --- a/pypy/rlib/_rweakkeydict.py +++ b/pypy/rlib/_rweakkeydict.py @@ -123,17 +123,17 @@ @jit.dont_look_inside def ll_get(d, llkey): hash = compute_identity_hash(llkey) - llop.debug_print(lltype.Void, "computed key", ll_debugrepr(llkey), - hex(hash)) + #llop.debug_print(lltype.Void, "computed key", ll_debugrepr(llkey), + # hex(hash)) i = rdict.ll_dict_lookup(d, llkey, hash) & rdict.MASK index = d.indexes[i] if index < 0: - llop.debug_print(lltype.Void, i, 'get', hex(hash), "null") + #llop.debug_print(lltype.Void, i, 'get', hex(hash), "null") return NULLVALUE - llop.debug_print(lltype.Void, i, "getting", index) - llop.debug_print(lltype.Void, i, 'get', hex(hash), - ll_debugrepr(d.entries[index].key), - ll_debugrepr(d.entries[index].value)) + #llop.debug_print(lltype.Void, i, "getting", index) + #llop.debug_print(lltype.Void, i, 'get', hex(hash), + # ll_debugrepr(d.entries[index].key), + # ll_debugrepr(d.entries[index].value)) # NB. ll_valid() above was just called at least on entry i, so if # it is an invalid entry with a dead weakref, the value was reset # to NULLVALUE. @@ -160,13 +160,13 @@ d.entries[index].key = keyref d.entries[index].value = llvalue d.entries[index].f_hash = hash - llop.debug_print(lltype.Void, i, 'stored', index, d.num_items, hex(hash), - ll_debugrepr(llkey), - ll_debugrepr(llvalue)) + #llop.debug_print(lltype.Void, i, 'stored', index, d.num_items, hex(hash), + # ll_debugrepr(llkey), + # ll_debugrepr(llvalue)) if not everused: d.resize_counter -= 1 if d.resize_counter <= 0: - llop.debug_print(lltype.Void, 'RESIZE') + #llop.debug_print(lltype.Void, 'RESIZE') ll_weakdict_resize(d) @jit.dont_look_inside @@ -177,10 +177,10 @@ if d.entries.valid(index): d.entries[index].key = llmemory.dead_wref d.entries[index].value = NULLVALUE - llop.debug_print(lltype.Void, i, index, 'zero') + #llop.debug_print(lltype.Void, i, index, 'zero') def ll_weakdict_resize(d): - llop.debug_print(lltype.Void, "weakdict resize") + #llop.debug_print(lltype.Void, "weakdict resize") old_entries = d.entries old_indexes = d.indexes old_size = len(old_indexes) @@ -203,7 +203,6 @@ new_item_size = new_size // 3 * 2 + 1 d.entries = lltype.typeOf(old_entries).TO.allocate(new_item_size) d.indexes = lltype.typeOf(d).TO.indexes.TO.allocate(new_size) - d.resize_counter = new_item_size - num_items i = 0 indexes = d.indexes j = 0 @@ -213,15 +212,16 @@ hash = old_entries.hash(index) lookup_i = rdict.ll_dict_lookup_clean(d, hash) indexes[lookup_i] = j - llop.debug_print(lltype.Void, "inserting", hex(hash), i, - "to", lookup_i, index, "=>", j) - llop.debug_print(lltype.Void, hex(old_entries[index].f_hash)) + #llop.debug_print(lltype.Void, "inserting", hex(hash), i, + # "to", lookup_i, index, "=>", j) + #llop.debug_print(lltype.Void, hex(old_entries[index].f_hash)) d.entries[j].key = old_entries[index].key d.entries[j].value = old_entries[index].value d.entries[j].f_hash = old_entries[index].f_hash j += 1 i += 1 d.num_items = j + d.resize_counter = new_item_size - j def ll_keyeq(d, weakkey1, realkey2): # only called by ll_dict_lookup() with the first arg coming from an @@ -230,14 +230,14 @@ assert bool(realkey2) return False realkey1 = weakref_deref(rclass.OBJECTPTR, weakkey1) - llop.debug_print(lltype.Void, "comparison", realkey1, realkey2) + #llop.debug_print(lltype.Void, "comparison", realkey1, realkey2) return realkey1 == realkey2 @jit.dont_look_inside def ll_length(d): # xxx slow, but it's only for debugging d.resize() - llop.debug_print(lltype.Void, 'length:', d.num_items) + #llop.debug_print(lltype.Void, 'length:', d.num_items) return d.num_items dictmeths = { diff --git a/pypy/rlib/_rweakvaldict.py b/pypy/rlib/_rweakvaldict.py --- a/pypy/rlib/_rweakvaldict.py +++ b/pypy/rlib/_rweakvaldict.py @@ -18,8 +18,10 @@ self.ll_keyhash = r_key.get_ll_hash_function() ll_keyeq = lltype.staticAdtMethod(r_key.get_ll_eq_function()) - def ll_valid(entries, i): - value = entries[i].value + def ll_valid(entries, index): + if index < 0: + return False + value = entries[index].value return bool(value) and bool(weakref_deref(rclass.OBJECTPTR, value)) def ll_everused(entries, i): @@ -30,7 +32,6 @@ entrymeths = { 'allocate': lltype.typeMethod(rdict._ll_malloc_entries), - 'delete': rdict._ll_free_entries, 'valid': ll_valid, 'everused': ll_everused, 'hash': ll_hash, @@ -41,6 +42,9 @@ WEAKDICTENTRYARRAY = lltype.GcArray(WEAKDICTENTRY, adtmeths=entrymeths, hints={'weakarray': 'value'}) + + WEAKINDEXESARRAY = lltype.GcArray(lltype.Signed, + adtmeths={'allocate': lltype.typeMethod(rdict._ll_malloc_indexes)}) # NB. the 'hints' is not used so far ^^^ dictmeths = { @@ -48,12 +52,14 @@ 'll_set': self.ll_set, 'keyeq': ll_keyeq, 'paranoia': False, + 'resize': self.ll_weakdict_resize, } self.WEAKDICT = lltype.GcStruct( "weakvaldict", ("num_items", lltype.Signed), ("resize_counter", lltype.Signed), + ('indexes', lltype.Ptr(WEAKINDEXESARRAY)), ("entries", lltype.Ptr(WEAKDICTENTRYARRAY)), adtmeths=dictmeths) @@ -62,7 +68,7 @@ def convert_const(self, weakdict): if not isinstance(weakdict, RWeakValueDictionary): - raise TyperError("expected an RWeakValueDictionary: %r" % ( + raise TypeError("expected an RWeakValueDictionary: %r" % ( weakdict,)) try: key = Constant(weakdict) @@ -105,9 +111,10 @@ @jit.dont_look_inside def ll_new_weakdict(self): d = lltype.malloc(self.WEAKDICT) - d.entries = self.WEAKDICT.entries.TO.allocate(rdict.DICT_INITSIZE) + d.entries = self.WEAKDICT.entries.TO.allocate(rdict.DICT_ITEMS_INITSIZE) + d.indexes = self.WEAKDICT.indexes.TO.allocate(rdict.DICT_INITSIZE) d.num_items = 0 - d.resize_counter = rdict.DICT_INITSIZE * 2 + d.resize_counter = rdict.DICT_ITEMS_INITSIZE return d @jit.dont_look_inside @@ -115,8 +122,11 @@ hash = self.ll_keyhash(llkey) i = rdict.ll_dict_lookup(d, llkey, hash) & rdict.MASK #llop.debug_print(lltype.Void, i, 'get') - valueref = d.entries[i].value - if valueref: + index = d.indexes[i] + if index >= 0: + valueref = d.entries[index].value + if not valueref: + return lltype.nullptr(rclass.OBJECTPTR.TO) return weakref_deref(rclass.OBJECTPTR, valueref) else: return lltype.nullptr(rclass.OBJECTPTR.TO) @@ -127,18 +137,26 @@ self.ll_set_nonnull(d, llkey, llvalue) else: self.ll_set_null(d, llkey) - + @jit.dont_look_inside def ll_set_nonnull(self, d, llkey, llvalue): hash = self.ll_keyhash(llkey) valueref = weakref_create(llvalue) # GC effects here, before the rest i = rdict.ll_dict_lookup(d, llkey, hash) & rdict.MASK - everused = d.entries.everused(i) - d.entries[i].key = llkey - d.entries[i].value = valueref - #llop.debug_print(lltype.Void, i, 'stored') + index = d.indexes[i] + everused = index != rdict.FREE + if index < 0: + index = d.num_items + d.indexes[i] = index + d.num_items += 1 + d.entries[index].key = llkey + d.entries[index].value = valueref + llop.debug_print(lltype.Void, "set nonnull", i, index) + #llop.debug_print(lltype.Void, i, 'stored', index, d.num_items, hex(hash), + # ll_debugrepr(llkey), + # ll_debugrepr(llvalue)) if not everused: - d.resize_counter -= 3 + d.resize_counter -= 1 if d.resize_counter <= 0: #llop.debug_print(lltype.Void, 'RESIZE') self.ll_weakdict_resize(d) @@ -147,26 +165,60 @@ def ll_set_null(self, d, llkey): hash = self.ll_keyhash(llkey) i = rdict.ll_dict_lookup(d, llkey, hash) & rdict.MASK - if d.entries.everused(i): + index = d.indexes[i] + if d.entries.valid(index): # If the entry was ever used, clean up its key and value. # We don't store a NULL value, but a dead weakref, because # the entry must still be marked as everused(). - d.entries[i].value = llmemory.dead_wref + d.entries[index].value = llmemory.dead_wref if isinstance(self.r_key.lowleveltype, lltype.Ptr): - d.entries[i].key = self.r_key.convert_const(None) + d.entries[index].key = self.r_key.convert_const(None) else: - d.entries[i].key = self.r_key.convert_const(0) + d.entries[index].key = self.r_key.convert_const(0) #llop.debug_print(lltype.Void, i, 'zero') def ll_weakdict_resize(self, d): - # first set num_items to its correct, up-to-date value - entries = d.entries + #llop.debug_print(lltype.Void, "weakdict resize") + old_entries = d.entries + old_indexes = d.indexes + old_size = len(old_indexes) + # make a 'new_size' estimate and shrink it if there are many + # deleted entry markers. See CPython for why it is a good idea to + # quadruple the dictionary size as long as it's not too big. + # count the number of valid entries + i = 0 num_items = 0 - for i in range(len(entries)): - if entries.valid(i): + while i < d.num_items: + if old_entries.valid(i): num_items += 1 - d.num_items = num_items - rdict.ll_dict_resize(d) + i += 1 + if num_items > 50000: new_estimate = (num_items + 1) * 2 + else: new_estimate = (num_items + 1) * 4 + new_size = rdict.DICT_INITSIZE + while new_size <= new_estimate: + new_size *= 2 + # + new_item_size = new_size // 3 * 2 + 1 + d.entries = lltype.typeOf(old_entries).TO.allocate(new_item_size) + d.indexes = lltype.typeOf(d).TO.indexes.TO.allocate(new_size) + i = 0 + indexes = d.indexes + j = 0 + while i < old_size: + index = old_indexes[i] + if old_entries.valid(index): + hash = old_entries.hash(index) + lookup_i = rdict.ll_dict_lookup_clean(d, hash) + indexes[lookup_i] = j + #llop.debug_print(lltype.Void, "inserting", hex(hash), i, + # "to", lookup_i, index, "=>", j) + #llop.debug_print(lltype.Void, hex(old_entries[index].f_hash)) + d.entries[j].key = old_entries[index].key + d.entries[j].value = old_entries[index].value + j += 1 + i += 1 + d.num_items = j + d.resize_counter = new_item_size - j def specialize_make_weakdict(hop): hop.exception_cannot_occur() diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py --- a/pypy/rpython/lltypesystem/rdict.py +++ b/pypy/rpython/lltypesystem/rdict.py @@ -523,7 +523,7 @@ # correct hash, maybe the key is e.g. a different pointer to # an equal object found = d.keyeq(checkingkey, key) - llop.debug_print(lltype.Void, "comparing keys", ll_debugrepr(checkingkey), ll_debugrepr(key), found) + #llop.debug_print(lltype.Void, "comparing keys", ll_debugrepr(checkingkey), ll_debugrepr(key), found) if d.paranoia: if (entries != d.entries or not entries.valid(indexes[i]) or entries[index].key != checkingkey): From noreply at buildbot.pypy.org Mon Jan 7 01:42:10 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 7 Jan 2013 01:42:10 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: dict bench Message-ID: <20130107004210.6416C1C0312@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59830:86960b883d97 Date: 2013-01-07 02:41 +0200 http://bitbucket.org/pypy/pypy/changeset/86960b883d97/ Log: dict bench diff --git a/pypy/translator/goal/targetbenchdict.py b/pypy/translator/goal/targetbenchdict.py new file mode 100644 --- /dev/null +++ b/pypy/translator/goal/targetbenchdict.py @@ -0,0 +1,45 @@ +""" +A simple standalone target. + +The target below specifies None as the argument types list. +This is a case treated specially in driver.py . If the list +of input types is empty, it is meant to be a list of strings, +actually implementing argv of the executable. +""" + + +# __________ Entry point __________ + +class A(object): + def __init__(self, a, b, c): + self.a = a + self.b = b + self.c = c + +def entry_point(argv): + bench_no = int(argv[1]) + bench_counter = int(argv[2]) + d = {} + if bench_no == 1: + for i in range(bench_counter): + d[str(i)] = None + if bench_no == 2: + for i in range(bench_counter): + d[i] = A(1, 2, 3) + if bench_no == 3: + a = A(1, 2, 3) + for i in range(bench_counter): + if i % 100 == 0: + a = A(1, 2, 3) + d[i] = a + if bench_no == 4: + s = 0 + for i in range(bench_counter): + d[i % 100] = i + s += d[i % 100] + return 0 + +# _____ Define and setup target ___ + +def target(*args): + return entry_point, None From noreply at buildbot.pypy.org Mon Jan 7 01:43:39 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 7 Jan 2013 01:43:39 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: rpythonize Message-ID: <20130107004339.4492B1C0312@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59831:237f60d5937f Date: 2013-01-07 02:43 +0200 http://bitbucket.org/pypy/pypy/changeset/237f60d5937f/ Log: rpythonize diff --git a/pypy/translator/goal/targetbenchdict.py b/pypy/translator/goal/targetbenchdict.py --- a/pypy/translator/goal/targetbenchdict.py +++ b/pypy/translator/goal/targetbenchdict.py @@ -19,20 +19,23 @@ def entry_point(argv): bench_no = int(argv[1]) bench_counter = int(argv[2]) - d = {} if bench_no == 1: + d = {} for i in range(bench_counter): d[str(i)] = None if bench_no == 2: + d = {} for i in range(bench_counter): d[i] = A(1, 2, 3) if bench_no == 3: + d = {} a = A(1, 2, 3) for i in range(bench_counter): if i % 100 == 0: a = A(1, 2, 3) d[i] = a if bench_no == 4: + d = {} s = 0 for i in range(bench_counter): d[i % 100] = i From noreply at buildbot.pypy.org Mon Jan 7 01:52:36 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 7 Jan 2013 01:52:36 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: Give up and use old rdict for all other places that happen to use rdict details Message-ID: <20130107005236.C2D6F1C0312@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59832:4927a203f8b8 Date: 2013-01-07 02:52 +0200 http://bitbucket.org/pypy/pypy/changeset/4927a203f8b8/ Log: Give up and use old rdict for all other places that happen to use rdict details diff --git a/pypy/rlib/_rweakkeydict.py b/pypy/rlib/_rweakkeydict.py --- a/pypy/rlib/_rweakkeydict.py +++ b/pypy/rlib/_rweakkeydict.py @@ -1,13 +1,13 @@ from pypy.objspace.flow.model import Constant -from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rdict +from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rdict_old as rdict +from pypy.rpython.lltypesystem.llmemory import weakref_create, weakref_deref from pypy.rpython.lltypesystem.lloperation import llop -from pypy.rpython.lltypesystem.llmemory import weakref_create, weakref_deref from pypy.rpython.rclass import getinstancerepr from pypy.rpython.rmodel import Repr from pypy.rlib.rweakref import RWeakKeyDictionary from pypy.rlib import jit -from pypy.rlib.debug import ll_assert from pypy.rlib.objectmodel import compute_identity_hash +from pypy.rlib.objectmodel import we_are_translated # Warning: this implementation of RWeakKeyDictionary is not exactly @@ -25,7 +25,7 @@ def convert_const(self, weakdict): if not isinstance(weakdict, RWeakKeyDictionary): - raise TypeError("expected an RWeakKeyDictionary: %r" % ( + raise TyperError("expected an RWeakKeyDictionary: %r" % ( weakdict,)) try: key = Constant(weakdict) @@ -33,7 +33,7 @@ except KeyError: self.setup() if weakdict.length() != 0: - raise TypeError("got a non-empty prebuilt RWeakKeyDictionary") + raise TyperError("got a non-empty prebuilt RWeakKeyDictionary") l_dict = ll_new_weakdict() self.dict_cache[key] = l_dict return l_dict @@ -83,10 +83,8 @@ h = 0 return '<%x>' % (h,) -def ll_valid(entries, index): - if index < 0: - return False - key = entries[index].key +def ll_valid(entries, i): + key = entries[i].key if not key: return False elif weakref_deref(rclass.OBJECTPTR, key): @@ -95,16 +93,19 @@ # The entry might be a dead weakref still holding a strong # reference to the value; for this case, we clear the old # value from the entry, if any. - entries[index].value = NULLVALUE + entries[i].value = NULLVALUE return False +def ll_everused(entries, i): + return bool(entries[i].key) + entrymeths = { 'allocate': lltype.typeMethod(rdict._ll_malloc_entries), + 'delete': rdict._ll_free_entries, + 'valid': ll_valid, + 'everused': ll_everused, 'hash': rdict.ll_hash_from_cache, 'no_direct_compare': True, - 'clear_key': lambda : llmemory.dead_wref, - 'clear_value': lambda : lltype.nullptr(rclass.OBJECTPTR.TO), - 'valid': ll_valid, } WEAKDICTENTRYARRAY = lltype.GcArray(WEAKDICTENTRY, adtmeths=entrymeths, @@ -114,30 +115,22 @@ @jit.dont_look_inside def ll_new_weakdict(): d = lltype.malloc(WEAKDICT) - d.entries = WEAKDICT.entries.TO.allocate(rdict.DICT_ITEMS_INITSIZE) - d.indexes = WEAKDICT.indexes.TO.allocate(rdict.DICT_INITSIZE) + d.entries = WEAKDICT.entries.TO.allocate(rdict.DICT_INITSIZE) d.num_items = 0 - d.resize_counter = rdict.DICT_ITEMS_INITSIZE + d.resize_counter = rdict.DICT_INITSIZE * 2 return d @jit.dont_look_inside def ll_get(d, llkey): hash = compute_identity_hash(llkey) - #llop.debug_print(lltype.Void, "computed key", ll_debugrepr(llkey), - # hex(hash)) i = rdict.ll_dict_lookup(d, llkey, hash) & rdict.MASK - index = d.indexes[i] - if index < 0: - #llop.debug_print(lltype.Void, i, 'get', hex(hash), "null") - return NULLVALUE - #llop.debug_print(lltype.Void, i, "getting", index) #llop.debug_print(lltype.Void, i, 'get', hex(hash), - # ll_debugrepr(d.entries[index].key), - # ll_debugrepr(d.entries[index].value)) + # ll_debugrepr(d.entries[i].key), + # ll_debugrepr(d.entries[i].value)) # NB. ll_valid() above was just called at least on entry i, so if # it is an invalid entry with a dead weakref, the value was reset # to NULLVALUE. - return d.entries[index].value + return d.entries[i].value @jit.dont_look_inside def ll_set(d, llkey, llvalue): @@ -151,20 +144,15 @@ hash = compute_identity_hash(llkey) keyref = weakref_create(llkey) # GC effects here, before the rest i = rdict.ll_dict_lookup(d, llkey, hash) & rdict.MASK - index = d.indexes[i] - everused = index != rdict.FREE - if index < 0: - index = d.num_items - d.indexes[i] = index - d.num_items += 1 - d.entries[index].key = keyref - d.entries[index].value = llvalue - d.entries[index].f_hash = hash - #llop.debug_print(lltype.Void, i, 'stored', index, d.num_items, hex(hash), + everused = d.entries.everused(i) + d.entries[i].key = keyref + d.entries[i].value = llvalue + d.entries[i].f_hash = hash + #llop.debug_print(lltype.Void, i, 'stored', hex(hash), # ll_debugrepr(llkey), # ll_debugrepr(llvalue)) if not everused: - d.resize_counter -= 1 + d.resize_counter -= 3 if d.resize_counter <= 0: #llop.debug_print(lltype.Void, 'RESIZE') ll_weakdict_resize(d) @@ -173,55 +161,26 @@ def ll_set_null(d, llkey): hash = compute_identity_hash(llkey) i = rdict.ll_dict_lookup(d, llkey, hash) & rdict.MASK - index = d.indexes[i] - if d.entries.valid(index): - d.entries[index].key = llmemory.dead_wref - d.entries[index].value = NULLVALUE - #llop.debug_print(lltype.Void, i, index, 'zero') + if d.entries.everused(i): + # If the entry was ever used, clean up its key and value. + # We don't store a NULL value, but a dead weakref, because + # the entry must still be marked as everused(). + d.entries[i].key = llmemory.dead_wref + d.entries[i].value = NULLVALUE + #llop.debug_print(lltype.Void, i, 'zero') + +def ll_update_num_items(d): + entries = d.entries + num_items = 0 + for i in range(len(entries)): + if entries.valid(i): + num_items += 1 + d.num_items = num_items def ll_weakdict_resize(d): - #llop.debug_print(lltype.Void, "weakdict resize") - old_entries = d.entries - old_indexes = d.indexes - old_size = len(old_indexes) - # make a 'new_size' estimate and shrink it if there are many - # deleted entry markers. See CPython for why it is a good idea to - # quadruple the dictionary size as long as it's not too big. - # count the number of valid entries - i = 0 - num_items = 0 - while i < d.num_items: - if old_entries.valid(i): - num_items += 1 - i += 1 - if num_items > 50000: new_estimate = (num_items + 1) * 2 - else: new_estimate = (num_items + 1) * 4 - new_size = rdict.DICT_INITSIZE - while new_size <= new_estimate: - new_size *= 2 - # - new_item_size = new_size // 3 * 2 + 1 - d.entries = lltype.typeOf(old_entries).TO.allocate(new_item_size) - d.indexes = lltype.typeOf(d).TO.indexes.TO.allocate(new_size) - i = 0 - indexes = d.indexes - j = 0 - while i < old_size: - index = old_indexes[i] - if old_entries.valid(index): - hash = old_entries.hash(index) - lookup_i = rdict.ll_dict_lookup_clean(d, hash) - indexes[lookup_i] = j - #llop.debug_print(lltype.Void, "inserting", hex(hash), i, - # "to", lookup_i, index, "=>", j) - #llop.debug_print(lltype.Void, hex(old_entries[index].f_hash)) - d.entries[j].key = old_entries[index].key - d.entries[j].value = old_entries[index].value - d.entries[j].f_hash = old_entries[index].f_hash - j += 1 - i += 1 - d.num_items = j - d.resize_counter = new_item_size - j + # first set num_items to its correct, up-to-date value + ll_update_num_items(d) + rdict.ll_dict_resize(d) def ll_keyeq(d, weakkey1, realkey2): # only called by ll_dict_lookup() with the first arg coming from an @@ -229,14 +188,12 @@ if not weakkey1: assert bool(realkey2) return False - realkey1 = weakref_deref(rclass.OBJECTPTR, weakkey1) - #llop.debug_print(lltype.Void, "comparison", realkey1, realkey2) - return realkey1 == realkey2 + return weakref_deref(rclass.OBJECTPTR, weakkey1) == realkey2 @jit.dont_look_inside def ll_length(d): # xxx slow, but it's only for debugging - d.resize() + ll_update_num_items(d) #llop.debug_print(lltype.Void, 'length:', d.num_items) return d.num_items @@ -245,15 +202,10 @@ 'll_set': ll_set, 'keyeq': ll_keyeq, 'paranoia': False, - 'resize': ll_weakdict_resize, } -INDEXESARRAY = lltype.GcArray(lltype.Signed, - adtmeths={'allocate' : lltype.typeMethod(rdict._ll_malloc_indexes)}) - WEAKDICT = lltype.GcStruct("weakkeydict", ("num_items", lltype.Signed), ("resize_counter", lltype.Signed), - ("indexes", lltype.Ptr(INDEXESARRAY)), ("entries", lltype.Ptr(WEAKDICTENTRYARRAY)), adtmeths=dictmeths) diff --git a/pypy/rlib/_rweakvaldict.py b/pypy/rlib/_rweakvaldict.py --- a/pypy/rlib/_rweakvaldict.py +++ b/pypy/rlib/_rweakvaldict.py @@ -1,5 +1,5 @@ from pypy.objspace.flow.model import Constant -from pypy.rpython.lltypesystem import lltype, llmemory, rstr, rclass, rdict +from pypy.rpython.lltypesystem import lltype, llmemory, rstr, rclass, rdict_old as rdict from pypy.rpython.lltypesystem.llmemory import weakref_create, weakref_deref from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.rclass import getinstancerepr @@ -18,10 +18,8 @@ self.ll_keyhash = r_key.get_ll_hash_function() ll_keyeq = lltype.staticAdtMethod(r_key.get_ll_eq_function()) - def ll_valid(entries, index): - if index < 0: - return False - value = entries[index].value + def ll_valid(entries, i): + value = entries[i].value return bool(value) and bool(weakref_deref(rclass.OBJECTPTR, value)) def ll_everused(entries, i): @@ -32,6 +30,7 @@ entrymeths = { 'allocate': lltype.typeMethod(rdict._ll_malloc_entries), + 'delete': rdict._ll_free_entries, 'valid': ll_valid, 'everused': ll_everused, 'hash': ll_hash, @@ -42,9 +41,6 @@ WEAKDICTENTRYARRAY = lltype.GcArray(WEAKDICTENTRY, adtmeths=entrymeths, hints={'weakarray': 'value'}) - - WEAKINDEXESARRAY = lltype.GcArray(lltype.Signed, - adtmeths={'allocate': lltype.typeMethod(rdict._ll_malloc_indexes)}) # NB. the 'hints' is not used so far ^^^ dictmeths = { @@ -52,14 +48,12 @@ 'll_set': self.ll_set, 'keyeq': ll_keyeq, 'paranoia': False, - 'resize': self.ll_weakdict_resize, } self.WEAKDICT = lltype.GcStruct( "weakvaldict", ("num_items", lltype.Signed), ("resize_counter", lltype.Signed), - ('indexes', lltype.Ptr(WEAKINDEXESARRAY)), ("entries", lltype.Ptr(WEAKDICTENTRYARRAY)), adtmeths=dictmeths) @@ -68,7 +62,7 @@ def convert_const(self, weakdict): if not isinstance(weakdict, RWeakValueDictionary): - raise TypeError("expected an RWeakValueDictionary: %r" % ( + raise TyperError("expected an RWeakValueDictionary: %r" % ( weakdict,)) try: key = Constant(weakdict) @@ -111,10 +105,9 @@ @jit.dont_look_inside def ll_new_weakdict(self): d = lltype.malloc(self.WEAKDICT) - d.entries = self.WEAKDICT.entries.TO.allocate(rdict.DICT_ITEMS_INITSIZE) - d.indexes = self.WEAKDICT.indexes.TO.allocate(rdict.DICT_INITSIZE) + d.entries = self.WEAKDICT.entries.TO.allocate(rdict.DICT_INITSIZE) d.num_items = 0 - d.resize_counter = rdict.DICT_ITEMS_INITSIZE + d.resize_counter = rdict.DICT_INITSIZE * 2 return d @jit.dont_look_inside @@ -122,11 +115,8 @@ hash = self.ll_keyhash(llkey) i = rdict.ll_dict_lookup(d, llkey, hash) & rdict.MASK #llop.debug_print(lltype.Void, i, 'get') - index = d.indexes[i] - if index >= 0: - valueref = d.entries[index].value - if not valueref: - return lltype.nullptr(rclass.OBJECTPTR.TO) + valueref = d.entries[i].value + if valueref: return weakref_deref(rclass.OBJECTPTR, valueref) else: return lltype.nullptr(rclass.OBJECTPTR.TO) @@ -137,26 +127,18 @@ self.ll_set_nonnull(d, llkey, llvalue) else: self.ll_set_null(d, llkey) - + @jit.dont_look_inside def ll_set_nonnull(self, d, llkey, llvalue): hash = self.ll_keyhash(llkey) valueref = weakref_create(llvalue) # GC effects here, before the rest i = rdict.ll_dict_lookup(d, llkey, hash) & rdict.MASK - index = d.indexes[i] - everused = index != rdict.FREE - if index < 0: - index = d.num_items - d.indexes[i] = index - d.num_items += 1 - d.entries[index].key = llkey - d.entries[index].value = valueref - llop.debug_print(lltype.Void, "set nonnull", i, index) - #llop.debug_print(lltype.Void, i, 'stored', index, d.num_items, hex(hash), - # ll_debugrepr(llkey), - # ll_debugrepr(llvalue)) + everused = d.entries.everused(i) + d.entries[i].key = llkey + d.entries[i].value = valueref + #llop.debug_print(lltype.Void, i, 'stored') if not everused: - d.resize_counter -= 1 + d.resize_counter -= 3 if d.resize_counter <= 0: #llop.debug_print(lltype.Void, 'RESIZE') self.ll_weakdict_resize(d) @@ -165,60 +147,26 @@ def ll_set_null(self, d, llkey): hash = self.ll_keyhash(llkey) i = rdict.ll_dict_lookup(d, llkey, hash) & rdict.MASK - index = d.indexes[i] - if d.entries.valid(index): + if d.entries.everused(i): # If the entry was ever used, clean up its key and value. # We don't store a NULL value, but a dead weakref, because # the entry must still be marked as everused(). - d.entries[index].value = llmemory.dead_wref + d.entries[i].value = llmemory.dead_wref if isinstance(self.r_key.lowleveltype, lltype.Ptr): - d.entries[index].key = self.r_key.convert_const(None) + d.entries[i].key = self.r_key.convert_const(None) else: - d.entries[index].key = self.r_key.convert_const(0) + d.entries[i].key = self.r_key.convert_const(0) #llop.debug_print(lltype.Void, i, 'zero') def ll_weakdict_resize(self, d): - #llop.debug_print(lltype.Void, "weakdict resize") - old_entries = d.entries - old_indexes = d.indexes - old_size = len(old_indexes) - # make a 'new_size' estimate and shrink it if there are many - # deleted entry markers. See CPython for why it is a good idea to - # quadruple the dictionary size as long as it's not too big. - # count the number of valid entries - i = 0 + # first set num_items to its correct, up-to-date value + entries = d.entries num_items = 0 - while i < d.num_items: - if old_entries.valid(i): + for i in range(len(entries)): + if entries.valid(i): num_items += 1 - i += 1 - if num_items > 50000: new_estimate = (num_items + 1) * 2 - else: new_estimate = (num_items + 1) * 4 - new_size = rdict.DICT_INITSIZE - while new_size <= new_estimate: - new_size *= 2 - # - new_item_size = new_size // 3 * 2 + 1 - d.entries = lltype.typeOf(old_entries).TO.allocate(new_item_size) - d.indexes = lltype.typeOf(d).TO.indexes.TO.allocate(new_size) - i = 0 - indexes = d.indexes - j = 0 - while i < old_size: - index = old_indexes[i] - if old_entries.valid(index): - hash = old_entries.hash(index) - lookup_i = rdict.ll_dict_lookup_clean(d, hash) - indexes[lookup_i] = j - #llop.debug_print(lltype.Void, "inserting", hex(hash), i, - # "to", lookup_i, index, "=>", j) - #llop.debug_print(lltype.Void, hex(old_entries[index].f_hash)) - d.entries[j].key = old_entries[index].key - d.entries[j].value = old_entries[index].value - j += 1 - i += 1 - d.num_items = j - d.resize_counter = new_item_size - j + d.num_items = num_items + rdict.ll_dict_resize(d) def specialize_make_weakdict(hop): hop.exception_cannot_occur() diff --git a/pypy/rlib/test/test_rweakkeydict.py b/pypy/rlib/test/test_rweakkeydict.py --- a/pypy/rlib/test/test_rweakkeydict.py +++ b/pypy/rlib/test/test_rweakkeydict.py @@ -36,25 +36,19 @@ d = prebuilt if d is None: d = RWeakKeyDictionary(KX, VX) - llop.debug_print(lltype.Void, "XXX 1") k1, k3, v1, v2, v3 = g(d) rgc.collect(); rgc.collect() - llop.debug_print(lltype.Void, "XXX 2") assert d.get(k1) is v1 assert d.get(k3) is v3 assert d.get(k1) is not v2 assert d.get(k3) is not v2 - llop.debug_print(lltype.Void, "XXX 3") assert d.length() == 2 - llop.debug_print(lltype.Void, "XXX 4") d.set(k1, None) assert d.get(k1) is None assert d.get(k3) is v3 assert d.length() == 1 - llop.debug_print(lltype.Void, "XXX 5") # resizing should also work lots_of_keys = [KX() for i in range(loop)] - llop.debug_print(lltype.Void, "XXX 6") for k in lots_of_keys: d.set(k, v1) for k in lots_of_keys: @@ -63,26 +57,19 @@ assert d.get(k3) is v3 assert d.length() == loop + 1 # a subclass - llop.debug_print(lltype.Void, "XXX 7") ky = KY() vy = VY() d.set(ky, vy) assert d.get(ky) is vy assert d.length() == loop + 2 # deleting by storing Nones - llop.debug_print(lltype.Void, "XXX 8") for k in lots_of_keys: d.set(k, None) - llop.debug_print(lltype.Void, "XXX 9") for k in lots_of_keys: assert d.get(k) is None - llop.debug_print(lltype.Void, "XXX 10") assert d.get(k1) is None - llop.debug_print(lltype.Void, "XXX 11") assert d.get(k3) is v3 - llop.debug_print(lltype.Void, "XXX 12") assert d.get(ky) is vy - llop.debug_print(lltype.Void, "XXX 13") assert d.length() == 2 return f diff --git a/pypy/rpython/lltypesystem/rdict_old.py b/pypy/rpython/lltypesystem/rdict_old.py new file mode 100644 --- /dev/null +++ b/pypy/rpython/lltypesystem/rdict_old.py @@ -0,0 +1,927 @@ +from pypy.tool.pairtype import pairtype +from pypy.objspace.flow.model import Constant +from pypy.rpython.rdict import (AbstractDictRepr, AbstractDictIteratorRepr, + rtype_newdict) +from pypy.rpython.lltypesystem import lltype +from pypy.rlib import objectmodel, jit +from pypy.rlib.debug import ll_assert +from pypy.rlib.rarithmetic import r_uint, intmask, LONG_BIT +from pypy.rpython import rmodel +from pypy.rpython.error import TyperError + + +HIGHEST_BIT = intmask(1 << (LONG_BIT - 1)) +MASK = intmask(HIGHEST_BIT - 1) + +# ____________________________________________________________ +# +# generic implementation of RPython dictionary, with parametric DICTKEY and +# DICTVALUE types. +# +# XXX for immutable dicts, the array should be inlined and +# resize_counter and everused are not needed. +# +# struct dictentry { +# DICTKEY key; +# bool f_valid; # (optional) the entry is filled +# bool f_everused; # (optional) the entry is or has ever been filled +# DICTVALUE value; +# int f_hash; # (optional) key hash, if hard to recompute +# } +# +# struct dicttable { +# int num_items; +# int resize_counter; +# Array *entries; +# (Function DICTKEY, DICTKEY -> bool) *fnkeyeq; +# (Function DICTKEY -> int) *fnkeyhash; +# } +# +# + +class DictRepr(AbstractDictRepr): + + def __init__(self, rtyper, key_repr, value_repr, dictkey, dictvalue, + custom_eq_hash=None, force_non_null=False): + self.rtyper = rtyper + self.DICT = lltype.GcForwardReference() + self.lowleveltype = lltype.Ptr(self.DICT) + self.custom_eq_hash = custom_eq_hash is not None + if not isinstance(key_repr, rmodel.Repr): # not computed yet, done by setup() + assert callable(key_repr) + self._key_repr_computer = key_repr + else: + self.external_key_repr, self.key_repr = self.pickkeyrepr(key_repr) + if not isinstance(value_repr, rmodel.Repr): # not computed yet, done by setup() + assert callable(value_repr) + self._value_repr_computer = value_repr + else: + self.external_value_repr, self.value_repr = self.pickrepr(value_repr) + self.dictkey = dictkey + self.dictvalue = dictvalue + self.dict_cache = {} + self._custom_eq_hash_repr = custom_eq_hash + self.force_non_null = force_non_null + # setup() needs to be called to finish this initialization + + def _externalvsinternal(self, rtyper, item_repr): + return rmodel.externalvsinternal(self.rtyper, item_repr) + + def _setup_repr(self): + if 'key_repr' not in self.__dict__: + key_repr = self._key_repr_computer() + self.external_key_repr, self.key_repr = self.pickkeyrepr(key_repr) + if 'value_repr' not in self.__dict__: + self.external_value_repr, self.value_repr = self.pickrepr(self._value_repr_computer()) + if isinstance(self.DICT, lltype.GcForwardReference): + self.DICTKEY = self.key_repr.lowleveltype + self.DICTVALUE = self.value_repr.lowleveltype + + # compute the shape of the DICTENTRY structure + entryfields = [] + entrymeths = { + 'allocate': lltype.typeMethod(_ll_malloc_entries), + 'delete': _ll_free_entries, + 'must_clear_key': (isinstance(self.DICTKEY, lltype.Ptr) + and self.DICTKEY._needsgc()), + 'must_clear_value': (isinstance(self.DICTVALUE, lltype.Ptr) + and self.DICTVALUE._needsgc()), + } + + # * the key + entryfields.append(("key", self.DICTKEY)) + + # * if NULL is not a valid ll value for the key or the value + # field of the entry, it can be used as a marker for + # never-used entries. Otherwise, we need an explicit flag. + s_key = self.dictkey.s_value + s_value = self.dictvalue.s_value + nullkeymarker = not self.key_repr.can_ll_be_null(s_key) + nullvaluemarker = not self.value_repr.can_ll_be_null(s_value) + if self.force_non_null: + if not nullkeymarker: + rmodel.warning("%s can be null, but forcing non-null in dict key" % s_key) + nullkeymarker = True + if not nullvaluemarker: + rmodel.warning("%s can be null, but forcing non-null in dict value" % s_value) + nullvaluemarker = True + dummykeyobj = self.key_repr.get_ll_dummyval_obj(self.rtyper, + s_key) + dummyvalueobj = self.value_repr.get_ll_dummyval_obj(self.rtyper, + s_value) + + # * the state of the entry - trying to encode it as dummy objects + if nullkeymarker and dummykeyobj: + # all the state can be encoded in the key + entrymeths['everused'] = ll_everused_from_key + entrymeths['dummy_obj'] = dummykeyobj + entrymeths['valid'] = ll_valid_from_key + entrymeths['mark_deleted'] = ll_mark_deleted_in_key + # the key is overwritten by 'dummy' when the entry is deleted + entrymeths['must_clear_key'] = False + + elif nullvaluemarker and dummyvalueobj: + # all the state can be encoded in the value + entrymeths['everused'] = ll_everused_from_value + entrymeths['dummy_obj'] = dummyvalueobj + entrymeths['valid'] = ll_valid_from_value + entrymeths['mark_deleted'] = ll_mark_deleted_in_value + # value is overwritten by 'dummy' when entry is deleted + entrymeths['must_clear_value'] = False + + else: + # we need a flag to know if the entry was ever used + # (we cannot use a NULL as a marker for this, because + # the key and value will be reset to NULL to clear their + # reference) + entryfields.append(("f_everused", lltype.Bool)) + entrymeths['everused'] = ll_everused_from_flag + + # can we still rely on a dummy obj to mark deleted entries? + if dummykeyobj: + entrymeths['dummy_obj'] = dummykeyobj + entrymeths['valid'] = ll_valid_from_key + entrymeths['mark_deleted'] = ll_mark_deleted_in_key + # key is overwritten by 'dummy' when entry is deleted + entrymeths['must_clear_key'] = False + elif dummyvalueobj: + entrymeths['dummy_obj'] = dummyvalueobj + entrymeths['valid'] = ll_valid_from_value + entrymeths['mark_deleted'] = ll_mark_deleted_in_value + # value is overwritten by 'dummy' when entry is deleted + entrymeths['must_clear_value'] = False + else: + entryfields.append(("f_valid", lltype.Bool)) + entrymeths['valid'] = ll_valid_from_flag + entrymeths['mark_deleted'] = ll_mark_deleted_in_flag + + # * the value + entryfields.append(("value", self.DICTVALUE)) + + # * the hash, if needed + if self.custom_eq_hash: + fasthashfn = None + else: + fasthashfn = self.key_repr.get_ll_fasthash_function() + if fasthashfn is None: + entryfields.append(("f_hash", lltype.Signed)) + entrymeths['hash'] = ll_hash_from_cache + else: + entrymeths['hash'] = ll_hash_recomputed + entrymeths['fasthashfn'] = fasthashfn + + # Build the lltype data structures + self.DICTENTRY = lltype.Struct("dictentry", *entryfields) + self.DICTENTRYARRAY = lltype.GcArray(self.DICTENTRY, + adtmeths=entrymeths) + fields = [ ("num_items", lltype.Signed), + ("resize_counter", lltype.Signed), + ("entries", lltype.Ptr(self.DICTENTRYARRAY)) ] + if self.custom_eq_hash: + self.r_rdict_eqfn, self.r_rdict_hashfn = self._custom_eq_hash_repr() + fields.extend([ ("fnkeyeq", self.r_rdict_eqfn.lowleveltype), + ("fnkeyhash", self.r_rdict_hashfn.lowleveltype) ]) + adtmeths = { + 'keyhash': ll_keyhash_custom, + 'keyeq': ll_keyeq_custom, + 'r_rdict_eqfn': self.r_rdict_eqfn, + 'r_rdict_hashfn': self.r_rdict_hashfn, + 'paranoia': True, + } + else: + # figure out which functions must be used to hash and compare + ll_keyhash = self.key_repr.get_ll_hash_function() + ll_keyeq = self.key_repr.get_ll_eq_function() # can be None + ll_keyhash = lltype.staticAdtMethod(ll_keyhash) + if ll_keyeq is not None: + ll_keyeq = lltype.staticAdtMethod(ll_keyeq) + adtmeths = { + 'keyhash': ll_keyhash, + 'keyeq': ll_keyeq, + 'paranoia': False, + } + adtmeths['KEY'] = self.DICTKEY + adtmeths['VALUE'] = self.DICTVALUE + adtmeths['allocate'] = lltype.typeMethod(_ll_malloc_dict) + self.DICT.become(lltype.GcStruct("dicttable", adtmeths=adtmeths, + *fields)) + + + def convert_const(self, dictobj): + from pypy.rpython.lltypesystem import llmemory + # get object from bound dict methods + #dictobj = getattr(dictobj, '__self__', dictobj) + if dictobj is None: + return lltype.nullptr(self.DICT) + if not isinstance(dictobj, (dict, objectmodel.r_dict)): + raise TypeError("expected a dict: %r" % (dictobj,)) + try: + key = Constant(dictobj) + return self.dict_cache[key] + except KeyError: + self.setup() + l_dict = ll_newdict_size(self.DICT, len(dictobj)) + self.dict_cache[key] = l_dict + r_key = self.key_repr + if r_key.lowleveltype == llmemory.Address: + raise TypeError("No prebuilt dicts of address keys") + r_value = self.value_repr + if isinstance(dictobj, objectmodel.r_dict): + if self.r_rdict_eqfn.lowleveltype != lltype.Void: + l_fn = self.r_rdict_eqfn.convert_const(dictobj.key_eq) + l_dict.fnkeyeq = l_fn + if self.r_rdict_hashfn.lowleveltype != lltype.Void: + l_fn = self.r_rdict_hashfn.convert_const(dictobj.key_hash) + l_dict.fnkeyhash = l_fn + + for dictkeycontainer, dictvalue in dictobj._dict.items(): + llkey = r_key.convert_const(dictkeycontainer.key) + llvalue = r_value.convert_const(dictvalue) + ll_dict_insertclean(l_dict, llkey, llvalue, + dictkeycontainer.hash) + return l_dict + + else: + for dictkey, dictvalue in dictobj.items(): + llkey = r_key.convert_const(dictkey) + llvalue = r_value.convert_const(dictvalue) + ll_dict_insertclean(l_dict, llkey, llvalue, + l_dict.keyhash(llkey)) + return l_dict + + def rtype_len(self, hop): + v_dict, = hop.inputargs(self) + return hop.gendirectcall(ll_dict_len, v_dict) + + def rtype_is_true(self, hop): + v_dict, = hop.inputargs(self) + return hop.gendirectcall(ll_dict_is_true, v_dict) + + def make_iterator_repr(self, *variant): + return DictIteratorRepr(self, *variant) + + def rtype_method_get(self, hop): + v_dict, v_key, v_default = hop.inputargs(self, self.key_repr, + self.value_repr) + hop.exception_cannot_occur() + v_res = hop.gendirectcall(ll_get, v_dict, v_key, v_default) + return self.recast_value(hop.llops, v_res) + + def rtype_method_setdefault(self, hop): + v_dict, v_key, v_default = hop.inputargs(self, self.key_repr, + self.value_repr) + hop.exception_cannot_occur() + v_res = hop.gendirectcall(ll_setdefault, v_dict, v_key, v_default) + return self.recast_value(hop.llops, v_res) + + def rtype_method_copy(self, hop): + v_dict, = hop.inputargs(self) + hop.exception_cannot_occur() + return hop.gendirectcall(ll_copy, v_dict) + + def rtype_method_update(self, hop): + v_dic1, v_dic2 = hop.inputargs(self, self) + hop.exception_cannot_occur() + return hop.gendirectcall(ll_update, v_dic1, v_dic2) + + def _rtype_method_kvi(self, hop, ll_func): + v_dic, = hop.inputargs(self) + r_list = hop.r_result + cLIST = hop.inputconst(lltype.Void, r_list.lowleveltype.TO) + hop.exception_cannot_occur() + return hop.gendirectcall(ll_func, cLIST, v_dic) + + def rtype_method_keys(self, hop): + return self._rtype_method_kvi(hop, ll_dict_keys) + + def rtype_method_values(self, hop): + return self._rtype_method_kvi(hop, ll_dict_values) + + def rtype_method_items(self, hop): + return self._rtype_method_kvi(hop, ll_dict_items) + + def rtype_method_iterkeys(self, hop): + hop.exception_cannot_occur() + return DictIteratorRepr(self, "keys").newiter(hop) + + def rtype_method_itervalues(self, hop): + hop.exception_cannot_occur() + return DictIteratorRepr(self, "values").newiter(hop) + + def rtype_method_iteritems(self, hop): + hop.exception_cannot_occur() + return DictIteratorRepr(self, "items").newiter(hop) + + def rtype_method_clear(self, hop): + v_dict, = hop.inputargs(self) + hop.exception_cannot_occur() + return hop.gendirectcall(ll_clear, v_dict) + + def rtype_method_popitem(self, hop): + v_dict, = hop.inputargs(self) + r_tuple = hop.r_result + cTUPLE = hop.inputconst(lltype.Void, r_tuple.lowleveltype) + hop.exception_is_here() + return hop.gendirectcall(ll_popitem, cTUPLE, v_dict) + + def rtype_method_pop(self, hop): + if hop.nb_args == 2: + v_args = hop.inputargs(self, self.key_repr) + target = ll_pop + elif hop.nb_args == 3: + v_args = hop.inputargs(self, self.key_repr, self.value_repr) + target = ll_pop_default + hop.exception_is_here() + v_res = hop.gendirectcall(target, *v_args) + return self.recast_value(hop.llops, v_res) + +class __extend__(pairtype(DictRepr, rmodel.Repr)): + + def rtype_getitem((r_dict, r_key), hop): + v_dict, v_key = hop.inputargs(r_dict, r_dict.key_repr) + if not r_dict.custom_eq_hash: + hop.has_implicit_exception(KeyError) # record that we know about it + hop.exception_is_here() + v_res = hop.gendirectcall(ll_dict_getitem, v_dict, v_key) + return r_dict.recast_value(hop.llops, v_res) + + def rtype_delitem((r_dict, r_key), hop): + v_dict, v_key = hop.inputargs(r_dict, r_dict.key_repr) + if not r_dict.custom_eq_hash: + hop.has_implicit_exception(KeyError) # record that we know about it + hop.exception_is_here() + return hop.gendirectcall(ll_dict_delitem, v_dict, v_key) + + def rtype_setitem((r_dict, r_key), hop): + v_dict, v_key, v_value = hop.inputargs(r_dict, r_dict.key_repr, r_dict.value_repr) + if r_dict.custom_eq_hash: + hop.exception_is_here() + else: + hop.exception_cannot_occur() + hop.gendirectcall(ll_dict_setitem, v_dict, v_key, v_value) + + def rtype_contains((r_dict, r_key), hop): + v_dict, v_key = hop.inputargs(r_dict, r_dict.key_repr) + hop.exception_is_here() + return hop.gendirectcall(ll_contains, v_dict, v_key) + +class __extend__(pairtype(DictRepr, DictRepr)): + def convert_from_to((r_dict1, r_dict2), v, llops): + # check that we don't convert from Dicts with + # different key/value types + if r_dict1.dictkey is None or r_dict2.dictkey is None: + return NotImplemented + if r_dict1.dictkey is not r_dict2.dictkey: + return NotImplemented + if r_dict1.dictvalue is None or r_dict2.dictvalue is None: + return NotImplemented + if r_dict1.dictvalue is not r_dict2.dictvalue: + return NotImplemented + return v + +# ____________________________________________________________ +# +# Low-level methods. These can be run for testing, but are meant to +# be direct_call'ed from rtyped flow graphs, which means that they will +# get flowed and annotated, mostly with SomePtr. + +def ll_everused_from_flag(entries, i): + return entries[i].f_everused + +def ll_everused_from_key(entries, i): + return bool(entries[i].key) + +def ll_everused_from_value(entries, i): + return bool(entries[i].value) + +def ll_valid_from_flag(entries, i): + return entries[i].f_valid + +def ll_mark_deleted_in_flag(entries, i): + entries[i].f_valid = False + +def ll_valid_from_key(entries, i): + ENTRIES = lltype.typeOf(entries).TO + dummy = ENTRIES.dummy_obj.ll_dummy_value + return entries.everused(i) and entries[i].key != dummy + +def ll_mark_deleted_in_key(entries, i): + ENTRIES = lltype.typeOf(entries).TO + dummy = ENTRIES.dummy_obj.ll_dummy_value + entries[i].key = dummy + +def ll_valid_from_value(entries, i): + ENTRIES = lltype.typeOf(entries).TO + dummy = ENTRIES.dummy_obj.ll_dummy_value + return entries.everused(i) and entries[i].value != dummy + +def ll_mark_deleted_in_value(entries, i): + ENTRIES = lltype.typeOf(entries).TO + dummy = ENTRIES.dummy_obj.ll_dummy_value + entries[i].value = dummy + +def ll_hash_from_cache(entries, i): + return entries[i].f_hash + +def ll_hash_recomputed(entries, i): + ENTRIES = lltype.typeOf(entries).TO + return ENTRIES.fasthashfn(entries[i].key) + +def ll_get_value(d, i): + return d.entries[i].value + +def ll_keyhash_custom(d, key): + DICT = lltype.typeOf(d).TO + return objectmodel.hlinvoke(DICT.r_rdict_hashfn, d.fnkeyhash, key) + +def ll_keyeq_custom(d, key1, key2): + DICT = lltype.typeOf(d).TO + return objectmodel.hlinvoke(DICT.r_rdict_eqfn, d.fnkeyeq, key1, key2) + +def ll_dict_len(d): + return d.num_items + +def ll_dict_is_true(d): + # check if a dict is True, allowing for None + return bool(d) and d.num_items != 0 + +def ll_dict_getitem(d, key): + i = ll_dict_lookup(d, key, d.keyhash(key)) + if not i & HIGHEST_BIT: + return ll_get_value(d, i) + else: + raise KeyError + +def ll_dict_setitem(d, key, value): + hash = d.keyhash(key) + i = ll_dict_lookup(d, key, hash) + return _ll_dict_setitem_lookup_done(d, key, value, hash, i) + +# It may be safe to look inside always, it has a few branches though, and their +# frequencies needs to be investigated. + at jit.look_inside_iff(lambda d, key, value, hash, i: jit.isvirtual(d) and jit.isconstant(key)) +def _ll_dict_setitem_lookup_done(d, key, value, hash, i): + valid = (i & HIGHEST_BIT) == 0 + i = i & MASK + ENTRY = lltype.typeOf(d.entries).TO.OF + entry = d.entries[i] + if not d.entries.everused(i): + # a new entry that was never used before + ll_assert(not valid, "valid but not everused") + rc = d.resize_counter - 3 + if rc <= 0: # if needed, resize the dict -- before the insertion + ll_dict_resize(d) + i = ll_dict_lookup_clean(d, hash) # then redo the lookup for 'key' + entry = d.entries[i] + rc = d.resize_counter - 3 + ll_assert(rc > 0, "ll_dict_resize failed?") + d.resize_counter = rc + if hasattr(ENTRY, 'f_everused'): entry.f_everused = True + entry.value = value + else: + # override an existing or deleted entry + entry.value = value + if valid: + return + entry.key = key + if hasattr(ENTRY, 'f_hash'): entry.f_hash = hash + if hasattr(ENTRY, 'f_valid'): entry.f_valid = True + d.num_items += 1 + +def ll_dict_insertclean(d, key, value, hash): + # Internal routine used by ll_dict_resize() to insert an item which is + # known to be absent from the dict. This routine also assumes that + # the dict contains no deleted entries. This routine has the advantage + # of never calling d.keyhash() and d.keyeq(), so it cannot call back + # to user code. ll_dict_insertclean() doesn't resize the dict, either. + i = ll_dict_lookup_clean(d, hash) + ENTRY = lltype.typeOf(d.entries).TO.OF + entry = d.entries[i] + entry.value = value + entry.key = key + if hasattr(ENTRY, 'f_hash'): entry.f_hash = hash + if hasattr(ENTRY, 'f_valid'): entry.f_valid = True + if hasattr(ENTRY, 'f_everused'): entry.f_everused = True + d.num_items += 1 + d.resize_counter -= 3 + +def ll_dict_delitem(d, key): + i = ll_dict_lookup(d, key, d.keyhash(key)) + if i & HIGHEST_BIT: + raise KeyError + _ll_dict_del(d, i) + + at jit.look_inside_iff(lambda d, i: jit.isvirtual(d) and jit.isconstant(i)) +def _ll_dict_del(d, i): + d.entries.mark_deleted(i) + d.num_items -= 1 + # clear the key and the value if they are GC pointers + ENTRIES = lltype.typeOf(d.entries).TO + ENTRY = ENTRIES.OF + entry = d.entries[i] + if ENTRIES.must_clear_key: + entry.key = lltype.nullptr(ENTRY.key.TO) + if ENTRIES.must_clear_value: + entry.value = lltype.nullptr(ENTRY.value.TO) + # + # The rest is commented out: like CPython we no longer shrink the + # dictionary here. It may shrink later if we try to append a number + # of new items to it. Unsure if this behavior was designed in + # CPython or is accidental. A design reason would be that if you + # delete all items in a dictionary (e.g. with a series of + # popitem()), then CPython avoids shrinking the table several times. + #num_entries = len(d.entries) + #if num_entries > DICT_INITSIZE and d.num_items <= num_entries / 4: + # ll_dict_resize(d) + # A previous xxx: move the size checking and resize into a single + # call which is opaque to the JIT when the dict isn't virtual, to + # avoid extra branches. + +def ll_dict_resize(d): + old_entries = d.entries + old_size = len(old_entries) + # make a 'new_size' estimate and shrink it if there are many + # deleted entry markers. See CPython for why it is a good idea to + # quadruple the dictionary size as long as it's not too big. + num_items = d.num_items + 1 + if num_items > 50000: new_estimate = num_items * 2 + else: new_estimate = num_items * 4 + new_size = DICT_INITSIZE + while new_size <= new_estimate: + new_size *= 2 + # + d.entries = lltype.typeOf(old_entries).TO.allocate(new_size) + d.num_items = 0 + d.resize_counter = new_size * 2 + i = 0 + while i < old_size: + if old_entries.valid(i): + hash = old_entries.hash(i) + entry = old_entries[i] + ll_dict_insertclean(d, entry.key, entry.value, hash) + i += 1 + old_entries.delete() +ll_dict_resize.oopspec = 'dict.resize(d)' + +# ------- a port of CPython's dictobject.c's lookdict implementation ------- +PERTURB_SHIFT = 5 + + at jit.look_inside_iff(lambda d, key, hash: jit.isvirtual(d) and jit.isconstant(key)) +def ll_dict_lookup(d, key, hash): + entries = d.entries + ENTRIES = lltype.typeOf(entries).TO + direct_compare = not hasattr(ENTRIES, 'no_direct_compare') + mask = len(entries) - 1 + i = hash & mask + # do the first try before any looping + if entries.valid(i): + checkingkey = entries[i].key + if direct_compare and checkingkey == key: + return i # found the entry + if d.keyeq is not None and entries.hash(i) == hash: + # correct hash, maybe the key is e.g. a different pointer to + # an equal object + found = d.keyeq(checkingkey, key) + if d.paranoia: + if (entries != d.entries or + not entries.valid(i) or entries[i].key != checkingkey): + # the compare did major nasty stuff to the dict: start over + return ll_dict_lookup(d, key, hash) + if found: + return i # found the entry + freeslot = -1 + elif entries.everused(i): + freeslot = i + else: + return i | HIGHEST_BIT # pristine entry -- lookup failed + + # In the loop, a deleted entry (everused and not valid) is by far + # (factor of 100s) the least likely outcome, so test for that last. + perturb = r_uint(hash) + while 1: + # compute the next index using unsigned arithmetic + i = r_uint(i) + i = (i << 2) + i + perturb + 1 + i = intmask(i) & mask + # keep 'i' as a signed number here, to consistently pass signed + # arguments to the small helper methods. + if not entries.everused(i): + if freeslot == -1: + freeslot = i + return freeslot | HIGHEST_BIT + elif entries.valid(i): + checkingkey = entries[i].key + if direct_compare and checkingkey == key: + return i + if d.keyeq is not None and entries.hash(i) == hash: + # correct hash, maybe the key is e.g. a different pointer to + # an equal object + found = d.keyeq(checkingkey, key) + if d.paranoia: + if (entries != d.entries or + not entries.valid(i) or entries[i].key != checkingkey): + # the compare did major nasty stuff to the dict: + # start over + return ll_dict_lookup(d, key, hash) + if found: + return i # found the entry + elif freeslot == -1: + freeslot = i + perturb >>= PERTURB_SHIFT + +def ll_dict_lookup_clean(d, hash): + # a simplified version of ll_dict_lookup() which assumes that the + # key is new, and the dictionary doesn't contain deleted entries. + # It only finds the next free slot for the given hash. + entries = d.entries + mask = len(entries) - 1 + i = hash & mask + perturb = r_uint(hash) + while entries.everused(i): + i = r_uint(i) + i = (i << 2) + i + perturb + 1 + i = intmask(i) & mask + perturb >>= PERTURB_SHIFT + return i + +# ____________________________________________________________ +# +# Irregular operations. + +DICT_INITSIZE = 8 + +def ll_newdict(DICT): + d = DICT.allocate() + d.entries = DICT.entries.TO.allocate(DICT_INITSIZE) + d.num_items = 0 + d.resize_counter = DICT_INITSIZE * 2 + return d + +def ll_newdict_size(DICT, length_estimate): + length_estimate = (length_estimate // 2) * 3 + n = DICT_INITSIZE + while n < length_estimate: + n *= 2 + d = DICT.allocate() + d.entries = DICT.entries.TO.allocate(n) + d.num_items = 0 + d.resize_counter = n * 2 + return d + +# pypy.rpython.memory.lldict uses a dict based on Struct and Array +# instead of GcStruct and GcArray, which is done by using different +# 'allocate' and 'delete' adtmethod implementations than the ones below +def _ll_malloc_dict(DICT): + return lltype.malloc(DICT) +def _ll_malloc_entries(ENTRIES, n): + return lltype.malloc(ENTRIES, n, zero=True) +def _ll_free_entries(entries): + pass + + +def rtype_r_dict(hop, i_force_non_null=None): + r_dict = hop.r_result + if not r_dict.custom_eq_hash: + raise TyperError("r_dict() call does not return an r_dict instance") + v_eqfn = hop.inputarg(r_dict.r_rdict_eqfn, arg=0) + v_hashfn = hop.inputarg(r_dict.r_rdict_hashfn, arg=1) + if i_force_non_null is not None: + assert i_force_non_null == 2 + hop.inputarg(lltype.Void, arg=2) + cDICT = hop.inputconst(lltype.Void, r_dict.DICT) + hop.exception_cannot_occur() + v_result = hop.gendirectcall(ll_newdict, cDICT) + if r_dict.r_rdict_eqfn.lowleveltype != lltype.Void: + cname = hop.inputconst(lltype.Void, 'fnkeyeq') + hop.genop('setfield', [v_result, cname, v_eqfn]) + if r_dict.r_rdict_hashfn.lowleveltype != lltype.Void: + cname = hop.inputconst(lltype.Void, 'fnkeyhash') + hop.genop('setfield', [v_result, cname, v_hashfn]) + return v_result + +# ____________________________________________________________ +# +# Iteration. + +class DictIteratorRepr(AbstractDictIteratorRepr): + + def __init__(self, r_dict, variant="keys"): + self.r_dict = r_dict + self.variant = variant + self.lowleveltype = lltype.Ptr(lltype.GcStruct('dictiter', + ('dict', r_dict.lowleveltype), + ('index', lltype.Signed))) + self.ll_dictiter = ll_dictiter + self.ll_dictnext = ll_dictnext_group[variant] + + +def ll_dictiter(ITERPTR, d): + iter = lltype.malloc(ITERPTR.TO) + iter.dict = d + iter.index = 0 + return iter + +def _make_ll_dictnext(kind): + # make three versions of the following function: keys, values, items + @jit.look_inside_iff(lambda RETURNTYPE, iter: jit.isvirtual(iter) + and (iter.dict is None or + jit.isvirtual(iter.dict))) + @jit.oopspec("dictiter.next%s(iter)" % kind) + def ll_dictnext(RETURNTYPE, iter): + # note that RETURNTYPE is None for keys and values + dict = iter.dict + if dict: + entries = dict.entries + index = iter.index + entries_len = len(entries) + while index < entries_len: + entry = entries[index] + is_valid = entries.valid(index) + index = index + 1 + if is_valid: + iter.index = index + if RETURNTYPE is lltype.Void: + return None + elif kind == 'items': + r = lltype.malloc(RETURNTYPE.TO) + r.item0 = recast(RETURNTYPE.TO.item0, entry.key) + r.item1 = recast(RETURNTYPE.TO.item1, entry.value) + return r + elif kind == 'keys': + return entry.key + elif kind == 'values': + return entry.value + # clear the reference to the dict and prevent restarts + iter.dict = lltype.nullptr(lltype.typeOf(iter).TO.dict.TO) + raise StopIteration + return ll_dictnext + +ll_dictnext_group = {'keys' : _make_ll_dictnext('keys'), + 'values': _make_ll_dictnext('values'), + 'items' : _make_ll_dictnext('items')} + +# _____________________________________________________________ +# methods + +def ll_get(dict, key, default): + i = ll_dict_lookup(dict, key, dict.keyhash(key)) + if not i & HIGHEST_BIT: + return ll_get_value(dict, i) + else: + return default + +def ll_setdefault(dict, key, default): + hash = dict.keyhash(key) + i = ll_dict_lookup(dict, key, hash) + if not i & HIGHEST_BIT: + return ll_get_value(dict, i) + else: + _ll_dict_setitem_lookup_done(dict, key, default, hash, i) + return default + +def ll_copy(dict): + DICT = lltype.typeOf(dict).TO + dictsize = len(dict.entries) + d = DICT.allocate() + d.entries = DICT.entries.TO.allocate(dictsize) + d.num_items = dict.num_items + d.resize_counter = dict.resize_counter + if hasattr(DICT, 'fnkeyeq'): d.fnkeyeq = dict.fnkeyeq + if hasattr(DICT, 'fnkeyhash'): d.fnkeyhash = dict.fnkeyhash + i = 0 + while i < dictsize: + d_entry = d.entries[i] + entry = dict.entries[i] + ENTRY = lltype.typeOf(d.entries).TO.OF + d_entry.key = entry.key + if hasattr(ENTRY, 'f_valid'): d_entry.f_valid = entry.f_valid + if hasattr(ENTRY, 'f_everused'): d_entry.f_everused = entry.f_everused + d_entry.value = entry.value + if hasattr(ENTRY, 'f_hash'): d_entry.f_hash = entry.f_hash + i += 1 + return d +ll_copy.oopspec = 'dict.copy(dict)' + +def ll_clear(d): + if (len(d.entries) == DICT_INITSIZE and + d.resize_counter == DICT_INITSIZE * 2): + return + old_entries = d.entries + d.entries = lltype.typeOf(old_entries).TO.allocate(DICT_INITSIZE) + d.num_items = 0 + d.resize_counter = DICT_INITSIZE * 2 + old_entries.delete() +ll_clear.oopspec = 'dict.clear(d)' + +def ll_update(dic1, dic2): + entries = dic2.entries + d2len = len(entries) + i = 0 + while i < d2len: + if entries.valid(i): + entry = entries[i] + hash = entries.hash(i) + key = entry.key + j = ll_dict_lookup(dic1, key, hash) + _ll_dict_setitem_lookup_done(dic1, key, entry.value, hash, j) + i += 1 +ll_update.oopspec = 'dict.update(dic1, dic2)' + +# this is an implementation of keys(), values() and items() +# in a single function. +# note that by specialization on func, three different +# and very efficient functions are created. + +def recast(P, v): + if isinstance(P, lltype.Ptr): + return lltype.cast_pointer(P, v) + else: + return v + +def _make_ll_keys_values_items(kind): + def ll_kvi(LIST, dic): + res = LIST.ll_newlist(dic.num_items) + entries = dic.entries + dlen = len(entries) + items = res.ll_items() + i = 0 + p = 0 + while i < dlen: + if entries.valid(i): + ELEM = lltype.typeOf(items).TO.OF + if ELEM is not lltype.Void: + entry = entries[i] + if kind == 'items': + r = lltype.malloc(ELEM.TO) + r.item0 = recast(ELEM.TO.item0, entry.key) + r.item1 = recast(ELEM.TO.item1, entry.value) + items[p] = r + elif kind == 'keys': + items[p] = recast(ELEM, entry.key) + elif kind == 'values': + items[p] = recast(ELEM, entry.value) + p += 1 + i += 1 + assert p == res.ll_length() + return res + ll_kvi.oopspec = 'dict.%s(dic)' % kind + return ll_kvi + +ll_dict_keys = _make_ll_keys_values_items('keys') +ll_dict_values = _make_ll_keys_values_items('values') +ll_dict_items = _make_ll_keys_values_items('items') + +def ll_contains(d, key): + i = ll_dict_lookup(d, key, d.keyhash(key)) + return not i & HIGHEST_BIT + +POPITEMINDEX = lltype.Struct('PopItemIndex', ('nextindex', lltype.Signed)) +global_popitem_index = lltype.malloc(POPITEMINDEX, zero=True, immortal=True) + +def _ll_getnextitem(dic): + entries = dic.entries + ENTRY = lltype.typeOf(entries).TO.OF + dmask = len(entries) - 1 + if hasattr(ENTRY, 'f_hash'): + if entries.valid(0): + return 0 + base = entries[0].f_hash + else: + base = global_popitem_index.nextindex + counter = 0 + while counter <= dmask: + i = (base + counter) & dmask + counter += 1 + if entries.valid(i): + break + else: + raise KeyError + if hasattr(ENTRY, 'f_hash'): + entries[0].f_hash = base + counter + else: + global_popitem_index.nextindex = base + counter + return i + +def ll_popitem(ELEM, dic): + i = _ll_getnextitem(dic) + entry = dic.entries[i] + r = lltype.malloc(ELEM.TO) + r.item0 = recast(ELEM.TO.item0, entry.key) + r.item1 = recast(ELEM.TO.item1, entry.value) + _ll_dict_del(dic, i) + return r + +def ll_pop(dic, key): + i = ll_dict_lookup(dic, key, dic.keyhash(key)) + if not i & HIGHEST_BIT: + value = ll_get_value(dic, i) + _ll_dict_del(dic, i) + return value + else: + raise KeyError + +def ll_pop_default(dic, key, dfl): + try: + return ll_pop(dic, key) + except KeyError: + return dfl diff --git a/pypy/rpython/memory/lldict.py b/pypy/rpython/memory/lldict.py --- a/pypy/rpython/memory/lldict.py +++ b/pypy/rpython/memory/lldict.py @@ -1,5 +1,5 @@ from pypy.rpython.lltypesystem import lltype, llmemory -from pypy.rpython.lltypesystem import rdict +from pypy.rpython.lltypesystem import rdict_old as rdict from pypy.rlib.objectmodel import we_are_translated from pypy.rpython.memory.support import mangle_hash From noreply at buildbot.pypy.org Mon Jan 7 01:56:04 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 7 Jan 2013 01:56:04 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: add repeats Message-ID: <20130107005604.C81CD1C0312@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59833:6bf8538f602f Date: 2013-01-07 02:55 +0200 http://bitbucket.org/pypy/pypy/changeset/6bf8538f602f/ Log: add repeats diff --git a/pypy/translator/goal/targetbenchdict.py b/pypy/translator/goal/targetbenchdict.py --- a/pypy/translator/goal/targetbenchdict.py +++ b/pypy/translator/goal/targetbenchdict.py @@ -19,27 +19,29 @@ def entry_point(argv): bench_no = int(argv[1]) bench_counter = int(argv[2]) - if bench_no == 1: - d = {} - for i in range(bench_counter): - d[str(i)] = None - if bench_no == 2: - d = {} - for i in range(bench_counter): - d[i] = A(1, 2, 3) - if bench_no == 3: - d = {} - a = A(1, 2, 3) - for i in range(bench_counter): - if i % 100 == 0: - a = A(1, 2, 3) - d[i] = a - if bench_no == 4: - d = {} - s = 0 - for i in range(bench_counter): - d[i % 100] = i - s += d[i % 100] + repeats = int(argv[3]) + for k in range(repeats): + if bench_no == 1: + d = {} + for i in range(bench_counter): + d[str(i)] = None + if bench_no == 2: + d = {} + for i in range(bench_counter): + d[i] = A(1, 2, 3) + if bench_no == 3: + d = {} + a = A(1, 2, 3) + for i in range(bench_counter): + if i % 100 == 0: + a = A(1, 2, 3) + d[i] = a + if bench_no == 4: + d = {} + s = 0 + for i in range(bench_counter): + d[i % 100] = i + s += d[i % 100] return 0 # _____ Define and setup target ___ From noreply at buildbot.pypy.org Mon Jan 7 02:22:07 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 7 Jan 2013 02:22:07 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: interface changes + fixes for strange hashes Message-ID: <20130107012207.35EE91C0AD6@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59834:10d7c1636f47 Date: 2013-01-07 03:21 +0200 http://bitbucket.org/pypy/pypy/changeset/10d7c1636f47/ Log: interface changes + fixes for strange hashes diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py --- a/pypy/rpython/lltypesystem/rdict.py +++ b/pypy/rpython/lltypesystem/rdict.py @@ -78,7 +78,10 @@ DICTENTRY = lltype.Struct("dictentry", *entryfields) DICTENTRYARRAY = lltype.GcArray(DICTENTRY, adtmeths=entrymeths) - array_adtmeths = {'allocate': lltype.typeMethod(_ll_malloc_indexes)} + array_adtmeths = {'allocate': lltype.typeMethod(_ll_malloc_indexes), + 'getitem': ll_dict_index_getitem, + 'setitem': ll_dict_index_setitem} + fields = [("num_items", lltype.Signed), ("resize_counter", lltype.Signed), ("entries", lltype.Ptr(DICTENTRYARRAY)), @@ -150,11 +153,14 @@ if isinstance(self.DICT, lltype.GcForwardReference): DICTKEY = self.key_repr.lowleveltype DICTVALUE = self.value_repr.lowleveltype - get_ll_dict(DICTKEY, DICTVALUE, DICT=self.DICT, - ll_fasthash_function=self.key_repr.get_ll_fasthash_function(), - ll_hash_function=self.key_repr.get_ll_hash_function(), - ll_eq_function=self.key_repr.get_ll_eq_function(), - get_custom_eq_hash=self.custom_eq_hash) + kwd = {} + if self.custom_eq_hash: + kwd['custom_eq_hash'] = self.custom_eq_hash + else: + kwd['ll_hash_function'] = self.key_repr.get_ll_hash_function() + kwd['ll_eq_function'] = self.key_repr.get_ll_eq_function() + kwd['ll_fasthash_function'] = self.key_repr.get_ll_fasthash_function() + get_ll_dict(DICTKEY, DICTVALUE, DICT=self.DICT, **kwd) if self.custom_eq_hash is not None: self.r_rdict_eqfn, self.r_rdict_hashfn = self.custom_eq_hash() @@ -344,8 +350,20 @@ ENTRIES = lltype.typeOf(entries).TO return ENTRIES.fasthashfn(entries[i].key) +def ll_dict_index_getitem(indexes, i): + return indexes[i] + +def ll_dict_index_setitem(indexes, i, v): + indexes[i] = v + +def ll_dict_copy_indexes(from_indexes, to_indexes): + i = 0 + while i < len(from_indexes): + to_indexes.setitem(i, from_indexes.getitem(i)) + i += 1 + def ll_get_value(d, i): - return d.entries[d.indexes[i]].value + return d.entries[d.indexes.getitem(i)].value def ll_keyhash_custom(d, key): DICT = lltype.typeOf(d).TO @@ -385,7 +403,7 @@ valid = (i & HIGHEST_BIT) == 0 i = i & MASK ENTRY = lltype.typeOf(d.entries).TO.OF - index = d.indexes[i] + index = d.indexes.getitem(i) if index == FREE: index = d.num_items entry = d.entries[index] @@ -401,12 +419,12 @@ ll_assert(rc > 0, "ll_dict_resize failed?") ll_assert(index < len(d.entries), "invalid insert") d.resize_counter = rc - d.indexes[i] = index + d.indexes.setitem(i, index) entry.value = value elif index == DELETED: index = d.num_items entry = d.entries[index] - d.indexes[i] = index + d.indexes.setitem(i, index) entry.value = value else: # override an existing or deleted entry @@ -426,7 +444,7 @@ @jit.look_inside_iff(lambda d, i: jit.isvirtual(d) and jit.isconstant(i)) def _ll_dict_del(d, i): - index = d.indexes[i] + index = d.indexes.getitem(i) ENTRIES = lltype.typeOf(d.entries).TO ENTRY = ENTRIES.OF if index != d.num_items - 1: @@ -434,7 +452,7 @@ key = old_entry.key to_insert_i = ll_dict_lookup(d, key, d.keyhash(key)) ll_assert(not to_insert_i & HIGHEST_BIT, "invalid entry") - d.indexes[to_insert_i] = index + d.indexes.setitem(to_insert_i, index) # copy the value new_entry = d.entries[index] new_entry.key = key @@ -442,7 +460,7 @@ if hasattr(ENTRY, 'f_hash'): new_entry.f_hash = old_entry.f_hash # clear the key and the value if they are GC pointers - d.indexes[i] = DELETED + d.indexes.setitem(i, DELETED) d.num_items -= 1 entry = d.entries[d.num_items] if ENTRIES.clear_key: @@ -478,18 +496,23 @@ new_size *= 2 # new_item_size = new_size // 3 * 2 + 1 - d.entries = lltype.typeOf(old_entries).TO.allocate(new_item_size) d.indexes = lltype.typeOf(d).TO.indexes.TO.allocate(new_size) d.resize_counter = new_item_size - d.num_items i = 0 indexes = d.indexes while i < old_size: - index = old_indexes[i] + index = old_indexes.getitem(i) if old_entries.valid(index): - indexes[ll_dict_lookup_clean(d, old_entries.hash(index))] = index + pos = ll_dict_lookup_clean(d, old_entries.hash(index)) + indexes.setitem(pos, index) i += 1 - rgc.ll_arraycopy(old_entries, d.entries, 0, 0, min(len(old_entries), - len(d.entries))) + if len(old_entries) != new_item_size: + d.entries = lltype.typeOf(old_entries).TO.allocate(new_item_size) + rgc.ll_arraycopy(old_entries, d.entries, 0, 0, min(len(old_entries), + len(d.entries))) + else: + # we just removed deleted items, but we didn't do anything else special + d.entries = old_entries ll_dict_resize.oopspec = 'dict.resize(d)' # ------- a port of CPython's dictobject.c's lookdict implementation ------- @@ -514,7 +537,7 @@ # do the first try before any looping ENTRIES = lltype.typeOf(entries).TO direct_compare = not hasattr(ENTRIES, 'no_direct_compare') - index = indexes[i] + index = indexes.getitem(i) if entries.valid(index): checkingkey = entries[index].key if direct_compare and checkingkey == key: @@ -526,7 +549,8 @@ #llop.debug_print(lltype.Void, "comparing keys", ll_debugrepr(checkingkey), ll_debugrepr(key), found) if d.paranoia: if (entries != d.entries or - not entries.valid(indexes[i]) or entries[index].key != checkingkey): + not entries.valid(indexes.getitem(i)) + or entries[index].key != checkingkey): # the compare did major nasty stuff to the dict: start over return ll_dict_lookup(d, key, hash) if found: @@ -545,7 +569,7 @@ i = r_uint(i) i = (i << 2) + i + perturb + 1 i = intmask(i) & mask - index = indexes[i] + index = indexes.getitem(i) # keep 'i' as a signed number here, to consistently pass signed # arguments to the small helper methods. if index == FREE: @@ -562,7 +586,7 @@ found = d.keyeq(checkingkey, key) if d.paranoia: if (entries != d.entries or - not entries.valid(indexes[i]) or + not entries.valid(indexes.getitem(i)) or entries[index].key != checkingkey): # the compare did major nasty stuff to the dict: # start over @@ -581,7 +605,7 @@ mask = len(indexes) - 1 i = hash & mask perturb = r_uint(hash) - while d.indexes[i] != FREE: + while indexes.getitem(i) != FREE: i = r_uint(i) i = (i << 2) + i + perturb + 1 i = intmask(i) & mask @@ -601,10 +625,10 @@ # XXX don't use _ll_items_allocate because of jit.unroll_safe, # should be *really* jit_unroll_iff d.indexes = lltype.malloc(DICT.indexes.TO, DICT_INITSIZE) + d.num_items = 0 for i in range(DICT_INITSIZE): - d.indexes[i] = FREE + d.indexes.setitem(i, FREE) d.entries = DICT.entries.TO.allocate(DICT_ITEMS_INITSIZE) - d.num_items = 0 d.resize_counter = DICT_ITEMS_INITSIZE return d @@ -739,7 +763,8 @@ d.resize_counter = dict.resize_counter if hasattr(DICT, 'fnkeyeq'): d.fnkeyeq = dict.fnkeyeq if hasattr(DICT, 'fnkeyhash'): d.fnkeyhash = dict.fnkeyhash - rgc.ll_arraycopy(dict.indexes, d.indexes, 0, 0, len(dict.indexes)) + ll_dict_copy_indexes(dict.indexes, d.indexes) + #rgc.ll_arraycopy(dict.indexes, d.indexes, 0, 0, len(dict.indexes)) rgc.ll_arraycopy(dict.entries, d.entries, 0, 0, len(dict.entries)) return d ll_copy.oopspec = 'dict.copy(dict)' From noreply at buildbot.pypy.org Mon Jan 7 02:23:03 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 7 Jan 2013 02:23:03 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: typo Message-ID: <20130107012303.10B0D1C0AD6@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59835:10958cb766a4 Date: 2013-01-07 03:22 +0200 http://bitbucket.org/pypy/pypy/changeset/10958cb766a4/ Log: typo diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py --- a/pypy/rpython/lltypesystem/rdict.py +++ b/pypy/rpython/lltypesystem/rdict.py @@ -155,7 +155,7 @@ DICTVALUE = self.value_repr.lowleveltype kwd = {} if self.custom_eq_hash: - kwd['custom_eq_hash'] = self.custom_eq_hash + kwd['get_custom_eq_hash'] = self.custom_eq_hash else: kwd['ll_hash_function'] = self.key_repr.get_ll_hash_function() kwd['ll_eq_function'] = self.key_repr.get_ll_eq_function() From noreply at buildbot.pypy.org Mon Jan 7 03:02:55 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 7 Jan 2013 03:02:55 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: work some more Message-ID: <20130107020255.0E4F21C0AD6@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59836:b1a3be05b6c4 Date: 2013-01-07 03:58 +0200 http://bitbucket.org/pypy/pypy/changeset/b1a3be05b6c4/ Log: work some more diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py --- a/pypy/rpython/lltypesystem/rdict.py +++ b/pypy/rpython/lltypesystem/rdict.py @@ -2,7 +2,7 @@ from pypy.objspace.flow.model import Constant from pypy.rpython.rdict import (AbstractDictRepr, AbstractDictIteratorRepr, rtype_newdict) -from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib import objectmodel, jit, rgc from pypy.rlib.debug import ll_assert @@ -31,6 +31,7 @@ # # struct dicttable { # int num_items; +# int size; # int resize_counter; # int *indexes; # note that this can be different int # Array *entries; @@ -78,15 +79,12 @@ DICTENTRY = lltype.Struct("dictentry", *entryfields) DICTENTRYARRAY = lltype.GcArray(DICTENTRY, adtmeths=entrymeths) - array_adtmeths = {'allocate': lltype.typeMethod(_ll_malloc_indexes), - 'getitem': ll_dict_index_getitem, - 'setitem': ll_dict_index_setitem} fields = [("num_items", lltype.Signed), + ("size", lltype.Signed), ("resize_counter", lltype.Signed), ("entries", lltype.Ptr(DICTENTRYARRAY)), - ("indexes", lltype.Ptr(lltype.GcArray(lltype.Signed, - adtmeths=array_adtmeths)))] + ("indexes", DICTINDEXOPAQUE)] if get_custom_eq_hash is not None: r_rdict_eqfn, r_rdict_hashfn = get_custom_eq_hash() fields.extend([ ("fnkeyeq", r_rdict_eqfn.lowleveltype), @@ -340,6 +338,40 @@ # be direct_call'ed from rtyped flow graphs, which means that they will # get flowed and annotated, mostly with SomePtr. + +# ------------------ indexes ---------------- + +DICTINDEXOPAQUE = llmemory.GCREF +DICTINDEX_SIGNED = lltype.Ptr(lltype.GcArray(lltype.Signed)) +DICTINDEX_INT = lltype.Ptr(lltype.GcArray(rffi.INT_real)) + +def _ll_malloc_indexes(n): + res = lltype.malloc(DICTINDEX_SIGNED.TO, n) + res = lltype.cast_opaque_ptr(DICTINDEXOPAQUE, res) + i = 0 + while i < n: + ll_index_setitem(n, res, i, FREE) + i += 1 + return res + +def ll_index_getitem(size, indexes, i): + return lltype.cast_opaque_ptr(DICTINDEX_SIGNED, indexes)[i] + +def ll_index_setitem(size, indexes, i, v): + lltype.cast_opaque_ptr(DICTINDEX_SIGNED, indexes)[i] = v + +def ll_dict_copy_indexes(size, from_indexes, to_indexes): + i = 0 + while i < size: + ll_index_setitem(size, to_indexes, i, + ll_index_getitem(size, from_indexes, i)) + i += 1 + +def ll_get_value(d, i): + return d.entries[ll_index_getitem(d.size, d.indexes, i)].value + +# ---------------- hashes ------------------ + def ll_hash_from_cache(entries, i): return entries[i].f_hash @@ -350,21 +382,6 @@ ENTRIES = lltype.typeOf(entries).TO return ENTRIES.fasthashfn(entries[i].key) -def ll_dict_index_getitem(indexes, i): - return indexes[i] - -def ll_dict_index_setitem(indexes, i, v): - indexes[i] = v - -def ll_dict_copy_indexes(from_indexes, to_indexes): - i = 0 - while i < len(from_indexes): - to_indexes.setitem(i, from_indexes.getitem(i)) - i += 1 - -def ll_get_value(d, i): - return d.entries[d.indexes.getitem(i)].value - def ll_keyhash_custom(d, key): DICT = lltype.typeOf(d).TO return objectmodel.hlinvoke(DICT.r_rdict_hashfn, d.fnkeyhash, key) @@ -403,7 +420,7 @@ valid = (i & HIGHEST_BIT) == 0 i = i & MASK ENTRY = lltype.typeOf(d.entries).TO.OF - index = d.indexes.getitem(i) + index = ll_index_getitem(d.size, d.indexes, i) if index == FREE: index = d.num_items entry = d.entries[index] @@ -419,12 +436,12 @@ ll_assert(rc > 0, "ll_dict_resize failed?") ll_assert(index < len(d.entries), "invalid insert") d.resize_counter = rc - d.indexes.setitem(i, index) + ll_index_setitem(d.size, d.indexes, i, index) entry.value = value elif index == DELETED: index = d.num_items entry = d.entries[index] - d.indexes.setitem(i, index) + ll_index_setitem(d.size, d.indexes, i, index) entry.value = value else: # override an existing or deleted entry @@ -444,7 +461,7 @@ @jit.look_inside_iff(lambda d, i: jit.isvirtual(d) and jit.isconstant(i)) def _ll_dict_del(d, i): - index = d.indexes.getitem(i) + index = ll_index_getitem(d.size, d.indexes, i) ENTRIES = lltype.typeOf(d.entries).TO ENTRY = ENTRIES.OF if index != d.num_items - 1: @@ -452,7 +469,7 @@ key = old_entry.key to_insert_i = ll_dict_lookup(d, key, d.keyhash(key)) ll_assert(not to_insert_i & HIGHEST_BIT, "invalid entry") - d.indexes.setitem(to_insert_i, index) + ll_index_setitem(d.size, d.indexes, to_insert_i, index) # copy the value new_entry = d.entries[index] new_entry.key = key @@ -460,7 +477,7 @@ if hasattr(ENTRY, 'f_hash'): new_entry.f_hash = old_entry.f_hash # clear the key and the value if they are GC pointers - d.indexes.setitem(i, DELETED) + ll_index_setitem(d.size, d.indexes, i, DELETED) d.num_items -= 1 entry = d.entries[d.num_items] if ENTRIES.clear_key: @@ -484,7 +501,7 @@ def ll_dict_resize(d): old_entries = d.entries old_indexes = d.indexes - old_size = len(old_indexes) + old_size = d.size # make a 'new_size' estimate and shrink it if there are many # deleted entry markers. See CPython for why it is a good idea to # quadruple the dictionary size as long as it's not too big. @@ -496,15 +513,16 @@ new_size *= 2 # new_item_size = new_size // 3 * 2 + 1 - d.indexes = lltype.typeOf(d).TO.indexes.TO.allocate(new_size) + d.indexes = _ll_malloc_indexes(new_size) d.resize_counter = new_item_size - d.num_items + i = 0 indexes = d.indexes while i < old_size: - index = old_indexes.getitem(i) + index = ll_index_getitem(old_size, old_indexes, i) if old_entries.valid(index): pos = ll_dict_lookup_clean(d, old_entries.hash(index)) - indexes.setitem(pos, index) + ll_index_setitem(new_size, indexes, pos, index) i += 1 if len(old_entries) != new_item_size: d.entries = lltype.typeOf(old_entries).TO.allocate(new_item_size) @@ -513,6 +531,7 @@ else: # we just removed deleted items, but we didn't do anything else special d.entries = old_entries + d.size = new_size ll_dict_resize.oopspec = 'dict.resize(d)' # ------- a port of CPython's dictobject.c's lookdict implementation ------- @@ -532,12 +551,12 @@ def ll_dict_lookup(d, key, hash): entries = d.entries indexes = d.indexes - mask = len(indexes) - 1 + mask = d.size - 1 i = hash & mask # do the first try before any looping ENTRIES = lltype.typeOf(entries).TO direct_compare = not hasattr(ENTRIES, 'no_direct_compare') - index = indexes.getitem(i) + index = ll_index_getitem(d.size, indexes, i) if entries.valid(index): checkingkey = entries[index].key if direct_compare and checkingkey == key: @@ -549,7 +568,7 @@ #llop.debug_print(lltype.Void, "comparing keys", ll_debugrepr(checkingkey), ll_debugrepr(key), found) if d.paranoia: if (entries != d.entries or - not entries.valid(indexes.getitem(i)) + not entries.valid(ll_index_getitem(d.size, indexes, i)) or entries[index].key != checkingkey): # the compare did major nasty stuff to the dict: start over return ll_dict_lookup(d, key, hash) @@ -569,7 +588,7 @@ i = r_uint(i) i = (i << 2) + i + perturb + 1 i = intmask(i) & mask - index = indexes.getitem(i) + index = ll_index_getitem(d.size, indexes, i) # keep 'i' as a signed number here, to consistently pass signed # arguments to the small helper methods. if index == FREE: @@ -586,7 +605,7 @@ found = d.keyeq(checkingkey, key) if d.paranoia: if (entries != d.entries or - not entries.valid(indexes.getitem(i)) or + not entries.valid(ll_index_getitem(d.size, indexes, i)) or entries[index].key != checkingkey): # the compare did major nasty stuff to the dict: # start over @@ -602,10 +621,10 @@ # key is new, and the dictionary doesn't contain deleted entries. # It only finds the next free slot for the given hash. indexes = d.indexes - mask = len(indexes) - 1 + mask = d.size - 1 i = hash & mask perturb = r_uint(hash) - while indexes.getitem(i) != FREE: + while ll_index_getitem(d.size, indexes, i) != FREE: i = r_uint(i) i = (i << 2) + i + perturb + 1 i = intmask(i) & mask @@ -619,15 +638,11 @@ DICT_INITSIZE = 8 DICT_ITEMS_INITSIZE = 5 - at jit.unroll_safe # we always unroll the small allocation def ll_newdict(DICT): d = DICT.allocate() - # XXX don't use _ll_items_allocate because of jit.unroll_safe, - # should be *really* jit_unroll_iff - d.indexes = lltype.malloc(DICT.indexes.TO, DICT_INITSIZE) + d.indexes = _ll_malloc_indexes(DICT_INITSIZE) d.num_items = 0 - for i in range(DICT_INITSIZE): - d.indexes.setitem(i, FREE) + d.size = DICT_INITSIZE d.entries = DICT.entries.TO.allocate(DICT_ITEMS_INITSIZE) d.resize_counter = DICT_ITEMS_INITSIZE return d @@ -640,7 +655,8 @@ items_size = n // 3 * 2 + 1 d = DICT.allocate() d.entries = DICT.entries.TO.allocate(items_size) - d.indexes = DICT.indexes.TO.allocate(n) + d.indexes = _ll_malloc_indexes(n) + d.size = n d.num_items = 0 d.resize_counter = items_size return d @@ -652,13 +668,6 @@ return lltype.malloc(DICT) def _ll_malloc_entries(ENTRIES, n): return lltype.malloc(ENTRIES, n, zero=True) -def _ll_malloc_indexes(ITEMS, n): - res = lltype.malloc(ITEMS, n) - i = 0 - while i < n: - res[i] = FREE - i += 1 - return res def rtype_r_dict(hop): r_dict = hop.r_result @@ -684,6 +693,7 @@ def get_ll_dictiter(DICT): return lltype.Ptr(lltype.GcStruct('dictiter', ('dict', DICT), + ('size', lltype.Signed), ('index', lltype.Signed))) class DictIteratorRepr(AbstractDictIteratorRepr): @@ -700,6 +710,7 @@ iter = lltype.malloc(ITERPTR.TO) iter.dict = d iter.index = 0 + iter.size = d.size return iter def _make_ll_dictnext(kind): @@ -711,6 +722,8 @@ def ll_dictnext(RETURNTYPE, iter): # note that RETURNTYPE is None for keys and values dict = iter.dict + if dict.size != iter.size: + raise RuntimeError if not dict or iter.index >= dict.num_items: # clear the reference to the dict and prevent restarts iter.dict = lltype.nullptr(lltype.typeOf(iter).TO.dict.TO) @@ -758,24 +771,25 @@ dictsize = len(dict.entries) d = DICT.allocate() d.entries = lltype.malloc(DICT.entries.TO, dictsize) - d.indexes = DICT.indexes.TO.allocate(len(dict.indexes)) + d.indexes = _ll_malloc_indexes(dict.size) d.num_items = dict.num_items + d.size = dict.size d.resize_counter = dict.resize_counter if hasattr(DICT, 'fnkeyeq'): d.fnkeyeq = dict.fnkeyeq if hasattr(DICT, 'fnkeyhash'): d.fnkeyhash = dict.fnkeyhash - ll_dict_copy_indexes(dict.indexes, d.indexes) + ll_dict_copy_indexes(dict.size, dict.indexes, d.indexes) #rgc.ll_arraycopy(dict.indexes, d.indexes, 0, 0, len(dict.indexes)) rgc.ll_arraycopy(dict.entries, d.entries, 0, 0, len(dict.entries)) return d ll_copy.oopspec = 'dict.copy(dict)' def ll_clear(d): - if (len(d.indexes) == DICT_INITSIZE and + if (d.size == DICT_INITSIZE and d.resize_counter == DICT_ITEMS_INITSIZE): return old_entries = d.entries d.entries = lltype.typeOf(old_entries).TO.allocate(DICT_ITEMS_INITSIZE) - d.indexes = lltype.typeOf(d).TO.indexes.TO.allocate(DICT_INITSIZE) + d.indexes = _ll_malloc_indexes(DICT_INITSIZE) d.num_items = 0 d.resize_counter = DICT_ITEMS_INITSIZE ll_clear.oopspec = 'dict.clear(d)' diff --git a/pypy/rpython/test/test_rdict.py b/pypy/rpython/test/test_rdict.py --- a/pypy/rpython/test/test_rdict.py +++ b/pypy/rpython/test/test_rdict.py @@ -24,6 +24,12 @@ yield x +def count_items(ll_d, TP): + return len([i for i in + [rdict.ll_index_getitem(ll_d.size, ll_d.indexes, i) + for i in range(ll_d.size)] if i == TP]) + + class TestRDictDirect(object): def _get_str_dict(self): # STR -> lltype.Signed @@ -37,8 +43,7 @@ DICT = self._get_str_dict() ll_d = rdict.ll_newdict(DICT) rdict.ll_dict_setitem(ll_d, llstr("abc"), 13) - assert (len([i for i in ll_d.indexes if i == rdict.FREE]) == - rdict.DICT_INITSIZE - 1) + assert count_items(ll_d, rdict.FREE) == rdict.DICT_INITSIZE - 1 assert rdict.ll_dict_getitem(ll_d, llstr("abc")) == 13 def test_dict_del_lastitem(self): @@ -48,9 +53,8 @@ rdict.ll_dict_setitem(ll_d, llstr("abc"), 13) py.test.raises(KeyError, rdict.ll_dict_delitem, ll_d, llstr("def")) rdict.ll_dict_delitem(ll_d, llstr("abc")) - assert (len([i for i in ll_d.indexes if i == rdict.FREE]) == - rdict.DICT_INITSIZE - 1) - assert (len([i for i in ll_d.indexes if i == rdict.DELETED]) == 1) + assert count_items(ll_d, rdict.FREE) == rdict.DICT_INITSIZE - 1 + assert count_items(ll_d, rdict.DELETED) == 1 py.test.raises(KeyError, rdict.ll_dict_getitem, ll_d, llstr("abc")) def test_dict_del_not_lastitem(self): @@ -59,9 +63,8 @@ rdict.ll_dict_setitem(ll_d, llstr("abc"), 13) rdict.ll_dict_setitem(ll_d, llstr("def"), 15) rdict.ll_dict_delitem(ll_d, llstr("abc")) - assert (len([i for i in ll_d.indexes if i == rdict.FREE]) == - rdict.DICT_INITSIZE - 2) - assert (len([i for i in ll_d.indexes if i == rdict.DELETED]) == 1) + assert count_items(ll_d, rdict.FREE) == rdict.DICT_INITSIZE - 2 + assert count_items(ll_d, rdict.DELETED) == 1 def test_dict_resize(self): DICT = self._get_str_dict() @@ -70,10 +73,10 @@ rdict.ll_dict_setitem(ll_d, llstr("b"), 2) rdict.ll_dict_setitem(ll_d, llstr("c"), 3) rdict.ll_dict_setitem(ll_d, llstr("d"), 4) - assert len(ll_d.indexes) == 8 + assert ll_d.size == 8 rdict.ll_dict_setitem(ll_d, llstr("e"), 5) rdict.ll_dict_setitem(ll_d, llstr("f"), 6) - assert len(ll_d.indexes) == 32 + assert ll_d.size == 32 for item in ['a', 'b', 'c', 'd', 'e', 'f']: assert rdict.ll_dict_getitem(ll_d, llstr(item)) == ord(item) - ord('a') + 1 @@ -115,8 +118,10 @@ for num in numbers: rdict.ll_dict_setitem(ll_d, num, 1) rdict.ll_dict_delitem(ll_d, num) - for k in ll_d.indexes: - assert k < 0 + tot = (count_items(ll_d, rdict.FREE) + + count_items(ll_d, rdict.DELETED)) + assert tot == ll_d.size + class BaseTestRdict(BaseRtypingTest): @@ -829,7 +834,11 @@ return d res = self.interpret(func2, [ord(x), ord(y)]) - assert len([i for i in res.indexes if i >= 0]) == 2 + c = 0 + for i in range(res.size): + if rdict.ll_index_getitem(res.size, res.indexes, i) >= 0: + c += 1 + assert c == 2 def func3(c0, c1, c2, c3, c4, c5, c6, c7): d = {} @@ -847,11 +856,7 @@ py.test.skip("make dict tests more indepdent from initsize") res = self.interpret(func3, [ord(char_by_hash[i][0]) for i in range(rdict.DICT_INITSIZE)]) - count_frees = 0 - for i in res.indexes: - if i == -1: - count_frees += 1 - assert count_frees >= 3 + assert count_items(res, rdict.FREE) >= 3 def test_dict_resize(self): # XXX we no longer automatically resize on 'del'. We need to @@ -870,11 +875,12 @@ del d[chr(ord('A') - i)] return d res = self.interpret(func, [0]) - assert len(res.indexes) > rdict.DICT_INITSIZE + assert res.size > rdict.DICT_INITSIZE res = self.interpret(func, [1]) - assert len(res.indexes) == rdict.DICT_INITSIZE + assert res.size == rdict.DICT_INITSIZE def test_dict_valid_resize(self): + py.test.skip("no longer valid, we're a bit too paranoid now") # see if we find our keys after resize def func(): d = {} From noreply at buildbot.pypy.org Mon Jan 7 03:02:56 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 7 Jan 2013 03:02:56 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: use different sizes of ints Message-ID: <20130107020256.70B881C0AD6@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59837:a02e95d6b5d1 Date: 2013-01-07 04:02 +0200 http://bitbucket.org/pypy/pypy/changeset/a02e95d6b5d1/ Log: use different sizes of ints diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py --- a/pypy/rpython/lltypesystem/rdict.py +++ b/pypy/rpython/lltypesystem/rdict.py @@ -345,9 +345,15 @@ DICTINDEX_SIGNED = lltype.Ptr(lltype.GcArray(lltype.Signed)) DICTINDEX_INT = lltype.Ptr(lltype.GcArray(rffi.INT_real)) +MAX_INT = 2 ** 31 - 1 + def _ll_malloc_indexes(n): - res = lltype.malloc(DICTINDEX_SIGNED.TO, n) - res = lltype.cast_opaque_ptr(DICTINDEXOPAQUE, res) + if n < MAX_INT: + res = lltype.malloc(DICTINDEX_INT.TO, n) + res = lltype.cast_opaque_ptr(DICTINDEXOPAQUE, res) + else: + res = lltype.malloc(DICTINDEX_SIGNED.TO, n) + res = lltype.cast_opaque_ptr(DICTINDEXOPAQUE, res) i = 0 while i < n: ll_index_setitem(n, res, i, FREE) @@ -355,9 +361,16 @@ return res def ll_index_getitem(size, indexes, i): + if size < MAX_INT: + res = lltype.cast_opaque_ptr(DICTINDEX_INT, indexes)[i] + return rffi.cast(lltype.Signed, res) return lltype.cast_opaque_ptr(DICTINDEX_SIGNED, indexes)[i] def ll_index_setitem(size, indexes, i, v): + if size < MAX_INT: + arg = rffi.cast(rffi.INT_real, v) + lltype.cast_opaque_ptr(DICTINDEX_INT, indexes)[i] = arg + return lltype.cast_opaque_ptr(DICTINDEX_SIGNED, indexes)[i] = v def ll_dict_copy_indexes(size, from_indexes, to_indexes): From noreply at buildbot.pypy.org Mon Jan 7 15:42:48 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 7 Jan 2013 15:42:48 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: introduce size Message-ID: <20130107144248.E43261C1241@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59838:198841ce3268 Date: 2013-01-07 16:13 +0200 http://bitbucket.org/pypy/pypy/changeset/198841ce3268/ Log: introduce size diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py --- a/pypy/rpython/lltypesystem/rdict.py +++ b/pypy/rpython/lltypesystem/rdict.py @@ -31,6 +31,7 @@ # # struct dicttable { # int num_items; +# int size; # int resize_counter; # int *indexes; # note that this can be different int # Array *entries; @@ -84,6 +85,7 @@ fields = [("num_items", lltype.Signed), ("resize_counter", lltype.Signed), + ("size", lltype.Signed), ("entries", lltype.Ptr(DICTENTRYARRAY)), ("indexes", lltype.Ptr(lltype.GcArray(lltype.Signed, adtmeths=array_adtmeths)))] @@ -356,9 +358,9 @@ def ll_dict_index_setitem(indexes, i, v): indexes[i] = v -def ll_dict_copy_indexes(from_indexes, to_indexes): +def ll_dict_copy_indexes(size, from_indexes, to_indexes): i = 0 - while i < len(from_indexes): + while i < size: to_indexes.setitem(i, from_indexes.getitem(i)) i += 1 @@ -484,7 +486,7 @@ def ll_dict_resize(d): old_entries = d.entries old_indexes = d.indexes - old_size = len(old_indexes) + old_size = d.size # make a 'new_size' estimate and shrink it if there are many # deleted entry markers. See CPython for why it is a good idea to # quadruple the dictionary size as long as it's not too big. @@ -497,6 +499,7 @@ # new_item_size = new_size // 3 * 2 + 1 d.indexes = lltype.typeOf(d).TO.indexes.TO.allocate(new_size) + d.size = new_size d.resize_counter = new_item_size - d.num_items i = 0 indexes = d.indexes @@ -532,7 +535,7 @@ def ll_dict_lookup(d, key, hash): entries = d.entries indexes = d.indexes - mask = len(indexes) - 1 + mask = d.size - 1 i = hash & mask # do the first try before any looping ENTRIES = lltype.typeOf(entries).TO @@ -602,7 +605,7 @@ # key is new, and the dictionary doesn't contain deleted entries. # It only finds the next free slot for the given hash. indexes = d.indexes - mask = len(indexes) - 1 + mask = d.size - 1 i = hash & mask perturb = r_uint(hash) while indexes.getitem(i) != FREE: @@ -625,6 +628,7 @@ # XXX don't use _ll_items_allocate because of jit.unroll_safe, # should be *really* jit_unroll_iff d.indexes = lltype.malloc(DICT.indexes.TO, DICT_INITSIZE) + d.size = DICT_INITSIZE d.num_items = 0 for i in range(DICT_INITSIZE): d.indexes.setitem(i, FREE) @@ -641,6 +645,7 @@ d = DICT.allocate() d.entries = DICT.entries.TO.allocate(items_size) d.indexes = DICT.indexes.TO.allocate(n) + d.size = n d.num_items = 0 d.resize_counter = items_size return d @@ -758,24 +763,26 @@ dictsize = len(dict.entries) d = DICT.allocate() d.entries = lltype.malloc(DICT.entries.TO, dictsize) - d.indexes = DICT.indexes.TO.allocate(len(dict.indexes)) + d.indexes = DICT.indexes.TO.allocate(dict.size) + d.size = dict.size d.num_items = dict.num_items d.resize_counter = dict.resize_counter if hasattr(DICT, 'fnkeyeq'): d.fnkeyeq = dict.fnkeyeq if hasattr(DICT, 'fnkeyhash'): d.fnkeyhash = dict.fnkeyhash - ll_dict_copy_indexes(dict.indexes, d.indexes) + ll_dict_copy_indexes(d.size, dict.indexes, d.indexes) #rgc.ll_arraycopy(dict.indexes, d.indexes, 0, 0, len(dict.indexes)) rgc.ll_arraycopy(dict.entries, d.entries, 0, 0, len(dict.entries)) return d ll_copy.oopspec = 'dict.copy(dict)' def ll_clear(d): - if (len(d.indexes) == DICT_INITSIZE and + if (d.size == DICT_INITSIZE and d.resize_counter == DICT_ITEMS_INITSIZE): return old_entries = d.entries d.entries = lltype.typeOf(old_entries).TO.allocate(DICT_ITEMS_INITSIZE) d.indexes = lltype.typeOf(d).TO.indexes.TO.allocate(DICT_INITSIZE) + d.size = DICT_INITSIZE d.num_items = 0 d.resize_counter = DICT_ITEMS_INITSIZE ll_clear.oopspec = 'dict.clear(d)' From noreply at buildbot.pypy.org Mon Jan 7 15:42:50 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 7 Jan 2013 15:42:50 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: close this head Message-ID: <20130107144250.2EDA41C1241@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59839:f1dfb9775129 Date: 2013-01-07 16:13 +0200 http://bitbucket.org/pypy/pypy/changeset/f1dfb9775129/ Log: close this head From noreply at buildbot.pypy.org Mon Jan 7 15:42:51 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 7 Jan 2013 15:42:51 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: change the API (again) Message-ID: <20130107144251.48DD51C1241@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59840:8ed7cb8c71e4 Date: 2013-01-07 16:20 +0200 http://bitbucket.org/pypy/pypy/changeset/8ed7cb8c71e4/ Log: change the API (again) diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py --- a/pypy/rpython/lltypesystem/rdict.py +++ b/pypy/rpython/lltypesystem/rdict.py @@ -79,16 +79,12 @@ DICTENTRY = lltype.Struct("dictentry", *entryfields) DICTENTRYARRAY = lltype.GcArray(DICTENTRY, adtmeths=entrymeths) - array_adtmeths = {'allocate': lltype.typeMethod(_ll_malloc_indexes), - 'getitem': ll_dict_index_getitem, - 'setitem': ll_dict_index_setitem} fields = [("num_items", lltype.Signed), ("resize_counter", lltype.Signed), ("size", lltype.Signed), ("entries", lltype.Ptr(DICTENTRYARRAY)), - ("indexes", lltype.Ptr(lltype.GcArray(lltype.Signed, - adtmeths=array_adtmeths)))] + ("indexes", DICTINDEXARRAY)] if get_custom_eq_hash is not None: r_rdict_eqfn, r_rdict_hashfn = get_custom_eq_hash() fields.extend([ ("fnkeyeq", r_rdict_eqfn.lowleveltype), @@ -342,6 +338,32 @@ # be direct_call'ed from rtyped flow graphs, which means that they will # get flowed and annotated, mostly with SomePtr. +# --------------------- indexes ------------------ + +DICTINDEXARRAY = lltype.Ptr(lltype.GcArray(lltype.Signed)) + +def _ll_malloc_indexes(n): + res = lltype.malloc(DICTINDEXARRAY.TO, n) + i = 0 + while i < n: + res[i] = FREE + i += 1 + return res + +def ll_index_getitem(size, indexes, i): + return indexes[i] + +def ll_index_setitem(size, indexes, i, v): + indexes[i] = v + +def ll_dict_copy_indexes(size, from_indexes, to_indexes): + i = 0 + while i < size: + ll_index_setitem(size, i, ll_index_getitem(size, from_indexes, i)) + i += 1 + +# ---------------------- hashes ------------------- + def ll_hash_from_cache(entries, i): return entries[i].f_hash @@ -352,20 +374,8 @@ ENTRIES = lltype.typeOf(entries).TO return ENTRIES.fasthashfn(entries[i].key) -def ll_dict_index_getitem(indexes, i): - return indexes[i] - -def ll_dict_index_setitem(indexes, i, v): - indexes[i] = v - -def ll_dict_copy_indexes(size, from_indexes, to_indexes): - i = 0 - while i < size: - to_indexes.setitem(i, from_indexes.getitem(i)) - i += 1 - def ll_get_value(d, i): - return d.entries[d.indexes.getitem(i)].value + return d.entries[ll_index_getitem(d.size, d.indexes, i)].value def ll_keyhash_custom(d, key): DICT = lltype.typeOf(d).TO @@ -405,7 +415,7 @@ valid = (i & HIGHEST_BIT) == 0 i = i & MASK ENTRY = lltype.typeOf(d.entries).TO.OF - index = d.indexes.getitem(i) + index = ll_index_getitem(d.size, d.indexes, i) if index == FREE: index = d.num_items entry = d.entries[index] @@ -421,12 +431,12 @@ ll_assert(rc > 0, "ll_dict_resize failed?") ll_assert(index < len(d.entries), "invalid insert") d.resize_counter = rc - d.indexes.setitem(i, index) + ll_index_setitem(d.size, d.indexes, i, index) entry.value = value elif index == DELETED: index = d.num_items entry = d.entries[index] - d.indexes.setitem(i, index) + ll_index_setitem(d.size, d.indexes, i, index) entry.value = value else: # override an existing or deleted entry @@ -446,7 +456,7 @@ @jit.look_inside_iff(lambda d, i: jit.isvirtual(d) and jit.isconstant(i)) def _ll_dict_del(d, i): - index = d.indexes.getitem(i) + index = ll_index_getitem(d.size, d.indexes, i) ENTRIES = lltype.typeOf(d.entries).TO ENTRY = ENTRIES.OF if index != d.num_items - 1: @@ -454,7 +464,7 @@ key = old_entry.key to_insert_i = ll_dict_lookup(d, key, d.keyhash(key)) ll_assert(not to_insert_i & HIGHEST_BIT, "invalid entry") - d.indexes.setitem(to_insert_i, index) + ll_index_setitem(d.size, d.indexes, to_insert_i, index) # copy the value new_entry = d.entries[index] new_entry.key = key @@ -462,7 +472,7 @@ if hasattr(ENTRY, 'f_hash'): new_entry.f_hash = old_entry.f_hash # clear the key and the value if they are GC pointers - d.indexes.setitem(i, DELETED) + ll_index_setitem(d.size, d.indexes, i, DELETED) d.num_items -= 1 entry = d.entries[d.num_items] if ENTRIES.clear_key: @@ -498,16 +508,16 @@ new_size *= 2 # new_item_size = new_size // 3 * 2 + 1 - d.indexes = lltype.typeOf(d).TO.indexes.TO.allocate(new_size) + d.indexes = _ll_malloc_indexes(new_size) d.size = new_size d.resize_counter = new_item_size - d.num_items i = 0 indexes = d.indexes while i < old_size: - index = old_indexes.getitem(i) + index = ll_index_getitem(old_size, old_indexes, i) if old_entries.valid(index): pos = ll_dict_lookup_clean(d, old_entries.hash(index)) - indexes.setitem(pos, index) + ll_index_setitem(d.size, indexes, pos, index) i += 1 if len(old_entries) != new_item_size: d.entries = lltype.typeOf(old_entries).TO.allocate(new_item_size) @@ -540,7 +550,7 @@ # do the first try before any looping ENTRIES = lltype.typeOf(entries).TO direct_compare = not hasattr(ENTRIES, 'no_direct_compare') - index = indexes.getitem(i) + index = ll_index_getitem(d.size, indexes, i) if entries.valid(index): checkingkey = entries[index].key if direct_compare and checkingkey == key: @@ -552,7 +562,7 @@ #llop.debug_print(lltype.Void, "comparing keys", ll_debugrepr(checkingkey), ll_debugrepr(key), found) if d.paranoia: if (entries != d.entries or - not entries.valid(indexes.getitem(i)) + not entries.valid(ll_index_getitem(d.size, indexes, i)) or entries[index].key != checkingkey): # the compare did major nasty stuff to the dict: start over return ll_dict_lookup(d, key, hash) @@ -572,7 +582,7 @@ i = r_uint(i) i = (i << 2) + i + perturb + 1 i = intmask(i) & mask - index = indexes.getitem(i) + index = ll_index_getitem(d.size, indexes, i) # keep 'i' as a signed number here, to consistently pass signed # arguments to the small helper methods. if index == FREE: @@ -589,7 +599,7 @@ found = d.keyeq(checkingkey, key) if d.paranoia: if (entries != d.entries or - not entries.valid(indexes.getitem(i)) or + not entries.valid(ll_index_getitem(d.size, indexes, i)) or entries[index].key != checkingkey): # the compare did major nasty stuff to the dict: # start over @@ -608,7 +618,7 @@ mask = d.size - 1 i = hash & mask perturb = r_uint(hash) - while indexes.getitem(i) != FREE: + while ll_index_getitem(d.size, indexes, i) != FREE: i = r_uint(i) i = (i << 2) + i + perturb + 1 i = intmask(i) & mask @@ -631,7 +641,7 @@ d.size = DICT_INITSIZE d.num_items = 0 for i in range(DICT_INITSIZE): - d.indexes.setitem(i, FREE) + ll_index_setitem(d.size, d.indexes, i, FREE) d.entries = DICT.entries.TO.allocate(DICT_ITEMS_INITSIZE) d.resize_counter = DICT_ITEMS_INITSIZE return d @@ -644,7 +654,7 @@ items_size = n // 3 * 2 + 1 d = DICT.allocate() d.entries = DICT.entries.TO.allocate(items_size) - d.indexes = DICT.indexes.TO.allocate(n) + d.indexes = _ll_malloc_indexes(n) d.size = n d.num_items = 0 d.resize_counter = items_size @@ -657,13 +667,6 @@ return lltype.malloc(DICT) def _ll_malloc_entries(ENTRIES, n): return lltype.malloc(ENTRIES, n, zero=True) -def _ll_malloc_indexes(ITEMS, n): - res = lltype.malloc(ITEMS, n) - i = 0 - while i < n: - res[i] = FREE - i += 1 - return res def rtype_r_dict(hop): r_dict = hop.r_result @@ -763,7 +766,7 @@ dictsize = len(dict.entries) d = DICT.allocate() d.entries = lltype.malloc(DICT.entries.TO, dictsize) - d.indexes = DICT.indexes.TO.allocate(dict.size) + d.indexes = _ll_malloc_indexes(dict.size) d.size = dict.size d.num_items = dict.num_items d.resize_counter = dict.resize_counter @@ -781,7 +784,7 @@ return old_entries = d.entries d.entries = lltype.typeOf(old_entries).TO.allocate(DICT_ITEMS_INITSIZE) - d.indexes = lltype.typeOf(d).TO.indexes.TO.allocate(DICT_INITSIZE) + d.indexes = _ll_malloc_indexes(DICT_INITSIZE) d.size = DICT_INITSIZE d.num_items = 0 d.resize_counter = DICT_ITEMS_INITSIZE From noreply at buildbot.pypy.org Mon Jan 7 15:42:52 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 7 Jan 2013 15:42:52 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: bring this branch to the state where it passes tests Message-ID: <20130107144252.73FD71C1241@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59841:09ee3d8253c3 Date: 2013-01-07 16:34 +0200 http://bitbucket.org/pypy/pypy/changeset/09ee3d8253c3/ Log: bring this branch to the state where it passes tests diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py --- a/pypy/rpython/lltypesystem/rdict.py +++ b/pypy/rpython/lltypesystem/rdict.py @@ -2,7 +2,7 @@ from pypy.objspace.flow.model import Constant from pypy.rpython.rdict import (AbstractDictRepr, AbstractDictIteratorRepr, rtype_newdict) -from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib import objectmodel, jit, rgc from pypy.rlib.debug import ll_assert @@ -84,7 +84,7 @@ ("resize_counter", lltype.Signed), ("size", lltype.Signed), ("entries", lltype.Ptr(DICTENTRYARRAY)), - ("indexes", DICTINDEXARRAY)] + ("indexes", llmemory.GCREF)] if get_custom_eq_hash is not None: r_rdict_eqfn, r_rdict_hashfn = get_custom_eq_hash() fields.extend([ ("fnkeyeq", r_rdict_eqfn.lowleveltype), @@ -340,26 +340,31 @@ # --------------------- indexes ------------------ -DICTINDEXARRAY = lltype.Ptr(lltype.GcArray(lltype.Signed)) +DICTINDEX_SIGNED = lltype.Ptr(lltype.GcArray(lltype.Signed)) +DICTINDEX_INT = lltype.Ptr(lltype.GcArray(rffi.INT_real)) +MAX_INT = 2 ** 31 - 1 def _ll_malloc_indexes(n): - res = lltype.malloc(DICTINDEXARRAY.TO, n) + # XXXX 64 bit only + res = lltype.cast_opaque_ptr(llmemory.GCREF, + lltype.malloc(DICTINDEX_SIGNED.TO, n)) i = 0 while i < n: - res[i] = FREE + ll_index_setitem(n, res, i, FREE) i += 1 return res def ll_index_getitem(size, indexes, i): - return indexes[i] + return lltype.cast_opaque_ptr(DICTINDEX_SIGNED, indexes)[i] def ll_index_setitem(size, indexes, i, v): - indexes[i] = v + lltype.cast_opaque_ptr(DICTINDEX_SIGNED, indexes)[i] = v def ll_dict_copy_indexes(size, from_indexes, to_indexes): i = 0 while i < size: - ll_index_setitem(size, i, ll_index_getitem(size, from_indexes, i)) + ll_index_setitem(size, to_indexes, i, + ll_index_getitem(size, from_indexes, i)) i += 1 # ---------------------- hashes ------------------- @@ -632,16 +637,11 @@ DICT_INITSIZE = 8 DICT_ITEMS_INITSIZE = 5 - at jit.unroll_safe # we always unroll the small allocation def ll_newdict(DICT): d = DICT.allocate() - # XXX don't use _ll_items_allocate because of jit.unroll_safe, - # should be *really* jit_unroll_iff - d.indexes = lltype.malloc(DICT.indexes.TO, DICT_INITSIZE) + d.indexes = _ll_malloc_indexes(DICT_INITSIZE) d.size = DICT_INITSIZE d.num_items = 0 - for i in range(DICT_INITSIZE): - ll_index_setitem(d.size, d.indexes, i, FREE) d.entries = DICT.entries.TO.allocate(DICT_ITEMS_INITSIZE) d.resize_counter = DICT_ITEMS_INITSIZE return d diff --git a/pypy/rpython/test/test_rdict.py b/pypy/rpython/test/test_rdict.py --- a/pypy/rpython/test/test_rdict.py +++ b/pypy/rpython/test/test_rdict.py @@ -23,6 +23,16 @@ assert 0 <= x < 4 yield x +def foreach_index(ll_d): + for i in range(ll_d.size): + yield rdict.ll_index_getitem(ll_d.size, ll_d.indexes, i) + +def count_items(ll_d, ITEM): + c = 0 + for item in foreach_index(ll_d): + if item == ITEM: + c += 1 + return c class TestRDictDirect(object): def _get_str_dict(self): @@ -37,8 +47,7 @@ DICT = self._get_str_dict() ll_d = rdict.ll_newdict(DICT) rdict.ll_dict_setitem(ll_d, llstr("abc"), 13) - assert (len([i for i in ll_d.indexes if i == rdict.FREE]) == - rdict.DICT_INITSIZE - 1) + assert count_items(ll_d, rdict.FREE) == rdict.DICT_INITSIZE - 1 assert rdict.ll_dict_getitem(ll_d, llstr("abc")) == 13 def test_dict_del_lastitem(self): @@ -48,9 +57,8 @@ rdict.ll_dict_setitem(ll_d, llstr("abc"), 13) py.test.raises(KeyError, rdict.ll_dict_delitem, ll_d, llstr("def")) rdict.ll_dict_delitem(ll_d, llstr("abc")) - assert (len([i for i in ll_d.indexes if i == rdict.FREE]) == - rdict.DICT_INITSIZE - 1) - assert (len([i for i in ll_d.indexes if i == rdict.DELETED]) == 1) + assert count_items(ll_d, rdict.FREE) == rdict.DICT_INITSIZE - 1 + assert count_items(ll_d, rdict.DELETED) == 1 py.test.raises(KeyError, rdict.ll_dict_getitem, ll_d, llstr("abc")) def test_dict_del_not_lastitem(self): @@ -59,9 +67,8 @@ rdict.ll_dict_setitem(ll_d, llstr("abc"), 13) rdict.ll_dict_setitem(ll_d, llstr("def"), 15) rdict.ll_dict_delitem(ll_d, llstr("abc")) - assert (len([i for i in ll_d.indexes if i == rdict.FREE]) == - rdict.DICT_INITSIZE - 2) - assert (len([i for i in ll_d.indexes if i == rdict.DELETED]) == 1) + assert count_items(ll_d, rdict.FREE) == rdict.DICT_INITSIZE - 2 + assert count_items(ll_d, rdict.DELETED) == 1 def test_dict_resize(self): DICT = self._get_str_dict() @@ -70,10 +77,10 @@ rdict.ll_dict_setitem(ll_d, llstr("b"), 2) rdict.ll_dict_setitem(ll_d, llstr("c"), 3) rdict.ll_dict_setitem(ll_d, llstr("d"), 4) - assert len(ll_d.indexes) == 8 + assert ll_d.size == 8 rdict.ll_dict_setitem(ll_d, llstr("e"), 5) rdict.ll_dict_setitem(ll_d, llstr("f"), 6) - assert len(ll_d.indexes) == 32 + assert ll_d.size == 32 for item in ['a', 'b', 'c', 'd', 'e', 'f']: assert rdict.ll_dict_getitem(ll_d, llstr(item)) == ord(item) - ord('a') + 1 @@ -115,7 +122,7 @@ for num in numbers: rdict.ll_dict_setitem(ll_d, num, 1) rdict.ll_dict_delitem(ll_d, num) - for k in ll_d.indexes: + for k in foreach_index(ll_d): assert k < 0 class BaseTestRdict(BaseRtypingTest): @@ -829,7 +836,7 @@ return d res = self.interpret(func2, [ord(x), ord(y)]) - assert len([i for i in res.indexes if i >= 0]) == 2 + assert len([i for i in foreach_index(res) if i >= 0]) == 2 def func3(c0, c1, c2, c3, c4, c5, c6, c7): d = {} @@ -848,7 +855,7 @@ res = self.interpret(func3, [ord(char_by_hash[i][0]) for i in range(rdict.DICT_INITSIZE)]) count_frees = 0 - for i in res.indexes: + for i in foreach_index(res): if i == -1: count_frees += 1 assert count_frees >= 3 @@ -870,9 +877,9 @@ del d[chr(ord('A') - i)] return d res = self.interpret(func, [0]) - assert len(res.indexes) > rdict.DICT_INITSIZE + assert res.size > rdict.DICT_INITSIZE res = self.interpret(func, [1]) - assert len(res.indexes) == rdict.DICT_INITSIZE + assert res.size == rdict.DICT_INITSIZE def test_dict_valid_resize(self): # see if we find our keys after resize @@ -884,7 +891,6 @@ # delete again for i in range(10): del d[str(i)] - res = 0 # if it does not crash, we are fine. It crashes if you forget the hash field. self.interpret(func, []) From noreply at buildbot.pypy.org Mon Jan 7 15:42:53 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 7 Jan 2013 15:42:53 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: use INTs Message-ID: <20130107144253.989391C1241@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59842:292fea80a9e1 Date: 2013-01-07 16:38 +0200 http://bitbucket.org/pypy/pypy/changeset/292fea80a9e1/ Log: use INTs diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py --- a/pypy/rpython/lltypesystem/rdict.py +++ b/pypy/rpython/lltypesystem/rdict.py @@ -346,8 +346,12 @@ def _ll_malloc_indexes(n): # XXXX 64 bit only - res = lltype.cast_opaque_ptr(llmemory.GCREF, - lltype.malloc(DICTINDEX_SIGNED.TO, n)) + if n < MAX_INT: + res = lltype.cast_opaque_ptr(llmemory.GCREF, + lltype.malloc(DICTINDEX_INT.TO, n)) + else: + res = lltype.cast_opaque_ptr(llmemory.GCREF, + lltype.malloc(DICTINDEX_SIGNED.TO, n)) i = 0 while i < n: ll_index_setitem(n, res, i, FREE) @@ -355,10 +359,17 @@ return res def ll_index_getitem(size, indexes, i): + if size < MAX_INT: + return rffi.cast(lltype.Signed, + lltype.cast_opaque_ptr(DICTINDEX_INT, indexes)[i]) return lltype.cast_opaque_ptr(DICTINDEX_SIGNED, indexes)[i] def ll_index_setitem(size, indexes, i, v): - lltype.cast_opaque_ptr(DICTINDEX_SIGNED, indexes)[i] = v + if size < MAX_INT: + lltype.cast_opaque_ptr(DICTINDEX_INT, indexes)[i] = rffi.cast( + rffi.INT_real, v) + else: + lltype.cast_opaque_ptr(DICTINDEX_SIGNED, indexes)[i] = v def ll_dict_copy_indexes(size, from_indexes, to_indexes): i = 0 From noreply at buildbot.pypy.org Mon Jan 7 15:42:54 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 7 Jan 2013 15:42:54 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: dummy merge Message-ID: <20130107144254.ADC521C1241@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59843:07a7e681aee9 Date: 2013-01-07 16:42 +0200 http://bitbucket.org/pypy/pypy/changeset/07a7e681aee9/ Log: dummy merge From noreply at buildbot.pypy.org Mon Jan 7 16:25:41 2013 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 7 Jan 2013 16:25:41 +0100 (CET) Subject: [pypy-commit] cffi default: Issue #52: Add ffi.verify(..modulename='foo'..) with test and documentation. Message-ID: <20130107152541.1968E1C1246@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r1120:c2d0ff47f54d Date: 2013-01-07 16:25 +0100 http://bitbucket.org/cffi/cffi/changeset/c2d0ff47f54d/ Log: Issue #52: Add ffi.verify(..modulename='foo'..) with test and documentation. diff --git a/cffi/verifier.py b/cffi/verifier.py --- a/cffi/verifier.py +++ b/cffi/verifier.py @@ -5,27 +5,32 @@ class Verifier(object): - def __init__(self, ffi, preamble, tmpdir=None, ext_package=None, - tag='', force_generic_engine=False, **kwds): + def __init__(self, ffi, preamble, tmpdir=None, modulename=None, + ext_package=None, tag='', force_generic_engine=False, **kwds): self.ffi = ffi self.preamble = preamble - flattened_kwds = ffiplatform.flatten(kwds) + if not modulename: + flattened_kwds = ffiplatform.flatten(kwds) vengine_class = _locate_engine_class(ffi, force_generic_engine) self._vengine = vengine_class(self) self._vengine.patch_extension_kwds(kwds) self.kwds = kwds # - key = '\x00'.join([sys.version[:3], __version__, preamble, - flattened_kwds] + - ffi._cdefsources) - if sys.version_info >= (3,): - key = key.encode('utf-8') - k1 = hex(binascii.crc32(key[0::2]) & 0xffffffff) - k1 = k1.lstrip('0x').rstrip('L') - k2 = hex(binascii.crc32(key[1::2]) & 0xffffffff) - k2 = k2.lstrip('0').rstrip('L') - modulename = '_cffi_%s_%s%s%s' % (tag, self._vengine._class_key, - k1, k2) + if modulename: + if tag: + raise TypeError("can't specify both 'modulename' and 'tag'") + else: + key = '\x00'.join([sys.version[:3], __version__, preamble, + flattened_kwds] + + ffi._cdefsources) + if sys.version_info >= (3,): + key = key.encode('utf-8') + k1 = hex(binascii.crc32(key[0::2]) & 0xffffffff) + k1 = k1.lstrip('0x').rstrip('L') + k2 = hex(binascii.crc32(key[1::2]) & 0xffffffff) + k2 = k2.lstrip('0').rstrip('L') + modulename = '_cffi_%s_%s%s%s' % (tag, self._vengine._class_key, + k1, k2) suffix = _get_so_suffix() self.tmpdir = tmpdir or _caller_dir_pycache() self.sourcefilename = os.path.join(self.tmpdir, modulename + '.c') diff --git a/doc/source/index.rst b/doc/source/index.rst --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -453,7 +453,7 @@ The verification step --------------------- -``ffi.verify(source, tmpdir=.., ext_package=.., tag='', **kwargs)``: +``ffi.verify(source, tmpdir=.., ext_package=.., modulename=.., **kwargs)``: verifies that the current ffi signatures compile on this machine, and return a dynamic library object. The dynamic library can be used to call functions and access global @@ -598,6 +598,18 @@ middle of the extension module's name: ``_cffi__``. Useful to give a bit more context, e.g. when debugging. +.. _`warning about modulename`: + +.. versionadded:: 0.5 + The ``modulename`` argument can be used to force a specific module + name, overriding the name ``_cffi__``. Use with care, + e.g. if you are passing variable information to ``verify()`` but + still want the module name to be always the same (e.g. absolute + paths to local files). In this case, no hash is computed and if + the module name already exists it will be reused without further + check. Be sure to have other means of clearing the ``tmpdir`` + whenever you change your sources. + Working with pointers, structures and arrays -------------------------------------------- @@ -1339,15 +1351,17 @@ can be instantiated directly. It is normally instantiated for you by ``ffi.verify()``, and the instance is attached as ``ffi.verifier``. -- ``Verifier(ffi, preamble, tmpdir=.., ext_package='', tag='', **kwds)``: +- ``Verifier(ffi, preamble, tmpdir=.., ext_package='', modulename=None, + tag='', **kwds)``: instantiate the class with an FFI object and a preamble, which is C text that will be pasted into the generated C source. The value of ``tmpdir`` defaults to the directory ``directory_of_the_caller/__pycache__``. The value of ``ext_package`` is used when looking up an already-compiled, already- installed version of the extension module. The module name is - ``_cffi__``. - The keyword arguments are passed directly + ``_cffi__``, unless overridden with ``modulename`` + (see the `warning about modulename`_ above). + The other keyword arguments are passed directly to `distutils when building the Extension object.`__ .. __: http://docs.python.org/distutils/setupscript.html#describing-extension-module diff --git a/testing/test_zdistutils.py b/testing/test_zdistutils.py --- a/testing/test_zdistutils.py +++ b/testing/test_zdistutils.py @@ -1,7 +1,7 @@ import sys, os, imp, math, shutil import py from cffi import FFI, FFIError -from cffi.verifier import Verifier, _locate_engine_class +from cffi.verifier import Verifier, _locate_engine_class, _get_so_suffix from cffi.ffiplatform import maybe_relative_path from testing.udir import udir @@ -241,6 +241,20 @@ assert lib.test1tag(143) == 101.0 assert '_cffi_xxtest_tagxx_' in ffi.verifier.modulefilename + def test_modulename(self): + ffi = FFI() + ffi.cdef("/* %s test_modulename */ double test1foo(double x);" % self) + csrc = "double test1foo(double x) { return x - 63.0; }" + modname = 'xxtest_modulenamexx%d' % (self.generic,) + lib = ffi.verify(csrc, force_generic_engine=self.generic, + modulename=modname) + assert lib.test1foo(143) == 80.0 + suffix = _get_so_suffix() + fn1 = os.path.join(ffi.verifier.tmpdir, modname + '.c') + fn2 = os.path.join(ffi.verifier.tmpdir, modname + suffix) + assert ffi.verifier.sourcefilename == fn1 + assert ffi.verifier.modulefilename == fn2 + class TestDistUtilsCPython(DistUtilsTest): generic = False From noreply at buildbot.pypy.org Mon Jan 7 16:27:46 2013 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 7 Jan 2013 16:27:46 +0100 (CET) Subject: [pypy-commit] pypy default: back out changeset 5d07b9c30222: did not change readthedocs build failure Message-ID: <20130107152746.DD3F61C1246@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r59846:7d092712837a Date: 2013-01-07 17:20 +0200 http://bitbucket.org/pypy/pypy/changeset/7d092712837a/ Log: back out changeset 5d07b9c30222: did not change readthedocs build failure diff --git a/pypy/doc/config/generate.py b/pypy/doc/config/generate.py --- a/pypy/doc/config/generate.py +++ b/pypy/doc/config/generate.py @@ -50,13 +50,6 @@ thisdir = py.path.local(__file__).dirpath() -#XXX hack - remove all old versions, try to be careful -for f in thisdir.listdir('objspace*.rst'): - f.remove() -for f in thisdir.listdir('translation*.rst'): - f.remove() - - for descr in all_optiondescrs: prefix = descr._name c = config.Config(descr) From noreply at buildbot.pypy.org Mon Jan 7 16:27:48 2013 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 7 Jan 2013 16:27:48 +0100 (CET) Subject: [pypy-commit] pypy default: touch problematic documentation file, will this clean up the readthedocs build? Message-ID: <20130107152748.10EFB1C1246@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r59847:05fa45adf925 Date: 2013-01-07 17:28 +0200 http://bitbucket.org/pypy/pypy/changeset/05fa45adf925/ Log: touch problematic documentation file, will this clean up the readthedocs build? diff --git a/pypy/doc/config/translation.no__thread.txt b/pypy/doc/config/translation.no__thread.txt --- a/pypy/doc/config/translation.no__thread.txt +++ b/pypy/doc/config/translation.no__thread.txt @@ -1,3 +1,3 @@ Don't use gcc __thread attribute for fast thread local storage -implementation . Increases the chance that moving the resulting +implementation. Increases the chance that moving the resulting executable to another same processor Linux machine will work. From noreply at buildbot.pypy.org Mon Jan 7 16:53:44 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 7 Jan 2013 16:53:44 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: revert the multi-int-size-thing Message-ID: <20130107155344.197B01C1233@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59848:c965b4839eb3 Date: 2013-01-07 17:53 +0200 http://bitbucket.org/pypy/pypy/changeset/c965b4839eb3/ Log: revert the multi-int-size-thing diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py --- a/pypy/rpython/lltypesystem/rdict.py +++ b/pypy/rpython/lltypesystem/rdict.py @@ -350,18 +350,18 @@ def _ll_malloc_indexes(n): # XXXX 64 bit only - if n & MAX_INT_MASK: - res = lltype.cast_opaque_ptr(llmemory.GCREF, - lltype.malloc(DICTINDEX_SIGNED.TO, n)) - elif n & MAX_SHORT_MASK: - res = lltype.cast_opaque_ptr(llmemory.GCREF, - lltype.malloc(DICTINDEX_INT.TO, n)) - elif n & MAX_BYTE_MASK: - res = lltype.cast_opaque_ptr(llmemory.GCREF, - lltype.malloc(DICTINDEX_SHORT.TO, n)) - else: - res = lltype.cast_opaque_ptr(llmemory.GCREF, - lltype.malloc(DICTINDEX_BYTE.TO, n)) + #if n & MAX_INT_MASK: + res = lltype.cast_opaque_ptr(llmemory.GCREF, + lltype.malloc(DICTINDEX_SIGNED.TO, n)) + #elif n & MAX_SHORT_MASK: + # res = lltype.cast_opaque_ptr(llmemory.GCREF, + # lltype.malloc(DICTINDEX_INT.TO, n)) + #elif n & MAX_BYTE_MASK: + # res = lltype.cast_opaque_ptr(llmemory.GCREF, + # lltype.malloc(DICTINDEX_SHORT.TO, n)) + #else: + # res = lltype.cast_opaque_ptr(llmemory.GCREF, + # lltype.malloc(DICTINDEX_BYTE.TO, n)) i = 0 while i < n: ll_index_setitem(n, res, i, FREE) @@ -369,30 +369,30 @@ return res def ll_index_getitem(size, indexes, i): - if size & MAX_INT_MASK: - return lltype.cast_opaque_ptr(DICTINDEX_SIGNED, indexes)[i] - elif size & MAX_SHORT_MASK: - return rffi.cast(lltype.Signed, - lltype.cast_opaque_ptr(DICTINDEX_INT, indexes)[i]) - elif size & MAX_BYTE_MASK: - return rffi.cast(lltype.Signed, - lltype.cast_opaque_ptr(DICTINDEX_SHORT, indexes)[i]) - else: - return rffi.cast(lltype.Signed, - lltype.cast_opaque_ptr(DICTINDEX_BYTE, indexes)[i]) + #if size & MAX_INT_MASK: + return lltype.cast_opaque_ptr(DICTINDEX_SIGNED, indexes)[i] + #elif size & MAX_SHORT_MASK: + # return rffi.cast(lltype.Signed, + # lltype.cast_opaque_ptr(DICTINDEX_INT, indexes)[i]) + #elif size & MAX_BYTE_MASK: + # return rffi.cast(lltype.Signed, + # lltype.cast_opaque_ptr(DICTINDEX_SHORT, indexes)[i]) + #else: + # return rffi.cast(lltype.Signed, + # lltype.cast_opaque_ptr(DICTINDEX_BYTE, indexes)[i]) def ll_index_setitem(size, indexes, i, v): - if size & MAX_INT_MASK: - lltype.cast_opaque_ptr(DICTINDEX_SIGNED, indexes)[i] = v - elif size & MAX_SHORT_MASK: - lltype.cast_opaque_ptr(DICTINDEX_INT, indexes)[i] = rffi.cast( - rffi.INT_real, v) - elif size & MAX_BYTE_MASK: - lltype.cast_opaque_ptr(DICTINDEX_SHORT, indexes)[i] = rffi.cast( - rffi.SHORT, v) - else: - lltype.cast_opaque_ptr(DICTINDEX_BYTE, indexes)[i] = rffi.cast( - rffi.SIGNEDCHAR, v) + #if size & MAX_INT_MASK: + lltype.cast_opaque_ptr(DICTINDEX_SIGNED, indexes)[i] = v + #elif size & MAX_SHORT_MASK: + # lltype.cast_opaque_ptr(DICTINDEX_INT, indexes)[i] = rffi.cast( + # rffi.INT_real, v) + #elif size & MAX_BYTE_MASK: + # lltype.cast_opaque_ptr(DICTINDEX_SHORT, indexes)[i] = rffi.cast( + # rffi.SHORT, v) + #else: + # lltype.cast_opaque_ptr(DICTINDEX_BYTE, indexes)[i] = rffi.cast( + # rffi.SIGNEDCHAR, v) def ll_dict_copy_indexes(size, from_indexes, to_indexes): i = 0 From noreply at buildbot.pypy.org Mon Jan 7 17:09:35 2013 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 7 Jan 2013 17:09:35 +0100 (CET) Subject: [pypy-commit] pypy default: Hack hack hack again at the same old problem: libffi.a. Message-ID: <20130107160935.85F111C1241@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r59849:a984a9dda83d Date: 2013-01-07 17:12 +0100 http://bitbucket.org/pypy/pypy/changeset/a984a9dda83d/ Log: Hack hack hack again at the same old problem: libffi.a. diff --git a/pypy/translator/platform/linux.py b/pypy/translator/platform/linux.py --- a/pypy/translator/platform/linux.py +++ b/pypy/translator/platform/linux.py @@ -36,7 +36,13 @@ # places where we need to look for libffi.a # XXX obscuuure! only look for libffi.a if run with translate.py if 'translate' in sys.modules: - return self.library_dirs_for_libffi() + ['/usr/lib'] + if sys.maxint > 2**32: + host = 'x86_64' + else: + host = 'x86' + return self.library_dirs_for_libffi() + [ + '/usr/lib', + '/usr/lib/%s-linux-gnu/' % host] else: return [] From noreply at buildbot.pypy.org Mon Jan 7 18:59:26 2013 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 7 Jan 2013 18:59:26 +0100 (CET) Subject: [pypy-commit] pypy default: move call to make_cmdline_overview so that readthedocs can find it Message-ID: <20130107175926.C3C841C1233@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r59850:832cba6f5385 Date: 2012-12-30 16:42 +0200 http://bitbucket.org/pypy/pypy/changeset/832cba6f5385/ Log: move call to make_cmdline_overview so that readthedocs can find it diff --git a/pypy/doc/Makefile b/pypy/doc/Makefile --- a/pypy/doc/Makefile +++ b/pypy/doc/Makefile @@ -32,38 +32,38 @@ -rm -rf $(BUILDDIR)/* html: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." pickle: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ @@ -73,7 +73,7 @@ @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/PyPy.qhc" latex: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @@ -81,26 +81,26 @@ "run these through (pdf)latex." man: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man" changes: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." doctest: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." diff --git a/pypy/doc/pypyconfig.py b/pypy/doc/pypyconfig.py --- a/pypy/doc/pypyconfig.py +++ b/pypy/doc/pypyconfig.py @@ -3,6 +3,10 @@ def setup(app): import sys, os sys.path.insert(0, os.path.abspath("../../")) + + #Autmatically calls make_cmdlline_overview + from pypy.doc.config import generate + from pypy.config import makerestdoc import py role = makerestdoc.register_config_role(py.path.local()) From noreply at buildbot.pypy.org Mon Jan 7 21:16:23 2013 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 7 Jan 2013 21:16:23 +0100 (CET) Subject: [pypy-commit] pypy py3k: hg merge default Message-ID: <20130107201623.DC6A41C1233@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r59851:544ca7a6d0ee Date: 2013-01-06 17:46 +0100 http://bitbucket.org/pypy/pypy/changeset/544ca7a6d0ee/ Log: hg merge default diff too long, truncating to 2000 out of 10942 lines diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -28,7 +28,7 @@ DEALINGS IN THE SOFTWARE. -PyPy Copyright holders 2003-2012 +PyPy Copyright holders 2003-2013 ----------------------------------- Except when otherwise stated (look for LICENSE files or information at diff --git a/lib-python/2.7/sre_parse.py b/lib-python/2.7/sre_parse.py --- a/lib-python/2.7/sre_parse.py +++ b/lib-python/2.7/sre_parse.py @@ -16,6 +16,12 @@ from sre_constants import * +try: + from __pypy__ import newdict +except ImportError: + def newdict(tp): + return {} + SPECIAL_CHARS = ".\\[{()*+?^$|" REPEAT_CHARS = "*+?{" @@ -68,7 +74,7 @@ self.flags = 0 self.open = [] self.groups = 1 - self.groupdict = {} + self.groupdict = newdict("module") def opengroup(self, name=None): gid = self.groups self.groups = gid + 1 diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst --- a/pypy/doc/index.rst +++ b/pypy/doc/index.rst @@ -67,7 +67,7 @@ .. _Python: http://docs.python.org/index.html .. _`more...`: architecture.html#mission-statement .. _`PyPy blog`: http://morepypy.blogspot.com/ -.. _`development bug/feature tracker`: https://codespeak.net/issue/pypy-dev/ +.. _`development bug/feature tracker`: https://bugs.pypy.org .. _here: http://tismerysoft.de/pypy/irc-logs/pypy .. _`Mercurial commit mailing list`: http://python.org/mailman/listinfo/pypy-commit .. _`development mailing list`: http://python.org/mailman/listinfo/pypy-dev diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -5,15 +5,35 @@ .. this is a revision shortly after release-2.0-beta1 .. startrev: 0e6161a009c6 +.. branch: callback-jit +Callbacks from C are now better JITted + +.. branch: remove-globals-in-jit + .. branch: length-hint Implement __lenght_hint__ according to PEP 424 +.. branch: numpypy-longdouble +Long double support for numpypy + +.. branch: signatures +Improved RPython typing + +.. branch: rpython-bytearray +Rudimentary support for bytearray in RPython + .. branches we don't care about .. branch: autoreds +.. branch: reflex-support +.. branch: kill-faking +.. branch: improved_ebnfparse_error +.. branch: task-decorator .. branch: release-2.0-beta1 .. branch: remove-PYPY_NOT_MAIN_FILE +.. branch: missing-jit-operations + .. branch: fix-lookinside-iff-oopspec Fixed the interaction between two internal tools for controlling the JIT. diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -56,11 +56,24 @@ ---------------------------- On Windows, there is no standard place where to download, build and -install third-party libraries. We chose to install them in the parent +install third-party libraries. We recommend installing them in the parent directory of the pypy checkout. For example, if you installed pypy in ``d:\pypy\trunk\`` (This directory contains a README file), the base -directory is ``d:\pypy``. You may choose different values by setting the -INCLUDE, LIB and PATH (for DLLs) +directory is ``d:\pypy``. You must then set the +INCLUDE, LIB and PATH (for DLLs) environment variables appropriately. + +Abridged method (for -Ojit builds using Visual Studio 2008) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Download the versions of all the external packages +from +https://bitbucket.org/pypy/pypy/downloads/local.zip +Then expand it into the base directory (base_dir) and modify your environment to reflect this:: + + set PATH=\bin;%PATH% + set INCLUDE=\include;%INCLUDE% + set LIB=\lib;%LIB% + +Now you should be good to go. Read on for more information. The Boehm garbage collector ~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py --- a/pypy/jit/backend/arm/assembler.py +++ b/pypy/jit/backend/arm/assembler.py @@ -1,5 +1,6 @@ from __future__ import with_statement import os +from pypy.jit.backend.llsupport import jitframe from pypy.jit.backend.arm.helper.assembler import saved_registers from pypy.jit.backend.arm import conditions as c from pypy.jit.backend.arm import registers as r @@ -25,12 +26,12 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.jit.backend.arm.opassembler import ResOpAssembler from pypy.rlib.debug import (debug_print, debug_start, debug_stop, - have_debug_prints) + have_debug_prints, fatalerror) from pypy.rlib.jit import AsmInfo from pypy.rlib.objectmodel import compute_unique_id # XXX Move to llsupport -from pypy.jit.backend.x86.support import values_array, memcpy_fn +from pypy.jit.backend.x86.support import memcpy_fn DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', ('i', lltype.Signed), ('type', lltype.Char), # 'b'ridge, 'l'abel or @@ -44,14 +45,8 @@ debug = True - def __init__(self, cpu, failargs_limit=1000): + def __init__(self, cpu, translate_support_code=False): self.cpu = cpu - self.fail_boxes_int = values_array(lltype.Signed, failargs_limit) - self.fail_boxes_float = values_array(longlong.FLOATSTORAGE, - failargs_limit) - self.fail_boxes_ptr = values_array(llmemory.GCREF, failargs_limit) - self.fail_boxes_count = 0 - self.fail_force_index = 0 self.setup_failure_recovery() self.mc = None self.memcpy_addr = 0 @@ -68,6 +63,7 @@ self._debug = False self.loop_run_counters = [] self.debug_counter_descr = cpu.fielddescrof(DEBUG_COUNTER, 'i') + self.force_token_to_dead_frame = {} # XXX temporary hack def set_debug(self, v): r = self._debug @@ -112,9 +108,13 @@ gc_ll_descr.initialize() self._build_wb_slowpath(False) self._build_wb_slowpath(True) + self._build_failure_recovery(exc=True, withfloats=False) + self._build_failure_recovery(exc=False, withfloats=False) if self.cpu.supports_floats: self._build_wb_slowpath(False, withfloats=True) self._build_wb_slowpath(True, withfloats=True) + self._build_failure_recovery(exc=True, withfloats=True) + self._build_failure_recovery(exc=False, withfloats=True) self._build_propagate_exception_path() if gc_ll_descr.get_malloc_slowpath_addr is not None: self._build_malloc_slowpath() @@ -122,10 +122,7 @@ if gc_ll_descr.gcrootmap and gc_ll_descr.gcrootmap.is_shadow_stack: self._build_release_gil(gc_ll_descr.gcrootmap) self.memcpy_addr = self.cpu.cast_ptr_to_int(memcpy_fn) - self._exit_code_addr = self._gen_exit_path() - self._leave_jitted_hook_save_exc = \ - self._gen_leave_jitted_hook_code(True) - self._leave_jitted_hook = self._gen_leave_jitted_hook_code(False) + if not self._debug: # if self._debug is already set it means that someone called # set_debug by hand before initializing the assembler. Leave it @@ -220,36 +217,17 @@ self.releasegil_addr = rffi.cast(lltype.Signed, releasegil_func) self.reacqgil_addr = rffi.cast(lltype.Signed, reacqgil_func) - def _gen_leave_jitted_hook_code(self, save_exc): - mc = ARMv7Builder() - if self.cpu.supports_floats: - floats = r.caller_vfp_resp - else: - floats = [] - with saved_registers(mc, r.caller_resp + [r.lr], floats): - addr = self.cpu.get_on_leave_jitted_int(save_exception=save_exc) - mc.BL(addr) - assert self._exit_code_addr != 0 - mc.B(self._exit_code_addr) - return mc.materialize(self.cpu.asmmemmgr, [], - self.cpu.gc_ll_descr.gcrootmap) - def _build_propagate_exception_path(self): if self.cpu.propagate_exception_v < 0: return # not supported (for tests, or non-translated) # mc = ARMv7Builder() - # call on_leave_jitted_save_exc() - if self.cpu.supports_floats: - floats = r.caller_vfp_resp - else: - floats = [] - with saved_registers(mc, r.caller_resp + [r.lr], floats): - addr = self.cpu.get_on_leave_jitted_int(save_exception=True, - default_to_memoryerror=True) - mc.BL(addr) - mc.gen_load_int(r.ip.value, self.cpu.propagate_exception_v) - mc.MOV_rr(r.r0.value, r.ip.value) + # + # Call the helper, which will return a dead frame object with + # the correct exception set, or MemoryError by default + # XXX make sure we return the correct value here + addr = rffi.cast(lltype.Signed, self.cpu.get_propagate_exception()) + mc.BL(addr) self.gen_func_epilog(mc=mc) self.propagate_exception_path = mc.materialize(self.cpu.asmmemmgr, []) @@ -282,12 +260,12 @@ # restore registers and return # We check for c.EQ here, meaning all bits zero in this case mc.POP([reg.value for reg in r.argument_regs] + [r.pc.value], cond=c.EQ) - # call on_leave_jitted_save_exc() - addr = self.cpu.get_on_leave_jitted_int(save_exception=True) + # + # Call the helper, which will return a dead frame object with + # the correct exception set, or MemoryError by default + addr = rffi.cast(lltype.Signed, self.cpu.get_propagate_exception()) mc.BL(addr) # - mc.gen_load_int(r.r0.value, self.cpu.propagate_exception_v) - # # footer -- note the ADD, which skips the return address of this # function, and will instead return to the caller's caller. Note # also that we completely ignore the saved arguments, because we @@ -339,39 +317,82 @@ def setup_failure_recovery(self): - @rgc.no_collect + #@rgc.no_collect -- XXX still true, but hacked gc_set_extra_threshold def failure_recovery_func(mem_loc, frame_pointer, stack_pointer): """mem_loc is a structure in memory describing where the values for the failargs are stored. frame loc is the address of the frame pointer for the frame to be decoded frame """ - vfp_registers = rffi.cast(rffi.LONGLONGP, stack_pointer) - registers = rffi.ptradd(vfp_registers, len(r.all_vfp_regs)) + vfp_registers = rffi.cast(rffi.LONGP, stack_pointer) + registers = rffi.ptradd(vfp_registers, 2*len(r.all_vfp_regs)) registers = rffi.cast(rffi.LONGP, registers) - return self.decode_registers_and_descr(mem_loc, frame_pointer, + bytecode = rffi.cast(rffi.UCHARP, mem_loc) + return self.grab_frame_values(self.cpu, bytecode, frame_pointer, registers, vfp_registers) + self.failure_recovery_code = [0, 0, 0, 0] self.failure_recovery_func = failure_recovery_func - recovery_func_sign = lltype.Ptr(lltype.FuncType([lltype.Signed] * 3, - lltype.Signed)) + _FAILURE_RECOVERY_FUNC = lltype.Ptr(lltype.FuncType([rffi.LONGP] * 3, + llmemory.GCREF)) - @rgc.no_collect - def decode_registers_and_descr(self, mem_loc, frame_pointer, + @staticmethod + #@rgc.no_collect -- XXX still true, but hacked gc_set_extra_threshold + def grab_frame_values(cpu, bytecode, frame_pointer, registers, vfp_registers): - """Decode locations encoded in memory at mem_loc and write the values - to the failboxes. Values for spilled vars and registers are stored on - stack at frame_loc """ - assert frame_pointer & 1 == 0 - self.fail_force_index = frame_pointer - bytecode = rffi.cast(rffi.UCHARP, mem_loc) + # no malloc allowed here!! xxx apart from one, hacking a lot + force_index = rffi.cast(lltype.Signed, frame_pointer) num = 0 - value = 0 - fvalue = 0 + deadframe = lltype.nullptr(jitframe.DEADFRAME) + # step 1: lots of mess just to count the final value of 'num' + bytecode1 = bytecode + while 1: + code = rffi.cast(lltype.Signed, bytecode1[0]) + bytecode1 = rffi.ptradd(bytecode1, 1) + if code >= AssemblerARM.CODE_FROMSTACK: + while code > 0x7F: + code = rffi.cast(lltype.Signed, bytecode1[0]) + bytecode1 = rffi.ptradd(bytecode1, 1) + else: + kind = code & 3 + if kind == AssemblerARM.DESCR_SPECIAL: + if code == AssemblerARM.CODE_HOLE: + num += 1 + continue + if code == AssemblerARM.CODE_INPUTARG: + continue + if code == AssemblerARM.CODE_FORCED: + # resuming from a GUARD_NOT_FORCED + token = force_index + deadframe = ( + cpu.assembler.force_token_to_dead_frame.pop(token)) + deadframe = lltype.cast_opaque_ptr( + jitframe.DEADFRAMEPTR, deadframe) + continue + assert code == AssemblerARM.CODE_STOP + break + num += 1 + + # allocate the deadframe + if not deadframe: + # Remove the "reserve" at the end of the nursery. This means + # that it is guaranteed that the following malloc() works + # without requiring a collect(), but it needs to be re-added + # as soon as possible. + cpu.gc_clear_extra_threshold() + assert num <= cpu.get_failargs_limit() + try: + deadframe = lltype.malloc(jitframe.DEADFRAME, num) + except MemoryError: + fatalerror("memory usage error in grab_frame_values") + # fill it code_inputarg = False - while True: + num = 0 + value_hi = 0 + while 1: + # decode the next instruction from the bytecode code = rffi.cast(lltype.Signed, bytecode[0]) bytecode = rffi.ptradd(bytecode, 1) - if code >= self.CODE_FROMSTACK: + if code >= AssemblerARM.CODE_FROMSTACK: if code > 0x7F: shift = 7 code &= 0x7F @@ -384,54 +405,63 @@ break # load the value from the stack kind = code & 3 - code = int((code - self.CODE_FROMSTACK) >> 2) + code = (code - AssemblerARM.CODE_FROMSTACK) >> 2 if code_inputarg: code = ~code code_inputarg = False - if kind == self.DESCR_FLOAT: - # we use code + 1 to get the hi word of the double worded float - stackloc = frame_pointer - get_fp_offset(int(code) + 1) - assert stackloc & 3 == 0 - fvalue = rffi.cast(rffi.LONGLONGP, stackloc)[0] - else: - stackloc = frame_pointer - get_fp_offset(int(code)) - assert stackloc & 1 == 0 - value = rffi.cast(rffi.LONGP, stackloc)[0] + stackloc = force_index - get_fp_offset(int(code)) + value = rffi.cast(rffi.LONGP, stackloc)[0] + if kind == AssemblerARM.DESCR_FLOAT: + assert WORD == 4 + value_hi = value + value = rffi.cast(rffi.LONGP, stackloc - WORD)[0] else: - # 'code' identifies a register: load its value kind = code & 3 - if kind == self.DESCR_SPECIAL: - if code == self.CODE_HOLE: + if kind == AssemblerARM.DESCR_SPECIAL: + if code == AssemblerARM.CODE_HOLE: num += 1 continue - if code == self.CODE_INPUTARG: + if code == AssemblerARM.CODE_INPUTARG: code_inputarg = True continue - assert code == self.CODE_STOP + if code == AssemblerARM.CODE_FORCED: + continue + assert code == AssemblerARM.CODE_STOP break + # 'code' identifies a register: load its value code >>= 2 - if kind == self.DESCR_FLOAT: - fvalue = vfp_registers[code] + if kind == AssemblerARM.DESCR_FLOAT: + if WORD == 4: + value = vfp_registers[2*code] + value_hi = vfp_registers[2*code + 1] + else: + value = registers[code] else: value = registers[code] # store the loaded value into fail_boxes_ - if kind == self.DESCR_FLOAT: - tgt = self.fail_boxes_float.get_addr_for_num(num) - rffi.cast(rffi.LONGLONGP, tgt)[0] = fvalue + if kind == AssemblerARM.DESCR_INT: + deadframe.jf_values[num].int = value + elif kind == AssemblerARM.DESCR_REF: + deadframe.jf_values[num].ref = rffi.cast(llmemory.GCREF, value) + elif kind == AssemblerARM.DESCR_FLOAT: + assert WORD == 4 + assert not longlong.is_64_bit + floatvalue = rffi.cast(lltype.SignedLongLong, value_hi) + floatvalue <<= 32 + floatvalue |= rffi.cast(lltype.SignedLongLong, + rffi.cast(lltype.Unsigned, value)) + deadframe.jf_values[num].float = floatvalue else: - if kind == self.DESCR_INT: - tgt = self.fail_boxes_int.get_addr_for_num(num) - elif kind == self.DESCR_REF: - assert (value & 3) == 0, "misaligned pointer" - tgt = self.fail_boxes_ptr.get_addr_for_num(num) - else: - assert 0, "bogus kind" - rffi.cast(rffi.LONGP, tgt)[0] = value + assert 0, "bogus kind" num += 1 - self.fail_boxes_count = num + # + assert num == len(deadframe.jf_values) + if not we_are_translated(): + assert bytecode[4] == 0xCC fail_index = rffi.cast(rffi.INTP, bytecode)[0] - fail_index = rffi.cast(lltype.Signed, fail_index) - return fail_index + fail_descr = cpu.get_fail_descr_from_number(fail_index) + deadframe.jf_descr = fail_descr.hide(cpu) + return lltype.cast_opaque_ptr(llmemory.GCREF, deadframe) def decode_inputargs(self, code): descr_to_box_type = [REF, INT, FLOAT] @@ -514,12 +544,26 @@ self.mc.CMP_ri(r.r0.value, 0) self.mc.B(self.propagate_exception_path, c=c.EQ) - def _gen_exit_path(self): + def _build_failure_recovery(self, exc, withfloats=False): mc = ARMv7Builder() - decode_registers_addr = llhelper(self.recovery_func_sign, + failure_recovery = llhelper(self._FAILURE_RECOVERY_FUNC, self.failure_recovery_func) self._insert_checks(mc) - with saved_registers(mc, r.all_regs, r.all_vfp_regs): + if withfloats: + f = r.all_vfp_regs + else: + f = [] + with saved_registers(mc, r.all_regs, f): + if exc: + # We might have an exception pending. Load it into r4 + # (this is a register saved across calls) + mc.gen_load_int(r.r5.value, self.cpu.pos_exc_value()) + mc.LDR_ri(r.r4.value, self.cpu.pos_exc_value()) + # clear the exc flags + mc.gen_load_int(r.r6.value, 0) + mc.STR_ri(r.r6.value, r.r5.value) + mc.gen_load_int(r.r5.value, self.cpu.pos_exception()) + mc.STR_ri(r.r6.value, r.r5.value) # move mem block address, to r0 to pass as mc.MOV_rr(r.r0.value, r.lr.value) # pass the current frame pointer as second param @@ -527,23 +571,32 @@ # pass the current stack pointer as third param mc.MOV_rr(r.r2.value, r.sp.value) self._insert_checks(mc) - mc.BL(rffi.cast(lltype.Signed, decode_registers_addr)) + mc.BL(rffi.cast(lltype.Signed, failure_recovery)) + if exc: + # save ebx into 'jf_guard_exc' + from pypy.jit.backend.llsupport.descr import unpack_fielddescr + descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) + offset, size, _ = unpack_fielddescr(descrs.jf_guard_exc) + mc.STR_rr(r.r4.value, r.r0.value, offset, cond=c.AL) mc.MOV_rr(r.ip.value, r.r0.value) mc.MOV_rr(r.r0.value, r.ip.value) self.gen_func_epilog(mc=mc) - return mc.materialize(self.cpu.asmmemmgr, [], + rawstart = mc.materialize(self.cpu.asmmemmgr, [], self.cpu.gc_ll_descr.gcrootmap) + self.failure_recovery_code[exc + 2 * withfloats] = rawstart + self.mc = None DESCR_REF = 0x00 DESCR_INT = 0x01 DESCR_FLOAT = 0x02 DESCR_SPECIAL = 0x03 CODE_FROMSTACK = 64 - CODE_STOP = 0 | DESCR_SPECIAL - CODE_HOLE = 4 | DESCR_SPECIAL - CODE_INPUTARG = 8 | DESCR_SPECIAL + CODE_STOP = 0 | DESCR_SPECIAL + CODE_HOLE = 4 | DESCR_SPECIAL + CODE_INPUTARG = 8 | DESCR_SPECIAL + CODE_FORCED = 12 | DESCR_SPECIAL #XXX where should this be written? - def gen_descr_encoding(self, descr, failargs, locs): + def write_failure_recovery_description(self, descr, failargs, locs): assert self.mc is not None for i in range(len(failargs)): arg = failargs[i] @@ -575,26 +628,33 @@ self.mc.writechar(chr(n)) self.mc.writechar(chr(self.CODE_STOP)) - fdescr = self.cpu.get_fail_descr_number(descr) - self.mc.write32(fdescr) + + def generate_quick_failure(self, guardtok, fcond=c.AL): + assert isinstance(guardtok.save_exc, bool) + fail_index = self.cpu.get_fail_descr_number(guardtok.descr) + startpos = self.mc.currpos() + withfloats = False + for box in guardtok.failargs: + if box is not None and box.type == FLOAT: + withfloats = True + break + exc = guardtok.save_exc + target = self.failure_recovery_code[exc + 2 * withfloats] + assert target != 0 + self.mc.BL(target) + # write tight data that describes the failure recovery + if guardtok.is_guard_not_forced: + self.mc.writechar(chr(self.CODE_FORCED)) + self.write_failure_recovery_description(guardtok.descr, + guardtok.failargs, guardtok.faillocs[1:]) + self.mc.write32(fail_index) + # for testing the decoding, write a final byte 0xCC + if not we_are_translated(): + self.mc.writechar('\xCC') + faillocs = [loc for loc in guardtok.faillocs if loc is not None] + guardtok.descr._arm_debug_faillocs = faillocs self.align() - - # assert that the fail_boxes lists are big enough - assert len(failargs) <= self.fail_boxes_int.SIZE - - def _gen_path_to_exit_path(self, descr, args, arglocs, - save_exc, fcond=c.AL): - assert isinstance(save_exc, bool) - self.gen_exit_code(self.mc, save_exc, fcond) - self.gen_descr_encoding(descr, args, arglocs[1:]) - - def gen_exit_code(self, mc, save_exc, fcond=c.AL): - assert isinstance(save_exc, bool) - if save_exc: - path = self._leave_jitted_hook_save_exc - else: - path = self._leave_jitted_hook - mc.BL(path) + return startpos def align(self): while(self.mc.currpos() % FUNC_ALIGN != 0): @@ -841,15 +901,10 @@ def write_pending_failure_recoveries(self): for tok in self.pending_guards: - descr = tok.descr #generate the exit stub and the encoded representation - pos = self.mc.currpos() - tok.pos_recovery_stub = pos - - self._gen_path_to_exit_path(descr, tok.failargs, - tok.faillocs, save_exc=tok.save_exc) + tok.pos_recovery_stub = self.generate_quick_failure(tok) # store info on the descr - descr._arm_current_frame_depth = tok.faillocs[0].getint() + tok.descr._arm_current_frame_depth = tok.faillocs[0].getint() def process_pending_guards(self, block_start): clt = self.current_clt @@ -860,7 +915,7 @@ descr._arm_failure_recovery_block = failure_recovery_pos relative_offset = tok.pos_recovery_stub - tok.offset guard_pos = block_start + tok.offset - if not tok.is_invalidate: + if not tok.is_guard_not_invalidated: # patch the guard jumpt to the stub # overwrite the generate NOP with a B_offs to the pos of the # stub @@ -1255,11 +1310,6 @@ else: raise AssertionError('Trying to pop to an invalid location') - def leave_jitted_hook(self): - ptrs = self.fail_boxes_ptr.ar - llop.gc_assume_young_pointers(lltype.Void, - llmemory.cast_ptr_to_adr(ptrs)) - def malloc_cond(self, nursery_free_adr, nursery_top_adr, size): assert size & (WORD-1) == 0 # must be correctly aligned diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py --- a/pypy/jit/backend/arm/opassembler.py +++ b/pypy/jit/backend/arm/opassembler.py @@ -16,6 +16,7 @@ gen_emit_unary_float_op, saved_registers, count_reg_args) +from pypy.jit.backend.arm.helper.regalloc import check_imm_arg from pypy.jit.backend.arm.codebuilder import ARMv7Builder, OverwritingBuilder from pypy.jit.backend.arm.jump import remap_frame_layout from pypy.jit.backend.arm.regalloc import TempInt, TempPtr @@ -27,18 +28,20 @@ from pypy.jit.metainterp.history import JitCellToken, TargetToken from pypy.jit.metainterp.resoperation import rop from pypy.rlib.objectmodel import we_are_translated -from pypy.rpython.lltypesystem import rstr +from pypy.rlib import rgc +from pypy.rpython.lltypesystem import rstr, rffi, lltype, llmemory NO_FORCE_INDEX = -1 class GuardToken(object): - def __init__(self, descr, failargs, faillocs, offset, - save_exc, fcond=c.AL, is_invalidate=False): + def __init__(self, descr, failargs, faillocs, offset, save_exc, fcond=c.AL, + is_guard_not_invalidated=False, is_guard_not_forced=False): assert isinstance(save_exc, bool) self.descr = descr self.offset = offset - self.is_invalidate = is_invalidate + self.is_guard_not_invalidated = is_guard_not_invalidated + self.is_guard_not_forced = is_guard_not_forced self.failargs = failargs self.faillocs = faillocs self.save_exc = save_exc @@ -190,7 +193,8 @@ return fcond def _emit_guard(self, op, arglocs, fcond, save_exc, - is_guard_not_invalidated=False): + is_guard_not_invalidated=False, + is_guard_not_forced=False): assert isinstance(save_exc, bool) assert isinstance(fcond, int) descr = op.getdescr() @@ -210,7 +214,8 @@ faillocs=arglocs, offset=pos, save_exc=save_exc, - is_invalidate=is_guard_not_invalidated, + is_guard_not_invalidated=is_guard_not_invalidated, + is_guard_not_forced=is_guard_not_forced, fcond=fcond)) return c.AL @@ -312,49 +317,10 @@ return fcond def emit_op_finish(self, op, arglocs, regalloc, fcond): - for i in range(len(arglocs) - 1): - loc = arglocs[i] - box = op.getarg(i) - if loc is None: - continue - if loc.is_reg(): - if box.type == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(i) - elif box.type == INT: - adr = self.fail_boxes_int.get_addr_for_num(i) - else: - assert 0 - self.mc.gen_load_int(r.ip.value, adr) - self.mc.STR_ri(loc.value, r.ip.value) - elif loc.is_vfp_reg(): - assert box.type == FLOAT - adr = self.fail_boxes_float.get_addr_for_num(i) - self.mc.gen_load_int(r.ip.value, adr) - self.mc.VSTR(loc.value, r.ip.value) - elif loc.is_stack() or loc.is_imm() or loc.is_imm_float(): - if box.type == FLOAT: - adr = self.fail_boxes_float.get_addr_for_num(i) - self.mov_loc_loc(loc, r.vfp_ip) - self.mc.gen_load_int(r.ip.value, adr) - self.mc.VSTR(r.vfp_ip.value, r.ip.value) - elif box.type == REF or box.type == INT: - if box.type == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(i) - elif box.type == INT: - adr = self.fail_boxes_int.get_addr_for_num(i) - else: - assert 0 - self.mov_loc_loc(loc, r.ip) - self.mc.gen_load_int(r.lr.value, adr) - self.mc.STR_ri(r.ip.value, r.lr.value) - else: - assert 0 - # note: no exception should currently be set in llop.get_exception_addr - # even if this finish may be an exit_frame_with_exception (in this case - # the exception instance is in arglocs[0]). - addr = self.cpu.get_on_leave_jitted_int(save_exception=False) - self.mc.BL(addr) - self.mc.gen_load_int(r.r0.value, arglocs[-1].value) + [argloc] = arglocs + if argloc is not r.r0: #XXX verify this + self.mov_loc_loc(argloc, r.r0, fcond) + # exit function self.gen_func_epilog() return fcond @@ -1152,8 +1118,26 @@ value = self.cpu.done_with_this_frame_float_v else: raise AssertionError(kind) - self.mc.gen_load_int(r.ip.value, value) - self.mc.CMP_rr(tmploc.value, r.ip.value) + from pypy.jit.backend.llsupport.descr import unpack_fielddescr + from pypy.jit.backend.llsupport.descr import unpack_interiorfielddescr + descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) + _offset, _size, _ = unpack_fielddescr(descrs.jf_descr) + fail_descr = self.cpu.get_fail_descr_from_number(value) + value = fail_descr.hide(self.cpu) + rgc._make_sure_does_not_move(value) + value = rffi.cast(lltype.Signed, value) + + if check_imm_arg(_offset): + self.mc.LDR_ri(r.ip.value, tmploc.value, imm=_offset) + else: + self.mc.gen_load_int(r.ip.value, _offset) + self.mc.LDR_rr(r.ip.value, tmploc.value, r.ip.value) + if check_imm_arg(value): + self.mc.CMP_ri(r.ip.value, imm=value) + else: + self.mc.gen_load_int(r.lr.value, value) + self.mc.CMP_rr(r.lr.value, r.ip.value) + #if values are equal we take the fast path # Slow path, calling helper @@ -1180,19 +1164,31 @@ if op.result is not None: # load the return value from fail_boxes_xxx[0] kind = op.result.type - if kind == INT: - adr = self.fail_boxes_int.get_addr_for_num(0) - elif kind == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(0) - elif kind == FLOAT: - adr = self.fail_boxes_float.get_addr_for_num(0) + if kind == FLOAT: + t = unpack_interiorfielddescr(descrs.as_float)[0] + if not check_imm_arg(t): + self.mc.gen_load_int(r.ip.value, t, cond=fast_path_cond) + self.mc.ADD_rr(r.ip.value, r.r0.value, r.ip.value, + cond=fast_path_cond) + t = 0 + base = r.ip + else: + base = r.r0 + self.mc.VLDR(resloc.value, base.value, imm=t, + cond=fast_path_cond) else: - raise AssertionError(kind) - self.mc.gen_load_int(r.ip.value, adr, cond=fast_path_cond) - if op.result.type == FLOAT: - self.mc.VLDR(resloc.value, r.ip.value, cond=fast_path_cond) - else: - self.mc.LDR_ri(resloc.value, r.ip.value, cond=fast_path_cond) + assert resloc is r.r0 + if kind == INT: + t = unpack_interiorfielddescr(descrs.as_int)[0] + else: + t = unpack_interiorfielddescr(descrs.as_ref)[0] + if not check_imm_arg(t): + self.mc.gen_load_int(r.ip.value, t, cond=fast_path_cond) + self.mc.LDR_rr(resloc.value, resloc.value, r.ip.value, + cond=fast_path_cond) + else: + self.mc.LDR_ri(resloc.value, resloc.value, imm=t, + cond=fast_path_cond) # jump to merge point jmp_pos = self.mc.currpos() self.mc.BKPT() @@ -1266,7 +1262,8 @@ self.mc.LDR_ri(r.ip.value, r.fp.value) self.mc.CMP_ri(r.ip.value, 0) - self._emit_guard(guard_op, arglocs[1 + numargs:], c.GE, save_exc=True) + self._emit_guard(guard_op, arglocs[1 + numargs:], c.GE, + save_exc=True, is_guard_not_forced=True) return fcond def emit_guard_call_release_gil(self, op, guard_op, arglocs, regalloc, diff --git a/pypy/jit/backend/arm/regalloc.py b/pypy/jit/backend/arm/regalloc.py --- a/pypy/jit/backend/arm/regalloc.py +++ b/pypy/jit/backend/arm/regalloc.py @@ -644,15 +644,9 @@ return args def prepare_op_finish(self, op, fcond): - args = [None] * (op.numargs() + 1) - for i in range(op.numargs()): - arg = op.getarg(i) - if arg: - args[i] = self.loc(arg) - self.possibly_free_var(arg) - n = self.cpu.get_fail_descr_number(op.getdescr()) - args[-1] = imm(n) - return args + loc = self.loc(op.getarg(0)) + self.possibly_free_var(op.getarg(0)) + return [loc] def prepare_op_guard_true(self, op, fcond): l0 = self.make_sure_var_in_reg(op.getarg(0)) diff --git a/pypy/jit/backend/arm/runner.py b/pypy/jit/backend/arm/runner.py --- a/pypy/jit/backend/arm/runner.py +++ b/pypy/jit/backend/arm/runner.py @@ -23,15 +23,21 @@ AbstractLLCPU.__init__(self, rtyper, stats, opts, translate_support_code, gcdescr) + from pypy.jit.backend.llsupport import jitframe + self.deadframe_size_max = llmemory.sizeof(jitframe.DEADFRAME, + self.get_failargs_limit()) + def set_debug(self, flag): return self.assembler.set_debug(flag) + def get_failargs_limit(self): + if self.opts is not None: + return self.opts.failargs_limit + else: + return 1000 + def setup(self): - if self.opts is not None: - failargs_limit = self.opts.failargs_limit - else: - failargs_limit = 1000 - self.assembler = AssemblerARM(self, failargs_limit=failargs_limit) + self.assembler = AssemblerARM(self, self.translate_support_code) def setup_once(self): self.assembler.setup_once() @@ -51,24 +57,6 @@ return self.assembler.assemble_bridge(faildescr, inputargs, operations, original_loop_token, log=log) - def get_latest_value_float(self, index): - return self.assembler.fail_boxes_float.getitem(index) - - def get_latest_value_int(self, index): - return self.assembler.fail_boxes_int.getitem(index) - - def get_latest_value_ref(self, index): - return self.assembler.fail_boxes_ptr.getitem(index) - - def get_latest_value_count(self): - return self.assembler.fail_boxes_count - - def get_latest_force_token(self): - return self.assembler.fail_force_index - - def get_on_leave_jitted_hook(self): - return self.assembler.leave_jitted_hook - def clear_latest_values(self, count): setitem = self.assembler.fail_boxes_ptr.setitem null = lltype.nullptr(llmemory.GCREF.TO) @@ -76,7 +64,7 @@ setitem(index, null) def make_execute_token(self, *ARGS): - FUNCPTR = lltype.Ptr(lltype.FuncType(ARGS, lltype.Signed)) + FUNCPTR = lltype.Ptr(lltype.FuncType(ARGS, llmemory.GCREF)) def execute_token(executable_token, *args): clt = executable_token.compiled_loop_token @@ -91,12 +79,13 @@ prev_interpreter = LLInterpreter.current_interpreter LLInterpreter.current_interpreter = self.debug_ll_interpreter try: - fail_index = func(*args) + deadframe = func(*args) finally: if not self.translate_support_code: LLInterpreter.current_interpreter = prev_interpreter #llop.debug_print(lltype.Void, "<<<< Back") - return self.get_fail_descr_from_number(fail_index) + self.gc_set_extra_threshold() + return deadframe return execute_token def cast_ptr_to_int(x): @@ -115,17 +104,22 @@ assert fail_index >= 0, "already forced!" faildescr = self.get_fail_descr_from_number(fail_index) rffi.cast(TP, addr_of_force_index)[0] = ~fail_index - bytecode = self.assembler._find_failure_recovery_bytecode(faildescr) + frb = self.assembler._find_failure_recovery_bytecode(faildescr) + bytecode = rffi.cast(rffi.UCHARP, frb) addr_all_null_regsiters = rffi.cast(rffi.LONG, self.all_null_registers) - # start of "no gc operation!" block - fail_index_2 = self.assembler.failure_recovery_func( - bytecode, - addr_of_force_index, - addr_all_null_regsiters) - self.assembler.leave_jitted_hook() - # end of "no gc operation!" block - assert fail_index == fail_index_2 - return faildescr + # + assert (rffi.cast(lltype.Signed, bytecode[0]) == + self.assembler.CODE_FORCED) + bytecode = rffi.ptradd(bytecode, 1) + deadframe = self.assembler.grab_frame_values(self, + bytecode, addr_of_force_index, + self.all_null_registers, + self.all_null_registers) + # + assert self.get_latest_descr(deadframe) is faildescr + self.assembler.force_token_to_dead_frame[addr_of_force_index] = ( + deadframe) + return deadframe def redirect_call_assembler(self, oldlooptoken, newlooptoken): self.assembler.redirect_call_assembler(oldlooptoken, newlooptoken) diff --git a/pypy/jit/backend/arm/test/test_gc_integration.py b/pypy/jit/backend/arm/test/test_gc_integration.py --- a/pypy/jit/backend/arm/test/test_gc_integration.py +++ b/pypy/jit/backend/arm/test/test_gc_integration.py @@ -20,7 +20,7 @@ from pypy.jit.backend.arm.test.test_regalloc import BaseTestRegalloc from pypy.jit.backend.arm.regalloc import ARMFrameManager, VFPRegisterManager from pypy.jit.codewriter.effectinfo import EffectInfo -from pypy.jit.backend.arm.regalloc import Regalloc, ARMv7RegisterManager +from pypy.jit.backend.arm.regalloc import Regalloc CPU = getcpuclass() diff --git a/pypy/jit/backend/arm/test/test_loop_unroll.py b/pypy/jit/backend/arm/test/test_loop_unroll.py --- a/pypy/jit/backend/arm/test/test_loop_unroll.py +++ b/pypy/jit/backend/arm/test/test_loop_unroll.py @@ -1,8 +1,8 @@ import py -from pypy.jit.backend.x86.test.test_basic import Jit386Mixin +from pypy.jit.backend.arm.test.support import JitARMMixin from pypy.jit.metainterp.test import test_loop_unroll -class TestLoopSpec(Jit386Mixin, test_loop_unroll.LoopUnrollTest): +class TestLoopSpec(JitARMMixin, test_loop_unroll.LoopUnrollTest): # for the individual tests see # ====> ../../../metainterp/test/test_loop.py pass diff --git a/pypy/jit/backend/arm/test/test_runner.py b/pypy/jit/backend/arm/test/test_runner.py --- a/pypy/jit/backend/arm/test/test_runner.py +++ b/pypy/jit/backend/arm/test/test_runner.py @@ -60,16 +60,18 @@ ] cpu.compile_loop(inp, operations, looptoken) args = [i for i in range(1, 15)] - self.cpu.execute_token(looptoken, *args) - output = [self.cpu.get_latest_value_int(i - 1) for i in range(1, 15)] + deadframe = self.cpu.execute_token(looptoken, *args) + output = [self.cpu.get_latest_value_int(deadframe, i - 1) for i in range(1, 15)] expected = [3, 7, 11, 15, 19, 23, 27, 3, 7, 11, 15, 19, 23, 27] assert output == expected - def test_redirect_call_assember2(self): - def assembler_helper(failindex, virtualizable): - return self.cpu.get_latest_value_int(0) + def test_redirect_call_assembler2(self): + def assembler_helper(deadframe, virtualizable): + x = self.cpu.get_latest_value_int(deadframe, 0) + assert x == 11 + return 7 - FUNCPTR = lltype.Ptr(lltype.FuncType([lltype.Signed, llmemory.GCREF], + FUNCPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF, llmemory.GCREF], lltype.Signed)) class FakeJitDriverSD: @@ -101,12 +103,12 @@ self.cpu.compile_loop(loop2.inputargs, loop2.operations, lt2) self.cpu.compile_loop(loop3.inputargs, loop3.operations, lt3) self.cpu.compile_loop(loop1.inputargs, loop1.operations, lt1) - self.cpu.execute_token(lt1, 11) - assert self.cpu.get_latest_value_int(0) == 12 + df = self.cpu.execute_token(lt1, 10) + assert self.cpu.get_latest_value_int(df, 0) == 7 self.cpu.redirect_call_assembler(lt2, lt3) - self.cpu.execute_token(lt1, 11) - assert self.cpu.get_latest_value_int(0) == 10 + df = self.cpu.execute_token(lt1, 12) + assert self.cpu.get_latest_value_int(df, 0) == 7 SFloat = lltype.GcForwardReference() SFloat.become(lltype.GcStruct('SFloat', ('parent', rclass.OBJECT), @@ -202,8 +204,8 @@ ARGS = [lltype.Signed] * numargs RES = lltype.Signed args = [i+1 for i in range(numargs)] - res = self.cpu.execute_token(looptoken, *args) - assert self.cpu.get_latest_value_int(0) == sum(args) + deadframe = self.cpu.execute_token(looptoken, *args) + assert self.cpu.get_latest_value_int(deadframe, 0) == sum(args) def test_debugger_on(self): from pypy.rlib import debug diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py deleted file mode 100644 --- a/pypy/jit/backend/llgraph/llimpl.py +++ /dev/null @@ -1,1968 +0,0 @@ -""" -The non-RPythonic part of the llgraph backend. -This contains all the code that is directly run -when executing on top of the llinterpreter. -""" - -import weakref -from pypy.objspace.flow.model import Variable, Constant -from pypy.annotation import model as annmodel -from pypy.jit.metainterp.history import REF, INT, FLOAT -from pypy.jit.metainterp import history -from pypy.jit.codewriter import heaptracker -from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr, rffi -from pypy.rpython.ootypesystem import ootype -from pypy.rpython.module.support import LLSupport, OOSupport -from pypy.rpython.llinterp import LLException -from pypy.rpython.extregistry import ExtRegistryEntry - -from pypy.jit.metainterp import resoperation -from pypy.jit.metainterp.resoperation import rop -from pypy.jit.backend.llgraph import symbolic -from pypy.jit.codewriter import longlong -from pypy.jit.codewriter.effectinfo import EffectInfo - -from pypy.rlib.objectmodel import ComputedIntSymbolic, we_are_translated -from pypy.rlib.rarithmetic import ovfcheck -from pypy.rlib.rarithmetic import r_longlong, r_ulonglong, r_uint -from pypy.rlib.rtimer import read_timestamp - -import py -from pypy.tool.ansi_print import ansi_log -log = py.log.Producer('runner') -py.log.setconsumer('runner', ansi_log) - -IS_32_BIT = r_ulonglong is not r_uint - - -def _from_opaque(opq): - return opq._obj.externalobj - -_TO_OPAQUE = {} - -def _to_opaque(value): - try: - return value._the_opaque_pointer - except AttributeError: - op = lltype.opaqueptr(_TO_OPAQUE[value.__class__], 'opaque', - externalobj=value) - value._the_opaque_pointer = op - return op - -def _normalize(value): - if isinstance(value, lltype._ptr): - value = lltype.top_container(value._obj) - return value - -def from_opaque_string(s): - if isinstance(s, str): - return s - elif isinstance(s, ootype._string): - return OOSupport.from_rstr(s) - else: - return LLSupport.from_rstr(s) - -FLOAT_ARRAY_TP = lltype.Ptr(lltype.Array(lltype.Float, hints={"nolength": True})) -def maybe_uncast(TP, array): - if array._TYPE.TO.OF != lltype.Float: - # array._TYPE.TO._hints.get("uncast_on_llgraph"): - array = rffi.cast(TP, array) - return array - -# a list of argtypes of all operations - couldn't find any and it's -# very useful. Note however that the table is half-broken here and -# there, in ways that are sometimes a bit hard to fix; that's why -# it is not "official". -TYPES = { - 'int_add' : (('int', 'int'), 'int'), - 'int_sub' : (('int', 'int'), 'int'), - 'int_mul' : (('int', 'int'), 'int'), - 'int_floordiv' : (('int', 'int'), 'int'), - 'int_mod' : (('int', 'int'), 'int'), - 'int_and' : (('int', 'int'), 'int'), - 'int_or' : (('int', 'int'), 'int'), - 'int_xor' : (('int', 'int'), 'int'), - 'int_lshift' : (('int', 'int'), 'int'), - 'int_rshift' : (('int', 'int'), 'int'), - 'int_lt' : (('int', 'int'), 'bool'), - 'int_gt' : (('int', 'int'), 'bool'), - 'int_ge' : (('int', 'int'), 'bool'), - 'int_le' : (('int', 'int'), 'bool'), - 'int_eq' : (('int', 'int'), 'bool'), - 'int_ne' : (('int', 'int'), 'bool'), - 'int_is_true' : (('int',), 'bool'), - 'int_is_zero' : (('int',), 'bool'), - 'int_neg' : (('int',), 'int'), - 'int_invert' : (('int',), 'int'), - 'int_add_ovf' : (('int', 'int'), 'int'), - 'int_sub_ovf' : (('int', 'int'), 'int'), - 'int_mul_ovf' : (('int', 'int'), 'int'), - 'int_force_ge_zero':(('int',), 'int'), - 'uint_add' : (('int', 'int'), 'int'), - 'uint_sub' : (('int', 'int'), 'int'), - 'uint_mul' : (('int', 'int'), 'int'), - 'uint_lt' : (('int', 'int'), 'bool'), - 'uint_le' : (('int', 'int'), 'bool'), - 'uint_eq' : (('int', 'int'), 'bool'), - 'uint_ne' : (('int', 'int'), 'bool'), - 'uint_gt' : (('int', 'int'), 'bool'), - 'uint_ge' : (('int', 'int'), 'bool'), - 'uint_xor' : (('int', 'int'), 'int'), - 'uint_rshift' : (('int', 'int'), 'int'), - 'uint_floordiv' : (('int', 'int'), 'int'), - 'float_add' : (('float', 'float'), 'float'), - 'float_sub' : (('float', 'float'), 'float'), - 'float_mul' : (('float', 'float'), 'float'), - 'float_truediv' : (('float', 'float'), 'float'), - 'float_lt' : (('float', 'float'), 'bool'), - 'float_le' : (('float', 'float'), 'bool'), - 'float_eq' : (('float', 'float'), 'bool'), - 'float_ne' : (('float', 'float'), 'bool'), - 'float_gt' : (('float', 'float'), 'bool'), - 'float_ge' : (('float', 'float'), 'bool'), - 'float_neg' : (('float',), 'float'), - 'float_abs' : (('float',), 'float'), - 'cast_float_to_int':(('float',), 'int'), - 'cast_int_to_float':(('int',), 'float'), - 'same_as' : (('int',), 'int'), # could also be ptr=>ptr - 'new_with_vtable' : (('ref',), 'ref'), - 'new' : ((), 'ref'), - 'new_array' : (('int',), 'ref'), - 'oois' : (('ref', 'ref'), 'bool'), - 'ooisnot' : (('ref', 'ref'), 'bool'), - 'instanceof' : (('ref',), 'bool'), - 'subclassof' : (('ref', 'ref'), 'bool'), - 'runtimenew' : (('ref',), 'ref'), - 'setfield_gc' : (('ref', 'intorptr'), None), - 'getfield_gc' : (('ref',), 'intorptr'), - 'getfield_gc_pure': (('ref',), 'intorptr'), - 'setfield_raw' : (('ref', 'intorptr'), None), - 'getfield_raw' : (('ref',), 'intorptr'), - 'getfield_raw_pure': (('ref',), 'intorptr'), - 'setarrayitem_gc' : (('ref', 'int', 'intorptr'), None), - 'getarrayitem_gc' : (('ref', 'int'), 'intorptr'), - 'getarrayitem_gc_pure' : (('ref', 'int'), 'intorptr'), - 'setarrayitem_raw' : (('ref', 'int', 'intorptr'), None), - 'getarrayitem_raw' : (('ref', 'int'), 'intorptr'), - 'getarrayitem_raw_pure' : (('ref', 'int'), 'intorptr'), - 'arraylen_gc' : (('ref',), 'int'), - 'call' : (('ref', 'varargs'), 'intorptr'), - 'call_assembler' : (('varargs',), 'intorptr'), - 'cond_call_gc_wb' : (('ptr', 'ptr'), None), - 'cond_call_gc_wb_array': (('ptr', 'int', 'ptr'), None), - 'oosend' : (('varargs',), 'intorptr'), - 'oosend_pure' : (('varargs',), 'intorptr'), - 'guard_true' : (('bool',), None), - 'guard_false' : (('bool',), None), - 'guard_value' : (('int', 'int'), None), - 'guard_class' : (('ref', 'ref'), None), - 'guard_no_exception' : ((), None), - 'guard_exception' : (('ref',), 'ref'), - 'guard_no_overflow' : ((), None), - 'guard_overflow' : ((), None), - 'guard_nonnull' : (('ref',), None), - 'guard_isnull' : (('ref',), None), - 'guard_nonnull_class' : (('ref', 'ref'), None), - 'newstr' : (('int',), 'ref'), - 'strlen' : (('ref',), 'int'), - 'strgetitem' : (('ref', 'int'), 'int'), - 'strsetitem' : (('ref', 'int', 'int'), None), - 'newunicode' : (('int',), 'ref'), - 'unicodelen' : (('ref',), 'int'), - 'unicodegetitem' : (('ref', 'int'), 'int'), - 'unicodesetitem' : (('ref', 'int', 'int'), 'int'), - 'cast_ptr_to_int' : (('ref',), 'int'), - 'cast_int_to_ptr' : (('int',), 'ref'), - 'debug_merge_point': (('ref', 'int', 'int'), None), - 'force_token' : ((), 'int'), - 'call_may_force' : (('int', 'varargs'), 'intorptr'), - 'guard_not_forced': ((), None), -} - -# ____________________________________________________________ - -class CompiledLoop(object): - has_been_freed = False - invalid = False - - def __init__(self): - self.inputargs = [] - self.operations = [] - - def getargtypes(self): - return [v.concretetype for v in self.inputargs] - - def __repr__(self): - lines = [] - self.as_text(lines, 1) - return 'CompiledLoop %s:\n%s' % (self.inputargs, '\n'.join(lines)) - - def as_text(self, lines, indent): - for op in self.operations: - lines.append('\t'*indent + repr(op)) - -class Operation(object): - result = None - descr = None - jump_target = None - fail_args = None - - def __init__(self, opnum): - self.opnum = opnum - self.args = [] - - def __repr__(self): - if self.result is not None: - sres = repr0(self.result) + ' = ' - else: - sres = '' - return '{%s%s(%s)}' % (sres, self.getopname(), - ', '.join(map(repr0, self.args))) - - def getopname(self): - try: - return resoperation.opname[self.opnum] - except KeyError: - return '<%d>' % self.opnum - - def is_guard(self): - return rop._GUARD_FIRST <= self.opnum <= rop._GUARD_LAST - - def is_final(self): - return rop._FINAL_FIRST <= self.opnum <= rop._FINAL_LAST - -def repr0(x): - if isinstance(x, list): - return '[' + ', '.join(repr0(y) for y in x) + ']' - elif isinstance(x, Constant): - return '(' + repr0(x.value) + ')' - elif isinstance(x, lltype._ptr): - x = llmemory.cast_ptr_to_adr(x) - if x.ptr: - try: - container = x.ptr._obj._normalizedcontainer() - return '* %s' % (container._TYPE._short_name(),) - except AttributeError: - return repr(x) - else: - return 'NULL' - else: - return repr(x) - -def repr_list(lst, types): - res_l = [] - if types and types[-1] == 'varargs': - types = types[:-1] + ('int',) * (len(lst) - len(types) + 1) - assert len(types) == len(lst) - for elem, tp in zip(lst, types): - if isinstance(elem, Constant): - res_l.append('(%s)' % repr1(elem, tp)) - else: - res_l.append(repr1(elem, tp)) - return '[%s]' % (', '.join(res_l)) - -def repr1(x, tp): - if tp == "intorptr": - TYPE = lltype.typeOf(x) - if isinstance(TYPE, lltype.Ptr) and TYPE.TO._gckind == 'gc': - tp = "ref" - else: - tp = "int" - if tp == 'int': - return str(x) - elif tp == 'void': - return '---' - elif tp == 'ref': - if not x: - return '(* None)' - if isinstance(x, int): - # XXX normalize? - ptr = str(llmemory.cast_int_to_adr(x)) - elif isinstance(ootype.typeOf(x), ootype.OOType): - return repr(x) - else: - if getattr(x, '_fake', None): - return repr(x) - if lltype.typeOf(x) == llmemory.GCREF: - TP = lltype.Ptr(lltype.typeOf(x._obj.container)) - ptr = lltype.cast_opaque_ptr(TP, x) - else: - ptr = x - try: - container = ptr._obj._normalizedcontainer() - return '(* %s)' % (container._TYPE._short_name(),) - except AttributeError: - return '(%r)' % (ptr,) - elif tp == 'bool': - assert x == 0 or x == 1 - return str(bool(x)) - #elif tp == 'fieldname': - # return str(symbolic.TokenToField[x...][1]) - elif tp == 'float': - return str(x) - else: - raise NotImplementedError("tp = %s" % tp) - -_variables = [] - -def compile_start(): - del _variables[:] - return _to_opaque(CompiledLoop()) - -def mark_as_free(loop): - loop = _from_opaque(loop) - assert not loop.has_been_freed - loop.has_been_freed = True - -def compile_start_int_var(loop): - return compile_start_ref_var(loop, lltype.Signed) - -def compile_start_float_var(loop): - return compile_start_ref_var(loop, longlong.FLOATSTORAGE) - -def compile_start_ref_var(loop, TYPE): - loop = _from_opaque(loop) - assert not loop.operations - v = Variable() - v.concretetype = TYPE - loop.inputargs.append(v) - r = len(_variables) - _variables.append(v) - return r - -def compile_started_vars(clt): - if not hasattr(clt, '_debug_argtypes'): # only when compiling the loop - argtypes = [v.concretetype for v in _variables] - try: - clt._debug_argtypes = argtypes - except AttributeError: # when 'clt' is actually a translated - pass # GcStruct - -def compile_add(loop, opnum): - loop = _from_opaque(loop) - loop.operations.append(Operation(opnum)) - -def compile_add_descr(loop, ofs, type, arg_types, extrainfo, width): - from pypy.jit.backend.llgraph.runner import Descr - loop = _from_opaque(loop) - op = loop.operations[-1] - assert isinstance(type, str) and len(type) == 1 - op.descr = Descr(ofs, type, arg_types=arg_types, extrainfo=extrainfo, width=width) - -def compile_add_descr_arg(loop, ofs, type, arg_types): - from pypy.jit.backend.llgraph.runner import Descr - loop = _from_opaque(loop) - op = loop.operations[-1] - assert isinstance(type, str) and len(type) == 1 - op.args.append(Descr(ofs, type, arg_types=arg_types)) - -def compile_add_loop_token(loop, descr): - if we_are_translated(): - raise ValueError("CALL_ASSEMBLER not supported") - loop = _from_opaque(loop) - op = loop.operations[-1] - op.descr = weakref.ref(descr) - -TARGET_TOKENS = weakref.WeakKeyDictionary() - -def compile_add_target_token(loop, descr, clt): - # here, 'clt' is the compiled_loop_token of the original loop that - # we are compiling - loop = _from_opaque(loop) - op = loop.operations[-1] - descrobj = _normalize(descr) - TARGET_TOKENS[descrobj] = loop, len(loop.operations), op.args, clt - -def compile_add_var(loop, intvar): - loop = _from_opaque(loop) - op = loop.operations[-1] - op.args.append(_variables[intvar]) - -def compile_add_int_const(loop, value): - compile_add_ref_const(loop, value, lltype.Signed) - -def compile_add_float_const(loop, value): - compile_add_ref_const(loop, value, longlong.FLOATSTORAGE) - -def compile_add_ref_const(loop, value, TYPE): - loop = _from_opaque(loop) - const = Constant(value) - const.concretetype = TYPE - op = loop.operations[-1] - op.args.append(const) - -def compile_add_int_result(loop): - return compile_add_ref_result(loop, lltype.Signed) - -def compile_add_float_result(loop): - return compile_add_ref_result(loop, longlong.FLOATSTORAGE) - -def compile_add_ref_result(loop, TYPE): - loop = _from_opaque(loop) - v = Variable() - v.concretetype = TYPE - op = loop.operations[-1] - op.result = v - r = len(_variables) - _variables.append(v) - return r - -def compile_add_jump_target(loop, targettoken, source_clt): - loop = _from_opaque(loop) - descrobj = _normalize(targettoken) - (loop_target, target_opindex, target_inputargs, target_clt - ) = TARGET_TOKENS[descrobj] - # - try: - assert source_clt._debug_argtypes == target_clt._debug_argtypes - except AttributeError: # when translated - pass - # - op = loop.operations[-1] - op.jump_target = loop_target - op.jump_target_opindex = target_opindex - op.jump_target_inputargs = target_inputargs - assert op.opnum == rop.JUMP - assert [v.concretetype for v in op.args] == ( - [v.concretetype for v in target_inputargs]) - # - if loop_target == loop: - log.info("compiling new loop") - else: - log.info("compiling new bridge") - -def compile_add_guard_jump_target(loop, loop_target): - loop = _from_opaque(loop) - loop_target = _from_opaque(loop_target) - op = loop.operations[-1] - assert op.is_guard() - op.jump_target = loop_target - -def compile_add_fail(loop, fail_index): - loop = _from_opaque(loop) - index = len(loop.operations)-1 - op = loop.operations[index] - op.fail_index = fail_index - return index - -def compile_add_fail_arg(loop, intvar): - loop = _from_opaque(loop) - op = loop.operations[-1] - if op.fail_args is None: - op.fail_args = [] - if intvar == -1: - op.fail_args.append(None) - else: - op.fail_args.append(_variables[intvar]) - -def compile_redirect_fail(old_loop, old_index, new_loop): - old_loop = _from_opaque(old_loop) - new_loop = _from_opaque(new_loop) - guard_op = old_loop.operations[old_index] - assert guard_op.is_guard() - guard_op.jump_target = new_loop - # check that the bridge's inputargs are of the correct number and - # kind for the guard - if guard_op.fail_args is not None: - argkinds = [v.concretetype for v in guard_op.fail_args if v] - else: - argkinds = [] - assert argkinds == [v.concretetype for v in new_loop.inputargs] - -# ------------------------------ - -class Frame(object): - OPHANDLERS = [None] * (rop._LAST+1) - - def __init__(self, cpu): - self.verbose = False - self.cpu = cpu - self.opindex = 1 - self._forced = False - self._may_force = -1 - - def getenv(self, v): - from pypy.jit.backend.llgraph.runner import Descr - if isinstance(v, Constant): - return v.value - elif isinstance(v, Descr): - return v - else: - return self.env[v] - - def _populate_fail_args(self, op, skip=None): - fail_args = [] - if op.fail_args: - for fail_arg in op.fail_args: - if fail_arg is None: - fail_args.append(None) - elif fail_arg is skip: - fail_args.append(fail_arg.concretetype._defl()) - else: - fail_args.append(self.getenv(fail_arg)) - self.fail_args = fail_args - self.fail_index = op.fail_index - - def execute(self): - """Execute all operations in a loop, - possibly following to other loops as well. - """ - global _last_exception - assert _last_exception is None, "exception left behind" - verbose = True - self.opindex = 0 - while True: - assert not self.loop.has_been_freed - op = self.loop.operations[self.opindex] - args = [self.getenv(v) for v in op.args] - if not op.is_final(): - try: - result = self.execute_operation(op.opnum, args, op.descr, - verbose) - except GuardFailed: - assert op.is_guard() - _stats.exec_conditional_jumps += 1 - if op.jump_target is not None: - # a patched guard, pointing to further code - if op.fail_args: - args = [self.getenv(v) for v in op.fail_args if v] - else: - args = [] - assert len(op.jump_target.inputargs) == len(args) - self.env = dict(zip(op.jump_target.inputargs, args)) - self.loop = op.jump_target - self.opindex = 0 - continue - else: - self._populate_fail_args(op) - # a non-patched guard - if self.verbose: - log.trace('failed: %s' % ( - ', '.join(map(str, fail_args)),)) - return op.fail_index - #verbose = self.verbose - assert (result is None) == (op.result is None) - if op.result is not None: - RESTYPE = op.result.concretetype - if RESTYPE is lltype.Signed: - x = self.as_int(result) - elif RESTYPE is llmemory.GCREF: - x = self.as_ptr(result) - elif RESTYPE is ootype.Object: - x = self.as_object(result) - elif RESTYPE is longlong.FLOATSTORAGE: - x = self.as_floatstorage(result) - else: - raise Exception("op.result.concretetype is %r" - % (RESTYPE,)) - self.env[op.result] = x - self.opindex += 1 - continue - if op.opnum == rop.JUMP: - inputargs = op.jump_target_inputargs - assert len(inputargs) == len(args) - self.env = dict(zip(inputargs, args)) - self.loop = op.jump_target - self.opindex = op.jump_target_opindex - _stats.exec_jumps += 1 - elif op.opnum == rop.FINISH: - if self.verbose: - log.trace('finished: %s' % ( - ', '.join(map(str, args)),)) - self.fail_args = args - return op.fail_index - - else: - assert 0, "unknown final operation %d" % (op.opnum,) - - def execute_operation(self, opnum, values, descr, verbose): - """Execute a single operation. - """ - ophandler = self.OPHANDLERS[opnum] - if ophandler is None: - self._define_impl(opnum) - ophandler = self.OPHANDLERS[opnum] - assert ophandler is not None, "missing impl for op %d" % opnum - opname = resoperation.opname[opnum].lower() - exec_counters = _stats.exec_counters - exec_counters[opname] = exec_counters.get(opname, 0) + 1 - for i in range(len(values)): - if isinstance(values[i], ComputedIntSymbolic): - values[i] = values[i].compute_fn() - res = NotImplemented - try: - res = ophandler(self, descr, *values) - finally: - if 0: # if verbose: - argtypes, restype = TYPES[opname] - if res is None: - resdata = '' - elif res is NotImplemented: - resdata = '*fail*' - else: - resdata = '-> ' + repr1(res, restype) - # fish the types - log.cpu('\t%s %s %s' % (opname, repr_list(values, argtypes), - resdata)) - return res - - def as_int(self, x): - return cast_to_int(x) - - def as_ptr(self, x): - return cast_to_ptr(x) - - def as_object(self, x): - return ootype.cast_to_object(x) - - def as_floatstorage(self, x): - return cast_to_floatstorage(x) - - def log_progress(self): - count = sum(_stats.exec_counters.values()) - count_jumps = _stats.exec_jumps - log.trace('ran %d operations, %d jumps' % (count, count_jumps)) - - # ---------- - - @classmethod - def _define_impl(cls, opnum): - opname = resoperation.opname[opnum] - try: - op = getattr(cls, 'op_' + opname.lower()) # op_guard_true etc. - except AttributeError: - try: - impl = globals()['do_' + opname.lower()] # do_arraylen_gc etc. - def op(self, descr, *args): - if descr is None: - return impl(*args) - else: - return impl(descr, *args) - except KeyError: - op = cls._make_impl_from_blackhole_interp(opname) - cls.OPHANDLERS[opnum] = op - - @classmethod - def _make_impl_from_blackhole_interp(cls, opname): - from pypy.jit.metainterp.blackhole import BlackholeInterpreter - name = 'bhimpl_' + opname.lower() - func = BlackholeInterpreter.__dict__[name] - for argtype in func.argtypes: - assert argtype in ('i', 'r', 'f') - # - def _op_default_implementation(self, descr, *args): - # for all operations implemented in the blackhole interpreter - return func(*args) - # - return _op_default_implementation - - def op_label(self, _, *args): - op = self.loop.operations[self.opindex] - assert op.opnum == rop.LABEL - assert len(op.args) == len(args) - newenv = {} - for v, value in zip(op.args, args): - newenv[v] = value - self.env = newenv - - def op_debug_merge_point(self, _, *args): - from pypy.jit.metainterp.warmspot import get_stats - try: - stats = get_stats() - except AttributeError: - pass - else: - stats.add_merge_point_location(args[1:]) - pass - - def op_guard_true(self, _, value): - if not value: - raise GuardFailed - - def op_guard_false(self, _, value): - if value: - raise GuardFailed - - op_guard_nonnull = op_guard_true - op_guard_isnull = op_guard_false - - def op_guard_class(self, _, value, expected_class): - value = lltype.cast_opaque_ptr(rclass.OBJECTPTR, value) - expected_class = llmemory.cast_adr_to_ptr( - llmemory.cast_int_to_adr(expected_class), - rclass.CLASSTYPE) - if value.typeptr != expected_class: - raise GuardFailed - - def op_guard_nonnull_class(self, _, value, expected_class): - if not value: - raise GuardFailed - self.op_guard_class(_, value, expected_class) - - def op_guard_value(self, _, value, expected_value): - if value != expected_value: - raise GuardFailed - - def op_guard_no_exception(self, _): - if _last_exception: - raise GuardFailed - - def _check_exception(self, expected_exception): - global _last_exception - expected_exception = self._cast_exception(expected_exception) - assert expected_exception - exc = _last_exception - if exc: - got = exc.args[0] - # exact match! - if got != expected_exception: - return False - return True - else: - return False - - def _cast_exception(self, exception): - return llmemory.cast_adr_to_ptr( - llmemory.cast_int_to_adr(exception), - rclass.CLASSTYPE) - - def _issubclass(self, cls1, cls2): - return rclass.ll_issubclass(cls1, cls2) - - def op_guard_exception(self, _, expected_exception): - global _last_exception - if not self._check_exception(expected_exception): - raise GuardFailed - res = _last_exception[1] - _last_exception = None - return res - - def op_guard_no_overflow(self, _): - flag = self.overflow_flag - del self.overflow_flag - if flag: - raise GuardFailed - - def op_guard_overflow(self, _): - flag = self.overflow_flag - del self.overflow_flag - if not flag: - raise GuardFailed - - def op_int_add_ovf(self, _, x, y): - try: - z = ovfcheck(x + y) - except OverflowError: - ovf = True - z = 0 - else: - ovf = False - self.overflow_flag = ovf - return z - - def op_int_sub_ovf(self, _, x, y): - try: - z = ovfcheck(x - y) - except OverflowError: - ovf = True - z = 0 - else: - ovf = False - self.overflow_flag = ovf - return z - - def op_int_mul_ovf(self, _, x, y): - try: - z = ovfcheck(x * y) - except OverflowError: - ovf = True - z = 0 - else: - ovf = False - self.overflow_flag = ovf - return z - - def op_keepalive(self, _, x): - pass - - # ---------- - # delegating to the builtins do_xxx() (done automatically for simple cases) - - def op_getarrayitem_gc(self, arraydescr, array, index): - if arraydescr.typeinfo == REF: - return do_getarrayitem_gc_ptr(array, index) - elif arraydescr.typeinfo == INT: - return do_getarrayitem_gc_int(array, index) - elif arraydescr.typeinfo == FLOAT: - return do_getarrayitem_gc_float(array, index) - else: - raise NotImplementedError - - op_getarrayitem_gc_pure = op_getarrayitem_gc - - def op_getarrayitem_raw(self, arraydescr, array, index): - if arraydescr.typeinfo == REF: - raise NotImplementedError("getarrayitem_raw -> gcref") - elif arraydescr.typeinfo == INT: - return do_getarrayitem_raw_int(array, index, arraydescr.ofs) - elif arraydescr.typeinfo == FLOAT: - return do_getarrayitem_raw_float(array, index) - else: - raise NotImplementedError - - op_getarrayitem_raw_pure = op_getarrayitem_raw - - def op_getfield_gc(self, fielddescr, struct): - if fielddescr.typeinfo == REF: - return do_getfield_gc_ptr(struct, fielddescr.ofs) - elif fielddescr.typeinfo == INT: - return do_getfield_gc_int(struct, fielddescr.ofs) - elif fielddescr.typeinfo == FLOAT: - return do_getfield_gc_float(struct, fielddescr.ofs) - else: - raise NotImplementedError - - op_getfield_gc_pure = op_getfield_gc - - def op_getfield_raw(self, fielddescr, struct): - if fielddescr.typeinfo == REF: - return do_getfield_raw_ptr(struct, fielddescr.ofs) - elif fielddescr.typeinfo == INT: - return do_getfield_raw_int(struct, fielddescr.ofs) - elif fielddescr.typeinfo == FLOAT: - return do_getfield_raw_float(struct, fielddescr.ofs) - else: - raise NotImplementedError - - op_getfield_raw_pure = op_getfield_raw - - def op_raw_store(self, arraydescr, addr, offset, value): - if arraydescr.typeinfo == REF: - raise AssertionError("cannot store GC pointer in raw storage") - elif arraydescr.typeinfo == INT: - do_raw_store_int(addr, offset, arraydescr.ofs, value) - elif arraydescr.typeinfo == FLOAT: - do_raw_store_float(addr, offset, value) - else: - raise NotImplementedError - - def op_raw_load(self, arraydescr, addr, offset): - if arraydescr.typeinfo == REF: - raise AssertionError("cannot store GC pointer in raw storage") - elif arraydescr.typeinfo == INT: - return do_raw_load_int(addr, offset, arraydescr.ofs) - elif arraydescr.typeinfo == FLOAT: - return do_raw_load_float(addr, offset) - else: - raise NotImplementedError - - def op_new(self, size): - return do_new(size.ofs) - - def op_new_with_vtable(self, descr, vtable): - assert descr is None - descr = heaptracker.vtable2descr(self.cpu, vtable) - result = do_new(descr.ofs) - value = lltype.cast_opaque_ptr(rclass.OBJECTPTR, result) - value.typeptr = cast_from_int(rclass.CLASSTYPE, vtable) - return result - - def op_setarrayitem_gc(self, arraydescr, array, index, newvalue): - if arraydescr.typeinfo == REF: - do_setarrayitem_gc_ptr(array, index, newvalue) - elif arraydescr.typeinfo == INT: - do_setarrayitem_gc_int(array, index, newvalue) - elif arraydescr.typeinfo == FLOAT: - do_setarrayitem_gc_float(array, index, newvalue) - else: - raise NotImplementedError - - def op_setarrayitem_raw(self, arraydescr, array, index, newvalue): - if arraydescr.typeinfo == REF: - raise NotImplementedError("setarrayitem_raw <- gcref") - elif arraydescr.typeinfo == INT: - do_setarrayitem_raw_int(array, index, newvalue, arraydescr.ofs) - elif arraydescr.typeinfo == FLOAT: - do_setarrayitem_raw_float(array, index, newvalue) - else: - raise NotImplementedError - - def op_getinteriorfield_gc(self, descr, array, index): - if descr.typeinfo == REF: - return do_getinteriorfield_gc_ptr(array, index, descr.ofs) - elif descr.typeinfo == INT: - return do_getinteriorfield_gc_int(array, index, descr.ofs) - elif descr.typeinfo == FLOAT: - return do_getinteriorfield_gc_float(array, index, descr.ofs) - else: - raise NotImplementedError - - def op_getinteriorfield_raw(self, descr, array, index): - if descr.typeinfo == REF: - return do_getinteriorfield_raw_ptr(array, index, descr.width, descr.ofs) - elif descr.typeinfo == INT: - return do_getinteriorfield_raw_int(array, index, descr.width, descr.ofs) - elif descr.typeinfo == FLOAT: - return do_getinteriorfield_raw_float(array, index, descr.width, descr.ofs) - else: - raise NotImplementedError - - def op_setinteriorfield_gc(self, descr, array, index, newvalue): - if descr.typeinfo == REF: - return do_setinteriorfield_gc_ptr(array, index, descr.ofs, - newvalue) - elif descr.typeinfo == INT: - return do_setinteriorfield_gc_int(array, index, descr.ofs, - newvalue) - elif descr.typeinfo == FLOAT: - return do_setinteriorfield_gc_float(array, index, descr.ofs, - newvalue) - else: - raise NotImplementedError - - def op_setinteriorfield_raw(self, descr, array, index, newvalue): - if descr.typeinfo == REF: - return do_setinteriorfield_raw_ptr(array, index, newvalue, descr.width, descr.ofs) - elif descr.typeinfo == INT: - return do_setinteriorfield_raw_int(array, index, newvalue, descr.width, descr.ofs) - elif descr.typeinfo == FLOAT: - return do_setinteriorfield_raw_float(array, index, newvalue, descr.width, descr.ofs) - else: - raise NotImplementedError - - def op_setfield_gc(self, fielddescr, struct, newvalue): - if fielddescr.typeinfo == REF: - do_setfield_gc_ptr(struct, fielddescr.ofs, newvalue) - elif fielddescr.typeinfo == INT: - do_setfield_gc_int(struct, fielddescr.ofs, newvalue) - elif fielddescr.typeinfo == FLOAT: - do_setfield_gc_float(struct, fielddescr.ofs, newvalue) - else: - raise NotImplementedError - - def op_setfield_raw(self, fielddescr, struct, newvalue): - if fielddescr.typeinfo == REF: - do_setfield_raw_ptr(struct, fielddescr.ofs, newvalue) - elif fielddescr.typeinfo == INT: - do_setfield_raw_int(struct, fielddescr.ofs, newvalue) - elif fielddescr.typeinfo == FLOAT: - do_setfield_raw_float(struct, fielddescr.ofs, newvalue) - else: - raise NotImplementedError - - def op_call(self, calldescr, func, *args): - effectinfo = calldescr.get_extra_info() - if effectinfo is not None and hasattr(effectinfo, 'oopspecindex'): - oopspecindex = effectinfo.oopspecindex - if oopspecindex == EffectInfo.OS_MATH_SQRT: - return do_math_sqrt(args[0]) - return self._do_call(calldescr, func, args, call_with_llptr=False) - - def op_call_release_gil(self, calldescr, func, *args): - return self._do_call(calldescr, func, args, call_with_llptr=True) - - def _do_call(self, calldescr, func, args, call_with_llptr): - global _last_exception - assert _last_exception is None, "exception left behind" - assert _call_args_i == _call_args_r == _call_args_f == [] - args_in_order = [] - for x in args: - T = lltype.typeOf(x) - if T is lltype.Signed: - args_in_order.append('i') - _call_args_i.append(x) - elif T == llmemory.GCREF: From noreply at buildbot.pypy.org Mon Jan 7 21:54:13 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Mon, 7 Jan 2013 21:54:13 +0100 (CET) Subject: [pypy-commit] pypy py3k: fix translation (for now) Message-ID: <20130107205413.9BF121C1241@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r59852:85478a2805be Date: 2013-01-07 12:34 -0800 http://bitbucket.org/pypy/pypy/changeset/85478a2805be/ Log: fix translation (for now) diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py --- a/pypy/module/imp/importing.py +++ b/pypy/module/imp/importing.py @@ -896,7 +896,7 @@ ext + PYC_TAG + '.pyc') return result - at signature(types.str0(), returns=types.str0()) +#@signature(types.str0(), returns=types.str0()) def make_source_pathname(pathname): "Given the path to a .pyc file, return the path to its .py file." # (...)/__pycache__/foo..pyc -> (...)/foo.py From noreply at buildbot.pypy.org Mon Jan 7 21:54:15 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Mon, 7 Jan 2013 21:54:15 +0100 (CET) Subject: [pypy-commit] pypy py3k: kill the lonepycfiles option entirely, it's not so useful now that there's Message-ID: <20130107205415.1BC351C1241@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r59853:eeddf44bbf34 Date: 2013-01-07 12:36 -0800 http://bitbucket.org/pypy/pypy/changeset/eeddf44bbf34/ Log: kill the lonepycfiles option entirely, it's not so useful now that there's __pycache__ dirs diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -171,10 +171,6 @@ BoolOption("usepycfiles", "Write and read pyc files when importing", default=True), - BoolOption("lonepycfiles", "Import pyc files with no matching py file", - default=False, - requires=[("objspace.usepycfiles", True)]), - StrOption("soabi", "Tag to differentiate extension modules built for different Python interpreters", cmdline="--soabi", diff --git a/pypy/doc/config/objspace.lonepycfiles.txt b/pypy/doc/config/objspace.lonepycfiles.txt deleted file mode 100644 --- a/pypy/doc/config/objspace.lonepycfiles.txt +++ /dev/null @@ -1,16 +0,0 @@ -If turned on, PyPy accepts to import a module ``x`` if it finds a -file ``x.pyc`` even if there is no file ``x.py``. - -This is the way that CPython behaves, but it is disabled by -default for PyPy because it is a common cause of issues: most -typically, the ``x.py`` file is removed (manually or by a -version control system) but the ``x`` module remains -accidentally importable because the ``x.pyc`` file stays -around. - -The usual reason for wanting this feature is to distribute -non-open-source Python programs by distributing ``pyc`` files -only, but this use case is not practical for PyPy at the -moment because multiple versions of PyPy compiled with various -optimizations might be unable to load each other's ``pyc`` -files. diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py --- a/pypy/module/imp/importing.py +++ b/pypy/module/imp/importing.py @@ -73,13 +73,8 @@ if file_exists(pyfile): return PY_SOURCE, ".pyw", "U" - # The .py file does not exist. By default on PyPy, lonepycfiles - # is False: if a .py file does not exist, we don't even try to - # look for a lone .pyc file. - # The "imp" module does not respect this, and is allowed to find - # lone .pyc files. - # check the .pyc file - if space.config.objspace.usepycfiles and space.config.objspace.lonepycfiles: + # The .py file does not exist, check the .pyc file + if space.config.objspace.usepycfiles: pycfile = filepart + ".pyc" if file_exists(pycfile): # existing .pyc file diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py --- a/pypy/module/imp/test/test_import.py +++ b/pypy/module/imp/test/test_import.py @@ -1283,13 +1283,10 @@ class AppTestNoPycFile(object): spaceconfig = { "objspace.usepycfiles": False, - "objspace.lonepycfiles": False } def setup_class(cls): usepycfiles = cls.spaceconfig['objspace.usepycfiles'] - lonepycfiles = cls.spaceconfig['objspace.lonepycfiles'] cls.w_usepycfiles = cls.space.wrap(usepycfiles) - cls.w_lonepycfiles = cls.space.wrap(lonepycfiles) cls.saved_modules = _setup(cls.space) def teardown_class(cls): @@ -1301,21 +1298,13 @@ try: from compiled import lone except ImportError: - assert not self.lonepycfiles, "should have found 'lone.pyc'" + assert not self.usepycfiles else: - assert self.lonepycfiles, "should not have found 'lone.pyc'" assert lone.__cached__.endswith('.pyc') class AppTestNoLonePycFile(AppTestNoPycFile): spaceconfig = { "objspace.usepycfiles": True, - "objspace.lonepycfiles": False - } - -class AppTestLonePycFile(AppTestNoPycFile): - spaceconfig = { - "objspace.usepycfiles": True, - "objspace.lonepycfiles": True } diff --git a/pypy/translator/goal/targetpypystandalone.py b/pypy/translator/goal/targetpypystandalone.py --- a/pypy/translator/goal/targetpypystandalone.py +++ b/pypy/translator/goal/targetpypystandalone.py @@ -194,7 +194,6 @@ # config.translation.backend == "cli" if config.translation.sandbox: - config.objspace.lonepycfiles = False config.objspace.usepycfiles = False config.translating = True From noreply at buildbot.pypy.org Mon Jan 7 21:54:16 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Mon, 7 Jan 2013 21:54:16 +0100 (CET) Subject: [pypy-commit] pypy py3k: revert .pyc skips now that lonepycfiles was killed Message-ID: <20130107205416.435A91C1241@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r59854:385839b6a744 Date: 2013-01-07 12:37 -0800 http://bitbucket.org/pypy/pypy/changeset/385839b6a744/ Log: revert .pyc skips now that lonepycfiles was killed diff --git a/lib-python/3.2/test/test_import.py b/lib-python/3.2/test/test_import.py --- a/lib-python/3.2/test/test_import.py +++ b/lib-python/3.2/test/test_import.py @@ -14,9 +14,9 @@ import errno from test.support import ( - EnvironmentVarGuard, TESTFN, check_impl_detail, check_warnings, forget, - impl_detail, is_jython, make_legacy_pyc, rmtree, run_unittest, swap_attr, - swap_item, temp_umask, unlink, unload) + EnvironmentVarGuard, TESTFN, check_warnings, forget, is_jython, + make_legacy_pyc, rmtree, run_unittest, swap_attr, swap_item, temp_umask, + unlink, unload) from test import script_helper @@ -166,9 +166,7 @@ # Bytecode must be relocated from the PEP 3147 bytecode-only location. py_compile.compile(filename) finally: - if check_impl_detail(pypy=False): - # pypy refuses to import a .pyc if the .py does not exist - unlink(filename) + unlink(filename) # Need to be able to load from current dir. sys.path.append('') @@ -251,7 +249,6 @@ remove_files(TESTFN) unload(TESTFN) - @impl_detail("pypy refuses to import without a .py source", pypy=False) def test_file_to_source(self): # check if __file__ points to the source file where available source = TESTFN + ".py" @@ -399,7 +396,6 @@ self.assertEqual(mod.code_filename, self.file_name) self.assertEqual(mod.func_filename, self.file_name) - @impl_detail("pypy refuses to import without a .py source", pypy=False) def test_module_without_source(self): target = "another_module.py" py_compile.compile(self.file_name, dfile=target) @@ -597,7 +593,6 @@ forget(TESTFN) self.assertRaises(ImportError, __import__, TESTFN) - @impl_detail("pypy refuses to import without a .py source", pypy=False) def test_missing_source_legacy(self): # Like test_missing_source() except that for backward compatibility, # when the pyc file lives where the py file would have been (and named @@ -618,7 +613,6 @@ pyc_file = imp.cache_from_source(TESTFN + '.py') self.assertEqual(m.__cached__, os.path.join(os.curdir, pyc_file)) - @impl_detail("pypy refuses to import without a .py source", pypy=False) def test___cached___legacy_pyc(self): # Like test___cached__() except that for backward compatibility, # when the pyc file lives where the py file would have been (and named From noreply at buildbot.pypy.org Mon Jan 7 21:54:17 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Mon, 7 Jan 2013 21:54:17 +0100 (CET) Subject: [pypy-commit] pypy py3k: fix find_module's encoding check to look at line 2 and be more forgiving Message-ID: <20130107205417.786961C1241@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r59855:0430488c1af7 Date: 2013-01-07 12:52 -0800 http://bitbucket.org/pypy/pypy/changeset/0430488c1af7/ Log: fix find_module's encoding check to look at line 2 and be more forgiving diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py --- a/pypy/module/imp/interp_imp.py +++ b/pypy/module/imp/interp_imp.py @@ -4,7 +4,7 @@ from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.module import Module from pypy.interpreter.gateway import unwrap_spec -from pypy.interpreter.pyparser import pytokenizer +from pypy.interpreter.pyparser import pyparse from pypy.objspace.std import unicodetype from pypy.rlib import streamio from pypy.module._io.interp_iobase import W_IOBase @@ -76,10 +76,10 @@ encoding = None if find_info.modtype == importing.PY_SOURCE: # try to find the declared encoding - firstline = stream.readline() + top = stream.readline() + top += stream.readline() stream.seek(0, 0) # reset position - if firstline.startswith('#'): - encoding = pytokenizer.match_encoding_declaration(firstline) + encoding = pyparse._check_for_encoding(top) if encoding is None: encoding = unicodetype.getdefaultencoding(space) # diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py --- a/pypy/module/imp/test/test_import.py +++ b/pypy/module/imp/test/test_import.py @@ -90,6 +90,9 @@ 'a=5\nb=6\rc="""hello\r\nworld"""\r', mode='wb') p.join('mod.py').write( 'a=15\nb=16\rc="""foo\r\nbar"""\r', mode='wb') + setuppkg("encoded", + # actually a line 2, setuppkg() sets up a line1 + line2 = "# encoding: iso-8859-1\n") # create compiled/x.py and a corresponding pyc file p = setuppkg("compiled", x = "x = 84") @@ -659,6 +662,12 @@ assert module.__name__ == 'a' assert module.__file__ == 'invalid_path_name' + def test_source_encoding(self): + import imp + import encoded + fd = imp.find_module('line2', encoded.__path__)[0] + assert fd.encoding == 'iso-8859-1' + class TestAbi: def test_abi_tag(self): From noreply at buildbot.pypy.org Mon Jan 7 23:10:11 2013 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 7 Jan 2013 23:10:11 +0100 (CET) Subject: [pypy-commit] pypy default: Add a passing test. Message-ID: <20130107221011.AABE81C1241@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r59856:2dc74be985d3 Date: 2013-01-07 21:18 +0100 http://bitbucket.org/pypy/pypy/changeset/2dc74be985d3/ Log: Add a passing test. diff --git a/pypy/module/_weakref/test/test_weakref.py b/pypy/module/_weakref/test/test_weakref.py --- a/pypy/module/_weakref/test/test_weakref.py +++ b/pypy/module/_weakref/test/test_weakref.py @@ -114,19 +114,28 @@ class A(object): def __eq__(self, other): return True + def __ne__(self, other): + return False a1 = A() a2 = A() ref1 = _weakref.ref(a1) ref2 = _weakref.ref(a2) assert ref1 == ref2 + assert not (ref1 != ref2) + assert not (ref1 == []) + assert ref1 != [] del a1 gc.collect() assert not ref1 == ref2 assert ref1 != ref2 + assert not (ref1 == []) + assert ref1 != [] del a2 gc.collect() assert not ref1 == ref2 assert ref1 != ref2 + assert not (ref1 == []) + assert ref1 != [] def test_getweakrefs(self): import _weakref, gc @@ -435,6 +444,8 @@ class A(object): def __eq__(self, other): return True + def __ne__(self, other): + return False a = A() assert _weakref.ref(a) == a From noreply at buildbot.pypy.org Mon Jan 7 23:10:13 2013 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 7 Jan 2013 23:10:13 +0100 (CET) Subject: [pypy-commit] pypy default: This test fails when run with "pypy -A". Still trying to figure out Message-ID: <20130107221013.150871C1241@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r59857:efb9453c7e81 Date: 2013-01-07 21:19 +0100 http://bitbucket.org/pypy/pypy/changeset/efb9453c7e81/ Log: This test fails when run with "pypy -A". Still trying to figure out what's going on. diff --git a/pypy/module/_weakref/test/test_weakref.py b/pypy/module/_weakref/test/test_weakref.py --- a/pypy/module/_weakref/test/test_weakref.py +++ b/pypy/module/_weakref/test/test_weakref.py @@ -307,6 +307,13 @@ if seen_callback: assert seen_callback == [True, True, True] + def test_type_weakrefable(self): + import _weakref, gc + w = _weakref.ref(list) + assert w() is list + gc.collect() + assert w() is list + class AppTestProxy(object): spaceconfig = dict(usemodules=('_weakref',)) From noreply at buildbot.pypy.org Mon Jan 7 23:10:14 2013 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 7 Jan 2013 23:10:14 +0100 (CET) Subject: [pypy-commit] pypy default: Test and fix for issue #1353: weakref to prebuilt object. Message-ID: <20130107221014.4EF751C1241@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r59858:50c910edeafa Date: 2013-01-07 23:07 +0100 http://bitbucket.org/pypy/pypy/changeset/50c910edeafa/ Log: Test and fix for issue #1353: weakref to prebuilt object. diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py --- a/pypy/rpython/memory/gc/minimark.py +++ b/pypy/rpython/memory/gc/minimark.py @@ -2000,6 +2000,17 @@ (obj + offset).address[0] = llmemory.NULL continue # no need to remember this weakref any longer # + elif self.header(pointing_to).tid & GCFLAG_NO_HEAP_PTRS: + # see test_weakref_to_prebuilt: it's not useful to put + # weakrefs into 'old_objects_with_weakrefs' if they point + # to a prebuilt object (they are immortal). If moreover + # the 'pointing_to' prebuilt object still has the + # GCFLAG_NO_HEAP_PTRS flag, then it's even wrong, because + # 'pointing_to' will not get the GCFLAG_VISITED during + # the next major collection. Solve this by not registering + # the weakref into 'old_objects_with_weakrefs'. + continue + # self.old_objects_with_weakrefs.append(obj) def invalidate_old_weakrefs(self): @@ -2013,6 +2024,9 @@ continue # weakref itself dies offset = self.weakpointer_offset(self.get_type_id(obj)) pointing_to = (obj + offset).address[0] + ll_assert((self.header(pointing_to).tid & GCFLAG_NO_HEAP_PTRS) + == 0, "registered old weakref should not " + "point to a NO_HEAP_PTRS obj") if self.header(pointing_to).tid & GCFLAG_VISITED: new_with_weakref.append(obj) else: diff --git a/pypy/rpython/memory/test/test_gc.py b/pypy/rpython/memory/test/test_gc.py --- a/pypy/rpython/memory/test/test_gc.py +++ b/pypy/rpython/memory/test/test_gc.py @@ -557,6 +557,19 @@ res = self.interpret(f, [20]) # for GenerationGC, enough for a minor collection assert res == True + def test_weakref_to_prebuilt(self): + import weakref + class A: + pass + a = A() + def f(x): + ref = weakref.ref(a) + assert ref() is a + llop.gc__collect(lltype.Void) + return ref() is a + res = self.interpret(f, [20]) # for GenerationGC, enough for a minor collection + assert res == True + def test_many_weakrefs(self): # test for the case where allocating the weakref itself triggers # a collection From noreply at buildbot.pypy.org Mon Jan 7 23:16:38 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Mon, 7 Jan 2013 23:16:38 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: I think its better to keep the original name. pypysandbox -> pypy_interact Message-ID: <20130107221638.2D3E31C1241@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59859:57edbe9fd86c Date: 2013-01-06 22:06 +0100 http://bitbucket.org/pypy/pypy/changeset/57edbe9fd86c/ Log: I think its better to keep the original name. pypysandbox -> pypy_interact diff --git a/pypy/bin/pypysandbox b/pypy/bin/pypy_interact rename from pypy/bin/pypysandbox rename to pypy/bin/pypy_interact --- a/pypy/bin/pypysandbox +++ b/pypy/bin/pypy_interact @@ -1,5 +1,5 @@ #!/usr/bin/env pypy -from pypy.sandbox import pypysandbox +from pypy.sandbox import pypy_interact -pypysandbox.main() \ No newline at end of file +pypy_interact.main() \ No newline at end of file diff --git a/pypy/sandbox/pypysandbox.py b/pypy/sandbox/pypy_interact.py rename from pypy/sandbox/pypysandbox.py rename to pypy/sandbox/pypy_interact.py diff --git a/pypy/sandbox/test/test_pypysandbox.py b/pypy/sandbox/test/test_pypy_interact.py rename from pypy/sandbox/test/test_pypysandbox.py rename to pypy/sandbox/test/test_pypy_interact.py --- a/pypy/sandbox/test/test_pypysandbox.py +++ b/pypy/sandbox/test/test_pypy_interact.py @@ -1,5 +1,5 @@ import os, sys, stat, errno -from pypy.sandbox.pypysandbox import PyPySandboxedProc +from pypy.sandbox.pypy_interact import PyPySandboxedProc from rpython.translator.interactive import Translation from pypy.module.sys.version import CPYTHON_VERSION From noreply at buildbot.pypy.org Mon Jan 7 23:16:39 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Mon, 7 Jan 2013 23:16:39 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Moved pypy.tool.lsprofcalltree to rpython Message-ID: <20130107221639.630351C1241@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59860:6722baba8a5d Date: 2013-01-07 02:53 +0100 http://bitbucket.org/pypy/pypy/changeset/6722baba8a5d/ Log: Moved pypy.tool.lsprofcalltree to rpython diff --git a/pypy/tool/lsprofcalltree.py b/rpython/tool/lsprofcalltree.py rename from pypy/tool/lsprofcalltree.py rename to rpython/tool/lsprofcalltree.py diff --git a/rpython/translator/driver.py b/rpython/translator/driver.py --- a/rpython/translator/driver.py +++ b/rpython/translator/driver.py @@ -258,7 +258,7 @@ def _profile(self, goal, func): from cProfile import Profile - from pypy.tool.lsprofcalltree import KCacheGrind + from rpython.tool.lsprofcalltree import KCacheGrind d = {'func':func} prof = Profile() prof.runctx("res = func()", globals(), d) From noreply at buildbot.pypy.org Mon Jan 7 23:16:40 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Mon, 7 Jan 2013 23:16:40 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Assume a working sys.path Message-ID: <20130107221640.8EAFB1C1241@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59861:3b0c754dbdac Date: 2013-01-07 03:29 +0100 http://bitbucket.org/pypy/pypy/changeset/3b0c754dbdac/ Log: Assume a working sys.path diff --git a/pypy/bin/checkmodule.py b/pypy/bin/checkmodule.py --- a/pypy/bin/checkmodule.py +++ b/pypy/bin/checkmodule.py @@ -7,7 +7,6 @@ modules compiles without doing a full translation. """ import sys, os -sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..'))) from pypy.objspace.fake.checkmodule import checkmodule diff --git a/pypy/bin/pyinteractive.py b/pypy/bin/pyinteractive.py --- a/pypy/bin/pyinteractive.py +++ b/pypy/bin/pyinteractive.py @@ -9,9 +9,6 @@ import os, sys import time -if os.path.exists(os.path.join(os.path.dirname(__file__), '..', '..', 'pypy', '__init__.py')): - sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..'))) - import pypy from pypy.tool import option from optparse import make_option diff --git a/pypy/bin/reportstaticdata.py b/pypy/bin/reportstaticdata.py --- a/pypy/bin/reportstaticdata.py +++ b/pypy/bin/reportstaticdata.py @@ -28,7 +28,6 @@ """ import sys -sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..'))) from rpython.translator.tool.staticsizereport import print_report diff --git a/pypy/bin/translatorshell.py b/pypy/bin/translatorshell.py --- a/pypy/bin/translatorshell.py +++ b/pypy/bin/translatorshell.py @@ -24,7 +24,6 @@ """ import os, sys -sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..'))) from rpython.translator.interactive import Translation from rpython.rtyper.rtyper import * From noreply at buildbot.pypy.org Mon Jan 7 23:16:41 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Mon, 7 Jan 2013 23:16:41 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Removed unused imports Message-ID: <20130107221641.D1E941C1241@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59862:b43d7331e165 Date: 2013-01-07 15:10 +0100 http://bitbucket.org/pypy/pypy/changeset/b43d7331e165/ Log: Removed unused imports diff --git a/rpython/rtyper/lltypesystem/test/test_ll2ctypes.py b/rpython/rtyper/lltypesystem/test/test_ll2ctypes.py --- a/rpython/rtyper/lltypesystem/test/test_ll2ctypes.py +++ b/rpython/rtyper/lltypesystem/test/test_ll2ctypes.py @@ -1017,7 +1017,6 @@ assert ftest == [-5, -5, -5] def test_c_callback_with_void_arg_3(self): - import pypy def f(i): x = 'X' * i return x[-2] diff --git a/rpython/rtyper/module/test/test_ll_os.py b/rpython/rtyper/module/test/test_ll_os.py --- a/rpython/rtyper/module/test/test_ll_os.py +++ b/rpython/rtyper/module/test/test_ll_os.py @@ -1,7 +1,6 @@ import os from py.path import local -import pypy from rpython.tool.udir import udir from rpython.translator.c.test.test_genc import compile from rpython.rtyper.module import ll_os #has side effect of registering functions From noreply at buildbot.pypy.org Mon Jan 7 23:16:43 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Mon, 7 Jan 2013 23:16:43 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Moved spli to toplevel Message-ID: <20130107221643.1E0CA1C1241@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59863:a0b190c85333 Date: 2013-01-07 16:21 +0100 http://bitbucket.org/pypy/pypy/changeset/a0b190c85333/ Log: Moved spli to toplevel diff --git a/rpython/translator/tool/benchmark.py b/rpython/translator/tool/benchmark.py deleted file mode 100644 --- a/rpython/translator/tool/benchmark.py +++ /dev/null @@ -1,33 +0,0 @@ -from pypy.tool import testit -from rpython.tool.udir import udir -from rpython.translator.tool.cbuild import build_cfunc -from rpython.translator.test.test_cltrans import global_cl, make_cl_func - -def benchmark(func): - try: - func = func.im_func - except AttributeError: - pass - c_func = build_cfunc(func, dot=False) - if global_cl: - cl_func = make_cl_func(func) - print "generated c-func for", func.func_name - t1 = timeit(100, func) - t2 = timeit(100, c_func) - if global_cl: - t3 = timeit(100, cl_func) - print "cpython func ", t1, "seconds" - print "pypy/pyrex/cmodule ", t2, "seconds" - if global_cl: - print "cl (experimental) ", t3, "seconds", global_cl - -def timeit(num, func, *args): - from time import time as now - start = now() - for i in xrange(num): - func(*args) - return now()-start - -if __name__ == '__main__': - from rpython.translator.test.snippet import sieve_of_eratosthenes - benchmark(sieve_of_eratosthenes) diff --git a/rpython/jit/tl/spli/__init__.py b/spli/__init__.py rename from rpython/jit/tl/spli/__init__.py rename to spli/__init__.py diff --git a/rpython/jit/tl/spli/examples.py b/spli/examples.py rename from rpython/jit/tl/spli/examples.py rename to spli/examples.py diff --git a/rpython/jit/tl/spli/execution.py b/spli/execution.py rename from rpython/jit/tl/spli/execution.py rename to spli/execution.py --- a/rpython/jit/tl/spli/execution.py +++ b/spli/execution.py @@ -1,4 +1,4 @@ -from rpython.jit.tl.spli import interpreter, objects, pycode +from spli import interpreter, objects, pycode def run_from_cpython_code(co, args=[], locs=None, globs=None): diff --git a/rpython/jit/tl/spli/interpreter.py b/spli/interpreter.py rename from rpython/jit/tl/spli/interpreter.py rename to spli/interpreter.py --- a/rpython/jit/tl/spli/interpreter.py +++ b/spli/interpreter.py @@ -1,6 +1,6 @@ import os from rpython.tool import stdlib_opcode -from rpython.jit.tl.spli import objects, pycode +from spli import objects, pycode from rpython.rlib.unroll import unrolling_iterable from rpython.rlib.jit import JitDriver, promote, dont_look_inside from rpython.rlib.objectmodel import we_are_translated diff --git a/rpython/jit/tl/spli/objects.py b/spli/objects.py rename from rpython/jit/tl/spli/objects.py rename to spli/objects.py --- a/rpython/jit/tl/spli/objects.py +++ b/spli/objects.py @@ -154,5 +154,5 @@ self.globs = globs def call(self, args): - from rpython.jit.tl.spli import execution + from spli import execution return execution.run(self.code, args, None, self.globs) diff --git a/rpython/jit/tl/spli/pycode.py b/spli/pycode.py rename from rpython/jit/tl/spli/pycode.py rename to spli/pycode.py --- a/rpython/jit/tl/spli/pycode.py +++ b/spli/pycode.py @@ -1,5 +1,5 @@ from pypy.interpreter import pycode -from rpython.jit.tl.spli import objects +from spli import objects class Code(objects.SPLIObject): diff --git a/rpython/jit/tl/spli/serializer.py b/spli/serializer.py rename from rpython/jit/tl/spli/serializer.py rename to spli/serializer.py --- a/rpython/jit/tl/spli/serializer.py +++ b/spli/serializer.py @@ -6,8 +6,8 @@ import py import sys import types -from rpython.jit.tl.spli.objects import Int, Str, spli_None -from rpython.jit.tl.spli.pycode import Code +from spli.objects import Int, Str, spli_None +from spli.pycode import Code from rpython.rlib.rstruct.runpack import runpack import struct diff --git a/rpython/jit/tl/spli/targetspli.py b/spli/targetspli.py rename from rpython/jit/tl/spli/targetspli.py rename to spli/targetspli.py --- a/rpython/jit/tl/spli/targetspli.py +++ b/spli/targetspli.py @@ -3,7 +3,7 @@ """ import sys, os -from rpython.jit.tl.spli import execution, serializer, objects +from spli import execution, serializer, objects from rpython.rlib.streamio import open_file_as_stream diff --git a/rpython/jit/tl/spli/test/__init__.py b/spli/test/__init__.py rename from rpython/jit/tl/spli/test/__init__.py rename to spli/test/__init__.py diff --git a/rpython/jit/tl/spli/test/test_interpreter.py b/spli/test/test_interpreter.py rename from rpython/jit/tl/spli/test/test_interpreter.py rename to spli/test/test_interpreter.py --- a/rpython/jit/tl/spli/test/test_interpreter.py +++ b/spli/test/test_interpreter.py @@ -1,6 +1,6 @@ import py import os -from rpython.jit.tl.spli import execution, objects +from spli import execution, objects class TestSPLIInterpreter: diff --git a/rpython/jit/tl/spli/test/test_jit.py b/spli/test/test_jit.py rename from rpython/jit/tl/spli/test/test_jit.py rename to spli/test/test_jit.py --- a/rpython/jit/tl/spli/test/test_jit.py +++ b/spli/test/test_jit.py @@ -1,7 +1,7 @@ import py from rpython.jit.metainterp.test.support import JitMixin -from rpython.jit.tl.spli import interpreter, objects, serializer +from spli import interpreter, objects, serializer from rpython.jit.metainterp.typesystem import LLTypeHelper, OOTypeHelper from rpython.jit.backend.llgraph import runner from rpython.rtyper.annlowlevel import llstr, hlstr diff --git a/rpython/jit/tl/spli/test/test_serializer.py b/spli/test/test_serializer.py rename from rpython/jit/tl/spli/test/test_serializer.py rename to spli/test/test_serializer.py --- a/rpython/jit/tl/spli/test/test_serializer.py +++ b/spli/test/test_serializer.py @@ -1,5 +1,5 @@ -from rpython.jit.tl.spli.serializer import serialize, deserialize -from rpython.jit.tl.spli import execution, pycode, objects +from spli.serializer import serialize, deserialize +from spli import execution, pycode, objects class TestSerializer(object): diff --git a/rpython/jit/tl/spli/test/test_translated.py b/spli/test/test_translated.py rename from rpython/jit/tl/spli/test/test_translated.py rename to spli/test/test_translated.py --- a/rpython/jit/tl/spli/test/test_translated.py +++ b/spli/test/test_translated.py @@ -1,7 +1,7 @@ from rpython.rtyper.test.test_llinterp import interpret -from rpython.jit.tl.spli import execution, objects -from rpython.jit.tl.spli.serializer import serialize, deserialize +from spli import execution, objects +from spli.serializer import serialize, deserialize class TestSPLITranslated(object): From noreply at buildbot.pypy.org Mon Jan 7 23:40:28 2013 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 7 Jan 2013 23:40:28 +0100 (CET) Subject: [pypy-commit] pypy default: add Makefile test of many files, try to fix windows implementation Message-ID: <20130107224028.C91C11C1233@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r59864:492d8777d15f Date: 2013-01-07 23:26 +0200 http://bitbucket.org/pypy/pypy/changeset/492d8777d15f/ Log: add Makefile test of many files, try to fix windows implementation diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -38,7 +38,7 @@ # General information about the project. project = u'PyPy' -copyright = u'2011, The PyPy Project' +copyright = u'2013, The PyPy Project' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/pypy/translator/platform/test/test_platform.py b/pypy/translator/platform/test/test_platform.py --- a/pypy/translator/platform/test/test_platform.py +++ b/pypy/translator/platform/test/test_platform.py @@ -59,6 +59,34 @@ res = self.platform.execute(executable) self.check_res(res) + def test_900_files(self): + txt = '#include \n' + for i in range(900): + txt += 'int func%03d();\n' % i + txt += 'int main() {\n int j=0;' + for i in range(900): + txt += ' j += func%03d();\n' % i + txt += ' printf("%d\\n", j);\n' + txt += ' return j;};\n' + cfile = udir.join('test_900_files.c') + cfile.write(txt) + cfiles = [cfile] + for i in range(900): + cfile2 = udir.join('implement%03d.c' %i) + cfile2.write(''' + int func%03d() + { + return %d; + } + ''' % (i, i)) + cfiles.append(cfile2) + mk = self.platform.gen_makefile(cfiles, ExternalCompilationInfo(), path=udir) + mk.write() + self.platform.execute_makefile(mk) + res = self.platform.execute(udir.join('test_900_files')) + self.check_res(res, sum(range(900))) + + def test_nice_errors(self): cfile = udir.join('test_nice_errors.c') cfile.write('') diff --git a/pypy/translator/platform/windows.py b/pypy/translator/platform/windows.py --- a/pypy/translator/platform/windows.py +++ b/pypy/translator/platform/windows.py @@ -326,17 +326,26 @@ for rule in rules: m.rule(*rule) - + + objects = ' $(OBJECTS)' + create_obj_response_file = '' + if len(' '.join(rel_ofiles)) > 4000: + create_obj_response_file = 'echo $(OBJECTS) > obj_names.rsp' + objects = ' @obj_names.rsp' if self.version < 80: m.rule('$(TARGET)', '$(OBJECTS)', - '$(CC_LINK) /nologo $(LDFLAGS) $(LDFLAGSEXTRA) $(OBJECTS) /out:$@ $(LIBDIRS) $(LIBS)') + [create_obj_response_file, + '$(CC_LINK) /nologo $(LDFLAGS) $(LDFLAGSEXTRA)' + objects + ' /out:$@ $(LIBDIRS) $(LIBS)', + ]) else: m.rule('$(TARGET)', '$(OBJECTS)', - ['$(CC_LINK) /nologo $(LDFLAGS) $(LDFLAGSEXTRA) $(OBJECTS) $(LINKFILES) /out:$@ $(LIBDIRS) $(LIBS) /MANIFEST /MANIFESTFILE:$*.manifest', + [create_obj_response_file, + '$(CC_LINK) /nologo $(LDFLAGS) $(LDFLAGSEXTRA)' + objects + ' $(LINKFILES) /out:$@ $(LIBDIRS) $(LIBS) /MANIFEST /MANIFESTFILE:$*.manifest', 'mt.exe -nologo -manifest $*.manifest -outputresource:$@;1', ]) m.rule('debugmode_$(TARGET)', '$(OBJECTS)', - ['$(CC_LINK) /nologo /DEBUG $(LDFLAGS) $(LDFLAGSEXTRA) $(OBJECTS) $(LINKFILES) /out:$@ $(LIBDIRS) $(LIBS)', + [create_obj_response_file, + '$(CC_LINK) /nologo /DEBUG $(LDFLAGS) $(LDFLAGSEXTRA)' + objects + ' $(LINKFILES) /out:$@ $(LIBDIRS) $(LIBS)', ]) if shared: From noreply at buildbot.pypy.org Mon Jan 7 23:40:30 2013 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 7 Jan 2013 23:40:30 +0100 (CET) Subject: [pypy-commit] pypy default: fix till test passes Message-ID: <20130107224030.0033B1C1233@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r59865:65392c9700e4 Date: 2013-01-08 00:10 +0200 http://bitbucket.org/pypy/pypy/changeset/65392c9700e4/ Log: fix till test passes diff --git a/pypy/translator/platform/test/test_platform.py b/pypy/translator/platform/test/test_platform.py --- a/pypy/translator/platform/test/test_platform.py +++ b/pypy/translator/platform/test/test_platform.py @@ -67,7 +67,7 @@ for i in range(900): txt += ' j += func%03d();\n' % i txt += ' printf("%d\\n", j);\n' - txt += ' return j;};\n' + txt += ' return 0;};\n' cfile = udir.join('test_900_files.c') cfile.write(txt) cfiles = [cfile] @@ -84,7 +84,7 @@ mk.write() self.platform.execute_makefile(mk) res = self.platform.execute(udir.join('test_900_files')) - self.check_res(res, sum(range(900))) + self.check_res(res, '%d\n' %sum(range(900))) def test_nice_errors(self): diff --git a/pypy/translator/platform/windows.py b/pypy/translator/platform/windows.py --- a/pypy/translator/platform/windows.py +++ b/pypy/translator/platform/windows.py @@ -328,23 +328,30 @@ m.rule(*rule) objects = ' $(OBJECTS)' - create_obj_response_file = '' + create_obj_response_file = [] if len(' '.join(rel_ofiles)) > 4000: - create_obj_response_file = 'echo $(OBJECTS) > obj_names.rsp' + # cmd.exe has a limit of ~4000 characters before a command line is too long. + # Use a response file instead, at the cost of making the Makefile very ugly. + for i in range(len(rel_ofiles) - 1): + create_obj_response_file.append('echo %s >> obj_names.rsp' % \ + rel_ofiles[i]) + # use cmd /c for the last one so that the file is flushed + create_obj_response_file.append('cmd /c echo %s >> obj_names.rsp' % \ + rel_ofiles[-1]) objects = ' @obj_names.rsp' if self.version < 80: m.rule('$(TARGET)', '$(OBJECTS)', - [create_obj_response_file, + create_obj_response_file + [\ '$(CC_LINK) /nologo $(LDFLAGS) $(LDFLAGSEXTRA)' + objects + ' /out:$@ $(LIBDIRS) $(LIBS)', ]) else: m.rule('$(TARGET)', '$(OBJECTS)', - [create_obj_response_file, + create_obj_response_file + [\ '$(CC_LINK) /nologo $(LDFLAGS) $(LDFLAGSEXTRA)' + objects + ' $(LINKFILES) /out:$@ $(LIBDIRS) $(LIBS) /MANIFEST /MANIFESTFILE:$*.manifest', 'mt.exe -nologo -manifest $*.manifest -outputresource:$@;1', ]) m.rule('debugmode_$(TARGET)', '$(OBJECTS)', - [create_obj_response_file, + create_obj_response_file + [\ '$(CC_LINK) /nologo /DEBUG $(LDFLAGS) $(LDFLAGSEXTRA)' + objects + ' $(LINKFILES) /out:$@ $(LIBDIRS) $(LIBS)', ]) From noreply at buildbot.pypy.org Mon Jan 7 23:40:31 2013 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 7 Jan 2013 23:40:31 +0100 (CET) Subject: [pypy-commit] pypy default: make all tests pass Message-ID: <20130107224031.44C7E1C1233@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r59866:f01c89bc3a9d Date: 2012-12-30 21:21 +0200 http://bitbucket.org/pypy/pypy/changeset/f01c89bc3a9d/ Log: make all tests pass diff --git a/pypy/translator/platform/test/test_distutils.py b/pypy/translator/platform/test/test_distutils.py --- a/pypy/translator/platform/test/test_distutils.py +++ b/pypy/translator/platform/test/test_distutils.py @@ -8,3 +8,6 @@ def test_nice_errors(self): py.test.skip("Unsupported") + + def test_900_files(self): + py.test.skip('Makefiles not suppoerted') From noreply at buildbot.pypy.org Mon Jan 7 23:40:32 2013 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 7 Jan 2013 23:40:32 +0100 (CET) Subject: [pypy-commit] pypy default: merge heads Message-ID: <20130107224032.BCAB41C1233@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r59867:19b73fa5c8cb Date: 2012-12-30 21:23 +0200 http://bitbucket.org/pypy/pypy/changeset/19b73fa5c8cb/ Log: merge heads diff --git a/pypy/doc/Makefile b/pypy/doc/Makefile --- a/pypy/doc/Makefile +++ b/pypy/doc/Makefile @@ -32,38 +32,38 @@ -rm -rf $(BUILDDIR)/* html: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." pickle: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ @@ -73,7 +73,7 @@ @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/PyPy.qhc" latex: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @@ -81,26 +81,26 @@ "run these through (pdf)latex." man: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man" changes: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." doctest: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." diff --git a/pypy/doc/config/translation.no__thread.txt b/pypy/doc/config/translation.no__thread.txt --- a/pypy/doc/config/translation.no__thread.txt +++ b/pypy/doc/config/translation.no__thread.txt @@ -1,3 +1,3 @@ Don't use gcc __thread attribute for fast thread local storage -implementation . Increases the chance that moving the resulting +implementation. Increases the chance that moving the resulting executable to another same processor Linux machine will work. diff --git a/pypy/doc/pypyconfig.py b/pypy/doc/pypyconfig.py --- a/pypy/doc/pypyconfig.py +++ b/pypy/doc/pypyconfig.py @@ -3,6 +3,10 @@ def setup(app): import sys, os sys.path.insert(0, os.path.abspath("../../")) + + #Autmatically calls make_cmdlline_overview + from pypy.doc.config import generate + from pypy.config import makerestdoc import py role = makerestdoc.register_config_role(py.path.local()) diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -5,6 +5,11 @@ .. this is a revision shortly after release-2.0-beta1 .. startrev: 0e6161a009c6 +.. branch: callback-jit +Callbacks from C are now better JITted + +.. branch: remove-globals-in-jit + .. branch: length-hint Implement __lenght_hint__ according to PEP 424 diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py --- a/pypy/interpreter/typedef.py +++ b/pypy/interpreter/typedef.py @@ -105,6 +105,13 @@ # typical usage (case 1 is the most common kind of app-level subclasses; # case 2 is the memory-saving kind defined with __slots__). # +# +----------------------------------------------------------------+ +# | NOTE: if withmapdict is enabled, the following doesn't apply! | +# | Map dicts can flexibly allow any slots/__dict__/__weakref__ to | +# | show up only when needed. In particular there is no way with | +# | mapdict to prevent some objects from being weakrefable. | +# +----------------------------------------------------------------+ +# # dict slots del weakrefable # # 1. Y N N Y UserDictWeakref diff --git a/pypy/jit/backend/llgraph/runner.py b/pypy/jit/backend/llgraph/runner.py --- a/pypy/jit/backend/llgraph/runner.py +++ b/pypy/jit/backend/llgraph/runner.py @@ -877,13 +877,16 @@ # assembler_helper_ptr = jd.assembler_helper_adr.ptr # fish try: - return assembler_helper_ptr(pframe, vable) + result = assembler_helper_ptr(pframe, vable) except LLException, lle: assert self.last_exception is None, "exception left behind" self.last_exception = lle # fish op op = self.current_op return op.result and op.result.value + if isinstance(result, float): + result = support.cast_to_floatstorage(result) + return result def execute_same_as(self, _, x): return x diff --git a/pypy/module/_weakref/test/test_weakref.py b/pypy/module/_weakref/test/test_weakref.py --- a/pypy/module/_weakref/test/test_weakref.py +++ b/pypy/module/_weakref/test/test_weakref.py @@ -114,19 +114,28 @@ class A(object): def __eq__(self, other): return True + def __ne__(self, other): + return False a1 = A() a2 = A() ref1 = _weakref.ref(a1) ref2 = _weakref.ref(a2) assert ref1 == ref2 + assert not (ref1 != ref2) + assert not (ref1 == []) + assert ref1 != [] del a1 gc.collect() assert not ref1 == ref2 assert ref1 != ref2 + assert not (ref1 == []) + assert ref1 != [] del a2 gc.collect() assert not ref1 == ref2 assert ref1 != ref2 + assert not (ref1 == []) + assert ref1 != [] def test_getweakrefs(self): import _weakref, gc @@ -298,6 +307,13 @@ if seen_callback: assert seen_callback == [True, True, True] + def test_type_weakrefable(self): + import _weakref, gc + w = _weakref.ref(list) + assert w() is list + gc.collect() + assert w() is list + class AppTestProxy(object): spaceconfig = dict(usemodules=('_weakref',)) @@ -435,6 +451,8 @@ class A(object): def __eq__(self, other): return True + def __ne__(self, other): + return False a = A() assert _weakref.ref(a) == a diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py --- a/pypy/rpython/lltypesystem/rffi.py +++ b/pypy/rpython/lltypesystem/rffi.py @@ -283,9 +283,10 @@ args = ', '.join(['a%d' % i for i in range(len(TP.TO.ARGS))]) source = py.code.Source(r""" def inner_wrapper(%(args)s): - callback_hook = aroundstate.callback_hook - if callback_hook: - callback_hook(llstr("%(callable_name_descr)s")) + if aroundstate is not None: + callback_hook = aroundstate.callback_hook + if callback_hook: + callback_hook(llstr("%(callable_name_descr)s")) return callable(%(args)s) inner_wrapper._never_inline_ = True diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py --- a/pypy/rpython/memory/gc/minimark.py +++ b/pypy/rpython/memory/gc/minimark.py @@ -2000,6 +2000,17 @@ (obj + offset).address[0] = llmemory.NULL continue # no need to remember this weakref any longer # + elif self.header(pointing_to).tid & GCFLAG_NO_HEAP_PTRS: + # see test_weakref_to_prebuilt: it's not useful to put + # weakrefs into 'old_objects_with_weakrefs' if they point + # to a prebuilt object (they are immortal). If moreover + # the 'pointing_to' prebuilt object still has the + # GCFLAG_NO_HEAP_PTRS flag, then it's even wrong, because + # 'pointing_to' will not get the GCFLAG_VISITED during + # the next major collection. Solve this by not registering + # the weakref into 'old_objects_with_weakrefs'. + continue + # self.old_objects_with_weakrefs.append(obj) def invalidate_old_weakrefs(self): @@ -2013,6 +2024,9 @@ continue # weakref itself dies offset = self.weakpointer_offset(self.get_type_id(obj)) pointing_to = (obj + offset).address[0] + ll_assert((self.header(pointing_to).tid & GCFLAG_NO_HEAP_PTRS) + == 0, "registered old weakref should not " + "point to a NO_HEAP_PTRS obj") if self.header(pointing_to).tid & GCFLAG_VISITED: new_with_weakref.append(obj) else: diff --git a/pypy/rpython/memory/test/test_gc.py b/pypy/rpython/memory/test/test_gc.py --- a/pypy/rpython/memory/test/test_gc.py +++ b/pypy/rpython/memory/test/test_gc.py @@ -557,6 +557,19 @@ res = self.interpret(f, [20]) # for GenerationGC, enough for a minor collection assert res == True + def test_weakref_to_prebuilt(self): + import weakref + class A: + pass + a = A() + def f(x): + ref = weakref.ref(a) + assert ref() is a + llop.gc__collect(lltype.Void) + return ref() is a + res = self.interpret(f, [20]) # for GenerationGC, enough for a minor collection + assert res == True + def test_many_weakrefs(self): # test for the case where allocating the weakref itself triggers # a collection diff --git a/pypy/translator/platform/linux.py b/pypy/translator/platform/linux.py --- a/pypy/translator/platform/linux.py +++ b/pypy/translator/platform/linux.py @@ -36,7 +36,13 @@ # places where we need to look for libffi.a # XXX obscuuure! only look for libffi.a if run with translate.py if 'translate' in sys.modules: - return self.library_dirs_for_libffi() + ['/usr/lib'] + if sys.maxint > 2**32: + host = 'x86_64' + else: + host = 'x86' + return self.library_dirs_for_libffi() + [ + '/usr/lib', + '/usr/lib/%s-linux-gnu/' % host] else: return [] From noreply at buildbot.pypy.org Tue Jan 8 00:10:43 2013 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 8 Jan 2013 00:10:43 +0100 (CET) Subject: [pypy-commit] pypy default: Issue #1337: Give a __file__ to PYTHONSTARTUP. Message-ID: <20130107231043.DBA2E1C1243@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r59868:b90d91783d29 Date: 2013-01-08 00:09 +0100 http://bitbucket.org/pypy/pypy/changeset/b90d91783d29/ Log: Issue #1337: Give a __file__ to PYTHONSTARTUP. diff --git a/pypy/translator/goal/app_main.py b/pypy/translator/goal/app_main.py --- a/pypy/translator/goal/app_main.py +++ b/pypy/translator/goal/app_main.py @@ -584,7 +584,12 @@ python_startup, 'exec') exec co_python_startup in mainmodule.__dict__ + mainmodule.__file__ = python_startup run_toplevel(run_it) + try: + del mainmodule.__file__ + except (AttributeError, TypeError): + pass # Then we need a prompt. inspect = True else: diff --git a/pypy/translator/goal/test2/test_app_main.py b/pypy/translator/goal/test2/test_app_main.py --- a/pypy/translator/goal/test2/test_app_main.py +++ b/pypy/translator/goal/test2/test_app_main.py @@ -376,6 +376,30 @@ child.expect('Traceback') child.expect('NameError') + def test_pythonstartup_file1(self, monkeypatch): + monkeypatch.setenv('PYTHONPATH', None) + monkeypatch.setenv('PYTHONSTARTUP', demo_script) + child = self.spawn([]) + child.expect('File: [^\n]+\.py') + child.expect('goodbye') + child.expect('>>> ') + child.sendline('[myvalue]') + child.expect(re.escape('[42]')) + child.expect('>>> ') + child.sendline('__file__') + child.expect('Traceback') + child.expect('NameError') + + def test_pythonstartup_file2(self, monkeypatch): + monkeypatch.setenv('PYTHONPATH', None) + monkeypatch.setenv('PYTHONSTARTUP', crashing_demo_script) + child = self.spawn([]) + child.expect('Traceback') + child.expect('>>> ') + child.sendline('__file__') + child.expect('Traceback') + child.expect('NameError') + def test_ignore_python_startup(self): old = os.environ.get('PYTHONSTARTUP', '') try: From noreply at buildbot.pypy.org Tue Jan 8 00:10:45 2013 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 8 Jan 2013 00:10:45 +0100 (CET) Subject: [pypy-commit] pypy default: merge heads Message-ID: <20130107231045.0AC9F1C1243@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r59869:2dd6e756b754 Date: 2013-01-08 00:10 +0100 http://bitbucket.org/pypy/pypy/changeset/2dd6e756b754/ Log: merge heads diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -38,7 +38,7 @@ # General information about the project. project = u'PyPy' -copyright = u'2011, The PyPy Project' +copyright = u'2013, The PyPy Project' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/pypy/translator/platform/test/test_distutils.py b/pypy/translator/platform/test/test_distutils.py --- a/pypy/translator/platform/test/test_distutils.py +++ b/pypy/translator/platform/test/test_distutils.py @@ -8,3 +8,6 @@ def test_nice_errors(self): py.test.skip("Unsupported") + + def test_900_files(self): + py.test.skip('Makefiles not suppoerted') diff --git a/pypy/translator/platform/test/test_platform.py b/pypy/translator/platform/test/test_platform.py --- a/pypy/translator/platform/test/test_platform.py +++ b/pypy/translator/platform/test/test_platform.py @@ -59,6 +59,34 @@ res = self.platform.execute(executable) self.check_res(res) + def test_900_files(self): + txt = '#include \n' + for i in range(900): + txt += 'int func%03d();\n' % i + txt += 'int main() {\n int j=0;' + for i in range(900): + txt += ' j += func%03d();\n' % i + txt += ' printf("%d\\n", j);\n' + txt += ' return 0;};\n' + cfile = udir.join('test_900_files.c') + cfile.write(txt) + cfiles = [cfile] + for i in range(900): + cfile2 = udir.join('implement%03d.c' %i) + cfile2.write(''' + int func%03d() + { + return %d; + } + ''' % (i, i)) + cfiles.append(cfile2) + mk = self.platform.gen_makefile(cfiles, ExternalCompilationInfo(), path=udir) + mk.write() + self.platform.execute_makefile(mk) + res = self.platform.execute(udir.join('test_900_files')) + self.check_res(res, '%d\n' %sum(range(900))) + + def test_nice_errors(self): cfile = udir.join('test_nice_errors.c') cfile.write('') diff --git a/pypy/translator/platform/windows.py b/pypy/translator/platform/windows.py --- a/pypy/translator/platform/windows.py +++ b/pypy/translator/platform/windows.py @@ -326,17 +326,33 @@ for rule in rules: m.rule(*rule) - + + objects = ' $(OBJECTS)' + create_obj_response_file = [] + if len(' '.join(rel_ofiles)) > 4000: + # cmd.exe has a limit of ~4000 characters before a command line is too long. + # Use a response file instead, at the cost of making the Makefile very ugly. + for i in range(len(rel_ofiles) - 1): + create_obj_response_file.append('echo %s >> obj_names.rsp' % \ + rel_ofiles[i]) + # use cmd /c for the last one so that the file is flushed + create_obj_response_file.append('cmd /c echo %s >> obj_names.rsp' % \ + rel_ofiles[-1]) + objects = ' @obj_names.rsp' if self.version < 80: m.rule('$(TARGET)', '$(OBJECTS)', - '$(CC_LINK) /nologo $(LDFLAGS) $(LDFLAGSEXTRA) $(OBJECTS) /out:$@ $(LIBDIRS) $(LIBS)') + create_obj_response_file + [\ + '$(CC_LINK) /nologo $(LDFLAGS) $(LDFLAGSEXTRA)' + objects + ' /out:$@ $(LIBDIRS) $(LIBS)', + ]) else: m.rule('$(TARGET)', '$(OBJECTS)', - ['$(CC_LINK) /nologo $(LDFLAGS) $(LDFLAGSEXTRA) $(OBJECTS) $(LINKFILES) /out:$@ $(LIBDIRS) $(LIBS) /MANIFEST /MANIFESTFILE:$*.manifest', + create_obj_response_file + [\ + '$(CC_LINK) /nologo $(LDFLAGS) $(LDFLAGSEXTRA)' + objects + ' $(LINKFILES) /out:$@ $(LIBDIRS) $(LIBS) /MANIFEST /MANIFESTFILE:$*.manifest', 'mt.exe -nologo -manifest $*.manifest -outputresource:$@;1', ]) m.rule('debugmode_$(TARGET)', '$(OBJECTS)', - ['$(CC_LINK) /nologo /DEBUG $(LDFLAGS) $(LDFLAGSEXTRA) $(OBJECTS) $(LINKFILES) /out:$@ $(LIBDIRS) $(LIBS)', + create_obj_response_file + [\ + '$(CC_LINK) /nologo /DEBUG $(LDFLAGS) $(LDFLAGSEXTRA)' + objects + ' $(LINKFILES) /out:$@ $(LIBDIRS) $(LIBS)', ]) if shared: From noreply at buildbot.pypy.org Tue Jan 8 02:07:37 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Tue, 8 Jan 2013 02:07:37 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Removed get_repo_version_info from udir. Moved tl/pypyjit to pypy/tool Message-ID: <20130108010737.3FC1B1C1241@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59870:c1e59dc0fb7b Date: 2013-01-08 02:07 +0100 http://bitbucket.org/pypy/pypy/changeset/c1e59dc0fb7b/ Log: Removed get_repo_version_info from udir. Moved tl/pypyjit to pypy/tool diff --git a/rpython/jit/tl/pypyjit.py b/pypy/tool/pypyjit.py rename from rpython/jit/tl/pypyjit.py rename to pypy/tool/pypyjit.py diff --git a/rpython/jit/tl/pypyjit_child.py b/pypy/tool/pypyjit_child.py rename from rpython/jit/tl/pypyjit_child.py rename to pypy/tool/pypyjit_child.py diff --git a/rpython/jit/tl/pypyjit_demo.py b/pypy/tool/pypyjit_demo.py rename from rpython/jit/tl/pypyjit_demo.py rename to pypy/tool/pypyjit_demo.py diff --git a/rpython/tool/udir.py b/rpython/tool/udir.py --- a/rpython/tool/udir.py +++ b/rpython/tool/udir.py @@ -20,7 +20,6 @@ import os, sys import py -from pypy.tool.version import get_repo_version_info from py.path import local PYPY_KEEP = int(os.environ.get('PYPY_USESSION_KEEP', '3')) @@ -29,17 +28,7 @@ if dir is not None: dir = local(dir) if basename is None: - info = get_repo_version_info() - if info: - project, hgtag, hgid = info - basename = hgtag - if basename == '?': - basename = 'unknown' # directories with ? are not fun - # especially on windows - if isinstance(basename, unicode): - basename = basename.encode(sys.getdefaultencoding()) - else: - basename = '' + basename = '' basename = basename.replace('/', '--') if not basename.startswith('-'): basename = '-' + basename From noreply at buildbot.pypy.org Tue Jan 8 09:02:10 2013 From: noreply at buildbot.pypy.org (antocuni) Date: Tue, 8 Jan 2013 09:02:10 +0100 (CET) Subject: [pypy-commit] pypy refactor-call_release_gil: a branch where to fix this failing test about call_release_gil, which is due to the fact that the op is generated inside pyjitpl and thus its resultbox is not included in the guard_not_forced failargs Message-ID: <20130108080210.23A071C10DD@cobra.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: refactor-call_release_gil Changeset: r59871:90482eb028fe Date: 2013-01-08 09:01 +0100 http://bitbucket.org/pypy/pypy/changeset/90482eb028fe/ Log: a branch where to fix this failing test about call_release_gil, which is due to the fact that the op is generated inside pyjitpl and thus its resultbox is not included in the guard_not_forced failargs diff --git a/pypy/jit/backend/llgraph/runner.py b/pypy/jit/backend/llgraph/runner.py --- a/pypy/jit/backend/llgraph/runner.py +++ b/pypy/jit/backend/llgraph/runner.py @@ -820,10 +820,22 @@ # manipulation here (as a hack, instead of really doing # the aroundstate manipulation ourselves) return self.execute_call_may_force(descr, func, *args) + guard_op = self.lltrace.operations[self.current_index + 1] + assert guard_op.getopnum() == rop.GUARD_NOT_FORCED + self.force_guard_op = guard_op call_args = support.cast_call_args_in_order(descr.ARGS, args) - FUNC = lltype.FuncType(descr.ARGS, descr.RESULT) - func_to_call = rffi.cast(lltype.Ptr(FUNC), func) - result = func_to_call(*call_args) + # + func_adr = llmemory.cast_int_to_adr(func) + if hasattr(func_adr.ptr._obj, '_callable'): + # this is needed e.g. by test_fficall.test_guard_not_forced_fails, + # because to actually force the virtualref we need to llinterp the + # graph, not to directly execute the python function + result = self.cpu.maybe_on_top_of_llinterp(func, call_args, descr.RESULT) + else: + FUNC = lltype.FuncType(descr.ARGS, descr.RESULT) + func_to_call = rffi.cast(lltype.Ptr(FUNC), func) + result = func_to_call(*call_args) + del self.force_guard_op return support.cast_result(descr.RESULT, result) def execute_call_assembler(self, descr, *args): diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py --- a/pypy/jit/metainterp/history.py +++ b/pypy/jit/metainterp/history.py @@ -769,8 +769,8 @@ def show(self, errmsg=None): "NOT_RPYTHON" - from pypy.jit.metainterp.graphpage import display_loops - display_loops([self], errmsg) + from pypy.jit.metainterp.graphpage import display_procedures + display_procedures([self], errmsg) def check_consistency(self): # for testing "NOT_RPYTHON" diff --git a/pypy/jit/metainterp/test/test_fficall.py b/pypy/jit/metainterp/test/test_fficall.py --- a/pypy/jit/metainterp/test/test_fficall.py +++ b/pypy/jit/metainterp/test/test_fficall.py @@ -1,4 +1,5 @@ import py +from pypy.rpython.annlowlevel import llhelper from pypy.rpython.lltypesystem import lltype, rffi from pypy.jit.metainterp.test.support import LLJitMixin from pypy.rlib import jit @@ -102,5 +103,74 @@ rffi.cast(rffi.SIGNEDCHAR, -42)) + def test_guard_not_forced_fails(self): + FUNC = lltype.FuncType([lltype.Signed], lltype.Signed) + + cif_description = get_description([types.slong], types.slong) + cif_description.exchange_args[0] = 16 + cif_description.exchange_result = 32 + + ARRAY = lltype.Ptr(rffi.CArray(lltype.Signed)) + + @jit.dont_look_inside + def fn(n): + if n >= 50: + exctx.m = exctx.topframeref().n # forces the frame + return n*2 + + @jit.oopspec("libffi_call(cif_description,func_addr,exchange_buffer)") + def fake_call(cif_description, func_addr, exchange_buffer): + # read the args from the buffer + data_in = rffi.ptradd(exchange_buffer, 16) + n = rffi.cast(ARRAY, data_in)[0] + # + # logic of the function + func_ptr = rffi.cast(lltype.Ptr(FUNC), func_addr) + n = func_ptr(n) + # + # write the result to the buffer + data_out = rffi.ptradd(exchange_buffer, 32) + rffi.cast(ARRAY, data_out)[0] = n + + def do_call(n): + func_ptr = llhelper(lltype.Ptr(FUNC), fn) + #func_addr = rffi.cast(rffi.VOIDP, func_ptr) + exbuf = lltype.malloc(rffi.CCHARP.TO, 48, flavor='raw', zero=True) + data_in = rffi.ptradd(exbuf, 16) + rffi.cast(ARRAY, data_in)[0] = n + fake_call(cif_description, func_ptr, exbuf) + data_out = rffi.ptradd(exbuf, 32) + res = rffi.cast(ARRAY, data_out)[0] + lltype.free(exbuf, flavor='raw') + return res + + # + # + class XY: + pass + class ExCtx: + pass + exctx = ExCtx() + myjitdriver = jit.JitDriver(greens = [], reds = ['n']) + def f(): + n = 0 + while n < 100: + myjitdriver.jit_merge_point(n=n) + xy = XY() + xy.n = n + exctx.topframeref = vref = jit.virtual_ref(xy) + res = do_call(n) # this is equivalent of a cffi call which + # sometimes forces a frame + assert res == n*2 + jit.virtual_ref_finish(vref, xy) + exctx.topframeref = jit.vref_None + n += 1 + return n + + assert f() == 100 + res = self.meta_interp(f, []) + assert res == 100 + + class TestFfiCall(FfiCallTests, LLJitMixin): pass From noreply at buildbot.pypy.org Tue Jan 8 09:39:30 2013 From: noreply at buildbot.pypy.org (bivab) Date: Tue, 8 Jan 2013 09:39:30 +0100 (CET) Subject: [pypy-commit] pypy default: avoid this test crashing on load time Message-ID: <20130108083930.0C6EF1C118C@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: Changeset: r59872:371fbdc16264 Date: 2013-01-08 09:38 +0100 http://bitbucket.org/pypy/pypy/changeset/371fbdc16264/ Log: avoid this test crashing on load time diff --git a/pypy/jit/backend/arm/test/test_gc_integration.py b/pypy/jit/backend/arm/test/test_gc_integration.py --- a/pypy/jit/backend/arm/test/test_gc_integration.py +++ b/pypy/jit/backend/arm/test/test_gc_integration.py @@ -7,7 +7,7 @@ BoxPtr, TreeLoop, TargetToken from pypy.jit.metainterp.resoperation import rop, ResOperation from pypy.jit.codewriter import heaptracker -from pypy.jit.backend.llsupport.descr import GcCache +from pypy.jit.backend.llsupport.gc import GcLLDescription, GcLLDescr_boehm from pypy.jit.backend.llsupport.gc import GcLLDescription from pypy.jit.backend.detect_cpu import getcpuclass from pypy.jit.backend.arm.arch import WORD @@ -44,23 +44,14 @@ return ['compressed'] + shape[1:] -class MockGcDescr(GcCache): - get_malloc_slowpath_addr = None - write_barrier_descr = None - moving_gc = True +class MockGcDescr(GcLLDescr_boehm): gcrootmap = MockGcRootMap() - def initialize(self): - pass - - _record_constptrs = GcLLDescr_framework._record_constptrs.im_func - rewrite_assembler = GcLLDescr_framework.rewrite_assembler.im_func - class TestRegallocGcIntegration(BaseTestRegalloc): cpu = CPU(None, None) - cpu.gc_ll_descr = MockGcDescr(False) + cpu.gc_ll_descr = MockGcDescr(None, None, None) cpu.setup_once() S = lltype.GcForwardReference() From noreply at buildbot.pypy.org Tue Jan 8 09:39:32 2013 From: noreply at buildbot.pypy.org (bivab) Date: Tue, 8 Jan 2013 09:39:32 +0100 (CET) Subject: [pypy-commit] pypy default: kill unused imports Message-ID: <20130108083932.16B431C118C@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: Changeset: r59873:bd749c887394 Date: 2013-01-08 09:39 +0100 http://bitbucket.org/pypy/pypy/changeset/bd749c887394/ Log: kill unused imports diff --git a/pypy/jit/backend/arm/test/test_gc_integration.py b/pypy/jit/backend/arm/test/test_gc_integration.py --- a/pypy/jit/backend/arm/test/test_gc_integration.py +++ b/pypy/jit/backend/arm/test/test_gc_integration.py @@ -2,25 +2,13 @@ """ Tests for register allocation for common constructs """ -import py -from pypy.jit.metainterp.history import BoxInt, \ - BoxPtr, TreeLoop, TargetToken -from pypy.jit.metainterp.resoperation import rop, ResOperation -from pypy.jit.codewriter import heaptracker +from pypy.jit.metainterp.history import TargetToken from pypy.jit.backend.llsupport.gc import GcLLDescription, GcLLDescr_boehm -from pypy.jit.backend.llsupport.gc import GcLLDescription from pypy.jit.backend.detect_cpu import getcpuclass from pypy.jit.backend.arm.arch import WORD from pypy.rpython.lltypesystem import lltype, llmemory, rffi -from pypy.rpython.annlowlevel import llhelper -from pypy.rpython.lltypesystem import rclass -from pypy.jit.backend.llsupport.gc import GcLLDescr_framework -from pypy.jit.backend.arm.test.test_regalloc import MockAssembler from pypy.jit.backend.arm.test.test_regalloc import BaseTestRegalloc -from pypy.jit.backend.arm.regalloc import ARMFrameManager, VFPRegisterManager -from pypy.jit.codewriter.effectinfo import EffectInfo -from pypy.jit.backend.arm.regalloc import Regalloc CPU = getcpuclass() From noreply at buildbot.pypy.org Tue Jan 8 16:08:20 2013 From: noreply at buildbot.pypy.org (stepahn) Date: Tue, 8 Jan 2013 16:08:20 +0100 (CET) Subject: [pypy-commit] lang-js default: moved some files out of js folder Message-ID: <20130108150820.6CC1A1C12B6@cobra.cs.uni-duesseldorf.de> Author: Stephan Branch: Changeset: r332:fd3b7c14fe7d Date: 2013-01-08 10:40 +0100 http://bitbucket.org/pypy/lang-js/changeset/fd3b7c14fe7d/ Log: moved some files out of js folder diff --git a/js/bench/TODO.txt b/bench/TODO.txt rename from js/bench/TODO.txt rename to bench/TODO.txt diff --git a/js/bench/ackermann.javascript b/bench/ackermann.javascript rename from js/bench/ackermann.javascript rename to bench/ackermann.javascript diff --git a/js/bench/ary.javascript b/bench/ary.javascript rename from js/bench/ary.javascript rename to bench/ary.javascript diff --git a/js/bench/binarytrees.javascript b/bench/binarytrees.javascript rename from js/bench/binarytrees.javascript rename to bench/binarytrees.javascript diff --git a/js/bench/fannkuch.javascript b/bench/fannkuch.javascript rename from js/bench/fannkuch.javascript rename to bench/fannkuch.javascript diff --git a/js/bench/fasta.javascript b/bench/fasta.javascript rename from js/bench/fasta.javascript rename to bench/fasta.javascript diff --git a/js/bench/fibo.javascript b/bench/fibo.javascript rename from js/bench/fibo.javascript rename to bench/fibo.javascript diff --git a/js/bench/harmonic.javascript b/bench/harmonic.javascript rename from js/bench/harmonic.javascript rename to bench/harmonic.javascript diff --git a/js/bench/hash.javascript b/bench/hash.javascript rename from js/bench/hash.javascript rename to bench/hash.javascript diff --git a/js/bench/hash2.javascript b/bench/hash2.javascript rename from js/bench/hash2.javascript rename to bench/hash2.javascript diff --git a/js/bench/heapsort.javascript b/bench/heapsort.javascript rename from js/bench/heapsort.javascript rename to bench/heapsort.javascript diff --git a/js/bench/loop/run.js b/bench/loop/run.js rename from js/bench/loop/run.js rename to bench/loop/run.js diff --git a/js/bench/sunspider/run.js b/bench/sunspider/run.js rename from js/bench/sunspider/run.js rename to bench/sunspider/run.js diff --git a/js/bench/sunspider/t/3d-cube.js b/bench/sunspider/t/3d-cube.js rename from js/bench/sunspider/t/3d-cube.js rename to bench/sunspider/t/3d-cube.js diff --git a/js/bench/sunspider/t/3d-morph.js b/bench/sunspider/t/3d-morph.js rename from js/bench/sunspider/t/3d-morph.js rename to bench/sunspider/t/3d-morph.js diff --git a/js/bench/sunspider/t/3d-raytrace.js b/bench/sunspider/t/3d-raytrace.js rename from js/bench/sunspider/t/3d-raytrace.js rename to bench/sunspider/t/3d-raytrace.js diff --git a/js/bench/sunspider/t/access-binary-trees.js b/bench/sunspider/t/access-binary-trees.js rename from js/bench/sunspider/t/access-binary-trees.js rename to bench/sunspider/t/access-binary-trees.js diff --git a/js/bench/sunspider/t/access-fannkuch.js b/bench/sunspider/t/access-fannkuch.js rename from js/bench/sunspider/t/access-fannkuch.js rename to bench/sunspider/t/access-fannkuch.js diff --git a/js/bench/sunspider/t/access-nbody.js b/bench/sunspider/t/access-nbody.js rename from js/bench/sunspider/t/access-nbody.js rename to bench/sunspider/t/access-nbody.js diff --git a/js/bench/sunspider/t/access-nsieve.js b/bench/sunspider/t/access-nsieve.js rename from js/bench/sunspider/t/access-nsieve.js rename to bench/sunspider/t/access-nsieve.js diff --git a/js/bench/sunspider/t/bitops-3bit-bits-in-byte.js b/bench/sunspider/t/bitops-3bit-bits-in-byte.js rename from js/bench/sunspider/t/bitops-3bit-bits-in-byte.js rename to bench/sunspider/t/bitops-3bit-bits-in-byte.js diff --git a/js/bench/sunspider/t/bitops-bits-in-byte.js b/bench/sunspider/t/bitops-bits-in-byte.js rename from js/bench/sunspider/t/bitops-bits-in-byte.js rename to bench/sunspider/t/bitops-bits-in-byte.js diff --git a/js/bench/sunspider/t/bitops-bitwise-and.js b/bench/sunspider/t/bitops-bitwise-and.js rename from js/bench/sunspider/t/bitops-bitwise-and.js rename to bench/sunspider/t/bitops-bitwise-and.js diff --git a/js/bench/sunspider/t/bitops-nsieve-bits.js b/bench/sunspider/t/bitops-nsieve-bits.js rename from js/bench/sunspider/t/bitops-nsieve-bits.js rename to bench/sunspider/t/bitops-nsieve-bits.js diff --git a/js/bench/sunspider/t/controlflow-recursive.js b/bench/sunspider/t/controlflow-recursive.js rename from js/bench/sunspider/t/controlflow-recursive.js rename to bench/sunspider/t/controlflow-recursive.js diff --git a/js/bench/sunspider/t/crypto-aes.js b/bench/sunspider/t/crypto-aes.js rename from js/bench/sunspider/t/crypto-aes.js rename to bench/sunspider/t/crypto-aes.js diff --git a/js/bench/sunspider/t/crypto-md5.js b/bench/sunspider/t/crypto-md5.js rename from js/bench/sunspider/t/crypto-md5.js rename to bench/sunspider/t/crypto-md5.js diff --git a/js/bench/sunspider/t/crypto-sha1.js b/bench/sunspider/t/crypto-sha1.js rename from js/bench/sunspider/t/crypto-sha1.js rename to bench/sunspider/t/crypto-sha1.js diff --git a/js/bench/sunspider/t/date-format-tofte.js b/bench/sunspider/t/date-format-tofte.js rename from js/bench/sunspider/t/date-format-tofte.js rename to bench/sunspider/t/date-format-tofte.js diff --git a/js/bench/sunspider/t/date-format-xparb.js b/bench/sunspider/t/date-format-xparb.js rename from js/bench/sunspider/t/date-format-xparb.js rename to bench/sunspider/t/date-format-xparb.js diff --git a/js/bench/sunspider/t/math-cordic.js b/bench/sunspider/t/math-cordic.js rename from js/bench/sunspider/t/math-cordic.js rename to bench/sunspider/t/math-cordic.js diff --git a/js/bench/sunspider/t/math-partial-sums.js b/bench/sunspider/t/math-partial-sums.js rename from js/bench/sunspider/t/math-partial-sums.js rename to bench/sunspider/t/math-partial-sums.js diff --git a/js/bench/sunspider/t/math-spectral-norm.js b/bench/sunspider/t/math-spectral-norm.js rename from js/bench/sunspider/t/math-spectral-norm.js rename to bench/sunspider/t/math-spectral-norm.js diff --git a/js/bench/sunspider/t/regexp-dna.js b/bench/sunspider/t/regexp-dna.js rename from js/bench/sunspider/t/regexp-dna.js rename to bench/sunspider/t/regexp-dna.js diff --git a/js/bench/sunspider/t/string-base64.js b/bench/sunspider/t/string-base64.js rename from js/bench/sunspider/t/string-base64.js rename to bench/sunspider/t/string-base64.js diff --git a/js/bench/sunspider/t/string-fasta.js b/bench/sunspider/t/string-fasta.js rename from js/bench/sunspider/t/string-fasta.js rename to bench/sunspider/t/string-fasta.js diff --git a/js/bench/sunspider/t/string-tagcloud.js b/bench/sunspider/t/string-tagcloud.js rename from js/bench/sunspider/t/string-tagcloud.js rename to bench/sunspider/t/string-tagcloud.js diff --git a/js/bench/sunspider/t/string-unpack-code.js b/bench/sunspider/t/string-unpack-code.js rename from js/bench/sunspider/t/string-unpack-code.js rename to bench/sunspider/t/string-unpack-code.js diff --git a/js/bench/sunspider/t/string-validate-input.js b/bench/sunspider/t/string-validate-input.js rename from js/bench/sunspider/t/string-validate-input.js rename to bench/sunspider/t/string-validate-input.js diff --git a/js/bench/v8/v1/README.txt b/bench/v8/v1/README.txt rename from js/bench/v8/v1/README.txt rename to bench/v8/v1/README.txt diff --git a/js/bench/v8/v1/base.js b/bench/v8/v1/base.js rename from js/bench/v8/v1/base.js rename to bench/v8/v1/base.js diff --git a/js/bench/v8/v1/crypto.js b/bench/v8/v1/crypto.js rename from js/bench/v8/v1/crypto.js rename to bench/v8/v1/crypto.js diff --git a/js/bench/v8/v1/deltablue.js b/bench/v8/v1/deltablue.js rename from js/bench/v8/v1/deltablue.js rename to bench/v8/v1/deltablue.js diff --git a/js/bench/v8/v1/earley-boyer.js b/bench/v8/v1/earley-boyer.js rename from js/bench/v8/v1/earley-boyer.js rename to bench/v8/v1/earley-boyer.js diff --git a/js/bench/v8/v1/raytrace.js b/bench/v8/v1/raytrace.js rename from js/bench/v8/v1/raytrace.js rename to bench/v8/v1/raytrace.js diff --git a/js/bench/v8/v1/richards.js b/bench/v8/v1/richards.js rename from js/bench/v8/v1/richards.js rename to bench/v8/v1/richards.js diff --git a/js/bench/v8/v1/run.html b/bench/v8/v1/run.html rename from js/bench/v8/v1/run.html rename to bench/v8/v1/run.html diff --git a/js/bench/v8/v1/run.js b/bench/v8/v1/run.js rename from js/bench/v8/v1/run.js rename to bench/v8/v1/run.js diff --git a/js/bench/v8/v1/v8-logo.png b/bench/v8/v1/v8-logo.png rename from js/bench/v8/v1/v8-logo.png rename to bench/v8/v1/v8-logo.png diff --git a/js/py-js.py b/py-js.py rename from js/py-js.py rename to py-js.py diff --git a/js/test/__init__.py b/test/__init__.py rename from js/test/__init__.py rename to test/__init__.py diff --git a/js/test/ecma/Array/15.4-1.js b/test/ecma/Array/15.4-1.js rename from js/test/ecma/Array/15.4-1.js rename to test/ecma/Array/15.4-1.js diff --git a/js/test/ecma/Array/15.4-2.js b/test/ecma/Array/15.4-2.js rename from js/test/ecma/Array/15.4-2.js rename to test/ecma/Array/15.4-2.js diff --git a/js/test/ecma/Array/15.4.1.1.js b/test/ecma/Array/15.4.1.1.js rename from js/test/ecma/Array/15.4.1.1.js rename to test/ecma/Array/15.4.1.1.js diff --git a/js/test/ecma/Array/15.4.1.2.js b/test/ecma/Array/15.4.1.2.js rename from js/test/ecma/Array/15.4.1.2.js rename to test/ecma/Array/15.4.1.2.js diff --git a/js/test/ecma/Array/15.4.1.3.js b/test/ecma/Array/15.4.1.3.js rename from js/test/ecma/Array/15.4.1.3.js rename to test/ecma/Array/15.4.1.3.js diff --git a/js/test/ecma/Array/15.4.1.js b/test/ecma/Array/15.4.1.js rename from js/test/ecma/Array/15.4.1.js rename to test/ecma/Array/15.4.1.js diff --git a/js/test/ecma/Array/15.4.2.1-1.js b/test/ecma/Array/15.4.2.1-1.js rename from js/test/ecma/Array/15.4.2.1-1.js rename to test/ecma/Array/15.4.2.1-1.js diff --git a/js/test/ecma/Array/15.4.2.1-2.js b/test/ecma/Array/15.4.2.1-2.js rename from js/test/ecma/Array/15.4.2.1-2.js rename to test/ecma/Array/15.4.2.1-2.js diff --git a/js/test/ecma/Array/15.4.2.1-3.js b/test/ecma/Array/15.4.2.1-3.js rename from js/test/ecma/Array/15.4.2.1-3.js rename to test/ecma/Array/15.4.2.1-3.js diff --git a/js/test/ecma/Array/15.4.2.2-1.js b/test/ecma/Array/15.4.2.2-1.js rename from js/test/ecma/Array/15.4.2.2-1.js rename to test/ecma/Array/15.4.2.2-1.js diff --git a/js/test/ecma/Array/15.4.2.2-2.js b/test/ecma/Array/15.4.2.2-2.js rename from js/test/ecma/Array/15.4.2.2-2.js rename to test/ecma/Array/15.4.2.2-2.js diff --git a/js/test/ecma/Array/15.4.2.3.js b/test/ecma/Array/15.4.2.3.js rename from js/test/ecma/Array/15.4.2.3.js rename to test/ecma/Array/15.4.2.3.js diff --git a/js/test/ecma/Array/15.4.3.1-2.js b/test/ecma/Array/15.4.3.1-2.js rename from js/test/ecma/Array/15.4.3.1-2.js rename to test/ecma/Array/15.4.3.1-2.js diff --git a/js/test/ecma/Array/15.4.3.2.js b/test/ecma/Array/15.4.3.2.js rename from js/test/ecma/Array/15.4.3.2.js rename to test/ecma/Array/15.4.3.2.js diff --git a/js/test/ecma/Array/15.4.3.js b/test/ecma/Array/15.4.3.js rename from js/test/ecma/Array/15.4.3.js rename to test/ecma/Array/15.4.3.js diff --git a/js/test/ecma/Array/15.4.4.1.js b/test/ecma/Array/15.4.4.1.js rename from js/test/ecma/Array/15.4.4.1.js rename to test/ecma/Array/15.4.4.1.js diff --git a/js/test/ecma/Array/15.4.4.2.js b/test/ecma/Array/15.4.4.2.js rename from js/test/ecma/Array/15.4.4.2.js rename to test/ecma/Array/15.4.4.2.js diff --git a/js/test/ecma/Array/15.4.4.3-1.js b/test/ecma/Array/15.4.4.3-1.js rename from js/test/ecma/Array/15.4.4.3-1.js rename to test/ecma/Array/15.4.4.3-1.js diff --git a/js/test/ecma/Array/15.4.4.4-1.js b/test/ecma/Array/15.4.4.4-1.js rename from js/test/ecma/Array/15.4.4.4-1.js rename to test/ecma/Array/15.4.4.4-1.js diff --git a/js/test/ecma/Array/15.4.4.4-2.js b/test/ecma/Array/15.4.4.4-2.js rename from js/test/ecma/Array/15.4.4.4-2.js rename to test/ecma/Array/15.4.4.4-2.js diff --git a/js/test/ecma/Array/15.4.4.5-1.js b/test/ecma/Array/15.4.4.5-1.js rename from js/test/ecma/Array/15.4.4.5-1.js rename to test/ecma/Array/15.4.4.5-1.js diff --git a/js/test/ecma/Array/15.4.4.5-2.js b/test/ecma/Array/15.4.4.5-2.js rename from js/test/ecma/Array/15.4.4.5-2.js rename to test/ecma/Array/15.4.4.5-2.js diff --git a/js/test/ecma/Array/15.4.4.5-3.js b/test/ecma/Array/15.4.4.5-3.js rename from js/test/ecma/Array/15.4.4.5-3.js rename to test/ecma/Array/15.4.4.5-3.js diff --git a/js/test/ecma/Array/15.4.4.js b/test/ecma/Array/15.4.4.js rename from js/test/ecma/Array/15.4.4.js rename to test/ecma/Array/15.4.4.js diff --git a/js/test/ecma/Array/15.4.5.1-1.js b/test/ecma/Array/15.4.5.1-1.js rename from js/test/ecma/Array/15.4.5.1-1.js rename to test/ecma/Array/15.4.5.1-1.js diff --git a/js/test/ecma/Array/15.4.5.1-2.js b/test/ecma/Array/15.4.5.1-2.js rename from js/test/ecma/Array/15.4.5.1-2.js rename to test/ecma/Array/15.4.5.1-2.js diff --git a/js/test/ecma/Array/15.4.5.2-1.js b/test/ecma/Array/15.4.5.2-1.js rename from js/test/ecma/Array/15.4.5.2-1.js rename to test/ecma/Array/15.4.5.2-1.js diff --git a/js/test/ecma/Array/15.4.5.2-2.js b/test/ecma/Array/15.4.5.2-2.js rename from js/test/ecma/Array/15.4.5.2-2.js rename to test/ecma/Array/15.4.5.2-2.js diff --git a/js/test/ecma/Array/browser.js b/test/ecma/Array/browser.js rename from js/test/ecma/Array/browser.js rename to test/ecma/Array/browser.js diff --git a/js/test/ecma/Array/shell.js b/test/ecma/Array/shell.js rename from js/test/ecma/Array/shell.js rename to test/ecma/Array/shell.js diff --git a/js/test/ecma/Boolean/15.6.1.js b/test/ecma/Boolean/15.6.1.js rename from js/test/ecma/Boolean/15.6.1.js rename to test/ecma/Boolean/15.6.1.js diff --git a/js/test/ecma/Boolean/15.6.2.js b/test/ecma/Boolean/15.6.2.js rename from js/test/ecma/Boolean/15.6.2.js rename to test/ecma/Boolean/15.6.2.js diff --git a/js/test/ecma/Boolean/15.6.3.1-1.js b/test/ecma/Boolean/15.6.3.1-1.js rename from js/test/ecma/Boolean/15.6.3.1-1.js rename to test/ecma/Boolean/15.6.3.1-1.js diff --git a/js/test/ecma/Boolean/15.6.3.1-2.js b/test/ecma/Boolean/15.6.3.1-2.js rename from js/test/ecma/Boolean/15.6.3.1-2.js rename to test/ecma/Boolean/15.6.3.1-2.js diff --git a/js/test/ecma/Boolean/15.6.3.1-3.js b/test/ecma/Boolean/15.6.3.1-3.js rename from js/test/ecma/Boolean/15.6.3.1-3.js rename to test/ecma/Boolean/15.6.3.1-3.js diff --git a/js/test/ecma/Boolean/15.6.3.1-4.js b/test/ecma/Boolean/15.6.3.1-4.js rename from js/test/ecma/Boolean/15.6.3.1-4.js rename to test/ecma/Boolean/15.6.3.1-4.js diff --git a/js/test/ecma/Boolean/15.6.3.1-5.js b/test/ecma/Boolean/15.6.3.1-5.js rename from js/test/ecma/Boolean/15.6.3.1-5.js rename to test/ecma/Boolean/15.6.3.1-5.js diff --git a/js/test/ecma/Boolean/15.6.3.1.js b/test/ecma/Boolean/15.6.3.1.js rename from js/test/ecma/Boolean/15.6.3.1.js rename to test/ecma/Boolean/15.6.3.1.js diff --git a/js/test/ecma/Boolean/15.6.3.js b/test/ecma/Boolean/15.6.3.js rename from js/test/ecma/Boolean/15.6.3.js rename to test/ecma/Boolean/15.6.3.js diff --git a/js/test/ecma/Boolean/15.6.4-1.js b/test/ecma/Boolean/15.6.4-1.js rename from js/test/ecma/Boolean/15.6.4-1.js rename to test/ecma/Boolean/15.6.4-1.js diff --git a/js/test/ecma/Boolean/15.6.4-2.js b/test/ecma/Boolean/15.6.4-2.js rename from js/test/ecma/Boolean/15.6.4-2.js rename to test/ecma/Boolean/15.6.4-2.js diff --git a/js/test/ecma/Boolean/15.6.4.1.js b/test/ecma/Boolean/15.6.4.1.js rename from js/test/ecma/Boolean/15.6.4.1.js rename to test/ecma/Boolean/15.6.4.1.js diff --git a/js/test/ecma/Boolean/15.6.4.2-1.js b/test/ecma/Boolean/15.6.4.2-1.js rename from js/test/ecma/Boolean/15.6.4.2-1.js rename to test/ecma/Boolean/15.6.4.2-1.js diff --git a/js/test/ecma/Boolean/15.6.4.2-2.js b/test/ecma/Boolean/15.6.4.2-2.js rename from js/test/ecma/Boolean/15.6.4.2-2.js rename to test/ecma/Boolean/15.6.4.2-2.js diff --git a/js/test/ecma/Boolean/15.6.4.2-3.js b/test/ecma/Boolean/15.6.4.2-3.js rename from js/test/ecma/Boolean/15.6.4.2-3.js rename to test/ecma/Boolean/15.6.4.2-3.js diff --git a/js/test/ecma/Boolean/15.6.4.2-4-n.js b/test/ecma/Boolean/15.6.4.2-4-n.js rename from js/test/ecma/Boolean/15.6.4.2-4-n.js rename to test/ecma/Boolean/15.6.4.2-4-n.js diff --git a/js/test/ecma/Boolean/15.6.4.3-1.js b/test/ecma/Boolean/15.6.4.3-1.js rename from js/test/ecma/Boolean/15.6.4.3-1.js rename to test/ecma/Boolean/15.6.4.3-1.js diff --git a/js/test/ecma/Boolean/15.6.4.3-2.js b/test/ecma/Boolean/15.6.4.3-2.js rename from js/test/ecma/Boolean/15.6.4.3-2.js rename to test/ecma/Boolean/15.6.4.3-2.js diff --git a/js/test/ecma/Boolean/15.6.4.3-3.js b/test/ecma/Boolean/15.6.4.3-3.js rename from js/test/ecma/Boolean/15.6.4.3-3.js rename to test/ecma/Boolean/15.6.4.3-3.js diff --git a/js/test/ecma/Boolean/15.6.4.3-4-n.js b/test/ecma/Boolean/15.6.4.3-4-n.js rename from js/test/ecma/Boolean/15.6.4.3-4-n.js rename to test/ecma/Boolean/15.6.4.3-4-n.js diff --git a/js/test/ecma/Boolean/15.6.4.3.js b/test/ecma/Boolean/15.6.4.3.js rename from js/test/ecma/Boolean/15.6.4.3.js rename to test/ecma/Boolean/15.6.4.3.js diff --git a/js/test/ecma/Boolean/15.6.4.js b/test/ecma/Boolean/15.6.4.js rename from js/test/ecma/Boolean/15.6.4.js rename to test/ecma/Boolean/15.6.4.js diff --git a/js/test/ecma/Boolean/browser.js b/test/ecma/Boolean/browser.js rename from js/test/ecma/Boolean/browser.js rename to test/ecma/Boolean/browser.js diff --git a/js/test/ecma/Boolean/shell.js b/test/ecma/Boolean/shell.js rename from js/test/ecma/Boolean/shell.js rename to test/ecma/Boolean/shell.js diff --git a/js/test/ecma/ExecutionContexts/10.1.3-1.js b/test/ecma/ExecutionContexts/10.1.3-1.js rename from js/test/ecma/ExecutionContexts/10.1.3-1.js rename to test/ecma/ExecutionContexts/10.1.3-1.js diff --git a/js/test/ecma/ExecutionContexts/10.1.3-2.js b/test/ecma/ExecutionContexts/10.1.3-2.js rename from js/test/ecma/ExecutionContexts/10.1.3-2.js rename to test/ecma/ExecutionContexts/10.1.3-2.js diff --git a/js/test/ecma/ExecutionContexts/10.1.3.js b/test/ecma/ExecutionContexts/10.1.3.js rename from js/test/ecma/ExecutionContexts/10.1.3.js rename to test/ecma/ExecutionContexts/10.1.3.js diff --git a/js/test/ecma/ExecutionContexts/10.1.4-1.js b/test/ecma/ExecutionContexts/10.1.4-1.js rename from js/test/ecma/ExecutionContexts/10.1.4-1.js rename to test/ecma/ExecutionContexts/10.1.4-1.js diff --git a/js/test/ecma/ExecutionContexts/10.1.4-10.js b/test/ecma/ExecutionContexts/10.1.4-10.js rename from js/test/ecma/ExecutionContexts/10.1.4-10.js rename to test/ecma/ExecutionContexts/10.1.4-10.js diff --git a/js/test/ecma/ExecutionContexts/10.1.4-2.js b/test/ecma/ExecutionContexts/10.1.4-2.js rename from js/test/ecma/ExecutionContexts/10.1.4-2.js rename to test/ecma/ExecutionContexts/10.1.4-2.js diff --git a/js/test/ecma/ExecutionContexts/10.1.4-3.js b/test/ecma/ExecutionContexts/10.1.4-3.js rename from js/test/ecma/ExecutionContexts/10.1.4-3.js rename to test/ecma/ExecutionContexts/10.1.4-3.js diff --git a/js/test/ecma/ExecutionContexts/10.1.4-4.js b/test/ecma/ExecutionContexts/10.1.4-4.js rename from js/test/ecma/ExecutionContexts/10.1.4-4.js rename to test/ecma/ExecutionContexts/10.1.4-4.js diff --git a/js/test/ecma/ExecutionContexts/10.1.4-5.js b/test/ecma/ExecutionContexts/10.1.4-5.js rename from js/test/ecma/ExecutionContexts/10.1.4-5.js rename to test/ecma/ExecutionContexts/10.1.4-5.js diff --git a/js/test/ecma/ExecutionContexts/10.1.4-6.js b/test/ecma/ExecutionContexts/10.1.4-6.js rename from js/test/ecma/ExecutionContexts/10.1.4-6.js rename to test/ecma/ExecutionContexts/10.1.4-6.js diff --git a/js/test/ecma/ExecutionContexts/10.1.4-7.js b/test/ecma/ExecutionContexts/10.1.4-7.js rename from js/test/ecma/ExecutionContexts/10.1.4-7.js rename to test/ecma/ExecutionContexts/10.1.4-7.js diff --git a/js/test/ecma/ExecutionContexts/10.1.4-8.js b/test/ecma/ExecutionContexts/10.1.4-8.js rename from js/test/ecma/ExecutionContexts/10.1.4-8.js rename to test/ecma/ExecutionContexts/10.1.4-8.js diff --git a/js/test/ecma/ExecutionContexts/10.1.4-9.js b/test/ecma/ExecutionContexts/10.1.4-9.js rename from js/test/ecma/ExecutionContexts/10.1.4-9.js rename to test/ecma/ExecutionContexts/10.1.4-9.js diff --git a/js/test/ecma/ExecutionContexts/10.1.5-1.js b/test/ecma/ExecutionContexts/10.1.5-1.js rename from js/test/ecma/ExecutionContexts/10.1.5-1.js rename to test/ecma/ExecutionContexts/10.1.5-1.js diff --git a/js/test/ecma/ExecutionContexts/10.1.5-2.js b/test/ecma/ExecutionContexts/10.1.5-2.js rename from js/test/ecma/ExecutionContexts/10.1.5-2.js rename to test/ecma/ExecutionContexts/10.1.5-2.js diff --git a/js/test/ecma/ExecutionContexts/10.1.5-3.js b/test/ecma/ExecutionContexts/10.1.5-3.js rename from js/test/ecma/ExecutionContexts/10.1.5-3.js rename to test/ecma/ExecutionContexts/10.1.5-3.js diff --git a/js/test/ecma/ExecutionContexts/10.1.5-4.js b/test/ecma/ExecutionContexts/10.1.5-4.js rename from js/test/ecma/ExecutionContexts/10.1.5-4.js rename to test/ecma/ExecutionContexts/10.1.5-4.js diff --git a/js/test/ecma/ExecutionContexts/10.1.6.js b/test/ecma/ExecutionContexts/10.1.6.js rename from js/test/ecma/ExecutionContexts/10.1.6.js rename to test/ecma/ExecutionContexts/10.1.6.js diff --git a/js/test/ecma/ExecutionContexts/10.1.8-1.js b/test/ecma/ExecutionContexts/10.1.8-1.js rename from js/test/ecma/ExecutionContexts/10.1.8-1.js rename to test/ecma/ExecutionContexts/10.1.8-1.js diff --git a/js/test/ecma/ExecutionContexts/10.1.8-2.js b/test/ecma/ExecutionContexts/10.1.8-2.js rename from js/test/ecma/ExecutionContexts/10.1.8-2.js rename to test/ecma/ExecutionContexts/10.1.8-2.js diff --git a/js/test/ecma/ExecutionContexts/10.2.1.js b/test/ecma/ExecutionContexts/10.2.1.js rename from js/test/ecma/ExecutionContexts/10.2.1.js rename to test/ecma/ExecutionContexts/10.2.1.js diff --git a/js/test/ecma/ExecutionContexts/10.2.2-1.js b/test/ecma/ExecutionContexts/10.2.2-1.js rename from js/test/ecma/ExecutionContexts/10.2.2-1.js rename to test/ecma/ExecutionContexts/10.2.2-1.js diff --git a/js/test/ecma/ExecutionContexts/10.2.2-2.js b/test/ecma/ExecutionContexts/10.2.2-2.js rename from js/test/ecma/ExecutionContexts/10.2.2-2.js rename to test/ecma/ExecutionContexts/10.2.2-2.js diff --git a/js/test/ecma/ExecutionContexts/10.2.3-1.js b/test/ecma/ExecutionContexts/10.2.3-1.js rename from js/test/ecma/ExecutionContexts/10.2.3-1.js rename to test/ecma/ExecutionContexts/10.2.3-1.js diff --git a/js/test/ecma/ExecutionContexts/10.2.3-2.js b/test/ecma/ExecutionContexts/10.2.3-2.js rename from js/test/ecma/ExecutionContexts/10.2.3-2.js rename to test/ecma/ExecutionContexts/10.2.3-2.js diff --git a/js/test/ecma/ExecutionContexts/browser.js b/test/ecma/ExecutionContexts/browser.js rename from js/test/ecma/ExecutionContexts/browser.js rename to test/ecma/ExecutionContexts/browser.js diff --git a/js/test/ecma/ExecutionContexts/shell.js b/test/ecma/ExecutionContexts/shell.js rename from js/test/ecma/ExecutionContexts/shell.js rename to test/ecma/ExecutionContexts/shell.js diff --git a/js/test/ecma/Expressions/11.1.1.js b/test/ecma/Expressions/11.1.1.js rename from js/test/ecma/Expressions/11.1.1.js rename to test/ecma/Expressions/11.1.1.js diff --git a/js/test/ecma/Expressions/11.10-1.js b/test/ecma/Expressions/11.10-1.js rename from js/test/ecma/Expressions/11.10-1.js rename to test/ecma/Expressions/11.10-1.js diff --git a/js/test/ecma/Expressions/11.10-2.js b/test/ecma/Expressions/11.10-2.js rename from js/test/ecma/Expressions/11.10-2.js rename to test/ecma/Expressions/11.10-2.js diff --git a/js/test/ecma/Expressions/11.10-3.js b/test/ecma/Expressions/11.10-3.js rename from js/test/ecma/Expressions/11.10-3.js rename to test/ecma/Expressions/11.10-3.js diff --git a/js/test/ecma/Expressions/11.12-1.js b/test/ecma/Expressions/11.12-1.js rename from js/test/ecma/Expressions/11.12-1.js rename to test/ecma/Expressions/11.12-1.js diff --git a/js/test/ecma/Expressions/11.12-2-n.js b/test/ecma/Expressions/11.12-2-n.js rename from js/test/ecma/Expressions/11.12-2-n.js rename to test/ecma/Expressions/11.12-2-n.js diff --git a/js/test/ecma/Expressions/11.12-3.js b/test/ecma/Expressions/11.12-3.js rename from js/test/ecma/Expressions/11.12-3.js rename to test/ecma/Expressions/11.12-3.js diff --git a/js/test/ecma/Expressions/11.12-4.js b/test/ecma/Expressions/11.12-4.js rename from js/test/ecma/Expressions/11.12-4.js rename to test/ecma/Expressions/11.12-4.js diff --git a/js/test/ecma/Expressions/11.13.1.js b/test/ecma/Expressions/11.13.1.js rename from js/test/ecma/Expressions/11.13.1.js rename to test/ecma/Expressions/11.13.1.js diff --git a/js/test/ecma/Expressions/11.13.2-1.js b/test/ecma/Expressions/11.13.2-1.js rename from js/test/ecma/Expressions/11.13.2-1.js rename to test/ecma/Expressions/11.13.2-1.js diff --git a/js/test/ecma/Expressions/11.13.2-2.js b/test/ecma/Expressions/11.13.2-2.js rename from js/test/ecma/Expressions/11.13.2-2.js rename to test/ecma/Expressions/11.13.2-2.js diff --git a/js/test/ecma/Expressions/11.13.2-3.js b/test/ecma/Expressions/11.13.2-3.js rename from js/test/ecma/Expressions/11.13.2-3.js rename to test/ecma/Expressions/11.13.2-3.js diff --git a/js/test/ecma/Expressions/11.13.2-4.js b/test/ecma/Expressions/11.13.2-4.js rename from js/test/ecma/Expressions/11.13.2-4.js rename to test/ecma/Expressions/11.13.2-4.js diff --git a/js/test/ecma/Expressions/11.13.2-5.js b/test/ecma/Expressions/11.13.2-5.js rename from js/test/ecma/Expressions/11.13.2-5.js rename to test/ecma/Expressions/11.13.2-5.js diff --git a/js/test/ecma/Expressions/11.13.js b/test/ecma/Expressions/11.13.js rename from js/test/ecma/Expressions/11.13.js rename to test/ecma/Expressions/11.13.js diff --git a/js/test/ecma/Expressions/11.14-1.js b/test/ecma/Expressions/11.14-1.js rename from js/test/ecma/Expressions/11.14-1.js rename to test/ecma/Expressions/11.14-1.js diff --git a/js/test/ecma/Expressions/11.2.1-1.js b/test/ecma/Expressions/11.2.1-1.js rename from js/test/ecma/Expressions/11.2.1-1.js rename to test/ecma/Expressions/11.2.1-1.js diff --git a/js/test/ecma/Expressions/11.2.1-2.js b/test/ecma/Expressions/11.2.1-2.js rename from js/test/ecma/Expressions/11.2.1-2.js rename to test/ecma/Expressions/11.2.1-2.js diff --git a/js/test/ecma/Expressions/11.2.1-3-n.js b/test/ecma/Expressions/11.2.1-3-n.js rename from js/test/ecma/Expressions/11.2.1-3-n.js rename to test/ecma/Expressions/11.2.1-3-n.js diff --git a/js/test/ecma/Expressions/11.2.1-4-n.js b/test/ecma/Expressions/11.2.1-4-n.js rename from js/test/ecma/Expressions/11.2.1-4-n.js rename to test/ecma/Expressions/11.2.1-4-n.js diff --git a/js/test/ecma/Expressions/11.2.1-5.js b/test/ecma/Expressions/11.2.1-5.js rename from js/test/ecma/Expressions/11.2.1-5.js rename to test/ecma/Expressions/11.2.1-5.js diff --git a/js/test/ecma/Expressions/11.2.2-1-n.js b/test/ecma/Expressions/11.2.2-1-n.js rename from js/test/ecma/Expressions/11.2.2-1-n.js rename to test/ecma/Expressions/11.2.2-1-n.js diff --git a/js/test/ecma/Expressions/11.2.2-1.js b/test/ecma/Expressions/11.2.2-1.js rename from js/test/ecma/Expressions/11.2.2-1.js rename to test/ecma/Expressions/11.2.2-1.js diff --git a/js/test/ecma/Expressions/11.2.2-10-n.js b/test/ecma/Expressions/11.2.2-10-n.js rename from js/test/ecma/Expressions/11.2.2-10-n.js rename to test/ecma/Expressions/11.2.2-10-n.js diff --git a/js/test/ecma/Expressions/11.2.2-11.js b/test/ecma/Expressions/11.2.2-11.js rename from js/test/ecma/Expressions/11.2.2-11.js rename to test/ecma/Expressions/11.2.2-11.js diff --git a/js/test/ecma/Expressions/11.2.2-2-n.js b/test/ecma/Expressions/11.2.2-2-n.js rename from js/test/ecma/Expressions/11.2.2-2-n.js rename to test/ecma/Expressions/11.2.2-2-n.js diff --git a/js/test/ecma/Expressions/11.2.2-3-n.js b/test/ecma/Expressions/11.2.2-3-n.js rename from js/test/ecma/Expressions/11.2.2-3-n.js rename to test/ecma/Expressions/11.2.2-3-n.js diff --git a/js/test/ecma/Expressions/11.2.2-4-n.js b/test/ecma/Expressions/11.2.2-4-n.js rename from js/test/ecma/Expressions/11.2.2-4-n.js rename to test/ecma/Expressions/11.2.2-4-n.js diff --git a/js/test/ecma/Expressions/11.2.2-5-n.js b/test/ecma/Expressions/11.2.2-5-n.js rename from js/test/ecma/Expressions/11.2.2-5-n.js rename to test/ecma/Expressions/11.2.2-5-n.js diff --git a/js/test/ecma/Expressions/11.2.2-6-n.js b/test/ecma/Expressions/11.2.2-6-n.js rename from js/test/ecma/Expressions/11.2.2-6-n.js rename to test/ecma/Expressions/11.2.2-6-n.js diff --git a/js/test/ecma/Expressions/11.2.2-7-n.js b/test/ecma/Expressions/11.2.2-7-n.js rename from js/test/ecma/Expressions/11.2.2-7-n.js rename to test/ecma/Expressions/11.2.2-7-n.js diff --git a/js/test/ecma/Expressions/11.2.2-8-n.js b/test/ecma/Expressions/11.2.2-8-n.js rename from js/test/ecma/Expressions/11.2.2-8-n.js rename to test/ecma/Expressions/11.2.2-8-n.js diff --git a/js/test/ecma/Expressions/11.2.2-9-n.js b/test/ecma/Expressions/11.2.2-9-n.js rename from js/test/ecma/Expressions/11.2.2-9-n.js rename to test/ecma/Expressions/11.2.2-9-n.js diff --git a/js/test/ecma/Expressions/11.2.3-1.js b/test/ecma/Expressions/11.2.3-1.js rename from js/test/ecma/Expressions/11.2.3-1.js rename to test/ecma/Expressions/11.2.3-1.js diff --git a/js/test/ecma/Expressions/11.2.3-2-n.js b/test/ecma/Expressions/11.2.3-2-n.js rename from js/test/ecma/Expressions/11.2.3-2-n.js rename to test/ecma/Expressions/11.2.3-2-n.js diff --git a/js/test/ecma/Expressions/11.2.3-3-n.js b/test/ecma/Expressions/11.2.3-3-n.js rename from js/test/ecma/Expressions/11.2.3-3-n.js rename to test/ecma/Expressions/11.2.3-3-n.js diff --git a/js/test/ecma/Expressions/11.2.3-4-n.js b/test/ecma/Expressions/11.2.3-4-n.js rename from js/test/ecma/Expressions/11.2.3-4-n.js rename to test/ecma/Expressions/11.2.3-4-n.js diff --git a/js/test/ecma/Expressions/11.2.3-5.js b/test/ecma/Expressions/11.2.3-5.js rename from js/test/ecma/Expressions/11.2.3-5.js rename to test/ecma/Expressions/11.2.3-5.js diff --git a/js/test/ecma/Expressions/11.3.1.js b/test/ecma/Expressions/11.3.1.js rename from js/test/ecma/Expressions/11.3.1.js rename to test/ecma/Expressions/11.3.1.js diff --git a/js/test/ecma/Expressions/11.3.2.js b/test/ecma/Expressions/11.3.2.js rename from js/test/ecma/Expressions/11.3.2.js rename to test/ecma/Expressions/11.3.2.js diff --git a/js/test/ecma/Expressions/11.4.1.js b/test/ecma/Expressions/11.4.1.js rename from js/test/ecma/Expressions/11.4.1.js rename to test/ecma/Expressions/11.4.1.js diff --git a/js/test/ecma/Expressions/11.4.2.js b/test/ecma/Expressions/11.4.2.js rename from js/test/ecma/Expressions/11.4.2.js rename to test/ecma/Expressions/11.4.2.js diff --git a/js/test/ecma/Expressions/11.4.3.js b/test/ecma/Expressions/11.4.3.js rename from js/test/ecma/Expressions/11.4.3.js rename to test/ecma/Expressions/11.4.3.js diff --git a/js/test/ecma/Expressions/11.4.4.js b/test/ecma/Expressions/11.4.4.js rename from js/test/ecma/Expressions/11.4.4.js rename to test/ecma/Expressions/11.4.4.js diff --git a/js/test/ecma/Expressions/11.4.5.js b/test/ecma/Expressions/11.4.5.js rename from js/test/ecma/Expressions/11.4.5.js rename to test/ecma/Expressions/11.4.5.js diff --git a/js/test/ecma/Expressions/11.4.6.js b/test/ecma/Expressions/11.4.6.js rename from js/test/ecma/Expressions/11.4.6.js rename to test/ecma/Expressions/11.4.6.js diff --git a/js/test/ecma/Expressions/11.4.8.js b/test/ecma/Expressions/11.4.8.js rename from js/test/ecma/Expressions/11.4.8.js rename to test/ecma/Expressions/11.4.8.js diff --git a/js/test/ecma/Expressions/11.4.9.js b/test/ecma/Expressions/11.4.9.js rename from js/test/ecma/Expressions/11.4.9.js rename to test/ecma/Expressions/11.4.9.js diff --git a/js/test/ecma/Expressions/11.5.1.js b/test/ecma/Expressions/11.5.1.js rename from js/test/ecma/Expressions/11.5.1.js rename to test/ecma/Expressions/11.5.1.js diff --git a/js/test/ecma/Expressions/11.5.2.js b/test/ecma/Expressions/11.5.2.js rename from js/test/ecma/Expressions/11.5.2.js rename to test/ecma/Expressions/11.5.2.js diff --git a/js/test/ecma/Expressions/11.5.3.js b/test/ecma/Expressions/11.5.3.js rename from js/test/ecma/Expressions/11.5.3.js rename to test/ecma/Expressions/11.5.3.js diff --git a/js/test/ecma/Expressions/11.6.1-1.js b/test/ecma/Expressions/11.6.1-1.js rename from js/test/ecma/Expressions/11.6.1-1.js rename to test/ecma/Expressions/11.6.1-1.js diff --git a/js/test/ecma/Expressions/11.6.1-2.js b/test/ecma/Expressions/11.6.1-2.js rename from js/test/ecma/Expressions/11.6.1-2.js rename to test/ecma/Expressions/11.6.1-2.js diff --git a/js/test/ecma/Expressions/11.6.1-3.js b/test/ecma/Expressions/11.6.1-3.js rename from js/test/ecma/Expressions/11.6.1-3.js rename to test/ecma/Expressions/11.6.1-3.js diff --git a/js/test/ecma/Expressions/11.6.2-1.js b/test/ecma/Expressions/11.6.2-1.js rename from js/test/ecma/Expressions/11.6.2-1.js rename to test/ecma/Expressions/11.6.2-1.js diff --git a/js/test/ecma/Expressions/11.6.3.js b/test/ecma/Expressions/11.6.3.js rename from js/test/ecma/Expressions/11.6.3.js rename to test/ecma/Expressions/11.6.3.js diff --git a/js/test/ecma/Expressions/11.7.1.js b/test/ecma/Expressions/11.7.1.js rename from js/test/ecma/Expressions/11.7.1.js rename to test/ecma/Expressions/11.7.1.js diff --git a/js/test/ecma/Expressions/11.7.2.js b/test/ecma/Expressions/11.7.2.js rename from js/test/ecma/Expressions/11.7.2.js rename to test/ecma/Expressions/11.7.2.js diff --git a/js/test/ecma/Expressions/11.7.3.js b/test/ecma/Expressions/11.7.3.js rename from js/test/ecma/Expressions/11.7.3.js rename to test/ecma/Expressions/11.7.3.js diff --git a/js/test/ecma/Expressions/11.8.1.js b/test/ecma/Expressions/11.8.1.js rename from js/test/ecma/Expressions/11.8.1.js rename to test/ecma/Expressions/11.8.1.js diff --git a/js/test/ecma/Expressions/11.8.2.js b/test/ecma/Expressions/11.8.2.js rename from js/test/ecma/Expressions/11.8.2.js rename to test/ecma/Expressions/11.8.2.js diff --git a/js/test/ecma/Expressions/11.8.3.js b/test/ecma/Expressions/11.8.3.js rename from js/test/ecma/Expressions/11.8.3.js rename to test/ecma/Expressions/11.8.3.js diff --git a/js/test/ecma/Expressions/11.8.4.js b/test/ecma/Expressions/11.8.4.js rename from js/test/ecma/Expressions/11.8.4.js rename to test/ecma/Expressions/11.8.4.js diff --git a/js/test/ecma/Expressions/11.9.1.js b/test/ecma/Expressions/11.9.1.js rename from js/test/ecma/Expressions/11.9.1.js rename to test/ecma/Expressions/11.9.1.js diff --git a/js/test/ecma/Expressions/11.9.2.js b/test/ecma/Expressions/11.9.2.js rename from js/test/ecma/Expressions/11.9.2.js rename to test/ecma/Expressions/11.9.2.js diff --git a/js/test/ecma/Expressions/11.9.3.js b/test/ecma/Expressions/11.9.3.js rename from js/test/ecma/Expressions/11.9.3.js rename to test/ecma/Expressions/11.9.3.js diff --git a/js/test/ecma/Expressions/browser.js b/test/ecma/Expressions/browser.js rename from js/test/ecma/Expressions/browser.js rename to test/ecma/Expressions/browser.js diff --git a/js/test/ecma/Expressions/shell.js b/test/ecma/Expressions/shell.js rename from js/test/ecma/Expressions/shell.js rename to test/ecma/Expressions/shell.js diff --git a/js/test/ecma/FunctionObjects/15.3.1.1-1.js b/test/ecma/FunctionObjects/15.3.1.1-1.js rename from js/test/ecma/FunctionObjects/15.3.1.1-1.js rename to test/ecma/FunctionObjects/15.3.1.1-1.js diff --git a/js/test/ecma/FunctionObjects/15.3.1.1-2.js b/test/ecma/FunctionObjects/15.3.1.1-2.js rename from js/test/ecma/FunctionObjects/15.3.1.1-2.js rename to test/ecma/FunctionObjects/15.3.1.1-2.js diff --git a/js/test/ecma/FunctionObjects/15.3.1.1-3.js b/test/ecma/FunctionObjects/15.3.1.1-3.js rename from js/test/ecma/FunctionObjects/15.3.1.1-3.js rename to test/ecma/FunctionObjects/15.3.1.1-3.js diff --git a/js/test/ecma/FunctionObjects/15.3.2.1-1.js b/test/ecma/FunctionObjects/15.3.2.1-1.js rename from js/test/ecma/FunctionObjects/15.3.2.1-1.js rename to test/ecma/FunctionObjects/15.3.2.1-1.js diff --git a/js/test/ecma/FunctionObjects/15.3.2.1-2.js b/test/ecma/FunctionObjects/15.3.2.1-2.js rename from js/test/ecma/FunctionObjects/15.3.2.1-2.js rename to test/ecma/FunctionObjects/15.3.2.1-2.js diff --git a/js/test/ecma/FunctionObjects/15.3.2.1-3.js b/test/ecma/FunctionObjects/15.3.2.1-3.js rename from js/test/ecma/FunctionObjects/15.3.2.1-3.js rename to test/ecma/FunctionObjects/15.3.2.1-3.js diff --git a/js/test/ecma/FunctionObjects/15.3.3.1-1.js b/test/ecma/FunctionObjects/15.3.3.1-1.js rename from js/test/ecma/FunctionObjects/15.3.3.1-1.js rename to test/ecma/FunctionObjects/15.3.3.1-1.js diff --git a/js/test/ecma/FunctionObjects/15.3.3.1-2.js b/test/ecma/FunctionObjects/15.3.3.1-2.js rename from js/test/ecma/FunctionObjects/15.3.3.1-2.js rename to test/ecma/FunctionObjects/15.3.3.1-2.js diff --git a/js/test/ecma/FunctionObjects/15.3.3.1-3.js b/test/ecma/FunctionObjects/15.3.3.1-3.js rename from js/test/ecma/FunctionObjects/15.3.3.1-3.js rename to test/ecma/FunctionObjects/15.3.3.1-3.js diff --git a/js/test/ecma/FunctionObjects/15.3.3.1-4.js b/test/ecma/FunctionObjects/15.3.3.1-4.js rename from js/test/ecma/FunctionObjects/15.3.3.1-4.js rename to test/ecma/FunctionObjects/15.3.3.1-4.js diff --git a/js/test/ecma/FunctionObjects/15.3.3.2.js b/test/ecma/FunctionObjects/15.3.3.2.js rename from js/test/ecma/FunctionObjects/15.3.3.2.js rename to test/ecma/FunctionObjects/15.3.3.2.js diff --git a/js/test/ecma/FunctionObjects/15.3.4-1.js b/test/ecma/FunctionObjects/15.3.4-1.js rename from js/test/ecma/FunctionObjects/15.3.4-1.js rename to test/ecma/FunctionObjects/15.3.4-1.js diff --git a/js/test/ecma/FunctionObjects/15.3.4.1.js b/test/ecma/FunctionObjects/15.3.4.1.js rename from js/test/ecma/FunctionObjects/15.3.4.1.js rename to test/ecma/FunctionObjects/15.3.4.1.js diff --git a/js/test/ecma/FunctionObjects/15.3.4.js b/test/ecma/FunctionObjects/15.3.4.js rename from js/test/ecma/FunctionObjects/15.3.4.js rename to test/ecma/FunctionObjects/15.3.4.js diff --git a/js/test/ecma/FunctionObjects/15.3.5-1.js b/test/ecma/FunctionObjects/15.3.5-1.js rename from js/test/ecma/FunctionObjects/15.3.5-1.js rename to test/ecma/FunctionObjects/15.3.5-1.js diff --git a/js/test/ecma/FunctionObjects/15.3.5-2.js b/test/ecma/FunctionObjects/15.3.5-2.js rename from js/test/ecma/FunctionObjects/15.3.5-2.js rename to test/ecma/FunctionObjects/15.3.5-2.js diff --git a/js/test/ecma/FunctionObjects/15.3.5.1.js b/test/ecma/FunctionObjects/15.3.5.1.js rename from js/test/ecma/FunctionObjects/15.3.5.1.js rename to test/ecma/FunctionObjects/15.3.5.1.js diff --git a/js/test/ecma/FunctionObjects/15.3.5.3.js b/test/ecma/FunctionObjects/15.3.5.3.js rename from js/test/ecma/FunctionObjects/15.3.5.3.js rename to test/ecma/FunctionObjects/15.3.5.3.js diff --git a/js/test/ecma/FunctionObjects/browser.js b/test/ecma/FunctionObjects/browser.js rename from js/test/ecma/FunctionObjects/browser.js rename to test/ecma/FunctionObjects/browser.js diff --git a/js/test/ecma/FunctionObjects/shell.js b/test/ecma/FunctionObjects/shell.js rename from js/test/ecma/FunctionObjects/shell.js rename to test/ecma/FunctionObjects/shell.js diff --git a/js/test/ecma/GlobalObject/15.1-1-n.js b/test/ecma/GlobalObject/15.1-1-n.js rename from js/test/ecma/GlobalObject/15.1-1-n.js rename to test/ecma/GlobalObject/15.1-1-n.js diff --git a/js/test/ecma/GlobalObject/15.1-2-n.js b/test/ecma/GlobalObject/15.1-2-n.js rename from js/test/ecma/GlobalObject/15.1-2-n.js rename to test/ecma/GlobalObject/15.1-2-n.js diff --git a/js/test/ecma/GlobalObject/15.1.1.1.js b/test/ecma/GlobalObject/15.1.1.1.js rename from js/test/ecma/GlobalObject/15.1.1.1.js rename to test/ecma/GlobalObject/15.1.1.1.js diff --git a/js/test/ecma/GlobalObject/15.1.1.2.js b/test/ecma/GlobalObject/15.1.1.2.js rename from js/test/ecma/GlobalObject/15.1.1.2.js rename to test/ecma/GlobalObject/15.1.1.2.js diff --git a/js/test/ecma/GlobalObject/15.1.2.1-1.js b/test/ecma/GlobalObject/15.1.2.1-1.js rename from js/test/ecma/GlobalObject/15.1.2.1-1.js rename to test/ecma/GlobalObject/15.1.2.1-1.js diff --git a/js/test/ecma/GlobalObject/15.1.2.1-2.js b/test/ecma/GlobalObject/15.1.2.1-2.js rename from js/test/ecma/GlobalObject/15.1.2.1-2.js rename to test/ecma/GlobalObject/15.1.2.1-2.js diff --git a/js/test/ecma/GlobalObject/15.1.2.2-1.js b/test/ecma/GlobalObject/15.1.2.2-1.js rename from js/test/ecma/GlobalObject/15.1.2.2-1.js rename to test/ecma/GlobalObject/15.1.2.2-1.js diff --git a/js/test/ecma/GlobalObject/15.1.2.2-2.js b/test/ecma/GlobalObject/15.1.2.2-2.js rename from js/test/ecma/GlobalObject/15.1.2.2-2.js rename to test/ecma/GlobalObject/15.1.2.2-2.js diff --git a/js/test/ecma/GlobalObject/15.1.2.3-1.js b/test/ecma/GlobalObject/15.1.2.3-1.js rename from js/test/ecma/GlobalObject/15.1.2.3-1.js rename to test/ecma/GlobalObject/15.1.2.3-1.js diff --git a/js/test/ecma/GlobalObject/15.1.2.3-2.js b/test/ecma/GlobalObject/15.1.2.3-2.js rename from js/test/ecma/GlobalObject/15.1.2.3-2.js rename to test/ecma/GlobalObject/15.1.2.3-2.js diff --git a/js/test/ecma/GlobalObject/15.1.2.4.js b/test/ecma/GlobalObject/15.1.2.4.js rename from js/test/ecma/GlobalObject/15.1.2.4.js rename to test/ecma/GlobalObject/15.1.2.4.js diff --git a/js/test/ecma/GlobalObject/15.1.2.5-1.js b/test/ecma/GlobalObject/15.1.2.5-1.js rename from js/test/ecma/GlobalObject/15.1.2.5-1.js rename to test/ecma/GlobalObject/15.1.2.5-1.js diff --git a/js/test/ecma/GlobalObject/15.1.2.5-2.js b/test/ecma/GlobalObject/15.1.2.5-2.js rename from js/test/ecma/GlobalObject/15.1.2.5-2.js rename to test/ecma/GlobalObject/15.1.2.5-2.js diff --git a/js/test/ecma/GlobalObject/15.1.2.5-3.js b/test/ecma/GlobalObject/15.1.2.5-3.js rename from js/test/ecma/GlobalObject/15.1.2.5-3.js rename to test/ecma/GlobalObject/15.1.2.5-3.js diff --git a/js/test/ecma/GlobalObject/15.1.2.6.js b/test/ecma/GlobalObject/15.1.2.6.js rename from js/test/ecma/GlobalObject/15.1.2.6.js rename to test/ecma/GlobalObject/15.1.2.6.js diff --git a/js/test/ecma/GlobalObject/15.1.2.7.js b/test/ecma/GlobalObject/15.1.2.7.js rename from js/test/ecma/GlobalObject/15.1.2.7.js rename to test/ecma/GlobalObject/15.1.2.7.js diff --git a/js/test/ecma/GlobalObject/browser.js b/test/ecma/GlobalObject/browser.js rename from js/test/ecma/GlobalObject/browser.js rename to test/ecma/GlobalObject/browser.js diff --git a/js/test/ecma/GlobalObject/shell.js b/test/ecma/GlobalObject/shell.js rename from js/test/ecma/GlobalObject/shell.js rename to test/ecma/GlobalObject/shell.js diff --git a/js/test/ecma/LexicalConventions/7.1-1.js b/test/ecma/LexicalConventions/7.1-1.js rename from js/test/ecma/LexicalConventions/7.1-1.js rename to test/ecma/LexicalConventions/7.1-1.js diff --git a/js/test/ecma/LexicalConventions/7.1-2.js b/test/ecma/LexicalConventions/7.1-2.js rename from js/test/ecma/LexicalConventions/7.1-2.js rename to test/ecma/LexicalConventions/7.1-2.js diff --git a/js/test/ecma/LexicalConventions/7.1-3.js b/test/ecma/LexicalConventions/7.1-3.js rename from js/test/ecma/LexicalConventions/7.1-3.js rename to test/ecma/LexicalConventions/7.1-3.js diff --git a/js/test/ecma/LexicalConventions/7.2-1.js b/test/ecma/LexicalConventions/7.2-1.js rename from js/test/ecma/LexicalConventions/7.2-1.js rename to test/ecma/LexicalConventions/7.2-1.js diff --git a/js/test/ecma/LexicalConventions/7.2-2-n.js b/test/ecma/LexicalConventions/7.2-2-n.js rename from js/test/ecma/LexicalConventions/7.2-2-n.js rename to test/ecma/LexicalConventions/7.2-2-n.js diff --git a/js/test/ecma/LexicalConventions/7.2-3-n.js b/test/ecma/LexicalConventions/7.2-3-n.js rename from js/test/ecma/LexicalConventions/7.2-3-n.js rename to test/ecma/LexicalConventions/7.2-3-n.js diff --git a/js/test/ecma/LexicalConventions/7.2-4-n.js b/test/ecma/LexicalConventions/7.2-4-n.js rename from js/test/ecma/LexicalConventions/7.2-4-n.js rename to test/ecma/LexicalConventions/7.2-4-n.js diff --git a/js/test/ecma/LexicalConventions/7.2-5-n.js b/test/ecma/LexicalConventions/7.2-5-n.js rename from js/test/ecma/LexicalConventions/7.2-5-n.js rename to test/ecma/LexicalConventions/7.2-5-n.js diff --git a/js/test/ecma/LexicalConventions/7.2-6.js b/test/ecma/LexicalConventions/7.2-6.js rename from js/test/ecma/LexicalConventions/7.2-6.js rename to test/ecma/LexicalConventions/7.2-6.js diff --git a/js/test/ecma/LexicalConventions/7.3-1.js b/test/ecma/LexicalConventions/7.3-1.js rename from js/test/ecma/LexicalConventions/7.3-1.js rename to test/ecma/LexicalConventions/7.3-1.js diff --git a/js/test/ecma/LexicalConventions/7.3-10.js b/test/ecma/LexicalConventions/7.3-10.js rename from js/test/ecma/LexicalConventions/7.3-10.js rename to test/ecma/LexicalConventions/7.3-10.js diff --git a/js/test/ecma/LexicalConventions/7.3-11.js b/test/ecma/LexicalConventions/7.3-11.js rename from js/test/ecma/LexicalConventions/7.3-11.js rename to test/ecma/LexicalConventions/7.3-11.js diff --git a/js/test/ecma/LexicalConventions/7.3-12.js b/test/ecma/LexicalConventions/7.3-12.js rename from js/test/ecma/LexicalConventions/7.3-12.js rename to test/ecma/LexicalConventions/7.3-12.js diff --git a/js/test/ecma/LexicalConventions/7.3-13-n.js b/test/ecma/LexicalConventions/7.3-13-n.js rename from js/test/ecma/LexicalConventions/7.3-13-n.js rename to test/ecma/LexicalConventions/7.3-13-n.js diff --git a/js/test/ecma/LexicalConventions/7.3-2.js b/test/ecma/LexicalConventions/7.3-2.js rename from js/test/ecma/LexicalConventions/7.3-2.js rename to test/ecma/LexicalConventions/7.3-2.js diff --git a/js/test/ecma/LexicalConventions/7.3-3.js b/test/ecma/LexicalConventions/7.3-3.js rename from js/test/ecma/LexicalConventions/7.3-3.js rename to test/ecma/LexicalConventions/7.3-3.js diff --git a/js/test/ecma/LexicalConventions/7.3-4.js b/test/ecma/LexicalConventions/7.3-4.js rename from js/test/ecma/LexicalConventions/7.3-4.js rename to test/ecma/LexicalConventions/7.3-4.js diff --git a/js/test/ecma/LexicalConventions/7.3-5.js b/test/ecma/LexicalConventions/7.3-5.js rename from js/test/ecma/LexicalConventions/7.3-5.js rename to test/ecma/LexicalConventions/7.3-5.js diff --git a/js/test/ecma/LexicalConventions/7.3-6.js b/test/ecma/LexicalConventions/7.3-6.js rename from js/test/ecma/LexicalConventions/7.3-6.js rename to test/ecma/LexicalConventions/7.3-6.js diff --git a/js/test/ecma/LexicalConventions/7.3-7.js b/test/ecma/LexicalConventions/7.3-7.js rename from js/test/ecma/LexicalConventions/7.3-7.js rename to test/ecma/LexicalConventions/7.3-7.js diff --git a/js/test/ecma/LexicalConventions/7.3-8.js b/test/ecma/LexicalConventions/7.3-8.js rename from js/test/ecma/LexicalConventions/7.3-8.js rename to test/ecma/LexicalConventions/7.3-8.js diff --git a/js/test/ecma/LexicalConventions/7.3-9.js b/test/ecma/LexicalConventions/7.3-9.js rename from js/test/ecma/LexicalConventions/7.3-9.js rename to test/ecma/LexicalConventions/7.3-9.js diff --git a/js/test/ecma/LexicalConventions/7.4.1-1-n.js b/test/ecma/LexicalConventions/7.4.1-1-n.js rename from js/test/ecma/LexicalConventions/7.4.1-1-n.js rename to test/ecma/LexicalConventions/7.4.1-1-n.js diff --git a/js/test/ecma/LexicalConventions/7.4.1-2-n.js b/test/ecma/LexicalConventions/7.4.1-2-n.js rename from js/test/ecma/LexicalConventions/7.4.1-2-n.js rename to test/ecma/LexicalConventions/7.4.1-2-n.js diff --git a/js/test/ecma/LexicalConventions/7.4.1-3-n.js b/test/ecma/LexicalConventions/7.4.1-3-n.js rename from js/test/ecma/LexicalConventions/7.4.1-3-n.js rename to test/ecma/LexicalConventions/7.4.1-3-n.js diff --git a/js/test/ecma/LexicalConventions/7.4.2-1-n.js b/test/ecma/LexicalConventions/7.4.2-1-n.js rename from js/test/ecma/LexicalConventions/7.4.2-1-n.js rename to test/ecma/LexicalConventions/7.4.2-1-n.js diff --git a/js/test/ecma/LexicalConventions/7.4.2-10-n.js b/test/ecma/LexicalConventions/7.4.2-10-n.js rename from js/test/ecma/LexicalConventions/7.4.2-10-n.js rename to test/ecma/LexicalConventions/7.4.2-10-n.js diff --git a/js/test/ecma/LexicalConventions/7.4.2-11-n.js b/test/ecma/LexicalConventions/7.4.2-11-n.js rename from js/test/ecma/LexicalConventions/7.4.2-11-n.js rename to test/ecma/LexicalConventions/7.4.2-11-n.js diff --git a/js/test/ecma/LexicalConventions/7.4.2-12-n.js b/test/ecma/LexicalConventions/7.4.2-12-n.js rename from js/test/ecma/LexicalConventions/7.4.2-12-n.js rename to test/ecma/LexicalConventions/7.4.2-12-n.js diff --git a/js/test/ecma/LexicalConventions/7.4.2-13-n.js b/test/ecma/LexicalConventions/7.4.2-13-n.js rename from js/test/ecma/LexicalConventions/7.4.2-13-n.js rename to test/ecma/LexicalConventions/7.4.2-13-n.js diff --git a/js/test/ecma/LexicalConventions/7.4.2-14-n.js b/test/ecma/LexicalConventions/7.4.2-14-n.js rename from js/test/ecma/LexicalConventions/7.4.2-14-n.js rename to test/ecma/LexicalConventions/7.4.2-14-n.js diff --git a/js/test/ecma/LexicalConventions/7.4.2-15-n.js b/test/ecma/LexicalConventions/7.4.2-15-n.js rename from js/test/ecma/LexicalConventions/7.4.2-15-n.js rename to test/ecma/LexicalConventions/7.4.2-15-n.js diff --git a/js/test/ecma/LexicalConventions/7.4.2-16-n.js b/test/ecma/LexicalConventions/7.4.2-16-n.js rename from js/test/ecma/LexicalConventions/7.4.2-16-n.js rename to test/ecma/LexicalConventions/7.4.2-16-n.js diff --git a/js/test/ecma/LexicalConventions/7.4.2-2-n.js b/test/ecma/LexicalConventions/7.4.2-2-n.js rename from js/test/ecma/LexicalConventions/7.4.2-2-n.js rename to test/ecma/LexicalConventions/7.4.2-2-n.js diff --git a/js/test/ecma/LexicalConventions/7.4.2-3-n.js b/test/ecma/LexicalConventions/7.4.2-3-n.js rename from js/test/ecma/LexicalConventions/7.4.2-3-n.js rename to test/ecma/LexicalConventions/7.4.2-3-n.js diff --git a/js/test/ecma/LexicalConventions/7.4.2-4-n.js b/test/ecma/LexicalConventions/7.4.2-4-n.js rename from js/test/ecma/LexicalConventions/7.4.2-4-n.js rename to test/ecma/LexicalConventions/7.4.2-4-n.js diff --git a/js/test/ecma/LexicalConventions/7.4.2-5-n.js b/test/ecma/LexicalConventions/7.4.2-5-n.js rename from js/test/ecma/LexicalConventions/7.4.2-5-n.js rename to test/ecma/LexicalConventions/7.4.2-5-n.js diff --git a/js/test/ecma/LexicalConventions/7.4.2-6-n.js b/test/ecma/LexicalConventions/7.4.2-6-n.js rename from js/test/ecma/LexicalConventions/7.4.2-6-n.js rename to test/ecma/LexicalConventions/7.4.2-6-n.js diff --git a/js/test/ecma/LexicalConventions/7.4.2-7-n.js b/test/ecma/LexicalConventions/7.4.2-7-n.js rename from js/test/ecma/LexicalConventions/7.4.2-7-n.js rename to test/ecma/LexicalConventions/7.4.2-7-n.js diff --git a/js/test/ecma/LexicalConventions/7.4.2-8-n.js b/test/ecma/LexicalConventions/7.4.2-8-n.js rename from js/test/ecma/LexicalConventions/7.4.2-8-n.js rename to test/ecma/LexicalConventions/7.4.2-8-n.js diff --git a/js/test/ecma/LexicalConventions/7.4.2-9-n.js b/test/ecma/LexicalConventions/7.4.2-9-n.js rename from js/test/ecma/LexicalConventions/7.4.2-9-n.js rename to test/ecma/LexicalConventions/7.4.2-9-n.js diff --git a/js/test/ecma/LexicalConventions/7.4.3-1-n.js b/test/ecma/LexicalConventions/7.4.3-1-n.js rename from js/test/ecma/LexicalConventions/7.4.3-1-n.js rename to test/ecma/LexicalConventions/7.4.3-1-n.js diff --git a/js/test/ecma/LexicalConventions/7.4.3-10-n.js b/test/ecma/LexicalConventions/7.4.3-10-n.js rename from js/test/ecma/LexicalConventions/7.4.3-10-n.js rename to test/ecma/LexicalConventions/7.4.3-10-n.js diff --git a/js/test/ecma/LexicalConventions/7.4.3-11-n.js b/test/ecma/LexicalConventions/7.4.3-11-n.js rename from js/test/ecma/LexicalConventions/7.4.3-11-n.js rename to test/ecma/LexicalConventions/7.4.3-11-n.js diff --git a/js/test/ecma/LexicalConventions/7.4.3-12-n.js b/test/ecma/LexicalConventions/7.4.3-12-n.js rename from js/test/ecma/LexicalConventions/7.4.3-12-n.js rename to test/ecma/LexicalConventions/7.4.3-12-n.js diff --git a/js/test/ecma/LexicalConventions/7.4.3-13-n.js b/test/ecma/LexicalConventions/7.4.3-13-n.js rename from js/test/ecma/LexicalConventions/7.4.3-13-n.js rename to test/ecma/LexicalConventions/7.4.3-13-n.js diff --git a/js/test/ecma/LexicalConventions/7.4.3-14-n.js b/test/ecma/LexicalConventions/7.4.3-14-n.js rename from js/test/ecma/LexicalConventions/7.4.3-14-n.js rename to test/ecma/LexicalConventions/7.4.3-14-n.js diff --git a/js/test/ecma/LexicalConventions/7.4.3-15-n.js b/test/ecma/LexicalConventions/7.4.3-15-n.js rename from js/test/ecma/LexicalConventions/7.4.3-15-n.js rename to test/ecma/LexicalConventions/7.4.3-15-n.js diff --git a/js/test/ecma/LexicalConventions/7.4.3-16-n.js b/test/ecma/LexicalConventions/7.4.3-16-n.js rename from js/test/ecma/LexicalConventions/7.4.3-16-n.js rename to test/ecma/LexicalConventions/7.4.3-16-n.js diff --git a/js/test/ecma/LexicalConventions/7.4.3-2-n.js b/test/ecma/LexicalConventions/7.4.3-2-n.js rename from js/test/ecma/LexicalConventions/7.4.3-2-n.js rename to test/ecma/LexicalConventions/7.4.3-2-n.js diff --git a/js/test/ecma/LexicalConventions/7.4.3-3-n.js b/test/ecma/LexicalConventions/7.4.3-3-n.js rename from js/test/ecma/LexicalConventions/7.4.3-3-n.js rename to test/ecma/LexicalConventions/7.4.3-3-n.js diff --git a/js/test/ecma/LexicalConventions/7.4.3-4-n.js b/test/ecma/LexicalConventions/7.4.3-4-n.js rename from js/test/ecma/LexicalConventions/7.4.3-4-n.js rename to test/ecma/LexicalConventions/7.4.3-4-n.js diff --git a/js/test/ecma/LexicalConventions/7.4.3-5-n.js b/test/ecma/LexicalConventions/7.4.3-5-n.js rename from js/test/ecma/LexicalConventions/7.4.3-5-n.js rename to test/ecma/LexicalConventions/7.4.3-5-n.js diff --git a/js/test/ecma/LexicalConventions/7.4.3-6-n.js b/test/ecma/LexicalConventions/7.4.3-6-n.js rename from js/test/ecma/LexicalConventions/7.4.3-6-n.js rename to test/ecma/LexicalConventions/7.4.3-6-n.js diff --git a/js/test/ecma/LexicalConventions/7.4.3-7-n.js b/test/ecma/LexicalConventions/7.4.3-7-n.js rename from js/test/ecma/LexicalConventions/7.4.3-7-n.js rename to test/ecma/LexicalConventions/7.4.3-7-n.js diff --git a/js/test/ecma/LexicalConventions/7.4.3-8-n.js b/test/ecma/LexicalConventions/7.4.3-8-n.js rename from js/test/ecma/LexicalConventions/7.4.3-8-n.js rename to test/ecma/LexicalConventions/7.4.3-8-n.js diff --git a/js/test/ecma/LexicalConventions/7.4.3-9-n.js b/test/ecma/LexicalConventions/7.4.3-9-n.js rename from js/test/ecma/LexicalConventions/7.4.3-9-n.js rename to test/ecma/LexicalConventions/7.4.3-9-n.js diff --git a/js/test/ecma/LexicalConventions/7.5-1.js b/test/ecma/LexicalConventions/7.5-1.js rename from js/test/ecma/LexicalConventions/7.5-1.js rename to test/ecma/LexicalConventions/7.5-1.js diff --git a/js/test/ecma/LexicalConventions/7.5-10-n.js b/test/ecma/LexicalConventions/7.5-10-n.js rename from js/test/ecma/LexicalConventions/7.5-10-n.js rename to test/ecma/LexicalConventions/7.5-10-n.js diff --git a/js/test/ecma/LexicalConventions/7.5-2-n.js b/test/ecma/LexicalConventions/7.5-2-n.js rename from js/test/ecma/LexicalConventions/7.5-2-n.js rename to test/ecma/LexicalConventions/7.5-2-n.js diff --git a/js/test/ecma/LexicalConventions/7.5-3-n.js b/test/ecma/LexicalConventions/7.5-3-n.js rename from js/test/ecma/LexicalConventions/7.5-3-n.js rename to test/ecma/LexicalConventions/7.5-3-n.js diff --git a/js/test/ecma/LexicalConventions/7.5-4-n.js b/test/ecma/LexicalConventions/7.5-4-n.js rename from js/test/ecma/LexicalConventions/7.5-4-n.js rename to test/ecma/LexicalConventions/7.5-4-n.js diff --git a/js/test/ecma/LexicalConventions/7.5-5-n.js b/test/ecma/LexicalConventions/7.5-5-n.js rename from js/test/ecma/LexicalConventions/7.5-5-n.js rename to test/ecma/LexicalConventions/7.5-5-n.js diff --git a/js/test/ecma/LexicalConventions/7.5-6.js b/test/ecma/LexicalConventions/7.5-6.js rename from js/test/ecma/LexicalConventions/7.5-6.js rename to test/ecma/LexicalConventions/7.5-6.js diff --git a/js/test/ecma/LexicalConventions/7.5-7.js b/test/ecma/LexicalConventions/7.5-7.js rename from js/test/ecma/LexicalConventions/7.5-7.js rename to test/ecma/LexicalConventions/7.5-7.js diff --git a/js/test/ecma/LexicalConventions/7.5-8-n.js b/test/ecma/LexicalConventions/7.5-8-n.js rename from js/test/ecma/LexicalConventions/7.5-8-n.js rename to test/ecma/LexicalConventions/7.5-8-n.js diff --git a/js/test/ecma/LexicalConventions/7.5-9-n.js b/test/ecma/LexicalConventions/7.5-9-n.js rename from js/test/ecma/LexicalConventions/7.5-9-n.js rename to test/ecma/LexicalConventions/7.5-9-n.js diff --git a/js/test/ecma/LexicalConventions/7.6.js b/test/ecma/LexicalConventions/7.6.js rename from js/test/ecma/LexicalConventions/7.6.js rename to test/ecma/LexicalConventions/7.6.js diff --git a/js/test/ecma/LexicalConventions/7.7.1.js b/test/ecma/LexicalConventions/7.7.1.js rename from js/test/ecma/LexicalConventions/7.7.1.js rename to test/ecma/LexicalConventions/7.7.1.js diff --git a/js/test/ecma/LexicalConventions/7.7.2.js b/test/ecma/LexicalConventions/7.7.2.js rename from js/test/ecma/LexicalConventions/7.7.2.js rename to test/ecma/LexicalConventions/7.7.2.js diff --git a/js/test/ecma/LexicalConventions/7.7.3-1.js b/test/ecma/LexicalConventions/7.7.3-1.js rename from js/test/ecma/LexicalConventions/7.7.3-1.js rename to test/ecma/LexicalConventions/7.7.3-1.js diff --git a/js/test/ecma/LexicalConventions/7.7.3-2.js b/test/ecma/LexicalConventions/7.7.3-2.js rename from js/test/ecma/LexicalConventions/7.7.3-2.js rename to test/ecma/LexicalConventions/7.7.3-2.js diff --git a/js/test/ecma/LexicalConventions/7.7.3.js b/test/ecma/LexicalConventions/7.7.3.js rename from js/test/ecma/LexicalConventions/7.7.3.js rename to test/ecma/LexicalConventions/7.7.3.js diff --git a/js/test/ecma/LexicalConventions/7.7.4.js b/test/ecma/LexicalConventions/7.7.4.js rename from js/test/ecma/LexicalConventions/7.7.4.js rename to test/ecma/LexicalConventions/7.7.4.js diff --git a/js/test/ecma/LexicalConventions/7.8.2-n.js b/test/ecma/LexicalConventions/7.8.2-n.js rename from js/test/ecma/LexicalConventions/7.8.2-n.js rename to test/ecma/LexicalConventions/7.8.2-n.js diff --git a/js/test/ecma/LexicalConventions/browser.js b/test/ecma/LexicalConventions/browser.js rename from js/test/ecma/LexicalConventions/browser.js rename to test/ecma/LexicalConventions/browser.js diff --git a/js/test/ecma/LexicalConventions/shell.js b/test/ecma/LexicalConventions/shell.js rename from js/test/ecma/LexicalConventions/shell.js rename to test/ecma/LexicalConventions/shell.js diff --git a/js/test/ecma/Math/15.8-1.js b/test/ecma/Math/15.8-1.js rename from js/test/ecma/Math/15.8-1.js rename to test/ecma/Math/15.8-1.js diff --git a/js/test/ecma/Math/15.8-2-n.js b/test/ecma/Math/15.8-2-n.js rename from js/test/ecma/Math/15.8-2-n.js rename to test/ecma/Math/15.8-2-n.js diff --git a/js/test/ecma/Math/15.8-3-n.js b/test/ecma/Math/15.8-3-n.js rename from js/test/ecma/Math/15.8-3-n.js rename to test/ecma/Math/15.8-3-n.js diff --git a/js/test/ecma/Math/15.8.1.1-1.js b/test/ecma/Math/15.8.1.1-1.js rename from js/test/ecma/Math/15.8.1.1-1.js rename to test/ecma/Math/15.8.1.1-1.js diff --git a/js/test/ecma/Math/15.8.1.1-2.js b/test/ecma/Math/15.8.1.1-2.js rename from js/test/ecma/Math/15.8.1.1-2.js rename to test/ecma/Math/15.8.1.1-2.js diff --git a/js/test/ecma/Math/15.8.1.2-1.js b/test/ecma/Math/15.8.1.2-1.js rename from js/test/ecma/Math/15.8.1.2-1.js rename to test/ecma/Math/15.8.1.2-1.js diff --git a/js/test/ecma/Math/15.8.1.2-2.js b/test/ecma/Math/15.8.1.2-2.js rename from js/test/ecma/Math/15.8.1.2-2.js rename to test/ecma/Math/15.8.1.2-2.js diff --git a/js/test/ecma/Math/15.8.1.3-1.js b/test/ecma/Math/15.8.1.3-1.js rename from js/test/ecma/Math/15.8.1.3-1.js rename to test/ecma/Math/15.8.1.3-1.js diff --git a/js/test/ecma/Math/15.8.1.3-2.js b/test/ecma/Math/15.8.1.3-2.js rename from js/test/ecma/Math/15.8.1.3-2.js rename to test/ecma/Math/15.8.1.3-2.js diff --git a/js/test/ecma/Math/15.8.1.4-1.js b/test/ecma/Math/15.8.1.4-1.js rename from js/test/ecma/Math/15.8.1.4-1.js rename to test/ecma/Math/15.8.1.4-1.js diff --git a/js/test/ecma/Math/15.8.1.4-2.js b/test/ecma/Math/15.8.1.4-2.js rename from js/test/ecma/Math/15.8.1.4-2.js rename to test/ecma/Math/15.8.1.4-2.js diff --git a/js/test/ecma/Math/15.8.1.5-1.js b/test/ecma/Math/15.8.1.5-1.js rename from js/test/ecma/Math/15.8.1.5-1.js rename to test/ecma/Math/15.8.1.5-1.js diff --git a/js/test/ecma/Math/15.8.1.5-2.js b/test/ecma/Math/15.8.1.5-2.js rename from js/test/ecma/Math/15.8.1.5-2.js rename to test/ecma/Math/15.8.1.5-2.js diff --git a/js/test/ecma/Math/15.8.1.6-1.js b/test/ecma/Math/15.8.1.6-1.js rename from js/test/ecma/Math/15.8.1.6-1.js rename to test/ecma/Math/15.8.1.6-1.js diff --git a/js/test/ecma/Math/15.8.1.6-2.js b/test/ecma/Math/15.8.1.6-2.js rename from js/test/ecma/Math/15.8.1.6-2.js rename to test/ecma/Math/15.8.1.6-2.js diff --git a/js/test/ecma/Math/15.8.1.7-1.js b/test/ecma/Math/15.8.1.7-1.js rename from js/test/ecma/Math/15.8.1.7-1.js rename to test/ecma/Math/15.8.1.7-1.js diff --git a/js/test/ecma/Math/15.8.1.7-2.js b/test/ecma/Math/15.8.1.7-2.js rename from js/test/ecma/Math/15.8.1.7-2.js rename to test/ecma/Math/15.8.1.7-2.js diff --git a/js/test/ecma/Math/15.8.1.8-1.js b/test/ecma/Math/15.8.1.8-1.js rename from js/test/ecma/Math/15.8.1.8-1.js rename to test/ecma/Math/15.8.1.8-1.js diff --git a/js/test/ecma/Math/15.8.1.8-2.js b/test/ecma/Math/15.8.1.8-2.js rename from js/test/ecma/Math/15.8.1.8-2.js rename to test/ecma/Math/15.8.1.8-2.js diff --git a/js/test/ecma/Math/15.8.1.8-3.js b/test/ecma/Math/15.8.1.8-3.js rename from js/test/ecma/Math/15.8.1.8-3.js rename to test/ecma/Math/15.8.1.8-3.js diff --git a/js/test/ecma/Math/15.8.1.js b/test/ecma/Math/15.8.1.js rename from js/test/ecma/Math/15.8.1.js rename to test/ecma/Math/15.8.1.js diff --git a/js/test/ecma/Math/15.8.2.1.js b/test/ecma/Math/15.8.2.1.js rename from js/test/ecma/Math/15.8.2.1.js rename to test/ecma/Math/15.8.2.1.js diff --git a/js/test/ecma/Math/15.8.2.10.js b/test/ecma/Math/15.8.2.10.js rename from js/test/ecma/Math/15.8.2.10.js rename to test/ecma/Math/15.8.2.10.js diff --git a/js/test/ecma/Math/15.8.2.11.js b/test/ecma/Math/15.8.2.11.js rename from js/test/ecma/Math/15.8.2.11.js rename to test/ecma/Math/15.8.2.11.js diff --git a/js/test/ecma/Math/15.8.2.12.js b/test/ecma/Math/15.8.2.12.js rename from js/test/ecma/Math/15.8.2.12.js rename to test/ecma/Math/15.8.2.12.js diff --git a/js/test/ecma/Math/15.8.2.13.js b/test/ecma/Math/15.8.2.13.js rename from js/test/ecma/Math/15.8.2.13.js rename to test/ecma/Math/15.8.2.13.js diff --git a/js/test/ecma/Math/15.8.2.14.js b/test/ecma/Math/15.8.2.14.js rename from js/test/ecma/Math/15.8.2.14.js rename to test/ecma/Math/15.8.2.14.js diff --git a/js/test/ecma/Math/15.8.2.15.js b/test/ecma/Math/15.8.2.15.js rename from js/test/ecma/Math/15.8.2.15.js rename to test/ecma/Math/15.8.2.15.js diff --git a/js/test/ecma/Math/15.8.2.16.js b/test/ecma/Math/15.8.2.16.js rename from js/test/ecma/Math/15.8.2.16.js rename to test/ecma/Math/15.8.2.16.js diff --git a/js/test/ecma/Math/15.8.2.17.js b/test/ecma/Math/15.8.2.17.js rename from js/test/ecma/Math/15.8.2.17.js rename to test/ecma/Math/15.8.2.17.js diff --git a/js/test/ecma/Math/15.8.2.18.js b/test/ecma/Math/15.8.2.18.js rename from js/test/ecma/Math/15.8.2.18.js rename to test/ecma/Math/15.8.2.18.js diff --git a/js/test/ecma/Math/15.8.2.2.js b/test/ecma/Math/15.8.2.2.js rename from js/test/ecma/Math/15.8.2.2.js rename to test/ecma/Math/15.8.2.2.js diff --git a/js/test/ecma/Math/15.8.2.3.js b/test/ecma/Math/15.8.2.3.js rename from js/test/ecma/Math/15.8.2.3.js rename to test/ecma/Math/15.8.2.3.js diff --git a/js/test/ecma/Math/15.8.2.4.js b/test/ecma/Math/15.8.2.4.js rename from js/test/ecma/Math/15.8.2.4.js rename to test/ecma/Math/15.8.2.4.js diff --git a/js/test/ecma/Math/15.8.2.5.js b/test/ecma/Math/15.8.2.5.js rename from js/test/ecma/Math/15.8.2.5.js rename to test/ecma/Math/15.8.2.5.js diff --git a/js/test/ecma/Math/15.8.2.6.js b/test/ecma/Math/15.8.2.6.js rename from js/test/ecma/Math/15.8.2.6.js rename to test/ecma/Math/15.8.2.6.js diff --git a/js/test/ecma/Math/15.8.2.7.js b/test/ecma/Math/15.8.2.7.js rename from js/test/ecma/Math/15.8.2.7.js rename to test/ecma/Math/15.8.2.7.js diff --git a/js/test/ecma/Math/15.8.2.8.js b/test/ecma/Math/15.8.2.8.js rename from js/test/ecma/Math/15.8.2.8.js rename to test/ecma/Math/15.8.2.8.js diff --git a/js/test/ecma/Math/15.8.2.9.js b/test/ecma/Math/15.8.2.9.js rename from js/test/ecma/Math/15.8.2.9.js rename to test/ecma/Math/15.8.2.9.js diff --git a/js/test/ecma/Math/browser.js b/test/ecma/Math/browser.js rename from js/test/ecma/Math/browser.js rename to test/ecma/Math/browser.js diff --git a/js/test/ecma/Math/shell.js b/test/ecma/Math/shell.js rename from js/test/ecma/Math/shell.js rename to test/ecma/Math/shell.js diff --git a/js/test/ecma/NativeObjects/15-1.js b/test/ecma/NativeObjects/15-1.js rename from js/test/ecma/NativeObjects/15-1.js rename to test/ecma/NativeObjects/15-1.js diff --git a/js/test/ecma/NativeObjects/15-2.js b/test/ecma/NativeObjects/15-2.js rename from js/test/ecma/NativeObjects/15-2.js rename to test/ecma/NativeObjects/15-2.js diff --git a/js/test/ecma/NativeObjects/browser.js b/test/ecma/NativeObjects/browser.js rename from js/test/ecma/NativeObjects/browser.js rename to test/ecma/NativeObjects/browser.js diff --git a/js/test/ecma/NativeObjects/shell.js b/test/ecma/NativeObjects/shell.js rename from js/test/ecma/NativeObjects/shell.js rename to test/ecma/NativeObjects/shell.js diff --git a/js/test/ecma/Number/15.7.1.js b/test/ecma/Number/15.7.1.js rename from js/test/ecma/Number/15.7.1.js rename to test/ecma/Number/15.7.1.js diff --git a/js/test/ecma/Number/15.7.2.js b/test/ecma/Number/15.7.2.js rename from js/test/ecma/Number/15.7.2.js rename to test/ecma/Number/15.7.2.js diff --git a/js/test/ecma/Number/15.7.3.1-1.js b/test/ecma/Number/15.7.3.1-1.js rename from js/test/ecma/Number/15.7.3.1-1.js rename to test/ecma/Number/15.7.3.1-1.js diff --git a/js/test/ecma/Number/15.7.3.1-2.js b/test/ecma/Number/15.7.3.1-2.js rename from js/test/ecma/Number/15.7.3.1-2.js rename to test/ecma/Number/15.7.3.1-2.js diff --git a/js/test/ecma/Number/15.7.3.1-3.js b/test/ecma/Number/15.7.3.1-3.js rename from js/test/ecma/Number/15.7.3.1-3.js rename to test/ecma/Number/15.7.3.1-3.js diff --git a/js/test/ecma/Number/15.7.3.2-1.js b/test/ecma/Number/15.7.3.2-1.js rename from js/test/ecma/Number/15.7.3.2-1.js rename to test/ecma/Number/15.7.3.2-1.js diff --git a/js/test/ecma/Number/15.7.3.2-2.js b/test/ecma/Number/15.7.3.2-2.js rename from js/test/ecma/Number/15.7.3.2-2.js rename to test/ecma/Number/15.7.3.2-2.js diff --git a/js/test/ecma/Number/15.7.3.2-3.js b/test/ecma/Number/15.7.3.2-3.js rename from js/test/ecma/Number/15.7.3.2-3.js rename to test/ecma/Number/15.7.3.2-3.js diff --git a/js/test/ecma/Number/15.7.3.2-4.js b/test/ecma/Number/15.7.3.2-4.js rename from js/test/ecma/Number/15.7.3.2-4.js rename to test/ecma/Number/15.7.3.2-4.js diff --git a/js/test/ecma/Number/15.7.3.3-1.js b/test/ecma/Number/15.7.3.3-1.js rename from js/test/ecma/Number/15.7.3.3-1.js rename to test/ecma/Number/15.7.3.3-1.js diff --git a/js/test/ecma/Number/15.7.3.3-2.js b/test/ecma/Number/15.7.3.3-2.js rename from js/test/ecma/Number/15.7.3.3-2.js rename to test/ecma/Number/15.7.3.3-2.js diff --git a/js/test/ecma/Number/15.7.3.3-3.js b/test/ecma/Number/15.7.3.3-3.js rename from js/test/ecma/Number/15.7.3.3-3.js rename to test/ecma/Number/15.7.3.3-3.js diff --git a/js/test/ecma/Number/15.7.3.3-4.js b/test/ecma/Number/15.7.3.3-4.js rename from js/test/ecma/Number/15.7.3.3-4.js rename to test/ecma/Number/15.7.3.3-4.js diff --git a/js/test/ecma/Number/15.7.3.4-1.js b/test/ecma/Number/15.7.3.4-1.js rename from js/test/ecma/Number/15.7.3.4-1.js rename to test/ecma/Number/15.7.3.4-1.js diff --git a/js/test/ecma/Number/15.7.3.4-2.js b/test/ecma/Number/15.7.3.4-2.js rename from js/test/ecma/Number/15.7.3.4-2.js rename to test/ecma/Number/15.7.3.4-2.js diff --git a/js/test/ecma/Number/15.7.3.4-3.js b/test/ecma/Number/15.7.3.4-3.js rename from js/test/ecma/Number/15.7.3.4-3.js rename to test/ecma/Number/15.7.3.4-3.js diff --git a/js/test/ecma/Number/15.7.3.4-4.js b/test/ecma/Number/15.7.3.4-4.js rename from js/test/ecma/Number/15.7.3.4-4.js rename to test/ecma/Number/15.7.3.4-4.js diff --git a/js/test/ecma/Number/15.7.3.5-1.js b/test/ecma/Number/15.7.3.5-1.js rename from js/test/ecma/Number/15.7.3.5-1.js rename to test/ecma/Number/15.7.3.5-1.js diff --git a/js/test/ecma/Number/15.7.3.5-2.js b/test/ecma/Number/15.7.3.5-2.js rename from js/test/ecma/Number/15.7.3.5-2.js rename to test/ecma/Number/15.7.3.5-2.js diff --git a/js/test/ecma/Number/15.7.3.5-3.js b/test/ecma/Number/15.7.3.5-3.js rename from js/test/ecma/Number/15.7.3.5-3.js rename to test/ecma/Number/15.7.3.5-3.js diff --git a/js/test/ecma/Number/15.7.3.5-4.js b/test/ecma/Number/15.7.3.5-4.js rename from js/test/ecma/Number/15.7.3.5-4.js rename to test/ecma/Number/15.7.3.5-4.js diff --git a/js/test/ecma/Number/15.7.3.6-1.js b/test/ecma/Number/15.7.3.6-1.js rename from js/test/ecma/Number/15.7.3.6-1.js rename to test/ecma/Number/15.7.3.6-1.js diff --git a/js/test/ecma/Number/15.7.3.6-2.js b/test/ecma/Number/15.7.3.6-2.js rename from js/test/ecma/Number/15.7.3.6-2.js rename to test/ecma/Number/15.7.3.6-2.js diff --git a/js/test/ecma/Number/15.7.3.6-3.js b/test/ecma/Number/15.7.3.6-3.js rename from js/test/ecma/Number/15.7.3.6-3.js rename to test/ecma/Number/15.7.3.6-3.js diff --git a/js/test/ecma/Number/15.7.3.6-4.js b/test/ecma/Number/15.7.3.6-4.js rename from js/test/ecma/Number/15.7.3.6-4.js rename to test/ecma/Number/15.7.3.6-4.js diff --git a/js/test/ecma/Number/15.7.3.js b/test/ecma/Number/15.7.3.js rename from js/test/ecma/Number/15.7.3.js rename to test/ecma/Number/15.7.3.js diff --git a/js/test/ecma/Number/15.7.4-1.js b/test/ecma/Number/15.7.4-1.js rename from js/test/ecma/Number/15.7.4-1.js rename to test/ecma/Number/15.7.4-1.js diff --git a/js/test/ecma/Number/15.7.4.1.js b/test/ecma/Number/15.7.4.1.js rename from js/test/ecma/Number/15.7.4.1.js rename to test/ecma/Number/15.7.4.1.js diff --git a/js/test/ecma/Number/15.7.4.2-1.js b/test/ecma/Number/15.7.4.2-1.js rename from js/test/ecma/Number/15.7.4.2-1.js rename to test/ecma/Number/15.7.4.2-1.js diff --git a/js/test/ecma/Number/15.7.4.2-2-n.js b/test/ecma/Number/15.7.4.2-2-n.js rename from js/test/ecma/Number/15.7.4.2-2-n.js rename to test/ecma/Number/15.7.4.2-2-n.js diff --git a/js/test/ecma/Number/15.7.4.2-3-n.js b/test/ecma/Number/15.7.4.2-3-n.js rename from js/test/ecma/Number/15.7.4.2-3-n.js rename to test/ecma/Number/15.7.4.2-3-n.js diff --git a/js/test/ecma/Number/15.7.4.2-4.js b/test/ecma/Number/15.7.4.2-4.js rename from js/test/ecma/Number/15.7.4.2-4.js rename to test/ecma/Number/15.7.4.2-4.js diff --git a/js/test/ecma/Number/15.7.4.3-1.js b/test/ecma/Number/15.7.4.3-1.js rename from js/test/ecma/Number/15.7.4.3-1.js rename to test/ecma/Number/15.7.4.3-1.js diff --git a/js/test/ecma/Number/15.7.4.3-2.js b/test/ecma/Number/15.7.4.3-2.js rename from js/test/ecma/Number/15.7.4.3-2.js rename to test/ecma/Number/15.7.4.3-2.js diff --git a/js/test/ecma/Number/15.7.4.3-3-n.js b/test/ecma/Number/15.7.4.3-3-n.js rename from js/test/ecma/Number/15.7.4.3-3-n.js rename to test/ecma/Number/15.7.4.3-3-n.js diff --git a/js/test/ecma/Number/15.7.4.js b/test/ecma/Number/15.7.4.js rename from js/test/ecma/Number/15.7.4.js rename to test/ecma/Number/15.7.4.js diff --git a/js/test/ecma/Number/browser.js b/test/ecma/Number/browser.js rename from js/test/ecma/Number/browser.js rename to test/ecma/Number/browser.js diff --git a/js/test/ecma/Number/shell.js b/test/ecma/Number/shell.js rename from js/test/ecma/Number/shell.js rename to test/ecma/Number/shell.js diff --git a/js/test/ecma/ObjectObjects/15.2.1.1.js b/test/ecma/ObjectObjects/15.2.1.1.js rename from js/test/ecma/ObjectObjects/15.2.1.1.js rename to test/ecma/ObjectObjects/15.2.1.1.js diff --git a/js/test/ecma/ObjectObjects/15.2.1.2.js b/test/ecma/ObjectObjects/15.2.1.2.js rename from js/test/ecma/ObjectObjects/15.2.1.2.js rename to test/ecma/ObjectObjects/15.2.1.2.js diff --git a/js/test/ecma/ObjectObjects/15.2.2.1.js b/test/ecma/ObjectObjects/15.2.2.1.js rename from js/test/ecma/ObjectObjects/15.2.2.1.js rename to test/ecma/ObjectObjects/15.2.2.1.js diff --git a/js/test/ecma/ObjectObjects/15.2.2.2.js b/test/ecma/ObjectObjects/15.2.2.2.js rename from js/test/ecma/ObjectObjects/15.2.2.2.js rename to test/ecma/ObjectObjects/15.2.2.2.js diff --git a/js/test/ecma/ObjectObjects/15.2.3-1.js b/test/ecma/ObjectObjects/15.2.3-1.js rename from js/test/ecma/ObjectObjects/15.2.3-1.js rename to test/ecma/ObjectObjects/15.2.3-1.js diff --git a/js/test/ecma/ObjectObjects/15.2.3.1-1.js b/test/ecma/ObjectObjects/15.2.3.1-1.js rename from js/test/ecma/ObjectObjects/15.2.3.1-1.js rename to test/ecma/ObjectObjects/15.2.3.1-1.js diff --git a/js/test/ecma/ObjectObjects/15.2.3.1-2.js b/test/ecma/ObjectObjects/15.2.3.1-2.js rename from js/test/ecma/ObjectObjects/15.2.3.1-2.js rename to test/ecma/ObjectObjects/15.2.3.1-2.js diff --git a/js/test/ecma/ObjectObjects/15.2.3.1-3.js b/test/ecma/ObjectObjects/15.2.3.1-3.js rename from js/test/ecma/ObjectObjects/15.2.3.1-3.js rename to test/ecma/ObjectObjects/15.2.3.1-3.js diff --git a/js/test/ecma/ObjectObjects/15.2.3.1-4.js b/test/ecma/ObjectObjects/15.2.3.1-4.js rename from js/test/ecma/ObjectObjects/15.2.3.1-4.js rename to test/ecma/ObjectObjects/15.2.3.1-4.js diff --git a/js/test/ecma/ObjectObjects/15.2.3.js b/test/ecma/ObjectObjects/15.2.3.js rename from js/test/ecma/ObjectObjects/15.2.3.js rename to test/ecma/ObjectObjects/15.2.3.js diff --git a/js/test/ecma/ObjectObjects/15.2.4.1.js b/test/ecma/ObjectObjects/15.2.4.1.js rename from js/test/ecma/ObjectObjects/15.2.4.1.js rename to test/ecma/ObjectObjects/15.2.4.1.js diff --git a/js/test/ecma/ObjectObjects/15.2.4.2.js b/test/ecma/ObjectObjects/15.2.4.2.js rename from js/test/ecma/ObjectObjects/15.2.4.2.js rename to test/ecma/ObjectObjects/15.2.4.2.js diff --git a/js/test/ecma/ObjectObjects/15.2.4.3.js b/test/ecma/ObjectObjects/15.2.4.3.js rename from js/test/ecma/ObjectObjects/15.2.4.3.js rename to test/ecma/ObjectObjects/15.2.4.3.js diff --git a/js/test/ecma/ObjectObjects/15.2.4.js b/test/ecma/ObjectObjects/15.2.4.js rename from js/test/ecma/ObjectObjects/15.2.4.js rename to test/ecma/ObjectObjects/15.2.4.js diff --git a/js/test/ecma/ObjectObjects/browser.js b/test/ecma/ObjectObjects/browser.js rename from js/test/ecma/ObjectObjects/browser.js rename to test/ecma/ObjectObjects/browser.js diff --git a/js/test/ecma/ObjectObjects/shell.js b/test/ecma/ObjectObjects/shell.js rename from js/test/ecma/ObjectObjects/shell.js rename to test/ecma/ObjectObjects/shell.js diff --git a/js/test/ecma/README b/test/ecma/README rename from js/test/ecma/README rename to test/ecma/README diff --git a/js/test/ecma/SourceText/6-1.js b/test/ecma/SourceText/6-1.js rename from js/test/ecma/SourceText/6-1.js rename to test/ecma/SourceText/6-1.js diff --git a/js/test/ecma/SourceText/6-2.js b/test/ecma/SourceText/6-2.js rename from js/test/ecma/SourceText/6-2.js rename to test/ecma/SourceText/6-2.js diff --git a/js/test/ecma/SourceText/browser.js b/test/ecma/SourceText/browser.js rename from js/test/ecma/SourceText/browser.js rename to test/ecma/SourceText/browser.js diff --git a/js/test/ecma/SourceText/shell.js b/test/ecma/SourceText/shell.js rename from js/test/ecma/SourceText/shell.js rename to test/ecma/SourceText/shell.js diff --git a/js/test/ecma/Statements/12.10-1.js b/test/ecma/Statements/12.10-1.js rename from js/test/ecma/Statements/12.10-1.js rename to test/ecma/Statements/12.10-1.js diff --git a/js/test/ecma/Statements/12.10.js b/test/ecma/Statements/12.10.js rename from js/test/ecma/Statements/12.10.js rename to test/ecma/Statements/12.10.js diff --git a/js/test/ecma/Statements/12.2-1.js b/test/ecma/Statements/12.2-1.js rename from js/test/ecma/Statements/12.2-1.js rename to test/ecma/Statements/12.2-1.js diff --git a/js/test/ecma/Statements/12.5-1.js b/test/ecma/Statements/12.5-1.js rename from js/test/ecma/Statements/12.5-1.js rename to test/ecma/Statements/12.5-1.js diff --git a/js/test/ecma/Statements/12.5-2.js b/test/ecma/Statements/12.5-2.js rename from js/test/ecma/Statements/12.5-2.js rename to test/ecma/Statements/12.5-2.js diff --git a/js/test/ecma/Statements/12.6.1-1.js b/test/ecma/Statements/12.6.1-1.js rename from js/test/ecma/Statements/12.6.1-1.js rename to test/ecma/Statements/12.6.1-1.js diff --git a/js/test/ecma/Statements/12.6.2-1.js b/test/ecma/Statements/12.6.2-1.js rename from js/test/ecma/Statements/12.6.2-1.js rename to test/ecma/Statements/12.6.2-1.js diff --git a/js/test/ecma/Statements/12.6.2-2.js b/test/ecma/Statements/12.6.2-2.js rename from js/test/ecma/Statements/12.6.2-2.js rename to test/ecma/Statements/12.6.2-2.js diff --git a/js/test/ecma/Statements/12.6.2-3.js b/test/ecma/Statements/12.6.2-3.js rename from js/test/ecma/Statements/12.6.2-3.js rename to test/ecma/Statements/12.6.2-3.js diff --git a/js/test/ecma/Statements/12.6.2-4.js b/test/ecma/Statements/12.6.2-4.js rename from js/test/ecma/Statements/12.6.2-4.js rename to test/ecma/Statements/12.6.2-4.js diff --git a/js/test/ecma/Statements/12.6.2-5.js b/test/ecma/Statements/12.6.2-5.js rename from js/test/ecma/Statements/12.6.2-5.js rename to test/ecma/Statements/12.6.2-5.js diff --git a/js/test/ecma/Statements/12.6.2-6.js b/test/ecma/Statements/12.6.2-6.js rename from js/test/ecma/Statements/12.6.2-6.js rename to test/ecma/Statements/12.6.2-6.js diff --git a/js/test/ecma/Statements/12.6.2-7.js b/test/ecma/Statements/12.6.2-7.js rename from js/test/ecma/Statements/12.6.2-7.js rename to test/ecma/Statements/12.6.2-7.js diff --git a/js/test/ecma/Statements/12.6.2-8.js b/test/ecma/Statements/12.6.2-8.js rename from js/test/ecma/Statements/12.6.2-8.js rename to test/ecma/Statements/12.6.2-8.js diff --git a/js/test/ecma/Statements/12.6.2-9-n.js b/test/ecma/Statements/12.6.2-9-n.js rename from js/test/ecma/Statements/12.6.2-9-n.js rename to test/ecma/Statements/12.6.2-9-n.js diff --git a/js/test/ecma/Statements/12.6.3-1.js b/test/ecma/Statements/12.6.3-1.js rename from js/test/ecma/Statements/12.6.3-1.js rename to test/ecma/Statements/12.6.3-1.js diff --git a/js/test/ecma/Statements/12.6.3-10.js b/test/ecma/Statements/12.6.3-10.js rename from js/test/ecma/Statements/12.6.3-10.js rename to test/ecma/Statements/12.6.3-10.js diff --git a/js/test/ecma/Statements/12.6.3-11.js b/test/ecma/Statements/12.6.3-11.js rename from js/test/ecma/Statements/12.6.3-11.js rename to test/ecma/Statements/12.6.3-11.js diff --git a/js/test/ecma/Statements/12.6.3-12.js b/test/ecma/Statements/12.6.3-12.js rename from js/test/ecma/Statements/12.6.3-12.js rename to test/ecma/Statements/12.6.3-12.js diff --git a/js/test/ecma/Statements/12.6.3-19.js b/test/ecma/Statements/12.6.3-19.js rename from js/test/ecma/Statements/12.6.3-19.js rename to test/ecma/Statements/12.6.3-19.js diff --git a/js/test/ecma/Statements/12.6.3-2.js b/test/ecma/Statements/12.6.3-2.js rename from js/test/ecma/Statements/12.6.3-2.js rename to test/ecma/Statements/12.6.3-2.js diff --git a/js/test/ecma/Statements/12.6.3-3.js b/test/ecma/Statements/12.6.3-3.js rename from js/test/ecma/Statements/12.6.3-3.js rename to test/ecma/Statements/12.6.3-3.js diff --git a/js/test/ecma/Statements/12.6.3-4.js b/test/ecma/Statements/12.6.3-4.js rename from js/test/ecma/Statements/12.6.3-4.js rename to test/ecma/Statements/12.6.3-4.js diff --git a/js/test/ecma/Statements/12.6.3-5-n.js b/test/ecma/Statements/12.6.3-5-n.js rename from js/test/ecma/Statements/12.6.3-5-n.js rename to test/ecma/Statements/12.6.3-5-n.js diff --git a/js/test/ecma/Statements/12.6.3-6-n.js b/test/ecma/Statements/12.6.3-6-n.js rename from js/test/ecma/Statements/12.6.3-6-n.js rename to test/ecma/Statements/12.6.3-6-n.js diff --git a/js/test/ecma/Statements/12.6.3-7-n.js b/test/ecma/Statements/12.6.3-7-n.js rename from js/test/ecma/Statements/12.6.3-7-n.js rename to test/ecma/Statements/12.6.3-7-n.js diff --git a/js/test/ecma/Statements/12.6.3-8-n.js b/test/ecma/Statements/12.6.3-8-n.js rename from js/test/ecma/Statements/12.6.3-8-n.js rename to test/ecma/Statements/12.6.3-8-n.js diff --git a/js/test/ecma/Statements/12.6.3-9-n.js b/test/ecma/Statements/12.6.3-9-n.js rename from js/test/ecma/Statements/12.6.3-9-n.js rename to test/ecma/Statements/12.6.3-9-n.js diff --git a/js/test/ecma/Statements/12.7-1-n.js b/test/ecma/Statements/12.7-1-n.js rename from js/test/ecma/Statements/12.7-1-n.js rename to test/ecma/Statements/12.7-1-n.js diff --git a/js/test/ecma/Statements/12.8-1-n.js b/test/ecma/Statements/12.8-1-n.js rename from js/test/ecma/Statements/12.8-1-n.js rename to test/ecma/Statements/12.8-1-n.js diff --git a/js/test/ecma/Statements/12.9-1-n.js b/test/ecma/Statements/12.9-1-n.js rename from js/test/ecma/Statements/12.9-1-n.js rename to test/ecma/Statements/12.9-1-n.js diff --git a/js/test/ecma/Statements/browser.js b/test/ecma/Statements/browser.js rename from js/test/ecma/Statements/browser.js rename to test/ecma/Statements/browser.js diff --git a/js/test/ecma/Statements/shell.js b/test/ecma/Statements/shell.js rename from js/test/ecma/Statements/shell.js rename to test/ecma/Statements/shell.js diff --git a/js/test/ecma/String/15.5.1.js b/test/ecma/String/15.5.1.js rename from js/test/ecma/String/15.5.1.js rename to test/ecma/String/15.5.1.js diff --git a/js/test/ecma/String/15.5.2.js b/test/ecma/String/15.5.2.js rename from js/test/ecma/String/15.5.2.js rename to test/ecma/String/15.5.2.js diff --git a/js/test/ecma/String/15.5.3.1-1.js b/test/ecma/String/15.5.3.1-1.js rename from js/test/ecma/String/15.5.3.1-1.js rename to test/ecma/String/15.5.3.1-1.js diff --git a/js/test/ecma/String/15.5.3.1-2.js b/test/ecma/String/15.5.3.1-2.js rename from js/test/ecma/String/15.5.3.1-2.js rename to test/ecma/String/15.5.3.1-2.js diff --git a/js/test/ecma/String/15.5.3.1-3.js b/test/ecma/String/15.5.3.1-3.js rename from js/test/ecma/String/15.5.3.1-3.js rename to test/ecma/String/15.5.3.1-3.js diff --git a/js/test/ecma/String/15.5.3.1-4.js b/test/ecma/String/15.5.3.1-4.js rename from js/test/ecma/String/15.5.3.1-4.js rename to test/ecma/String/15.5.3.1-4.js diff --git a/js/test/ecma/String/15.5.3.2-1.js b/test/ecma/String/15.5.3.2-1.js rename from js/test/ecma/String/15.5.3.2-1.js rename to test/ecma/String/15.5.3.2-1.js diff --git a/js/test/ecma/String/15.5.3.2-2.js b/test/ecma/String/15.5.3.2-2.js rename from js/test/ecma/String/15.5.3.2-2.js rename to test/ecma/String/15.5.3.2-2.js diff --git a/js/test/ecma/String/15.5.3.2-3.js b/test/ecma/String/15.5.3.2-3.js rename from js/test/ecma/String/15.5.3.2-3.js rename to test/ecma/String/15.5.3.2-3.js diff --git a/js/test/ecma/String/15.5.3.js b/test/ecma/String/15.5.3.js rename from js/test/ecma/String/15.5.3.js rename to test/ecma/String/15.5.3.js diff --git a/js/test/ecma/String/15.5.4.1.js b/test/ecma/String/15.5.4.1.js rename from js/test/ecma/String/15.5.4.1.js rename to test/ecma/String/15.5.4.1.js diff --git a/js/test/ecma/String/15.5.4.10-1.js b/test/ecma/String/15.5.4.10-1.js rename from js/test/ecma/String/15.5.4.10-1.js rename to test/ecma/String/15.5.4.10-1.js diff --git a/js/test/ecma/String/15.5.4.11-1.js b/test/ecma/String/15.5.4.11-1.js rename from js/test/ecma/String/15.5.4.11-1.js rename to test/ecma/String/15.5.4.11-1.js diff --git a/js/test/ecma/String/15.5.4.11-2.js b/test/ecma/String/15.5.4.11-2.js rename from js/test/ecma/String/15.5.4.11-2.js rename to test/ecma/String/15.5.4.11-2.js diff --git a/js/test/ecma/String/15.5.4.11-3.js b/test/ecma/String/15.5.4.11-3.js rename from js/test/ecma/String/15.5.4.11-3.js rename to test/ecma/String/15.5.4.11-3.js diff --git a/js/test/ecma/String/15.5.4.11-4.js b/test/ecma/String/15.5.4.11-4.js rename from js/test/ecma/String/15.5.4.11-4.js rename to test/ecma/String/15.5.4.11-4.js diff --git a/js/test/ecma/String/15.5.4.11-5.js b/test/ecma/String/15.5.4.11-5.js rename from js/test/ecma/String/15.5.4.11-5.js rename to test/ecma/String/15.5.4.11-5.js diff --git a/js/test/ecma/String/15.5.4.11-6.js b/test/ecma/String/15.5.4.11-6.js rename from js/test/ecma/String/15.5.4.11-6.js rename to test/ecma/String/15.5.4.11-6.js diff --git a/js/test/ecma/String/15.5.4.12-1.js b/test/ecma/String/15.5.4.12-1.js rename from js/test/ecma/String/15.5.4.12-1.js rename to test/ecma/String/15.5.4.12-1.js diff --git a/js/test/ecma/String/15.5.4.12-2.js b/test/ecma/String/15.5.4.12-2.js rename from js/test/ecma/String/15.5.4.12-2.js rename to test/ecma/String/15.5.4.12-2.js diff --git a/js/test/ecma/String/15.5.4.12-3.js b/test/ecma/String/15.5.4.12-3.js rename from js/test/ecma/String/15.5.4.12-3.js rename to test/ecma/String/15.5.4.12-3.js diff --git a/js/test/ecma/String/15.5.4.12-4.js b/test/ecma/String/15.5.4.12-4.js rename from js/test/ecma/String/15.5.4.12-4.js rename to test/ecma/String/15.5.4.12-4.js diff --git a/js/test/ecma/String/15.5.4.12-5.js b/test/ecma/String/15.5.4.12-5.js rename from js/test/ecma/String/15.5.4.12-5.js rename to test/ecma/String/15.5.4.12-5.js diff --git a/js/test/ecma/String/15.5.4.2-1.js b/test/ecma/String/15.5.4.2-1.js rename from js/test/ecma/String/15.5.4.2-1.js rename to test/ecma/String/15.5.4.2-1.js diff --git a/js/test/ecma/String/15.5.4.2-2-n.js b/test/ecma/String/15.5.4.2-2-n.js rename from js/test/ecma/String/15.5.4.2-2-n.js rename to test/ecma/String/15.5.4.2-2-n.js diff --git a/js/test/ecma/String/15.5.4.2-3.js b/test/ecma/String/15.5.4.2-3.js rename from js/test/ecma/String/15.5.4.2-3.js rename to test/ecma/String/15.5.4.2-3.js diff --git a/js/test/ecma/String/15.5.4.2.js b/test/ecma/String/15.5.4.2.js rename from js/test/ecma/String/15.5.4.2.js rename to test/ecma/String/15.5.4.2.js diff --git a/js/test/ecma/String/15.5.4.3-1.js b/test/ecma/String/15.5.4.3-1.js rename from js/test/ecma/String/15.5.4.3-1.js rename to test/ecma/String/15.5.4.3-1.js diff --git a/js/test/ecma/String/15.5.4.3-2.js b/test/ecma/String/15.5.4.3-2.js rename from js/test/ecma/String/15.5.4.3-2.js rename to test/ecma/String/15.5.4.3-2.js diff --git a/js/test/ecma/String/15.5.4.3-3-n.js b/test/ecma/String/15.5.4.3-3-n.js rename from js/test/ecma/String/15.5.4.3-3-n.js rename to test/ecma/String/15.5.4.3-3-n.js diff --git a/js/test/ecma/String/15.5.4.4-1.js b/test/ecma/String/15.5.4.4-1.js rename from js/test/ecma/String/15.5.4.4-1.js rename to test/ecma/String/15.5.4.4-1.js diff --git a/js/test/ecma/String/15.5.4.4-2.js b/test/ecma/String/15.5.4.4-2.js rename from js/test/ecma/String/15.5.4.4-2.js rename to test/ecma/String/15.5.4.4-2.js diff --git a/js/test/ecma/String/15.5.4.4-3.js b/test/ecma/String/15.5.4.4-3.js rename from js/test/ecma/String/15.5.4.4-3.js rename to test/ecma/String/15.5.4.4-3.js diff --git a/js/test/ecma/String/15.5.4.4-4.js b/test/ecma/String/15.5.4.4-4.js rename from js/test/ecma/String/15.5.4.4-4.js rename to test/ecma/String/15.5.4.4-4.js diff --git a/js/test/ecma/String/15.5.4.5-1.js b/test/ecma/String/15.5.4.5-1.js rename from js/test/ecma/String/15.5.4.5-1.js rename to test/ecma/String/15.5.4.5-1.js diff --git a/js/test/ecma/String/15.5.4.5-2.js b/test/ecma/String/15.5.4.5-2.js rename from js/test/ecma/String/15.5.4.5-2.js rename to test/ecma/String/15.5.4.5-2.js diff --git a/js/test/ecma/String/15.5.4.5-3.js b/test/ecma/String/15.5.4.5-3.js rename from js/test/ecma/String/15.5.4.5-3.js rename to test/ecma/String/15.5.4.5-3.js diff --git a/js/test/ecma/String/15.5.4.5-4.js b/test/ecma/String/15.5.4.5-4.js rename from js/test/ecma/String/15.5.4.5-4.js rename to test/ecma/String/15.5.4.5-4.js diff --git a/js/test/ecma/String/15.5.4.5-5.js b/test/ecma/String/15.5.4.5-5.js rename from js/test/ecma/String/15.5.4.5-5.js rename to test/ecma/String/15.5.4.5-5.js diff --git a/js/test/ecma/String/15.5.4.5-6.js b/test/ecma/String/15.5.4.5-6.js rename from js/test/ecma/String/15.5.4.5-6.js rename to test/ecma/String/15.5.4.5-6.js diff --git a/js/test/ecma/String/15.5.4.6-1.js b/test/ecma/String/15.5.4.6-1.js rename from js/test/ecma/String/15.5.4.6-1.js rename to test/ecma/String/15.5.4.6-1.js diff --git a/js/test/ecma/String/15.5.4.6-2.js b/test/ecma/String/15.5.4.6-2.js rename from js/test/ecma/String/15.5.4.6-2.js rename to test/ecma/String/15.5.4.6-2.js diff --git a/js/test/ecma/String/15.5.4.7-1.js b/test/ecma/String/15.5.4.7-1.js rename from js/test/ecma/String/15.5.4.7-1.js rename to test/ecma/String/15.5.4.7-1.js diff --git a/js/test/ecma/String/15.5.4.7-2.js b/test/ecma/String/15.5.4.7-2.js rename from js/test/ecma/String/15.5.4.7-2.js rename to test/ecma/String/15.5.4.7-2.js diff --git a/js/test/ecma/String/15.5.4.7-3.js b/test/ecma/String/15.5.4.7-3.js rename from js/test/ecma/String/15.5.4.7-3.js rename to test/ecma/String/15.5.4.7-3.js diff --git a/js/test/ecma/String/15.5.4.8-1.js b/test/ecma/String/15.5.4.8-1.js rename from js/test/ecma/String/15.5.4.8-1.js rename to test/ecma/String/15.5.4.8-1.js diff --git a/js/test/ecma/String/15.5.4.8-2.js b/test/ecma/String/15.5.4.8-2.js rename from js/test/ecma/String/15.5.4.8-2.js rename to test/ecma/String/15.5.4.8-2.js diff --git a/js/test/ecma/String/15.5.4.8-3.js b/test/ecma/String/15.5.4.8-3.js rename from js/test/ecma/String/15.5.4.8-3.js rename to test/ecma/String/15.5.4.8-3.js diff --git a/js/test/ecma/String/15.5.4.9-1.js b/test/ecma/String/15.5.4.9-1.js rename from js/test/ecma/String/15.5.4.9-1.js rename to test/ecma/String/15.5.4.9-1.js diff --git a/js/test/ecma/String/15.5.4.js b/test/ecma/String/15.5.4.js rename from js/test/ecma/String/15.5.4.js rename to test/ecma/String/15.5.4.js diff --git a/js/test/ecma/String/15.5.5.1.js b/test/ecma/String/15.5.5.1.js rename from js/test/ecma/String/15.5.5.1.js rename to test/ecma/String/15.5.5.1.js diff --git a/js/test/ecma/String/browser.js b/test/ecma/String/browser.js rename from js/test/ecma/String/browser.js rename to test/ecma/String/browser.js diff --git a/js/test/ecma/String/shell.js b/test/ecma/String/shell.js rename from js/test/ecma/String/shell.js rename to test/ecma/String/shell.js diff --git a/js/test/ecma/TypeConversion/9.2.js b/test/ecma/TypeConversion/9.2.js rename from js/test/ecma/TypeConversion/9.2.js rename to test/ecma/TypeConversion/9.2.js diff --git a/js/test/ecma/TypeConversion/9.3-1.js b/test/ecma/TypeConversion/9.3-1.js rename from js/test/ecma/TypeConversion/9.3-1.js rename to test/ecma/TypeConversion/9.3-1.js diff --git a/js/test/ecma/TypeConversion/9.3.1-1.js b/test/ecma/TypeConversion/9.3.1-1.js rename from js/test/ecma/TypeConversion/9.3.1-1.js rename to test/ecma/TypeConversion/9.3.1-1.js diff --git a/js/test/ecma/TypeConversion/9.3.1-2.js b/test/ecma/TypeConversion/9.3.1-2.js rename from js/test/ecma/TypeConversion/9.3.1-2.js rename to test/ecma/TypeConversion/9.3.1-2.js diff --git a/js/test/ecma/TypeConversion/9.3.1-3.js b/test/ecma/TypeConversion/9.3.1-3.js rename from js/test/ecma/TypeConversion/9.3.1-3.js rename to test/ecma/TypeConversion/9.3.1-3.js diff --git a/js/test/ecma/TypeConversion/9.3.js b/test/ecma/TypeConversion/9.3.js rename from js/test/ecma/TypeConversion/9.3.js rename to test/ecma/TypeConversion/9.3.js diff --git a/js/test/ecma/TypeConversion/9.4-1.js b/test/ecma/TypeConversion/9.4-1.js rename from js/test/ecma/TypeConversion/9.4-1.js rename to test/ecma/TypeConversion/9.4-1.js diff --git a/js/test/ecma/TypeConversion/9.4-2.js b/test/ecma/TypeConversion/9.4-2.js rename from js/test/ecma/TypeConversion/9.4-2.js rename to test/ecma/TypeConversion/9.4-2.js diff --git a/js/test/ecma/TypeConversion/9.5-2.js b/test/ecma/TypeConversion/9.5-2.js rename from js/test/ecma/TypeConversion/9.5-2.js rename to test/ecma/TypeConversion/9.5-2.js diff --git a/js/test/ecma/TypeConversion/9.6.js b/test/ecma/TypeConversion/9.6.js rename from js/test/ecma/TypeConversion/9.6.js rename to test/ecma/TypeConversion/9.6.js diff --git a/js/test/ecma/TypeConversion/9.7.js b/test/ecma/TypeConversion/9.7.js rename from js/test/ecma/TypeConversion/9.7.js rename to test/ecma/TypeConversion/9.7.js diff --git a/js/test/ecma/TypeConversion/9.8.1.js b/test/ecma/TypeConversion/9.8.1.js rename from js/test/ecma/TypeConversion/9.8.1.js rename to test/ecma/TypeConversion/9.8.1.js diff --git a/js/test/ecma/TypeConversion/9.9-1.js b/test/ecma/TypeConversion/9.9-1.js rename from js/test/ecma/TypeConversion/9.9-1.js rename to test/ecma/TypeConversion/9.9-1.js diff --git a/js/test/ecma/TypeConversion/browser.js b/test/ecma/TypeConversion/browser.js rename from js/test/ecma/TypeConversion/browser.js rename to test/ecma/TypeConversion/browser.js diff --git a/js/test/ecma/TypeConversion/shell.js b/test/ecma/TypeConversion/shell.js rename from js/test/ecma/TypeConversion/shell.js rename to test/ecma/TypeConversion/shell.js diff --git a/js/test/ecma/Types/8.1.js b/test/ecma/Types/8.1.js rename from js/test/ecma/Types/8.1.js rename to test/ecma/Types/8.1.js diff --git a/js/test/ecma/Types/8.4.js b/test/ecma/Types/8.4.js rename from js/test/ecma/Types/8.4.js rename to test/ecma/Types/8.4.js diff --git a/js/test/ecma/Types/8.6.2.1-1.js b/test/ecma/Types/8.6.2.1-1.js rename from js/test/ecma/Types/8.6.2.1-1.js rename to test/ecma/Types/8.6.2.1-1.js diff --git a/js/test/ecma/Types/browser.js b/test/ecma/Types/browser.js rename from js/test/ecma/Types/browser.js rename to test/ecma/Types/browser.js diff --git a/js/test/ecma/Types/shell.js b/test/ecma/Types/shell.js rename from js/test/ecma/Types/shell.js rename to test/ecma/Types/shell.js diff --git a/js/test/ecma/conftest.py b/test/ecma/conftest.py rename from js/test/ecma/conftest.py rename to test/ecma/conftest.py diff --git a/js/test/ecma/shell.js b/test/ecma/shell.js rename from js/test/ecma/shell.js rename to test/ecma/shell.js diff --git a/js/test/jit_view.py b/test/jit_view.py rename from js/test/jit_view.py rename to test/jit_view.py diff --git a/js/test/test_array.py b/test/test_array.py rename from js/test/test_array.py rename to test/test_array.py --- a/js/test/test_array.py +++ b/test/test_array.py @@ -1,4 +1,4 @@ -from js.test.test_interp import assertv, assertp +from test.test_interp import assertv, assertp def test_array_push(capsys): assertv("var x = []; x.push(42); x.length;", 1) diff --git a/js/test/test_astbuilder.py b/test/test_astbuilder.py rename from js/test/test_astbuilder.py rename to test/test_astbuilder.py diff --git a/js/test/test_environment_record.py b/test/test_environment_record.py rename from js/test/test_environment_record.py rename to test/test_environment_record.py diff --git a/js/test/test_execution_context.py b/test/test_execution_context.py rename from js/test/test_execution_context.py rename to test/test_execution_context.py diff --git a/js/test/test_frames.py b/test/test_frames.py rename from js/test/test_frames.py rename to test/test_frames.py diff --git a/js/test/test_helpers.py b/test/test_helpers.py rename from js/test/test_helpers.py rename to test/test_helpers.py diff --git a/js/test/test_interactive.py b/test/test_interactive.py rename from js/test/test_interactive.py rename to test/test_interactive.py diff --git a/js/test/test_interp.py b/test/test_interp.py rename from js/test/test_interp.py rename to test/test_interp.py diff --git a/js/test/test_interpreter.py b/test/test_interpreter.py rename from js/test/test_interpreter.py rename to test/test_interpreter.py diff --git a/js/test/test_jscode.py b/test/test_jscode.py rename from js/test/test_jscode.py rename to test/test_jscode.py diff --git a/js/test/test_jsfunction.py b/test/test_jsfunction.py rename from js/test/test_jsfunction.py rename to test/test_jsfunction.py diff --git a/js/test/test_jsobj.py b/test/test_jsobj.py rename from js/test/test_jsobj.py rename to test/test_jsobj.py diff --git a/js/test/test_lexical_environment.py b/test/test_lexical_environment.py rename from js/test/test_lexical_environment.py rename to test/test_lexical_environment.py diff --git a/js/test/test_load.js b/test/test_load.js rename from js/test/test_load.js rename to test/test_load.js diff --git a/js/test/test_load.py b/test/test_load.py rename from js/test/test_load.py rename to test/test_load.py --- a/js/test/test_load.py +++ b/test/test_load.py @@ -1,5 +1,5 @@ import py -from js.test.test_interp import assertv +from test.test_interp import assertv def test_load(): pwd = py.path.local(__file__) diff --git a/js/test/test_locals.py b/test/test_locals.py rename from js/test/test_locals.py rename to test/test_locals.py diff --git a/js/test/test_numbers.py b/test/test_numbers.py rename from js/test/test_numbers.py rename to test/test_numbers.py --- a/js/test/test_numbers.py +++ b/test/test_numbers.py @@ -1,5 +1,5 @@ -from js.test.test_interp import assertp +from test.test_interp import assertp def test_infinity_nan(capsys): assertp('print(1/0)', 'Infinity', capsys) diff --git a/js/test/test_object_map.py b/test/test_object_map.py rename from js/test/test_object_map.py rename to test/test_object_map.py diff --git a/js/test/test_parser.py b/test/test_parser.py rename from js/test/test_parser.py rename to test/test_parser.py --- a/js/test/test_parser.py +++ b/test/test_parser.py @@ -9,33 +9,35 @@ from js.astbuilder import FakeParseError from js.astbuilder import ASTBuilder from js.jscode import JsCode +from pypy.rlib.parsing.parsing import ParseError xfail = py.test.mark.xfail -GFILE = py.path.local(__file__).dirpath().join('../jsgrammar.txt') +GFILE = py.path.local(__file__).dirpath().join('../js/jsgrammar.txt') try: t = GFILE.read(mode='U') regexs, rules, ToAST = parse_ebnf(t) -except ParseError,e: - print e.nice_error_message(filename=str(GFILE),source=t) +except ParseError, e: + print e.nice_error_message(filename=str(GFILE), source=t) raise parse_function = make_parse_function(regexs, rules, eof=True) + def setstartrule(rules, start): "takes the rule start and put it on the beginning of the rules" - oldpos = 0 newrules = [Rule("hacked_first_symbol", [[start, "EOF"]])] + rules return newrules + def get_defaultparse(): return parse_function + def parse_func(start=None): if start is not None: - parse_function = make_parse_function(regexs, setstartrule(rules, start), - eof=True) + parse_function = make_parse_function(regexs, setstartrule(rules, start), eof=True) else: parse_function = get_defaultparse() @@ -50,6 +52,7 @@ return tree return methodparse + class CountingVisitor(RPythonVisitor): def __init__(self): self.counts = {} @@ -63,10 +66,12 @@ def general_symbol_visit(self, node): self.counts[node.symbol] = self.counts.get(node.symbol, 0) + 1 + class BaseGrammarTest(object): def setup_class(cls): cls.parse = parse_func() + class TestLiterals(BaseGrammarTest): def setup_class(cls): cls.parse = parse_func('literal') @@ -77,6 +82,7 @@ self.parse(str(i)).visit(dc) assert dc.counts["DECIMALLITERAL"] == 1 + class IntEvaluationVisitor(RPythonVisitor): def general_symbol_visit(self, node): return node.additional_info @@ -100,7 +106,7 @@ else: arg1 = self.dispatch(node.children[1]) op = self.dispatch(node.children[0]) - return self.evalop(arg1,op) + return self.evalop(arg1, op) opmap = {'+': lambda x,y: x+y, '-': lambda x,y: x-y, diff --git a/js/test/test_scope.py b/test/test_scope.py rename from js/test/test_scope.py rename to test/test_scope.py --- a/js/test/test_scope.py +++ b/test/test_scope.py @@ -1,9 +1,9 @@ import py -from js.test.test_interp import assertv +from test.test_interp import assertv + @py.test.mark.xfail def test_variable_deletion(): assertv("var x = 3; delete this.x;", False) assertv("x = 3; delete this.x;", True) assertv("var x = 3; delete this.x; x", 3) - diff --git a/js/test/test_stack.py b/test/test_stack.py rename from js/test/test_stack.py rename to test/test_stack.py diff --git a/js/test/test_w_string.py b/test/test_w_string.py rename from js/test/test_w_string.py rename to test/test_w_string.py From noreply at buildbot.pypy.org Tue Jan 8 16:08:21 2013 From: noreply at buildbot.pypy.org (stepahn) Date: Tue, 8 Jan 2013 16:08:21 +0100 (CET) Subject: [pypy-commit] lang-js default: moved doc folder Message-ID: <20130108150821.AFD221C12C1@cobra.cs.uni-duesseldorf.de> Author: Stephan Branch: Changeset: r333:1ee936b6c7c5 Date: 2013-01-08 16:07 +0100 http://bitbucket.org/pypy/lang-js/changeset/1ee936b6c7c5/ Log: moved doc folder diff --git a/js/doc/bytecode.txt b/doc/bytecode.txt rename from js/doc/bytecode.txt rename to doc/bytecode.txt From noreply at buildbot.pypy.org Tue Jan 8 17:22:04 2013 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 8 Jan 2013 17:22:04 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: revive continulet-jit-3 branch in new reality. For starters, a JIT frame Message-ID: <20130108162204.D48051C1305@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r59874:46a6aeacf807 Date: 2013-01-08 18:20 +0200 http://bitbucket.org/pypy/pypy/changeset/46a6aeacf807/ Log: revive continulet-jit-3 branch in new reality. For starters, a JIT frame description diff --git a/pypy/jit/backend/llsupport/jitframe.py b/pypy/jit/backend/llsupport/jitframe.py --- a/pypy/jit/backend/llsupport/jitframe.py +++ b/pypy/jit/backend/llsupport/jitframe.py @@ -1,33 +1,29 @@ -from pypy.rpython.lltypesystem import lltype, llmemory -from pypy.jit.codewriter import longlong +from pypy.rpython.lltypesystem import lltype, llmemory, rffi +GCINDEXLIST = lltype.GcArray(rffi.UINT) -# XXX not an actual union for now -VALUEUNION = lltype.Struct('VALUEUNION', - ('int', lltype.Signed), - ('ref', llmemory.GCREF), - ('float', longlong.FLOATSTORAGE)) +# the JITFRAME that's stored on the heap. See backend//arch.py for +# detailed explanation how it is on your architecture -DEADFRAME = lltype.GcStruct( - 'DEADFRAME', - +JITFRAME = lltype.GcStruct( + 'JITFRAME', + # gcindexlist is a list of indexes of GC ptrs + # in the actual array + ('jf_gcindexlist', lltype.Ptr(GCINDEXLIST)), # Once the execute_token() returns, the field 'jf_descr' stores the # descr of the last executed operation (either a GUARD, or FINISH). # This field is also set immediately before doing CALL_MAY_FORCE # or CALL_ASSEMBLER. ('jf_descr', llmemory.GCREF), - # For the front-end: a GCREF for the savedata ('jf_savedata', llmemory.GCREF), - # For GUARD_(NO)_EXCEPTION and GUARD_NOT_FORCED: the exception we # got. (Note that in case of a regular FINISH generated from # RPython code that finishes the function with an exception, the # exception is not stored there, but in jf_values[0].ref.) ('jf_guard_exc', llmemory.GCREF), + # the actual frame + ('jf_frame', lltype.Array(lltype.Signed)) +) - # All values are stored in the following array, for now not - # compactly at all - ('jf_values', lltype.Array(VALUEUNION))) - -DEADFRAMEPTR = lltype.Ptr(DEADFRAME) +JITFRAMEPTR = lltype.Ptr(JITFRAME) diff --git a/pypy/jit/backend/x86/arch.py b/pypy/jit/backend/x86/arch.py --- a/pypy/jit/backend/x86/arch.py +++ b/pypy/jit/backend/x86/arch.py @@ -8,22 +8,52 @@ import sys if sys.maxint == (2**31 - 1): WORD = 4 - # ebp + ebx + esi + edi + 4 extra words + force_index = 9 words - FRAME_FIXED_SIZE = 9 - FORCE_INDEX_OFS = -8*WORD - MY_COPY_OF_REGS = -7*WORD IS_X86_32 = True IS_X86_64 = False else: WORD = 8 - # rbp + rbx + r12 + r13 + r14 + r15 + 11 extra words + force_index = 18 - FRAME_FIXED_SIZE = 18 - FORCE_INDEX_OFS = -17*WORD - MY_COPY_OF_REGS = -16*WORD IS_X86_32 = False IS_X86_64 = True -# The extra space has room for almost all registers, apart from eax and edx +# The stack for a JIT call is fixed, but it contains only scratch space +# used e.g. for storing arguments to further calls: +# +# +--------------------+ <== aligned to 16 bytes +# | return address | +# +--------------------+ +# | scratch | +# | space | +# +--------------------+ <== aligned to 16 bytes + +if WORD == 4: + SCRATCH_SIZE = 7 # total size: 32 bytes +else: + SCRATCH_SIZE = 3 # total size: 32 bytes + +# All the rest of the data is in a GC-managed variable-size "frame". +# This frame object's address is always stored in the register EBP/RBP. +# A frame is a jit.backend.llsupport.llmodel.JITFRAME = GcArray(Signed). +# The following locations are indices in this array. + +# The frame's fixed size gives the standard fixed part at the +# start of every frame: the saved value of some registers, +# one word for the force_index, and some extra space used only +# during a malloc that needs to go via its slow path. + +if WORD == 4: + # ebp + ebx + esi + edi + 4 extra words + force_index = 9 words + FRAME_FIXED_SIZE = 9 + FORCE_INDEX_OFS = 0 + SAVED_REGISTERS = 1 # range(1, 5) + MY_COPY_OF_REGS = 5 # range(5, 9) +else: + # rbp + rbx + r12 + r13 + r14 + r15 + 11 extra words + force_index = 18 + FRAME_FIXED_SIZE = 18 + FORCE_INDEX_OFS = 0 + SAVED_REGISTERS = 1 # range(1, 7) + MY_COPY_OF_REGS = 7 # range(7, 18) + +# "My copy of regs" has room for almost all registers, apart from eax and edx # which are used in the malloc itself. They are: # ecx, ebx, esi, edi [32 and 64 bits] # r8, r9, r10, r12, r13, r14, r15 [64 bits only] From noreply at buildbot.pypy.org Tue Jan 8 17:35:02 2013 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 8 Jan 2013 17:35:02 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: Kill this hack Message-ID: <20130108163502.063DD1C1305@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r59875:dce88c3d5657 Date: 2013-01-08 18:34 +0200 http://bitbucket.org/pypy/pypy/changeset/dce88c3d5657/ Log: Kill this hack diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py --- a/pypy/jit/backend/llsupport/llmodel.py +++ b/pypy/jit/backend/llsupport/llmodel.py @@ -193,12 +193,6 @@ # ------------------- helpers and descriptions -------------------- - def gc_set_extra_threshold(self): - llop.gc_set_extra_threshold(lltype.Void, self.deadframe_size_max) - - def gc_clear_extra_threshold(self): - llop.gc_set_extra_threshold(lltype.Void, 0) - @staticmethod def _cast_int_to_gcref(x): # dangerous! only use if you are sure no collection could occur diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -1958,7 +1958,7 @@ return arglocs[:] @staticmethod - #@rgc.no_collect -- XXX still true, but hacked gc_set_extra_threshold + @rgc.no_collect def grab_frame_values(cpu, bytecode, frame_addr, allregisters): # no malloc allowed here!! xxx apart from one, hacking a lot #self.fail_ebp = allregisters[16 + ebp.value] @@ -1998,7 +1998,6 @@ # that it is guaranteed that the following malloc() works # without requiring a collect(), but it needs to be re-added # as soon as possible. - cpu.gc_clear_extra_threshold() assert num <= cpu.get_failargs_limit() try: deadframe = lltype.malloc(jitframe.DEADFRAME, num) @@ -2088,8 +2087,7 @@ return lltype.cast_opaque_ptr(llmemory.GCREF, deadframe) def setup_failure_recovery(self): - - #@rgc.no_collect -- XXX still true, but hacked gc_set_extra_threshold + @rgc.no_collect def failure_recovery_func(registers): # 'registers' is a pointer to a structure containing the # original value of the registers, optionally the original diff --git a/pypy/jit/backend/x86/runner.py b/pypy/jit/backend/x86/runner.py --- a/pypy/jit/backend/x86/runner.py +++ b/pypy/jit/backend/x86/runner.py @@ -45,10 +45,6 @@ self.profile_agent = profile_agent - from pypy.jit.backend.llsupport import jitframe - self.deadframe_size_max = llmemory.sizeof(jitframe.DEADFRAME, - self.get_failargs_limit()) - def set_debug(self, flag): return self.assembler.set_debug(flag) @@ -123,7 +119,6 @@ if not self.translate_support_code: LLInterpreter.current_interpreter = prev_interpreter #llop.debug_print(lltype.Void, "<<<< Back") - self.gc_set_extra_threshold() return deadframe return execute_token diff --git a/pypy/jit/backend/x86/test/test_assembler.py b/pypy/jit/backend/x86/test/test_assembler.py --- a/pypy/jit/backend/x86/test/test_assembler.py +++ b/pypy/jit/backend/x86/test/test_assembler.py @@ -32,9 +32,6 @@ assert num == 0x1C3 return FakeFailDescr() - def gc_clear_extra_threshold(self): - pass - def get_failargs_limit(self): return 1000 diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py --- a/pypy/jit/metainterp/warmspot.py +++ b/pypy/jit/metainterp/warmspot.py @@ -959,11 +959,8 @@ EffectInfo.MOST_GENERAL) vinfo = jd.virtualizable_info - gc_set_extra_threshold = getattr(self.cpu, 'gc_set_extra_threshold', - lambda: None) def assembler_call_helper(deadframe, virtualizableref): - gc_set_extra_threshold() # XXX temporary hack fail_descr = self.cpu.get_latest_descr(deadframe) if vinfo is not None: virtualizable = lltype.cast_opaque_ptr( diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py --- a/pypy/rpython/lltypesystem/lloperation.py +++ b/pypy/rpython/lltypesystem/lloperation.py @@ -499,7 +499,6 @@ 'gc_typeids_z' : LLOp(), 'gc_gcflag_extra' : LLOp(), 'gc_add_memory_pressure': LLOp(), - 'gc_set_extra_threshold': LLOp(canrun=True, canmallocgc=True), # ------- JIT & GC interaction, only for some GCs ---------- diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py --- a/pypy/rpython/lltypesystem/opimpl.py +++ b/pypy/rpython/lltypesystem/opimpl.py @@ -644,9 +644,6 @@ def op_gc_assume_young_pointers(addr): pass -def op_gc_set_extra_threshold(threshold): - pass - def op_shrink_array(array, smallersize): return False diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py --- a/pypy/rpython/memory/gc/minimark.py +++ b/pypy/rpython/memory/gc/minimark.py @@ -1820,20 +1820,6 @@ return self.id_or_identityhash(gcobj, True) # ---------- - # set_extra_threshold support - - def set_extra_threshold(self, reserved_size): - ll_assert(reserved_size <= self.nonlarge_max, - "set_extra_threshold: too big!") - diff = reserved_size - self.extra_threshold - if diff > 0 and self.nursery_free + diff > self.nursery_top: - self.minor_collection() - self.nursery_size -= diff - self.nursery_top -= diff - self.extra_threshold += diff - - - # ---------- # Finalizers def deal_with_young_objects_with_finalizers(self): diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py --- a/pypy/rpython/memory/gctransform/framework.py +++ b/pypy/rpython/memory/gctransform/framework.py @@ -385,10 +385,6 @@ self.obtainfreespace_ptr = getfn(GCClass.obtain_free_space.im_func, [s_gc, annmodel.SomeInteger()], annmodel.SomeAddress()) - if getattr(GCClass, 'set_extra_threshold', False): - self.setextrathreshold_ptr = getfn( - GCClass.set_extra_threshold.im_func, - [s_gc, annmodel.SomeInteger()], annmodel.s_None) if GCClass.moving_gc: self.id_ptr = getfn(GCClass.id.im_func, @@ -959,13 +955,6 @@ resultvar=hop.spaceop.result) self.pop_roots(hop, livevars) - def gct_gc_set_extra_threshold(self, hop): - livevars = self.push_roots(hop) - [v_size] = hop.spaceop.args - hop.genop("direct_call", - [self.setextrathreshold_ptr, self.c_const_gc, v_size]) - self.pop_roots(hop, livevars) - def gct_gc_set_max_heap_size(self, hop): [v_size] = hop.spaceop.args hop.genop("direct_call", [self.set_max_heap_size_ptr, diff --git a/pypy/rpython/memory/gctransform/test/test_framework.py b/pypy/rpython/memory/gctransform/test/test_framework.py --- a/pypy/rpython/memory/gctransform/test/test_framework.py +++ b/pypy/rpython/memory/gctransform/test/test_framework.py @@ -97,13 +97,6 @@ gg = graphof(t, g) assert CollectAnalyzer(t).analyze_direct_call(gg) -def test_cancollect_some_operation(): - def g(): - llop.gc_set_extra_threshold(lltype.Void, 32) - t = rtype(g, []) - gg = graphof(t, g) - assert CollectAnalyzer(t).analyze_direct_call(gg) - def test_no_collect(): from pypy.rlib import rgc from pypy.translator.c.genc import CStandaloneBuilder From noreply at buildbot.pypy.org Tue Jan 8 19:41:06 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Tue, 8 Jan 2013 19:41:06 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Removed rpython.tool.compat, moved test_annmm to pypy.objspace.std.test Message-ID: <20130108184106.B46681C1305@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59876:b4a8a57e8427 Date: 2013-01-08 19:00 +0100 http://bitbucket.org/pypy/pypy/changeset/b4a8a57e8427/ Log: Removed rpython.tool.compat, moved test_annmm to pypy.objspace.std.test diff --git a/rpython/annotator/test/test_annmm.py b/pypy/objspace/std/test/test_annmm.py rename from rpython/annotator/test/test_annmm.py rename to pypy/objspace/std/test/test_annmm.py diff --git a/rpython/jit/tl/jittest.py b/rpython/jit/tl/jittest.py --- a/rpython/jit/tl/jittest.py +++ b/rpython/jit/tl/jittest.py @@ -1,5 +1,5 @@ """ -This file is imported by pypy.translation.driver when running the +This file is imported by rpython.translator.driver when running the target --jittest. Feel free to hack it as needed; it is imported only after the '---> Checkpoint' fork. """ diff --git a/rpython/tool/compat.py b/rpython/tool/compat.py deleted file mode 100644 --- a/rpython/tool/compat.py +++ /dev/null @@ -1,13 +0,0 @@ - -try: - # Preferred way since python 2.6 - from hashlib import md5 -except ImportError: - try: - from md5 import md5 - except ImportError: - # no _md5 module on this platform. Try hard to find a pure-python one - # by fishing it from lib_pypy - from pypy.tool.lib_pypy import import_from_lib_pypy - md5 = import_from_lib_pypy('md5') - del import_from_lib_pypy diff --git a/rpython/tool/gcc_cache.py b/rpython/tool/gcc_cache.py --- a/rpython/tool/gcc_cache.py +++ b/rpython/tool/gcc_cache.py @@ -1,7 +1,7 @@ from pypy.conftest import pypydir from rpython.translator.platform import CompilationError from rpython.translator.tool.cbuild import ExternalCompilationInfo -from rpython.tool.compat import md5 +from hashlib import md5 import py cache_dir_root = py.path.local(pypydir).join('_cache').ensure(dir=1) diff --git a/rpython/translator/backendopt/stat.py b/rpython/translator/backendopt/stat.py --- a/rpython/translator/backendopt/stat.py +++ b/rpython/translator/backendopt/stat.py @@ -1,5 +1,5 @@ from rpython.translator.simplify import get_graph -from rpython.tool.compat import md5 +from hashlib import md5 def get_statistics(graph, translator, save_per_graph_details=None, ignore_stack_checks=False): seen_graphs = {} diff --git a/rpython/translator/backendopt/support.py b/rpython/translator/backendopt/support.py --- a/rpython/translator/backendopt/support.py +++ b/rpython/translator/backendopt/support.py @@ -116,7 +116,7 @@ return loop def md5digest(translator): - from rpython.tool.compat import md5 + from hashlib import md5 graph2digest = {} for graph in translator.graphs: m = md5() diff --git a/rpython/translator/driver.py b/rpython/translator/driver.py --- a/rpython/translator/driver.py +++ b/rpython/translator/driver.py @@ -395,7 +395,7 @@ # print a message, and restart from rpython.translator.goal import unixcheckpoint unixcheckpoint.restartable_point(auto='run') - # load the module pypy/jit/tl/jittest.py, which you can hack at + # load the module rpython/jit/tl/jittest.py, which you can hack at # and restart without needing to restart the whole translation process from rpython.jit.tl import jittest jittest.jittest(self) From noreply at buildbot.pypy.org Tue Jan 8 19:41:07 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Tue, 8 Jan 2013 19:41:07 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Don't use pypy.module.signal.interp_signal.SignalActionFlag in rpython/jit/metainterp/test/test_del.py Message-ID: <20130108184107.EC3F71C1305@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59877:35e52e793d41 Date: 2013-01-08 19:29 +0100 http://bitbucket.org/pypy/pypy/changeset/35e52e793d41/ Log: Don't use pypy.module.signal.interp_signal.SignalActionFlag in rpython/jit/metainterp/test/test_del.py diff --git a/rpython/jit/metainterp/test/test_del.py b/rpython/jit/metainterp/test/test_del.py --- a/rpython/jit/metainterp/test/test_del.py +++ b/rpython/jit/metainterp/test/test_del.py @@ -123,12 +123,28 @@ res = self.meta_interp(main, [20]) assert res == 1001 +# Minimal copy of pypy.module.signal.interp_signal.SignalActionFlag for +# TestLLtype +from rpython.rtyper.lltypesystem import lltype, rffi + +class Ticker(object): + def __init__(self): + self.ticker = rffi.llexternal('ticker', [], + lltype.Ptr(LONG_STRUCT), + compilation_info=eci, + sandboxsafe=True, _nowrapper=True, + elidable_function=True) + + def reset_ticker(self, value): + self.ticker().c_value = value + + def decrement_ticker(self, by): + self.ticker().c_value -= by + return self.ticker().c_value class TestLLtype(DelTests, LLJitMixin): def test_signal_action(self): - from pypy.module.signal.interp_signal import SignalActionFlag - action = SignalActionFlag() - action.has_bytecode_counter = True + action = Ticker() # myjitdriver = JitDriver(greens = [], reds = ['n', 'x']) class X: From noreply at buildbot.pypy.org Tue Jan 8 20:06:46 2013 From: noreply at buildbot.pypy.org (stepahn) Date: Tue, 8 Jan 2013 20:06:46 +0100 (CET) Subject: [pypy-commit] lang-js default: re-organized builtins Message-ID: <20130108190646.352651C1305@cobra.cs.uni-duesseldorf.de> Author: Stephan Branch: Changeset: r334:7c7a58b2bf0b Date: 2013-01-08 17:21 +0100 http://bitbucket.org/pypy/lang-js/changeset/7c7a58b2bf0b/ Log: re-organized builtins diff --git a/js/astbuilder.py b/js/astbuilder.py --- a/js/astbuilder.py +++ b/js/astbuilder.py @@ -218,8 +218,8 @@ return operations.IntNumber(pos, int(node.additional_info, 8)) def string(self, node): - from operations import string_unquote - from runistr import unicode_unescape, decode_str_utf8 + from js.operations import string_unquote + from js.runistr import unicode_unescape, decode_str_utf8 # TODO decode utf-8 pos = self.get_pos(node) @@ -717,7 +717,7 @@ def parse_to_ast(code): #assert isinstance(code, unicode) from js.jsparser import parse - from runistr import encode_unicode_utf8 + from js.runistr import encode_unicode_utf8 src = encode_unicode_utf8(code) parse_tree = parse(src) ast = parse_tree_to_ast(parse_tree) diff --git a/js/baseop.py b/js/baseop.py --- a/js/baseop.py +++ b/js/baseop.py @@ -6,7 +6,7 @@ from pypy.rlib.rarithmetic import ovfcheck from pypy.rlib.rfloat import isnan, isinf -from js.builtins_number import w_NAN, w_POSITIVE_INFINITY, w_NEGATIVE_INFINITY +from js.builtins.number import w_NAN, w_POSITIVE_INFINITY, w_NEGATIVE_INFINITY import math diff --git a/js/builtins.py b/js/builtins/__init__.py rename from js/builtins.py rename to js/builtins/__init__.py --- a/js/builtins.py +++ b/js/builtins/__init__.py @@ -48,10 +48,10 @@ put_property(global_object, u'Function', w_Function) # 15.3.4 Properties of the Function Prototype Object - import js.builtins_function as function_builtins from js.functions import JsNativeFunction - empty_func = JsNativeFunction(function_builtins.empty, u'Empty') + import js.builtins.function + empty_func = JsNativeFunction(js.builtins.function.empty, u'Empty') w_FunctionPrototype = object_space.new_func(empty_func) object_space.assign_proto(w_FunctionPrototype, object_space.proto_object) object_space.proto_function = w_FunctionPrototype @@ -75,13 +75,13 @@ # 14.2.4.1 Object.prototype.constructor put_property(w_ObjectPrototype, u'constructor', w_Object) - import js.builtins_object as object_builtins + import js.builtins.object # 15.2.4.2 Object.prototype.toString() - put_native_function(w_ObjectPrototype, u'toString', object_builtins.to_string) - put_native_function(w_ObjectPrototype, u'toLocaleString', object_builtins.to_string) + put_native_function(w_ObjectPrototype, u'toString', js.builtins.object.to_string) + put_native_function(w_ObjectPrototype, u'toLocaleString', js.builtins.object.to_string) # 15.2.4.3 Object.prototype.valueOf() - put_native_function(w_ObjectPrototype, u'valueOf', object_builtins.value_of) + put_native_function(w_ObjectPrototype, u'valueOf', js.builtins.object.value_of) # 15.3 Function Objects # 15.3.3 Properties of the Function Constructor @@ -95,38 +95,35 @@ # 14.3.4.1 Function.prototype.constructor put_property(w_FunctionPrototype, u'constructor', w_Function) - import js.builtins_function as function_builtins - # 15.3.4.2 Function.prototype.toString() - put_native_function(w_FunctionPrototype, u'toString', function_builtins.to_string) + put_native_function(w_FunctionPrototype, u'toString', js.builtins.function.to_string) # 15.3.4.3 Function.prototype.apply - put_intimate_function(w_FunctionPrototype, u'apply', function_builtins.js_apply) + put_intimate_function(w_FunctionPrototype, u'apply', js.builtins.function.js_apply) # 15.3.4.4 Function.prototype.call - put_intimate_function(w_FunctionPrototype, u'call', function_builtins.js_call) + put_intimate_function(w_FunctionPrototype, u'call', js.builtins.function.js_call) - import js.builtins_boolean - js.builtins_boolean.setup(global_object) + import js.builtins.boolean + js.builtins.boolean.setup(global_object) - import js.builtins_number - js.builtins_number.setup(global_object) + import js.builtins.number + js.builtins.number.setup(global_object) - import js.builtins_string - js.builtins_string.setup(global_object) + import js.builtins.string + js.builtins.string.setup(global_object) - import js.builtins_array - js.builtins_array.setup(global_object) + import js.builtins.array + js.builtins.array.setup(global_object) - #Math - import js.builtins_math - js.builtins_math.setup(global_object) + import js.builtins.js_math + js.builtins.js_math.setup(global_object) - import js.builtins_date - js.builtins_date.setup(global_object) + import js.builtins.date + js.builtins.date.setup(global_object) - import js.builtins_global - js.builtins_global.setup(global_object) + import js.builtins.js_global + js.builtins.js_global.setup(global_object) def get_arg(args, index, default=w_Undefined): diff --git a/js/builtins_array.py b/js/builtins/array.py rename from js/builtins_array.py rename to js/builtins/array.py diff --git a/js/builtins_boolean.py b/js/builtins/boolean.py rename from js/builtins_boolean.py rename to js/builtins/boolean.py diff --git a/js/builtins_date.py b/js/builtins/date.py rename from js/builtins_date.py rename to js/builtins/date.py diff --git a/js/builtins_function.py b/js/builtins/function.py rename from js/builtins_function.py rename to js/builtins/function.py diff --git a/js/builtins_interpreter.py b/js/builtins/interpreter.py rename from js/builtins_interpreter.py rename to js/builtins/interpreter.py diff --git a/js/builtins_global.py b/js/builtins/js_global.py rename from js/builtins_global.py rename to js/builtins/js_global.py --- a/js/builtins_global.py +++ b/js/builtins/js_global.py @@ -8,8 +8,8 @@ def setup(global_object): from js.builtins import put_intimate_function, put_native_function, put_property - from js.builtins_number import w_NAN - from js.builtins_number import w_POSITIVE_INFINITY + from js.builtins.number import w_NAN + from js.builtins.number import w_POSITIVE_INFINITY from js.jsobj import w_Undefined from pypy.rlib.objectmodel import we_are_translated @@ -183,7 +183,7 @@ # 15.1.2.3 @w_return def parse_float(this, args): - from runistr import encode_unicode_utf8 + from js.runistr import encode_unicode_utf8 from js.constants import num_lit_rexp string = get_arg(args, 0) @@ -222,7 +222,7 @@ return from pypy.rlib.rstring import UnicodeBuilder - from runistr import encode_unicode_utf8 + from js.runistr import encode_unicode_utf8 builder = UnicodeBuilder() for arg in args[:-1]: diff --git a/js/builtins_math.py b/js/builtins/js_math.py rename from js/builtins_math.py rename to js/builtins/js_math.py diff --git a/js/builtins_number.py b/js/builtins/number.py rename from js/builtins_number.py rename to js/builtins/number.py diff --git a/js/builtins_object.py b/js/builtins/object.py rename from js/builtins_object.py rename to js/builtins/object.py diff --git a/js/builtins_string.py b/js/builtins/string.py rename from js/builtins_string.py rename to js/builtins/string.py diff --git a/js/interpreter.py b/js/interpreter.py --- a/js/interpreter.py +++ b/js/interpreter.py @@ -17,8 +17,7 @@ def __init__(self, config={}): from js.jsobj import W_GlobalObject from js.object_space import object_space - import js.builtins - import js.builtins_interpreter + import js.builtins.interpreter self.config = InterpreterConfig(config) self.global_object = W_GlobalObject() @@ -26,7 +25,7 @@ object_space.interpreter = self js.builtins.setup_builtins(self.global_object) - js.builtins_interpreter.setup_builtins(self.global_object) + js.builtins.interpreter.setup_builtins(self.global_object) object_space.assign_proto(self.global_object) @@ -40,7 +39,7 @@ def run_src(self, src): from js.astbuilder import parse_to_ast - from runistr import decode_str_utf8 + from js.runistr import decode_str_utf8 ast = parse_to_ast(decode_str_utf8(src)) return self.run_ast(ast) diff --git a/js/jsobj.py b/js/jsobj.py --- a/js/jsobj.py +++ b/js/jsobj.py @@ -1042,8 +1042,8 @@ return True def ToNumber(self): - from js.builtins_global import _strip - from runistr import encode_unicode_utf8 + from js.builtins.js_global import _strip + from js.runistr import encode_unicode_utf8 from js.constants import hex_rexp, oct_rexp, num_rexp u_strval = self._strval_ @@ -1067,7 +1067,7 @@ return float(num_lit) - from builtins_global import _parse_int + from js.builtins.js_global import _parse_int match_data = hex_rexp.match(s) if match_data is not None: diff --git a/test/test_helpers.py b/test/test_helpers.py --- a/test/test_helpers.py +++ b/test/test_helpers.py @@ -1,13 +1,12 @@ -import py - def test_string_match_chars(): - from js.builtins_global import _string_match_chars + from js.builtins.js_global import _string_match_chars assert _string_match_chars(u'abccab', u'abc') is True assert _string_match_chars(u'ABCcba', u'abc') is True assert _string_match_chars(u'2921', u'0123456789') is True assert _string_match_chars(u'abcdabcd', u'abc') is False assert _string_match_chars(u'29x21', u'0123456789') is False + def test_unescape(): from js import runistr assert runistr.unicode_unescape(u'\\u0041B\\u0043') == u'ABC' diff --git a/test/test_w_string.py b/test/test_w_string.py --- a/test/test_w_string.py +++ b/test/test_w_string.py @@ -11,7 +11,7 @@ def test_isspace(): - from js.builtins_global import _strip + from js.builtins.js_global import _strip assert _strip(u' ') == u'' assert _strip(u' ') == u'' assert _strip(u' \t\t\r\n ') == u'' From noreply at buildbot.pypy.org Tue Jan 8 20:06:47 2013 From: noreply at buildbot.pypy.org (stepahn) Date: Tue, 8 Jan 2013 20:06:47 +0100 (CET) Subject: [pypy-commit] lang-js default: moved _w to js.object_space Message-ID: <20130108190647.705A41C1305@cobra.cs.uni-duesseldorf.de> Author: Stephan Branch: Changeset: r335:77a0a486f6e4 Date: 2013-01-08 18:03 +0100 http://bitbucket.org/pypy/lang-js/changeset/77a0a486f6e4/ Log: moved _w to js.object_space diff --git a/js/baseop.py b/js/baseop.py --- a/js/baseop.py +++ b/js/baseop.py @@ -2,7 +2,8 @@ """ Base operations implementations """ -from js.jsobj import W_String, W_IntNumber, W_FloatNumber, _w +from js.jsobj import W_String, W_IntNumber, W_FloatNumber +from js.object_space import _w from pypy.rlib.rarithmetic import ovfcheck from pypy.rlib.rfloat import isnan, isinf diff --git a/js/builtins/__init__.py b/js/builtins/__init__.py --- a/js/builtins/__init__.py +++ b/js/builtins/__init__.py @@ -1,4 +1,5 @@ -from js.jsobj import w_Undefined, _w +from js.jsobj import w_Undefined +from js.object_space import _w #from pypy.rlib import jit diff --git a/js/builtins/array.py b/js/builtins/array.py --- a/js/builtins/array.py +++ b/js/builtins/array.py @@ -1,6 +1,6 @@ -from js.jsobj import isnull_or_undefined, _w, w_Undefined +from js.jsobj import isnull_or_undefined, w_Undefined from js.builtins import get_arg -from js.object_space import w_return +from js.object_space import w_return, _w def setup(global_object): diff --git a/js/builtins/boolean.py b/js/builtins/boolean.py --- a/js/builtins/boolean.py +++ b/js/builtins/boolean.py @@ -1,7 +1,6 @@ from js.jsobj import W_Boolean, W_BooleanObject from js.execution import JsTypeError -from js.jsobj import _w -from js.object_space import w_return +from js.object_space import w_return, _w def setup(global_object): diff --git a/js/builtins/date.py b/js/builtins/date.py --- a/js/builtins/date.py +++ b/js/builtins/date.py @@ -1,9 +1,8 @@ from pypy.rlib.rfloat import NAN, isnan -from js.jsobj import _w import datetime from js.builtins import get_arg -from js.object_space import w_return, hide_on_translate +from js.object_space import w_return, hide_on_translate, _w from pypy.rlib.objectmodel import we_are_translated diff --git a/js/builtins/function.py b/js/builtins/function.py --- a/js/builtins/function.py +++ b/js/builtins/function.py @@ -1,9 +1,9 @@ from js.jsobj import isnull_or_undefined from js.execution import JsTypeError -from js.jsobj import w_Undefined, _w +from js.jsobj import w_Undefined from js.builtins import get_arg from js.completion import NormalCompletion -from js.object_space import w_return +from js.object_space import w_return, _w @w_return diff --git a/js/builtins/js_math.py b/js/builtins/js_math.py --- a/js/builtins/js_math.py +++ b/js/builtins/js_math.py @@ -1,9 +1,8 @@ import math -from js.jsobj import _w from pypy.rlib.rfloat import NAN, INFINITY, isnan, isinf from js.builtins import get_arg -from js.object_space import w_return +from js.object_space import w_return, _w def setup(global_object): diff --git a/js/builtins/number.py b/js/builtins/number.py --- a/js/builtins/number.py +++ b/js/builtins/number.py @@ -1,7 +1,7 @@ from pypy.rlib.rfloat import NAN, INFINITY from js.execution import JsRangeError, JsTypeError -from js.jsobj import W_Number, W_NumericObject, _w -from js.object_space import w_return +from js.jsobj import W_Number, W_NumericObject +from js.object_space import w_return, _w def setup(global_object): diff --git a/js/builtins/object.py b/js/builtins/object.py --- a/js/builtins/object.py +++ b/js/builtins/object.py @@ -1,4 +1,4 @@ -from js.jsobj import _w +from js.object_space import _w def to_string(this, args): diff --git a/js/builtins/string.py b/js/builtins/string.py --- a/js/builtins/string.py +++ b/js/builtins/string.py @@ -1,8 +1,8 @@ -from js.jsobj import _w, w_Undefined, W_String, W_StringObject +from js.jsobj import w_Undefined, W_String, W_StringObject from pypy.rlib.rfloat import NAN from js.execution import JsTypeError from js.builtins import get_arg -from js.object_space import w_return +from js.object_space import w_return, _w from pypy.rlib.rstring import UnicodeBuilder diff --git a/js/execution.py b/js/execution.py --- a/js/execution.py +++ b/js/execution.py @@ -40,7 +40,7 @@ return self.message def msg(self): - from js.jsobj import _w + from js.object_space import _w return _w(self._msg()) diff --git a/js/functions.py b/js/functions.py --- a/js/functions.py +++ b/js/functions.py @@ -1,4 +1,4 @@ -from js.jsobj import _w +from js.object_space import _w class JsBaseFunction(object): diff --git a/js/jsobj.py b/js/jsobj.py --- a/js/jsobj.py +++ b/js/jsobj.py @@ -559,6 +559,7 @@ _class_ = 'String' def __init__(self, primitive_value): + from js.object_space import _w W__PrimitiveObject.__init__(self, primitive_value) length = len(self._primitive_value_.to_string()) descr = PropertyDescriptor(value=_w(length), enumerable=False, configurable=False, writable=False) @@ -580,6 +581,7 @@ return None result_string = string[index] + from js.object_space import _w d = PropertyDescriptor(value=_w(result_string), enumerable=True, writable=False, configurable=False) return d @@ -730,6 +732,7 @@ class W_NumberConstructor(W_BasicFunction): # 15.7.1.1 def Call(self, args=[], this=None, calling_context=None): + from js.object_space import _w if len(args) >= 1 and not isnull_or_undefined(args[0]): return _w(args[0].ToNumber()) elif len(args) >= 1 and args[0] is w_Undefined: @@ -749,6 +752,7 @@ class W_StringConstructor(W_BasicFunction): def Call(self, args=[], this=None, calling_context=None): from js.builtins import get_arg + from js.object_space import _w arg0 = get_arg(args, 0, _w(u"")) strval = arg0.to_string() return W_String(strval) @@ -763,6 +767,7 @@ # 15.6.2 class W_BooleanConstructor(W_BasicFunction): def Call(self, args=[], this=None, calling_context=None): + from js.object_space import _w if len(args) >= 1 and not isnull_or_undefined(args[0]): boolval = args[0].to_boolean() return _w(boolval) @@ -808,6 +813,7 @@ # value = _w(int(num)) #else: # value = _w(int(time.time() * 1000)) + from js.object_space import _w value = _w(int(time.time() * 1000)) from js.object_space import object_space @@ -828,6 +834,7 @@ def __init__(self, function_body, formal_parameter_list=[], scope=None, strict=False): W_BasicFunction.__init__(self) from js.object_space import object_space + from js.object_space import _w self._function_ = function_body self._scope_ = scope self._params_ = formal_parameter_list @@ -902,6 +909,7 @@ _class_ = 'Arguments' def __init__(self, func, names, args, env, strict=False): + from js.object_space import _w W__Object.__init__(self) self.strict = strict _len = len(args) @@ -948,6 +956,7 @@ # 15.4.2 class W_ArrayConstructor(W_BasicFunction): def __init__(self): + from js.object_space import _w W_BasicFunction.__init__(self) put_property(self, u'length', _w(1), writable=False, enumerable=False, configurable=False) @@ -956,6 +965,7 @@ def Call(self, args=[], this=None, calling_context=None): from js.object_space import object_space + from js.object_space import _w if len(args) == 1: _len = args[0] @@ -1268,6 +1278,7 @@ # 15.4.5.1 def define_own_property(self, p, desc, throw=False): + from js.object_space import _w old_len_desc = self.get_own_property(u'length') assert old_len_desc is not None old_len = old_len_desc.value.ToUInt32() @@ -1351,36 +1362,6 @@ return W_BasicObject.define_own_property(self, p, desc, throw) -from pypy.rlib.objectmodel import specialize - - - at specialize.argtype(0) -def _w(value): - if value is None: - return w_Null - elif isinstance(value, W_Root): - return value - elif isinstance(value, bool): - return newbool(value) - elif isinstance(value, int): - return W_IntNumber(value) - elif isinstance(value, float): - return W_FloatNumber(value) - elif isinstance(value, unicode): - return W_String(value) - elif isinstance(value, str): - u_str = unicode(value) - return W_String(u_str) - elif isinstance(value, list): - from js.object_space import object_space - a = object_space.new_array() - for index, item in enumerate(value): - put_property(a, unicode(str(index)), _w(item), writable=True, enumerable=True, configurable=True) - return a - - raise TypeError("ffffuuu %s" % (value,)) - - def put_property(obj, name, value, writable=False, configurable=False, enumerable=False, throw=False): descriptor = PropertyDescriptor(value=value, writable=writable, configurable=configurable, enumerable=enumerable) obj.define_own_property(name, descriptor, throw) diff --git a/js/object_space.py b/js/object_space.py --- a/js/object_space.py +++ b/js/object_space.py @@ -1,8 +1,36 @@ -from js.jsobj import _w, W__Object, W_BasicFunction, W__Function, W_DateObject, W_BooleanObject, W_StringObject, W_NumericObject, W__Array, w_Null +from pypy.rlib.objectmodel import specialize + + + at specialize.argtype(0) +def _w(value): + from js.jsobj import w_Null, newbool, W_IntNumber, W_FloatNumber, W_String, W_Root, put_property + if value is None: + return w_Null + elif isinstance(value, W_Root): + return value + elif isinstance(value, bool): + return newbool(value) + elif isinstance(value, int): + return W_IntNumber(value) + elif isinstance(value, float): + return W_FloatNumber(value) + elif isinstance(value, unicode): + return W_String(value) + elif isinstance(value, str): + u_str = unicode(value) + return W_String(u_str) + elif isinstance(value, list): + a = object_space.new_array() + for index, item in enumerate(value): + put_property(a, unicode(str(index)), _w(item), writable=True, enumerable=True, configurable=True) + return a + + raise TypeError("ffffuuu %s" % (value,)) class ObjectSpace(object): def __init__(self): + from js.jsobj import w_Null self.global_context = None self.global_object = None self.proto_function = w_Null @@ -18,6 +46,7 @@ return self.global_context.variable_environment() def assign_proto(self, obj, proto=None): + from js.jsobj import W_BasicFunction, W_DateObject, W_BooleanObject, W_StringObject, W_NumericObject, W__Array if proto is not None: obj._prototype_ = proto return obj @@ -39,46 +68,53 @@ return obj def new_obj(self): + from js.jsobj import W__Object obj = W__Object() self.assign_proto(obj) return obj def new_func(self, function_body, formal_parameter_list=[], scope=None, strict=False): + from js.jsobj import W__Function obj = W__Function(function_body, formal_parameter_list, scope, strict) self.assign_proto(obj) return obj def new_date(self, value): + from js.jsobj import W_DateObject obj = W_DateObject(value) self.assign_proto(obj) return obj def new_array(self, length=_w(0)): + from js.jsobj import W__Array obj = W__Array(length) self.assign_proto(obj) return obj def new_bool(self, value): + from js.jsobj import W_BooleanObject obj = W_BooleanObject(value) self.assign_proto(obj) return obj def new_string(self, value): + from js.jsobj import W_StringObject obj = W_StringObject(value) self.assign_proto(obj) return obj def new_number(self, value): + from js.jsobj import W_NumericObject obj = W_NumericObject(value) self.assign_proto(obj) return obj + object_space = ObjectSpace() def w_return(fn): def f(*args): - from js.jsobj import _w return _w(fn(*args)) return f diff --git a/js/opcodes.py b/js/opcodes.py --- a/js/opcodes.py +++ b/js/opcodes.py @@ -1,5 +1,6 @@ from js.jsobj import W_IntNumber, W_FloatNumber, W_String, \ - w_Undefined, newbool, w_Null, _w + w_Undefined, newbool, w_Null +from js.object_space import _w from js.execution import JsTypeError from js.baseop import plus, sub, compare, AbstractEC, StrictEC,\ compare_e, increment, decrement, mult, division, uminus, mod diff --git a/test/ecma/conftest.py b/test/ecma/conftest.py --- a/test/ecma/conftest.py +++ b/test/ecma/conftest.py @@ -3,7 +3,7 @@ from js.interpreter import Interpreter, load_file from _pytest.runner import Failed -from js.jsobj import _w +from js.object_space import _w from js.execution import JsException from pypy.rlib.parsing.parsing import ParseError diff --git a/test/test_interp.py b/test/test_interp.py --- a/test/test_interp.py +++ b/test/test_interp.py @@ -54,7 +54,7 @@ def assertv(code, value): from js.interpreter import Interpreter - from js.jsobj import _w + from js.object_space import _w jsint = Interpreter() ret_val = jsint.run_src(code) diff --git a/test/test_interpreter.py b/test/test_interpreter.py --- a/test/test_interpreter.py +++ b/test/test_interpreter.py @@ -1,4 +1,4 @@ -from js.jsobj import _w +from js.object_space import _w from js.interpreter import Interpreter from js.astbuilder import parse_to_ast diff --git a/test/test_jsfunction.py b/test/test_jsfunction.py --- a/test/test_jsfunction.py +++ b/test/test_jsfunction.py @@ -1,4 +1,4 @@ -from js.jsobj import _w +from js.object_space import _w from js.jscode import JsCode from js.execution_context import ExecutionContext, FunctionExecutionContext, GlobalExecutionContext, EvalExecutionContext from js.functions import JsFunction, JsExecutableCode, JsNativeFunction, JsGlobalCode, JsEvalCode From noreply at buildbot.pypy.org Tue Jan 8 20:06:48 2013 From: noreply at buildbot.pypy.org (stepahn) Date: Tue, 8 Jan 2013 20:06:48 +0100 (CET) Subject: [pypy-commit] lang-js default: removed unused files Message-ID: <20130108190648.775921C1305@cobra.cs.uni-duesseldorf.de> Author: Stephan Branch: Changeset: r336:ef8b7e480059 Date: 2013-01-08 20:05 +0100 http://bitbucket.org/pypy/lang-js/changeset/ef8b7e480059/ Log: removed unused files diff --git a/js/console.py b/js/console.py deleted file mode 100644 --- a/js/console.py +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env python - -import os, sys -from js.interpreter import load_source, Interpreter, load_file -from js.jsparser import parse, ParseError -from js.jsobj import W_NewBuiltin, W_String, JsThrowException, w_Undefined -from pypy.rlib.streamio import open_file_as_stream - -def printmessage(msg): - if msg is not None: - os.write(1, msg) - -def readline(): - result = [] - while 1: - s = os.read(0, 1) - if s == '\n': - break - else: - result.append(s) - if s == '': - if len(result) > 1: - break - raise SystemExit - return ''.join(result) - -class W_Quit(W_NewBuiltin): - def Call(self, ctx, args=[], this=None): - raise SystemExit, 0 - -class W_Load(W_NewBuiltin): - def __init__(self, ctx, interpreter): - W_NewBuiltin.__init__(self, ctx) - self.interpreter = interpreter - - def Call(self, ctx, args=[], this=None): - if len(args) >= 1: - filename = args[0].to_string(self.interpreter.global_context) - try: - assert filename is not None - program = load_file(filename) - except EnvironmentError, e: - msg = W_String(u"Can't open %s: %s" % (filename, e)) - raise JsThrowException(msg) - self.interpreter.run(program) - return w_Undefined - -class W_ReadLine(W_NewBuiltin): - def Call(self, ctx, args=[], this=None): - line = unicode(readline()) - return W_String(line) - -class JSConsole(object): - prompt_ok = 'js> ' - prompt_more = '... ' - - def __init__(self): - interp = Interpreter() - ctx = interp.global_context - - interp.w_Global.Put('quit', W_Quit(ctx)) - interp.w_Global.Put('load', W_Load(ctx, interp)) - interp.w_Global.Put('readline', W_ReadLine(ctx)) - - self.interpreter = interp - - def runsource(self, source, filename=''): - try: - ast = load_source(source, filename) - except ParseError, exc: - if exc.source_pos.i == len(source): - # more input needed - return True - else: - # syntax error - self.showsyntaxerror(filename, exc) - return False - - # execute it - self.runcode(ast) - return False - - def runcode(self, ast): - """Run the javascript code in the AST. All exceptions raised - by javascript code must be caught and handled here. When an - exception occurs, self.showtraceback() is called to display a - traceback. - """ - try: - res = self.interpreter.run(ast, interactive=True) - if res is not None and res != w_Undefined: - try: - printmessage(res.to_string(self.interpreter.global_context)) - except ThrowException, exc: - printmessage(exc.exception.to_string(self.interpreter.global_context)) - printmessage('\n') - except SystemExit: - raise - except ThrowException, exc: - self.showtraceback(exc) - - def showtraceback(self, exc): - printmessage(exc.exception.to_string(self.interpreter.global_context)) - printmessage('\n') - - def showsyntaxerror(self, filename, exc): - printmessage(' ' * 4 + \ - ' ' * exc.source_pos.columnno + \ - '^\n') - printmessage('Syntax Error\n') - - def interact(self): - printmessage('PyPy JavaScript Interpreter\n') - printmessage(self.prompt_ok) - - # input for the current statement - lines = [] - - while True: - try: - line = readline() - except SystemExit, e: - printmessage('\n') - return - - lines.append(line) - - source = ''.join(lines) - need_more = self.runsource(source) - - if need_more: - printmessage(self.prompt_more) - else: - printmessage(self.prompt_ok) - lines = [] - -if __name__ == '__main__': - console = JSConsole() - console.interact() diff --git a/js/contants.py b/js/contants.py deleted file mode 100644 --- a/js/contants.py +++ /dev/null @@ -1,6 +0,0 @@ -from js import jsobj - -w_Null = jsobj.W_Null() -w_Undefined = jsobj.W_Undefined() -w_False = jsobj.W_Boolean(False) -w_True = jsobj.W_Boolean(True) From noreply at buildbot.pypy.org Tue Jan 8 20:06:49 2013 From: noreply at buildbot.pypy.org (stepahn) Date: Tue, 8 Jan 2013 20:06:49 +0100 (CET) Subject: [pypy-commit] lang-js default: refactored boxing Message-ID: <20130108190649.84C441C1305@cobra.cs.uni-duesseldorf.de> Author: Stephan Branch: Changeset: r337:8a151cf97de6 Date: 2013-01-08 20:06 +0100 http://bitbucket.org/pypy/lang-js/changeset/8a151cf97de6/ Log: refactored boxing diff --git a/js/builtins/__init__.py b/js/builtins/__init__.py --- a/js/builtins/__init__.py +++ b/js/builtins/__init__.py @@ -1,4 +1,3 @@ -from js.jsobj import w_Undefined from js.object_space import _w #from pypy.rlib import jit @@ -127,7 +126,10 @@ js.builtins.js_global.setup(global_object) -def get_arg(args, index, default=w_Undefined): +from js.object_space import newundefined + + +def get_arg(args, index, default=newundefined()): if len(args) > index: return args[index] return default diff --git a/js/builtins/array.py b/js/builtins/array.py --- a/js/builtins/array.py +++ b/js/builtins/array.py @@ -1,6 +1,5 @@ -from js.jsobj import isnull_or_undefined, w_Undefined from js.builtins import get_arg -from js.object_space import w_return, _w +from js.object_space import w_return, _w, isnull_or_undefined, newundefined def setup(global_object): @@ -73,13 +72,15 @@ # 15.4.4.5 @w_return def join(this, args): + from js.object_space import isundefined + separator = get_arg(args, 0) o = this.ToObject() len_val = o.get(u'length') length = len_val.ToUInt32() - if separator is w_Undefined: + if isundefined(separator): sep = u',' else: sep = separator.to_string() @@ -117,7 +118,7 @@ if l == 0: o.put(u'length', _w(0)) - return w_Undefined + return newundefined() else: indx = l - 1 indxs = unicode(str(indx)) @@ -188,7 +189,9 @@ return obj -def sort_compare(obj, j, k, comparefn=w_Undefined): +def sort_compare(obj, j, k, comparefn=newundefined()): + from js.object_space import isundefined + j_string = j k_string = k has_j = obj.has_property(j) @@ -204,19 +207,19 @@ x = obj.get(j_string) y = obj.get(k_string) - if x is w_Undefined and y is w_Undefined: + if isundefined(x) and isundefined(y): return 0 - if x is w_Undefined: + if isundefined(x): return 1 - if y is w_Undefined: + if isundefined(y): return -1 - if comparefn is not w_Undefined: + if not isundefined(comparefn): if not comparefn.is_callable(): from js.execution import JsTypeError raise JsTypeError(u'') - res = comparefn.Call(args=[x, y], this=w_Undefined) + res = comparefn.Call(args=[x, y], this=newundefined()) return res.ToInteger() x_string = x.to_string() diff --git a/js/builtins/function.py b/js/builtins/function.py --- a/js/builtins/function.py +++ b/js/builtins/function.py @@ -1,6 +1,4 @@ -from js.jsobj import isnull_or_undefined from js.execution import JsTypeError -from js.jsobj import w_Undefined from js.builtins import get_arg from js.completion import NormalCompletion from js.object_space import w_return, _w @@ -17,7 +15,8 @@ @w_return def empty(this, args): - return w_Undefined + from js.object_space import newundefined + return newundefined() # 15.3.4.4 Function.prototype.call @@ -38,6 +37,7 @@ # 15.3.4.3 Function.prototype.apply (thisArg, argArray) def js_apply(ctx): + from js.object_space import isnull_or_undefined func = ctx.this_binding() args = ctx.argv() diff --git a/js/builtins/js_global.py b/js/builtins/js_global.py --- a/js/builtins/js_global.py +++ b/js/builtins/js_global.py @@ -10,8 +10,8 @@ from js.builtins import put_intimate_function, put_native_function, put_property from js.builtins.number import w_NAN from js.builtins.number import w_POSITIVE_INFINITY - from js.jsobj import w_Undefined from pypy.rlib.objectmodel import we_are_translated + from js.object_space import newundefined # 15.1.1.1 put_property(global_object, u'NaN', w_NAN, writable=False, enumerable=False, configurable=False) @@ -20,7 +20,7 @@ put_property(global_object, u'Infinity', w_POSITIVE_INFINITY, writable=False, enumerable=False, configurable=False) # 15.1.1.3 - put_property(global_object, u'undefined', w_Undefined, writable=False, enumerable=False, configurable=False) + put_property(global_object, u'undefined', newundefined(), writable=False, enumerable=False, configurable=False) # 15.1.2.1 put_intimate_function(global_object, u'eval', js_eval, params=[u'x']) diff --git a/js/builtins/string.py b/js/builtins/string.py --- a/js/builtins/string.py +++ b/js/builtins/string.py @@ -1,4 +1,4 @@ -from js.jsobj import w_Undefined, W_String, W_StringObject +from js.jsobj import W_String, W_StringObject from pypy.rlib.rfloat import NAN from js.execution import JsTypeError from js.builtins import get_arg @@ -111,10 +111,7 @@ # 15.5.4.4 @w_return def char_at(this, args): - pos = w_Undefined - - if len(args) > 0: - pos = args[0] + pos = get_arg(args, 0) position = pos.ToInt32() @@ -230,6 +227,8 @@ # 15.5.4.14 @w_return def split(this, args): + from js.object_space import isundefined + this.check_object_coercible() separator = get_arg(args, 0, None) @@ -237,7 +236,7 @@ string = this.to_string() - if limit is w_Undefined: + if isundefined(limit): import math lim = int(math.pow(2, 32) - 1) else: diff --git a/js/environment_record.py b/js/environment_record.py --- a/js/environment_record.py +++ b/js/environment_record.py @@ -1,4 +1,3 @@ -from js.jsobj import w_Undefined from js.object_map import ROOT_MAP @@ -89,9 +88,10 @@ # 10.2.1.1.2 def create_mutuable_binding(self, identifier, deletable): + from js.object_space import newundefined assert identifier is not None and isinstance(identifier, unicode) assert not self.has_binding(identifier) - self._add_binding(identifier, w_Undefined) + self._add_binding(identifier, newundefined()) self._set_mutable_binding(identifier) if deletable: self._set_deletable_binding(identifier) @@ -107,13 +107,14 @@ # 10.2.1.1.4 def get_binding_value(self, identifier, strict=False): + from js.object_space import newundefined assert identifier is not None and isinstance(identifier, unicode) if not self.has_binding(identifier): if strict: from js.execution import JsReferenceError raise JsReferenceError(identifier) else: - return w_Undefined + return newundefined() return self._get_binding(identifier) # 10.2.1.1.5 @@ -132,7 +133,8 @@ # 10.2.1.1.6 def implicit_this_value(self): - return w_Undefined + from js.object_space import newundefined + return newundefined() # 10.2.1.1.7 def create_immutable_bining(self, identifier): @@ -169,7 +171,8 @@ config_value = True from js.jsobj import PropertyDescriptor - desc = PropertyDescriptor(value=w_Undefined, writable=True, enumerable=True, configurable=config_value) + from js.object_space import newundefined + desc = PropertyDescriptor(value=newundefined(), writable=True, enumerable=True, configurable=config_value) bindings.define_own_property(n, desc, True) # 10.2.1.2.3 @@ -180,12 +183,13 @@ # 10.2.1.2.4 def get_binding_value(self, n, s=False): + from js.object_space import newundefined assert n is not None and isinstance(n, unicode) bindings = self.binding_object value = bindings.has_property(n) if value is False: if s is False: - return w_Undefined + return newundefined() else: from execution import JsReferenceError raise JsReferenceError(self.__class__) @@ -200,9 +204,10 @@ # 10.2.1.2.6 def implicit_this_value(self): + from js.object_space import newundefined if self.provide_this is True: return self.binding_object - return w_Undefined + return newundefined() class GlobalEnvironmentRecord(ObjectEnvironmentRecord): diff --git a/js/execution_context.py b/js/execution_context.py --- a/js/execution_context.py +++ b/js/execution_context.py @@ -1,5 +1,5 @@ -from js.jsobj import w_Undefined from js.utils import StackMixin +from js.object_space import newundefined class ExecutionContext(StackMixin): @@ -52,6 +52,8 @@ # 10.5 def declaration_binding_initialization(self): + from js.object_space import newundefined + env = self._variable_environment_.environment_record strict = self._strict_ code = self._code_ @@ -71,7 +73,7 @@ for arg_name in names: n += 1 if n > arg_count: - v = w_Undefined + v = newundefined() else: v = args[n - 1] arg_already_declared = env.has_binding(arg_name) @@ -113,7 +115,7 @@ var_already_declared = env.has_binding(dn) if var_already_declared is False: env.create_mutuable_binding(dn, configurable_bindings) - env.set_mutable_binding(dn, w_Undefined, False) + env.set_mutable_binding(dn, newundefined(), False) def _get_refs(self, index): assert index <= len(self._refs_) @@ -191,9 +193,9 @@ _immutable_fields_ = ['_scope_', '_calling_context_'] _refs_resizable_ = False - def __init__(self, code, formal_parameters=[], argv=[], this=w_Undefined, strict=False, scope=None, w_func=None): - from js.jsobj import isnull_or_undefined, W_BasicObject - from js.object_space import object_space + def __init__(self, code, formal_parameters=[], argv=[], this=newundefined(), strict=False, scope=None, w_func=None): + from js.jsobj import W_BasicObject + from js.object_space import object_space, isnull_or_undefined stack_size = code.estimated_stack_size() env_size = code.env_size() + 1 # neet do add one for the arguments object diff --git a/js/jsobj.py b/js/jsobj.py --- a/js/jsobj.py +++ b/js/jsobj.py @@ -132,12 +132,6 @@ def ToObject(self): raise JsTypeError(u'W_Null.ToObject') -w_Undefined = W_Undefined() -jit.promote(w_Undefined) - -w_Null = W_Null() -jit.promote(w_Null) - class PropertyIdenfidier(object): def __init__(self, name, descriptor): @@ -187,10 +181,11 @@ _immutable_fields_ = ['_type_', '_class_', '_extensible_'] def __init__(self): + from js.object_space import newnull self._property_map_ = _new_map() self._property_slots_ = [] - self._prototype_ = w_Null + self._prototype_ = newnull() W_BasicObject.define_own_property(self, u'__proto__', proto_desc) def __str__(self): @@ -209,17 +204,18 @@ # 8.12.3 def get(self, p): + from js.object_space import newundefined assert p is not None and isinstance(p, unicode) desc = self.get_property(p) if desc is None: - return w_Undefined + return newundefined() if is_data_descriptor(desc): return desc.value if desc.has_set_getter() is False: - return w_Undefined + return newundefined() getter = desc.getter res = getter.Call(this=self) @@ -271,6 +267,7 @@ # 8.12.2 def get_property(self, p): + from js.object_space import isnull assert p is not None and isinstance(p, unicode) prop = self.get_own_property(p) @@ -278,7 +275,7 @@ return prop proto = self.prototype() - if proto is w_Null: + if isnull(proto): return None assert isinstance(proto, W_BasicObject) @@ -311,10 +308,11 @@ # 8.12.4 def can_put(self, p): + from js.object_space import isundefined, isnull_or_undefined desc = self.get_own_property(p) if desc is not None: if is_accessor_descriptor(desc) is True: - if desc.setter is w_Undefined: + if isundefined(desc.setter): return False else: return True @@ -322,7 +320,7 @@ proto = self.prototype() - if proto is w_Null or proto is w_Undefined: + if isnull_or_undefined(proto): return self.extensible() assert isinstance(proto, W_BasicObject) @@ -331,7 +329,7 @@ return self.extensible() if is_accessor_descriptor(inherited) is True: - if inherited.setter is w_Undefined: + if isundefined(inherited.setter): return False else: return True @@ -511,6 +509,7 @@ ### def _named_properties_dict(self): + from js.object_space import isnull_or_undefined my_d = {} for i in self._property_map_.keys(): my_d[i] = None @@ -637,6 +636,7 @@ # 15.3.5.3 def has_instance(self, v): + from js.object_space import isnull_or_undefined if not isinstance(v, W_BasicObject): return False @@ -656,6 +656,7 @@ class W_ObjectConstructor(W_BasicFunction): def Call(self, args=[], this=None, calling_context=None): + from js.object_space import isnull_or_undefined from js.builtins import get_arg value = get_arg(args, 0) @@ -732,10 +733,11 @@ class W_NumberConstructor(W_BasicFunction): # 15.7.1.1 def Call(self, args=[], this=None, calling_context=None): - from js.object_space import _w + from js.object_space import _w, isnull_or_undefined, isundefined + if len(args) >= 1 and not isnull_or_undefined(args[0]): return _w(args[0].ToNumber()) - elif len(args) >= 1 and args[0] is w_Undefined: + elif len(args) >= 1 and isundefined(args[0]): return _w(NAN) else: return _w(0.0) @@ -767,7 +769,7 @@ # 15.6.2 class W_BooleanConstructor(W_BasicFunction): def Call(self, args=[], this=None, calling_context=None): - from js.object_space import _w + from js.object_space import _w, isnull_or_undefined if len(args) >= 1 and not isnull_or_undefined(args[0]): boolval = args[0].to_boolean() return _w(boolval) @@ -833,8 +835,7 @@ def __init__(self, function_body, formal_parameter_list=[], scope=None, strict=False): W_BasicFunction.__init__(self) - from js.object_space import object_space - from js.object_space import _w + from js.object_space import _w, newnull, object_space self._function_ = function_body self._scope_ = scope self._params_ = formal_parameter_list @@ -855,8 +856,8 @@ if strict is True: raise NotImplementedError() else: - put_property(self, u'caller', w_Null, writable=True, enumerable=False, configurable=False) - put_property(self, u'arguments', w_Null, writable=True, enumerable=False, configurable=False) + put_property(self, u'caller', newnull(), writable=True, enumerable=False, configurable=False) + put_property(self, u'arguments', newnull(), writable=True, enumerable=False, configurable=False) def _to_string(self): return self._function_.to_string() @@ -1220,24 +1221,6 @@ return intmask(int(self._floatval_)) -def isnull_or_undefined(obj): - if obj is w_Null or obj is w_Undefined: - return True - return False - -w_True = W_Boolean(True) -jit.promote(w_True) - -w_False = W_Boolean(False) -jit.promote(w_False) - - -def newbool(val): - if val: - return w_True - return w_False - - class W_List(W_Root): def __init__(self, values): self.values = values diff --git a/js/object_space.py b/js/object_space.py --- a/js/object_space.py +++ b/js/object_space.py @@ -1,24 +1,102 @@ -from pypy.rlib.objectmodel import specialize +from pypy.rlib.objectmodel import specialize, enforceargs +from pypy.rlib import jit + + + at enforceargs(int) +def newint(i): + from js.jsobj import W_IntNumber + return W_IntNumber(i) + + + at enforceargs(float) +def newfloat(f): + from js.jsobj import W_FloatNumber + return W_FloatNumber(f) + + + at enforceargs(unicode) +def newstring(s): + from js.jsobj import W_String + return W_String(s) + + + at enforceargs(bool) +def _makebool(b): + from js.jsobj import W_Boolean + return W_Boolean(b) + + +w_True = _makebool(True) +jit.promote(w_True) + + +w_False = _makebool(False) +jit.promote(w_False) + + +def _makeundefined(): + from js.jsobj import W_Undefined + return W_Undefined() + +w_Undefined = _makeundefined() +jit.promote(w_Undefined) + + +def _makenull(): + from js.jsobj import W_Null + return W_Null() + +w_Null = _makenull() +jit.promote(w_Null) + + +def isnull(value): + return value is w_Null + + +def newnull(): + return w_Null + + +def isundefined(value): + return value is w_Undefined + + +def newundefined(): + return w_Undefined + + +def isnull_or_undefined(obj): + if isnull(obj) or isundefined(obj): + return True + return False + + + at enforceargs(bool) +def newbool(val): + if val is True: + return w_True + return w_False @specialize.argtype(0) def _w(value): - from js.jsobj import w_Null, newbool, W_IntNumber, W_FloatNumber, W_String, W_Root, put_property + from js.jsobj import W_Root, put_property if value is None: - return w_Null + return newnull() elif isinstance(value, W_Root): return value elif isinstance(value, bool): return newbool(value) elif isinstance(value, int): - return W_IntNumber(value) + return newint(value) elif isinstance(value, float): - return W_FloatNumber(value) + return newfloat(value) elif isinstance(value, unicode): - return W_String(value) + return newstring(value) elif isinstance(value, str): u_str = unicode(value) - return W_String(u_str) + return newstring(u_str) elif isinstance(value, list): a = object_space.new_array() for index, item in enumerate(value): @@ -30,16 +108,15 @@ class ObjectSpace(object): def __init__(self): - from js.jsobj import w_Null self.global_context = None self.global_object = None - self.proto_function = w_Null - self.proto_boolean = w_Null - self.proto_number = w_Null - self.proto_string = w_Null - self.proto_array = w_Null - self.proto_date = w_Null - self.proto_object = w_Null + self.proto_function = newnull() + self.proto_boolean = newnull() + self.proto_number = newnull() + self.proto_string = newnull() + self.proto_array = newnull() + self.proto_date = newnull() + self.proto_object = newnull() self.interpreter = None def get_global_environment(self): diff --git a/js/opcodes.py b/js/opcodes.py --- a/js/opcodes.py +++ b/js/opcodes.py @@ -1,5 +1,3 @@ -from js.jsobj import W_IntNumber, W_FloatNumber, W_String, \ - w_Undefined, newbool, w_Null from js.object_space import _w from js.execution import JsTypeError from js.baseop import plus, sub, compare, AbstractEC, StrictEC,\ @@ -31,9 +29,13 @@ _stack_change = 0 def eval(self, ctx): + from js.object_space import newbool s4 = ctx.stack_pop() s2 = ctx.stack_pop() - ctx.stack_append(self.decision(ctx, s2, s4)) + res = self.decision(ctx, s2, s4) + # XXX mimik behaviour of old newbool + res_true = res is True + ctx.stack_append(newbool(res_true)) def decision(self, ctx, op1, op2): raise NotImplementedError @@ -66,14 +68,16 @@ class Undefined(Opcode): def eval(self, ctx): - ctx.stack_append(w_Undefined) + from js.object_space import newundefined + ctx.stack_append(newundefined()) class LOAD_INTCONSTANT(Opcode): _immutable_fields_ = ['w_intvalue'] def __init__(self, value): - self.w_intvalue = W_IntNumber(int(value)) + from js.object_space import newint + self.w_intvalue = newint(int(value)) def eval(self, ctx): ctx.stack_append(self.w_intvalue) @@ -84,20 +88,22 @@ class LOAD_BOOLCONSTANT(Opcode): def __init__(self, value): - self.boolval = value + from js.object_space import newbool + self.w_boolval = newbool(value) def eval(self, ctx): - ctx.stack_append(newbool(self.boolval)) + ctx.stack_append(self.w_boolval) def __str__(self): - if self.boolval: + if self.w_boolval.to_boolean(): return 'LOAD_BOOLCONSTANT true' return 'LOAD_BOOLCONSTANT false' class LOAD_FLOATCONSTANT(Opcode): def __init__(self, value): - self.w_floatvalue = W_FloatNumber(float(value)) + from js.object_space import newfloat + self.w_floatvalue = newfloat(float(value)) def eval(self, ctx): ctx.stack_append(self.w_floatvalue) @@ -110,27 +116,27 @@ _immutable_fields_ = ['strval'] def __init__(self, value): - #assert isinstance(value, unicode) - self.strval = value + from js.object_space import newstring + self.w_strval = newstring(value) def eval(self, ctx): - strval = self.strval - #assert isinstance(strval, unicode) - w_string = W_String(strval) - ctx.stack_append(w_string) + w_strval = self.w_strval + ctx.stack_append(w_strval) def __str__(self): - return u'LOAD_STRINGCONSTANT "%s"' % (self.strval) + return u'LOAD_STRINGCONSTANT "%s"' % (self.w_strval.to_string()) class LOAD_UNDEFINED(Opcode): def eval(self, ctx): - ctx.stack_append(w_Undefined) + from js.object_space import newundefined + ctx.stack_append(newundefined()) class LOAD_NULL(Opcode): def eval(self, ctx): - ctx.stack_append(w_Null) + from js.object_space import newnull + ctx.stack_append(newnull()) class LOAD_VARIABLE(Opcode): @@ -264,6 +270,7 @@ class IN(BaseBinaryOperation): def operation(self, ctx, left, right): from js.jsobj import W_BasicObject + from js.object_space import newbool if not isinstance(right, W_BasicObject): raise JsTypeError(u"TypeError: fffuuu!") # + repr(right) name = left.to_string() @@ -292,6 +299,7 @@ self.name = name def eval(self, ctx): + from js.object_space import newstring ref = ctx.get_ref(self.name) if ref.is_unresolvable_reference(): var_type = u'undefined' @@ -299,7 +307,7 @@ var = ref.get_value() var_type = type_of(var) - w_string = W_String(var_type) + w_string = newstring(var_type) ctx.stack_append(w_string) def __str__(self): @@ -313,23 +321,27 @@ class BITAND(BaseBinaryBitwiseOp): def operation(self, ctx, op1, op2): - return W_IntNumber(op1 & op2) + from js.object_space import newint + return newint(op1 & op2) class BITXOR(BaseBinaryBitwiseOp): def operation(self, ctx, op1, op2): - return W_IntNumber(op1 ^ op2) + from js.object_space import newint + return newint(op1 ^ op2) class BITOR(BaseBinaryBitwiseOp): def operation(self, ctx, op1, op2): - return W_IntNumber(op1 | op2) + from js.object_space import newint + return newint(op1 | op2) class BITNOT(BaseUnaryOperation): def eval(self, ctx): op = ctx.stack_pop().ToInt32() - ctx.stack_append(W_IntNumber(~op)) + from js.object_space import newint + ctx.stack_append(newint(~op)) class URSH(BaseBinaryBitwiseOp): @@ -436,42 +448,42 @@ class GT(BaseBinaryComparison): def decision(self, ctx, op1, op2): - return newbool(compare(ctx, op1, op2)) + return compare(ctx, op1, op2) class GE(BaseBinaryComparison): def decision(self, ctx, op1, op2): - return newbool(compare_e(ctx, op1, op2)) + return compare_e(ctx, op1, op2) class LT(BaseBinaryComparison): def decision(self, ctx, op1, op2): - return newbool(compare(ctx, op2, op1)) + return compare(ctx, op2, op1) class LE(BaseBinaryComparison): def decision(self, ctx, op1, op2): - return newbool(compare_e(ctx, op2, op1)) + return compare_e(ctx, op2, op1) class EQ(BaseBinaryComparison): def decision(self, ctx, op1, op2): - return newbool(AbstractEC(ctx, op1, op2)) + return AbstractEC(ctx, op1, op2) class NE(BaseBinaryComparison): def decision(self, ctx, op1, op2): - return newbool(not AbstractEC(ctx, op1, op2)) + return not AbstractEC(ctx, op1, op2) class IS(BaseBinaryComparison): def decision(self, ctx, op1, op2): - return newbool(StrictEC(op1, op2)) + return StrictEC(op1, op2) class ISNOT(BaseBinaryComparison): def decision(self, ctx, op1, op2): - return newbool(not StrictEC(op1, op2)) + return not StrictEC(op1, op2) class STORE_MEMBER(Opcode): diff --git a/test/test_interp.py b/test/test_interp.py --- a/test/test_interp.py +++ b/test/test_interp.py @@ -1,10 +1,4 @@ import py -#from js import interpreter -#from js.operations import IntNumber, FloatNumber, Position, Plus -#from js.jsobj import W_Root, w_Null, W___Root -#from js.execution import ThrowException -#from js.jscode import JsCode -#from js.baseop import AbstractEC xfail = py.test.mark.xfail @@ -527,7 +521,7 @@ def test_null(): - from js.jsobj import w_Null + from js.object_space import w_Null assertv("null;", w_Null) diff --git a/test/test_lexical_environment.py b/test/test_lexical_environment.py --- a/test/test_lexical_environment.py +++ b/test/test_lexical_environment.py @@ -1,5 +1,5 @@ from js.lexical_environment import DeclarativeEnvironment, ObjectEnvironment -from js.jsobj import w_Undefined, W_BasicObject +from js.jsobj import W_BasicObject class TestDeclarativeEnvironment(object): From noreply at buildbot.pypy.org Tue Jan 8 23:09:13 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Tue, 8 Jan 2013 23:09:13 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Moved MathTests to test_ll_math Message-ID: <20130108220913.6F5E31C1305@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59878:616f56ea2576 Date: 2013-01-08 23:08 +0100 http://bitbucket.org/pypy/pypy/changeset/616f56ea2576/ Log: Moved MathTests to test_ll_math diff --git a/pypy/module/math/test/test_direct.py b/pypy/module/math/test/test_direct.py --- a/pypy/module/math/test/test_direct.py +++ b/pypy/module/math/test/test_direct.py @@ -4,220 +4,17 @@ import py, sys, math from rpython.rlib import rfloat from rpython.rlib.rfloat import isinf, isnan, INFINITY, NAN +from rpython.rtyper.lltypesystem.module.test.test_ll_math import (MathTests, + getTester) consistent_host = True if '__pypy__' not in sys.builtin_module_names: if sys.version_info < (2, 6): consistent_host = False -def positiveinf(x): - return isinf(x) and x > 0.0 - -def negativeinf(x): - return isinf(x) and x < 0.0 - -def finite(x): - return not isinf(x) and not isnan(x) - - -unary_math_functions = ['acos', 'asin', 'atan', - 'ceil', 'cos', 'cosh', 'exp', 'fabs', 'floor', - 'sin', 'sinh', 'sqrt', 'tan', 'tanh', 'log', 'log10', - 'acosh', 'asinh', 'atanh', 'log1p', 'expm1'] -binary_math_functions = ['atan2', 'fmod', 'hypot', 'pow'] - - -class MathTests: - - REGCASES = [] - for name in unary_math_functions: - try: - input, output = (0.3,), getattr(math, name)(0.3) - except AttributeError: - # cannot test this function - continue - except ValueError: - input, output = (1.3,), getattr(math, name)(1.3) - REGCASES.append((name, input, output)) - - IRREGCASES = [ - ('atan2', (0.31, 0.123), math.atan2(0.31, 0.123)), - ('fmod', (0.31, 0.123), math.fmod(0.31, 0.123)), - ('hypot', (0.31, 0.123), math.hypot(0.31, 0.123)), - ('pow', (0.31, 0.123), math.pow(0.31, 0.123)), - ('pow', (-0.31, 0.123), ValueError), - ('pow', (-0.5, 2.0), 0.25), - ('pow', (-0.5, 1.0), -0.5), - ('pow', (-0.5, 0.0), 1.0), - ('pow', (-0.5, -1.0), -2.0), - ('ldexp', (3.375, 2), 13.5), - ('ldexp', (1.0, -10000), 0.0), # underflow - ('frexp', (-1.25,), lambda x: x == (-0.625, 1)), - ('modf', (4.25,), lambda x: x == (0.25, 4.0)), - ('modf', (-4.25,), lambda x: x == (-0.25, -4.0)), - ('copysign', (1.5, 0.0), 1.5), - ('copysign', (1.5, -0.0), -1.5), - ('copysign', (1.5, INFINITY), 1.5), - ('copysign', (1.5, -INFINITY), -1.5), - ] - if sys.platform != 'win32': # all NaNs seem to be negative there...? - IRREGCASES += [ - ('copysign', (1.5, NAN), 1.5), - ('copysign', (1.75, -NAN), -1.75), # special case for -NAN here - ] - - OVFCASES = [ - ('cosh', (9999.9,), OverflowError), - ('sinh', (9999.9,), OverflowError), - ('exp', (9999.9,), OverflowError), - ('pow', (10.0, 40000.0), OverflowError), - ('ldexp', (10.0, 40000), OverflowError), - ('log', (0.0,), ValueError), #cpython does it this way - ('log1p', (-1.0,), OverflowError), - ('log', (-1.,), ValueError), - ('log10', (0.0,), ValueError), #cpython does it this way - ] - - INFCASES = [ - ('acos', (INFINITY,), ValueError), - ('acos', (-INFINITY,), ValueError), - ('asin', (INFINITY,), ValueError), - ('asin', (-INFINITY,), ValueError), - ('atan', (INFINITY,), math.pi / 2), - ('atan', (-INFINITY,), -math.pi / 2), - ('atanh', (INFINITY,), ValueError), - ('atanh', (-INFINITY,), ValueError), - ('ceil', (INFINITY,), positiveinf), - ('ceil', (-INFINITY,), negativeinf), - ('cos', (INFINITY,), ValueError), - ('cos', (-INFINITY,), ValueError), - ('cosh', (INFINITY,), positiveinf), - ('cosh', (-INFINITY,), positiveinf), - ('exp', (INFINITY,), positiveinf), - ('exp', (-INFINITY,), 0.0), - ('fabs', (INFINITY,), positiveinf), - ('fabs', (-INFINITY,), positiveinf), - ('floor', (INFINITY,), positiveinf), - ('floor', (-INFINITY,), negativeinf), - ('sin', (INFINITY,), ValueError), - ('sin', (-INFINITY,), ValueError), - ('sinh', (INFINITY,), positiveinf), - ('sinh', (-INFINITY,), negativeinf), - ('sqrt', (INFINITY,), positiveinf), - ('sqrt', (-INFINITY,), ValueError), - ('tan', (INFINITY,), ValueError), - ('tan', (-INFINITY,), ValueError), - ('tanh', (INFINITY,), 1.0), - ('tanh', (-INFINITY,), -1.0), - ('log', (INFINITY,), positiveinf), - ('log', (-INFINITY,), ValueError), - ('log10', (INFINITY,), positiveinf), - ('log10', (-INFINITY,), ValueError), - ('frexp', (INFINITY,), lambda x: isinf(x[0])), - ('ldexp', (INFINITY, 3), positiveinf), - ('ldexp', (-INFINITY, 3), negativeinf), - ('modf', (INFINITY,), lambda x: positiveinf(x[1])), - ('modf', (-INFINITY,), lambda x: negativeinf(x[1])), - ('pow', (INFINITY, 0.0), 1.0), - ('pow', (INFINITY, 0.001), positiveinf), - ('pow', (INFINITY, -0.001), 0.0), - ('pow', (-INFINITY, 0.0), 1.0), - ('pow', (-INFINITY, 0.001), positiveinf), - ('pow', (-INFINITY, -0.001), 0.0), - ('pow', (-INFINITY, 3.0), negativeinf), - ('pow', (-INFINITY, 6.0), positiveinf), - ('pow', (-INFINITY, -13.0), -0.0), - ('pow', (-INFINITY, -128.0), 0.0), - ('pow', (1.001, INFINITY), positiveinf), - ('pow', (1.0, INFINITY), 1.0), - ('pow', (0.999, INFINITY), 0.0), - ('pow', (-0.999,INFINITY), 0.0), - #('pow', (-1.0, INFINITY), 1.0, but strange, could also be -1.0), - ('pow', (-1.001,INFINITY), positiveinf), - ('pow', (1.001, -INFINITY), 0.0), - ('pow', (1.0, -INFINITY), 1.0), - #('pow', (0.999, -INFINITY), positiveinf, but get OverflowError), - #('pow', (INFINITY, INFINITY), positiveinf, but get OverflowError), - ('pow', (INFINITY, -INFINITY), 0.0), - ('pow', (-INFINITY, INFINITY), positiveinf), - ] - - IRREGERRCASES = [ - # - ('atan2', (INFINITY, -2.3), math.pi / 2), - ('atan2', (INFINITY, 0.0), math.pi / 2), - ('atan2', (INFINITY, 3.0), math.pi / 2), - #('atan2', (INFINITY, INFINITY), ?strange), - ('atan2', (2.1, INFINITY), 0.0), - ('atan2', (0.0, INFINITY), 0.0), - ('atan2', (-0.1, INFINITY), -0.0), - ('atan2', (-INFINITY, 0.4), -math.pi / 2), - ('atan2', (2.1, -INFINITY), math.pi), - ('atan2', (0.0, -INFINITY), math.pi), - ('atan2', (-0.1, -INFINITY), -math.pi), - # - ('fmod', (INFINITY, 1.0), ValueError), - ('fmod', (1.0, INFINITY), 1.0), - ('fmod', (1.0, -INFINITY), 1.0), - ('fmod', (INFINITY, INFINITY), ValueError), - # - ('hypot', (-INFINITY, 1.0), positiveinf), - ('hypot', (-2.3, -INFINITY), positiveinf), - ] - - binary_math_functions = ['atan2', 'fmod', 'hypot', 'pow'] - - NANCASES1 = [ - (name, (NAN,), isnan) for name in unary_math_functions] - NANCASES2 = [ - (name, (NAN, 0.1), isnan) for name in binary_math_functions] - NANCASES3 = [ - (name, (-0.2, NAN), isnan) for name in binary_math_functions] - NANCASES4 = [ - (name, (NAN, -INFINITY), isnan) for name in binary_math_functions - if name != 'hypot'] - NANCASES5 = [ - (name, (INFINITY, NAN), isnan) for name in binary_math_functions - if name != 'hypot'] - NANCASES6 = [ - ('frexp', (NAN,), lambda x: isnan(x[0])), - ('ldexp', (NAN, 2), isnan), - ('hypot', (NAN, INFINITY), positiveinf), - ('hypot', (NAN, -INFINITY), positiveinf), - ('hypot', (INFINITY, NAN), positiveinf), - ('hypot', (-INFINITY, NAN), positiveinf), - ('modf', (NAN,), lambda x: (isnan(x[0]) and isnan(x[1]))), - ] - - # The list of test cases. Note that various tests import this, - # notably in rpython/lltypesystem/module and in translator/c/test. - TESTCASES = (REGCASES + IRREGCASES + OVFCASES + INFCASES + IRREGERRCASES - + NANCASES1 + NANCASES2 + NANCASES3 + NANCASES4 + NANCASES5 - + NANCASES6) - - class TestDirect(MathTests): pass -def get_tester(expected): - if type(expected) is type(Exception): - def tester(value): - return False - elif callable(expected): - def tester(value): - ok = expected(value) - assert isinstance(ok, bool) - return ok - else: - assert finite(expected), "badly written test" - def tester(got): - gotsign = expectedsign = 1 - if got < 0.0: gotsign = -gotsign - if expected < 0.0: expectedsign = -expectedsign - return finite(got) and (got == expected and - gotsign == expectedsign) - return tester - def do_test(fn, fnname, args, expected): repr = "%s(%s)" % (fnname, ', '.join(map(str, args))) try: diff --git a/rpython/jit/backend/x86/test/test_zmath.py b/rpython/jit/backend/x86/test/test_zmath.py --- a/rpython/jit/backend/x86/test/test_zmath.py +++ b/rpython/jit/backend/x86/test/test_zmath.py @@ -2,13 +2,13 @@ compiled to C with SSE2 enabled. """ import py, math -from pypy.module.math.test import test_direct from rpython.translator.c.test.test_genc import compile from rpython.jit.backend.x86.support import ensure_sse2_floats from rpython.rlib import rfloat from rpython.rlib.unroll import unrolling_iterable from rpython.rlib.debug import debug_print - +from rpython.rtyper.lltypesystem.module.test.test_ll_math import (MathTests, + getTester) def get_test_case((fnname, args, expected)): try: @@ -17,7 +17,7 @@ fn = getattr(rfloat, fnname) expect_valueerror = (expected == ValueError) expect_overflowerror = (expected == OverflowError) - check = test_direct.get_tester(expected) + check = get_tester(expected) unroll_args = unrolling_iterable(args) # def testfn(): @@ -50,7 +50,7 @@ testfnlist = [get_test_case(testcase) - for testcase in test_direct.MathTests.TESTCASES] + for testcase in MathTests.TESTCASES] def fn(): ensure_sse2_floats() @@ -64,6 +64,6 @@ f = compile(fn, []) res = f() if res >= 0: - py.test.fail(repr(test_direct.MathTests.TESTCASES[res])) + py.test.fail(repr(MathTests.TESTCASES[res])) else: assert res == -42 diff --git a/rpython/jit/tool/pypytrace-mode.el b/rpython/jit/tool/pypytrace-mode.el --- a/rpython/jit/tool/pypytrace-mode.el +++ b/rpython/jit/tool/pypytrace-mode.el @@ -18,7 +18,7 @@ ;; to generate the list of keywords: -;; from pypy.jit.metainterp import resoperation +;; from rpython.jit.metainterp import resoperation ;; print ' '.join(sorted('"%s"' % op.lower() for op in resoperation.opname.values() if not op.startswith('GUARD'))) diff --git a/rpython/rtyper/lltypesystem/module/test/test_ll_math.py b/rpython/rtyper/lltypesystem/module/test/test_ll_math.py --- a/rpython/rtyper/lltypesystem/module/test/test_ll_math.py +++ b/rpython/rtyper/lltypesystem/module/test/test_ll_math.py @@ -2,10 +2,194 @@ """ from rpython.rtyper.lltypesystem.module import ll_math -from pypy.module.math.test.test_direct import MathTests, get_tester from rpython.translator.c.test.test_genc import compile +def positiveinf(x): + return isinf(x) and x > 0.0 + +def negativeinf(x): + return isinf(x) and x < 0.0 + +def finite(x): + return not isinf(x) and not isnan(x) + + +unary_math_functions = ['acos', 'asin', 'atan', + 'ceil', 'cos', 'cosh', 'exp', 'fabs', 'floor', + 'sin', 'sinh', 'sqrt', 'tan', 'tanh', 'log', 'log10', + 'acosh', 'asinh', 'atanh', 'log1p', 'expm1'] +binary_math_functions = ['atan2', 'fmod', 'hypot', 'pow'] + +class MathTests: + + REGCASES = [] + for name in unary_math_functions: + try: + input, output = (0.3,), getattr(math, name)(0.3) + except AttributeError: + # cannot test this function + continue + except ValueError: + input, output = (1.3,), getattr(math, name)(1.3) + REGCASES.append((name, input, output)) + + IRREGCASES = [ + ('atan2', (0.31, 0.123), math.atan2(0.31, 0.123)), + ('fmod', (0.31, 0.123), math.fmod(0.31, 0.123)), + ('hypot', (0.31, 0.123), math.hypot(0.31, 0.123)), + ('pow', (0.31, 0.123), math.pow(0.31, 0.123)), + ('pow', (-0.31, 0.123), ValueError), + ('pow', (-0.5, 2.0), 0.25), + ('pow', (-0.5, 1.0), -0.5), + ('pow', (-0.5, 0.0), 1.0), + ('pow', (-0.5, -1.0), -2.0), + ('ldexp', (3.375, 2), 13.5), + ('ldexp', (1.0, -10000), 0.0), # underflow + ('frexp', (-1.25,), lambda x: x == (-0.625, 1)), + ('modf', (4.25,), lambda x: x == (0.25, 4.0)), + ('modf', (-4.25,), lambda x: x == (-0.25, -4.0)), + ('copysign', (1.5, 0.0), 1.5), + ('copysign', (1.5, -0.0), -1.5), + ('copysign', (1.5, INFINITY), 1.5), + ('copysign', (1.5, -INFINITY), -1.5), + ] + if sys.platform != 'win32': # all NaNs seem to be negative there...? + IRREGCASES += [ + ('copysign', (1.5, NAN), 1.5), + ('copysign', (1.75, -NAN), -1.75), # special case for -NAN here + ] + + OVFCASES = [ + ('cosh', (9999.9,), OverflowError), + ('sinh', (9999.9,), OverflowError), + ('exp', (9999.9,), OverflowError), + ('pow', (10.0, 40000.0), OverflowError), + ('ldexp', (10.0, 40000), OverflowError), + ('log', (0.0,), ValueError), #cpython does it this way + ('log1p', (-1.0,), OverflowError), + ('log', (-1.,), ValueError), + ('log10', (0.0,), ValueError), #cpython does it this way + ] + + INFCASES = [ + ('acos', (INFINITY,), ValueError), + ('acos', (-INFINITY,), ValueError), + ('asin', (INFINITY,), ValueError), + ('asin', (-INFINITY,), ValueError), + ('atan', (INFINITY,), math.pi / 2), + ('atan', (-INFINITY,), -math.pi / 2), + ('atanh', (INFINITY,), ValueError), + ('atanh', (-INFINITY,), ValueError), + ('ceil', (INFINITY,), positiveinf), + ('ceil', (-INFINITY,), negativeinf), + ('cos', (INFINITY,), ValueError), + ('cos', (-INFINITY,), ValueError), + ('cosh', (INFINITY,), positiveinf), + ('cosh', (-INFINITY,), positiveinf), + ('exp', (INFINITY,), positiveinf), + ('exp', (-INFINITY,), 0.0), + ('fabs', (INFINITY,), positiveinf), + ('fabs', (-INFINITY,), positiveinf), + ('floor', (INFINITY,), positiveinf), + ('floor', (-INFINITY,), negativeinf), + ('sin', (INFINITY,), ValueError), + ('sin', (-INFINITY,), ValueError), + ('sinh', (INFINITY,), positiveinf), + ('sinh', (-INFINITY,), negativeinf), + ('sqrt', (INFINITY,), positiveinf), + ('sqrt', (-INFINITY,), ValueError), + ('tan', (INFINITY,), ValueError), + ('tan', (-INFINITY,), ValueError), + ('tanh', (INFINITY,), 1.0), + ('tanh', (-INFINITY,), -1.0), + ('log', (INFINITY,), positiveinf), + ('log', (-INFINITY,), ValueError), + ('log10', (INFINITY,), positiveinf), + ('log10', (-INFINITY,), ValueError), + ('frexp', (INFINITY,), lambda x: isinf(x[0])), + ('ldexp', (INFINITY, 3), positiveinf), + ('ldexp', (-INFINITY, 3), negativeinf), + ('modf', (INFINITY,), lambda x: positiveinf(x[1])), + ('modf', (-INFINITY,), lambda x: negativeinf(x[1])), + ('pow', (INFINITY, 0.0), 1.0), + ('pow', (INFINITY, 0.001), positiveinf), + ('pow', (INFINITY, -0.001), 0.0), + ('pow', (-INFINITY, 0.0), 1.0), + ('pow', (-INFINITY, 0.001), positiveinf), + ('pow', (-INFINITY, -0.001), 0.0), + ('pow', (-INFINITY, 3.0), negativeinf), + ('pow', (-INFINITY, 6.0), positiveinf), + ('pow', (-INFINITY, -13.0), -0.0), + ('pow', (-INFINITY, -128.0), 0.0), + ('pow', (1.001, INFINITY), positiveinf), + ('pow', (1.0, INFINITY), 1.0), + ('pow', (0.999, INFINITY), 0.0), + ('pow', (-0.999,INFINITY), 0.0), + #('pow', (-1.0, INFINITY), 1.0, but strange, could also be -1.0), + ('pow', (-1.001,INFINITY), positiveinf), + ('pow', (1.001, -INFINITY), 0.0), + ('pow', (1.0, -INFINITY), 1.0), + #('pow', (0.999, -INFINITY), positiveinf, but get OverflowError), + #('pow', (INFINITY, INFINITY), positiveinf, but get OverflowError), + ('pow', (INFINITY, -INFINITY), 0.0), + ('pow', (-INFINITY, INFINITY), positiveinf), + ] + + IRREGERRCASES = [ + # + ('atan2', (INFINITY, -2.3), math.pi / 2), + ('atan2', (INFINITY, 0.0), math.pi / 2), + ('atan2', (INFINITY, 3.0), math.pi / 2), + #('atan2', (INFINITY, INFINITY), ?strange), + ('atan2', (2.1, INFINITY), 0.0), + ('atan2', (0.0, INFINITY), 0.0), + ('atan2', (-0.1, INFINITY), -0.0), + ('atan2', (-INFINITY, 0.4), -math.pi / 2), + ('atan2', (2.1, -INFINITY), math.pi), + ('atan2', (0.0, -INFINITY), math.pi), + ('atan2', (-0.1, -INFINITY), -math.pi), + # + ('fmod', (INFINITY, 1.0), ValueError), + ('fmod', (1.0, INFINITY), 1.0), + ('fmod', (1.0, -INFINITY), 1.0), + ('fmod', (INFINITY, INFINITY), ValueError), + # + ('hypot', (-INFINITY, 1.0), positiveinf), + ('hypot', (-2.3, -INFINITY), positiveinf), + ] + + binary_math_functions = ['atan2', 'fmod', 'hypot', 'pow'] + + NANCASES1 = [ + (name, (NAN,), isnan) for name in unary_math_functions] + NANCASES2 = [ + (name, (NAN, 0.1), isnan) for name in binary_math_functions] + NANCASES3 = [ + (name, (-0.2, NAN), isnan) for name in binary_math_functions] + NANCASES4 = [ + (name, (NAN, -INFINITY), isnan) for name in binary_math_functions + if name != 'hypot'] + NANCASES5 = [ + (name, (INFINITY, NAN), isnan) for name in binary_math_functions + if name != 'hypot'] + NANCASES6 = [ + ('frexp', (NAN,), lambda x: isnan(x[0])), + ('ldexp', (NAN, 2), isnan), + ('hypot', (NAN, INFINITY), positiveinf), + ('hypot', (NAN, -INFINITY), positiveinf), + ('hypot', (INFINITY, NAN), positiveinf), + ('hypot', (-INFINITY, NAN), positiveinf), + ('modf', (NAN,), lambda x: (isnan(x[0]) and isnan(x[1]))), + ] + + # The list of test cases. Note that various tests import this, + # notably in rpython/lltypesystem/module and in translator/c/test. + TESTCASES = (REGCASES + IRREGCASES + OVFCASES + INFCASES + IRREGERRCASES + + NANCASES1 + NANCASES2 + NANCASES3 + NANCASES4 + NANCASES5 + + NANCASES6) + + class TestMath(MathTests): def test_isinf(self): inf = 1e200 * 1e200 @@ -68,6 +252,25 @@ assert not f(1e200, 1e200) # nan +def get_tester(expected): + if type(expected) is type(Exception): + def tester(value): + return False + elif callable(expected): + def tester(value): + ok = expected(value) + assert isinstance(ok, bool) + return ok + else: + assert finite(expected), "badly written test" + def tester(got): + gotsign = expectedsign = 1 + if got < 0.0: gotsign = -gotsign + if expected < 0.0: expectedsign = -expectedsign + return finite(got) and (got == expected and + gotsign == expectedsign) + return tester + from rpython.rtyper.lltypesystem import lltype _A = lltype.GcArray(lltype.Float) def normalize(x): @@ -91,7 +294,7 @@ assert expected == OverflowError, "%s: got an OverflowError" % ( repr,) else: - if not get_tester(expected)(got): + if not (expected)(got): raise AssertionError("%r: got %r, expected %r" % (repr, got, expected)) # From noreply at buildbot.pypy.org Tue Jan 8 23:26:37 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Tue, 8 Jan 2013 23:26:37 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed missing import of math in test_ll_math Message-ID: <20130108222637.107FE1C1305@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59879:668748ed1fc4 Date: 2013-01-08 23:26 +0100 http://bitbucket.org/pypy/pypy/changeset/668748ed1fc4/ Log: Fixed missing import of math in test_ll_math diff --git a/rpython/rtyper/lltypesystem/module/test/test_ll_math.py b/rpython/rtyper/lltypesystem/module/test/test_ll_math.py --- a/rpython/rtyper/lltypesystem/module/test/test_ll_math.py +++ b/rpython/rtyper/lltypesystem/module/test/test_ll_math.py @@ -1,6 +1,7 @@ """ Try to test systematically all cases of ll_math.py. """ +import math from rpython.rtyper.lltypesystem.module import ll_math from rpython.translator.c.test.test_genc import compile From noreply at buildbot.pypy.org Tue Jan 8 23:43:23 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Tue, 8 Jan 2013 23:43:23 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Removed unneeded ctype existence check. Removed resulting dead code. Message-ID: <20130108224323.8FF591C1305@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59880:ea633cd73b5b Date: 2013-01-08 23:43 +0100 http://bitbucket.org/pypy/pypy/changeset/ea633cd73b5b/ Log: Removed unneeded ctype existence check. Removed resulting dead code. diff --git a/pypy/tool/pytest/modcheck.py b/pypy/tool/pytest/modcheck.py deleted file mode 100644 --- a/pypy/tool/pytest/modcheck.py +++ /dev/null @@ -1,14 +0,0 @@ -import py - -def skipimporterror(name): - if not hasimport(name): - __tracebackhide__ = True - py.test.skip("cannot import %r module" % (name,)) - -def hasimport(name): - try: - __import__(name) - except ImportError: - return False - else: - return True diff --git a/rpython/rtyper/module/test/test_ll_os_path.py b/rpython/rtyper/module/test/test_ll_os_path.py --- a/rpython/rtyper/module/test/test_ll_os_path.py +++ b/rpython/rtyper/module/test/test_ll_os_path.py @@ -2,9 +2,6 @@ import sys, os -from pypy.tool.pytest.modcheck import skipimporterror -skipimporterror("ctypes") - from rpython.rtyper.lltypesystem.module.ll_os_path import Implementation as impl from rpython.rtyper.module.support import ll_strcpy from rpython.rtyper.test.test_llinterp import interpret From noreply at buildbot.pypy.org Tue Jan 8 23:47:24 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Tue, 8 Jan 2013 23:47:24 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed missing import of math in test_ll_math #2 Message-ID: <20130108224724.064161C1305@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59881:ccb69ed1d20c Date: 2013-01-08 23:47 +0100 http://bitbucket.org/pypy/pypy/changeset/ccb69ed1d20c/ Log: Fixed missing import of math in test_ll_math #2 diff --git a/pypy/module/math/test/test_direct.py b/pypy/module/math/test/test_direct.py --- a/pypy/module/math/test/test_direct.py +++ b/pypy/module/math/test/test_direct.py @@ -3,7 +3,6 @@ import py, sys, math from rpython.rlib import rfloat -from rpython.rlib.rfloat import isinf, isnan, INFINITY, NAN from rpython.rtyper.lltypesystem.module.test.test_ll_math import (MathTests, getTester) diff --git a/rpython/rtyper/lltypesystem/module/test/test_ll_math.py b/rpython/rtyper/lltypesystem/module/test/test_ll_math.py --- a/rpython/rtyper/lltypesystem/module/test/test_ll_math.py +++ b/rpython/rtyper/lltypesystem/module/test/test_ll_math.py @@ -4,6 +4,7 @@ import math from rpython.rtyper.lltypesystem.module import ll_math from rpython.translator.c.test.test_genc import compile +from rpython.rlib.rfloat import isinf, isnan, INFINITY, NAN def positiveinf(x): From noreply at buildbot.pypy.org Wed Jan 9 04:28:19 2013 From: noreply at buildbot.pypy.org (wlav) Date: Wed, 9 Jan 2013 04:28:19 +0100 (CET) Subject: [pypy-commit] pypy reflex-support: merge default into branch Message-ID: <20130109032819.71F0D1C1305@cobra.cs.uni-duesseldorf.de> Author: Wim Lavrijsen Branch: reflex-support Changeset: r59882:a06891d6aa9c Date: 2013-01-05 11:23 -0800 http://bitbucket.org/pypy/pypy/changeset/a06891d6aa9c/ Log: merge default into branch diff too long, truncating to 2000 out of 15618 lines diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -28,7 +28,7 @@ DEALINGS IN THE SOFTWARE. -PyPy Copyright holders 2003-2012 +PyPy Copyright holders 2003-2013 ----------------------------------- Except when otherwise stated (look for LICENSE files or information at diff --git a/README b/README deleted file mode 100644 --- a/README +++ /dev/null @@ -1,24 +0,0 @@ -===================================== -PyPy: Python in Python Implementation -===================================== - -Welcome to PyPy! - -PyPy is both an implementation of the Python programming language, and -an extensive compiler framework for dynamic language implementations. -You can build self-contained Python implementations which execute -independently from CPython. - -The home page is: - - http://pypy.org/ - -The getting-started document will help guide you: - - http://doc.pypy.org/en/latest/getting-started.html - -It will also point you to the rest of the documentation which is generated -from files in the pypy/doc directory within the source repositories. Enjoy -and send us feedback! - - the pypy-dev team diff --git a/README.rst b/README.rst new file mode 100644 --- /dev/null +++ b/README.rst @@ -0,0 +1,24 @@ +===================================== +PyPy: Python in Python Implementation +===================================== + +Welcome to PyPy! + +PyPy is both an implementation of the Python programming language, and +an extensive compiler framework for dynamic language implementations. +You can build self-contained Python implementations which execute +independently from CPython. + +The home page is: + + http://pypy.org/ + +The getting-started document will help guide you: + + http://doc.pypy.org/en/latest/getting-started.html + +It will also point you to the rest of the documentation which is generated +from files in the pypy/doc directory within the source repositories. Enjoy +and send us feedback! + + the pypy-dev team diff --git a/lib-python/2.7/ctypes/test/test_internals.py b/lib-python/2.7/ctypes/test/test_internals.py --- a/lib-python/2.7/ctypes/test/test_internals.py +++ b/lib-python/2.7/ctypes/test/test_internals.py @@ -1,7 +1,10 @@ # This tests the internal _objects attribute import unittest from ctypes import * -from sys import getrefcount as grc +try: + from sys import getrefcount as grc +except ImportError: + grc = None # e.g. PyPy # XXX This test must be reviewed for correctness!!! @@ -22,6 +25,8 @@ self.assertEqual(id(a), id(b)) def test_ints(self): + if grc is None: + return unittest.skip("no sys.getrefcount()") i = 42000123 refcnt = grc(i) ci = c_int(i) @@ -29,6 +34,8 @@ self.assertEqual(ci._objects, None) def test_c_char_p(self): + if grc is None: + return unittest.skip("no sys.getrefcount()") s = "Hello, World" refcnt = grc(s) cs = c_char_p(s) diff --git a/lib-python/2.7/ctypes/test/test_memfunctions.py b/lib-python/2.7/ctypes/test/test_memfunctions.py --- a/lib-python/2.7/ctypes/test/test_memfunctions.py +++ b/lib-python/2.7/ctypes/test/test_memfunctions.py @@ -53,7 +53,8 @@ s = string_at("foo bar") # XXX The following may be wrong, depending on how Python # manages string instances - self.assertEqual(2, sys.getrefcount(s)) + if hasattr(sys, 'getrefcount'): + self.assertEqual(2, sys.getrefcount(s)) self.assertTrue(s, "foo bar") self.assertEqual(string_at("foo bar", 8), "foo bar\0") diff --git a/lib-python/2.7/ctypes/test/test_python_api.py b/lib-python/2.7/ctypes/test/test_python_api.py --- a/lib-python/2.7/ctypes/test/test_python_api.py +++ b/lib-python/2.7/ctypes/test/test_python_api.py @@ -9,7 +9,10 @@ ################################################################ -from sys import getrefcount as grc +try: + from sys import getrefcount as grc +except ImportError: + grc = None # e.g. PyPy if sys.version_info > (2, 4): c_py_ssize_t = c_size_t else: diff --git a/lib-python/2.7/ctypes/test/test_refcounts.py b/lib-python/2.7/ctypes/test/test_refcounts.py --- a/lib-python/2.7/ctypes/test/test_refcounts.py +++ b/lib-python/2.7/ctypes/test/test_refcounts.py @@ -11,7 +11,10 @@ class RefcountTestCase(unittest.TestCase): def test_1(self): - from sys import getrefcount as grc + try: + from sys import getrefcount as grc + except ImportError: + return unittest.skip("no sys.getrefcount()") f = dll._testfunc_callback_i_if f.restype = ctypes.c_int @@ -35,7 +38,10 @@ def test_refcount(self): - from sys import getrefcount as grc + try: + from sys import getrefcount as grc + except ImportError: + return unittest.skip("no sys.getrefcount()") def func(*args): pass # this is the standard refcount for func @@ -84,6 +90,10 @@ class AnotherLeak(unittest.TestCase): def test_callback(self): import sys + try: + from sys import getrefcount + except ImportError: + return unittest.skip("no sys.getrefcount()") proto = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int, ctypes.c_int) def func(a, b): diff --git a/lib-python/2.7/sre_parse.py b/lib-python/2.7/sre_parse.py --- a/lib-python/2.7/sre_parse.py +++ b/lib-python/2.7/sre_parse.py @@ -16,6 +16,12 @@ from sre_constants import * +try: + from __pypy__ import newdict +except ImportError: + def newdict(tp): + return {} + SPECIAL_CHARS = ".\\[{()*+?^$|" REPEAT_CHARS = "*+?{" @@ -68,7 +74,7 @@ self.flags = 0 self.open = [] self.groups = 1 - self.groupdict = {} + self.groupdict = newdict("module") def opengroup(self, name=None): gid = self.groups self.groups = gid + 1 diff --git a/lib-python/2.7/timeit.py b/lib-python/2.7/timeit.py --- a/lib-python/2.7/timeit.py +++ b/lib-python/2.7/timeit.py @@ -190,7 +190,8 @@ else: it = [None] * number gcold = gc.isenabled() - gc.disable() + if '__pypy__' not in sys.builtin_module_names: + gc.disable() # only do that on CPython try: timing = self.inner(it, self.timer) finally: diff --git a/lib_pypy/_ctypes_test.py b/lib_pypy/_ctypes_test.py --- a/lib_pypy/_ctypes_test.py +++ b/lib_pypy/_ctypes_test.py @@ -2,10 +2,6 @@ import tempfile import gc -# Monkeypatch & hacks to let ctypes.tests import. -# This should be removed at some point. -sys.getrefcount = lambda x: len(gc.get_referrers(x)) - 1 - def compile_shared(): """Compile '_ctypes_test.c' into an extension module, and import it """ diff --git a/lib_pypy/numpypy/core/arrayprint.py b/lib_pypy/numpypy/core/arrayprint.py --- a/lib_pypy/numpypy/core/arrayprint.py +++ b/lib_pypy/numpypy/core/arrayprint.py @@ -248,9 +248,9 @@ 'int' : IntegerFormat(data), 'float' : FloatFormat(data, precision, suppress_small), 'longfloat' : LongFloatFormat(precision), - #'complexfloat' : ComplexFormat(data, precision, - # suppress_small), - #'longcomplexfloat' : LongComplexFormat(precision), + 'complexfloat' : ComplexFormat(data, precision, + suppress_small), + 'longcomplexfloat' : LongComplexFormat(precision), 'datetime' : DatetimeFormat(data), 'timedelta' : TimedeltaFormat(data), 'numpystr' : repr_format, @@ -294,19 +294,19 @@ #else: format_function = formatdict['int'] elif issubclass(dtypeobj, _nt.floating): - #if issubclass(dtypeobj, _nt.longfloat): - # format_function = formatdict['longfloat'] - #else: - format_function = formatdict['float'] - #elif issubclass(dtypeobj, _nt.complexfloating): - # if issubclass(dtypeobj, _nt.clongfloat): - # format_function = formatdict['longcomplexfloat'] - # else: - # format_function = formatdict['complexfloat'] + if issubclass(dtypeobj, _nt.longfloat): + format_function = formatdict['longfloat'] + else: + format_function = formatdict['float'] + elif issubclass(dtypeobj, _nt.complexfloating): + if issubclass(dtypeobj, _nt.clongfloat): + format_function = formatdict['longcomplexfloat'] + else: + format_function = formatdict['complexfloat'] elif issubclass(dtypeobj, (_nt.unicode_, _nt.string_)): format_function = formatdict['numpystr'] - elif issubclass(dtypeobj, _nt.datetime64): - format_function = formatdict['datetime'] + #elif issubclass(dtypeobj, _nt.datetime64): + # format_function = formatdict['datetime'] else: format_function = formatdict['str'] diff --git a/lib_pypy/stackless.py b/lib_pypy/stackless.py --- a/lib_pypy/stackless.py +++ b/lib_pypy/stackless.py @@ -434,8 +434,8 @@ def insert(self): if self.blocked: raise RuntimeError, "You cannot run a blocked tasklet" - if not self.alive: - raise RuntimeError, "You cannot run an unbound(dead) tasklet" + if not self.alive: + raise RuntimeError, "You cannot run an unbound(dead) tasklet" _scheduler_append(self) def remove(self): diff --git a/pypy/annotation/annrpython.py b/pypy/annotation/annrpython.py --- a/pypy/annotation/annrpython.py +++ b/pypy/annotation/annrpython.py @@ -8,7 +8,7 @@ from pypy.objspace.flow.model import (Variable, Constant, FunctionGraph, c_last_exception, checkgraph) from pypy.translator import simplify, transform -from pypy.annotation import model as annmodel, signature +from pypy.annotation import model as annmodel, signature, unaryop, binaryop from pypy.annotation.bookkeeper import Bookkeeper import py log = py.log.Producer("annrpython") @@ -453,12 +453,12 @@ # occour for this specific, typed operation. if block.exitswitch == c_last_exception: op = block.operations[-1] - if op.opname in annmodel.BINARY_OPERATIONS: + if op.opname in binaryop.BINARY_OPERATIONS: arg1 = self.binding(op.args[0]) arg2 = self.binding(op.args[1]) binop = getattr(pair(arg1, arg2), op.opname, None) can_only_throw = annmodel.read_can_only_throw(binop, arg1, arg2) - elif op.opname in annmodel.UNARY_OPERATIONS: + elif op.opname in unaryop.UNARY_OPERATIONS: arg1 = self.binding(op.args[0]) opname = op.opname if opname == 'contains': opname = 'op_contains' @@ -629,10 +629,10 @@ return self.bookkeeper.newdict() - def _registeroperations(cls, model): + def _registeroperations(cls, unary_ops, binary_ops): # All unary operations d = {} - for opname in model.UNARY_OPERATIONS: + for opname in unary_ops: fnname = 'consider_op_' + opname exec py.code.Source(""" def consider_op_%s(self, arg, *args): @@ -640,7 +640,7 @@ """ % (opname, opname)).compile() in globals(), d setattr(cls, fnname, d[fnname]) # All binary operations - for opname in model.BINARY_OPERATIONS: + for opname in binary_ops: fnname = 'consider_op_' + opname exec py.code.Source(""" def consider_op_%s(self, arg1, arg2, *args): @@ -650,7 +650,7 @@ _registeroperations = classmethod(_registeroperations) # register simple operations handling -RPythonAnnotator._registeroperations(annmodel) +RPythonAnnotator._registeroperations(unaryop.UNARY_OPERATIONS, binaryop.BINARY_OPERATIONS) class BlockedInference(Exception): diff --git a/pypy/annotation/binaryop.py b/pypy/annotation/binaryop.py --- a/pypy/annotation/binaryop.py +++ b/pypy/annotation/binaryop.py @@ -7,10 +7,10 @@ from pypy.tool.pairtype import pair, pairtype from pypy.annotation.model import SomeObject, SomeInteger, SomeBool, s_Bool from pypy.annotation.model import SomeString, SomeChar, SomeList, SomeDict -from pypy.annotation.model import SomeUnicodeCodePoint, SomeStringOrUnicode +from pypy.annotation.model import SomeUnicodeCodePoint, SomeUnicodeString from pypy.annotation.model import SomeTuple, SomeImpossibleValue, s_ImpossibleValue from pypy.annotation.model import SomeInstance, SomeBuiltin, SomeIterator -from pypy.annotation.model import SomePBC, SomeFloat, s_None +from pypy.annotation.model import SomePBC, SomeFloat, s_None, SomeByteArray from pypy.annotation.model import SomeExternalObject, SomeWeakRef from pypy.annotation.model import SomeAddress, SomeTypedAddressAccess from pypy.annotation.model import SomeSingleFloat, SomeLongFloat, SomeType @@ -19,7 +19,6 @@ from pypy.annotation.model import read_can_only_throw from pypy.annotation.model import add_knowntypedata, merge_knowntypedata from pypy.annotation.model import SomeGenericCallable -from pypy.annotation.model import SomeUnicodeString from pypy.annotation.bookkeeper import getbookkeeper from pypy.objspace.flow.model import Variable, Constant from pypy.rlib import rarithmetic @@ -416,6 +415,34 @@ result.const = str1.const + str2.const return result +class __extend__(pairtype(SomeByteArray, SomeByteArray)): + def union((b1, b2)): + can_be_None = b1.can_be_None or b2.can_be_None + return SomeByteArray(can_be_None=can_be_None) + + def add((b1, b2)): + result = SomeByteArray() + if b1.is_immutable_constant() and b2.is_immutable_constant(): + result.const = b1.const + b2.const + return result + +class __extend__(pairtype(SomeByteArray, SomeInteger)): + def getitem((s_b, s_i)): + return SomeInteger() + + def setitem((s_b, s_i), s_i2): + assert isinstance(s_i2, SomeInteger) + +class __extend__(pairtype(SomeString, SomeByteArray), + pairtype(SomeByteArray, SomeString), + pairtype(SomeChar, SomeByteArray), + pairtype(SomeByteArray, SomeChar)): + def add((b1, b2)): + result = SomeByteArray() + if b1.is_immutable_constant() and b2.is_immutable_constant(): + result.const = b1.const + b2.const + return result + class __extend__(pairtype(SomeChar, SomeChar)): def union((chr1, chr2)): @@ -458,7 +485,8 @@ for s_item in s_tuple.items: if isinstance(s_item, SomeFloat): pass # or s_item is a subclass, like SomeInteger - elif isinstance(s_item, SomeStringOrUnicode) and s_item.no_nul: + elif (isinstance(s_item, SomeString) or + isinstance(s_item, SomeUnicodeString)) and s_item.no_nul: pass else: no_nul = False diff --git a/pypy/annotation/bookkeeper.py b/pypy/annotation/bookkeeper.py --- a/pypy/annotation/bookkeeper.py +++ b/pypy/annotation/bookkeeper.py @@ -13,7 +13,7 @@ SomeUnicodeCodePoint, SomeOOStaticMeth, s_None, s_ImpossibleValue, \ SomeLLADTMeth, SomeBool, SomeTuple, SomeOOClass, SomeImpossibleValue, \ SomeUnicodeString, SomeList, SomeObject, HarmlesslyBlocked, \ - SomeWeakRef, lltype_to_annotation, SomeType + SomeWeakRef, lltype_to_annotation, SomeType, SomeByteArray from pypy.annotation.classdef import InstanceSource, ClassDef from pypy.annotation.listdef import ListDef, ListItem from pypy.annotation.dictdef import DictDef @@ -349,6 +349,8 @@ result = SomeUnicodeCodePoint() else: result = SomeUnicodeString() + elif tp is bytearray: + result = SomeByteArray() elif tp is tuple: result = SomeTuple(items = [self.immutablevalue(e, need_const) for e in x]) elif tp is float: diff --git a/pypy/annotation/builtin.py b/pypy/annotation/builtin.py --- a/pypy/annotation/builtin.py +++ b/pypy/annotation/builtin.py @@ -9,7 +9,7 @@ from pypy.annotation.model import SomeFloat, unionof, SomeUnicodeString from pypy.annotation.model import SomePBC, SomeInstance, SomeDict, SomeList from pypy.annotation.model import SomeWeakRef, SomeIterator -from pypy.annotation.model import SomeOOObject +from pypy.annotation.model import SomeOOObject, SomeByteArray from pypy.annotation.model import annotation_to_lltype, lltype_to_annotation, ll_to_annotation from pypy.annotation.model import add_knowntypedata from pypy.annotation.model import s_ImpossibleValue @@ -119,6 +119,9 @@ def builtin_unicode(s_unicode): return constpropagate(unicode, [s_unicode], SomeUnicodeString()) +def builtin_bytearray(s_str): + return constpropagate(bytearray, [s_str], SomeByteArray()) + def our_issubclass(cls1, cls2): """ we're going to try to be less silly in the face of old-style classes""" from pypy.annotation.classdef import ClassDef @@ -253,24 +256,6 @@ s = SomeInteger(nonneg=True, knowntype=s.knowntype) return s -def builtin_apply(*stuff): - getbookkeeper().warning("ignoring apply%r" % (stuff,)) - return SomeObject() - -##def builtin_slice(*args): -## bk = getbookkeeper() -## if len(args) == 1: -## return SomeSlice( -## bk.immutablevalue(None), args[0], bk.immutablevalue(None)) -## elif len(args) == 2: -## return SomeSlice( -## args[0], args[1], bk.immutablevalue(None)) -## elif len(args) == 3: -## return SomeSlice( -## args[0], args[1], args[2]) -## else: -## raise Exception, "bogus call to slice()" - def OSError_init(s_self, *args): pass diff --git a/pypy/annotation/model.py b/pypy/annotation/model.py --- a/pypy/annotation/model.py +++ b/pypy/annotation/model.py @@ -202,11 +202,16 @@ self.knowntypedata = knowntypedata class SomeStringOrUnicode(SomeObject): + """Base class for shared implementation of SomeString and SomeUnicodeString. + + Cannot be an annotation.""" + immutable = True can_be_None=False no_nul = False # No NUL character in the string. def __init__(self, can_be_None=False, no_nul=False): + assert type(self) is not SomeStringOrUnicode if can_be_None: self.can_be_None = True if no_nul: @@ -225,19 +230,19 @@ d2 = d2.copy(); d2['no_nul'] = 0 # ignored return d1 == d2 + def nonnoneify(self): + return self.__class__(can_be_None=False, no_nul=self.no_nul) + class SomeString(SomeStringOrUnicode): "Stands for an object which is known to be a string." knowntype = str - def nonnoneify(self): - return SomeString(can_be_None=False, no_nul=self.no_nul) - class SomeUnicodeString(SomeStringOrUnicode): "Stands for an object which is known to be an unicode string" knowntype = unicode - def nonnoneify(self): - return SomeUnicodeString(can_be_None=False, no_nul=self.no_nul) +class SomeByteArray(SomeStringOrUnicode): + knowntype = bytearray class SomeChar(SomeString): "Stands for an object known to be a string of length 1." @@ -773,7 +778,3 @@ else: raise RuntimeError("The annotator relies on 'assert' statements from the\n" "\tannotated program: you cannot run it with 'python -O'.") - -# this has the side-effect of registering the unary and binary operations -from pypy.annotation.unaryop import UNARY_OPERATIONS -from pypy.annotation.binaryop import BINARY_OPERATIONS diff --git a/pypy/annotation/signature.py b/pypy/annotation/signature.py --- a/pypy/annotation/signature.py +++ b/pypy/annotation/signature.py @@ -151,4 +151,9 @@ actualtypes[:] = params_s def enforce_signature_return(funcdesc, sigtype, inferredtype): - return finish_type(sigtype, funcdesc.bookkeeper, funcdesc.pyobj) + s_sigret = finish_type(sigtype, funcdesc.bookkeeper, funcdesc.pyobj) + if not s_sigret.contains(inferredtype): + raise Exception("%r return value:\n" + "expected %s,\n" + " got %s" % (funcdesc, s_sigret, inferredtype)) + return s_sigret diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py --- a/pypy/annotation/test/test_annrpython.py +++ b/pypy/annotation/test/test_annrpython.py @@ -255,13 +255,6 @@ assert getcdef(snippet.H).about_attribute('attr') == ( a.bookkeeper.immutablevalue(1)) - def DISABLED_test_knownkeysdict(self): - # disabled, SomeDict() is now a general {s_key: s_value} dict - a = self.RPythonAnnotator() - s = a.build_types(snippet.knownkeysdict, [int]) - # result should be an integer - assert s.knowntype == int - def test_generaldict(self): a = self.RPythonAnnotator() s = a.build_types(snippet.generaldict, [str, int, str, int]) @@ -483,6 +476,13 @@ s = a.build_types(f, [str]) assert isinstance(s, annmodel.SomeString) + def test_str_isalpha(self): + def f(s): + return s.isalpha() + a = self.RPythonAnnotator() + s = a.build_types(f, [str]) + assert isinstance(s, annmodel.SomeBool) + def test_simple_slicing(self): a = self.RPythonAnnotator() s = a.build_types(snippet.simple_slice, [somelist(annmodel.s_Int)]) @@ -3819,6 +3819,37 @@ a = self.RPythonAnnotator() a.build_types(f, []) # assert did not explode + def test_bytearray(self): + def f(): + return bytearray("xyz") + + a = self.RPythonAnnotator() + assert isinstance(a.build_types(f, []), annmodel.SomeByteArray) + + def test_bytearray_add(self): + def f(a): + return a + bytearray("xyz") + + a = self.RPythonAnnotator() + assert isinstance(a.build_types(f, [annmodel.SomeByteArray()]), + annmodel.SomeByteArray) + a = self.RPythonAnnotator() + assert isinstance(a.build_types(f, [str]), + annmodel.SomeByteArray) + a = self.RPythonAnnotator() + assert isinstance(a.build_types(f, [annmodel.SomeChar()]), + annmodel.SomeByteArray) + + def test_bytearray_setitem_getitem(self): + def f(b, i, c): + b[i] = c + return b[i + 1] + + a = self.RPythonAnnotator() + assert isinstance(a.build_types(f, [annmodel.SomeByteArray(), + int, int]), + annmodel.SomeInteger) + def g(n): return [0,1,2,n] diff --git a/pypy/annotation/unaryop.py b/pypy/annotation/unaryop.py --- a/pypy/annotation/unaryop.py +++ b/pypy/annotation/unaryop.py @@ -526,7 +526,14 @@ return SomeString() method_encode.can_only_throw = [UnicodeEncodeError] + class __extend__(SomeString): + def method_isdigit(chr): + return s_Bool + + def method_isalpha(chr): + return s_Bool + def method_upper(str): return SomeString() @@ -562,12 +569,6 @@ def method_isspace(chr): return s_Bool - def method_isdigit(chr): - return s_Bool - - def method_isalpha(chr): - return s_Bool - def method_isalnum(chr): return s_Bool diff --git a/pypy/doc/config/objspace.usemodules.time.txt b/pypy/doc/config/objspace.usemodules.time.txt --- a/pypy/doc/config/objspace.usemodules.time.txt +++ b/pypy/doc/config/objspace.usemodules.time.txt @@ -1,4 +1,5 @@ Use the 'time' module. Obsolete; use :config:`objspace.usemodules.rctime` for our up-to-date version -of the application-level 'time' module. +of the application-level 'time' module, at least for C-like targets (the C +and LLVM backends). diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst --- a/pypy/doc/index.rst +++ b/pypy/doc/index.rst @@ -67,7 +67,7 @@ .. _Python: http://docs.python.org/index.html .. _`more...`: architecture.html#mission-statement .. _`PyPy blog`: http://morepypy.blogspot.com/ -.. _`development bug/feature tracker`: https://codespeak.net/issue/pypy-dev/ +.. _`development bug/feature tracker`: https://bugs.pypy.org .. _here: http://tismerysoft.de/pypy/irc-logs/pypy .. _`Mercurial commit mailing list`: http://python.org/mailman/listinfo/pypy-commit .. _`development mailing list`: http://python.org/mailman/listinfo/pypy-dev diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -8,12 +8,27 @@ .. branch: length-hint Implement __lenght_hint__ according to PEP 424 +.. branch: numpypy-longdouble +Long double support for numpypy + +.. branch: signatures +Improved RPython typing + +.. branch: rpython-bytearray +Rudimentary support for bytearray in RPython + .. branches we don't care about .. branch: autoreds +.. branch: reflex-support +.. branch: kill-faking +.. branch: improved_ebnfparse_error +.. branch: task-decorator .. branch: release-2.0-beta1 .. branch: remove-PYPY_NOT_MAIN_FILE +.. branch: missing-jit-operations + .. branch: fix-lookinside-iff-oopspec Fixed the interaction between two internal tools for controlling the JIT. diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py --- a/pypy/jit/backend/arm/assembler.py +++ b/pypy/jit/backend/arm/assembler.py @@ -1,5 +1,6 @@ from __future__ import with_statement import os +from pypy.jit.backend.llsupport import jitframe from pypy.jit.backend.arm.helper.assembler import saved_registers from pypy.jit.backend.arm import conditions as c from pypy.jit.backend.arm import registers as r @@ -8,7 +9,7 @@ from pypy.jit.backend.arm.codebuilder import ARMv7Builder, OverwritingBuilder from pypy.jit.backend.arm.locations import get_fp_offset from pypy.jit.backend.arm.regalloc import (Regalloc, ARMFrameManager, - ARMv7RegisterManager, check_imm_arg, + CoreRegisterManager, check_imm_arg, operations as regalloc_operations, operations_with_guard as regalloc_operations_with_guard) from pypy.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper @@ -25,12 +26,12 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.jit.backend.arm.opassembler import ResOpAssembler from pypy.rlib.debug import (debug_print, debug_start, debug_stop, - have_debug_prints) + have_debug_prints, fatalerror) from pypy.rlib.jit import AsmInfo from pypy.rlib.objectmodel import compute_unique_id # XXX Move to llsupport -from pypy.jit.backend.x86.support import values_array, memcpy_fn +from pypy.jit.backend.x86.support import memcpy_fn DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', ('i', lltype.Signed), ('type', lltype.Char), # 'b'ridge, 'l'abel or @@ -44,14 +45,8 @@ debug = True - def __init__(self, cpu, failargs_limit=1000): + def __init__(self, cpu, translate_support_code=False): self.cpu = cpu - self.fail_boxes_int = values_array(lltype.Signed, failargs_limit) - self.fail_boxes_float = values_array(longlong.FLOATSTORAGE, - failargs_limit) - self.fail_boxes_ptr = values_array(llmemory.GCREF, failargs_limit) - self.fail_boxes_count = 0 - self.fail_force_index = 0 self.setup_failure_recovery() self.mc = None self.memcpy_addr = 0 @@ -68,6 +63,7 @@ self._debug = False self.loop_run_counters = [] self.debug_counter_descr = cpu.fielddescrof(DEBUG_COUNTER, 'i') + self.force_token_to_dead_frame = {} # XXX temporary hack def set_debug(self, v): r = self._debug @@ -112,9 +108,13 @@ gc_ll_descr.initialize() self._build_wb_slowpath(False) self._build_wb_slowpath(True) + self._build_failure_recovery(exc=True, withfloats=False) + self._build_failure_recovery(exc=False, withfloats=False) if self.cpu.supports_floats: self._build_wb_slowpath(False, withfloats=True) self._build_wb_slowpath(True, withfloats=True) + self._build_failure_recovery(exc=True, withfloats=True) + self._build_failure_recovery(exc=False, withfloats=True) self._build_propagate_exception_path() if gc_ll_descr.get_malloc_slowpath_addr is not None: self._build_malloc_slowpath() @@ -122,10 +122,7 @@ if gc_ll_descr.gcrootmap and gc_ll_descr.gcrootmap.is_shadow_stack: self._build_release_gil(gc_ll_descr.gcrootmap) self.memcpy_addr = self.cpu.cast_ptr_to_int(memcpy_fn) - self._exit_code_addr = self._gen_exit_path() - self._leave_jitted_hook_save_exc = \ - self._gen_leave_jitted_hook_code(True) - self._leave_jitted_hook = self._gen_leave_jitted_hook_code(False) + if not self._debug: # if self._debug is already set it means that someone called # set_debug by hand before initializing the assembler. Leave it @@ -220,36 +217,17 @@ self.releasegil_addr = rffi.cast(lltype.Signed, releasegil_func) self.reacqgil_addr = rffi.cast(lltype.Signed, reacqgil_func) - def _gen_leave_jitted_hook_code(self, save_exc): - mc = ARMv7Builder() - if self.cpu.supports_floats: - floats = r.caller_vfp_resp - else: - floats = [] - with saved_registers(mc, r.caller_resp + [r.lr], floats): - addr = self.cpu.get_on_leave_jitted_int(save_exception=save_exc) - mc.BL(addr) - assert self._exit_code_addr != 0 - mc.B(self._exit_code_addr) - return mc.materialize(self.cpu.asmmemmgr, [], - self.cpu.gc_ll_descr.gcrootmap) - def _build_propagate_exception_path(self): if self.cpu.propagate_exception_v < 0: return # not supported (for tests, or non-translated) # mc = ARMv7Builder() - # call on_leave_jitted_save_exc() - if self.cpu.supports_floats: - floats = r.caller_vfp_resp - else: - floats = [] - with saved_registers(mc, r.caller_resp + [r.lr], floats): - addr = self.cpu.get_on_leave_jitted_int(save_exception=True, - default_to_memoryerror=True) - mc.BL(addr) - mc.gen_load_int(r.ip.value, self.cpu.propagate_exception_v) - mc.MOV_rr(r.r0.value, r.ip.value) + # + # Call the helper, which will return a dead frame object with + # the correct exception set, or MemoryError by default + # XXX make sure we return the correct value here + addr = rffi.cast(lltype.Signed, self.cpu.get_propagate_exception()) + mc.BL(addr) self.gen_func_epilog(mc=mc) self.propagate_exception_path = mc.materialize(self.cpu.asmmemmgr, []) @@ -282,12 +260,12 @@ # restore registers and return # We check for c.EQ here, meaning all bits zero in this case mc.POP([reg.value for reg in r.argument_regs] + [r.pc.value], cond=c.EQ) - # call on_leave_jitted_save_exc() - addr = self.cpu.get_on_leave_jitted_int(save_exception=True) + # + # Call the helper, which will return a dead frame object with + # the correct exception set, or MemoryError by default + addr = rffi.cast(lltype.Signed, self.cpu.get_propagate_exception()) mc.BL(addr) # - mc.gen_load_int(r.r0.value, self.cpu.propagate_exception_v) - # # footer -- note the ADD, which skips the return address of this # function, and will instead return to the caller's caller. Note # also that we completely ignore the saved arguments, because we @@ -339,39 +317,82 @@ def setup_failure_recovery(self): - @rgc.no_collect + #@rgc.no_collect -- XXX still true, but hacked gc_set_extra_threshold def failure_recovery_func(mem_loc, frame_pointer, stack_pointer): """mem_loc is a structure in memory describing where the values for the failargs are stored. frame loc is the address of the frame pointer for the frame to be decoded frame """ - vfp_registers = rffi.cast(rffi.LONGLONGP, stack_pointer) - registers = rffi.ptradd(vfp_registers, len(r.all_vfp_regs)) + vfp_registers = rffi.cast(rffi.LONGP, stack_pointer) + registers = rffi.ptradd(vfp_registers, 2*len(r.all_vfp_regs)) registers = rffi.cast(rffi.LONGP, registers) - return self.decode_registers_and_descr(mem_loc, frame_pointer, + bytecode = rffi.cast(rffi.UCHARP, mem_loc) + return self.grab_frame_values(self.cpu, bytecode, frame_pointer, registers, vfp_registers) + self.failure_recovery_code = [0, 0, 0, 0] self.failure_recovery_func = failure_recovery_func - recovery_func_sign = lltype.Ptr(lltype.FuncType([lltype.Signed] * 3, - lltype.Signed)) + _FAILURE_RECOVERY_FUNC = lltype.Ptr(lltype.FuncType([rffi.LONGP] * 3, + llmemory.GCREF)) - @rgc.no_collect - def decode_registers_and_descr(self, mem_loc, frame_pointer, + @staticmethod + #@rgc.no_collect -- XXX still true, but hacked gc_set_extra_threshold + def grab_frame_values(cpu, bytecode, frame_pointer, registers, vfp_registers): - """Decode locations encoded in memory at mem_loc and write the values - to the failboxes. Values for spilled vars and registers are stored on - stack at frame_loc """ - assert frame_pointer & 1 == 0 - self.fail_force_index = frame_pointer - bytecode = rffi.cast(rffi.UCHARP, mem_loc) + # no malloc allowed here!! xxx apart from one, hacking a lot + force_index = rffi.cast(lltype.Signed, frame_pointer) num = 0 - value = 0 - fvalue = 0 + deadframe = lltype.nullptr(jitframe.DEADFRAME) + # step 1: lots of mess just to count the final value of 'num' + bytecode1 = bytecode + while 1: + code = rffi.cast(lltype.Signed, bytecode1[0]) + bytecode1 = rffi.ptradd(bytecode1, 1) + if code >= AssemblerARM.CODE_FROMSTACK: + while code > 0x7F: + code = rffi.cast(lltype.Signed, bytecode1[0]) + bytecode1 = rffi.ptradd(bytecode1, 1) + else: + kind = code & 3 + if kind == AssemblerARM.DESCR_SPECIAL: + if code == AssemblerARM.CODE_HOLE: + num += 1 + continue + if code == AssemblerARM.CODE_INPUTARG: + continue + if code == AssemblerARM.CODE_FORCED: + # resuming from a GUARD_NOT_FORCED + token = force_index + deadframe = ( + cpu.assembler.force_token_to_dead_frame.pop(token)) + deadframe = lltype.cast_opaque_ptr( + jitframe.DEADFRAMEPTR, deadframe) + continue + assert code == AssemblerARM.CODE_STOP + break + num += 1 + + # allocate the deadframe + if not deadframe: + # Remove the "reserve" at the end of the nursery. This means + # that it is guaranteed that the following malloc() works + # without requiring a collect(), but it needs to be re-added + # as soon as possible. + cpu.gc_clear_extra_threshold() + assert num <= cpu.get_failargs_limit() + try: + deadframe = lltype.malloc(jitframe.DEADFRAME, num) + except MemoryError: + fatalerror("memory usage error in grab_frame_values") + # fill it code_inputarg = False - while True: + num = 0 + value_hi = 0 + while 1: + # decode the next instruction from the bytecode code = rffi.cast(lltype.Signed, bytecode[0]) bytecode = rffi.ptradd(bytecode, 1) - if code >= self.CODE_FROMSTACK: + if code >= AssemblerARM.CODE_FROMSTACK: if code > 0x7F: shift = 7 code &= 0x7F @@ -384,54 +405,63 @@ break # load the value from the stack kind = code & 3 - code = int((code - self.CODE_FROMSTACK) >> 2) + code = (code - AssemblerARM.CODE_FROMSTACK) >> 2 if code_inputarg: code = ~code code_inputarg = False - if kind == self.DESCR_FLOAT: - # we use code + 1 to get the hi word of the double worded float - stackloc = frame_pointer - get_fp_offset(int(code) + 1) - assert stackloc & 3 == 0 - fvalue = rffi.cast(rffi.LONGLONGP, stackloc)[0] - else: - stackloc = frame_pointer - get_fp_offset(int(code)) - assert stackloc & 1 == 0 - value = rffi.cast(rffi.LONGP, stackloc)[0] + stackloc = force_index - get_fp_offset(int(code)) + value = rffi.cast(rffi.LONGP, stackloc)[0] + if kind == AssemblerARM.DESCR_FLOAT: + assert WORD == 4 + value_hi = value + value = rffi.cast(rffi.LONGP, stackloc - WORD)[0] else: - # 'code' identifies a register: load its value kind = code & 3 - if kind == self.DESCR_SPECIAL: - if code == self.CODE_HOLE: + if kind == AssemblerARM.DESCR_SPECIAL: + if code == AssemblerARM.CODE_HOLE: num += 1 continue - if code == self.CODE_INPUTARG: + if code == AssemblerARM.CODE_INPUTARG: code_inputarg = True continue - assert code == self.CODE_STOP + if code == AssemblerARM.CODE_FORCED: + continue + assert code == AssemblerARM.CODE_STOP break + # 'code' identifies a register: load its value code >>= 2 - if kind == self.DESCR_FLOAT: - fvalue = vfp_registers[code] + if kind == AssemblerARM.DESCR_FLOAT: + if WORD == 4: + value = vfp_registers[2*code] + value_hi = vfp_registers[2*code + 1] + else: + value = registers[code] else: value = registers[code] # store the loaded value into fail_boxes_ - if kind == self.DESCR_FLOAT: - tgt = self.fail_boxes_float.get_addr_for_num(num) - rffi.cast(rffi.LONGLONGP, tgt)[0] = fvalue + if kind == AssemblerARM.DESCR_INT: + deadframe.jf_values[num].int = value + elif kind == AssemblerARM.DESCR_REF: + deadframe.jf_values[num].ref = rffi.cast(llmemory.GCREF, value) + elif kind == AssemblerARM.DESCR_FLOAT: + assert WORD == 4 + assert not longlong.is_64_bit + floatvalue = rffi.cast(lltype.SignedLongLong, value_hi) + floatvalue <<= 32 + floatvalue |= rffi.cast(lltype.SignedLongLong, + rffi.cast(lltype.Unsigned, value)) + deadframe.jf_values[num].float = floatvalue else: - if kind == self.DESCR_INT: - tgt = self.fail_boxes_int.get_addr_for_num(num) - elif kind == self.DESCR_REF: - assert (value & 3) == 0, "misaligned pointer" - tgt = self.fail_boxes_ptr.get_addr_for_num(num) - else: - assert 0, "bogus kind" - rffi.cast(rffi.LONGP, tgt)[0] = value + assert 0, "bogus kind" num += 1 - self.fail_boxes_count = num + # + assert num == len(deadframe.jf_values) + if not we_are_translated(): + assert bytecode[4] == 0xCC fail_index = rffi.cast(rffi.INTP, bytecode)[0] - fail_index = rffi.cast(lltype.Signed, fail_index) - return fail_index + fail_descr = cpu.get_fail_descr_from_number(fail_index) + deadframe.jf_descr = fail_descr.hide(cpu) + return lltype.cast_opaque_ptr(llmemory.GCREF, deadframe) def decode_inputargs(self, code): descr_to_box_type = [REF, INT, FLOAT] @@ -492,10 +522,10 @@ # are stored in r0 and r1. mc.SUB_rr(r.r0.value, r.r1.value, r.r0.value) addr = self.cpu.gc_ll_descr.get_malloc_slowpath_addr() - for reg, ofs in ARMv7RegisterManager.REGLOC_TO_COPY_AREA_OFS.items(): + for reg, ofs in CoreRegisterManager.REGLOC_TO_COPY_AREA_OFS.items(): mc.STR_ri(reg.value, r.fp.value, imm=ofs) mc.BL(addr) - for reg, ofs in ARMv7RegisterManager.REGLOC_TO_COPY_AREA_OFS.items(): + for reg, ofs in CoreRegisterManager.REGLOC_TO_COPY_AREA_OFS.items(): mc.LDR_ri(reg.value, r.fp.value, imm=ofs) mc.CMP_ri(r.r0.value, 0) @@ -514,12 +544,26 @@ self.mc.CMP_ri(r.r0.value, 0) self.mc.B(self.propagate_exception_path, c=c.EQ) - def _gen_exit_path(self): + def _build_failure_recovery(self, exc, withfloats=False): mc = ARMv7Builder() - decode_registers_addr = llhelper(self.recovery_func_sign, + failure_recovery = llhelper(self._FAILURE_RECOVERY_FUNC, self.failure_recovery_func) self._insert_checks(mc) - with saved_registers(mc, r.all_regs, r.all_vfp_regs): + if withfloats: + f = r.all_vfp_regs + else: + f = [] + with saved_registers(mc, r.all_regs, f): + if exc: + # We might have an exception pending. Load it into r4 + # (this is a register saved across calls) + mc.gen_load_int(r.r5.value, self.cpu.pos_exc_value()) + mc.LDR_ri(r.r4.value, self.cpu.pos_exc_value()) + # clear the exc flags + mc.gen_load_int(r.r6.value, 0) + mc.STR_ri(r.r6.value, r.r5.value) + mc.gen_load_int(r.r5.value, self.cpu.pos_exception()) + mc.STR_ri(r.r6.value, r.r5.value) # move mem block address, to r0 to pass as mc.MOV_rr(r.r0.value, r.lr.value) # pass the current frame pointer as second param @@ -527,23 +571,32 @@ # pass the current stack pointer as third param mc.MOV_rr(r.r2.value, r.sp.value) self._insert_checks(mc) - mc.BL(rffi.cast(lltype.Signed, decode_registers_addr)) + mc.BL(rffi.cast(lltype.Signed, failure_recovery)) + if exc: + # save ebx into 'jf_guard_exc' + from pypy.jit.backend.llsupport.descr import unpack_fielddescr + descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) + offset, size, _ = unpack_fielddescr(descrs.jf_guard_exc) + mc.STR_rr(r.r4.value, r.r0.value, offset, cond=c.AL) mc.MOV_rr(r.ip.value, r.r0.value) mc.MOV_rr(r.r0.value, r.ip.value) self.gen_func_epilog(mc=mc) - return mc.materialize(self.cpu.asmmemmgr, [], + rawstart = mc.materialize(self.cpu.asmmemmgr, [], self.cpu.gc_ll_descr.gcrootmap) + self.failure_recovery_code[exc + 2 * withfloats] = rawstart + self.mc = None DESCR_REF = 0x00 DESCR_INT = 0x01 DESCR_FLOAT = 0x02 DESCR_SPECIAL = 0x03 CODE_FROMSTACK = 64 - CODE_STOP = 0 | DESCR_SPECIAL - CODE_HOLE = 4 | DESCR_SPECIAL - CODE_INPUTARG = 8 | DESCR_SPECIAL + CODE_STOP = 0 | DESCR_SPECIAL + CODE_HOLE = 4 | DESCR_SPECIAL + CODE_INPUTARG = 8 | DESCR_SPECIAL + CODE_FORCED = 12 | DESCR_SPECIAL #XXX where should this be written? - def gen_descr_encoding(self, descr, failargs, locs): + def write_failure_recovery_description(self, descr, failargs, locs): assert self.mc is not None for i in range(len(failargs)): arg = failargs[i] @@ -575,26 +628,33 @@ self.mc.writechar(chr(n)) self.mc.writechar(chr(self.CODE_STOP)) - fdescr = self.cpu.get_fail_descr_number(descr) - self.mc.write32(fdescr) + + def generate_quick_failure(self, guardtok, fcond=c.AL): + assert isinstance(guardtok.save_exc, bool) + fail_index = self.cpu.get_fail_descr_number(guardtok.descr) + startpos = self.mc.currpos() + withfloats = False + for box in guardtok.failargs: + if box is not None and box.type == FLOAT: + withfloats = True + break + exc = guardtok.save_exc + target = self.failure_recovery_code[exc + 2 * withfloats] + assert target != 0 + self.mc.BL(target) + # write tight data that describes the failure recovery + if guardtok.is_guard_not_forced: + self.mc.writechar(chr(self.CODE_FORCED)) + self.write_failure_recovery_description(guardtok.descr, + guardtok.failargs, guardtok.faillocs[1:]) + self.mc.write32(fail_index) + # for testing the decoding, write a final byte 0xCC + if not we_are_translated(): + self.mc.writechar('\xCC') + faillocs = [loc for loc in guardtok.faillocs if loc is not None] + guardtok.descr._arm_debug_faillocs = faillocs self.align() - - # assert that the fail_boxes lists are big enough - assert len(failargs) <= self.fail_boxes_int.SIZE - - def _gen_path_to_exit_path(self, descr, args, arglocs, - save_exc, fcond=c.AL): - assert isinstance(save_exc, bool) - self.gen_exit_code(self.mc, save_exc, fcond) - self.gen_descr_encoding(descr, args, arglocs[1:]) - - def gen_exit_code(self, mc, save_exc, fcond=c.AL): - assert isinstance(save_exc, bool) - if save_exc: - path = self._leave_jitted_hook_save_exc - else: - path = self._leave_jitted_hook - mc.BL(path) + return startpos def align(self): while(self.mc.currpos() % FUNC_ALIGN != 0): @@ -841,15 +901,10 @@ def write_pending_failure_recoveries(self): for tok in self.pending_guards: - descr = tok.descr #generate the exit stub and the encoded representation - pos = self.mc.currpos() - tok.pos_recovery_stub = pos - - self._gen_path_to_exit_path(descr, tok.failargs, - tok.faillocs, save_exc=tok.save_exc) + tok.pos_recovery_stub = self.generate_quick_failure(tok) # store info on the descr - descr._arm_current_frame_depth = tok.faillocs[0].getint() + tok.descr._arm_current_frame_depth = tok.faillocs[0].getint() def process_pending_guards(self, block_start): clt = self.current_clt @@ -860,7 +915,7 @@ descr._arm_failure_recovery_block = failure_recovery_pos relative_offset = tok.pos_recovery_stub - tok.offset guard_pos = block_start + tok.offset - if not tok.is_invalidate: + if not tok.is_guard_not_invalidated: # patch the guard jumpt to the stub # overwrite the generate NOP with a B_offs to the pos of the # stub @@ -1255,11 +1310,6 @@ else: raise AssertionError('Trying to pop to an invalid location') - def leave_jitted_hook(self): - ptrs = self.fail_boxes_ptr.ar - llop.gc_assume_young_pointers(lltype.Void, - llmemory.cast_ptr_to_adr(ptrs)) - def malloc_cond(self, nursery_free_adr, nursery_top_adr, size): assert size & (WORD-1) == 0 # must be correctly aligned diff --git a/pypy/jit/backend/arm/helper/regalloc.py b/pypy/jit/backend/arm/helper/regalloc.py --- a/pypy/jit/backend/arm/helper/regalloc.py +++ b/pypy/jit/backend/arm/helper/regalloc.py @@ -32,14 +32,14 @@ imm_a0 = check_imm_box(a0, imm_size, allow_zero=allow_zero) imm_a1 = check_imm_box(a1, imm_size, allow_zero=allow_zero) if not imm_a0 and imm_a1: - l0 = self._ensure_value_is_boxed(a0) + l0 = self.make_sure_var_in_reg(a0) l1 = self.convert_to_imm(a1) elif commutative and imm_a0 and not imm_a1: l1 = self.convert_to_imm(a0) - l0 = self._ensure_value_is_boxed(a1, boxes) + l0 = self.make_sure_var_in_reg(a1, boxes) else: - l0 = self._ensure_value_is_boxed(a0, boxes) - l1 = self._ensure_value_is_boxed(a1, boxes) + l0 = self.make_sure_var_in_reg(a0, boxes) + l1 = self.make_sure_var_in_reg(a1, boxes) self.possibly_free_vars_for_op(op) self.free_temp_vars() res = self.force_allocate_reg(op.result, boxes) @@ -52,10 +52,10 @@ if guard: def f(self, op, guard_op, fcond): locs = [] - loc1 = self._ensure_value_is_boxed(op.getarg(0)) + loc1 = self.make_sure_var_in_reg(op.getarg(0)) locs.append(loc1) if base: - loc2 = self._ensure_value_is_boxed(op.getarg(1)) + loc2 = self.make_sure_var_in_reg(op.getarg(1)) locs.append(loc2) self.possibly_free_vars_for_op(op) self.free_temp_vars() @@ -70,10 +70,10 @@ else: def f(self, op, fcond): locs = [] - loc1 = self._ensure_value_is_boxed(op.getarg(0)) + loc1 = self.make_sure_var_in_reg(op.getarg(0)) locs.append(loc1) if base: - loc2 = self._ensure_value_is_boxed(op.getarg(1)) + loc2 = self.make_sure_var_in_reg(op.getarg(1)) locs.append(loc2) self.possibly_free_vars_for_op(op) self.free_temp_vars() @@ -111,11 +111,11 @@ arg0, arg1 = boxes imm_a1 = check_imm_box(arg1) - l0 = self._ensure_value_is_boxed(arg0, forbidden_vars=boxes) + l0 = self.make_sure_var_in_reg(arg0, forbidden_vars=boxes) if imm_a1: l1 = self.convert_to_imm(arg1) else: - l1 = self._ensure_value_is_boxed(arg1, forbidden_vars=boxes) + l1 = self.make_sure_var_in_reg(arg1, forbidden_vars=boxes) self.possibly_free_vars_for_op(op) self.free_temp_vars() @@ -134,7 +134,7 @@ assert fcond is not None a0 = op.getarg(0) assert isinstance(a0, Box) - reg = self._ensure_value_is_boxed(a0) + reg = self.make_sure_var_in_reg(a0) self.possibly_free_vars_for_op(op) if guard_op is None: res = self.force_allocate_reg(op.result, [a0]) diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py --- a/pypy/jit/backend/arm/opassembler.py +++ b/pypy/jit/backend/arm/opassembler.py @@ -16,6 +16,7 @@ gen_emit_unary_float_op, saved_registers, count_reg_args) +from pypy.jit.backend.arm.helper.regalloc import check_imm_arg from pypy.jit.backend.arm.codebuilder import ARMv7Builder, OverwritingBuilder from pypy.jit.backend.arm.jump import remap_frame_layout from pypy.jit.backend.arm.regalloc import TempInt, TempPtr @@ -27,18 +28,20 @@ from pypy.jit.metainterp.history import JitCellToken, TargetToken from pypy.jit.metainterp.resoperation import rop from pypy.rlib.objectmodel import we_are_translated -from pypy.rpython.lltypesystem import rstr +from pypy.rlib import rgc +from pypy.rpython.lltypesystem import rstr, rffi, lltype, llmemory NO_FORCE_INDEX = -1 class GuardToken(object): - def __init__(self, descr, failargs, faillocs, offset, - save_exc, fcond=c.AL, is_invalidate=False): + def __init__(self, descr, failargs, faillocs, offset, save_exc, fcond=c.AL, + is_guard_not_invalidated=False, is_guard_not_forced=False): assert isinstance(save_exc, bool) self.descr = descr self.offset = offset - self.is_invalidate = is_invalidate + self.is_guard_not_invalidated = is_guard_not_invalidated + self.is_guard_not_forced = is_guard_not_forced self.failargs = failargs self.faillocs = faillocs self.save_exc = save_exc @@ -190,7 +193,8 @@ return fcond def _emit_guard(self, op, arglocs, fcond, save_exc, - is_guard_not_invalidated=False): + is_guard_not_invalidated=False, + is_guard_not_forced=False): assert isinstance(save_exc, bool) assert isinstance(fcond, int) descr = op.getdescr() @@ -210,7 +214,8 @@ faillocs=arglocs, offset=pos, save_exc=save_exc, - is_invalidate=is_guard_not_invalidated, + is_guard_not_invalidated=is_guard_not_invalidated, + is_guard_not_forced=is_guard_not_forced, fcond=fcond)) return c.AL @@ -312,49 +317,10 @@ return fcond def emit_op_finish(self, op, arglocs, regalloc, fcond): - for i in range(len(arglocs) - 1): - loc = arglocs[i] - box = op.getarg(i) - if loc is None: - continue - if loc.is_reg(): - if box.type == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(i) - elif box.type == INT: - adr = self.fail_boxes_int.get_addr_for_num(i) - else: - assert 0 - self.mc.gen_load_int(r.ip.value, adr) - self.mc.STR_ri(loc.value, r.ip.value) - elif loc.is_vfp_reg(): - assert box.type == FLOAT - adr = self.fail_boxes_float.get_addr_for_num(i) - self.mc.gen_load_int(r.ip.value, adr) - self.mc.VSTR(loc.value, r.ip.value) - elif loc.is_stack() or loc.is_imm() or loc.is_imm_float(): - if box.type == FLOAT: - adr = self.fail_boxes_float.get_addr_for_num(i) - self.mov_loc_loc(loc, r.vfp_ip) - self.mc.gen_load_int(r.ip.value, adr) - self.mc.VSTR(r.vfp_ip.value, r.ip.value) - elif box.type == REF or box.type == INT: - if box.type == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(i) - elif box.type == INT: - adr = self.fail_boxes_int.get_addr_for_num(i) - else: - assert 0 - self.mov_loc_loc(loc, r.ip) - self.mc.gen_load_int(r.lr.value, adr) - self.mc.STR_ri(r.ip.value, r.lr.value) - else: - assert 0 - # note: no exception should currently be set in llop.get_exception_addr - # even if this finish may be an exit_frame_with_exception (in this case - # the exception instance is in arglocs[0]). - addr = self.cpu.get_on_leave_jitted_int(save_exception=False) - self.mc.BL(addr) - self.mc.gen_load_int(r.r0.value, arglocs[-1].value) + [argloc] = arglocs + if argloc is not r.r0: #XXX verify this + self.mov_loc_loc(argloc, r.r0, fcond) + # exit function self.gen_func_epilog() return fcond @@ -988,8 +954,8 @@ def _emit_copystrcontent(self, op, regalloc, fcond, is_unicode): # compute the source address args = op.getarglist() - base_loc = regalloc._ensure_value_is_boxed(args[0], args) - ofs_loc = regalloc._ensure_value_is_boxed(args[2], args) + base_loc = regalloc.make_sure_var_in_reg(args[0], args) + ofs_loc = regalloc.make_sure_var_in_reg(args[2], args) assert args[0] is not args[1] # forbidden case of aliasing regalloc.possibly_free_var(args[0]) regalloc.free_temp_vars() @@ -1009,8 +975,8 @@ dstaddr_loc = regalloc.force_allocate_reg(dstaddr_box, selected_reg=r.r0) forbidden_vars.append(dstaddr_box) - base_loc = regalloc._ensure_value_is_boxed(args[1], forbidden_vars) - ofs_loc = regalloc._ensure_value_is_boxed(args[3], forbidden_vars) + base_loc = regalloc.make_sure_var_in_reg(args[1], forbidden_vars) + ofs_loc = regalloc.make_sure_var_in_reg(args[3], forbidden_vars) assert base_loc.is_reg() assert ofs_loc.is_reg() regalloc.possibly_free_var(args[1]) @@ -1026,7 +992,7 @@ # need the box here if isinstance(args[4], Box): length_box = args[4] - length_loc = regalloc._ensure_value_is_boxed(args[4], + length_loc = regalloc.make_sure_var_in_reg(args[4], forbidden_vars) else: length_box = TempInt() @@ -1152,8 +1118,26 @@ value = self.cpu.done_with_this_frame_float_v else: raise AssertionError(kind) - self.mc.gen_load_int(r.ip.value, value) - self.mc.CMP_rr(tmploc.value, r.ip.value) + from pypy.jit.backend.llsupport.descr import unpack_fielddescr + from pypy.jit.backend.llsupport.descr import unpack_interiorfielddescr + descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) + _offset, _size, _ = unpack_fielddescr(descrs.jf_descr) + fail_descr = self.cpu.get_fail_descr_from_number(value) + value = fail_descr.hide(self.cpu) + rgc._make_sure_does_not_move(value) + value = rffi.cast(lltype.Signed, value) + + if check_imm_arg(_offset): + self.mc.LDR_ri(r.ip.value, tmploc.value, imm=_offset) + else: + self.mc.gen_load_int(r.ip.value, _offset) + self.mc.LDR_rr(r.ip.value, tmploc.value, r.ip.value) + if check_imm_arg(value): + self.mc.CMP_ri(r.ip.value, imm=value) + else: + self.mc.gen_load_int(r.lr.value, value) + self.mc.CMP_rr(r.lr.value, r.ip.value) + #if values are equal we take the fast path # Slow path, calling helper @@ -1180,19 +1164,31 @@ if op.result is not None: # load the return value from fail_boxes_xxx[0] kind = op.result.type - if kind == INT: - adr = self.fail_boxes_int.get_addr_for_num(0) - elif kind == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(0) - elif kind == FLOAT: - adr = self.fail_boxes_float.get_addr_for_num(0) + if kind == FLOAT: + t = unpack_interiorfielddescr(descrs.as_float)[0] + if not check_imm_arg(t): + self.mc.gen_load_int(r.ip.value, t, cond=fast_path_cond) + self.mc.ADD_rr(r.ip.value, r.r0.value, r.ip.value, + cond=fast_path_cond) + t = 0 + base = r.ip + else: + base = r.r0 + self.mc.VLDR(resloc.value, base.value, imm=t, + cond=fast_path_cond) else: - raise AssertionError(kind) - self.mc.gen_load_int(r.ip.value, adr, cond=fast_path_cond) - if op.result.type == FLOAT: - self.mc.VLDR(resloc.value, r.ip.value, cond=fast_path_cond) - else: - self.mc.LDR_ri(resloc.value, r.ip.value, cond=fast_path_cond) + assert resloc is r.r0 + if kind == INT: + t = unpack_interiorfielddescr(descrs.as_int)[0] + else: + t = unpack_interiorfielddescr(descrs.as_ref)[0] + if not check_imm_arg(t): + self.mc.gen_load_int(r.ip.value, t, cond=fast_path_cond) + self.mc.LDR_rr(resloc.value, resloc.value, r.ip.value, + cond=fast_path_cond) + else: + self.mc.LDR_ri(resloc.value, resloc.value, imm=t, + cond=fast_path_cond) # jump to merge point jmp_pos = self.mc.currpos() self.mc.BKPT() @@ -1266,7 +1262,8 @@ self.mc.LDR_ri(r.ip.value, r.fp.value) self.mc.CMP_ri(r.ip.value, 0) - self._emit_guard(guard_op, arglocs[1 + numargs:], c.GE, save_exc=True) + self._emit_guard(guard_op, arglocs[1 + numargs:], c.GE, + save_exc=True, is_guard_not_forced=True) return fcond def emit_guard_call_release_gil(self, op, guard_op, arglocs, regalloc, diff --git a/pypy/jit/backend/arm/regalloc.py b/pypy/jit/backend/arm/regalloc.py --- a/pypy/jit/backend/arm/regalloc.py +++ b/pypy/jit/backend/arm/regalloc.py @@ -84,8 +84,28 @@ def void(self, op, fcond): return [] +class ARMRegisterManager(RegisterManager): + def return_constant(self, v, forbidden_vars=[], selected_reg=None): + self._check_type(v) + if isinstance(v, Const): + if isinstance(v, ConstPtr): + tp = REF + elif isinstance(v, ConstFloat): + tp = FLOAT + else: + tp = INT + loc = self.get_scratch_reg(tp, + self.temp_boxes + forbidden_vars, + selected_reg=selected_reg) + immvalue = self.convert_to_imm(v) + self.assembler.load(loc, immvalue) + return loc + else: + return RegisterManager.return_constant(self, v, + forbidden_vars, selected_reg) -class VFPRegisterManager(RegisterManager): + +class VFPRegisterManager(ARMRegisterManager): all_regs = r.all_vfp_regs box_types = [FLOAT] save_around_call_regs = r.all_vfp_regs @@ -107,20 +127,7 @@ reg = self.force_allocate_reg(v, selected_reg=r.d0) return reg - def ensure_value_is_boxed(self, thing, forbidden_vars=[]): - loc = None - if isinstance(thing, Const): - assert isinstance(thing, ConstFloat) - loc = self.get_scratch_reg(FLOAT, self.temp_boxes + forbidden_vars) - immvalue = self.convert_to_imm(thing) - self.assembler.load(loc, immvalue) - else: - loc = self.make_sure_var_in_reg(thing, - forbidden_vars=self.temp_boxes + forbidden_vars) - return loc - - def get_scratch_reg(self, type=FLOAT, forbidden_vars=[], - selected_reg=None): + def get_scratch_reg(self, type=FLOAT, forbidden_vars=[], selected_reg=None): assert type == FLOAT # for now box = TempFloat() self.temp_boxes.append(box) @@ -129,7 +136,7 @@ return reg -class ARMv7RegisterManager(RegisterManager): +class CoreRegisterManager(ARMRegisterManager): all_regs = r.all_regs box_types = None # or a list of acceptable types no_lower_byte_regs = all_regs @@ -162,22 +169,6 @@ return locations.ImmLocation(rffi.cast(lltype.Signed, c.value)) assert 0 - def ensure_value_is_boxed(self, thing, forbidden_vars=None): - loc = None - if isinstance(thing, Const): - if isinstance(thing, ConstPtr): - tp = REF - else: - tp = INT - loc = self.get_scratch_reg(tp, forbidden_vars=self.temp_boxes - + forbidden_vars) - immvalue = self.convert_to_imm(thing) - self.assembler.load(loc, immvalue) - else: - loc = self.make_sure_var_in_reg(thing, - forbidden_vars=self.temp_boxes + forbidden_vars) - return loc - def get_scratch_reg(self, type=INT, forbidden_vars=[], selected_reg=None): assert type == INT or type == REF box = TempBox() @@ -277,7 +268,12 @@ def make_sure_var_in_reg(self, var, forbidden_vars=[], selected_reg=None, need_lower_byte=False): - assert 0, 'should not be called directly' + if var.type == FLOAT: + return self.vfprm.make_sure_var_in_reg(var, forbidden_vars, + selected_reg, need_lower_byte) + else: + return self.rm.make_sure_var_in_reg(var, forbidden_vars, + selected_reg, need_lower_byte) def convert_to_imm(self, value): if isinstance(value, ConstInt): @@ -294,7 +290,7 @@ fm = self.frame_manager asm = self.assembler self.vfprm = VFPRegisterManager(longevity, fm, asm) - self.rm = ARMv7RegisterManager(longevity, fm, asm) + self.rm = CoreRegisterManager(longevity, fm, asm) def prepare_loop(self, inputargs, operations): self._prepare(inputargs, operations) @@ -426,12 +422,6 @@ self.rm.before_call(force_store, save_all_regs) self.vfprm.before_call(force_store, save_all_regs) - def _ensure_value_is_boxed(self, thing, forbidden_vars=[]): - if thing.type == FLOAT: - return self.vfprm.ensure_value_is_boxed(thing, forbidden_vars) - else: - return self.rm.ensure_value_is_boxed(thing, forbidden_vars) - def _sync_var(self, v): if v.type == FLOAT: self.vfprm._sync_var(v) @@ -444,14 +434,14 @@ imm_a0 = check_imm_box(a0) imm_a1 = check_imm_box(a1) if not imm_a0 and imm_a1: - l0 = self._ensure_value_is_boxed(a0, boxes) + l0 = self.make_sure_var_in_reg(a0, boxes) l1 = self.convert_to_imm(a1) elif imm_a0 and not imm_a1: l0 = self.convert_to_imm(a0) - l1 = self._ensure_value_is_boxed(a1, boxes) + l1 = self.make_sure_var_in_reg(a1, boxes) else: - l0 = self._ensure_value_is_boxed(a0, boxes) - l1 = self._ensure_value_is_boxed(a1, boxes) + l0 = self.make_sure_var_in_reg(a0, boxes) + l1 = self.make_sure_var_in_reg(a1, boxes) return [l0, l1] def prepare_op_int_add(self, op, fcond): @@ -466,14 +456,14 @@ imm_a0 = check_imm_box(a0) imm_a1 = check_imm_box(a1) if not imm_a0 and imm_a1: - l0 = self._ensure_value_is_boxed(a0, boxes) + l0 = self.make_sure_var_in_reg(a0, boxes) l1 = self.convert_to_imm(a1) elif imm_a0 and not imm_a1: l0 = self.convert_to_imm(a0) - l1 = self._ensure_value_is_boxed(a1, boxes) + l1 = self.make_sure_var_in_reg(a1, boxes) else: - l0 = self._ensure_value_is_boxed(a0, boxes) - l1 = self._ensure_value_is_boxed(a1, boxes) + l0 = self.make_sure_var_in_reg(a0, boxes) + l1 = self.make_sure_var_in_reg(a1, boxes) return [l0, l1] def prepare_op_int_sub(self, op, fcond): @@ -487,8 +477,8 @@ boxes = op.getarglist() a0, a1 = boxes - reg1 = self._ensure_value_is_boxed(a0, forbidden_vars=boxes) - reg2 = self._ensure_value_is_boxed(a1, forbidden_vars=boxes) + reg1 = self.make_sure_var_in_reg(a0, forbidden_vars=boxes) + reg2 = self.make_sure_var_in_reg(a1, forbidden_vars=boxes) self.possibly_free_vars(boxes) self.possibly_free_vars_for_op(op) @@ -497,14 +487,14 @@ return [reg1, reg2, res] def prepare_op_int_force_ge_zero(self, op, fcond): - argloc = self._ensure_value_is_boxed(op.getarg(0)) + argloc = self.make_sure_var_in_reg(op.getarg(0)) resloc = self.force_allocate_reg(op.result, [op.getarg(0)]) return [argloc, resloc] def prepare_guard_int_mul_ovf(self, op, guard, fcond): boxes = op.getarglist() - reg1 = self._ensure_value_is_boxed(boxes[0], forbidden_vars=boxes) - reg2 = self._ensure_value_is_boxed(boxes[1], forbidden_vars=boxes) + reg1 = self.make_sure_var_in_reg(boxes[0], forbidden_vars=boxes) + reg2 = self.make_sure_var_in_reg(boxes[1], forbidden_vars=boxes) res = self.force_allocate_reg(op.result) return self._prepare_guard(guard, [reg1, reg2, res]) @@ -576,7 +566,7 @@ prepare_guard_int_is_zero = prepare_op_unary_cmp('int_is_zero') def prepare_op_int_neg(self, op, fcond): - l0 = self._ensure_value_is_boxed(op.getarg(0)) + l0 = self.make_sure_var_in_reg(op.getarg(0)) self.possibly_free_vars_for_op(op) self.free_temp_vars() resloc = self.force_allocate_reg(op.result) @@ -629,15 +619,15 @@ def _prepare_llong_binop_xx(self, op, fcond): # arg 0 is the address of the function - loc0 = self._ensure_value_is_boxed(op.getarg(1)) - loc1 = self._ensure_value_is_boxed(op.getarg(2)) + loc0 = self.make_sure_var_in_reg(op.getarg(1)) + loc1 = self.make_sure_var_in_reg(op.getarg(2)) self.possibly_free_vars_for_op(op) self.free_temp_vars() res = self.vfprm.force_allocate_reg(op.result) return [loc0, loc1, res] def _prepare_llong_to_int(self, op, fcond): - loc0 = self._ensure_value_is_boxed(op.getarg(1)) + loc0 = self.make_sure_var_in_reg(op.getarg(1)) res = self.force_allocate_reg(op.result) return [loc0, res] @@ -654,18 +644,12 @@ return args def prepare_op_finish(self, op, fcond): - args = [None] * (op.numargs() + 1) - for i in range(op.numargs()): - arg = op.getarg(i) - if arg: - args[i] = self.loc(arg) - self.possibly_free_var(arg) - n = self.cpu.get_fail_descr_number(op.getdescr()) - args[-1] = imm(n) - return args + loc = self.loc(op.getarg(0)) + self.possibly_free_var(op.getarg(0)) + return [loc] def prepare_op_guard_true(self, op, fcond): - l0 = self._ensure_value_is_boxed(op.getarg(0)) + l0 = self.make_sure_var_in_reg(op.getarg(0)) args = self._prepare_guard(op, [l0]) return args @@ -677,9 +661,9 @@ boxes = op.getarglist() a0, a1 = boxes imm_a1 = check_imm_box(a1) - l0 = self._ensure_value_is_boxed(a0, boxes) + l0 = self.make_sure_var_in_reg(a0, boxes) if not imm_a1: - l1 = self._ensure_value_is_boxed(a1, boxes) + l1 = self.make_sure_var_in_reg(a1, boxes) else: l1 = self.convert_to_imm(a1) assert op.result is None @@ -699,7 +683,7 @@ def prepare_op_guard_exception(self, op, fcond): boxes = op.getarglist() arg0 = ConstInt(rffi.cast(lltype.Signed, op.getarg(0).getint())) - loc = self._ensure_value_is_boxed(arg0) + loc = self.make_sure_var_in_reg(arg0) loc1 = self.get_scratch_reg(INT, boxes) if op.result in self.longevity: resloc = self.force_allocate_reg(op.result, boxes) @@ -713,7 +697,7 @@ return arglocs def prepare_op_guard_no_exception(self, op, fcond): - loc = self._ensure_value_is_boxed( + loc = self.make_sure_var_in_reg( ConstInt(self.cpu.pos_exception())) arglocs = self._prepare_guard(op, [loc]) return arglocs @@ -727,7 +711,7 @@ assert isinstance(op.getarg(0), Box) boxes = op.getarglist() - x = self._ensure_value_is_boxed(boxes[0], boxes) + x = self.make_sure_var_in_reg(boxes[0], boxes) y_val = rffi.cast(lltype.Signed, op.getarg(1).getint()) arglocs = [x, None, None] @@ -837,8 +821,8 @@ boxes = op.getarglist() a0, a1 = boxes ofs, size, sign = unpack_fielddescr(op.getdescr()) - base_loc = self._ensure_value_is_boxed(a0, boxes) - value_loc = self._ensure_value_is_boxed(a1, boxes) + base_loc = self.make_sure_var_in_reg(a0, boxes) + value_loc = self.make_sure_var_in_reg(a1, boxes) if check_imm_arg(ofs): ofs_loc = imm(ofs) else: @@ -851,7 +835,7 @@ def prepare_op_getfield_gc(self, op, fcond): a0 = op.getarg(0) ofs, size, sign = unpack_fielddescr(op.getdescr()) - base_loc = self._ensure_value_is_boxed(a0) + base_loc = self.make_sure_var_in_reg(a0) immofs = imm(ofs) if check_imm_arg(ofs): ofs_loc = immofs @@ -871,8 +855,8 @@ t = unpack_interiorfielddescr(op.getdescr()) ofs, itemsize, fieldsize, sign = t args = op.getarglist() - base_loc = self._ensure_value_is_boxed(op.getarg(0), args) - index_loc = self._ensure_value_is_boxed(op.getarg(1), args) + base_loc = self.make_sure_var_in_reg(op.getarg(0), args) + index_loc = self.make_sure_var_in_reg(op.getarg(1), args) immofs = imm(ofs) if check_imm_arg(ofs): ofs_loc = immofs @@ -889,9 +873,9 @@ t = unpack_interiorfielddescr(op.getdescr()) ofs, itemsize, fieldsize, sign = t args = op.getarglist() - base_loc = self._ensure_value_is_boxed(op.getarg(0), args) - index_loc = self._ensure_value_is_boxed(op.getarg(1), args) - value_loc = self._ensure_value_is_boxed(op.getarg(2), args) + base_loc = self.make_sure_var_in_reg(op.getarg(0), args) + index_loc = self.make_sure_var_in_reg(op.getarg(1), args) + value_loc = self.make_sure_var_in_reg(op.getarg(2), args) immofs = imm(ofs) if check_imm_arg(ofs): ofs_loc = immofs @@ -907,7 +891,7 @@ assert isinstance(arraydescr, ArrayDescr) ofs = arraydescr.lendescr.offset arg = op.getarg(0) - base_loc = self._ensure_value_is_boxed(arg) + base_loc = self.make_sure_var_in_reg(arg) self.possibly_free_vars_for_op(op) self.free_temp_vars() res = self.force_allocate_reg(op.result) @@ -917,9 +901,9 @@ size, ofs, _ = unpack_arraydescr(op.getdescr()) scale = get_scale(size) args = op.getarglist() - base_loc = self._ensure_value_is_boxed(args[0], args) - ofs_loc = self._ensure_value_is_boxed(args[1], args) - value_loc = self._ensure_value_is_boxed(args[2], args) + base_loc = self.make_sure_var_in_reg(args[0], args) + ofs_loc = self.make_sure_var_in_reg(args[1], args) + value_loc = self.make_sure_var_in_reg(args[2], args) assert check_imm_arg(ofs) return [value_loc, base_loc, ofs_loc, imm(scale), imm(ofs)] prepare_op_setarrayitem_raw = prepare_op_setarrayitem_gc @@ -929,8 +913,8 @@ boxes = op.getarglist() size, ofs, _ = unpack_arraydescr(op.getdescr()) scale = get_scale(size) - base_loc = self._ensure_value_is_boxed(boxes[0], boxes) - ofs_loc = self._ensure_value_is_boxed(boxes[1], boxes) + base_loc = self.make_sure_var_in_reg(boxes[0], boxes) + ofs_loc = self.make_sure_var_in_reg(boxes[1], boxes) self.possibly_free_vars_for_op(op) self.free_temp_vars() res = self.force_allocate_reg(op.result) @@ -944,7 +928,7 @@ def prepare_op_strlen(self, op, fcond): args = op.getarglist() - l0 = self._ensure_value_is_boxed(op.getarg(0)) + l0 = self.make_sure_var_in_reg(op.getarg(0)) basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, self.cpu.translate_support_code) immofs = imm(ofs_length) @@ -963,14 +947,14 @@ def prepare_op_strgetitem(self, op, fcond): boxes = op.getarglist() - base_loc = self._ensure_value_is_boxed(boxes[0]) + base_loc = self.make_sure_var_in_reg(boxes[0]) a1 = boxes[1] imm_a1 = check_imm_box(a1) if imm_a1: ofs_loc = self.convert_to_imm(a1) else: - ofs_loc = self._ensure_value_is_boxed(a1, boxes) + ofs_loc = self.make_sure_var_in_reg(a1, boxes) self.possibly_free_vars_for_op(op) self.free_temp_vars() @@ -983,9 +967,9 @@ def prepare_op_strsetitem(self, op, fcond): boxes = op.getarglist() - base_loc = self._ensure_value_is_boxed(boxes[0], boxes) - ofs_loc = self._ensure_value_is_boxed(boxes[1], boxes) - value_loc = self._ensure_value_is_boxed(boxes[2], boxes) + base_loc = self.make_sure_var_in_reg(boxes[0], boxes) + ofs_loc = self.make_sure_var_in_reg(boxes[1], boxes) + value_loc = self.make_sure_var_in_reg(boxes[2], boxes) basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, self.cpu.translate_support_code) assert itemsize == 1 @@ -995,7 +979,7 @@ prepare_op_copyunicodecontent = void def prepare_op_unicodelen(self, op, fcond): - l0 = self._ensure_value_is_boxed(op.getarg(0)) + l0 = self.make_sure_var_in_reg(op.getarg(0)) basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, self.cpu.translate_support_code) immofs = imm(ofs_length) @@ -1012,8 +996,8 @@ def prepare_op_unicodegetitem(self, op, fcond): boxes = op.getarglist() - base_loc = self._ensure_value_is_boxed(boxes[0], boxes) - ofs_loc = self._ensure_value_is_boxed(boxes[1], boxes) + base_loc = self.make_sure_var_in_reg(boxes[0], boxes) + ofs_loc = self.make_sure_var_in_reg(boxes[1], boxes) self.possibly_free_vars_for_op(op) self.free_temp_vars() @@ -1027,9 +1011,9 @@ def prepare_op_unicodesetitem(self, op, fcond): boxes = op.getarglist() - base_loc = self._ensure_value_is_boxed(boxes[0], boxes) - ofs_loc = self._ensure_value_is_boxed(boxes[1], boxes) - value_loc = self._ensure_value_is_boxed(boxes[2], boxes) + base_loc = self.make_sure_var_in_reg(boxes[0], boxes) + ofs_loc = self.make_sure_var_in_reg(boxes[1], boxes) + value_loc = self.make_sure_var_in_reg(boxes[2], boxes) basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, self.cpu.translate_support_code) scale = itemsize / 2 @@ -1042,7 +1026,7 @@ if imm_arg: argloc = self.convert_to_imm(arg) else: - argloc = self._ensure_value_is_boxed(arg) + argloc = self.make_sure_var_in_reg(arg) self.possibly_free_vars_for_op(op) self.free_temp_vars() resloc = self.force_allocate_reg(op.result) @@ -1093,7 +1077,7 @@ # twice from the memory. N = op.numargs() args = op.getarglist() - arglocs = [self._ensure_value_is_boxed(op.getarg(i), args) + arglocs = [self.make_sure_var_in_reg(op.getarg(i), args) for i in range(N)] tmp = self.get_scratch_reg(INT, args) assert tmp not in arglocs @@ -1215,7 +1199,7 @@ float_result=False, name='prepare_guard_float_ge') def prepare_op_math_sqrt(self, op, fcond): - loc = self._ensure_value_is_boxed(op.getarg(1)) + loc = self.make_sure_var_in_reg(op.getarg(1)) self.possibly_free_vars_for_op(op) self.free_temp_vars() res = self.vfprm.force_allocate_reg(op.result) @@ -1223,12 +1207,12 @@ return [loc, res] def prepare_op_cast_float_to_int(self, op, fcond): - loc1 = self._ensure_value_is_boxed(op.getarg(0)) + loc1 = self.make_sure_var_in_reg(op.getarg(0)) res = self.rm.force_allocate_reg(op.result) return [loc1, res] def prepare_op_cast_int_to_float(self, op, fcond): - loc1 = self._ensure_value_is_boxed(op.getarg(0)) + loc1 = self.make_sure_var_in_reg(op.getarg(0)) res = self.vfprm.force_allocate_reg(op.result) return [loc1, res] @@ -1247,12 +1231,12 @@ return [loc, res] def prepare_op_cast_float_to_singlefloat(self, op, fcond): - loc1 = self._ensure_value_is_boxed(op.getarg(0)) + loc1 = self.make_sure_var_in_reg(op.getarg(0)) res = self.force_allocate_reg(op.result) return [loc1, res] - + def prepare_op_cast_singlefloat_to_float(self, op, fcond): - loc1 = self._ensure_value_is_boxed(op.getarg(0)) + loc1 = self.make_sure_var_in_reg(op.getarg(0)) res = self.force_allocate_reg(op.result) return [loc1, res] diff --git a/pypy/jit/backend/arm/runner.py b/pypy/jit/backend/arm/runner.py --- a/pypy/jit/backend/arm/runner.py +++ b/pypy/jit/backend/arm/runner.py @@ -23,15 +23,21 @@ AbstractLLCPU.__init__(self, rtyper, stats, opts, translate_support_code, gcdescr) From noreply at buildbot.pypy.org Wed Jan 9 04:28:20 2013 From: noreply at buildbot.pypy.org (wlav) Date: Wed, 9 Jan 2013 04:28:20 +0100 (CET) Subject: [pypy-commit] pypy reflex-support: for CINT backend: TTree reading of builtin types Message-ID: <20130109032820.C2A3E1C1339@cobra.cs.uni-duesseldorf.de> Author: Wim Lavrijsen Branch: reflex-support Changeset: r59883:3f8bc9293373 Date: 2013-01-08 15:35 -0800 http://bitbucket.org/pypy/pypy/changeset/3f8bc9293373/ Log: for CINT backend: TTree reading of builtin types diff --git a/pypy/module/cppyy/capi/cint_capi.py b/pypy/module/cppyy/capi/cint_capi.py --- a/pypy/module/cppyy/capi/cint_capi.py +++ b/pypy/module/cppyy/capi/cint_capi.py @@ -7,10 +7,9 @@ from pypy.translator.tool.cbuild import ExternalCompilationInfo from pypy.rpython.lltypesystem import rffi + from pypy.rlib import libffi, rdynload -from pypy.module.itertools import interp_itertools - __all__ = ['identify', 'eci', 'c_load_dictionary'] @@ -162,20 +161,51 @@ from pypy.module.cppyy import interp_cppyy tree = space.interp_w(interp_cppyy.W_CPPInstance, w_self) + space = tree.space # holds the class cache in State + + # prevent recursion + attr = space.str_w(args_w[0]) + if attr and attr[0] == '_': + raise OperationError(space.w_AttributeError, args_w[0]) + + # try the saved cdata (for builtin types) + try: + w_cdata = space.getattr(w_self, space.wrap('_'+attr)) + from pypy.module._cffi_backend import cdataobj + cdata = space.interp_w(cdataobj.W_CData, w_cdata, can_be_None=False) + return cdata.convert_to_object() + except OperationError: + pass + # setup branch as a data member and enable it for reading - space = tree.space # holds the class cache in State w_branch = space.call_method(w_self, "GetBranch", args_w[0]) if not space.is_true(w_branch): raise OperationError(space.w_AttributeError, args_w[0]) + activate_branch(space, w_branch) w_klassname = space.call_method(w_branch, "GetClassName") - klass = interp_cppyy.scope_byname(space, space.str_w(w_klassname)) - w_obj = klass.construct() - #space.call_method(w_branch, "SetStatus", space.wrap(1)) - activate_branch(space, w_branch) - space.call_method(w_branch, "SetObject", w_obj) - space.call_method(w_branch, "GetEntry", space.wrap(0)) - space.setattr(w_self, args_w[0], w_obj) - return w_obj + if space.is_true(w_klassname): + # some instance + klass = interp_cppyy.scope_byname(space, space.str_w(w_klassname)) + w_obj = klass.construct() + space.call_method(w_branch, "SetObject", w_obj) + space.call_method(w_branch, "GetEntry", space.wrap(0)) + space.setattr(w_self, args_w[0], w_obj) + return w_obj + else: + # builtin data + w_leaf = space.call_method(w_self, "GetLeaf", args_w[0]) + w_typename = space.call_method(w_leaf, "GetTypeName" ) + from pypy.module.cppyy import capi + typename = capi.c_resolve_name(space.str_w(w_typename)) + w_address = space.call_method(w_leaf, "GetValuePointer") + buf = space.buffer_w(w_address) + from pypy.module._rawffi import buffer + assert isinstance(buf, buffer.RawFFIBuffer) + address = rffi.cast(rffi.CCHARP, buf.datainstance.ll_buffer) + from pypy.module._cffi_backend import cdataobj, newtype + cdata = cdataobj.W_CData(space, address, newtype.new_primitive_type(space, typename)) + space.setattr(w_self, space.wrap('_'+attr), space.wrap(cdata)) + return space.getattr(w_self, args_w[0]) class W_TTreeIter(Wrappable): def __init__(self, space, w_tree): diff --git a/pypy/module/cppyy/interp_cppyy.py b/pypy/module/cppyy/interp_cppyy.py --- a/pypy/module/cppyy/interp_cppyy.py +++ b/pypy/module/cppyy/interp_cppyy.py @@ -921,7 +921,10 @@ try: return self.space.call_method(self.space.wrap(self), "_cppyy_as_builtin") except OperationError, e: - if not e.match(self.space, self.space.w_AttributeError): + if not (e.match(self.space, self.space.w_TypeError) or + e.match(self.space, self.space.w_AttributeError)): + # TODO: TypeError is raised by call_method if the method is not found; + # it'd be a lot nicer if only AttributeError were raise raise return None @@ -945,7 +948,7 @@ if not e.match(self.space, self.space.w_TypeError): raise - # fallback 1: convert the object to a builin equivalent + # fallback 1: convert the object to a builtin equivalent w_as_builtin = self._get_as_builtin() if w_as_builtin is not None: return self.space.eq(w_as_builtin, w_other) diff --git a/pypy/module/cppyy/src/cintcwrapper.cxx b/pypy/module/cppyy/src/cintcwrapper.cxx --- a/pypy/module/cppyy/src/cintcwrapper.cxx +++ b/pypy/module/cppyy/src/cintcwrapper.cxx @@ -274,6 +274,10 @@ } cppyy_scope_t cppyy_get_scope(const char* scope_name) { + // CINT still has trouble with std:: sometimes ... + if (strncmp(scope_name, "std::", 5) == 0) + scope_name = &scope_name[5]; + ClassRefIndices_t::iterator icr = g_classref_indices.find(scope_name); if (icr != g_classref_indices.end()) return (cppyy_type_t)icr->second; @@ -281,7 +285,7 @@ if (strcmp(scope_name, "#define") == 0) return (cppyy_type_t)NULL; - // use TClass directly, to enable auto-loading + // use TClass directly, to enable auto-loading TClassRef cr(TClass::GetClass(scope_name, kTRUE, kTRUE)); if (!cr.GetClass()) return (cppyy_type_t)NULL; diff --git a/pypy/module/cppyy/test/test_cint.py b/pypy/module/cppyy/test/test_cint.py --- a/pypy/module/cppyy/test/test_cint.py +++ b/pypy/module/cppyy/test/test_cint.py @@ -139,7 +139,7 @@ class AppTestCINTTTree: - spaceconfig = dict(usemodules=['cppyy']) + spaceconfig = dict(usemodules=['cppyy', 'array', '_rawffi', '_cffi_backend']) def setup_class(cls): cls.w_N = cls.space.wrap(5) @@ -148,7 +148,8 @@ cls.w_tname = cls.space.wrap("test") cls.w_title = cls.space.wrap("test tree") cls.w_iotypes = cls.space.appexec([], """(): - import cppyy + import cppyy, _cffi_backend + _cffi_backend.new_primitive_type # prevents leak-checking complaints on _cffi_backend return cppyy.load_reflection_info(%r)""" % (iotypes_dct,)) def test01_write_stdvector(self): @@ -176,7 +177,17 @@ f.Write() f.Close() - def test02_read_stdvector(self): + def test02_file_open(self): + + from cppyy import gbl + + f = gbl.TFile.Open(self.fname) + s = str(f) # should not raise + r = repr(f) + + f.Close() + + def test03_read_stdvector(self): """Test reading of a single branched TTree with an std::vector""" from cppyy import gbl @@ -195,7 +206,7 @@ f.Close() - def test03_write_some_data_object(self): + def test04_write_some_data_object(self): """Test writing of a complex data object""" from cppyy import gbl @@ -220,7 +231,7 @@ f.Write() f.Close() - def test04_read_some_data_object(self): + def test05_read_some_data_object(self): """Test reading of a complex data object""" from cppyy import gbl @@ -251,7 +262,7 @@ # f.Close() - def test05_branch_activation(self): + def test06_branch_activation(self): """Test of automatic branch activation""" from cppyy import gbl @@ -306,6 +317,49 @@ i += 1 assert i == self.N + def test07_write_builtin(self): + """Test writing of a builtins""" + + from cppyy import gbl # bootstraps, only needed for tests + from cppyy.gbl import TFile, TTree + from cppyy.gbl.std import vector + + f = TFile(self.fname, "RECREATE") + mytree = TTree(self.tname, self.title) + mytree._python_owns = False + + import array + a = array.array('i', [0]) + b = array.array('d', [0.]) + + mytree.Branch("myi", a, "myi/I") + mytree.Branch("myd", b, "myd/D") + + for i in range(self.N): + a[0] = i + b[0] = i/2. + mytree.Fill() + f.Write() + f.Close() + + def test08_read_builtin(self): + """Test reading of a single branched TTree with an std::vector""" + + from cppyy import gbl + from cppyy.gbl import TFile + + f = TFile(self.fname) + mytree = f.Get(self.tname) + + i = 0 + for event in mytree: + assert event.myi == i + assert event.myd == i/2. + i += 1 + assert i == self.N + + f.Close() + class AppTestRegression: spaceconfig = dict(usemodules=['cppyy']) From noreply at buildbot.pypy.org Wed Jan 9 04:28:21 2013 From: noreply at buildbot.pypy.org (wlav) Date: Wed, 9 Jan 2013 04:28:21 +0100 (CET) Subject: [pypy-commit] pypy reflex-support: significant optimization for TTree iteration Message-ID: <20130109032821.F135A1C133C@cobra.cs.uni-duesseldorf.de> Author: Wim Lavrijsen Branch: reflex-support Changeset: r59884:a14198d01269 Date: 2013-01-08 18:32 -0800 http://bitbucket.org/pypy/pypy/changeset/a14198d01269/ Log: significant optimization for TTree iteration diff --git a/pypy/module/cppyy/capi/cint_capi.py b/pypy/module/cppyy/capi/cint_capi.py --- a/pypy/module/cppyy/capi/cint_capi.py +++ b/pypy/module/cppyy/capi/cint_capi.py @@ -153,6 +153,12 @@ space.call_method(w_branch, "SetStatus", space.wrap(1)) space.call_method(w_branch, "ResetReadEntry") +c_ttree_GetEntry = rffi.llexternal( + "cppyy_ttree_GetEntry", + [rffi.VOIDP, rffi.LONGLONG], rffi.LONGLONG, + threadsafe=False, + compilation_info=eci) + @unwrap_spec(args_w='args_w') def ttree_getattr(space, w_self, args_w): """Specialized __getattr__ for TTree's that allows switching on/off the @@ -211,10 +217,9 @@ def __init__(self, space, w_tree): from pypy.module.cppyy import interp_cppyy tree = space.interp_w(interp_cppyy.W_CPPInstance, w_tree) - self.tree = tree.get_cppthis(tree.cppclass) + self.vtree = rffi.cast(rffi.VOIDP, tree.get_cppthis(tree.cppclass)) self.w_tree = w_tree - self.getentry = tree.cppclass.get_overload("GetEntry").functions[0] self.current = 0 self.maxentry = space.int_w(space.call_method(w_tree, "GetEntriesFast")) @@ -228,7 +233,7 @@ if self.current == self.maxentry: raise OperationError(self.space.w_StopIteration, self.space.w_None) # TODO: check bytes read? - self.getentry.call(self.tree, [self.space.wrap(self.current)]) + c_ttree_GetEntry(self.vtree, self.current) self.current += 1 return self.w_tree diff --git a/pypy/module/cppyy/include/cintcwrapper.h b/pypy/module/cppyy/include/cintcwrapper.h --- a/pypy/module/cppyy/include/cintcwrapper.h +++ b/pypy/module/cppyy/include/cintcwrapper.h @@ -15,6 +15,8 @@ void* vtree, const char* branchname, const char* classname, void* addobj, int bufsize, int splitlevel); + long long cppyy_ttree_GetEntry(void* vtree, long long entry); + #ifdef __cplusplus } #endif // ifdef __cplusplus diff --git a/pypy/module/cppyy/src/cintcwrapper.cxx b/pypy/module/cppyy/src/cintcwrapper.cxx --- a/pypy/module/cppyy/src/cintcwrapper.cxx +++ b/pypy/module/cppyy/src/cintcwrapper.cxx @@ -950,3 +950,7 @@ if (b) b->SetObject(addobj); return (cppyy_object_t)b; } + +long long cppyy_ttree_GetEntry(void* vtree, long long entry) { + return (long long)((TTree*)vtree)->GetEntry((Long64_t)entry); +} From noreply at buildbot.pypy.org Wed Jan 9 04:28:23 2013 From: noreply at buildbot.pypy.org (wlav) Date: Wed, 9 Jan 2013 04:28:23 +0100 (CET) Subject: [pypy-commit] pypy reflex-support: merge default into branch Message-ID: <20130109032823.8BCEF1C133D@cobra.cs.uni-duesseldorf.de> Author: Wim Lavrijsen Branch: reflex-support Changeset: r59885:4297455b87c7 Date: 2013-01-08 19:28 -0800 http://bitbucket.org/pypy/pypy/changeset/4297455b87c7/ Log: merge default into branch diff --git a/pypy/doc/Makefile b/pypy/doc/Makefile --- a/pypy/doc/Makefile +++ b/pypy/doc/Makefile @@ -32,38 +32,38 @@ -rm -rf $(BUILDDIR)/* html: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." pickle: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ @@ -73,7 +73,7 @@ @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/PyPy.qhc" latex: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @@ -81,26 +81,26 @@ "run these through (pdf)latex." man: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man" changes: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." doctest: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -38,7 +38,7 @@ # General information about the project. project = u'PyPy' -copyright = u'2011, The PyPy Project' +copyright = u'2013, The PyPy Project' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/pypy/doc/config/translation.no__thread.txt b/pypy/doc/config/translation.no__thread.txt --- a/pypy/doc/config/translation.no__thread.txt +++ b/pypy/doc/config/translation.no__thread.txt @@ -1,3 +1,3 @@ Don't use gcc __thread attribute for fast thread local storage -implementation . Increases the chance that moving the resulting +implementation. Increases the chance that moving the resulting executable to another same processor Linux machine will work. diff --git a/pypy/doc/pypyconfig.py b/pypy/doc/pypyconfig.py --- a/pypy/doc/pypyconfig.py +++ b/pypy/doc/pypyconfig.py @@ -3,6 +3,10 @@ def setup(app): import sys, os sys.path.insert(0, os.path.abspath("../../")) + + #Autmatically calls make_cmdlline_overview + from pypy.doc.config import generate + from pypy.config import makerestdoc import py role = makerestdoc.register_config_role(py.path.local()) diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -5,6 +5,11 @@ .. this is a revision shortly after release-2.0-beta1 .. startrev: 0e6161a009c6 +.. branch: callback-jit +Callbacks from C are now better JITted + +.. branch: remove-globals-in-jit + .. branch: length-hint Implement __lenght_hint__ according to PEP 424 diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -56,11 +56,24 @@ ---------------------------- On Windows, there is no standard place where to download, build and -install third-party libraries. We chose to install them in the parent +install third-party libraries. We recommend installing them in the parent directory of the pypy checkout. For example, if you installed pypy in ``d:\pypy\trunk\`` (This directory contains a README file), the base -directory is ``d:\pypy``. You may choose different values by setting the -INCLUDE, LIB and PATH (for DLLs) +directory is ``d:\pypy``. You must then set the +INCLUDE, LIB and PATH (for DLLs) environment variables appropriately. + +Abridged method (for -Ojit builds using Visual Studio 2008) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Download the versions of all the external packages +from +https://bitbucket.org/pypy/pypy/downloads/local.zip +Then expand it into the base directory (base_dir) and modify your environment to reflect this:: + + set PATH=\bin;%PATH% + set INCLUDE=\include;%INCLUDE% + set LIB=\lib;%LIB% + +Now you should be good to go. Read on for more information. The Boehm garbage collector ~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py --- a/pypy/interpreter/typedef.py +++ b/pypy/interpreter/typedef.py @@ -105,6 +105,13 @@ # typical usage (case 1 is the most common kind of app-level subclasses; # case 2 is the memory-saving kind defined with __slots__). # +# +----------------------------------------------------------------+ +# | NOTE: if withmapdict is enabled, the following doesn't apply! | +# | Map dicts can flexibly allow any slots/__dict__/__weakref__ to | +# | show up only when needed. In particular there is no way with | +# | mapdict to prevent some objects from being weakrefable. | +# +----------------------------------------------------------------+ +# # dict slots del weakrefable # # 1. Y N N Y UserDictWeakref diff --git a/pypy/jit/backend/arm/test/test_gc_integration.py b/pypy/jit/backend/arm/test/test_gc_integration.py --- a/pypy/jit/backend/arm/test/test_gc_integration.py +++ b/pypy/jit/backend/arm/test/test_gc_integration.py @@ -2,25 +2,13 @@ """ Tests for register allocation for common constructs """ -import py -from pypy.jit.metainterp.history import BoxInt, \ - BoxPtr, TreeLoop, TargetToken -from pypy.jit.metainterp.resoperation import rop, ResOperation -from pypy.jit.codewriter import heaptracker -from pypy.jit.backend.llsupport.descr import GcCache -from pypy.jit.backend.llsupport.gc import GcLLDescription +from pypy.jit.metainterp.history import TargetToken +from pypy.jit.backend.llsupport.gc import GcLLDescription, GcLLDescr_boehm from pypy.jit.backend.detect_cpu import getcpuclass from pypy.jit.backend.arm.arch import WORD from pypy.rpython.lltypesystem import lltype, llmemory, rffi -from pypy.rpython.annlowlevel import llhelper -from pypy.rpython.lltypesystem import rclass -from pypy.jit.backend.llsupport.gc import GcLLDescr_framework -from pypy.jit.backend.arm.test.test_regalloc import MockAssembler from pypy.jit.backend.arm.test.test_regalloc import BaseTestRegalloc -from pypy.jit.backend.arm.regalloc import ARMFrameManager, VFPRegisterManager -from pypy.jit.codewriter.effectinfo import EffectInfo -from pypy.jit.backend.arm.regalloc import Regalloc CPU = getcpuclass() @@ -44,23 +32,14 @@ return ['compressed'] + shape[1:] -class MockGcDescr(GcCache): - get_malloc_slowpath_addr = None - write_barrier_descr = None - moving_gc = True +class MockGcDescr(GcLLDescr_boehm): gcrootmap = MockGcRootMap() - def initialize(self): - pass - - _record_constptrs = GcLLDescr_framework._record_constptrs.im_func - rewrite_assembler = GcLLDescr_framework.rewrite_assembler.im_func - class TestRegallocGcIntegration(BaseTestRegalloc): cpu = CPU(None, None) - cpu.gc_ll_descr = MockGcDescr(False) + cpu.gc_ll_descr = MockGcDescr(None, None, None) cpu.setup_once() S = lltype.GcForwardReference() diff --git a/pypy/jit/backend/llgraph/runner.py b/pypy/jit/backend/llgraph/runner.py --- a/pypy/jit/backend/llgraph/runner.py +++ b/pypy/jit/backend/llgraph/runner.py @@ -877,13 +877,16 @@ # assembler_helper_ptr = jd.assembler_helper_adr.ptr # fish try: - return assembler_helper_ptr(pframe, vable) + result = assembler_helper_ptr(pframe, vable) except LLException, lle: assert self.last_exception is None, "exception left behind" self.last_exception = lle # fish op op = self.current_op return op.result and op.result.value + if isinstance(result, float): + result = support.cast_to_floatstorage(result) + return result def execute_same_as(self, _, x): return x diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py --- a/pypy/jit/metainterp/pyjitpl.py +++ b/pypy/jit/metainterp/pyjitpl.py @@ -1379,16 +1379,15 @@ """The 'residual_call' operation is emitted in two cases: when we have to generate a residual CALL operation, but also to handle an indirect_call that may need to be inlined.""" - assert isinstance(funcbox, Const) - sd = self.metainterp.staticdata - key = sd.cpu.ts.getaddr_for_box(funcbox) - jitcode = sd.bytecode_for_address(key) - if jitcode is not None: - # we should follow calls to this graph - return self.metainterp.perform_call(jitcode, argboxes) - else: - # but we should not follow calls to that graph - return self.do_residual_call(funcbox, argboxes, calldescr) + if isinstance(funcbox, Const): + sd = self.metainterp.staticdata + key = sd.cpu.ts.getaddr_for_box(funcbox) + jitcode = sd.bytecode_for_address(key) + if jitcode is not None: + # we should follow calls to this graph + return self.metainterp.perform_call(jitcode, argboxes) + # but we should not follow calls to that graph + return self.do_residual_call(funcbox, argboxes, calldescr) # ____________________________________________________________ diff --git a/pypy/jit/metainterp/resume.py b/pypy/jit/metainterp/resume.py --- a/pypy/jit/metainterp/resume.py +++ b/pypy/jit/metainterp/resume.py @@ -1085,6 +1085,9 @@ def consume_virtualref_info(self, vrefinfo, numb, end): # we have to decode a list of references containing pairs # [..., virtual, vref, ...] stopping at 'end' + if vrefinfo is None: + assert end == 0 + return assert (end & 1) == 0 for i in range(0, end, 2): virtual = self.decode_ref(numb.nums[i]) diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py --- a/pypy/jit/metainterp/test/test_ajit.py +++ b/pypy/jit/metainterp/test/test_ajit.py @@ -3979,6 +3979,8 @@ rgc.add_memory_pressure(1234) return 3 + self.interp_operations(f, []) + def test_external_call(self): from pypy.rlib.objectmodel import invoke_around_extcall diff --git a/pypy/jit/metainterp/test/test_call.py b/pypy/jit/metainterp/test/test_call.py new file mode 100644 --- /dev/null +++ b/pypy/jit/metainterp/test/test_call.py @@ -0,0 +1,27 @@ + +from pypy.jit.metainterp.test.support import LLJitMixin +from pypy.rlib import jit + +class TestCall(LLJitMixin): + def test_indirect_call(self): + @jit.dont_look_inside + def f1(x): + return x + 1 + + @jit.dont_look_inside + def f2(x): + return x + 2 + + @jit.dont_look_inside + def choice(i): + if i: + return f1 + return f2 + + def f(i): + func = choice(i) + return func(i) + + res = self.interp_operations(f, [3]) + assert res == f(3) + diff --git a/pypy/jit/metainterp/test/test_warmspot.py b/pypy/jit/metainterp/test/test_warmspot.py --- a/pypy/jit/metainterp/test/test_warmspot.py +++ b/pypy/jit/metainterp/test/test_warmspot.py @@ -519,6 +519,44 @@ self.check_trace_count(1) + def test_callback_jit_merge_point(self): + from pypy.rlib.objectmodel import register_around_callback_hook + from pypy.rpython.lltypesystem import lltype, rffi + from pypy.translator.tool.cbuild import ExternalCompilationInfo + + callback_jit_driver = JitDriver(greens = ['name'], reds = 'auto') + + def callback_merge_point(name): + callback_jit_driver.jit_merge_point(name=name) + + @callback_jit_driver.inline(callback_merge_point) + def callback_hook(name): + pass + + def callback(a, b): + if a > b: + return 1 + return -1 + + CB_TP = rffi.CCallback([lltype.Signed, lltype.Signed], lltype.Signed) + eci = ExternalCompilationInfo(includes=['stdlib.h']) + qsort = rffi.llexternal('qsort', + [rffi.VOIDP, lltype.Signed, lltype.Signed, + CB_TP], lltype.Void, compilation_info=eci) + ARR = rffi.CArray(lltype.Signed) + + def main(): + register_around_callback_hook(callback_hook) + raw = lltype.malloc(ARR, 10, flavor='raw') + for i in range(10): + raw[i] = 10 - i + qsort(raw, 10, rffi.sizeof(lltype.Signed), callback) + lltype.free(raw, flavor='raw') + + self.meta_interp(main, []) + self.check_trace_count(1) + + class TestLLWarmspot(WarmspotTests, LLJitMixin): CPUClass = runner.LLGraphCPU type_system = 'lltype' diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py --- a/pypy/jit/metainterp/warmspot.py +++ b/pypy/jit/metainterp/warmspot.py @@ -290,11 +290,13 @@ callgraph = inlinable_static_callers(self.translator.graphs, store_calls=True) new_callgraph = [] new_portals = set() + inlined_jit_merge_points = set() for caller, block, op_call, callee in callgraph: func = getattr(callee, 'func', None) _inline_jit_merge_point_ = getattr(func, '_inline_jit_merge_point_', None) if _inline_jit_merge_point_: _inline_jit_merge_point_._always_inline_ = True + inlined_jit_merge_points.add(_inline_jit_merge_point_) op_jmp_call, jmp_graph = get_jmp_call(callee, _inline_jit_merge_point_) # # now we move the op_jmp_call from callee to caller, just @@ -315,6 +317,9 @@ # inline them! inline_threshold = 0.1 # we rely on the _always_inline_ set above auto_inlining(self.translator, inline_threshold, new_callgraph) + # clean up _always_inline_ = True, it can explode later + for item in inlined_jit_merge_points: + del item._always_inline_ # make a fresh copy of the JitDriver in all newly created # jit_merge_points @@ -1011,6 +1016,9 @@ origblock.operations.append(newop) origblock.exitswitch = None origblock.recloseblock(Link([v_result], origportalgraph.returnblock)) + # the origportal now can raise (even if it did not raise before), + # which means that we cannot inline it anywhere any more, but that's + # fine since any forced inlining has been done before # checkgraph(origportalgraph) diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py --- a/pypy/module/_cffi_backend/ccallback.py +++ b/pypy/module/_cffi_backend/ccallback.py @@ -3,10 +3,10 @@ """ import os from pypy.interpreter.error import OperationError, operationerrfmt -from pypy.rpython.lltypesystem import lltype, llmemory, rffi +from pypy.rpython.lltypesystem import lltype, rffi from pypy.rlib.objectmodel import compute_unique_id, keepalive_until_here -from pypy.rlib import clibffi, rweakref, rgc -from pypy.rlib.rarithmetic import r_ulonglong +from pypy.rlib import clibffi, rweakref +from pypy.rlib import jit from pypy.module._cffi_backend.cdataobj import W_CData from pypy.module._cffi_backend.ctypefunc import SIZE_OF_FFI_ARG, BIG_ENDIAN @@ -77,6 +77,7 @@ space.wrap("expected a function ctype")) return ctype + @jit.unroll_safe def invoke(self, ll_args): space = self.space ctype = self.getfunctype() diff --git a/pypy/module/_weakref/test/test_weakref.py b/pypy/module/_weakref/test/test_weakref.py --- a/pypy/module/_weakref/test/test_weakref.py +++ b/pypy/module/_weakref/test/test_weakref.py @@ -114,19 +114,28 @@ class A(object): def __eq__(self, other): return True + def __ne__(self, other): + return False a1 = A() a2 = A() ref1 = _weakref.ref(a1) ref2 = _weakref.ref(a2) assert ref1 == ref2 + assert not (ref1 != ref2) + assert not (ref1 == []) + assert ref1 != [] del a1 gc.collect() assert not ref1 == ref2 assert ref1 != ref2 + assert not (ref1 == []) + assert ref1 != [] del a2 gc.collect() assert not ref1 == ref2 assert ref1 != ref2 + assert not (ref1 == []) + assert ref1 != [] def test_getweakrefs(self): import _weakref, gc @@ -298,6 +307,13 @@ if seen_callback: assert seen_callback == [True, True, True] + def test_type_weakrefable(self): + import _weakref, gc + w = _weakref.ref(list) + assert w() is list + gc.collect() + assert w() is list + class AppTestProxy(object): spaceconfig = dict(usemodules=('_weakref',)) @@ -435,6 +451,8 @@ class A(object): def __eq__(self, other): return True + def __ne__(self, other): + return False a = A() assert _weakref.ref(a) == a diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py --- a/pypy/module/pyexpat/interp_pyexpat.py +++ b/pypy/module/pyexpat/interp_pyexpat.py @@ -182,6 +182,8 @@ def clear(self): self.next_id = 0 + self._last_object_id = -1 + self._last_object = None self.storage = {} @staticmethod @@ -194,10 +196,18 @@ @staticmethod def get_object(id): - return global_storage.storage[id] + if id == global_storage._last_object_id: + return global_storage._last_object + result = global_storage.storage[id] + global_storage._last_object_id = id + global_storage._last_object = result + return result @staticmethod def free_nonmoving_id(id): + if id == global_storage._last_object_id: + global_storage._last_object = None + global_storage._last_object_id = -1 del global_storage.storage[id] global_storage = Storage() diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py --- a/pypy/module/pypyjit/interp_jit.py +++ b/pypy/module/pypyjit/interp_jit.py @@ -6,7 +6,7 @@ from pypy.tool.pairtype import extendabletype from pypy.rlib.rarithmetic import r_uint, intmask from pypy.rlib.jit import JitDriver, hint, we_are_jitted, dont_look_inside -from pypy.rlib import jit +from pypy.rlib import jit, objectmodel from pypy.rlib.jit import current_trace_length, unroll_parameters import pypy.interpreter.pyopcode # for side-effects from pypy.interpreter.error import OperationError, operationerrfmt @@ -97,6 +97,15 @@ is_being_profiled=self.is_being_profiled) return jumpto +callback_jit_driver = JitDriver(greens = ['name'], reds = 'auto') + +def callback_merge_point(name): + callback_jit_driver.jit_merge_point(name=name) + + at callback_jit_driver.inline(callback_merge_point) +def callback_hook(name): + pass + def _get_adapted_tick_counter(): # Normally, the tick counter is decremented by 100 for every # Python opcode. Here, to better support JIT compilation of diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py --- a/pypy/module/pypyjit/policy.py +++ b/pypy/module/pypyjit/policy.py @@ -106,7 +106,7 @@ 'posix', '_socket', '_sre', '_lsprof', '_weakref', '__pypy__', 'cStringIO', '_collections', 'struct', 'mmap', 'marshal', '_codecs', 'rctime', 'cppyy', - '_cffi_backend']: + '_cffi_backend', 'pyexpat']: if modname == 'pypyjit' and 'interp_resop' in rest: return False return True diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py --- a/pypy/rlib/objectmodel.py +++ b/pypy/rlib/objectmodel.py @@ -595,6 +595,16 @@ llhelper(rffi.AroundFnPtr, before) llhelper(rffi.AroundFnPtr, after) +def register_around_callback_hook(hook): + """ Register a hook that's called before a callback from C calls RPython. + Primary usage is for JIT to have 'started from' hook. + """ + from pypy.rpython.lltypesystem import rffi + from pypy.rpython.annlowlevel import llhelper + + rffi.aroundstate.callback_hook = hook + llhelper(rffi.CallbackHookPtr, hook) + def is_in_callback(): from pypy.rpython.lltypesystem import rffi return rffi.stackcounter.stacks_counter > 1 diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py --- a/pypy/rlib/rgc.py +++ b/pypy/rlib/rgc.py @@ -139,6 +139,52 @@ hop.exception_cannot_occur() return hop.genop(opname, vlist, resulttype = hop.r_result.lowleveltype) +def copy_struct_item(source, dest, si, di): + TP = lltype.typeOf(source).TO.OF + i = 0 + while i < len(TP._names): + setattr(dest[di], TP._names[i], getattr(source[si], TP._names[i])) + i += 1 + +class CopyStructEntry(ExtRegistryEntry): + _about_ = copy_struct_item + + def compute_result_annotation(self, s_source, s_dest, si, di): + pass + + def specialize_call(self, hop): + v_source, v_dest, v_si, v_di = hop.inputargs(hop.args_r[0], + hop.args_r[1], + lltype.Signed, + lltype.Signed) + hop.exception_cannot_occur() + TP = v_source.concretetype.TO.OF + for name, TP in TP._flds.iteritems(): + c_name = hop.inputconst(lltype.Void, name) + v_fld = hop.genop('getinteriorfield', [v_source, v_si, c_name], + resulttype=TP) + hop.genop('setinteriorfield', [v_dest, v_di, c_name, v_fld]) + + + at specialize.ll() +def copy_item(source, dest, si, di): + TP = lltype.typeOf(source) + if isinstance(TP.TO.OF, lltype.Struct): + copy_struct_item(source, dest, si, di) + else: + dest[di] = source[si] + + at specialize.memo() +def _contains_gcptr(TP): + if not isinstance(TP, lltype.Struct): + if isinstance(TP, lltype.Ptr) and TP.TO._gckind == 'gc': + return True + return False + for TP in TP._flds.itervalues(): + if _contains_gcptr(TP): + return True + return False + @jit.oopspec('list.ll_arraycopy(source, dest, source_start, dest_start, length)') @enforceargs(None, None, int, int, int) @specialize.ll() @@ -150,7 +196,7 @@ # and also, maybe, speed up very small cases if length <= 1: if length == 1: - dest[dest_start] = source[source_start] + copy_item(source, dest, source_start, dest_start) return # supports non-overlapping copies only @@ -161,7 +207,7 @@ TP = lltype.typeOf(source).TO assert TP == lltype.typeOf(dest).TO - if isinstance(TP.OF, lltype.Ptr) and TP.OF.TO._gckind == 'gc': + if _contains_gcptr(TP.OF): # perform a write barrier that copies necessary flags from # source to dest if not llop.gc_writebarrier_before_copy(lltype.Bool, source, dest, @@ -170,7 +216,7 @@ # if the write barrier is not supported, copy by hand i = 0 while i < length: - dest[i + dest_start] = source[i + source_start] + copy_item(source, dest, i + source_start, i + dest_start) i += 1 return source_addr = llmemory.cast_ptr_to_adr(source) diff --git a/pypy/rlib/test/test_rgc.py b/pypy/rlib/test/test_rgc.py --- a/pypy/rlib/test/test_rgc.py +++ b/pypy/rlib/test/test_rgc.py @@ -134,6 +134,46 @@ assert check.called +def test_ll_arraycopy_array_of_structs(): + TP = lltype.GcArray(lltype.Struct('x', ('x', lltype.Signed), + ('y', lltype.Signed))) + def f(): + a1 = lltype.malloc(TP, 3) + a2 = lltype.malloc(TP, 3) + for i in range(3): + a1[i].x = 2 * i + a1[i].y = 2 * i + 1 + rgc.ll_arraycopy(a1, a2, 0, 0, 3) + for i in range(3): + assert a2[i].x == 2 * i + assert a2[i].y == 2 * i + 1 + + + interpret(f, []) + a1 = lltype.malloc(TP, 3) + a2 = lltype.malloc(TP, 3) + a1[1].x = 3 + a1[1].y = 15 + rgc.copy_struct_item(a1, a2, 1, 2) + assert a2[2].x == 3 + assert a2[2].y == 15 + +def test__contains_gcptr(): + assert not rgc._contains_gcptr(lltype.Signed) + assert not rgc._contains_gcptr( + lltype.Struct('x', ('x', lltype.Signed))) + assert rgc._contains_gcptr( + lltype.Struct('x', ('x', lltype.Signed), + ('y', lltype.Ptr(lltype.GcArray(lltype.Signed))))) + assert rgc._contains_gcptr( + lltype.Struct('x', ('x', lltype.Signed), + ('y', llmemory.GCREF))) + assert rgc._contains_gcptr(lltype.Ptr(lltype.GcStruct('x'))) + assert not rgc._contains_gcptr(lltype.Ptr(lltype.Struct('x'))) + GCPTR = lltype.Ptr(lltype.GcStruct('x')) + assert rgc._contains_gcptr( + lltype.Struct('FOO', ('s', lltype.Struct('BAR', ('y', GCPTR))))) + def test_ll_arraycopy_small(): TYPE = lltype.GcArray(lltype.Signed) for length in range(5): diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py --- a/pypy/rpython/lltypesystem/rffi.py +++ b/pypy/rpython/lltypesystem/rffi.py @@ -1,6 +1,6 @@ import py from pypy.annotation import model as annmodel -from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem import lltype, rstr from pypy.rpython.lltypesystem import ll2ctypes from pypy.rpython.lltypesystem.llmemory import cast_adr_to_ptr, cast_ptr_to_adr from pypy.rpython.lltypesystem.llmemory import itemoffsetof, raw_memcopy @@ -13,7 +13,7 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.rpython.tool.rfficache import platform, sizeof_c_type from pypy.translator.tool.cbuild import ExternalCompilationInfo -from pypy.rpython.annlowlevel import llhelper +from pypy.rpython.annlowlevel import llhelper, llstr from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.rstring import StringBuilder, UnicodeBuilder, assert_str0 from pypy.rlib import jit @@ -279,9 +279,18 @@ callable_name = getattr(callable, '__name__', '?') if callbackholder is not None: callbackholder.callbacks[callable] = True + callable_name_descr = str(callable).replace('"', '\\"') args = ', '.join(['a%d' % i for i in range(len(TP.TO.ARGS))]) source = py.code.Source(r""" - def wrapper(%s): # no *args - no GIL for mallocing the tuple + def inner_wrapper(%(args)s): + if aroundstate is not None: + callback_hook = aroundstate.callback_hook + if callback_hook: + callback_hook(llstr("%(callable_name_descr)s")) + return callable(%(args)s) + inner_wrapper._never_inline_ = True + + def wrapper(%(args)s): # no *args - no GIL for mallocing the tuple llop.gc_stack_bottom(lltype.Void) # marker for trackgcroot.py if aroundstate is not None: after = aroundstate.after @@ -290,7 +299,7 @@ # from now on we hold the GIL stackcounter.stacks_counter += 1 try: - result = callable(%s) + result = inner_wrapper(%(args)s) except Exception, e: os.write(2, "Warning: uncaught exception in callback: %%s %%s\n" %% @@ -308,10 +317,11 @@ # by llexternal, it is essential that no exception checking occurs # after the call to before(). return result - """ % (args, args)) + """ % locals()) miniglobals = locals().copy() miniglobals['Exception'] = Exception miniglobals['os'] = os + miniglobals['llstr'] = llstr miniglobals['we_are_translated'] = we_are_translated miniglobals['stackcounter'] = stackcounter exec source.compile() in miniglobals @@ -319,10 +329,14 @@ _make_wrapper_for._annspecialcase_ = 'specialize:memo' AroundFnPtr = lltype.Ptr(lltype.FuncType([], lltype.Void)) +CallbackHookPtr = lltype.Ptr(lltype.FuncType([lltype.Ptr(rstr.STR)], lltype.Void)) + class AroundState: + callback_hook = None + def _cleanup_(self): - self.before = None # or a regular RPython function - self.after = None # or a regular RPython function + self.before = None # or a regular RPython function + self.after = None # or a regular RPython function aroundstate = AroundState() aroundstate._cleanup_() diff --git a/pypy/rpython/lltypesystem/test/test_lltype.py b/pypy/rpython/lltypesystem/test/test_lltype.py --- a/pypy/rpython/lltypesystem/test/test_lltype.py +++ b/pypy/rpython/lltypesystem/test/test_lltype.py @@ -808,7 +808,6 @@ assert F.RESULT == Signed assert F.ARGS == (Signed,) - class TestTrackAllocation: def test_automatic_tracking(self): # calls to start_tracking_allocations/stop_tracking_allocations diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py --- a/pypy/rpython/memory/gc/minimark.py +++ b/pypy/rpython/memory/gc/minimark.py @@ -2000,6 +2000,17 @@ (obj + offset).address[0] = llmemory.NULL continue # no need to remember this weakref any longer # + elif self.header(pointing_to).tid & GCFLAG_NO_HEAP_PTRS: + # see test_weakref_to_prebuilt: it's not useful to put + # weakrefs into 'old_objects_with_weakrefs' if they point + # to a prebuilt object (they are immortal). If moreover + # the 'pointing_to' prebuilt object still has the + # GCFLAG_NO_HEAP_PTRS flag, then it's even wrong, because + # 'pointing_to' will not get the GCFLAG_VISITED during + # the next major collection. Solve this by not registering + # the weakref into 'old_objects_with_weakrefs'. + continue + # self.old_objects_with_weakrefs.append(obj) def invalidate_old_weakrefs(self): @@ -2013,6 +2024,9 @@ continue # weakref itself dies offset = self.weakpointer_offset(self.get_type_id(obj)) pointing_to = (obj + offset).address[0] + ll_assert((self.header(pointing_to).tid & GCFLAG_NO_HEAP_PTRS) + == 0, "registered old weakref should not " + "point to a NO_HEAP_PTRS obj") if self.header(pointing_to).tid & GCFLAG_VISITED: new_with_weakref.append(obj) else: diff --git a/pypy/rpython/memory/test/test_gc.py b/pypy/rpython/memory/test/test_gc.py --- a/pypy/rpython/memory/test/test_gc.py +++ b/pypy/rpython/memory/test/test_gc.py @@ -557,6 +557,19 @@ res = self.interpret(f, [20]) # for GenerationGC, enough for a minor collection assert res == True + def test_weakref_to_prebuilt(self): + import weakref + class A: + pass + a = A() + def f(x): + ref = weakref.ref(a) + assert ref() is a + llop.gc__collect(lltype.Void) + return ref() is a + res = self.interpret(f, [20]) # for GenerationGC, enough for a minor collection + assert res == True + def test_many_weakrefs(self): # test for the case where allocating the weakref itself triggers # a collection diff --git a/pypy/translator/goal/app_main.py b/pypy/translator/goal/app_main.py --- a/pypy/translator/goal/app_main.py +++ b/pypy/translator/goal/app_main.py @@ -584,7 +584,12 @@ python_startup, 'exec') exec co_python_startup in mainmodule.__dict__ + mainmodule.__file__ = python_startup run_toplevel(run_it) + try: + del mainmodule.__file__ + except (AttributeError, TypeError): + pass # Then we need a prompt. inspect = True else: diff --git a/pypy/translator/goal/targetpypystandalone.py b/pypy/translator/goal/targetpypystandalone.py --- a/pypy/translator/goal/targetpypystandalone.py +++ b/pypy/translator/goal/targetpypystandalone.py @@ -28,6 +28,11 @@ w_call_startup_gateway = space.wrap(gateway.interp2app(call_startup)) withjit = space.config.objspace.usemodules.pypyjit + if withjit: + from pypy.module.pypyjit.interp_jit import callback_hook + from pypy.rlib import objectmodel + objectmodel.register_around_callback_hook(callback_hook) + def entry_point(argv): if withjit: from pypy.jit.backend.hlinfo import highleveljitinfo diff --git a/pypy/translator/goal/test2/test_app_main.py b/pypy/translator/goal/test2/test_app_main.py --- a/pypy/translator/goal/test2/test_app_main.py +++ b/pypy/translator/goal/test2/test_app_main.py @@ -376,6 +376,30 @@ child.expect('Traceback') child.expect('NameError') + def test_pythonstartup_file1(self, monkeypatch): + monkeypatch.setenv('PYTHONPATH', None) + monkeypatch.setenv('PYTHONSTARTUP', demo_script) + child = self.spawn([]) + child.expect('File: [^\n]+\.py') + child.expect('goodbye') + child.expect('>>> ') + child.sendline('[myvalue]') + child.expect(re.escape('[42]')) + child.expect('>>> ') + child.sendline('__file__') + child.expect('Traceback') + child.expect('NameError') + + def test_pythonstartup_file2(self, monkeypatch): + monkeypatch.setenv('PYTHONPATH', None) + monkeypatch.setenv('PYTHONSTARTUP', crashing_demo_script) + child = self.spawn([]) + child.expect('Traceback') + child.expect('>>> ') + child.sendline('__file__') + child.expect('Traceback') + child.expect('NameError') + def test_ignore_python_startup(self): old = os.environ.get('PYTHONSTARTUP', '') try: diff --git a/pypy/translator/platform/linux.py b/pypy/translator/platform/linux.py --- a/pypy/translator/platform/linux.py +++ b/pypy/translator/platform/linux.py @@ -36,7 +36,13 @@ # places where we need to look for libffi.a # XXX obscuuure! only look for libffi.a if run with translate.py if 'translate' in sys.modules: - return self.library_dirs_for_libffi() + ['/usr/lib'] + if sys.maxint > 2**32: + host = 'x86_64' + else: + host = 'x86' + return self.library_dirs_for_libffi() + [ + '/usr/lib', + '/usr/lib/%s-linux-gnu/' % host] else: return [] diff --git a/pypy/translator/platform/test/test_distutils.py b/pypy/translator/platform/test/test_distutils.py --- a/pypy/translator/platform/test/test_distutils.py +++ b/pypy/translator/platform/test/test_distutils.py @@ -8,3 +8,6 @@ def test_nice_errors(self): py.test.skip("Unsupported") + + def test_900_files(self): + py.test.skip('Makefiles not suppoerted') diff --git a/pypy/translator/platform/test/test_platform.py b/pypy/translator/platform/test/test_platform.py --- a/pypy/translator/platform/test/test_platform.py +++ b/pypy/translator/platform/test/test_platform.py @@ -59,6 +59,34 @@ res = self.platform.execute(executable) self.check_res(res) + def test_900_files(self): + txt = '#include \n' + for i in range(900): + txt += 'int func%03d();\n' % i + txt += 'int main() {\n int j=0;' + for i in range(900): + txt += ' j += func%03d();\n' % i + txt += ' printf("%d\\n", j);\n' + txt += ' return 0;};\n' + cfile = udir.join('test_900_files.c') + cfile.write(txt) + cfiles = [cfile] + for i in range(900): + cfile2 = udir.join('implement%03d.c' %i) + cfile2.write(''' + int func%03d() + { + return %d; + } + ''' % (i, i)) + cfiles.append(cfile2) + mk = self.platform.gen_makefile(cfiles, ExternalCompilationInfo(), path=udir) + mk.write() + self.platform.execute_makefile(mk) + res = self.platform.execute(udir.join('test_900_files')) + self.check_res(res, '%d\n' %sum(range(900))) + + def test_nice_errors(self): cfile = udir.join('test_nice_errors.c') cfile.write('') diff --git a/pypy/translator/platform/windows.py b/pypy/translator/platform/windows.py --- a/pypy/translator/platform/windows.py +++ b/pypy/translator/platform/windows.py @@ -326,17 +326,33 @@ for rule in rules: m.rule(*rule) - + + objects = ' $(OBJECTS)' + create_obj_response_file = [] + if len(' '.join(rel_ofiles)) > 4000: + # cmd.exe has a limit of ~4000 characters before a command line is too long. + # Use a response file instead, at the cost of making the Makefile very ugly. + for i in range(len(rel_ofiles) - 1): + create_obj_response_file.append('echo %s >> obj_names.rsp' % \ + rel_ofiles[i]) + # use cmd /c for the last one so that the file is flushed + create_obj_response_file.append('cmd /c echo %s >> obj_names.rsp' % \ + rel_ofiles[-1]) + objects = ' @obj_names.rsp' if self.version < 80: m.rule('$(TARGET)', '$(OBJECTS)', - '$(CC_LINK) /nologo $(LDFLAGS) $(LDFLAGSEXTRA) $(OBJECTS) /out:$@ $(LIBDIRS) $(LIBS)') + create_obj_response_file + [\ + '$(CC_LINK) /nologo $(LDFLAGS) $(LDFLAGSEXTRA)' + objects + ' /out:$@ $(LIBDIRS) $(LIBS)', + ]) else: m.rule('$(TARGET)', '$(OBJECTS)', - ['$(CC_LINK) /nologo $(LDFLAGS) $(LDFLAGSEXTRA) $(OBJECTS) $(LINKFILES) /out:$@ $(LIBDIRS) $(LIBS) /MANIFEST /MANIFESTFILE:$*.manifest', + create_obj_response_file + [\ + '$(CC_LINK) /nologo $(LDFLAGS) $(LDFLAGSEXTRA)' + objects + ' $(LINKFILES) /out:$@ $(LIBDIRS) $(LIBS) /MANIFEST /MANIFESTFILE:$*.manifest', 'mt.exe -nologo -manifest $*.manifest -outputresource:$@;1', ]) m.rule('debugmode_$(TARGET)', '$(OBJECTS)', - ['$(CC_LINK) /nologo /DEBUG $(LDFLAGS) $(LDFLAGSEXTRA) $(OBJECTS) $(LINKFILES) /out:$@ $(LIBDIRS) $(LIBS)', + create_obj_response_file + [\ + '$(CC_LINK) /nologo /DEBUG $(LDFLAGS) $(LDFLAGSEXTRA)' + objects + ' $(LINKFILES) /out:$@ $(LIBDIRS) $(LIBS)', ]) if shared: From noreply at buildbot.pypy.org Wed Jan 9 16:01:34 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 9 Jan 2013 16:01:34 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: store (probably temporarily) rd_count on descr here. this allows us Message-ID: <20130109150134.3F1BA1C0246@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r59886:927621febe6c Date: 2013-01-08 23:02 +0200 http://bitbucket.org/pypy/pypy/changeset/927621febe6c/ Log: store (probably temporarily) rd_count on descr here. this allows us to not have to fish it from the CPU diff --git a/pypy/jit/metainterp/compile.py b/pypy/jit/metainterp/compile.py --- a/pypy/jit/metainterp/compile.py +++ b/pypy/jit/metainterp/compile.py @@ -484,9 +484,13 @@ _counters = None # they get stored in _counters then. # this class also gets the following attributes stored by resume.py code + + # XXX move all of unused stuff to guard_op, now that we can have + # a separate class, so it does not survive that long rd_snapshot = None rd_frame_info_list = None rd_numb = lltype.nullptr(NUMBERING) + rd_count = 0 rd_consts = None rd_virtuals = None rd_pendingfields = lltype.nullptr(PENDINGFIELDSP.TO) @@ -501,6 +505,7 @@ def store_final_boxes(self, guard_op, boxes): guard_op.setfailargs(boxes) + self.rd_count = len(boxes) self.guard_opnum = guard_op.getopnum() def make_a_counter_per_value(self, guard_value_op): @@ -630,6 +635,7 @@ res.rd_consts = self.rd_consts res.rd_virtuals = self.rd_virtuals res.rd_pendingfields = self.rd_pendingfields + res.rd_count = self.rd_count def _clone_if_mutable(self): res = ResumeGuardDescr() diff --git a/pypy/jit/metainterp/resume.py b/pypy/jit/metainterp/resume.py --- a/pypy/jit/metainterp/resume.py +++ b/pypy/jit/metainterp/resume.py @@ -798,8 +798,7 @@ self._init(metainterp.cpu, storage) self.deadframe = deadframe self.metainterp = metainterp - count = metainterp.cpu.get_latest_value_count(deadframe) - self.liveboxes = [None] * count + self.liveboxes = [None] * storage.rd_count self._prepare(storage) def consume_boxes(self, info, boxes_i, boxes_r, boxes_f): From noreply at buildbot.pypy.org Wed Jan 9 16:01:35 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 9 Jan 2013 16:01:35 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: support renaming Message-ID: <20130109150135.A805B1C0246@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r59887:6dfdca1bddb3 Date: 2013-01-08 23:25 +0200 http://bitbucket.org/pypy/pypy/changeset/6dfdca1bddb3/ Log: support renaming diff --git a/pypy/jit/metainterp/resume.py b/pypy/jit/metainterp/resume.py --- a/pypy/jit/metainterp/resume.py +++ b/pypy/jit/metainterp/resume.py @@ -1279,6 +1279,17 @@ def write_a_float(self, index, float): self.blackholeinterp.setarg_f(index, float) +def resume_renum(list_of_positions, storage): + num = storage.rd_numb + while num: + i = 0 + while i < len(num.nums): + pos, flags = untag(num.nums[i]) + if flags == TAGBOX: + num.nums[i] = tag(list_of_positions[pos], TAGBOX) + i += 1 + num = num.prev + # ____________________________________________________________ def dump_storage(storage, liveboxes): diff --git a/pypy/jit/metainterp/test/test_fficall.py b/pypy/jit/metainterp/test/test_fficall.py --- a/pypy/jit/metainterp/test/test_fficall.py +++ b/pypy/jit/metainterp/test/test_fficall.py @@ -1,4 +1,5 @@ +import py import ctypes, math from pypy.rpython.lltypesystem import lltype, rffi from pypy.jit.metainterp.test.support import LLJitMixin @@ -106,6 +107,7 @@ class TestFfiCall(FfiCallTests, LLJitMixin): def test_jit_fii_vref(self): + py.test.skip("unsupported so far") from pypy.rlib import clibffi from pypy.rlib.jit_libffi import jit_ffi_prep_cif, jit_ffi_call diff --git a/pypy/jit/metainterp/test/test_resume.py b/pypy/jit/metainterp/test/test_resume.py --- a/pypy/jit/metainterp/test/test_resume.py +++ b/pypy/jit/metainterp/test/test_resume.py @@ -17,6 +17,7 @@ rd_consts = [] rd_virtuals = None rd_pendingfields = None + rd_count = 0 class FakeOptimizer(object): @@ -102,9 +103,6 @@ CONST_NULL = ConstPtr(gcrefnull) def __init__(self, values): self.values = values - def get_latest_value_count(self, deadframe): - assert deadframe == "deadframe" - return len(self.values) def get_latest_value_int(self, deadframe, index): assert deadframe == "deadframe" return self.values[index] @@ -182,6 +180,7 @@ tag(0, TAGBOX), tag(1, TAGBOX)]) storage.rd_numb = numb + storage.rd_count = 3 # cpu = MyCPU([42, gcref1, -66]) metainterp = MyMetaInterp(cpu) @@ -500,6 +499,21 @@ assert snapshot.prev is fs[2].parent_resumedata_snapshot assert snapshot.boxes == fs[2]._env +def test_renaming(): + b1, b2, b3 = [BoxInt(), BoxPtr(), BoxInt()] + c1, c2, c3 = [ConstInt(1), ConstInt(2), ConstInt(3)] + fs = [FakeFrame("code0", 0, b1, c1, b2)] + + storage = Storage() + capture_resumedata(fs, None, [], storage) + memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) + modifier = ResumeDataVirtualAdder(storage, memo) + modifier.finish(FakeOptimizer({})) + resume_renum([10, 20], storage) + for num, x in zip(storage.rd_numb.prev.nums, [10, 20]): + pos, tag = untag(num) + if tag == TAGBOX: + assert pos == x class FakeMetaInterpStaticData: cpu = LLtypeMixin.cpu From noreply at buildbot.pypy.org Wed Jan 9 16:01:37 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 9 Jan 2013 16:01:37 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: start moving jit frames onto heap Message-ID: <20130109150137.0053D1C0246@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r59888:39ae410c6f62 Date: 2013-01-09 17:00 +0200 http://bitbucket.org/pypy/pypy/changeset/39ae410c6f62/ Log: start moving jit frames onto heap diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py --- a/pypy/jit/backend/llsupport/gc.py +++ b/pypy/jit/backend/llsupport/gc.py @@ -113,16 +113,14 @@ @specialize.memo() def getframedescrs(self, cpu): descrs = JitFrameDescrs() - descrs.arraydescr = cpu.arraydescrof(jitframe.DEADFRAME) - descrs.as_int = cpu.interiorfielddescrof(jitframe.DEADFRAME, - 'int', 'jf_values') - descrs.as_ref = cpu.interiorfielddescrof(jitframe.DEADFRAME, - 'ref', 'jf_values') - descrs.as_float = cpu.interiorfielddescrof(jitframe.DEADFRAME, - 'float', 'jf_values') - descrs.jf_descr = cpu.fielddescrof(jitframe.DEADFRAME, 'jf_descr') - descrs.jf_guard_exc = cpu.fielddescrof(jitframe.DEADFRAME, + descrs.arraydescr = cpu.arraydescrof(jitframe.JITFRAME) + descrs.jf_descr = cpu.fielddescrof(jitframe.JITFRAME, 'jf_descr') + descrs.jf_guard_exc = cpu.fielddescrof(jitframe.JITFRAME, 'jf_guard_exc') + descrs.jf_frame_info = cpu.fielddescrof(jitframe.JITFRAME, + 'jf_frame_info') + descrs.jfi_frame_depth = cpu.fielddescrof(jitframe.JITFRAMEINFO, + 'jfi_frame_depth') return descrs class JitFrameDescrs: diff --git a/pypy/jit/backend/llsupport/jitframe.py b/pypy/jit/backend/llsupport/jitframe.py --- a/pypy/jit/backend/llsupport/jitframe.py +++ b/pypy/jit/backend/llsupport/jitframe.py @@ -1,15 +1,24 @@ -from pypy.rpython.lltypesystem import lltype, llmemory, rffi +from pypy.rpython.lltypesystem import lltype, llmemory -GCINDEXLIST = lltype.GcArray(rffi.UINT) +# this is an info that only depends on the assembler executed, copied from +# compiled loop token (in fact we could use this as a compiled loop token +# XXX do this + +JITFRAMEINFO = lltype.GcStruct( + 'JITFRAMEINFO', + # the depth of frame + ('jfi_frame_depth', lltype.Signed), + # gcindexlist is a list of indexes of GC ptrs + # in the actual array jf_frame of JITFRAME + ('jfi_gcindexlist', lltype.Ptr(lltype.GcArray(lltype.Signed))), + ) # the JITFRAME that's stored on the heap. See backend//arch.py for # detailed explanation how it is on your architecture JITFRAME = lltype.GcStruct( 'JITFRAME', - # gcindexlist is a list of indexes of GC ptrs - # in the actual array - ('jf_gcindexlist', lltype.Ptr(GCINDEXLIST)), + ('jf_frame_info', lltype.Ptr(JITFRAMEINFO)), # Once the execute_token() returns, the field 'jf_descr' stores the # descr of the last executed operation (either a GUARD, or FINISH). # This field is also set immediately before doing CALL_MAY_FORCE @@ -20,7 +29,7 @@ # For GUARD_(NO)_EXCEPTION and GUARD_NOT_FORCED: the exception we # got. (Note that in case of a regular FINISH generated from # RPython code that finishes the function with an exception, the - # exception is not stored there, but in jf_values[0].ref.) + # exception is not stored there, but is simply kept as a variable there) ('jf_guard_exc', llmemory.GCREF), # the actual frame ('jf_frame', lltype.Array(lltype.Signed)) diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py --- a/pypy/jit/backend/llsupport/llmodel.py +++ b/pypy/jit/backend/llsupport/llmodel.py @@ -77,9 +77,10 @@ return (rffi.cast(lltype.Signed, _exception_emulator) + rffi.sizeof(lltype.Signed)) - self._memoryerror_emulated = rffi.cast(llmemory.GCREF, -123) - self.deadframe_memoryerror = lltype.malloc(jitframe.DEADFRAME, 0) - self.deadframe_memoryerror.jf_guard_exc = self._memoryerror_emulated + # XXX I think we don't need it any more + #self._memoryerror_emulated = rffi.cast(llmemory.GCREF, -123) + #self.deadframe_memoryerror = lltype.malloc(jitframe.DEADFRAME, 0) + #self.deadframe_memoryerror.jf_guard_exc = self._memoryerror_emulated def propagate_exception(): exc = _exception_emulator[1] @@ -96,6 +97,7 @@ else: assert deadframe.jf_descr == faildescr else: + XXX deadframe = lltype.malloc(jitframe.DEADFRAME, 0) deadframe.jf_guard_exc = rffi.cast(llmemory.GCREF, exc) deadframe.jf_descr = faildescr @@ -126,7 +128,7 @@ slowpathaddr = rffi.cast(lltype.Signed, f) return endaddr, lengthaddr, slowpathaddr - self.deadframe_memoryerror = lltype.malloc(jitframe.DEADFRAME, 0) + #self.deadframe_memoryerror = lltype.malloc(jitframe.DEADFRAME, 0) def propagate_exception(): addr = llop.get_exception_addr(llmemory.Address) @@ -138,6 +140,7 @@ faildescr = self.get_fail_descr_from_number( self.propagate_exception_v) faildescr = faildescr.hide(self) + XXX deadframe = lltype.nullptr(jitframe.DEADFRAME) if exc: try: @@ -169,16 +172,19 @@ return llhelper(self.PROPAGATE_EXCEPTION, self._propagate_exception) def grab_exc_value(self, deadframe): + XXX deadframe = lltype.cast_opaque_ptr(jitframe.DEADFRAMEPTR, deadframe) if not we_are_translated() and deadframe == self.deadframe_memoryerror: return "memoryerror!" # for tests return deadframe.jf_guard_exc def set_savedata_ref(self, deadframe, data): + XXX deadframe = lltype.cast_opaque_ptr(jitframe.DEADFRAMEPTR, deadframe) deadframe.jf_savedata = data def get_savedata_ref(self, deadframe): + XXXX deadframe = lltype.cast_opaque_ptr(jitframe.DEADFRAMEPTR, deadframe) return deadframe.jf_savedata @@ -265,23 +271,28 @@ abiname) def get_latest_descr(self, deadframe): + XXX deadframe = lltype.cast_opaque_ptr(jitframe.DEADFRAMEPTR, deadframe) descr = deadframe.jf_descr return history.AbstractDescr.show(self, descr) def get_latest_value_int(self, deadframe, index): + XXX deadframe = lltype.cast_opaque_ptr(jitframe.DEADFRAMEPTR, deadframe) return deadframe.jf_values[index].int def get_latest_value_ref(self, deadframe, index): + XXX deadframe = lltype.cast_opaque_ptr(jitframe.DEADFRAMEPTR, deadframe) return deadframe.jf_values[index].ref def get_latest_value_float(self, deadframe, index): + XXX deadframe = lltype.cast_opaque_ptr(jitframe.DEADFRAMEPTR, deadframe) return deadframe.jf_values[index].float def get_latest_value_count(self, deadframe): + XXX deadframe = lltype.cast_opaque_ptr(jitframe.DEADFRAMEPTR, deadframe) return len(deadframe.jf_values) diff --git a/pypy/jit/backend/llsupport/regalloc.py b/pypy/jit/backend/llsupport/regalloc.py --- a/pypy/jit/backend/llsupport/regalloc.py +++ b/pypy/jit/backend/llsupport/regalloc.py @@ -21,8 +21,6 @@ self.used = [] # list of bools self.hint_frame_locations = {} - frame_depth = property(lambda:xxx, lambda:xxx) # XXX kill me - def get_frame_depth(self): return len(self.used) diff --git a/pypy/jit/backend/llsupport/rewrite.py b/pypy/jit/backend/llsupport/rewrite.py --- a/pypy/jit/backend/llsupport/rewrite.py +++ b/pypy/jit/backend/llsupport/rewrite.py @@ -67,10 +67,6 @@ continue # ---------- self.newops.append(op) - # ---------- FINISH ---------- - if len(self.newops) != 0 and self.newops[-1].getopnum() == rop.FINISH: - self.handle_finish(self.newops.pop()) - # ---------- return self.newops # ---------- @@ -343,40 +339,3 @@ # assume that "self.gc_ll_descr.minimal_size_in_nursery" is 2 WORDs size = max(size, 2 * WORD) return (size + WORD-1) & ~(WORD-1) # round up - - # ---------- - - def handle_finish(self, finish_op): - v_deadframe = BoxPtr() - args_boxes = finish_op.getarglist() # may contain Consts too - # - descrs = self.gc_ll_descr.getframedescrs(self.cpu) - # - op = ResOperation(rop.NEW_ARRAY, - [ConstInt(len(args_boxes))], v_deadframe, - descr=descrs.arraydescr) - self.handle_malloc_operation(op) - # - for i in range(len(args_boxes)): - # Generate setinteriorfields to write the args inside the - # deadframe object. Ignore write barriers because it's a - # recent object. - box = args_boxes[i] - if box.type == history.INT: descr = descrs.as_int - elif box.type == history.REF: descr = descrs.as_ref - elif box.type == history.FLOAT: descr = descrs.as_float - else: assert 0, "bad box type?" - op = ResOperation(rop.SETINTERIORFIELD_GC, - [v_deadframe, ConstInt(i), box], None, - descr=descr) - self.newops.append(op) - # - # Generate a setfield to write the finish_op's descr. - gcref_descr = finish_op.getdescr().hide(self.cpu) - op = ResOperation(rop.SETFIELD_GC, - [v_deadframe, ConstPtr(gcref_descr)], None, - descr=descrs.jf_descr) - self.newops.append(op) - # - op = ResOperation(rop.FINISH, [v_deadframe], None) - self.newops.append(op) diff --git a/pypy/jit/backend/x86/arch.py b/pypy/jit/backend/x86/arch.py --- a/pypy/jit/backend/x86/arch.py +++ b/pypy/jit/backend/x86/arch.py @@ -40,15 +40,17 @@ # one word for the force_index, and some extra space used only # during a malloc that needs to go via its slow path. +# XXX FORCE_INDEX might need to go to GC frame + if WORD == 4: # ebp + ebx + esi + edi + 4 extra words + force_index = 9 words - FRAME_FIXED_SIZE = 9 + FRAME_FIXED_SIZE = 12 # 9 aligned to 16 bytes = 4 * WORD FORCE_INDEX_OFS = 0 SAVED_REGISTERS = 1 # range(1, 5) MY_COPY_OF_REGS = 5 # range(5, 9) else: # rbp + rbx + r12 + r13 + r14 + r15 + 11 extra words + force_index = 18 - FRAME_FIXED_SIZE = 18 + FRAME_FIXED_SIZE = 18 # 18 aligned to 16 bytes = 2 * WORD FORCE_INDEX_OFS = 0 SAVED_REGISTERS = 1 # range(1, 7) MY_COPY_OF_REGS = 7 # range(7, 18) diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -4,6 +4,7 @@ from pypy.jit.metainterp.history import Const, Box, BoxInt, ConstInt from pypy.jit.metainterp.history import AbstractFailDescr, INT, REF, FLOAT from pypy.jit.metainterp.history import JitCellToken +from pypy.jit.backend.llsupport.descr import unpack_arraydescr from pypy.rpython.lltypesystem import lltype, rffi, rstr, llmemory from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.annlowlevel import llhelper @@ -498,6 +499,7 @@ # for the duration of compiling one loop or a one bridge. clt = CompiledLoopToken(self.cpu, looptoken.number) + clt.frame_info = lltype.malloc(jitframe.JITFRAMEINFO) clt.allgcrefs = [] looptoken.compiled_loop_token = clt if not we_are_translated(): @@ -512,15 +514,14 @@ regalloc = RegAlloc(self, self.cpu.translate_support_code) # self._call_header_with_stack_check() - stackadjustpos = self._patchable_stackadjust() clt._debug_nbargs = len(inputargs) operations = regalloc.prepare_loop(inputargs, operations, looptoken, clt.allgcrefs) looppos = self.mc.get_relative_pos() looptoken._x86_loop_code = looppos - clt.frame_depth = -1 # temporarily frame_depth = self._assemble(regalloc, operations) clt.frame_depth = frame_depth + clt.frame_info.jfi_frame_depth = frame_depth # size_excluding_failure_stuff = self.mc.get_relative_pos() self.write_pending_failure_recoveries() @@ -534,7 +535,6 @@ rawstart + size_excluding_failure_stuff, rawstart)) debug_stop("jit-backend-addr") - self._patch_stackadjust(rawstart + stackadjustpos, frame_depth) self.patch_pending_failure_recoveries(rawstart) # ops_offset = self.mc.ops_offset @@ -579,7 +579,6 @@ operations, self.current_clt.allgcrefs) - stackadjustpos = self._patchable_stackadjust() frame_depth = self._assemble(regalloc, operations) codeendpos = self.mc.get_relative_pos() self.write_pending_failure_recoveries() @@ -590,7 +589,6 @@ debug_print("bridge out of Guard %d has address %x to %x" % (descr_number, rawstart, rawstart + codeendpos)) debug_stop("jit-backend-addr") - self._patch_stackadjust(rawstart + stackadjustpos, frame_depth) self.patch_pending_failure_recoveries(rawstart) if not we_are_translated(): # for the benefit of tests @@ -599,7 +597,9 @@ self.patch_jump_for_descr(faildescr, rawstart) ops_offset = self.mc.ops_offset self.fixup_target_tokens(rawstart) - self.current_clt.frame_depth = max(self.current_clt.frame_depth, frame_depth) + frame_depth = max(self.current_clt.frame_depth, frame_depth) + self.current_clt.frame_depth = frame_depth + self.current_clt.frame_info.jfi_frame_depth = frame_depth self.teardown() # oprofile support if self.cpu.profile_agent is not None: @@ -776,31 +776,19 @@ frame_depth = max(frame_depth, target_frame_depth) return frame_depth - def _patchable_stackadjust(self): - # stack adjustment LEA - self.mc.LEA32_rb(esp.value, 0) - return self.mc.get_relative_pos() - 4 - - def _patch_stackadjust(self, adr_lea, allocated_depth): - # patch stack adjustment LEA - mc = codebuf.MachineCodeBlockWrapper() - # Compute the correct offset for the instruction LEA ESP, [EBP-4*words] - mc.writeimm32(self._get_offset_of_ebp_from_esp(allocated_depth)) - mc.copy_to_raw_memory(adr_lea) - - def _get_offset_of_ebp_from_esp(self, allocated_depth): - # Given that [EBP] is where we saved EBP, i.e. in the last word - # of our fixed frame, then the 'words' value is: - words = (FRAME_FIXED_SIZE - 1) + allocated_depth - # align, e.g. for Mac OS X - aligned_words = align_stack_words(words+2)-2 # 2 = EIP+EBP - return -WORD * aligned_words - def _call_header(self): # NB. the shape of the frame is hard-coded in get_basic_shape() too. # Also, make sure this is consistent with FRAME_FIXED_SIZE. self.mc.PUSH_r(ebp.value) - self.mc.MOV_rr(ebp.value, esp.value) + # XXX should be LEA? + self.mc.ADD_ri(esp.value, FRAME_FIXED_SIZE * WORD) + if IS_X86_64: + descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) + _, ofs, _ = unpack_arraydescr(descrs.arraydescr) + self.mc.LEA_rm(ebp.value, (edi.value, ofs)) + else: + xxx + for loc in self.cpu.CALLEE_SAVE_REGISTERS: self.mc.PUSH_r(loc.value) @@ -827,7 +815,7 @@ self._call_header() def _call_footer(self): - self.mc.LEA_rb(esp.value, -len(self.cpu.CALLEE_SAVE_REGISTERS) * WORD) + self.mc.SUB_ri(esp.value, FRAME_FIXED_SIZE * WORD) gcrootmap = self.cpu.gc_ll_descr.gcrootmap if gcrootmap and gcrootmap.is_shadow_stack: @@ -1963,6 +1951,7 @@ # no malloc allowed here!! xxx apart from one, hacking a lot #self.fail_ebp = allregisters[16 + ebp.value] num = 0 + XXX deadframe = lltype.nullptr(jitframe.DEADFRAME) # step 1: lots of mess just to count the final value of 'num' bytecode1 = bytecode diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py --- a/pypy/jit/backend/x86/regalloc.py +++ b/pypy/jit/backend/x86/regalloc.py @@ -221,55 +221,64 @@ return self.min_frame_depth def _set_initial_bindings(self, inputargs): - if IS_X86_64: - inputargs = self._set_initial_bindings_regs_64(inputargs) - # ... - # stack layout: arg2 - # arg1 - # arg0 - # return address - # saved ebp <-- ebp points here - # ... - cur_frame_pos = - 1 - FRAME_FIXED_SIZE - assert get_ebp_ofs(cur_frame_pos-1) == 2*WORD - assert get_ebp_ofs(cur_frame_pos-2) == 3*WORD - # for box in inputargs: assert isinstance(box, Box) - # - if IS_X86_32 and box.type == FLOAT: - cur_frame_pos -= 2 - else: - cur_frame_pos -= 1 - loc = self.fm.frame_pos(cur_frame_pos, box.type) - self.fm.set_binding(box, loc) + self.fm.get_new_loc(box) + #loc = self.fm.frame_pos(cur_frame_pos, box.type) + #self.fm.set_binding(box, loc) - def _set_initial_bindings_regs_64(self, inputargs): - # In reverse order for use with pop() - unused_gpr = [r9, r8, ecx, edx, esi, edi] - unused_xmm = [xmm7, xmm6, xmm5, xmm4, xmm3, xmm2, xmm1, xmm0] - # - pass_on_stack = [] - # - for box in inputargs: - assert isinstance(box, Box) - # - if box.type == FLOAT: - if len(unused_xmm) > 0: - ask = unused_xmm.pop() - got = self.xrm.try_allocate_reg(box, selected_reg=ask) - assert ask == got - else: - pass_on_stack.append(box) - else: - if len(unused_gpr) > 0: - ask = unused_gpr.pop() - got = self.rm.try_allocate_reg(box, selected_reg=ask) - assert ask == got - else: - pass_on_stack.append(box) - # - return pass_on_stack + # if IS_X86_64: + # inputargs = self._set_initial_bindings_regs_64(inputargs) + # # ... + # # stack layout: arg2 + # # arg1 + # # arg0 + # # return address + # # saved ebp <-- ebp points here + # # ... + # XXX # adjust the address to count for the fact that we're passing + # # jitframe as a first arg + # cur_frame_pos = - 1 - FRAME_FIXED_SIZE + # assert get_ebp_ofs(cur_frame_pos-1) == 2*WORD + # assert get_ebp_ofs(cur_frame_pos-2) == 3*WORD + # # + # for box in inputargs: + # assert isinstance(box, Box) + # # + # if IS_X86_32 and box.type == FLOAT: + # cur_frame_pos -= 2 + # else: + # cur_frame_pos -= 1 + # loc = self.fm.frame_pos(cur_frame_pos, box.type) + # self.fm.set_binding(box, loc) + + # def _set_initial_bindings_regs_64(self, inputargs): + # # In reverse order for use with pop() + # unused_gpr = [r9, r8, ecx, edx, esi] # jitframe comes in edi. don't use + # # it for parameter parsing + # unused_xmm = [xmm7, xmm6, xmm5, xmm4, xmm3, xmm2, xmm1, xmm0] + # # + # pass_on_stack = [] + # # + # for box in inputargs: + # assert isinstance(box, Box) + # # + # if box.type == FLOAT: + # if len(unused_xmm) > 0: + # ask = unused_xmm.pop() + # got = self.xrm.try_allocate_reg(box, selected_reg=ask) + # assert ask == got + # else: + # pass_on_stack.append(box) + # else: + # if len(unused_gpr) > 0: + # ask = unused_gpr.pop() + # got = self.rm.try_allocate_reg(box, selected_reg=ask) + # assert ask == got + # else: + # pass_on_stack.append(box) + # # + # return pass_on_stack def possibly_free_var(self, var): if var.type == FLOAT: @@ -486,8 +495,14 @@ consider_guard_isnull = _consider_guard def consider_finish(self, op): + # the frame is in ebp, but we have to point where in the frame is + # the potential argument to FINISH + descr = op.getdescr() + self.force_spill_var(op.getarg(0)) loc = self.loc(op.getarg(0)) - self.Perform(op, [loc], None) + descr._x86_result_offset = loc.value + fail_no = self.assembler.cpu.get_fail_descr_number(descr) + self.Perform(op, [imm(fail_no)], None) self.possibly_free_var(op.getarg(0)) def consider_guard_no_exception(self, op): @@ -1468,7 +1483,7 @@ # Argument is a frame position (0, 1, 2...). # Returns (ebp-20), (ebp-24), (ebp-28)... # i.e. the n'th word beyond the fixed frame size. - return -WORD * (FRAME_FIXED_SIZE + position) + return WORD * position def _valid_addressing_size(size): return size == 1 or size == 2 or size == 4 or size == 8 diff --git a/pypy/jit/backend/x86/regloc.py b/pypy/jit/backend/x86/regloc.py --- a/pypy/jit/backend/x86/regloc.py +++ b/pypy/jit/backend/x86/regloc.py @@ -49,12 +49,16 @@ # _getregkey() returns self.value; the value returned must not # conflict with RegLoc._getregkey(). It doesn't a bit by chance, # so let it fail the following assert if it no longer does. - assert not (0 <= ebp_offset < 8 + 8 * IS_X86_64) + assert ebp_offset >= 0 + #assert not (0 <= ebp_offset < 8 + 8 * IS_X86_64) self.position = position self.value = ebp_offset # One of INT, REF, FLOAT self.type = type + def _getregkey(self): + return -(self.value + 1) + def get_width(self): if self.type == FLOAT: return 8 diff --git a/pypy/jit/backend/x86/runner.py b/pypy/jit/backend/x86/runner.py --- a/pypy/jit/backend/x86/runner.py +++ b/pypy/jit/backend/x86/runner.py @@ -10,6 +10,7 @@ from pypy.jit.backend.x86.arch import FORCE_INDEX_OFS, IS_X86_32 from pypy.jit.backend.x86.profagent import ProfileAgent from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU +from pypy.jit.backend.llsupport import jitframe from pypy.jit.backend.x86 import regloc import sys @@ -100,7 +101,8 @@ setitem(index, null) def make_execute_token(self, *ARGS): - FUNCPTR = lltype.Ptr(lltype.FuncType(ARGS, llmemory.GCREF)) + FUNCPTR = lltype.Ptr(lltype.FuncType([jitframe.JITFRAME], + lltype.Signed)) # def execute_token(executable_token, *args): clt = executable_token.compiled_loop_token @@ -109,17 +111,22 @@ addr = executable_token._x86_function_addr func = rffi.cast(FUNCPTR, addr) #llop.debug_print(lltype.Void, ">>>> Entering", addr) + frame = lltype.malloc(jitframe.JITFRAME, clt.frame_depth) + frame.jf_frame_info = clt.frame_info prev_interpreter = None # help flow space if not self.translate_support_code: prev_interpreter = LLInterpreter.current_interpreter LLInterpreter.current_interpreter = self.debug_ll_interpreter try: - deadframe = func(*args) + import pdb + pdb.set_trace() + descr_no = func(frame) + xxx finally: if not self.translate_support_code: LLInterpreter.current_interpreter = prev_interpreter #llop.debug_print(lltype.Void, "<<<< Back") - return deadframe + return frame return execute_token def cast_ptr_to_int(x): diff --git a/pypy/jit/backend/x86/test/test_assembler.py b/pypy/jit/backend/x86/test/test_assembler.py --- a/pypy/jit/backend/x86/test/test_assembler.py +++ b/pypy/jit/backend/x86/test/test_assembler.py @@ -236,6 +236,8 @@ registers[ACTUAL_CPU.NUM_REGS] = rffi.cast(rffi.LONG, descr_bytes) registers[ebp.value] = rffi.cast(rffi.LONG, stack) + WORD*stacklen + XXX # rewrite + # run! assembler = Assembler386(FakeCPU()) deadframe = assembler.failure_recovery_func(registers) From noreply at buildbot.pypy.org Wed Jan 9 16:28:48 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 9 Jan 2013 16:28:48 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: enough wrangling to pass the test Message-ID: <20130109152848.AA0AF1C0041@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r59889:a94c9ca09e9e Date: 2013-01-09 17:28 +0200 http://bitbucket.org/pypy/pypy/changeset/a94c9ca09e9e/ Log: enough wrangling to pass the test diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py --- a/pypy/jit/backend/llsupport/llmodel.py +++ b/pypy/jit/backend/llsupport/llmodel.py @@ -271,8 +271,7 @@ abiname) def get_latest_descr(self, deadframe): - XXX - deadframe = lltype.cast_opaque_ptr(jitframe.DEADFRAMEPTR, deadframe) + deadframe = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, deadframe) descr = deadframe.jf_descr return history.AbstractDescr.show(self, descr) diff --git a/pypy/jit/backend/x86/runner.py b/pypy/jit/backend/x86/runner.py --- a/pypy/jit/backend/x86/runner.py +++ b/pypy/jit/backend/x86/runner.py @@ -2,6 +2,7 @@ from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.llinterp import LLInterpreter +from pypy.rpython.annlowlevel import cast_instance_to_gcref from pypy.rlib.objectmodel import we_are_translated, specialize from pypy.rlib.jit_hooks import LOOP_RUN_CONTAINER from pypy.jit.codewriter import longlong @@ -101,7 +102,7 @@ setitem(index, null) def make_execute_token(self, *ARGS): - FUNCPTR = lltype.Ptr(lltype.FuncType([jitframe.JITFRAME], + FUNCPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF], lltype.Signed)) # def execute_token(executable_token, *args): @@ -113,20 +114,21 @@ #llop.debug_print(lltype.Void, ">>>> Entering", addr) frame = lltype.malloc(jitframe.JITFRAME, clt.frame_depth) frame.jf_frame_info = clt.frame_info + ll_frame = lltype.cast_opaque_ptr(llmemory.GCREF, frame) prev_interpreter = None # help flow space if not self.translate_support_code: prev_interpreter = LLInterpreter.current_interpreter LLInterpreter.current_interpreter = self.debug_ll_interpreter try: - import pdb - pdb.set_trace() - descr_no = func(frame) - xxx + # XXX parameters + descr_no = func(ll_frame) finally: if not self.translate_support_code: LLInterpreter.current_interpreter = prev_interpreter #llop.debug_print(lltype.Void, "<<<< Back") - return frame + descr = self.get_fail_descr_from_number(descr_no) + frame.jf_descr = cast_instance_to_gcref(descr) + return ll_frame return execute_token def cast_ptr_to_int(x): diff --git a/pypy/rpython/annlowlevel.py b/pypy/rpython/annlowlevel.py --- a/pypy/rpython/annlowlevel.py +++ b/pypy/rpython/annlowlevel.py @@ -6,7 +6,7 @@ from pypy.annotation import model as annmodel from pypy.annotation.policy import AnnotatorPolicy, Sig from pypy.annotation.specialize import flatten_star_args -from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.ootypesystem import ootype from pypy.rpython import extregistry from pypy.objspace.flow.model import Constant @@ -512,6 +512,11 @@ def cast_instance_to_base_obj(instance): return cast_object_to_ptr(base_obj_ootype(), instance) + at specialize.argtype(0) +def cast_instance_to_gcref(instance): + return lltype.cast_opaque_ptr(llmemory.GCREF, + cast_instance_to_base_ptr(instance)) + def base_ptr_lltype(): from pypy.rpython.lltypesystem.rclass import OBJECTPTR return OBJECTPTR From noreply at buildbot.pypy.org Wed Jan 9 17:30:44 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 9 Jan 2013 17:30:44 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: enough to make one test pass, hooray! Message-ID: <20130109163044.5B70C1C0041@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r59890:8c1f9de1dc64 Date: 2013-01-09 18:30 +0200 http://bitbucket.org/pypy/pypy/changeset/8c1f9de1dc64/ Log: enough to make one test pass, hooray! diff --git a/pypy/jit/backend/llgraph/runner.py b/pypy/jit/backend/llgraph/runner.py --- a/pypy/jit/backend/llgraph/runner.py +++ b/pypy/jit/backend/llgraph/runner.py @@ -232,17 +232,17 @@ except ExecutionFinished, e: return e.deadframe - def get_latest_value_int(self, deadframe, index): + def get_int_value(self, deadframe, index): v = deadframe._values[index] assert lltype.typeOf(v) == lltype.Signed return v - def get_latest_value_ref(self, deadframe, index): + def get_ref_value(self, deadframe, index): v = deadframe._values[index] assert lltype.typeOf(v) == llmemory.GCREF return v - def get_latest_value_float(self, deadframe, index): + def get_float_value(self, deadframe, index): v = deadframe._values[index] assert lltype.typeOf(v) == longlong.FLOATSTORAGE return v @@ -250,9 +250,6 @@ def get_latest_descr(self, deadframe): return deadframe._latest_descr - def get_latest_value_count(self, deadframe): - return len(deadframe._values) - def grab_exc_value(self, deadframe): if deadframe._last_exception is not None: result = deadframe._last_exception.args[1] @@ -864,13 +861,13 @@ failindex = self.cpu.get_fail_descr_number(faildescr) if failindex == self.cpu.done_with_this_frame_int_v: reset_vable(jd, vable) - return self.cpu.get_latest_value_int(pframe, 0) + return self.cpu.get_int_value(pframe, 0) if failindex == self.cpu.done_with_this_frame_ref_v: reset_vable(jd, vable) - return self.cpu.get_latest_value_ref(pframe, 0) + return self.cpu.get_ref_value(pframe, 0) if failindex == self.cpu.done_with_this_frame_float_v: reset_vable(jd, vable) - return self.cpu.get_latest_value_float(pframe, 0) + return self.cpu.get_float_value(pframe, 0) if failindex == self.cpu.done_with_this_frame_void_v: reset_vable(jd, vable) return None diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py --- a/pypy/jit/backend/llsupport/llmodel.py +++ b/pypy/jit/backend/llsupport/llmodel.py @@ -271,29 +271,33 @@ abiname) def get_latest_descr(self, deadframe): - deadframe = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, deadframe) descr = deadframe.jf_descr return history.AbstractDescr.show(self, descr) - def get_latest_value_int(self, deadframe, index): - XXX - deadframe = lltype.cast_opaque_ptr(jitframe.DEADFRAMEPTR, deadframe) - return deadframe.jf_values[index].int + def get_int_value(self, deadframe, index): + return deadframe.jf_frame[index // WORD] - def get_latest_value_ref(self, deadframe, index): + def get_ref_value(self, deadframe, index): XXX deadframe = lltype.cast_opaque_ptr(jitframe.DEADFRAMEPTR, deadframe) return deadframe.jf_values[index].ref - def get_latest_value_float(self, deadframe, index): + def get_float_value(self, deadframe, index): XXX deadframe = lltype.cast_opaque_ptr(jitframe.DEADFRAMEPTR, deadframe) return deadframe.jf_values[index].float - def get_latest_value_count(self, deadframe): - XXX - deadframe = lltype.cast_opaque_ptr(jitframe.DEADFRAMEPTR, deadframe) - return len(deadframe.jf_values) + def set_int_value(self, newframe, index, value): + """ Note that we keep index multiplied by WORD here mostly + for completeness with get_int_value and friends + """ + newframe.jf_frame[index // WORD] = value + + def set_ref_value(self, deadframe, index, value): + xxx + + def set_float_value(self, deadframe, index, value): + xxx # ____________________________________________________________ diff --git a/pypy/jit/backend/model.py b/pypy/jit/backend/model.py --- a/pypy/jit/backend/model.py +++ b/pypy/jit/backend/model.py @@ -132,30 +132,24 @@ """Returns the Descr for the last operation executed by the frame.""" raise NotImplementedError - def get_latest_value_int(self, deadframe, index): + def get_int_value(self, deadframe, index): """Returns the value for the index'th argument to the last executed operation (from 'fail_args' if it was a guard, or from 'args' if it was a FINISH). Returns an int.""" raise NotImplementedError - def get_latest_value_float(self, deadframe, index): + def get_float_value(self, deadframe, index): """Returns the value for the index'th argument to the last executed operation (from 'fail_args' if it was a guard, or from 'args' if it was a FINISH). Returns a FLOATSTORAGE.""" raise NotImplementedError - def get_latest_value_ref(self, deadframe, index): + def get_ref_value(self, deadframe, index): """Returns the value for the index'th argument to the last executed operation (from 'fail_args' if it was a guard, or from 'args' if it was a FINISH). Returns a GCREF.""" raise NotImplementedError - def get_latest_value_count(self, deadframe): - """Return how many values are ready to be returned by - get_latest_value_xxx(). Only after a guard failure; not - necessarily correct after a FINISH.""" - raise NotImplementedError - def grab_exc_value(self, deadframe): """Return the exception set by the latest execute_token(), when it exits due to a failure of a GUARD_EXCEPTION or diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py --- a/pypy/jit/backend/test/runner_test.py +++ b/pypy/jit/backend/test/runner_test.py @@ -66,11 +66,11 @@ else: self.guard_failed = True if result_type == 'int': - return BoxInt(self.cpu.get_latest_value_int(deadframe, 0)) + return BoxInt(self.cpu.get_int_value(deadframe, 0)) elif result_type == 'ref': - return BoxPtr(self.cpu.get_latest_value_ref(deadframe, 0)) + return BoxPtr(self.cpu.get_ref_value(deadframe, 0)) elif result_type == 'float': - return BoxFloat(self.cpu.get_latest_value_float(deadframe, 0)) + return BoxFloat(self.cpu.get_float_value(deadframe, 0)) elif result_type == 'void': return None else: @@ -111,6 +111,10 @@ avoid_instances = False + def find_pos_on_faildescr(self, fail): + # overloaded by backends + return 0 + def test_compile_linear_loop(self): i0 = BoxInt() i1 = BoxInt() @@ -123,7 +127,8 @@ self.cpu.compile_loop(inputargs, operations, looptoken) deadframe = self.cpu.execute_token(looptoken, 2) fail = self.cpu.get_latest_descr(deadframe) - res = self.cpu.get_latest_value_int(deadframe, 0) + res = self.cpu.get_int_value(deadframe, + self.find_pos_on_faildescr(fail)) assert res == 3 assert fail.identifier == 1 @@ -139,7 +144,7 @@ self.cpu.compile_loop(inputargs, operations, looptoken) deadframe = self.cpu.execute_token(looptoken, longlong.getfloatstorage(2.8)) - res = self.cpu.get_latest_value_float(deadframe, 0) + res = self.cpu.get_float_value(deadframe, 0) assert longlong.getrealfloat(res) == 5.1 fail = self.cpu.get_latest_descr(deadframe) assert fail.identifier == 1 @@ -164,7 +169,7 @@ deadframe = self.cpu.execute_token(looptoken, 2) fail = self.cpu.get_latest_descr(deadframe) assert fail.identifier == 2 - res = self.cpu.get_latest_value_int(deadframe, 0) + res = self.cpu.get_int_value(deadframe, 0) assert res == 10 def test_compile_with_holes_in_fail_args(self): @@ -189,7 +194,7 @@ deadframe = self.cpu.execute_token(looptoken, 44) fail = self.cpu.get_latest_descr(deadframe) assert fail.identifier == 2 - res = self.cpu.get_latest_value_int(deadframe, 2) + res = self.cpu.get_int_value(deadframe, 2) assert res == 10 def test_backends_dont_keep_loops_alive(self): @@ -255,7 +260,7 @@ deadframe = self.cpu.execute_token(looptoken, 2) fail = self.cpu.get_latest_descr(deadframe) assert fail.identifier == 2 - res = self.cpu.get_latest_value_int(deadframe, 0) + res = self.cpu.get_int_value(deadframe, 0) assert res == 20 assert self.cpu.total_compiled_loops == 1 @@ -297,7 +302,7 @@ deadframe = self.cpu.execute_token(looptoken, 2) fail = self.cpu.get_latest_descr(deadframe) assert fail.identifier == 2 - res = self.cpu.get_latest_value_int(deadframe, 0) + res = self.cpu.get_int_value(deadframe, 0) assert res == 20 def test_compile_big_bridge_out_of_small_loop(self): @@ -330,37 +335,9 @@ fail = self.cpu.get_latest_descr(deadframe) assert fail.identifier == 3 for i in range(1000): - res = self.cpu.get_latest_value_int(deadframe, i) + res = self.cpu.get_int_value(deadframe, i) assert res == 2 + i - def test_get_latest_value_count(self): - i0 = BoxInt() - i1 = BoxInt() - i2 = BoxInt() - faildescr1 = BasicFailDescr(1) - looptoken = JitCellToken() - targettoken = TargetToken() - operations = [ - ResOperation(rop.LABEL, [i0], None, descr=targettoken), - ResOperation(rop.INT_ADD, [i0, ConstInt(1)], i1), - ResOperation(rop.INT_LE, [i1, ConstInt(9)], i2), - ResOperation(rop.GUARD_TRUE, [i2], None, descr=faildescr1), - ResOperation(rop.JUMP, [i1], None, descr=targettoken), - ] - inputargs = [i0] - operations[3].setfailargs([None, i1, None]) - self.cpu.compile_loop(inputargs, operations, looptoken) - - deadframe = self.cpu.execute_token(looptoken, 2) - fail = self.cpu.get_latest_descr(deadframe) - assert fail is faildescr1 - - count = self.cpu.get_latest_value_count(deadframe) - assert count == 3 - assert self.cpu.get_latest_value_int(deadframe, 1) == 10 - # multiple reads ok - assert self.cpu.get_latest_value_int(deadframe, 1) == 10 - def test_finish(self): i0 = BoxInt() class UntouchableFailDescr(AbstractFailDescr): @@ -378,7 +355,7 @@ deadframe = self.cpu.execute_token(looptoken, 99) fail = self.cpu.get_latest_descr(deadframe) assert fail is faildescr - res = self.cpu.get_latest_value_int(deadframe, 0) + res = self.cpu.get_int_value(deadframe, 0) assert res == 99 looptoken = JitCellToken() @@ -389,7 +366,7 @@ deadframe = self.cpu.execute_token(looptoken) fail = self.cpu.get_latest_descr(deadframe) assert fail is faildescr - res = self.cpu.get_latest_value_int(deadframe, 0) + res = self.cpu.get_int_value(deadframe, 0) assert res == 42 looptoken = JitCellToken() @@ -412,7 +389,7 @@ deadframe = self.cpu.execute_token(looptoken, value) fail = self.cpu.get_latest_descr(deadframe) assert fail is faildescr - res = self.cpu.get_latest_value_float(deadframe, 0) + res = self.cpu.get_float_value(deadframe, 0) assert longlong.getrealfloat(res) == -61.25 looptoken = JitCellToken() @@ -423,7 +400,7 @@ deadframe = self.cpu.execute_token(looptoken) fail = self.cpu.get_latest_descr(deadframe) assert fail is faildescr - res = self.cpu.get_latest_value_float(deadframe, 0) + res = self.cpu.get_float_value(deadframe, 0) assert longlong.getrealfloat(res) == 42.5 def test_execute_operations_in_env(self): @@ -447,8 +424,8 @@ operations[-2].setfailargs([t, z]) cpu.compile_loop([x, y], operations, looptoken) deadframe = self.cpu.execute_token(looptoken, 0, 10) - assert self.cpu.get_latest_value_int(deadframe, 0) == 0 - assert self.cpu.get_latest_value_int(deadframe, 1) == 55 + assert self.cpu.get_int_value(deadframe, 0) == 0 + assert self.cpu.get_int_value(deadframe, 1) == 55 def test_int_operations(self): from pypy.jit.metainterp.test.test_executor import get_int_tests @@ -512,7 +489,7 @@ else: assert fail.identifier == 2 if z != boom: - assert self.cpu.get_latest_value_int(deadframe, 0) == z + assert self.cpu.get_int_value(deadframe, 0) == z excvalue = self.cpu.grab_exc_value(deadframe) assert not excvalue @@ -1235,9 +1212,9 @@ # for k in range(len(retvalues)): if isinstance(retboxes[k], BoxInt): - got = self.cpu.get_latest_value_int(deadframe, k) + got = self.cpu.get_int_value(deadframe, k) else: - got = self.cpu.get_latest_value_float(deadframe, k) + got = self.cpu.get_float_value(deadframe, k) assert got == retvalues[k] def test_jump(self): @@ -1326,11 +1303,11 @@ dstvalues[index_counter] = 0 for i, (box, val) in enumerate(zip(inputargs, dstvalues)): if isinstance(box, BoxInt): - got = self.cpu.get_latest_value_int(deadframe, i) + got = self.cpu.get_int_value(deadframe, i) elif isinstance(box, BoxPtr): - got = self.cpu.get_latest_value_ref(deadframe, i) + got = self.cpu.get_ref_value(deadframe, i) elif isinstance(box, BoxFloat): - got = self.cpu.get_latest_value_float(deadframe, i) + got = self.cpu.get_float_value(deadframe, i) else: assert 0 assert type(got) == type(val) @@ -1370,10 +1347,10 @@ deadframe = self.cpu.execute_token(looptoken, *args) fail = self.cpu.get_latest_descr(deadframe) assert fail.identifier == 2 - res = self.cpu.get_latest_value_float(deadframe, 0) + res = self.cpu.get_float_value(deadframe, 0) assert longlong.getrealfloat(res) == 8.5 for i in range(1, len(fboxes)): - got = longlong.getrealfloat(self.cpu.get_latest_value_float( + got = longlong.getrealfloat(self.cpu.get_float_value( deadframe, i)) assert got == 13.5 + 6.73 * i @@ -1399,9 +1376,9 @@ deadframe = self.cpu.execute_token(looptoken, *args) #xxx check fail = self.cpu.get_latest_descr(deadframe) assert loop.operations[-2].getdescr() == fail - f1 = self.cpu.get_latest_value_float(deadframe, 0) - f2 = self.cpu.get_latest_value_float(deadframe, 1) - f3 = self.cpu.get_latest_value_float(deadframe, 2) + f1 = self.cpu.get_float_value(deadframe, 0) + f2 = self.cpu.get_float_value(deadframe, 1) + f3 = self.cpu.get_float_value(deadframe, 2) assert longlong.getrealfloat(f1) == 132.25 assert longlong.getrealfloat(f2) == 0.75 assert longlong.getrealfloat(f3) == 133.0 @@ -1417,9 +1394,9 @@ deadframe = self.cpu.execute_token(looptoken, *args) fail = self.cpu.get_latest_descr(deadframe) assert fail.identifier == 100 - f1 = self.cpu.get_latest_value_float(deadframe, 0) - f2 = self.cpu.get_latest_value_float(deadframe, 1) - f3 = self.cpu.get_latest_value_float(deadframe, 2) + f1 = self.cpu.get_float_value(deadframe, 0) + f2 = self.cpu.get_float_value(deadframe, 1) + f3 = self.cpu.get_float_value(deadframe, 2) assert longlong.getrealfloat(f1) == 132.25 assert longlong.getrealfloat(f2) == 0.75 assert longlong.getrealfloat(f3) == 133.0 @@ -2023,12 +2000,12 @@ looptoken = JitCellToken() self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) deadframe = self.cpu.execute_token(looptoken, 1) - assert self.cpu.get_latest_value_int(deadframe, 0) == 0 - assert self.cpu.get_latest_value_ref(deadframe, 1) == xptr + assert self.cpu.get_int_value(deadframe, 0) == 0 + assert self.cpu.get_ref_value(deadframe, 1) == xptr excvalue = self.cpu.grab_exc_value(deadframe) assert not excvalue deadframe = self.cpu.execute_token(looptoken, 0) - assert self.cpu.get_latest_value_int(deadframe, 0) == 1 + assert self.cpu.get_int_value(deadframe, 0) == 1 excvalue = self.cpu.grab_exc_value(deadframe) assert not excvalue @@ -2047,7 +2024,7 @@ looptoken = JitCellToken() self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) deadframe = self.cpu.execute_token(looptoken, 1) - assert self.cpu.get_latest_value_int(deadframe, 0) == 1 + assert self.cpu.get_int_value(deadframe, 0) == 1 excvalue = self.cpu.grab_exc_value(deadframe) assert excvalue == yptr @@ -2064,11 +2041,11 @@ looptoken = JitCellToken() self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) deadframe = self.cpu.execute_token(looptoken, 1) - assert self.cpu.get_latest_value_int(deadframe, 0) == 1 + assert self.cpu.get_int_value(deadframe, 0) == 1 excvalue = self.cpu.grab_exc_value(deadframe) assert excvalue == xptr deadframe = self.cpu.execute_token(looptoken, 0) - assert self.cpu.get_latest_value_int(deadframe, 0) == 0 + assert self.cpu.get_int_value(deadframe, 0) == 0 excvalue = self.cpu.grab_exc_value(deadframe) assert not excvalue @@ -2228,8 +2205,8 @@ if flag: deadframe = self.cpu.force(token) values.append(self.cpu.get_latest_descr(deadframe)) - values.append(self.cpu.get_latest_value_int(deadframe, 0)) - values.append(self.cpu.get_latest_value_int(deadframe, 1)) + values.append(self.cpu.get_int_value(deadframe, 0)) + values.append(self.cpu.get_int_value(deadframe, 1)) self.cpu.set_savedata_ref(deadframe, random_gcref) FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Void) @@ -2255,14 +2232,14 @@ deadframe = self.cpu.execute_token(looptoken, 20, 0) fail = self.cpu.get_latest_descr(deadframe) assert fail.identifier == 0 - assert self.cpu.get_latest_value_int(deadframe, 0) == 20 + assert self.cpu.get_int_value(deadframe, 0) == 20 assert values == [] deadframe = self.cpu.execute_token(looptoken, 10, 1) fail = self.cpu.get_latest_descr(deadframe) assert fail.identifier == 1 - assert self.cpu.get_latest_value_int(deadframe, 0) == 1 - assert self.cpu.get_latest_value_int(deadframe, 1) == 10 + assert self.cpu.get_int_value(deadframe, 0) == 1 + assert self.cpu.get_int_value(deadframe, 1) == 10 assert values == [faildescr, 1, 10] assert self.cpu.get_savedata_ref(deadframe) # not NULL assert self.cpu.get_savedata_ref(deadframe) == random_gcref @@ -2272,8 +2249,8 @@ def maybe_force(token, flag): if flag: deadframe = self.cpu.force(token) - values.append(self.cpu.get_latest_value_int(deadframe, 0)) - values.append(self.cpu.get_latest_value_int(deadframe, 2)) + values.append(self.cpu.get_int_value(deadframe, 0)) + values.append(self.cpu.get_int_value(deadframe, 2)) self.cpu.set_savedata_ref(deadframe, random_gcref) return 42 @@ -2301,15 +2278,15 @@ deadframe = self.cpu.execute_token(looptoken, 20, 0) fail = self.cpu.get_latest_descr(deadframe) assert fail.identifier == 0 - assert self.cpu.get_latest_value_int(deadframe, 0) == 42 + assert self.cpu.get_int_value(deadframe, 0) == 42 assert values == [] deadframe = self.cpu.execute_token(looptoken, 10, 1) fail = self.cpu.get_latest_descr(deadframe) assert fail.identifier == 1 - assert self.cpu.get_latest_value_int(deadframe, 0) == 1 - assert self.cpu.get_latest_value_int(deadframe, 1) == 42 - assert self.cpu.get_latest_value_int(deadframe, 2) == 10 + assert self.cpu.get_int_value(deadframe, 0) == 1 + assert self.cpu.get_int_value(deadframe, 1) == 42 + assert self.cpu.get_int_value(deadframe, 2) == 10 assert values == [1, 10] assert self.cpu.get_savedata_ref(deadframe) == random_gcref @@ -2320,8 +2297,8 @@ def maybe_force(token, flag): if flag: deadframe = self.cpu.force(token) - values.append(self.cpu.get_latest_value_int(deadframe, 0)) - values.append(self.cpu.get_latest_value_int(deadframe, 2)) + values.append(self.cpu.get_int_value(deadframe, 0)) + values.append(self.cpu.get_int_value(deadframe, 2)) self.cpu.set_savedata_ref(deadframe, random_gcref) return 42.5 @@ -2349,17 +2326,17 @@ deadframe = self.cpu.execute_token(looptoken, 20, 0) fail = self.cpu.get_latest_descr(deadframe) assert fail.identifier == 0 - x = self.cpu.get_latest_value_float(deadframe, 0) + x = self.cpu.get_float_value(deadframe, 0) assert longlong.getrealfloat(x) == 42.5 assert values == [] deadframe = self.cpu.execute_token(looptoken, 10, 1) fail = self.cpu.get_latest_descr(deadframe) assert fail.identifier == 1 - assert self.cpu.get_latest_value_int(deadframe, 0) == 1 - x = self.cpu.get_latest_value_float(deadframe, 1) + assert self.cpu.get_int_value(deadframe, 0) == 1 + x = self.cpu.get_float_value(deadframe, 1) assert longlong.getrealfloat(x) == 42.5 - assert self.cpu.get_latest_value_int(deadframe, 2) == 10 + assert self.cpu.get_int_value(deadframe, 2) == 10 assert values == [1, 10] assert self.cpu.get_savedata_ref(deadframe) == random_gcref @@ -2391,7 +2368,7 @@ deadframe = self.cpu.execute_token(looptoken, ord('G')) fail = self.cpu.get_latest_descr(deadframe) assert fail.identifier == 0 - assert self.cpu.get_latest_value_int(deadframe, 0) == ord('g') + assert self.cpu.get_int_value(deadframe, 0) == ord('g') def test_call_to_c_function_with_callback(self): from pypy.rlib.libffi import CDLL, types, ArgChain, clibffi @@ -2509,7 +2486,7 @@ deadframe = self.cpu.execute_token(looptoken, *args) fail = self.cpu.get_latest_descr(deadframe) assert fail.identifier == 0 - assert self.cpu.get_latest_value_int(deadframe, 0) == len(cwd) + assert self.cpu.get_int_value(deadframe, 0) == len(cwd) assert rffi.charp2strn(buffer, buflen) == cwd lltype.free(buffer, flavor='raw') @@ -2529,7 +2506,7 @@ deadframe = self.cpu.execute_token(looptoken, -42, 9) fail = self.cpu.get_latest_descr(deadframe) assert fail.identifier == 0 - assert self.cpu.get_latest_value_int(deadframe, 0) == -42 + assert self.cpu.get_int_value(deadframe, 0) == -42 print 'step 1 ok' print '-'*79 @@ -2539,7 +2516,7 @@ deadframe = self.cpu.execute_token(looptoken, -42, 9) fail = self.cpu.get_latest_descr(deadframe) assert fail is faildescr - assert self.cpu.get_latest_value_int(deadframe, 0) == 9 + assert self.cpu.get_int_value(deadframe, 0) == 9 print 'step 2 ok' print '-'*79 @@ -2556,7 +2533,7 @@ deadframe = self.cpu.execute_token(looptoken, -42, 9) fail = self.cpu.get_latest_descr(deadframe) assert fail.identifier == 3 - assert self.cpu.get_latest_value_int(deadframe, 0) == 9 + assert self.cpu.get_int_value(deadframe, 0) == 9 print 'step 3 ok' print '-'*79 @@ -2597,7 +2574,7 @@ deadframe = self.cpu.execute_token(looptoken, 16) fail = self.cpu.get_latest_descr(deadframe) assert fail.identifier == 3 - assert self.cpu.get_latest_value_int(deadframe, 0) == 333 + assert self.cpu.get_int_value(deadframe, 0) == 333 # pure do_ / descr features @@ -2770,7 +2747,7 @@ def test_assembler_call(self): called = [] def assembler_helper(deadframe, virtualizable): - assert self.cpu.get_latest_value_int(deadframe, 0) == 97 + assert self.cpu.get_int_value(deadframe, 0) == 97 called.append(self.cpu.get_latest_descr(deadframe)) return 4 + 9 @@ -2809,7 +2786,7 @@ EffectInfo.MOST_GENERAL) args = [i+1 for i in range(10)] deadframe = self.cpu.execute_token(looptoken, *args) - assert self.cpu.get_latest_value_int(deadframe, 0) == 55 + assert self.cpu.get_int_value(deadframe, 0) == 55 ops = ''' [i0, i1, i2, i3, i4, i5, i6, i7, i8, i9] i10 = int_add(i0, 42) @@ -2822,7 +2799,7 @@ self.cpu.compile_loop(loop.inputargs, loop.operations, othertoken) args = [i+1 for i in range(10)] deadframe = self.cpu.execute_token(othertoken, *args) - assert self.cpu.get_latest_value_int(deadframe, 0) == 13 + assert self.cpu.get_int_value(deadframe, 0) == 13 assert called == [finish_descr] # test the fast path, which should not call assembler_helper() @@ -2834,7 +2811,7 @@ self.cpu.compile_loop(loop.inputargs, loop.operations, othertoken) args = [i+1 for i in range(10)] deadframe = self.cpu.execute_token(othertoken, *args) - assert self.cpu.get_latest_value_int(deadframe, 0) == 97 + assert self.cpu.get_int_value(deadframe, 0) == 97 assert not called finally: del self.cpu.done_with_this_frame_int_v @@ -2844,7 +2821,7 @@ py.test.skip("requires floats") called = [] def assembler_helper(deadframe, virtualizable): - x = self.cpu.get_latest_value_float(deadframe, 0) + x = self.cpu.get_float_value(deadframe, 0) assert longlong.getrealfloat(x) == 1.2 + 3.2 called.append(self.cpu.get_latest_descr(deadframe)) print '!' * 30 + 'assembler_helper' @@ -2877,7 +2854,7 @@ args = [longlong.getfloatstorage(1.2), longlong.getfloatstorage(2.3)] deadframe = self.cpu.execute_token(looptoken, *args) - x = self.cpu.get_latest_value_float(deadframe, 0) + x = self.cpu.get_float_value(deadframe, 0) assert longlong.getrealfloat(x) == 1.2 + 2.3 ops = ''' [f4, f5] @@ -2891,7 +2868,7 @@ args = [longlong.getfloatstorage(1.2), longlong.getfloatstorage(3.2)] deadframe = self.cpu.execute_token(othertoken, *args) - x = self.cpu.get_latest_value_float(deadframe, 0) + x = self.cpu.get_float_value(deadframe, 0) assert longlong.getrealfloat(x) == 13.5 assert called == [finish_descr] @@ -2905,7 +2882,7 @@ args = [longlong.getfloatstorage(1.2), longlong.getfloatstorage(4.2)] deadframe = self.cpu.execute_token(othertoken, *args) - x = self.cpu.get_latest_value_float(deadframe, 0) + x = self.cpu.get_float_value(deadframe, 0) assert longlong.getrealfloat(x) == 1.2 + 4.2 assert not called finally: @@ -2940,7 +2917,7 @@ py.test.skip("requires floats") called = [] def assembler_helper(deadframe, virtualizable): - x = self.cpu.get_latest_value_float(deadframe, 0) + x = self.cpu.get_float_value(deadframe, 0) assert longlong.getrealfloat(x) == 1.25 + 3.25 called.append(self.cpu.get_latest_descr(deadframe)) return 13.5 @@ -2972,7 +2949,7 @@ args = [longlong.getfloatstorage(1.25), longlong.getfloatstorage(2.35)] deadframe = self.cpu.execute_token(looptoken, *args) - x = self.cpu.get_latest_value_float(deadframe, 0) + x = self.cpu.get_float_value(deadframe, 0) assert longlong.getrealfloat(x) == 1.25 + 2.35 assert not called @@ -2990,7 +2967,7 @@ args = [longlong.getfloatstorage(1.25), longlong.getfloatstorage(3.25)] deadframe = self.cpu.execute_token(othertoken, *args) - x = self.cpu.get_latest_value_float(deadframe, 0) + x = self.cpu.get_float_value(deadframe, 0) assert longlong.getrealfloat(x) == 13.5 assert called == [finish_descr] del called[:] @@ -3013,7 +2990,7 @@ args = [longlong.getfloatstorage(6.0), longlong.getfloatstorage(1.5)] # 6.0-1.5 == 1.25+3.25 deadframe = self.cpu.execute_token(othertoken, *args) - x = self.cpu.get_latest_value_float(deadframe, 0) + x = self.cpu.get_float_value(deadframe, 0) assert longlong.getrealfloat(x) == 13.5 assert called == [finish_descr2] @@ -3421,7 +3398,7 @@ deadframe = self.cpu.execute_token(looptoken, 2) fail = self.cpu.get_latest_descr(deadframe) assert fail.identifier == 2 - res = self.cpu.get_latest_value_int(deadframe, 0) + res = self.cpu.get_int_value(deadframe, 0) assert res == 10 inputargs2 = [i0] @@ -3434,7 +3411,7 @@ deadframe = self.cpu.execute_token(looptoken, 2) fail = self.cpu.get_latest_descr(deadframe) assert fail.identifier == 3 - res = self.cpu.get_latest_value_int(deadframe, 0) + res = self.cpu.get_int_value(deadframe, 0) assert res == -10 def test_int_force_ge_zero(self): @@ -3449,7 +3426,7 @@ self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) for inp, outp in [(2,2), (-3, 0)]: deadframe = self.cpu.execute_token(looptoken, inp) - assert outp == self.cpu.get_latest_value_int(deadframe, 0) + assert outp == self.cpu.get_int_value(deadframe, 0) def test_compile_asmlen(self): from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU @@ -3628,7 +3605,7 @@ self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) deadframe = self.cpu.execute_token(looptoken, rffi.cast(lltype.Signed, p), 16) - result = self.cpu.get_latest_value_int(deadframe, 0) + result = self.cpu.get_int_value(deadframe, 0) assert result == rffi.cast(lltype.Signed, value) rawstorage.free_raw_storage(p) @@ -3653,7 +3630,7 @@ self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) deadframe = self.cpu.execute_token(looptoken, rffi.cast(lltype.Signed, p), 16) - result = self.cpu.get_latest_value_float(deadframe, 0) + result = self.cpu.get_float_value(deadframe, 0) result = longlong.getrealfloat(result) assert result == rffi.cast(lltype.Float, value) rawstorage.free_raw_storage(p) @@ -3712,7 +3689,7 @@ values = [] def maybe_force(token, flag): deadframe = self.cpu.force(token) - values.append(self.cpu.get_latest_value_int(deadframe, 0)) + values.append(self.cpu.get_int_value(deadframe, 0)) return 42 FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Signed) @@ -3738,7 +3715,7 @@ deadframe = self.cpu.execute_token(looptoken, 20, 0) fail = self.cpu.get_latest_descr(deadframe) assert fail.identifier == 23 - assert self.cpu.get_latest_value_int(deadframe, 0) == 42 + assert self.cpu.get_int_value(deadframe, 0) == 42 # make sure that force reads the registers from a zeroed piece of # memory assert values[0] == 0 diff --git a/pypy/jit/backend/x86/runner.py b/pypy/jit/backend/x86/runner.py --- a/pypy/jit/backend/x86/runner.py +++ b/pypy/jit/backend/x86/runner.py @@ -13,6 +13,7 @@ from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU from pypy.jit.backend.llsupport import jitframe from pypy.jit.backend.x86 import regloc +from pypy.jit.backend.llsupport.symbolic import WORD import sys from pypy.tool.ansi_print import ansi_log @@ -120,7 +121,14 @@ prev_interpreter = LLInterpreter.current_interpreter LLInterpreter.current_interpreter = self.debug_ll_interpreter try: - # XXX parameters + # XXX RPythonize + num = 0 + for arg in args: + if isinstance(arg, int): + self.set_int_value(frame, num, arg) + else: + xxx + num += WORD descr_no = func(ll_frame) finally: if not self.translate_support_code: @@ -128,7 +136,7 @@ #llop.debug_print(lltype.Void, "<<<< Back") descr = self.get_fail_descr_from_number(descr_no) frame.jf_descr = cast_instance_to_gcref(descr) - return ll_frame + return frame return execute_token def cast_ptr_to_int(x): diff --git a/pypy/jit/backend/x86/test/test_runner.py b/pypy/jit/backend/x86/test/test_runner.py --- a/pypy/jit/backend/x86/test/test_runner.py +++ b/pypy/jit/backend/x86/test/test_runner.py @@ -33,6 +33,9 @@ # for the individual tests see # ====> ../../test/runner_test.py + def find_pos_on_faildescr(self, descr): + return descr._x86_result_offset + add_loop_instructions = ['mov', 'add', 'test', 'je', 'jmp'] if WORD == 4: bridge_loop_instructions = ['lea', 'jmp'] From noreply at buildbot.pypy.org Wed Jan 9 18:27:00 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 9 Jan 2013 18:27:00 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: enough refactoring to pass ints and floats there Message-ID: <20130109172700.704251C0041@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r59891:91afb0f2dfc0 Date: 2013-01-09 19:26 +0200 http://bitbucket.org/pypy/pypy/changeset/91afb0f2dfc0/ Log: enough refactoring to pass ints and floats there diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py --- a/pypy/jit/backend/llsupport/llmodel.py +++ b/pypy/jit/backend/llsupport/llmodel.py @@ -275,32 +275,90 @@ return history.AbstractDescr.show(self, descr) def get_int_value(self, deadframe, index): - return deadframe.jf_frame[index // WORD] + descr = self.gc_ll_descr.getframedescrs(self).arraydescr + ofs = self.unpack_arraydescr(descr) + return self.read_int_at_mem(deadframe, index + ofs, 1, WORD) def get_ref_value(self, deadframe, index): - XXX - deadframe = lltype.cast_opaque_ptr(jitframe.DEADFRAMEPTR, deadframe) - return deadframe.jf_values[index].ref + descr = self.gc_ll_descr.getframedescrs(self).arraydescr + ofs = self.unpack_arraydescr(descr) + return self.read_ref_at_mem(deadframe, index + ofs) def get_float_value(self, deadframe, index): - XXX - deadframe = lltype.cast_opaque_ptr(jitframe.DEADFRAMEPTR, deadframe) - return deadframe.jf_values[index].float + descr = self.gc_ll_descr.getframedescrs(self).arraydescr + ofs = self.unpack_arraydescr(descr) + return self.read_float_at_mem(deadframe, index + ofs) - def set_int_value(self, newframe, index, value): - """ Note that we keep index multiplied by WORD here mostly - for completeness with get_int_value and friends - """ - newframe.jf_frame[index // WORD] = value + # ____________________ RAW PRIMITIVES ________________________ - def set_ref_value(self, deadframe, index, value): - xxx + @specialize.argtype(1) + def read_int_at_mem(self, gcref, ofs, size, sign): + # --- start of GC unsafe code (no GC operation!) --- + items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) + for STYPE, UTYPE, itemsize in unroll_basic_sizes: + if size == itemsize: + if sign: + items = rffi.cast(rffi.CArrayPtr(STYPE), items) + val = items[0] + val = rffi.cast(lltype.Signed, val) + else: + items = rffi.cast(rffi.CArrayPtr(UTYPE), items) + val = items[0] + val = rffi.cast(lltype.Signed, val) + # --- end of GC unsafe code --- + return val + else: + raise NotImplementedError("size = %d" % size) - def set_float_value(self, deadframe, index, value): - xxx + @specialize.argtype(1) + def write_int_at_mem(self, gcref, ofs, size, sign, newvalue): + # --- start of GC unsafe code (no GC operation!) --- + items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) + for TYPE, _, itemsize in unroll_basic_sizes: + if size == itemsize: + items = rffi.cast(rffi.CArrayPtr(TYPE), items) + items[0] = rffi.cast(TYPE, newvalue) + # --- end of GC unsafe code --- + return + else: + raise NotImplementedError("size = %d" % size) + + def read_ref_at_mem(self, gcref, ofs): + # --- start of GC unsafe code (no GC operation!) --- + items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) + items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items) + pval = self._cast_int_to_gcref(items[0]) + # --- end of GC unsafe code --- + return pval + + def write_ref_at_mem(self, gcref, ofs, newvalue): + self.gc_ll_descr.do_write_barrier(gcref, newvalue) + # --- start of GC unsafe code (no GC operation!) --- + items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) + items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items) + items[0] = self.cast_gcref_to_int(newvalue) + # --- end of GC unsafe code --- + + @specialize.argtype(1) + def read_float_at_mem(self, gcref, ofs): + # --- start of GC unsafe code (no GC operation!) --- + items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) + items = rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), items) + fval = items[0] + # --- end of GC unsafe code --- + return fval + + @specialize.argtype(1) + def write_float_at_mem(self, gcref, ofs, newvalue): + # --- start of GC unsafe code (no GC operation!) --- + items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) + items = rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), items) + items[0] = newvalue + # --- end of GC unsafe code --- # ____________________________________________________________ + def bh_arraylen_gc(self, array, arraydescr): assert isinstance(arraydescr, ArrayDescr) ofs = arraydescr.lendescr.offset @@ -309,73 +367,34 @@ @specialize.argtype(1) def bh_getarrayitem_gc_i(self, gcref, itemindex, arraydescr): ofs, size, sign = self.unpack_arraydescr_size(arraydescr) - # --- start of GC unsafe code (no GC operation!) --- - items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) - for STYPE, UTYPE, itemsize in unroll_basic_sizes: - if size == itemsize: - if sign: - items = rffi.cast(rffi.CArrayPtr(STYPE), items) - val = items[itemindex] - val = rffi.cast(lltype.Signed, val) - else: - items = rffi.cast(rffi.CArrayPtr(UTYPE), items) - val = items[itemindex] - val = rffi.cast(lltype.Signed, val) - # --- end of GC unsafe code --- - return val - else: - raise NotImplementedError("size = %d" % size) + return self.read_int_at_mem(gcref, ofs + itemindex * size, size, + sign) def bh_getarrayitem_gc_r(self, gcref, itemindex, arraydescr): ofs = self.unpack_arraydescr(arraydescr) - # --- start of GC unsafe code (no GC operation!) --- - items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) - items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items) - pval = self._cast_int_to_gcref(items[itemindex]) - # --- end of GC unsafe code --- - return pval + return self.read_ref_at_mem(gcref, itemindex * WORD + ofs) @specialize.argtype(1) def bh_getarrayitem_gc_f(self, gcref, itemindex, arraydescr): ofs = self.unpack_arraydescr(arraydescr) - # --- start of GC unsafe code (no GC operation!) --- - items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) - items = rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), items) - fval = items[itemindex] - # --- end of GC unsafe code --- - return fval + fsize = rffi.sizeof(longlong.FLOATSTORAGE) + return self.read_float_at_mem(gcref, itemindex * fsize + ofs) @specialize.argtype(1) def bh_setarrayitem_gc_i(self, gcref, itemindex, newvalue, arraydescr): ofs, size, sign = self.unpack_arraydescr_size(arraydescr) - # --- start of GC unsafe code (no GC operation!) --- - items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) - for TYPE, _, itemsize in unroll_basic_sizes: - if size == itemsize: - items = rffi.cast(rffi.CArrayPtr(TYPE), items) - items[itemindex] = rffi.cast(TYPE, newvalue) - # --- end of GC unsafe code --- - return - else: - raise NotImplementedError("size = %d" % size) + self.write_int_at_mem(gcref, ofs + itemindex * size, size, sign, + newvalue) def bh_setarrayitem_gc_r(self, gcref, itemindex, newvalue, arraydescr): ofs = self.unpack_arraydescr(arraydescr) - self.gc_ll_descr.do_write_barrier(gcref, newvalue) - # --- start of GC unsafe code (no GC operation!) --- - items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) - items = rffi.cast(rffi.CArrayPtr(lltype.Signed), items) - items[itemindex] = self.cast_gcref_to_int(newvalue) - # --- end of GC unsafe code --- + self.write_ref_at_mem(gcref, itemindex * WORD + ofs, newvalue) @specialize.argtype(1) def bh_setarrayitem_gc_f(self, gcref, itemindex, newvalue, arraydescr): ofs = self.unpack_arraydescr(arraydescr) - # --- start of GC unsafe code (no GC operation!) --- - items = rffi.ptradd(rffi.cast(rffi.CCHARP, gcref), ofs) - items = rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), items) - items[itemindex] = newvalue - # --- end of GC unsafe code --- + fsize = rffi.sizeof(longlong.FLOATSTORAGE) + self.write_float_at_mem(gcref, ofs + itemindex * fsize, newvalue) bh_setarrayitem_raw_i = bh_setarrayitem_gc_i bh_setarrayitem_raw_f = bh_setarrayitem_gc_f diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py --- a/pypy/jit/backend/test/runner_test.py +++ b/pypy/jit/backend/test/runner_test.py @@ -144,7 +144,9 @@ self.cpu.compile_loop(inputargs, operations, looptoken) deadframe = self.cpu.execute_token(looptoken, longlong.getfloatstorage(2.8)) - res = self.cpu.get_float_value(deadframe, 0) + fail = self.cpu.get_latest_descr(deadframe) + res = self.cpu.get_float_value(deadframe, + self.find_pos_on_faildescr(fail)) assert longlong.getrealfloat(res) == 5.1 fail = self.cpu.get_latest_descr(deadframe) assert fail.identifier == 1 diff --git a/pypy/jit/backend/x86/runner.py b/pypy/jit/backend/x86/runner.py --- a/pypy/jit/backend/x86/runner.py +++ b/pypy/jit/backend/x86/runner.py @@ -14,6 +14,7 @@ from pypy.jit.backend.llsupport import jitframe from pypy.jit.backend.x86 import regloc from pypy.jit.backend.llsupport.symbolic import WORD +from pypy.jit.backend.llsupport.descr import unpack_arraydescr import sys from pypy.tool.ansi_print import ansi_log @@ -126,6 +127,10 @@ for arg in args: if isinstance(arg, int): self.set_int_value(frame, num, arg) + elif isinstance(arg, float): + self.set_float_value(frame, num, arg) + if IS_X86_32: + num += WORD else: xxx num += WORD @@ -196,6 +201,24 @@ l[i].counter = ll_s.i return l + def set_int_value(self, newframe, index, value): + """ Note that we keep index multiplied by WORD here mostly + for completeness with get_int_value and friends + """ + descr = self.gc_ll_descr.getframedescrs(self).arraydescr + ofs = self.unpack_arraydescr(descr) + self.write_int_at_mem(newframe, ofs + index * WORD, WORD, 1, value) + + def set_ref_value(self, newframe, index, value): + descr = self.gc_ll_descr.getframedescrs(self).arraydescr + ofs = self.unpack_arraydescr(descr) + self.write_ref_at_mem(newframe, ofs + index * WORD, value) + + def set_float_value(self, newframe, index, value): + descr = self.gc_ll_descr.getframedescrs(self).arraydescr + ofs = self.unpack_arraydescr(descr) + self.write_float_at_mem(newframe, ofs + index * WORD, value) + class CPU386(AbstractX86CPU): backend_name = 'x86' WORD = 4 From noreply at buildbot.pypy.org Wed Jan 9 18:46:16 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Wed, 9 Jan 2013 18:46:16 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: py3k's December update Message-ID: <20130109174616.BDF8D1C0041@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: extradoc Changeset: r4939:ca4f1f89f7b7 Date: 2013-01-09 09:44 -0800 http://bitbucket.org/pypy/extradoc/changeset/ca4f1f89f7b7/ Log: py3k's December update diff --git a/blog/draft/py3k-status-update-9.rst b/blog/draft/py3k-status-update-9.rst new file mode 100644 --- /dev/null +++ b/blog/draft/py3k-status-update-9.rst @@ -0,0 +1,32 @@ +Py3k status update #9 +--------------------- + +This is the ninth status update about our work on the `py3k branch`_, which +we can work on thanks to all of the people who donated_ to the `py3k +proposal`_. + +Just a very short update on December's work: we're now passing about 223 of +approximately 355 modules of CPython's regression test suite, up from passing +194 last month. + +Some brief highlights: + +* More encoding related issues were addressed. e.g. now most if not all the + multibytecodec test modules pass. + +* Fixed some path handling issues (``test_os``, ``test_ntpath`` and + ``test_posixpath`` now pass) + +* We now pass ``test_class``, ``test_descr`` and almost ``test_builtin`` (among + other things): these are notable as they are fairly extensive test suites of + core aspects of the langauge. + +* Amaury Forgeot d'Arc continued making progress on `CPyExt`_ (thanks again!) + +cheers, +Phil + +.. _donated: http://morepypy.blogspot.com/2012/01/py3k-and-numpy-first-stage-thanks-to.html +.. _`py3k proposal`: http://pypy.org/py3donate.html +.. _`py3k branch`: https://bitbucket.org/pypy/pypy/commits/all/tip/branch%28%22py3k%22%29 +.. _`CPyExt`: http://morepypy.blogspot.com/2010/04/using-cpython-extension-modules-with.html From noreply at buildbot.pypy.org Wed Jan 9 18:46:18 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Wed, 9 Jan 2013 18:46:18 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: merge upstream Message-ID: <20130109174618.13E651C0041@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: extradoc Changeset: r4940:065d72cf8cdf Date: 2013-01-09 09:45 -0800 http://bitbucket.org/pypy/extradoc/changeset/065d72cf8cdf/ Log: merge upstream diff --git a/blog/draft/numpy-internship-2.rst b/blog/draft/numpy-internship-2.rst new file mode 100644 --- /dev/null +++ b/blog/draft/numpy-internship-2.rst @@ -0,0 +1,28 @@ + +Hello everyone + +I would like to advertise a PyPy-related summer internship at +the National Center for Atmospheric Research, which is located in lovely +Boulder, Colorado. As for the last year, the mentor will be Davide del Vento, +with my possible support on the PyPy side. + +The full details of the application are to be found on +`the internship description`_ and make sure you read `the requirements`_ +first. Important requirements: + +* Must currently be enrolled in a United States university. + +* Only students authorized to work for any employer in the United + States will be considered for the SIParCS program. + +* Must be a graduate or under graduate who has completed their sophomore year. + +If you happen to fulfill the requirements, to me this sounds like +a great opportunity to spend a summer at NCAR in Boulder hacking on atmospheric +models using PyPy. + +.. _`the internship description`: http://cisl.catsone.com/careers/index.php?m=portal&a=details&jobOrderID=1694159 +.. _`the requirements`: https://www2.cisl.ucar.edu/siparcs + +Cheers, +fijal diff --git a/blog/draft/numpy-status-update-6.rst b/blog/draft/numpy-status-update-6.rst new file mode 100644 --- /dev/null +++ b/blog/draft/numpy-status-update-6.rst @@ -0,0 +1,29 @@ +NumPy status update #6 +---------------------- + +Hello. + +This is the last two months update of the activities on the NumPyPy project. + +First the update: + +* **dtype support** - NumPy on PyPy now supports all the numeric dtypes in numpy, + including non-native storage formats, longdouble, clongdouble and friends. + +* **missing ndarray attributes** - work has been made toward supporting the attributes + on ndarrays. We are progressing alphabetically, and have made it to d. + +* **pickling support for numarray** - hasn't started yet, but next on the list + +* There has been some work on exposing FFT routines into numpypy. + +More importantly, we're closing on being able to run the pure-python part of +numpy without modifications. This is not getting us close to passing all +the tests, but it's a good start. + +The most important part is the funding. While we managed to get a significant +amount of money in donations, we only managed to spend around $10 000 from it +so far. XXX + +Cheers, +Matti Picus, Maciej Fijalkowski diff --git a/talk/stm2012/stmimpl.rst b/talk/stm2012/stmimpl.rst --- a/talk/stm2012/stmimpl.rst +++ b/talk/stm2012/stmimpl.rst @@ -149,6 +149,8 @@ ``list_of_read_objects`` is a set of all global objects read from, in the revision that was used for reading. It is actually implemented as a list, but the order or repetition of elements in the list is irrelevant. +This list includes all objects that are keys in ``global_to_local``: we +don't have the notion of "write-only object". ``recent_reads_cache`` is a fixed-size cache that remembers recent additions to the preceeding list, in order to avoid inserting too much @@ -233,7 +235,7 @@ if v > start_time: # object too recent? if V >= LOCKED: # object actually locked? goto retry # spin-loop to start of func - ValidateDuringTransaction() # try to move start_time forward + ValidateNow() # try to move start_time forward goto retry # restart searching from R PossiblyUpdateChain(G, R, ...) # see below return R @@ -292,6 +294,7 @@ L->h_revision = R # back-reference to the original L->objectbody... = R->objectbody... global_to_local[R] = L + list_of_read_objects.append(R) return L def LocalizeReadReady(R): @@ -443,12 +446,12 @@ Validation ------------------------------------ -``ValidateDuringTransaction`` is called during a transaction to update -``start_time``. It makes sure that none of the read objects have been -modified since ``start_time``. If one of these objects is modified by -another commit in parallel, then we want this transaction to eventually -fail. More precisely, it will fail the next time one of the -``ValidateDuring*`` functions is called. +``ValidateDuringTransaction`` is called during a transaction just after +``start_time`` has been updated. It makes sure that none of the read +objects have been modified since ``start_time``. If one of these +objects is modified by another commit in parallel, then we want this +transaction to eventually fail. More precisely, it will fail the next +time ``ValidateDuringTransaction`` is called. Note a subtle point: if an object is currently locked, we have to wait until it gets unlocked, because it might turn out to point to a more @@ -456,30 +459,30 @@ Here is ``ValidateDuringTransaction``:: - def ValidateDuringTransaction(): - start_time = GetGlobalCurTime() # copy from the global time + def ValidateDuringTransaction(during_commit): for R in list_of_read_objects: v = R->h_revision if not (v & 1): # "is a pointer", i.e. - AbortTransaction() # "has a more recent revision" + return False # "has a more recent revision" if v >= LOCKED: # locked - spin loop retry # jump back to the "v = ..." line + if not during_commit: + assert v != my_lock # we don't hold any lock + spin loop retry # jump back to the "v = ..." line + else: + if v != my_lock: # not locked by me: conflict + return False + return True -The last detection for inconsistency is during commit, when -``ValidateDuringCommit`` is called. It is a slightly more complex -version than ``ValidateDuringTransaction`` because it has to handle -"locks" correctly. It also returns a True/False result instead of -aborting:: + def ValidateNow(): + start_time = GetGlobalCurTime() # copy from the global time + if not ValidateDuringTransaction(0): # do validation + AbortTransaction() # if it fails, abort - def ValidateDuringCommit(): - for R in list_of_read_objects: - v = R->h_revision - if not (v & 1): # "is a pointer", i.e. - return False # "has a more recent revision" - if v >= LOCKED: # locked - if v != my_lock: # and not by me - return False - return True +Checking for ``my_lock`` is only useful when ``ValidateDuringTransaction`` +is called during commit, which is when we actually hold locks. In that +case, detecting other already-locked objects causes a conflict. Note that +we should in general never spin-loop during commit; other threads might be +blocked by the fact that we own locks already, causing a deadlock. Local garbage collection @@ -583,8 +586,8 @@ while not CMPXCHG(&global_cur_time, cur_time, cur_time + 2): cur_time = global_cur_time # try again if cur_time != start_time: - if not ValidateDuringCommit(): # only call it if needed - AbortTransaction() # last abort point + if not ValidateDuringTransaction(1): # only call it if needed + AbortTransaction() # last abort point UpdateChainHeads(cur_time) Note the general style of usage of CMPXCHG: we first read normally the @@ -627,7 +630,7 @@ After this, ``CommitTransaction`` increases the global time and then -calls ``ValidateDuringCommit`` defined above. It may still abort. In +calls ``ValidateDuringTransaction`` defined above. It may still abort. In case ``AbortTransaction`` is called, it must release the locks. This is done by writing back the original timestamps in the ``h_revision`` fields:: @@ -707,7 +710,7 @@ cur_time = global_cur_time # try again if start_time != cur_time: start_time = cur_time - if not ValidateDuringCommit(): + if not ValidateDuringTransaction(0): global_cur_time = cur_time # must restore the value inevitable_mutex.release() AbortTransaction() @@ -741,8 +744,8 @@ while not CMPXCHG(&global_cur_time, cur_time, cur_time + 2): cur_time = GetGlobalCurTimeInCommit() # try again if cur_time != start_time: - if not ValidateDuringCommit(): # only call it if needed - AbortTransaction() # last abort point + if not ValidateDuringTransaction(1): # only call it if needed + AbortTransaction() # last abort point UpdateChainHeads(cur_time) def GetGlobalCurTimeInCommit(): From noreply at buildbot.pypy.org Wed Jan 9 19:29:16 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Wed, 9 Jan 2013 19:29:16 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Created rpython/_cache. Added cache_dir to rpython.conftest Message-ID: <20130109182916.41DBB1C0041@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59892:38643cd3eaf1 Date: 2013-01-09 19:28 +0100 http://bitbucket.org/pypy/pypy/changeset/38643cd3eaf1/ Log: Created rpython/_cache. Added cache_dir to rpython.conftest diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -85,3 +85,4 @@ ^compiled ^.git/ ^release/ +^rpython/_cache$ diff --git a/rpython/conftest.py b/rpython/conftest.py --- a/rpython/conftest.py +++ b/rpython/conftest.py @@ -1,3 +1,4 @@ -import os +from os.path import * -cdir = os.path.join(os.path.dirname(__file__), 'translator', 'c') +cdir = realpath(join(dirname(__file__), 'translator', 'c')) +cache_dir = realpath(join(dirname(__file__), '_cache')) diff --git a/rpython/tool/gcc_cache.py b/rpython/tool/gcc_cache.py --- a/rpython/tool/gcc_cache.py +++ b/rpython/tool/gcc_cache.py @@ -1,10 +1,10 @@ -from pypy.conftest import pypydir from rpython.translator.platform import CompilationError from rpython.translator.tool.cbuild import ExternalCompilationInfo +from rpython.conftest import cache_dir from hashlib import md5 import py -cache_dir_root = py.path.local(pypydir).join('_cache').ensure(dir=1) +cache_dir_root = py.path.local(cache_dir).ensure(dir=1) def cache_file_path(c_files, eci, cachename): "Builds a filename to cache compilation data" diff --git a/rpython/translator/cli/query.py b/rpython/translator/cli/query.py --- a/rpython/translator/cli/query.py +++ b/rpython/translator/cli/query.py @@ -65,8 +65,8 @@ def get_cachedir(): - import pypy - _cache = py.path.local(pypy.__file__).new(basename='_cache').ensure(dir=1) + from rpython import cache_dir + _cache = py.path.local(cache_dir).ensure(dir=1) return _cache def load_and_cache_assembly(name, outfile): diff --git a/rpython/translator/cli/src/query.cs b/rpython/translator/cli/src/query.cs --- a/rpython/translator/cli/src/query.cs +++ b/rpython/translator/cli/src/query.cs @@ -42,7 +42,7 @@ { Type[] types = ass.GetTypes(); outfile.WriteLine("# This file has been autogenerated by query.exe -- DO NOT EDIT"); - outfile.WriteLine("from pypy.translator.cli.query import ClassDesc"); + outfile.WriteLine("from rpython.translator.cli.query import ClassDesc"); outfile.WriteLine("types = {}"); foreach(Type t in types) { if (IgnoreType(t)) diff --git a/rpython/translator/goal/translate.py b/rpython/translator/goal/translate.py --- a/rpython/translator/goal/translate.py +++ b/rpython/translator/goal/translate.py @@ -7,12 +7,12 @@ import os import sys -from pypy.conftest import pypydir +from rpython.conftest import cache_dir import py -# clean up early pypy/_cache +# clean up early rpython/_cache try: - py.path.local(pypydir).join('_cache').remove() + py.path.local(cache_dir).remove() except Exception: pass From noreply at buildbot.pypy.org Wed Jan 9 19:47:13 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Wed, 9 Jan 2013 19:47:13 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: pypy.module.binascii.interp_crc32.crc_32_tab -> rpython.rlib.rzip.crc_32_tab Message-ID: <20130109184713.16A351C0041@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59893:0f89ba1b1dd1 Date: 2013-01-09 19:42 +0100 http://bitbucket.org/pypy/pypy/changeset/0f89ba1b1dd1/ Log: pypy.module.binascii.interp_crc32.crc_32_tab -> rpython.rlib.rzip.crc_32_tab diff --git a/pypy/module/binascii/interp_crc32.py b/pypy/module/binascii/interp_crc32.py --- a/pypy/module/binascii/interp_crc32.py +++ b/pypy/module/binascii/interp_crc32.py @@ -1,65 +1,7 @@ from pypy.interpreter.gateway import unwrap_spec from rpython.rlib.rarithmetic import r_uint, intmask from rpython.rtyper.lltypesystem import rffi - -# ____________________________________________________________ - -crc_32_tab = [ - 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, - 0x706af48f, 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, - 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, - 0x90bf1d91, 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, - 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, 0x136c9856, - 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, - 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, - 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, - 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, - 0x45df5c75, 0xdcd60dcf, 0xabd13d59, 0x26d930ac, 0x51de003a, - 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, - 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, - 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, - 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, - 0x9fbfe4a5, 0xe8b8d433, 0x7807c9a2, 0x0f00f934, 0x9609a88e, - 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01, - 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, - 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950, - 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, - 0xfbd44c65, 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, - 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, - 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, - 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, 0xbe0b1010, - 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, - 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, - 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, - 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615, - 0x73dc1683, 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, - 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, 0xf00f9344, - 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, - 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, - 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, - 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, - 0xa6bc5767, 0x3fb506dd, 0x48b2364b, 0xd80d2bda, 0xaf0a1b4c, - 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, - 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, - 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, - 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, - 0x2cd99e8b, 0x5bdeae1d, 0x9b64c2b0, 0xec63f226, 0x756aa39c, - 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, - 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, - 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, - 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, - 0x18b74777, 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, - 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, 0xa00ae278, - 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, - 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, 0x40df0b66, - 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, - 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, - 0xcdd70693, 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, - 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, - 0x2d02ef8d -] -crc_32_tab = map(r_uint, crc_32_tab) - +from rpython.rlib.rzipfile import crc_32_tab @unwrap_spec(data='bufferstr', oldcrc='truncatedint_w') def crc32(space, data, oldcrc=0): diff --git a/rpython/rlib/rzipfile.py b/rpython/rlib/rzipfile.py --- a/rpython/rlib/rzipfile.py +++ b/rpython/rlib/rzipfile.py @@ -11,8 +11,63 @@ except (ImportError, CompilationError): rzlib = None +crc_32_tab = [ + 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, + 0x706af48f, 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, + 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, + 0x90bf1d91, 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, + 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, 0x136c9856, + 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, + 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, + 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, + 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, + 0x45df5c75, 0xdcd60dcf, 0xabd13d59, 0x26d930ac, 0x51de003a, + 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, + 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, + 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, + 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, + 0x9fbfe4a5, 0xe8b8d433, 0x7807c9a2, 0x0f00f934, 0x9609a88e, + 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01, + 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, + 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950, + 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, + 0xfbd44c65, 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, + 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, + 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, + 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, 0xbe0b1010, + 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, + 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, + 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, + 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615, + 0x73dc1683, 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, + 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, 0xf00f9344, + 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, + 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, + 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, + 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, + 0xa6bc5767, 0x3fb506dd, 0x48b2364b, 0xd80d2bda, 0xaf0a1b4c, + 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, + 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, + 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, + 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, + 0x2cd99e8b, 0x5bdeae1d, 0x9b64c2b0, 0xec63f226, 0x756aa39c, + 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, + 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, + 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, + 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, + 0x18b74777, 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, + 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, 0xa00ae278, + 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, + 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, 0x40df0b66, + 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, + 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, + 0xcdd70693, 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, + 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, + 0x2d02ef8d +] +crc_32_tab = map(r_uint, crc_32_tab) + # XXX hack to get crc32 to work -from pypy.module.binascii.interp_crc32 import crc_32_tab rcrc_32_tab = [r_uint(i) for i in crc_32_tab] From noreply at buildbot.pypy.org Wed Jan 9 19:47:14 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Wed, 9 Jan 2013 19:47:14 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed import in rpython/translator/c/test/test_math.py Message-ID: <20130109184714.4575E1C0041@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59894:2746bd823bf1 Date: 2013-01-09 19:46 +0100 http://bitbucket.org/pypy/pypy/changeset/2746bd823bf1/ Log: Fixed import in rpython/translator/c/test/test_math.py diff --git a/rpython/translator/c/test/test_math.py b/rpython/translator/c/test/test_math.py --- a/rpython/translator/c/test/test_math.py +++ b/rpython/translator/c/test/test_math.py @@ -1,5 +1,6 @@ import py, math -from pypy.module.math.test import test_direct +from rpython.rtyper.lltypesystem.module.test.test_ll_math import (MathTests, + getTester) from rpython.translator.c.test.test_standalone import StandaloneTests from rpython.rlib import rfloat @@ -11,7 +12,7 @@ fn = getattr(rfloat, fnname) expect_valueerror = (expected == ValueError) expect_overflowerror = (expected == OverflowError) - check = test_direct.get_tester(expected) + check = get_tester(expected) # def testfn(): try: @@ -28,9 +29,9 @@ testfnlist = [get_test_case(testcase) - for testcase in test_direct.MathTests.TESTCASES] + for testcase in MathTests.TESTCASES] reprlist = [repr(testcase) - for testcase in test_direct.MathTests.TESTCASES] + for testcase in MathTests.TESTCASES] def fn(args): err = False From noreply at buildbot.pypy.org Wed Jan 9 20:40:40 2013 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 9 Jan 2013 20:40:40 +0100 (CET) Subject: [pypy-commit] pypy default: Remove the 'from stdlib_opcode import bytecode_spec' from this file. Message-ID: <20130109194040.E89381C11F3@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r59895:1f11f9231b68 Date: 2013-01-09 20:40 +0100 http://bitbucket.org/pypy/pypy/changeset/1f11f9231b68/ Log: Remove the 'from stdlib_opcode import bytecode_spec' from this file. diff --git a/pypy/objspace/flow/test/test_objspace.py b/pypy/objspace/flow/test/test_objspace.py --- a/pypy/objspace/flow/test/test_objspace.py +++ b/pypy/objspace/flow/test/test_objspace.py @@ -8,18 +8,17 @@ from pypy.objspace.flow.objspace import FlowObjSpace from pypy.objspace.flow.flowcontext import FlowingError, FlowSpaceFrame from pypy import conftest -from pypy.tool.stdlib_opcode import bytecode_spec, host_bytecode_spec +from pypy.tool.stdlib_opcode import host_bytecode_spec import os import operator is_operator = getattr(operator, 'is_', operator.eq) # it's not there 2.2 @contextmanager -def patching_opcodes(*opcodes): +def patching_opcodes(**opcodes): meth_names = host_bytecode_spec.method_names - opnums = [bytecode_spec.opmap[name] for name in opcodes] old_name = {} - for name, num in zip(opcodes, opnums): + for name, num in opcodes.items(): old_name[num] = meth_names[num] meth_names[num] = name yield @@ -898,7 +897,7 @@ """ Tests code generated by pypy-c compiled with CALL_METHOD bytecode """ - with patching_opcodes('CALL_METHOD', 'LOOKUP_METHOD'): + with patching_opcodes(CALL_METHOD=202, LOOKUP_METHOD=201): class X: def m(self): return 3 @@ -922,7 +921,7 @@ """ Tests code generated by pypy-c compiled with BUILD_LIST_FROM_ARG bytecode """ - with patching_opcodes('BUILD_LIST_FROM_ARG'): + with patching_opcodes(BUILD_LIST_FROM_ARG=203): def f(): return [i for i in "abc"] From noreply at buildbot.pypy.org Wed Jan 9 21:18:37 2013 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 9 Jan 2013 21:18:37 +0100 (CET) Subject: [pypy-commit] pypy numpypy-real-as-view: some tests pass Message-ID: <20130109201837.ACF191C0041@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: numpypy-real-as-view Changeset: r59896:b36dcabcd239 Date: 2013-01-09 18:50 +0200 http://bitbucket.org/pypy/pypy/changeset/b36dcabcd239/ Log: some tests pass diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py --- a/pypy/module/micronumpy/arrayimpl/concrete.py +++ b/pypy/module/micronumpy/arrayimpl/concrete.py @@ -222,21 +222,26 @@ return None def get_real(self): + strides = self.get_strides() + backstrides = self.get_backstrides() if self.dtype.is_complex_type(): - raise NotImplementedError('waiting for astype()') - return SliceArray(self.start, self.get_strides(), - self.get_backstrides(), self.get_shape(), self) + dtype = self.dtype.float_type + return SliceArray(self.start, strides, backstrides, + self.get_shape(), self, dtype=dtype) + return SliceArray(self.start, strides, backstrides, + self.get_shape(), self) def get_imag(self): + strides = self.get_strides() + backstrides = self.get_backstrides() if self.dtype.is_complex_type(): - raise NotImplementedError('waiting for astype()') + dtype = self.dtype.float_type + return SliceArray(self.start + dtype.get_size(), strides, + backstrides, self.get_shape(), self, dtype=dtype) if self.dtype.is_flexible_type(): # numpy returns self for self.imag - return SliceArray(self.start, self.get_strides(), - self.get_backstrides(), self.get_shape(), self) - - strides, backstrides = support.calc_strides(self.get_shape(), self.dtype, - self.order) + return SliceArray(self.start, strides, backstrides, + self.get_shape(), self) impl = NonWritableArray(self.get_shape(), self.dtype, self.order, strides, backstrides) impl.fill(self.dtype.box(0)) diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -136,7 +136,7 @@ return self.kind == SIGNEDLTR def is_complex_type(self): - return (self.num == 14 or self.num == 15 or self.num == 16) + return False def is_bool_type(self): return self.kind == BOOLLTR @@ -155,6 +155,19 @@ def get_size(self): return self.itemtype.get_element_size() +class W_ComplexDtype(W_Dtype): + + def __init__(self, itemtype, num, kind, name, char, w_box_type, + alternate_constructors=[], aliases=[], + fields=None, fieldnames=None, native=True, float_type=None): + W_Dtype.__init__(self, itemtype, num, kind, name, char, w_box_type, + alternate_constructors=[], aliases=[], + fields=None, fieldnames=None, native=True) + self.float_type = float_type + + def is_complex_type(self): + return True + def dtype_from_list(space, w_lst): lst_w = space.listview(w_lst) fields = {} @@ -413,15 +426,16 @@ alternate_constructors=[space.w_float], aliases=["float"], ) - self.w_complex64dtype = W_Dtype( + self.w_complex64dtype = W_ComplexDtype( types.Complex64(), num=14, kind=COMPLEXLTR, name="complex64", char="F", w_box_type = space.gettypefor(interp_boxes.W_Complex64Box), + float_type = self.w_float32dtype, ) - self.w_complex128dtype = W_Dtype( + self.w_complex128dtype = W_ComplexDtype( types.Complex128(), num=15, kind=COMPLEXLTR, @@ -430,6 +444,7 @@ w_box_type = space.gettypefor(interp_boxes.W_Complex128Box), alternate_constructors=[space.w_complex], aliases=["complex"], + float_type = self.w_float64dtype, ) if interp_boxes.long_double_size == 12: self.w_float96dtype = W_Dtype( @@ -443,7 +458,7 @@ ) self.w_longdouble = self.w_float96dtype - self.w_complex192dtype = W_Dtype( + self.w_complex192dtype = W_ComplexDtype( types.Complex192(), num=16, kind=COMPLEXLTR, @@ -452,6 +467,7 @@ w_box_type = space.gettypefor(interp_boxes.W_Complex192Box), alternate_constructors=[space.w_complex], aliases=["clongdouble", "clongfloat"], + float_type = self.w_float96dtype, ) self.w_clongdouble = self.w_complex192dtype @@ -467,7 +483,7 @@ ) self.w_longdouble = self.w_float128dtype - self.w_complex256dtype = W_Dtype( + self.w_complex256dtype = W_ComplexDtype( types.Complex256(), num=16, kind=COMPLEXLTR, @@ -476,6 +492,7 @@ w_box_type = space.gettypefor(interp_boxes.W_Complex256Box), alternate_constructors=[space.w_complex], aliases=["clongdouble", "clongfloat"], + float_type = self.w_float128dtype, ) self.w_clongdouble = self.w_complex256dtype else: diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1360,10 +1360,14 @@ assert a[1] == b[1] b[1] = 'xyz' assert a[1] == 'xyz' - a=array([1+1j, 2-3j]) - assert a.real[1] == 2 - a.real[1] = -20 - assert a[1].real == -20 + a=array([[1+1j, 2-3j, 4+5j],[-6+7j, 8-9j, -2-1j]]) + assert a.real[0,1] == 2 + a.real[0,1] = -20 + assert a[0,1].real == -20 + b = a.imag + assert b[1,2] == -1 + b[1,2] = 30 + assert a[1,2].imag == 30 def test_tolist_scalar(self): from _numpypy import int32, bool_ From noreply at buildbot.pypy.org Wed Jan 9 21:18:39 2013 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 9 Jan 2013 21:18:39 +0100 (CET) Subject: [pypy-commit] pypy numpypy-real-as-view: discover that descr_set_real requires a true view iterator, modify View iterators to allow dtype to differ from array.dtype Message-ID: <20130109201839.1CEB01C0041@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: numpypy-real-as-view Changeset: r59897:36d57babaa6e Date: 2013-01-09 21:50 +0200 http://bitbucket.org/pypy/pypy/changeset/36d57babaa6e/ Log: discover that descr_set_real requires a true view iterator, modify View iterators to allow dtype to differ from array.dtype diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py --- a/pypy/module/micronumpy/arrayimpl/concrete.py +++ b/pypy/module/micronumpy/arrayimpl/concrete.py @@ -13,6 +13,7 @@ from pypy.rlib.debug import make_sure_not_resized class ConcreteArrayIterator(base.BaseArrayIterator): + _immutable_fields_ = ['dtype', 'skip', 'size'] def __init__(self, array): self.array = array self.offset = 0 @@ -21,10 +22,10 @@ self.size = array.size def setitem(self, elem): - self.array.setitem(self.offset, elem) + self.dtype.setitem(self.array, self.offset, elem) def getitem(self): - return self.array.getitem(self.offset) + return self.dtype.getitem(self.array, self.offset) def getitem_bool(self): return self.dtype.getitem_bool(self.array, self.offset) @@ -42,13 +43,16 @@ self.offset %= self.size class OneDimViewIterator(ConcreteArrayIterator): - def __init__(self, array): + ''' The view iterator dtype can be different from the + array.dtype, this is what makes it a View + ''' + def __init__(self, array, dtype, start, strides, shape): self.array = array - self.offset = array.start - self.skip = array.get_strides()[0] - self.dtype = array.dtype + self.dtype = dtype + self.offset = start + self.skip = strides[0] self.index = 0 - self.size = array.get_shape()[0] + self.size = shape[0] def next(self): self.offset += self.skip @@ -65,9 +69,13 @@ self.offset %= self.size class MultiDimViewIterator(ConcreteArrayIterator): - def __init__(self, array, start, strides, backstrides, shape): + ''' The view iterator dtype can be different from the + array.dtype, this is what makes it a View + ''' + def __init__(self, array, dtype, start, strides, backstrides, shape): self.indexes = [0] * len(shape) self.array = array + self.dtype = dtype self.shape = shape self.offset = start self.shapelen = len(shape) @@ -114,7 +122,7 @@ self.offset %= self.size class AxisIterator(base.BaseArrayIterator): - def __init__(self, array, shape, dim): + def __init__(self, array, dtype, shape, dim): self.shape = shape strides = array.get_strides() backstrides = array.get_backstrides() @@ -131,12 +139,13 @@ self.offset = array.start self.dim = dim self.array = array + self.dtype = array.dtype def setitem(self, elem): - self.array.setitem(self.offset, elem) + self.dtype.setitem(self.array, self.offset, elem) def getitem(self): - return self.array.getitem(self.offset) + return self.dtype.getitem(self.array, self.offset) @jit.unroll_safe def next(self): @@ -403,7 +412,7 @@ def create_dot_iter(self, shape, skip): r = calculate_dot_strides(self.get_strides(), self.get_backstrides(), shape, skip) - return MultiDimViewIterator(self, self.start, r[0], r[1], shape) + return MultiDimViewIterator(self, self.dtype, self.start, r[0], r[1], shape) def swapaxes(self, axis1, axis2): shape = self.get_shape()[:] @@ -437,7 +446,7 @@ r = calculate_broadcast_strides(self.get_strides(), self.get_backstrides(), self.get_shape(), shape) - return MultiDimViewIterator(self, 0, r[0], r[1], shape) + return MultiDimViewIterator(self, self.dtype, 0, r[0], r[1], shape) def fill(self, box): self.dtype.fill(self.storage, box, 0, self.size) @@ -480,11 +489,12 @@ r = calculate_broadcast_strides(self.get_strides(), self.get_backstrides(), self.get_shape(), shape) - return MultiDimViewIterator(self.parent, + return MultiDimViewIterator(self.parent, self.dtype, self.start, r[0], r[1], shape) if len(self.get_shape()) == 1: - return OneDimViewIterator(self) - return MultiDimViewIterator(self.parent, self.start, + return OneDimViewIterator(self.parent, self.dtype, self.start, + self.get_strides(), self.get_shape()) + return MultiDimViewIterator(self.parent, self.dtype, self.start, self.get_strides(), self.get_backstrides(), self.get_shape()) diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -245,9 +245,10 @@ raise OperationError(space.w_NotImplementedError, space.wrap('imag not implemented for this dtype')) - def descr_set_real(self, space, w_new_val): + def descr_set_real(self, space, w_value): # copy (broadcast) values into self - self.implementation.descr_set_real(space, w_new_val) + tmp = self.implementation.get_real() + tmp.setslice(space, convert_to_array(space, w_value)) def descr_set_imag(self, space, w_new_val): # if possible, copy (broadcast) values into self diff --git a/pypy/module/micronumpy/test/test_iter.py b/pypy/module/micronumpy/test/test_iter.py --- a/pypy/module/micronumpy/test/test_iter.py +++ b/pypy/module/micronumpy/test/test_iter.py @@ -12,7 +12,7 @@ strides = [5, 1] backstrides = [x * (y - 1) for x,y in zip(strides, shape)] assert backstrides == [10, 4] - i = MultiDimViewIterator(MockArray, start, strides, backstrides, shape) + i = MultiDimViewIterator(MockArray, None, start, strides, backstrides, shape) i.next() i.next() i.next() @@ -30,7 +30,7 @@ strides = [1, 3] backstrides = [x * (y - 1) for x,y in zip(strides, shape)] assert backstrides == [2, 12] - i = MultiDimViewIterator(MockArray, start, strides, backstrides, shape) + i = MultiDimViewIterator(MockArray, None, start, strides, backstrides, shape) i.next() i.next() i.next() @@ -51,7 +51,7 @@ strides = [5, 1] backstrides = [x * (y - 1) for x,y in zip(strides, shape)] assert backstrides == [10, 4] - i = MultiDimViewIterator(MockArray, start, strides, backstrides, shape) + i = MultiDimViewIterator(MockArray, None, start, strides, backstrides, shape) i.next_skip_x(2) i.next_skip_x(2) i.next_skip_x(2) @@ -74,7 +74,7 @@ strides = [1, 3] backstrides = [x * (y - 1) for x,y in zip(strides, shape)] assert backstrides == [2, 12] - i = MultiDimViewIterator(MockArray, start, strides, backstrides, shape) + i = MultiDimViewIterator(MockArray, None, start, strides, backstrides, shape) i.next_skip_x(2) i.next_skip_x(2) i.next_skip_x(2) diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1368,6 +1368,11 @@ assert b[1,2] == -1 b[1,2] = 30 assert a[1,2].imag == 30 + a.real = 13 + assert a[1,1].real == 13 + a=array([1+1j, 2-3j, 4+5j, -6+7j, 8-9j, -2-1j]) + a.real = 13 + assert a[3].real == 13 def test_tolist_scalar(self): from _numpypy import int32, bool_ From noreply at buildbot.pypy.org Wed Jan 9 21:18:40 2013 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 9 Jan 2013 21:18:40 +0100 (CET) Subject: [pypy-commit] pypy numpypy-real-as-view: fixes, now tests pass Message-ID: <20130109201840.43DB71C0041@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: numpypy-real-as-view Changeset: r59898:5bfae8c3b2fc Date: 2013-01-09 22:18 +0200 http://bitbucket.org/pypy/pypy/changeset/5bfae8c3b2fc/ Log: fixes, now tests pass diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py --- a/pypy/module/micronumpy/arrayimpl/concrete.py +++ b/pypy/module/micronumpy/arrayimpl/concrete.py @@ -122,7 +122,7 @@ self.offset %= self.size class AxisIterator(base.BaseArrayIterator): - def __init__(self, array, dtype, shape, dim): + def __init__(self, array, shape, dim): self.shape = shape strides = array.get_strides() backstrides = array.get_backstrides() diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -161,8 +161,8 @@ alternate_constructors=[], aliases=[], fields=None, fieldnames=None, native=True, float_type=None): W_Dtype.__init__(self, itemtype, num, kind, name, char, w_box_type, - alternate_constructors=[], aliases=[], - fields=None, fieldnames=None, native=True) + alternate_constructors=alternate_constructors, aliases=aliases, + fields=fields, fieldnames=fieldnames, native=native) self.float_type = float_type def is_complex_type(self): From noreply at buildbot.pypy.org Wed Jan 9 21:31:24 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Wed, 9 Jan 2013 21:31:24 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed import in rpython/translator/cli/query.py Message-ID: <20130109203124.DBD7E1C0041@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59899:0043ae3d83f9 Date: 2013-01-09 20:02 +0100 http://bitbucket.org/pypy/pypy/changeset/0043ae3d83f9/ Log: Fixed import in rpython/translator/cli/query.py diff --git a/rpython/translator/cli/query.py b/rpython/translator/cli/query.py --- a/rpython/translator/cli/query.py +++ b/rpython/translator/cli/query.py @@ -65,7 +65,7 @@ def get_cachedir(): - from rpython import cache_dir + from rpython.conftest import cache_dir _cache = py.path.local(cache_dir).ensure(dir=1) return _cache From noreply at buildbot.pypy.org Wed Jan 9 21:31:26 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Wed, 9 Jan 2013 21:31:26 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Split up stdlib_opcode Message-ID: <20130109203126.3BAAF1C0041@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59900:d2ee25531ada Date: 2013-01-09 21:30 +0100 http://bitbucket.org/pypy/pypy/changeset/d2ee25531ada/ Log: Split up stdlib_opcode diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py --- a/pypy/interpreter/astcompiler/assemble.py +++ b/pypy/interpreter/astcompiler/assemble.py @@ -4,7 +4,7 @@ from pypy.interpreter.astcompiler import ast, symtable from pypy.interpreter import pycode -from rpython.tool import stdlib_opcode as ops +from pypy.tool import stdlib_opcode as ops from pypy.interpreter.error import OperationError from rpython.rlib.objectmodel import we_are_translated diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py --- a/pypy/interpreter/astcompiler/codegen.py +++ b/pypy/interpreter/astcompiler/codegen.py @@ -10,7 +10,7 @@ from pypy.interpreter.astcompiler import ast, assemble, symtable, consts, misc from pypy.interpreter.astcompiler import optimize # For side effects from pypy.interpreter.pyparser.error import SyntaxError -from rpython.tool import stdlib_opcode as ops +from pypy.tool import stdlib_opcode as ops from pypy.interpreter.error import OperationError diff --git a/pypy/interpreter/astcompiler/optimize.py b/pypy/interpreter/astcompiler/optimize.py --- a/pypy/interpreter/astcompiler/optimize.py +++ b/pypy/interpreter/astcompiler/optimize.py @@ -2,7 +2,7 @@ import sys from pypy.interpreter.astcompiler import ast, consts, misc -from rpython.tool import stdlib_opcode as ops +from pypy.tool import stdlib_opcode as ops from pypy.interpreter.error import OperationError from rpython.rlib.unroll import unrolling_iterable from rpython.rlib.runicode import MAXUNICODE diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py --- a/pypy/interpreter/astcompiler/test/test_compiler.py +++ b/pypy/interpreter/astcompiler/test/test_compiler.py @@ -4,7 +4,7 @@ from pypy.interpreter.pyparser.test import expressions from pypy.interpreter.pycode import PyCode from pypy.interpreter.pyparser.error import SyntaxError, IndentationError -from rpython.tool import stdlib_opcode as ops +from pypy.tool import stdlib_opcode as ops def compile_with_astcompiler(expr, mode, space): p = pyparse.PythonParser(space) diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -13,11 +13,11 @@ from pypy.interpreter.astcompiler.consts import ( CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS, CO_NESTED, CO_GENERATOR, CO_CONTAINSGLOBALS) +from pypy.tool.stdlib_opcode import opcodedesc, HAVE_ARGUMENT from rpython.rlib.rarithmetic import intmask from rpython.rlib.debug import make_sure_not_resized from rpython.rlib import jit from rpython.rlib.objectmodel import compute_hash -from rpython.tool.stdlib_opcode import opcodedesc, HAVE_ARGUMENT class BytecodeCorruption(Exception): diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py --- a/pypy/interpreter/pyframe.py +++ b/pypy/interpreter/pyframe.py @@ -12,7 +12,7 @@ from rpython.rlib.debug import make_sure_not_resized, check_nonneg from rpython.rlib.rarithmetic import intmask, r_uint from rpython.rlib import jit -from rpython.tool import stdlib_opcode +from pypy.tool import stdlib_opcode from rpython.tool.stdlib_opcode import host_bytecode_spec # Define some opcodes used diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -15,7 +15,7 @@ from rpython.rlib.rarithmetic import r_uint, intmask from rpython.rlib.unroll import unrolling_iterable from rpython.rlib.debug import check_nonneg -from rpython.tool.stdlib_opcode import (bytecode_spec, +from pypy.tool.stdlib_opcode import (bytecode_spec, unrolling_all_opcode_descs) def unaryoperation(operationname): diff --git a/pypy/module/_continuation/interp_pickle.py b/pypy/module/_continuation/interp_pickle.py --- a/pypy/module/_continuation/interp_pickle.py +++ b/pypy/module/_continuation/interp_pickle.py @@ -1,4 +1,4 @@ -from rpython.tool import stdlib_opcode as pythonopcode +from pypy.tool import stdlib_opcode as pythonopcode from rpython.rlib import jit from pypy.interpreter.error import OperationError from pypy.interpreter.pyframe import PyFrame diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py --- a/pypy/module/pypyjit/interp_jit.py +++ b/pypy/module/pypyjit/interp_jit.py @@ -27,7 +27,7 @@ JUMP_ABSOLUTE = opmap['JUMP_ABSOLUTE'] def get_printable_location(next_instr, is_being_profiled, bytecode): - from rpython.tool.stdlib_opcode import opcode_method_names + from pypy.tool.stdlib_opcode import opcode_method_names name = opcode_method_names[ord(bytecode.co_code[next_instr])] return '%s #%d %s' % (bytecode.get_repr(), next_instr, name) diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py --- a/pypy/module/sys/vm.py +++ b/pypy/module/sys/vm.py @@ -106,7 +106,7 @@ space.w_None]) def exc_info_direct(space, frame): - from rpython.tool import stdlib_opcode + from pypy.tool import stdlib_opcode # In order to make the JIT happy, we try to return (exc, val, None) # instead of (exc, val, tb). We can do that only if we recognize # the following pattern in the bytecode: diff --git a/pypy/tool/pydis.py b/pypy/tool/pydis.py --- a/pypy/tool/pydis.py +++ b/pypy/tool/pydis.py @@ -7,8 +7,8 @@ import sys -from rpython.tool import stdlib_opcode -from rpython.tool.stdlib_opcode import * +from pypy.tool import stdlib_opcode +from pypy.tool.stdlib_opcode import * __all__ = ["dis","pydisassemble","distb","disco"] + stdlib_opcode.__all__ diff --git a/rpython/tool/stdlib_opcode.py b/pypy/tool/stdlib_opcode.py rename from rpython/tool/stdlib_opcode.py rename to pypy/tool/stdlib_opcode.py --- a/rpython/tool/stdlib_opcode.py +++ b/pypy/tool/stdlib_opcode.py @@ -10,97 +10,10 @@ 'hasconst', 'hasname', 'hasjrel', 'hasjabs', 'haslocal', 'hascompare', 'hasfree', 'cmp_op'] -# ____________________________________________________________ -# RPython-friendly helpers and structures - -class _BaseOpcodeDesc(object): - def __init__(self, bytecode_spec, name, index, methodname): - self.bytecode_spec = bytecode_spec - self.name = name - self.methodname = methodname - self.index = index - self.hasarg = index >= self.HAVE_ARGUMENT - - def _freeze_(self): - return True - - def is_enabled(self, space): - """Check if the opcode should be enabled in the space's configuration. - (Returns True for all standard opcodes.)""" - opt = space.config.objspace.opcodes - return getattr(opt, self.name, True) - is_enabled._annspecialcase_ = 'specialize:memo' - - # for predictable results, we try to order opcodes most-used-first - opcodeorder = [124, 125, 100, 105, 1, 131, 116, 111, 106, 83, 23, 93, 113, 25, 95, 64, 112, 66, 102, 110, 60, 92, 62, 120, 68, 87, 32, 136, 4, 103, 24, 63, 18, 65, 15, 55, 121, 3, 101, 22, 12, 80, 86, 135, 126, 90, 140, 104, 2, 33, 20, 108, 107, 31, 134, 132, 88, 30, 133, 130, 137, 141, 61, 122, 11, 40, 74, 73, 51, 96, 21, 42, 56, 85, 82, 89, 142, 77, 78, 79, 91, 76, 97, 57, 19, 43, 84, 50, 41, 99, 53, 26] - - def sortkey(self): - try: - i = self.opcodeorder.index(self.index) - except ValueError: - i = 1000000 - return i, self.index - - def __cmp__(self, other): - return (cmp(self.__class__, other.__class__) or - cmp(self.sortkey(), other.sortkey())) - - def __str__(self): - return "" % (self.index, self.name, id(self)) - - __repr__ = __str__ - -class _baseopcodedesc: - """A namespace mapping OPCODE_NAME to _BaseOpcodeDescs.""" - pass - - -class BytecodeSpec(object): - """A bunch of mappings describing a bytecode instruction set.""" - - def __init__(self, name, opmap, HAVE_ARGUMENT): - """NOT_RPYTHON.""" - class OpcodeDesc(_BaseOpcodeDesc): - HAVE_ARGUMENT = HAVE_ARGUMENT - class opcodedesc(_baseopcodedesc): - """A namespace mapping OPCODE_NAME to OpcodeDescs.""" - - self.name = name - self.OpcodeDesc = OpcodeDesc - self.opcodedesc = opcodedesc - self.HAVE_ARGUMENT = HAVE_ARGUMENT - # opname -> opcode - self.opmap = opmap - # opcode -> method name - self.method_names = tbl = ['MISSING_OPCODE'] * 256 - # opcode -> opdesc - self.opdescmap = {} - for name, index in opmap.items(): - tbl[index] = methodname = name.replace('+', '_') - desc = OpcodeDesc(self, name, index, methodname) - setattr(self.opcodedesc, name, desc) - self.opdescmap[index] = desc - # fill the ordered opdesc list - self.ordered_opdescs = lst = self.opdescmap.values() - lst.sort() - - def to_globals(self): - """NOT_RPYTHON. Add individual opcodes to the module constants.""" - g = globals() - g.update(self.opmap) - g['SLICE'] = self.opmap["SLICE+0"] - g['STORE_SLICE'] = self.opmap["STORE_SLICE+0"] - g['DELETE_SLICE'] = self.opmap["DELETE_SLICE+0"] - - def __str__(self): - return "<%s bytecode>" % (self.name,) - - __repr__ = __str__ - - # Initialization from rpython.rlib.unroll import unrolling_iterable +from rpython.tool.stdlib_opcode import BytecodeSpec, host_bytecode_spec from opcode import ( opmap as host_opmap, HAVE_ARGUMENT as host_HAVE_ARGUMENT) @@ -119,7 +32,6 @@ del load_pypy_opcode bytecode_spec = BytecodeSpec('pypy', opmap, HAVE_ARGUMENT) -host_bytecode_spec = BytecodeSpec('host', host_opmap, host_HAVE_ARGUMENT) bytecode_spec.to_globals() opcode_method_names = bytecode_spec.method_names diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py --- a/rpython/flowspace/bytecode.py +++ b/rpython/flowspace/bytecode.py @@ -1,8 +1,8 @@ """ Bytecode handling classes and functions for use by the flow space. """ -from rpython.tool.stdlib_opcode import (host_bytecode_spec, EXTENDED_ARG, - HAVE_ARGUMENT) +from rpython.tool.stdlib_opcode import host_bytecode_spec +from opcode import EXTENDED_ARG, HAVE_ARGUMENT from rpython.flowspace.argument import Signature from rpython.flowspace.flowcontext import BytecodeCorruption From noreply at buildbot.pypy.org Wed Jan 9 21:48:02 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Wed, 9 Jan 2013 21:48:02 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Forgot to add rpython/tool/stdlib_opcode.py Message-ID: <20130109204802.4052C1C0041@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59901:da5f04dfa943 Date: 2013-01-09 21:47 +0100 http://bitbucket.org/pypy/pypy/changeset/da5f04dfa943/ Log: Forgot to add rpython/tool/stdlib_opcode.py diff --git a/rpython/tool/stdlib_opcode.py b/rpython/tool/stdlib_opcode.py new file mode 100644 --- /dev/null +++ b/rpython/tool/stdlib_opcode.py @@ -0,0 +1,87 @@ +class _BaseOpcodeDesc(object): + def __init__(self, bytecode_spec, name, index, methodname): + self.bytecode_spec = bytecode_spec + self.name = name + self.methodname = methodname + self.index = index + self.hasarg = index >= self.HAVE_ARGUMENT + + def _freeze_(self): + return True + + def is_enabled(self, space): + """Check if the opcode should be enabled in the space's configuration. + (Returns True for all standard opcodes.)""" + opt = space.config.objspace.opcodes + return getattr(opt, self.name, True) + is_enabled._annspecialcase_ = 'specialize:memo' + + # for predictable results, we try to order opcodes most-used-first + opcodeorder = [124, 125, 100, 105, 1, 131, 116, 111, 106, 83, 23, 93, 113, 25, 95, 64, 112, 66, 102, 110, 60, 92, 62, 120, 68, 87, 32, 136, 4, 103, 24, 63, 18, 65, 15, 55, 121, 3, 101, 22, 12, 80, 86, 135, 126, 90, 140, 104, 2, 33, 20, 108, 107, 31, 134, 132, 88, 30, 133, 130, 137, 141, 61, 122, 11, 40, 74, 73, 51, 96, 21, 42, 56, 85, 82, 89, 142, 77, 78, 79, 91, 76, 97, 57, 19, 43, 84, 50, 41, 99, 53, 26] + + def sortkey(self): + try: + i = self.opcodeorder.index(self.index) + except ValueError: + i = 1000000 + return i, self.index + + def __cmp__(self, other): + return (cmp(self.__class__, other.__class__) or + cmp(self.sortkey(), other.sortkey())) + + def __str__(self): + return "" % (self.index, self.name, id(self)) + + __repr__ = __str__ + +class _baseopcodedesc: + """A namespace mapping OPCODE_NAME to _BaseOpcodeDescs.""" + pass + + +class BytecodeSpec(object): + """A bunch of mappings describing a bytecode instruction set.""" + + def __init__(self, name, opmap, HAVE_ARGUMENT): + """NOT_RPYTHON.""" + class OpcodeDesc(_BaseOpcodeDesc): + HAVE_ARGUMENT = HAVE_ARGUMENT + class opcodedesc(_baseopcodedesc): + """A namespace mapping OPCODE_NAME to OpcodeDescs.""" + + self.name = name + self.OpcodeDesc = OpcodeDesc + self.opcodedesc = opcodedesc + self.HAVE_ARGUMENT = HAVE_ARGUMENT + # opname -> opcode + self.opmap = opmap + # opcode -> method name + self.method_names = tbl = ['MISSING_OPCODE'] * 256 + # opcode -> opdesc + self.opdescmap = {} + for name, index in opmap.items(): + tbl[index] = methodname = name.replace('+', '_') + desc = OpcodeDesc(self, name, index, methodname) + setattr(self.opcodedesc, name, desc) + self.opdescmap[index] = desc + # fill the ordered opdesc list + self.ordered_opdescs = lst = self.opdescmap.values() + lst.sort() + + def to_globals(self): + """NOT_RPYTHON. Add individual opcodes to the module constants.""" + g = globals() + g.update(self.opmap) + g['SLICE'] = self.opmap["SLICE+0"] + g['STORE_SLICE'] = self.opmap["STORE_SLICE+0"] + g['DELETE_SLICE'] = self.opmap["DELETE_SLICE+0"] + + def __str__(self): + return "<%s bytecode>" % (self.name,) + + __repr__ = __str__ + +from opcode import opmap, HAVE_ARGUMENT + +host_bytecode_spec = BytecodeSpec('host', opmap, HAVE_ARGUMENT) \ No newline at end of file From noreply at buildbot.pypy.org Wed Jan 9 22:25:47 2013 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 9 Jan 2013 22:25:47 +0100 (CET) Subject: [pypy-commit] pypy numpypy-real-as-view: test, implement forgotten method Message-ID: <20130109212547.1533C1C0041@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: numpypy-real-as-view Changeset: r59902:7e8979164700 Date: 2013-01-09 23:25 +0200 http://bitbucket.org/pypy/pypy/changeset/7e8979164700/ Log: test, implement forgotten method diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -250,9 +250,13 @@ tmp = self.implementation.get_real() tmp.setslice(space, convert_to_array(space, w_value)) - def descr_set_imag(self, space, w_new_val): + def descr_set_imag(self, space, w_value): # if possible, copy (broadcast) values into self - self.implementation.descr_set_imag(space, w_new_val) + if not self.get_dtype().is_complex_type(): + raise OperationError(space.w_TypeError, + space.wrap('array does not have imaginary part to set')) + tmp = self.implementation.get_imag() + tmp.setslice(space, convert_to_array(space, w_value)) def descr_reshape(self, space, args_w): """reshape(...) diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1354,12 +1354,15 @@ b = a.imag assert b[7] == 0 raises(RuntimeError, 'b[7] = -2') + raises(TypeError, 'a.imag = -2') a = array(['abc','def'],dtype='S3') b = a.real assert a[0] == b[0] assert a[1] == b[1] b[1] = 'xyz' assert a[1] == 'xyz' + assert a.imag[0] == 'abc' + raises(TypeError, 'a.imag = "qop"') a=array([[1+1j, 2-3j, 4+5j],[-6+7j, 8-9j, -2-1j]]) assert a.real[0,1] == 2 a.real[0,1] = -20 @@ -1373,6 +1376,10 @@ a=array([1+1j, 2-3j, 4+5j, -6+7j, 8-9j, -2-1j]) a.real = 13 assert a[3].real == 13 + a.imag = -5 + a.imag[3] = -10 + assert a[3].imag == -10 + assert a[2].imag == -5 def test_tolist_scalar(self): from _numpypy import int32, bool_ From noreply at buildbot.pypy.org Wed Jan 9 22:36:25 2013 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 9 Jan 2013 22:36:25 +0100 (CET) Subject: [pypy-commit] pypy numpypy-real-as-view: merge default into branch Message-ID: <20130109213625.B19CD1C0041@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: numpypy-real-as-view Changeset: r59903:c95c67c88a2b Date: 2013-01-09 23:36 +0200 http://bitbucket.org/pypy/pypy/changeset/c95c67c88a2b/ Log: merge default into branch diff too long, truncating to 2000 out of 11246 lines diff --git a/lib-python/2.7/sre_parse.py b/lib-python/2.7/sre_parse.py --- a/lib-python/2.7/sre_parse.py +++ b/lib-python/2.7/sre_parse.py @@ -16,6 +16,12 @@ from sre_constants import * +try: + from __pypy__ import newdict +except ImportError: + def newdict(tp): + return {} + SPECIAL_CHARS = ".\\[{()*+?^$|" REPEAT_CHARS = "*+?{" @@ -68,7 +74,7 @@ self.flags = 0 self.open = [] self.groups = 1 - self.groupdict = {} + self.groupdict = newdict("module") def opengroup(self, name=None): gid = self.groups self.groups = gid + 1 diff --git a/pypy/doc/Makefile b/pypy/doc/Makefile --- a/pypy/doc/Makefile +++ b/pypy/doc/Makefile @@ -32,38 +32,38 @@ -rm -rf $(BUILDDIR)/* html: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." pickle: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ @@ -73,7 +73,7 @@ @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/PyPy.qhc" latex: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @@ -81,26 +81,26 @@ "run these through (pdf)latex." man: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man" changes: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." doctest: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -38,7 +38,7 @@ # General information about the project. project = u'PyPy' -copyright = u'2011, The PyPy Project' +copyright = u'2013, The PyPy Project' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/pypy/doc/config/translation.no__thread.txt b/pypy/doc/config/translation.no__thread.txt --- a/pypy/doc/config/translation.no__thread.txt +++ b/pypy/doc/config/translation.no__thread.txt @@ -1,3 +1,3 @@ Don't use gcc __thread attribute for fast thread local storage -implementation . Increases the chance that moving the resulting +implementation. Increases the chance that moving the resulting executable to another same processor Linux machine will work. diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst --- a/pypy/doc/index.rst +++ b/pypy/doc/index.rst @@ -67,7 +67,7 @@ .. _Python: http://docs.python.org/index.html .. _`more...`: architecture.html#mission-statement .. _`PyPy blog`: http://morepypy.blogspot.com/ -.. _`development bug/feature tracker`: https://codespeak.net/issue/pypy-dev/ +.. _`development bug/feature tracker`: https://bugs.pypy.org .. _here: http://tismerysoft.de/pypy/irc-logs/pypy .. _`Mercurial commit mailing list`: http://python.org/mailman/listinfo/pypy-commit .. _`development mailing list`: http://python.org/mailman/listinfo/pypy-dev diff --git a/pypy/doc/pypyconfig.py b/pypy/doc/pypyconfig.py --- a/pypy/doc/pypyconfig.py +++ b/pypy/doc/pypyconfig.py @@ -3,6 +3,10 @@ def setup(app): import sys, os sys.path.insert(0, os.path.abspath("../../")) + + #Autmatically calls make_cmdlline_overview + from pypy.doc.config import generate + from pypy.config import makerestdoc import py role = makerestdoc.register_config_role(py.path.local()) diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -5,6 +5,11 @@ .. this is a revision shortly after release-2.0-beta1 .. startrev: 0e6161a009c6 +.. branch: callback-jit +Callbacks from C are now better JITted + +.. branch: remove-globals-in-jit + .. branch: length-hint Implement __lenght_hint__ according to PEP 424 @@ -28,5 +33,7 @@ .. branch: remove-PYPY_NOT_MAIN_FILE +.. branch: missing-jit-operations + .. branch: fix-lookinside-iff-oopspec Fixed the interaction between two internal tools for controlling the JIT. diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -56,11 +56,24 @@ ---------------------------- On Windows, there is no standard place where to download, build and -install third-party libraries. We chose to install them in the parent +install third-party libraries. We recommend installing them in the parent directory of the pypy checkout. For example, if you installed pypy in ``d:\pypy\trunk\`` (This directory contains a README file), the base -directory is ``d:\pypy``. You may choose different values by setting the -INCLUDE, LIB and PATH (for DLLs) +directory is ``d:\pypy``. You must then set the +INCLUDE, LIB and PATH (for DLLs) environment variables appropriately. + +Abridged method (for -Ojit builds using Visual Studio 2008) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Download the versions of all the external packages +from +https://bitbucket.org/pypy/pypy/downloads/local.zip +Then expand it into the base directory (base_dir) and modify your environment to reflect this:: + + set PATH=\bin;%PATH% + set INCLUDE=\include;%INCLUDE% + set LIB=\lib;%LIB% + +Now you should be good to go. Read on for more information. The Boehm garbage collector ~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py --- a/pypy/interpreter/typedef.py +++ b/pypy/interpreter/typedef.py @@ -105,6 +105,13 @@ # typical usage (case 1 is the most common kind of app-level subclasses; # case 2 is the memory-saving kind defined with __slots__). # +# +----------------------------------------------------------------+ +# | NOTE: if withmapdict is enabled, the following doesn't apply! | +# | Map dicts can flexibly allow any slots/__dict__/__weakref__ to | +# | show up only when needed. In particular there is no way with | +# | mapdict to prevent some objects from being weakrefable. | +# +----------------------------------------------------------------+ +# # dict slots del weakrefable # # 1. Y N N Y UserDictWeakref diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py --- a/pypy/jit/backend/arm/assembler.py +++ b/pypy/jit/backend/arm/assembler.py @@ -1,5 +1,6 @@ from __future__ import with_statement import os +from pypy.jit.backend.llsupport import jitframe from pypy.jit.backend.arm.helper.assembler import saved_registers from pypy.jit.backend.arm import conditions as c from pypy.jit.backend.arm import registers as r @@ -25,12 +26,12 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.jit.backend.arm.opassembler import ResOpAssembler from pypy.rlib.debug import (debug_print, debug_start, debug_stop, - have_debug_prints) + have_debug_prints, fatalerror) from pypy.rlib.jit import AsmInfo from pypy.rlib.objectmodel import compute_unique_id # XXX Move to llsupport -from pypy.jit.backend.x86.support import values_array, memcpy_fn +from pypy.jit.backend.x86.support import memcpy_fn DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', ('i', lltype.Signed), ('type', lltype.Char), # 'b'ridge, 'l'abel or @@ -44,14 +45,8 @@ debug = True - def __init__(self, cpu, failargs_limit=1000): + def __init__(self, cpu, translate_support_code=False): self.cpu = cpu - self.fail_boxes_int = values_array(lltype.Signed, failargs_limit) - self.fail_boxes_float = values_array(longlong.FLOATSTORAGE, - failargs_limit) - self.fail_boxes_ptr = values_array(llmemory.GCREF, failargs_limit) - self.fail_boxes_count = 0 - self.fail_force_index = 0 self.setup_failure_recovery() self.mc = None self.memcpy_addr = 0 @@ -68,6 +63,7 @@ self._debug = False self.loop_run_counters = [] self.debug_counter_descr = cpu.fielddescrof(DEBUG_COUNTER, 'i') + self.force_token_to_dead_frame = {} # XXX temporary hack def set_debug(self, v): r = self._debug @@ -112,9 +108,13 @@ gc_ll_descr.initialize() self._build_wb_slowpath(False) self._build_wb_slowpath(True) + self._build_failure_recovery(exc=True, withfloats=False) + self._build_failure_recovery(exc=False, withfloats=False) if self.cpu.supports_floats: self._build_wb_slowpath(False, withfloats=True) self._build_wb_slowpath(True, withfloats=True) + self._build_failure_recovery(exc=True, withfloats=True) + self._build_failure_recovery(exc=False, withfloats=True) self._build_propagate_exception_path() if gc_ll_descr.get_malloc_slowpath_addr is not None: self._build_malloc_slowpath() @@ -122,10 +122,7 @@ if gc_ll_descr.gcrootmap and gc_ll_descr.gcrootmap.is_shadow_stack: self._build_release_gil(gc_ll_descr.gcrootmap) self.memcpy_addr = self.cpu.cast_ptr_to_int(memcpy_fn) - self._exit_code_addr = self._gen_exit_path() - self._leave_jitted_hook_save_exc = \ - self._gen_leave_jitted_hook_code(True) - self._leave_jitted_hook = self._gen_leave_jitted_hook_code(False) + if not self._debug: # if self._debug is already set it means that someone called # set_debug by hand before initializing the assembler. Leave it @@ -220,36 +217,17 @@ self.releasegil_addr = rffi.cast(lltype.Signed, releasegil_func) self.reacqgil_addr = rffi.cast(lltype.Signed, reacqgil_func) - def _gen_leave_jitted_hook_code(self, save_exc): - mc = ARMv7Builder() - if self.cpu.supports_floats: - floats = r.caller_vfp_resp - else: - floats = [] - with saved_registers(mc, r.caller_resp + [r.lr], floats): - addr = self.cpu.get_on_leave_jitted_int(save_exception=save_exc) - mc.BL(addr) - assert self._exit_code_addr != 0 - mc.B(self._exit_code_addr) - return mc.materialize(self.cpu.asmmemmgr, [], - self.cpu.gc_ll_descr.gcrootmap) - def _build_propagate_exception_path(self): if self.cpu.propagate_exception_v < 0: return # not supported (for tests, or non-translated) # mc = ARMv7Builder() - # call on_leave_jitted_save_exc() - if self.cpu.supports_floats: - floats = r.caller_vfp_resp - else: - floats = [] - with saved_registers(mc, r.caller_resp + [r.lr], floats): - addr = self.cpu.get_on_leave_jitted_int(save_exception=True, - default_to_memoryerror=True) - mc.BL(addr) - mc.gen_load_int(r.ip.value, self.cpu.propagate_exception_v) - mc.MOV_rr(r.r0.value, r.ip.value) + # + # Call the helper, which will return a dead frame object with + # the correct exception set, or MemoryError by default + # XXX make sure we return the correct value here + addr = rffi.cast(lltype.Signed, self.cpu.get_propagate_exception()) + mc.BL(addr) self.gen_func_epilog(mc=mc) self.propagate_exception_path = mc.materialize(self.cpu.asmmemmgr, []) @@ -282,12 +260,12 @@ # restore registers and return # We check for c.EQ here, meaning all bits zero in this case mc.POP([reg.value for reg in r.argument_regs] + [r.pc.value], cond=c.EQ) - # call on_leave_jitted_save_exc() - addr = self.cpu.get_on_leave_jitted_int(save_exception=True) + # + # Call the helper, which will return a dead frame object with + # the correct exception set, or MemoryError by default + addr = rffi.cast(lltype.Signed, self.cpu.get_propagate_exception()) mc.BL(addr) # - mc.gen_load_int(r.r0.value, self.cpu.propagate_exception_v) - # # footer -- note the ADD, which skips the return address of this # function, and will instead return to the caller's caller. Note # also that we completely ignore the saved arguments, because we @@ -339,39 +317,82 @@ def setup_failure_recovery(self): - @rgc.no_collect + #@rgc.no_collect -- XXX still true, but hacked gc_set_extra_threshold def failure_recovery_func(mem_loc, frame_pointer, stack_pointer): """mem_loc is a structure in memory describing where the values for the failargs are stored. frame loc is the address of the frame pointer for the frame to be decoded frame """ - vfp_registers = rffi.cast(rffi.LONGLONGP, stack_pointer) - registers = rffi.ptradd(vfp_registers, len(r.all_vfp_regs)) + vfp_registers = rffi.cast(rffi.LONGP, stack_pointer) + registers = rffi.ptradd(vfp_registers, 2*len(r.all_vfp_regs)) registers = rffi.cast(rffi.LONGP, registers) - return self.decode_registers_and_descr(mem_loc, frame_pointer, + bytecode = rffi.cast(rffi.UCHARP, mem_loc) + return self.grab_frame_values(self.cpu, bytecode, frame_pointer, registers, vfp_registers) + self.failure_recovery_code = [0, 0, 0, 0] self.failure_recovery_func = failure_recovery_func - recovery_func_sign = lltype.Ptr(lltype.FuncType([lltype.Signed] * 3, - lltype.Signed)) + _FAILURE_RECOVERY_FUNC = lltype.Ptr(lltype.FuncType([rffi.LONGP] * 3, + llmemory.GCREF)) - @rgc.no_collect - def decode_registers_and_descr(self, mem_loc, frame_pointer, + @staticmethod + #@rgc.no_collect -- XXX still true, but hacked gc_set_extra_threshold + def grab_frame_values(cpu, bytecode, frame_pointer, registers, vfp_registers): - """Decode locations encoded in memory at mem_loc and write the values - to the failboxes. Values for spilled vars and registers are stored on - stack at frame_loc """ - assert frame_pointer & 1 == 0 - self.fail_force_index = frame_pointer - bytecode = rffi.cast(rffi.UCHARP, mem_loc) + # no malloc allowed here!! xxx apart from one, hacking a lot + force_index = rffi.cast(lltype.Signed, frame_pointer) num = 0 - value = 0 - fvalue = 0 + deadframe = lltype.nullptr(jitframe.DEADFRAME) + # step 1: lots of mess just to count the final value of 'num' + bytecode1 = bytecode + while 1: + code = rffi.cast(lltype.Signed, bytecode1[0]) + bytecode1 = rffi.ptradd(bytecode1, 1) + if code >= AssemblerARM.CODE_FROMSTACK: + while code > 0x7F: + code = rffi.cast(lltype.Signed, bytecode1[0]) + bytecode1 = rffi.ptradd(bytecode1, 1) + else: + kind = code & 3 + if kind == AssemblerARM.DESCR_SPECIAL: + if code == AssemblerARM.CODE_HOLE: + num += 1 + continue + if code == AssemblerARM.CODE_INPUTARG: + continue + if code == AssemblerARM.CODE_FORCED: + # resuming from a GUARD_NOT_FORCED + token = force_index + deadframe = ( + cpu.assembler.force_token_to_dead_frame.pop(token)) + deadframe = lltype.cast_opaque_ptr( + jitframe.DEADFRAMEPTR, deadframe) + continue + assert code == AssemblerARM.CODE_STOP + break + num += 1 + + # allocate the deadframe + if not deadframe: + # Remove the "reserve" at the end of the nursery. This means + # that it is guaranteed that the following malloc() works + # without requiring a collect(), but it needs to be re-added + # as soon as possible. + cpu.gc_clear_extra_threshold() + assert num <= cpu.get_failargs_limit() + try: + deadframe = lltype.malloc(jitframe.DEADFRAME, num) + except MemoryError: + fatalerror("memory usage error in grab_frame_values") + # fill it code_inputarg = False - while True: + num = 0 + value_hi = 0 + while 1: + # decode the next instruction from the bytecode code = rffi.cast(lltype.Signed, bytecode[0]) bytecode = rffi.ptradd(bytecode, 1) - if code >= self.CODE_FROMSTACK: + if code >= AssemblerARM.CODE_FROMSTACK: if code > 0x7F: shift = 7 code &= 0x7F @@ -384,54 +405,63 @@ break # load the value from the stack kind = code & 3 - code = int((code - self.CODE_FROMSTACK) >> 2) + code = (code - AssemblerARM.CODE_FROMSTACK) >> 2 if code_inputarg: code = ~code code_inputarg = False - if kind == self.DESCR_FLOAT: - # we use code + 1 to get the hi word of the double worded float - stackloc = frame_pointer - get_fp_offset(int(code) + 1) - assert stackloc & 3 == 0 - fvalue = rffi.cast(rffi.LONGLONGP, stackloc)[0] - else: - stackloc = frame_pointer - get_fp_offset(int(code)) - assert stackloc & 1 == 0 - value = rffi.cast(rffi.LONGP, stackloc)[0] + stackloc = force_index - get_fp_offset(int(code)) + value = rffi.cast(rffi.LONGP, stackloc)[0] + if kind == AssemblerARM.DESCR_FLOAT: + assert WORD == 4 + value_hi = value + value = rffi.cast(rffi.LONGP, stackloc - WORD)[0] else: - # 'code' identifies a register: load its value kind = code & 3 - if kind == self.DESCR_SPECIAL: - if code == self.CODE_HOLE: + if kind == AssemblerARM.DESCR_SPECIAL: + if code == AssemblerARM.CODE_HOLE: num += 1 continue - if code == self.CODE_INPUTARG: + if code == AssemblerARM.CODE_INPUTARG: code_inputarg = True continue - assert code == self.CODE_STOP + if code == AssemblerARM.CODE_FORCED: + continue + assert code == AssemblerARM.CODE_STOP break + # 'code' identifies a register: load its value code >>= 2 - if kind == self.DESCR_FLOAT: - fvalue = vfp_registers[code] + if kind == AssemblerARM.DESCR_FLOAT: + if WORD == 4: + value = vfp_registers[2*code] + value_hi = vfp_registers[2*code + 1] + else: + value = registers[code] else: value = registers[code] # store the loaded value into fail_boxes_ - if kind == self.DESCR_FLOAT: - tgt = self.fail_boxes_float.get_addr_for_num(num) - rffi.cast(rffi.LONGLONGP, tgt)[0] = fvalue + if kind == AssemblerARM.DESCR_INT: + deadframe.jf_values[num].int = value + elif kind == AssemblerARM.DESCR_REF: + deadframe.jf_values[num].ref = rffi.cast(llmemory.GCREF, value) + elif kind == AssemblerARM.DESCR_FLOAT: + assert WORD == 4 + assert not longlong.is_64_bit + floatvalue = rffi.cast(lltype.SignedLongLong, value_hi) + floatvalue <<= 32 + floatvalue |= rffi.cast(lltype.SignedLongLong, + rffi.cast(lltype.Unsigned, value)) + deadframe.jf_values[num].float = floatvalue else: - if kind == self.DESCR_INT: - tgt = self.fail_boxes_int.get_addr_for_num(num) - elif kind == self.DESCR_REF: - assert (value & 3) == 0, "misaligned pointer" - tgt = self.fail_boxes_ptr.get_addr_for_num(num) - else: - assert 0, "bogus kind" - rffi.cast(rffi.LONGP, tgt)[0] = value + assert 0, "bogus kind" num += 1 - self.fail_boxes_count = num + # + assert num == len(deadframe.jf_values) + if not we_are_translated(): + assert bytecode[4] == 0xCC fail_index = rffi.cast(rffi.INTP, bytecode)[0] - fail_index = rffi.cast(lltype.Signed, fail_index) - return fail_index + fail_descr = cpu.get_fail_descr_from_number(fail_index) + deadframe.jf_descr = fail_descr.hide(cpu) + return lltype.cast_opaque_ptr(llmemory.GCREF, deadframe) def decode_inputargs(self, code): descr_to_box_type = [REF, INT, FLOAT] @@ -514,12 +544,26 @@ self.mc.CMP_ri(r.r0.value, 0) self.mc.B(self.propagate_exception_path, c=c.EQ) - def _gen_exit_path(self): + def _build_failure_recovery(self, exc, withfloats=False): mc = ARMv7Builder() - decode_registers_addr = llhelper(self.recovery_func_sign, + failure_recovery = llhelper(self._FAILURE_RECOVERY_FUNC, self.failure_recovery_func) self._insert_checks(mc) - with saved_registers(mc, r.all_regs, r.all_vfp_regs): + if withfloats: + f = r.all_vfp_regs + else: + f = [] + with saved_registers(mc, r.all_regs, f): + if exc: + # We might have an exception pending. Load it into r4 + # (this is a register saved across calls) + mc.gen_load_int(r.r5.value, self.cpu.pos_exc_value()) + mc.LDR_ri(r.r4.value, self.cpu.pos_exc_value()) + # clear the exc flags + mc.gen_load_int(r.r6.value, 0) + mc.STR_ri(r.r6.value, r.r5.value) + mc.gen_load_int(r.r5.value, self.cpu.pos_exception()) + mc.STR_ri(r.r6.value, r.r5.value) # move mem block address, to r0 to pass as mc.MOV_rr(r.r0.value, r.lr.value) # pass the current frame pointer as second param @@ -527,23 +571,32 @@ # pass the current stack pointer as third param mc.MOV_rr(r.r2.value, r.sp.value) self._insert_checks(mc) - mc.BL(rffi.cast(lltype.Signed, decode_registers_addr)) + mc.BL(rffi.cast(lltype.Signed, failure_recovery)) + if exc: + # save ebx into 'jf_guard_exc' + from pypy.jit.backend.llsupport.descr import unpack_fielddescr + descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) + offset, size, _ = unpack_fielddescr(descrs.jf_guard_exc) + mc.STR_rr(r.r4.value, r.r0.value, offset, cond=c.AL) mc.MOV_rr(r.ip.value, r.r0.value) mc.MOV_rr(r.r0.value, r.ip.value) self.gen_func_epilog(mc=mc) - return mc.materialize(self.cpu.asmmemmgr, [], + rawstart = mc.materialize(self.cpu.asmmemmgr, [], self.cpu.gc_ll_descr.gcrootmap) + self.failure_recovery_code[exc + 2 * withfloats] = rawstart + self.mc = None DESCR_REF = 0x00 DESCR_INT = 0x01 DESCR_FLOAT = 0x02 DESCR_SPECIAL = 0x03 CODE_FROMSTACK = 64 - CODE_STOP = 0 | DESCR_SPECIAL - CODE_HOLE = 4 | DESCR_SPECIAL - CODE_INPUTARG = 8 | DESCR_SPECIAL + CODE_STOP = 0 | DESCR_SPECIAL + CODE_HOLE = 4 | DESCR_SPECIAL + CODE_INPUTARG = 8 | DESCR_SPECIAL + CODE_FORCED = 12 | DESCR_SPECIAL #XXX where should this be written? - def gen_descr_encoding(self, descr, failargs, locs): + def write_failure_recovery_description(self, descr, failargs, locs): assert self.mc is not None for i in range(len(failargs)): arg = failargs[i] @@ -575,26 +628,33 @@ self.mc.writechar(chr(n)) self.mc.writechar(chr(self.CODE_STOP)) - fdescr = self.cpu.get_fail_descr_number(descr) - self.mc.write32(fdescr) + + def generate_quick_failure(self, guardtok, fcond=c.AL): + assert isinstance(guardtok.save_exc, bool) + fail_index = self.cpu.get_fail_descr_number(guardtok.descr) + startpos = self.mc.currpos() + withfloats = False + for box in guardtok.failargs: + if box is not None and box.type == FLOAT: + withfloats = True + break + exc = guardtok.save_exc + target = self.failure_recovery_code[exc + 2 * withfloats] + assert target != 0 + self.mc.BL(target) + # write tight data that describes the failure recovery + if guardtok.is_guard_not_forced: + self.mc.writechar(chr(self.CODE_FORCED)) + self.write_failure_recovery_description(guardtok.descr, + guardtok.failargs, guardtok.faillocs[1:]) + self.mc.write32(fail_index) + # for testing the decoding, write a final byte 0xCC + if not we_are_translated(): + self.mc.writechar('\xCC') + faillocs = [loc for loc in guardtok.faillocs if loc is not None] + guardtok.descr._arm_debug_faillocs = faillocs self.align() - - # assert that the fail_boxes lists are big enough - assert len(failargs) <= self.fail_boxes_int.SIZE - - def _gen_path_to_exit_path(self, descr, args, arglocs, - save_exc, fcond=c.AL): - assert isinstance(save_exc, bool) - self.gen_exit_code(self.mc, save_exc, fcond) - self.gen_descr_encoding(descr, args, arglocs[1:]) - - def gen_exit_code(self, mc, save_exc, fcond=c.AL): - assert isinstance(save_exc, bool) - if save_exc: - path = self._leave_jitted_hook_save_exc - else: - path = self._leave_jitted_hook - mc.BL(path) + return startpos def align(self): while(self.mc.currpos() % FUNC_ALIGN != 0): @@ -841,15 +901,10 @@ def write_pending_failure_recoveries(self): for tok in self.pending_guards: - descr = tok.descr #generate the exit stub and the encoded representation - pos = self.mc.currpos() - tok.pos_recovery_stub = pos - - self._gen_path_to_exit_path(descr, tok.failargs, - tok.faillocs, save_exc=tok.save_exc) + tok.pos_recovery_stub = self.generate_quick_failure(tok) # store info on the descr - descr._arm_current_frame_depth = tok.faillocs[0].getint() + tok.descr._arm_current_frame_depth = tok.faillocs[0].getint() def process_pending_guards(self, block_start): clt = self.current_clt @@ -860,7 +915,7 @@ descr._arm_failure_recovery_block = failure_recovery_pos relative_offset = tok.pos_recovery_stub - tok.offset guard_pos = block_start + tok.offset - if not tok.is_invalidate: + if not tok.is_guard_not_invalidated: # patch the guard jumpt to the stub # overwrite the generate NOP with a B_offs to the pos of the # stub @@ -1255,11 +1310,6 @@ else: raise AssertionError('Trying to pop to an invalid location') - def leave_jitted_hook(self): - ptrs = self.fail_boxes_ptr.ar - llop.gc_assume_young_pointers(lltype.Void, - llmemory.cast_ptr_to_adr(ptrs)) - def malloc_cond(self, nursery_free_adr, nursery_top_adr, size): assert size & (WORD-1) == 0 # must be correctly aligned diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py --- a/pypy/jit/backend/arm/opassembler.py +++ b/pypy/jit/backend/arm/opassembler.py @@ -16,6 +16,7 @@ gen_emit_unary_float_op, saved_registers, count_reg_args) +from pypy.jit.backend.arm.helper.regalloc import check_imm_arg from pypy.jit.backend.arm.codebuilder import ARMv7Builder, OverwritingBuilder from pypy.jit.backend.arm.jump import remap_frame_layout from pypy.jit.backend.arm.regalloc import TempInt, TempPtr @@ -27,18 +28,20 @@ from pypy.jit.metainterp.history import JitCellToken, TargetToken from pypy.jit.metainterp.resoperation import rop from pypy.rlib.objectmodel import we_are_translated -from pypy.rpython.lltypesystem import rstr +from pypy.rlib import rgc +from pypy.rpython.lltypesystem import rstr, rffi, lltype, llmemory NO_FORCE_INDEX = -1 class GuardToken(object): - def __init__(self, descr, failargs, faillocs, offset, - save_exc, fcond=c.AL, is_invalidate=False): + def __init__(self, descr, failargs, faillocs, offset, save_exc, fcond=c.AL, + is_guard_not_invalidated=False, is_guard_not_forced=False): assert isinstance(save_exc, bool) self.descr = descr self.offset = offset - self.is_invalidate = is_invalidate + self.is_guard_not_invalidated = is_guard_not_invalidated + self.is_guard_not_forced = is_guard_not_forced self.failargs = failargs self.faillocs = faillocs self.save_exc = save_exc @@ -190,7 +193,8 @@ return fcond def _emit_guard(self, op, arglocs, fcond, save_exc, - is_guard_not_invalidated=False): + is_guard_not_invalidated=False, + is_guard_not_forced=False): assert isinstance(save_exc, bool) assert isinstance(fcond, int) descr = op.getdescr() @@ -210,7 +214,8 @@ faillocs=arglocs, offset=pos, save_exc=save_exc, - is_invalidate=is_guard_not_invalidated, + is_guard_not_invalidated=is_guard_not_invalidated, + is_guard_not_forced=is_guard_not_forced, fcond=fcond)) return c.AL @@ -312,49 +317,10 @@ return fcond def emit_op_finish(self, op, arglocs, regalloc, fcond): - for i in range(len(arglocs) - 1): - loc = arglocs[i] - box = op.getarg(i) - if loc is None: - continue - if loc.is_reg(): - if box.type == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(i) - elif box.type == INT: - adr = self.fail_boxes_int.get_addr_for_num(i) - else: - assert 0 - self.mc.gen_load_int(r.ip.value, adr) - self.mc.STR_ri(loc.value, r.ip.value) - elif loc.is_vfp_reg(): - assert box.type == FLOAT - adr = self.fail_boxes_float.get_addr_for_num(i) - self.mc.gen_load_int(r.ip.value, adr) - self.mc.VSTR(loc.value, r.ip.value) - elif loc.is_stack() or loc.is_imm() or loc.is_imm_float(): - if box.type == FLOAT: - adr = self.fail_boxes_float.get_addr_for_num(i) - self.mov_loc_loc(loc, r.vfp_ip) - self.mc.gen_load_int(r.ip.value, adr) - self.mc.VSTR(r.vfp_ip.value, r.ip.value) - elif box.type == REF or box.type == INT: - if box.type == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(i) - elif box.type == INT: - adr = self.fail_boxes_int.get_addr_for_num(i) - else: - assert 0 - self.mov_loc_loc(loc, r.ip) - self.mc.gen_load_int(r.lr.value, adr) - self.mc.STR_ri(r.ip.value, r.lr.value) - else: - assert 0 - # note: no exception should currently be set in llop.get_exception_addr - # even if this finish may be an exit_frame_with_exception (in this case - # the exception instance is in arglocs[0]). - addr = self.cpu.get_on_leave_jitted_int(save_exception=False) - self.mc.BL(addr) - self.mc.gen_load_int(r.r0.value, arglocs[-1].value) + [argloc] = arglocs + if argloc is not r.r0: #XXX verify this + self.mov_loc_loc(argloc, r.r0, fcond) + # exit function self.gen_func_epilog() return fcond @@ -1152,8 +1118,26 @@ value = self.cpu.done_with_this_frame_float_v else: raise AssertionError(kind) - self.mc.gen_load_int(r.ip.value, value) - self.mc.CMP_rr(tmploc.value, r.ip.value) + from pypy.jit.backend.llsupport.descr import unpack_fielddescr + from pypy.jit.backend.llsupport.descr import unpack_interiorfielddescr + descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) + _offset, _size, _ = unpack_fielddescr(descrs.jf_descr) + fail_descr = self.cpu.get_fail_descr_from_number(value) + value = fail_descr.hide(self.cpu) + rgc._make_sure_does_not_move(value) + value = rffi.cast(lltype.Signed, value) + + if check_imm_arg(_offset): + self.mc.LDR_ri(r.ip.value, tmploc.value, imm=_offset) + else: + self.mc.gen_load_int(r.ip.value, _offset) + self.mc.LDR_rr(r.ip.value, tmploc.value, r.ip.value) + if check_imm_arg(value): + self.mc.CMP_ri(r.ip.value, imm=value) + else: + self.mc.gen_load_int(r.lr.value, value) + self.mc.CMP_rr(r.lr.value, r.ip.value) + #if values are equal we take the fast path # Slow path, calling helper @@ -1180,19 +1164,31 @@ if op.result is not None: # load the return value from fail_boxes_xxx[0] kind = op.result.type - if kind == INT: - adr = self.fail_boxes_int.get_addr_for_num(0) - elif kind == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(0) - elif kind == FLOAT: - adr = self.fail_boxes_float.get_addr_for_num(0) + if kind == FLOAT: + t = unpack_interiorfielddescr(descrs.as_float)[0] + if not check_imm_arg(t): + self.mc.gen_load_int(r.ip.value, t, cond=fast_path_cond) + self.mc.ADD_rr(r.ip.value, r.r0.value, r.ip.value, + cond=fast_path_cond) + t = 0 + base = r.ip + else: + base = r.r0 + self.mc.VLDR(resloc.value, base.value, imm=t, + cond=fast_path_cond) else: - raise AssertionError(kind) - self.mc.gen_load_int(r.ip.value, adr, cond=fast_path_cond) - if op.result.type == FLOAT: - self.mc.VLDR(resloc.value, r.ip.value, cond=fast_path_cond) - else: - self.mc.LDR_ri(resloc.value, r.ip.value, cond=fast_path_cond) + assert resloc is r.r0 + if kind == INT: + t = unpack_interiorfielddescr(descrs.as_int)[0] + else: + t = unpack_interiorfielddescr(descrs.as_ref)[0] + if not check_imm_arg(t): + self.mc.gen_load_int(r.ip.value, t, cond=fast_path_cond) + self.mc.LDR_rr(resloc.value, resloc.value, r.ip.value, + cond=fast_path_cond) + else: + self.mc.LDR_ri(resloc.value, resloc.value, imm=t, + cond=fast_path_cond) # jump to merge point jmp_pos = self.mc.currpos() self.mc.BKPT() @@ -1266,7 +1262,8 @@ self.mc.LDR_ri(r.ip.value, r.fp.value) self.mc.CMP_ri(r.ip.value, 0) - self._emit_guard(guard_op, arglocs[1 + numargs:], c.GE, save_exc=True) + self._emit_guard(guard_op, arglocs[1 + numargs:], c.GE, + save_exc=True, is_guard_not_forced=True) return fcond def emit_guard_call_release_gil(self, op, guard_op, arglocs, regalloc, diff --git a/pypy/jit/backend/arm/regalloc.py b/pypy/jit/backend/arm/regalloc.py --- a/pypy/jit/backend/arm/regalloc.py +++ b/pypy/jit/backend/arm/regalloc.py @@ -644,15 +644,9 @@ return args def prepare_op_finish(self, op, fcond): - args = [None] * (op.numargs() + 1) - for i in range(op.numargs()): - arg = op.getarg(i) - if arg: - args[i] = self.loc(arg) - self.possibly_free_var(arg) - n = self.cpu.get_fail_descr_number(op.getdescr()) - args[-1] = imm(n) - return args + loc = self.loc(op.getarg(0)) + self.possibly_free_var(op.getarg(0)) + return [loc] def prepare_op_guard_true(self, op, fcond): l0 = self.make_sure_var_in_reg(op.getarg(0)) diff --git a/pypy/jit/backend/arm/runner.py b/pypy/jit/backend/arm/runner.py --- a/pypy/jit/backend/arm/runner.py +++ b/pypy/jit/backend/arm/runner.py @@ -23,15 +23,21 @@ AbstractLLCPU.__init__(self, rtyper, stats, opts, translate_support_code, gcdescr) + from pypy.jit.backend.llsupport import jitframe + self.deadframe_size_max = llmemory.sizeof(jitframe.DEADFRAME, + self.get_failargs_limit()) + def set_debug(self, flag): return self.assembler.set_debug(flag) + def get_failargs_limit(self): + if self.opts is not None: + return self.opts.failargs_limit + else: + return 1000 + def setup(self): - if self.opts is not None: - failargs_limit = self.opts.failargs_limit - else: - failargs_limit = 1000 - self.assembler = AssemblerARM(self, failargs_limit=failargs_limit) + self.assembler = AssemblerARM(self, self.translate_support_code) def setup_once(self): self.assembler.setup_once() @@ -51,24 +57,6 @@ return self.assembler.assemble_bridge(faildescr, inputargs, operations, original_loop_token, log=log) - def get_latest_value_float(self, index): - return self.assembler.fail_boxes_float.getitem(index) - - def get_latest_value_int(self, index): - return self.assembler.fail_boxes_int.getitem(index) - - def get_latest_value_ref(self, index): - return self.assembler.fail_boxes_ptr.getitem(index) - - def get_latest_value_count(self): - return self.assembler.fail_boxes_count - - def get_latest_force_token(self): - return self.assembler.fail_force_index - - def get_on_leave_jitted_hook(self): - return self.assembler.leave_jitted_hook - def clear_latest_values(self, count): setitem = self.assembler.fail_boxes_ptr.setitem null = lltype.nullptr(llmemory.GCREF.TO) @@ -76,7 +64,7 @@ setitem(index, null) def make_execute_token(self, *ARGS): - FUNCPTR = lltype.Ptr(lltype.FuncType(ARGS, lltype.Signed)) + FUNCPTR = lltype.Ptr(lltype.FuncType(ARGS, llmemory.GCREF)) def execute_token(executable_token, *args): clt = executable_token.compiled_loop_token @@ -91,12 +79,13 @@ prev_interpreter = LLInterpreter.current_interpreter LLInterpreter.current_interpreter = self.debug_ll_interpreter try: - fail_index = func(*args) + deadframe = func(*args) finally: if not self.translate_support_code: LLInterpreter.current_interpreter = prev_interpreter #llop.debug_print(lltype.Void, "<<<< Back") - return self.get_fail_descr_from_number(fail_index) + self.gc_set_extra_threshold() + return deadframe return execute_token def cast_ptr_to_int(x): @@ -115,17 +104,22 @@ assert fail_index >= 0, "already forced!" faildescr = self.get_fail_descr_from_number(fail_index) rffi.cast(TP, addr_of_force_index)[0] = ~fail_index - bytecode = self.assembler._find_failure_recovery_bytecode(faildescr) + frb = self.assembler._find_failure_recovery_bytecode(faildescr) + bytecode = rffi.cast(rffi.UCHARP, frb) addr_all_null_regsiters = rffi.cast(rffi.LONG, self.all_null_registers) - # start of "no gc operation!" block - fail_index_2 = self.assembler.failure_recovery_func( - bytecode, - addr_of_force_index, - addr_all_null_regsiters) - self.assembler.leave_jitted_hook() - # end of "no gc operation!" block - assert fail_index == fail_index_2 - return faildescr + # + assert (rffi.cast(lltype.Signed, bytecode[0]) == + self.assembler.CODE_FORCED) + bytecode = rffi.ptradd(bytecode, 1) + deadframe = self.assembler.grab_frame_values(self, + bytecode, addr_of_force_index, + self.all_null_registers, + self.all_null_registers) + # + assert self.get_latest_descr(deadframe) is faildescr + self.assembler.force_token_to_dead_frame[addr_of_force_index] = ( + deadframe) + return deadframe def redirect_call_assembler(self, oldlooptoken, newlooptoken): self.assembler.redirect_call_assembler(oldlooptoken, newlooptoken) diff --git a/pypy/jit/backend/arm/test/test_gc_integration.py b/pypy/jit/backend/arm/test/test_gc_integration.py --- a/pypy/jit/backend/arm/test/test_gc_integration.py +++ b/pypy/jit/backend/arm/test/test_gc_integration.py @@ -2,25 +2,13 @@ """ Tests for register allocation for common constructs """ -import py -from pypy.jit.metainterp.history import BoxInt, \ - BoxPtr, TreeLoop, TargetToken -from pypy.jit.metainterp.resoperation import rop, ResOperation -from pypy.jit.codewriter import heaptracker -from pypy.jit.backend.llsupport.descr import GcCache -from pypy.jit.backend.llsupport.gc import GcLLDescription +from pypy.jit.metainterp.history import TargetToken +from pypy.jit.backend.llsupport.gc import GcLLDescription, GcLLDescr_boehm from pypy.jit.backend.detect_cpu import getcpuclass from pypy.jit.backend.arm.arch import WORD from pypy.rpython.lltypesystem import lltype, llmemory, rffi -from pypy.rpython.annlowlevel import llhelper -from pypy.rpython.lltypesystem import rclass -from pypy.jit.backend.llsupport.gc import GcLLDescr_framework -from pypy.jit.backend.arm.test.test_regalloc import MockAssembler from pypy.jit.backend.arm.test.test_regalloc import BaseTestRegalloc -from pypy.jit.backend.arm.regalloc import ARMFrameManager, VFPRegisterManager -from pypy.jit.codewriter.effectinfo import EffectInfo -from pypy.jit.backend.arm.regalloc import Regalloc CPU = getcpuclass() @@ -44,23 +32,14 @@ return ['compressed'] + shape[1:] -class MockGcDescr(GcCache): - get_malloc_slowpath_addr = None - write_barrier_descr = None - moving_gc = True +class MockGcDescr(GcLLDescr_boehm): gcrootmap = MockGcRootMap() - def initialize(self): - pass - - _record_constptrs = GcLLDescr_framework._record_constptrs.im_func - rewrite_assembler = GcLLDescr_framework.rewrite_assembler.im_func - class TestRegallocGcIntegration(BaseTestRegalloc): cpu = CPU(None, None) - cpu.gc_ll_descr = MockGcDescr(False) + cpu.gc_ll_descr = MockGcDescr(None, None, None) cpu.setup_once() S = lltype.GcForwardReference() diff --git a/pypy/jit/backend/arm/test/test_loop_unroll.py b/pypy/jit/backend/arm/test/test_loop_unroll.py --- a/pypy/jit/backend/arm/test/test_loop_unroll.py +++ b/pypy/jit/backend/arm/test/test_loop_unroll.py @@ -1,8 +1,8 @@ import py -from pypy.jit.backend.x86.test.test_basic import Jit386Mixin +from pypy.jit.backend.arm.test.support import JitARMMixin from pypy.jit.metainterp.test import test_loop_unroll -class TestLoopSpec(Jit386Mixin, test_loop_unroll.LoopUnrollTest): +class TestLoopSpec(JitARMMixin, test_loop_unroll.LoopUnrollTest): # for the individual tests see # ====> ../../../metainterp/test/test_loop.py pass diff --git a/pypy/jit/backend/arm/test/test_runner.py b/pypy/jit/backend/arm/test/test_runner.py --- a/pypy/jit/backend/arm/test/test_runner.py +++ b/pypy/jit/backend/arm/test/test_runner.py @@ -60,16 +60,18 @@ ] cpu.compile_loop(inp, operations, looptoken) args = [i for i in range(1, 15)] - self.cpu.execute_token(looptoken, *args) - output = [self.cpu.get_latest_value_int(i - 1) for i in range(1, 15)] + deadframe = self.cpu.execute_token(looptoken, *args) + output = [self.cpu.get_latest_value_int(deadframe, i - 1) for i in range(1, 15)] expected = [3, 7, 11, 15, 19, 23, 27, 3, 7, 11, 15, 19, 23, 27] assert output == expected - def test_redirect_call_assember2(self): - def assembler_helper(failindex, virtualizable): - return self.cpu.get_latest_value_int(0) + def test_redirect_call_assembler2(self): + def assembler_helper(deadframe, virtualizable): + x = self.cpu.get_latest_value_int(deadframe, 0) + assert x == 11 + return 7 - FUNCPTR = lltype.Ptr(lltype.FuncType([lltype.Signed, llmemory.GCREF], + FUNCPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF, llmemory.GCREF], lltype.Signed)) class FakeJitDriverSD: @@ -101,12 +103,12 @@ self.cpu.compile_loop(loop2.inputargs, loop2.operations, lt2) self.cpu.compile_loop(loop3.inputargs, loop3.operations, lt3) self.cpu.compile_loop(loop1.inputargs, loop1.operations, lt1) - self.cpu.execute_token(lt1, 11) - assert self.cpu.get_latest_value_int(0) == 12 + df = self.cpu.execute_token(lt1, 10) + assert self.cpu.get_latest_value_int(df, 0) == 7 self.cpu.redirect_call_assembler(lt2, lt3) - self.cpu.execute_token(lt1, 11) - assert self.cpu.get_latest_value_int(0) == 10 + df = self.cpu.execute_token(lt1, 12) + assert self.cpu.get_latest_value_int(df, 0) == 7 SFloat = lltype.GcForwardReference() SFloat.become(lltype.GcStruct('SFloat', ('parent', rclass.OBJECT), @@ -202,8 +204,8 @@ ARGS = [lltype.Signed] * numargs RES = lltype.Signed args = [i+1 for i in range(numargs)] - res = self.cpu.execute_token(looptoken, *args) - assert self.cpu.get_latest_value_int(0) == sum(args) + deadframe = self.cpu.execute_token(looptoken, *args) + assert self.cpu.get_latest_value_int(deadframe, 0) == sum(args) def test_debugger_on(self): from pypy.rlib import debug diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py deleted file mode 100644 --- a/pypy/jit/backend/llgraph/llimpl.py +++ /dev/null @@ -1,1968 +0,0 @@ -""" -The non-RPythonic part of the llgraph backend. -This contains all the code that is directly run -when executing on top of the llinterpreter. -""" - -import weakref -from pypy.objspace.flow.model import Variable, Constant -from pypy.annotation import model as annmodel -from pypy.jit.metainterp.history import REF, INT, FLOAT -from pypy.jit.metainterp import history -from pypy.jit.codewriter import heaptracker -from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr, rffi -from pypy.rpython.ootypesystem import ootype -from pypy.rpython.module.support import LLSupport, OOSupport -from pypy.rpython.llinterp import LLException -from pypy.rpython.extregistry import ExtRegistryEntry - -from pypy.jit.metainterp import resoperation -from pypy.jit.metainterp.resoperation import rop -from pypy.jit.backend.llgraph import symbolic -from pypy.jit.codewriter import longlong -from pypy.jit.codewriter.effectinfo import EffectInfo - -from pypy.rlib.objectmodel import ComputedIntSymbolic, we_are_translated -from pypy.rlib.rarithmetic import ovfcheck -from pypy.rlib.rarithmetic import r_longlong, r_ulonglong, r_uint -from pypy.rlib.rtimer import read_timestamp - -import py -from pypy.tool.ansi_print import ansi_log -log = py.log.Producer('runner') -py.log.setconsumer('runner', ansi_log) - -IS_32_BIT = r_ulonglong is not r_uint - - -def _from_opaque(opq): - return opq._obj.externalobj - -_TO_OPAQUE = {} - -def _to_opaque(value): - try: - return value._the_opaque_pointer - except AttributeError: - op = lltype.opaqueptr(_TO_OPAQUE[value.__class__], 'opaque', - externalobj=value) - value._the_opaque_pointer = op - return op - -def _normalize(value): - if isinstance(value, lltype._ptr): - value = lltype.top_container(value._obj) - return value - -def from_opaque_string(s): - if isinstance(s, str): - return s - elif isinstance(s, ootype._string): - return OOSupport.from_rstr(s) - else: - return LLSupport.from_rstr(s) - -FLOAT_ARRAY_TP = lltype.Ptr(lltype.Array(lltype.Float, hints={"nolength": True})) -def maybe_uncast(TP, array): - if array._TYPE.TO.OF != lltype.Float: - # array._TYPE.TO._hints.get("uncast_on_llgraph"): - array = rffi.cast(TP, array) - return array - -# a list of argtypes of all operations - couldn't find any and it's -# very useful. Note however that the table is half-broken here and -# there, in ways that are sometimes a bit hard to fix; that's why -# it is not "official". -TYPES = { - 'int_add' : (('int', 'int'), 'int'), - 'int_sub' : (('int', 'int'), 'int'), - 'int_mul' : (('int', 'int'), 'int'), - 'int_floordiv' : (('int', 'int'), 'int'), - 'int_mod' : (('int', 'int'), 'int'), - 'int_and' : (('int', 'int'), 'int'), - 'int_or' : (('int', 'int'), 'int'), - 'int_xor' : (('int', 'int'), 'int'), - 'int_lshift' : (('int', 'int'), 'int'), - 'int_rshift' : (('int', 'int'), 'int'), - 'int_lt' : (('int', 'int'), 'bool'), - 'int_gt' : (('int', 'int'), 'bool'), - 'int_ge' : (('int', 'int'), 'bool'), - 'int_le' : (('int', 'int'), 'bool'), - 'int_eq' : (('int', 'int'), 'bool'), - 'int_ne' : (('int', 'int'), 'bool'), - 'int_is_true' : (('int',), 'bool'), - 'int_is_zero' : (('int',), 'bool'), - 'int_neg' : (('int',), 'int'), - 'int_invert' : (('int',), 'int'), - 'int_add_ovf' : (('int', 'int'), 'int'), - 'int_sub_ovf' : (('int', 'int'), 'int'), - 'int_mul_ovf' : (('int', 'int'), 'int'), - 'int_force_ge_zero':(('int',), 'int'), - 'uint_add' : (('int', 'int'), 'int'), - 'uint_sub' : (('int', 'int'), 'int'), - 'uint_mul' : (('int', 'int'), 'int'), - 'uint_lt' : (('int', 'int'), 'bool'), - 'uint_le' : (('int', 'int'), 'bool'), - 'uint_eq' : (('int', 'int'), 'bool'), - 'uint_ne' : (('int', 'int'), 'bool'), - 'uint_gt' : (('int', 'int'), 'bool'), - 'uint_ge' : (('int', 'int'), 'bool'), - 'uint_xor' : (('int', 'int'), 'int'), - 'uint_rshift' : (('int', 'int'), 'int'), - 'uint_floordiv' : (('int', 'int'), 'int'), - 'float_add' : (('float', 'float'), 'float'), - 'float_sub' : (('float', 'float'), 'float'), - 'float_mul' : (('float', 'float'), 'float'), - 'float_truediv' : (('float', 'float'), 'float'), - 'float_lt' : (('float', 'float'), 'bool'), - 'float_le' : (('float', 'float'), 'bool'), - 'float_eq' : (('float', 'float'), 'bool'), - 'float_ne' : (('float', 'float'), 'bool'), - 'float_gt' : (('float', 'float'), 'bool'), - 'float_ge' : (('float', 'float'), 'bool'), - 'float_neg' : (('float',), 'float'), - 'float_abs' : (('float',), 'float'), - 'cast_float_to_int':(('float',), 'int'), - 'cast_int_to_float':(('int',), 'float'), - 'same_as' : (('int',), 'int'), # could also be ptr=>ptr - 'new_with_vtable' : (('ref',), 'ref'), - 'new' : ((), 'ref'), - 'new_array' : (('int',), 'ref'), - 'oois' : (('ref', 'ref'), 'bool'), - 'ooisnot' : (('ref', 'ref'), 'bool'), - 'instanceof' : (('ref',), 'bool'), - 'subclassof' : (('ref', 'ref'), 'bool'), - 'runtimenew' : (('ref',), 'ref'), - 'setfield_gc' : (('ref', 'intorptr'), None), - 'getfield_gc' : (('ref',), 'intorptr'), - 'getfield_gc_pure': (('ref',), 'intorptr'), - 'setfield_raw' : (('ref', 'intorptr'), None), - 'getfield_raw' : (('ref',), 'intorptr'), - 'getfield_raw_pure': (('ref',), 'intorptr'), - 'setarrayitem_gc' : (('ref', 'int', 'intorptr'), None), - 'getarrayitem_gc' : (('ref', 'int'), 'intorptr'), - 'getarrayitem_gc_pure' : (('ref', 'int'), 'intorptr'), - 'setarrayitem_raw' : (('ref', 'int', 'intorptr'), None), - 'getarrayitem_raw' : (('ref', 'int'), 'intorptr'), - 'getarrayitem_raw_pure' : (('ref', 'int'), 'intorptr'), - 'arraylen_gc' : (('ref',), 'int'), - 'call' : (('ref', 'varargs'), 'intorptr'), - 'call_assembler' : (('varargs',), 'intorptr'), - 'cond_call_gc_wb' : (('ptr', 'ptr'), None), - 'cond_call_gc_wb_array': (('ptr', 'int', 'ptr'), None), - 'oosend' : (('varargs',), 'intorptr'), - 'oosend_pure' : (('varargs',), 'intorptr'), - 'guard_true' : (('bool',), None), - 'guard_false' : (('bool',), None), - 'guard_value' : (('int', 'int'), None), - 'guard_class' : (('ref', 'ref'), None), - 'guard_no_exception' : ((), None), - 'guard_exception' : (('ref',), 'ref'), - 'guard_no_overflow' : ((), None), - 'guard_overflow' : ((), None), - 'guard_nonnull' : (('ref',), None), - 'guard_isnull' : (('ref',), None), - 'guard_nonnull_class' : (('ref', 'ref'), None), - 'newstr' : (('int',), 'ref'), - 'strlen' : (('ref',), 'int'), - 'strgetitem' : (('ref', 'int'), 'int'), - 'strsetitem' : (('ref', 'int', 'int'), None), - 'newunicode' : (('int',), 'ref'), - 'unicodelen' : (('ref',), 'int'), - 'unicodegetitem' : (('ref', 'int'), 'int'), - 'unicodesetitem' : (('ref', 'int', 'int'), 'int'), - 'cast_ptr_to_int' : (('ref',), 'int'), - 'cast_int_to_ptr' : (('int',), 'ref'), - 'debug_merge_point': (('ref', 'int', 'int'), None), - 'force_token' : ((), 'int'), - 'call_may_force' : (('int', 'varargs'), 'intorptr'), - 'guard_not_forced': ((), None), -} - -# ____________________________________________________________ - -class CompiledLoop(object): - has_been_freed = False - invalid = False - - def __init__(self): - self.inputargs = [] - self.operations = [] - - def getargtypes(self): - return [v.concretetype for v in self.inputargs] - - def __repr__(self): - lines = [] - self.as_text(lines, 1) - return 'CompiledLoop %s:\n%s' % (self.inputargs, '\n'.join(lines)) - - def as_text(self, lines, indent): - for op in self.operations: - lines.append('\t'*indent + repr(op)) - -class Operation(object): - result = None - descr = None - jump_target = None - fail_args = None - - def __init__(self, opnum): - self.opnum = opnum - self.args = [] - - def __repr__(self): - if self.result is not None: - sres = repr0(self.result) + ' = ' - else: - sres = '' - return '{%s%s(%s)}' % (sres, self.getopname(), - ', '.join(map(repr0, self.args))) - - def getopname(self): - try: - return resoperation.opname[self.opnum] - except KeyError: - return '<%d>' % self.opnum - - def is_guard(self): - return rop._GUARD_FIRST <= self.opnum <= rop._GUARD_LAST - - def is_final(self): - return rop._FINAL_FIRST <= self.opnum <= rop._FINAL_LAST - -def repr0(x): - if isinstance(x, list): - return '[' + ', '.join(repr0(y) for y in x) + ']' - elif isinstance(x, Constant): - return '(' + repr0(x.value) + ')' - elif isinstance(x, lltype._ptr): - x = llmemory.cast_ptr_to_adr(x) - if x.ptr: - try: - container = x.ptr._obj._normalizedcontainer() - return '* %s' % (container._TYPE._short_name(),) - except AttributeError: - return repr(x) - else: - return 'NULL' - else: - return repr(x) - -def repr_list(lst, types): - res_l = [] - if types and types[-1] == 'varargs': - types = types[:-1] + ('int',) * (len(lst) - len(types) + 1) - assert len(types) == len(lst) - for elem, tp in zip(lst, types): - if isinstance(elem, Constant): - res_l.append('(%s)' % repr1(elem, tp)) - else: - res_l.append(repr1(elem, tp)) - return '[%s]' % (', '.join(res_l)) - -def repr1(x, tp): - if tp == "intorptr": - TYPE = lltype.typeOf(x) - if isinstance(TYPE, lltype.Ptr) and TYPE.TO._gckind == 'gc': - tp = "ref" - else: - tp = "int" - if tp == 'int': - return str(x) - elif tp == 'void': - return '---' - elif tp == 'ref': - if not x: - return '(* None)' - if isinstance(x, int): - # XXX normalize? - ptr = str(llmemory.cast_int_to_adr(x)) - elif isinstance(ootype.typeOf(x), ootype.OOType): - return repr(x) - else: - if getattr(x, '_fake', None): - return repr(x) - if lltype.typeOf(x) == llmemory.GCREF: - TP = lltype.Ptr(lltype.typeOf(x._obj.container)) - ptr = lltype.cast_opaque_ptr(TP, x) - else: - ptr = x - try: - container = ptr._obj._normalizedcontainer() - return '(* %s)' % (container._TYPE._short_name(),) - except AttributeError: - return '(%r)' % (ptr,) - elif tp == 'bool': - assert x == 0 or x == 1 - return str(bool(x)) - #elif tp == 'fieldname': - # return str(symbolic.TokenToField[x...][1]) - elif tp == 'float': - return str(x) - else: - raise NotImplementedError("tp = %s" % tp) - -_variables = [] - -def compile_start(): - del _variables[:] - return _to_opaque(CompiledLoop()) - -def mark_as_free(loop): - loop = _from_opaque(loop) - assert not loop.has_been_freed - loop.has_been_freed = True - -def compile_start_int_var(loop): - return compile_start_ref_var(loop, lltype.Signed) - -def compile_start_float_var(loop): - return compile_start_ref_var(loop, longlong.FLOATSTORAGE) - -def compile_start_ref_var(loop, TYPE): - loop = _from_opaque(loop) - assert not loop.operations - v = Variable() - v.concretetype = TYPE - loop.inputargs.append(v) - r = len(_variables) - _variables.append(v) - return r - -def compile_started_vars(clt): - if not hasattr(clt, '_debug_argtypes'): # only when compiling the loop - argtypes = [v.concretetype for v in _variables] - try: - clt._debug_argtypes = argtypes - except AttributeError: # when 'clt' is actually a translated - pass # GcStruct - -def compile_add(loop, opnum): - loop = _from_opaque(loop) - loop.operations.append(Operation(opnum)) - -def compile_add_descr(loop, ofs, type, arg_types, extrainfo, width): - from pypy.jit.backend.llgraph.runner import Descr - loop = _from_opaque(loop) - op = loop.operations[-1] - assert isinstance(type, str) and len(type) == 1 - op.descr = Descr(ofs, type, arg_types=arg_types, extrainfo=extrainfo, width=width) - -def compile_add_descr_arg(loop, ofs, type, arg_types): - from pypy.jit.backend.llgraph.runner import Descr - loop = _from_opaque(loop) - op = loop.operations[-1] - assert isinstance(type, str) and len(type) == 1 - op.args.append(Descr(ofs, type, arg_types=arg_types)) - -def compile_add_loop_token(loop, descr): - if we_are_translated(): - raise ValueError("CALL_ASSEMBLER not supported") - loop = _from_opaque(loop) - op = loop.operations[-1] - op.descr = weakref.ref(descr) - -TARGET_TOKENS = weakref.WeakKeyDictionary() - -def compile_add_target_token(loop, descr, clt): - # here, 'clt' is the compiled_loop_token of the original loop that - # we are compiling - loop = _from_opaque(loop) - op = loop.operations[-1] - descrobj = _normalize(descr) - TARGET_TOKENS[descrobj] = loop, len(loop.operations), op.args, clt - -def compile_add_var(loop, intvar): - loop = _from_opaque(loop) - op = loop.operations[-1] - op.args.append(_variables[intvar]) - -def compile_add_int_const(loop, value): - compile_add_ref_const(loop, value, lltype.Signed) - -def compile_add_float_const(loop, value): - compile_add_ref_const(loop, value, longlong.FLOATSTORAGE) - -def compile_add_ref_const(loop, value, TYPE): - loop = _from_opaque(loop) - const = Constant(value) - const.concretetype = TYPE - op = loop.operations[-1] - op.args.append(const) - -def compile_add_int_result(loop): - return compile_add_ref_result(loop, lltype.Signed) - -def compile_add_float_result(loop): - return compile_add_ref_result(loop, longlong.FLOATSTORAGE) - -def compile_add_ref_result(loop, TYPE): - loop = _from_opaque(loop) - v = Variable() - v.concretetype = TYPE - op = loop.operations[-1] - op.result = v - r = len(_variables) - _variables.append(v) - return r - -def compile_add_jump_target(loop, targettoken, source_clt): - loop = _from_opaque(loop) - descrobj = _normalize(targettoken) - (loop_target, target_opindex, target_inputargs, target_clt - ) = TARGET_TOKENS[descrobj] - # - try: - assert source_clt._debug_argtypes == target_clt._debug_argtypes - except AttributeError: # when translated - pass - # - op = loop.operations[-1] - op.jump_target = loop_target - op.jump_target_opindex = target_opindex - op.jump_target_inputargs = target_inputargs - assert op.opnum == rop.JUMP - assert [v.concretetype for v in op.args] == ( - [v.concretetype for v in target_inputargs]) - # - if loop_target == loop: - log.info("compiling new loop") - else: - log.info("compiling new bridge") - -def compile_add_guard_jump_target(loop, loop_target): - loop = _from_opaque(loop) - loop_target = _from_opaque(loop_target) - op = loop.operations[-1] - assert op.is_guard() - op.jump_target = loop_target - -def compile_add_fail(loop, fail_index): - loop = _from_opaque(loop) - index = len(loop.operations)-1 - op = loop.operations[index] - op.fail_index = fail_index - return index - -def compile_add_fail_arg(loop, intvar): - loop = _from_opaque(loop) - op = loop.operations[-1] - if op.fail_args is None: - op.fail_args = [] - if intvar == -1: - op.fail_args.append(None) - else: - op.fail_args.append(_variables[intvar]) - -def compile_redirect_fail(old_loop, old_index, new_loop): - old_loop = _from_opaque(old_loop) - new_loop = _from_opaque(new_loop) - guard_op = old_loop.operations[old_index] - assert guard_op.is_guard() - guard_op.jump_target = new_loop - # check that the bridge's inputargs are of the correct number and - # kind for the guard - if guard_op.fail_args is not None: - argkinds = [v.concretetype for v in guard_op.fail_args if v] - else: - argkinds = [] - assert argkinds == [v.concretetype for v in new_loop.inputargs] - -# ------------------------------ - -class Frame(object): - OPHANDLERS = [None] * (rop._LAST+1) - - def __init__(self, cpu): - self.verbose = False - self.cpu = cpu - self.opindex = 1 - self._forced = False - self._may_force = -1 - - def getenv(self, v): - from pypy.jit.backend.llgraph.runner import Descr - if isinstance(v, Constant): - return v.value - elif isinstance(v, Descr): - return v - else: - return self.env[v] - - def _populate_fail_args(self, op, skip=None): - fail_args = [] - if op.fail_args: - for fail_arg in op.fail_args: - if fail_arg is None: - fail_args.append(None) - elif fail_arg is skip: - fail_args.append(fail_arg.concretetype._defl()) - else: - fail_args.append(self.getenv(fail_arg)) - self.fail_args = fail_args - self.fail_index = op.fail_index - - def execute(self): - """Execute all operations in a loop, - possibly following to other loops as well. - """ - global _last_exception - assert _last_exception is None, "exception left behind" - verbose = True - self.opindex = 0 - while True: - assert not self.loop.has_been_freed - op = self.loop.operations[self.opindex] - args = [self.getenv(v) for v in op.args] - if not op.is_final(): - try: - result = self.execute_operation(op.opnum, args, op.descr, - verbose) - except GuardFailed: - assert op.is_guard() - _stats.exec_conditional_jumps += 1 - if op.jump_target is not None: - # a patched guard, pointing to further code - if op.fail_args: - args = [self.getenv(v) for v in op.fail_args if v] - else: - args = [] - assert len(op.jump_target.inputargs) == len(args) - self.env = dict(zip(op.jump_target.inputargs, args)) - self.loop = op.jump_target - self.opindex = 0 - continue - else: - self._populate_fail_args(op) - # a non-patched guard - if self.verbose: - log.trace('failed: %s' % ( - ', '.join(map(str, fail_args)),)) - return op.fail_index - #verbose = self.verbose - assert (result is None) == (op.result is None) - if op.result is not None: - RESTYPE = op.result.concretetype - if RESTYPE is lltype.Signed: - x = self.as_int(result) - elif RESTYPE is llmemory.GCREF: - x = self.as_ptr(result) - elif RESTYPE is ootype.Object: - x = self.as_object(result) - elif RESTYPE is longlong.FLOATSTORAGE: - x = self.as_floatstorage(result) - else: - raise Exception("op.result.concretetype is %r" - % (RESTYPE,)) - self.env[op.result] = x - self.opindex += 1 - continue - if op.opnum == rop.JUMP: - inputargs = op.jump_target_inputargs - assert len(inputargs) == len(args) - self.env = dict(zip(inputargs, args)) - self.loop = op.jump_target - self.opindex = op.jump_target_opindex - _stats.exec_jumps += 1 - elif op.opnum == rop.FINISH: - if self.verbose: - log.trace('finished: %s' % ( - ', '.join(map(str, args)),)) - self.fail_args = args - return op.fail_index - - else: - assert 0, "unknown final operation %d" % (op.opnum,) - - def execute_operation(self, opnum, values, descr, verbose): - """Execute a single operation. - """ - ophandler = self.OPHANDLERS[opnum] - if ophandler is None: - self._define_impl(opnum) - ophandler = self.OPHANDLERS[opnum] - assert ophandler is not None, "missing impl for op %d" % opnum - opname = resoperation.opname[opnum].lower() - exec_counters = _stats.exec_counters - exec_counters[opname] = exec_counters.get(opname, 0) + 1 - for i in range(len(values)): - if isinstance(values[i], ComputedIntSymbolic): - values[i] = values[i].compute_fn() - res = NotImplemented - try: - res = ophandler(self, descr, *values) - finally: - if 0: # if verbose: - argtypes, restype = TYPES[opname] - if res is None: - resdata = '' - elif res is NotImplemented: - resdata = '*fail*' - else: - resdata = '-> ' + repr1(res, restype) - # fish the types - log.cpu('\t%s %s %s' % (opname, repr_list(values, argtypes), - resdata)) - return res - - def as_int(self, x): - return cast_to_int(x) - - def as_ptr(self, x): - return cast_to_ptr(x) - - def as_object(self, x): - return ootype.cast_to_object(x) - - def as_floatstorage(self, x): - return cast_to_floatstorage(x) - - def log_progress(self): - count = sum(_stats.exec_counters.values()) - count_jumps = _stats.exec_jumps - log.trace('ran %d operations, %d jumps' % (count, count_jumps)) - - # ---------- - - @classmethod - def _define_impl(cls, opnum): - opname = resoperation.opname[opnum] - try: - op = getattr(cls, 'op_' + opname.lower()) # op_guard_true etc. - except AttributeError: - try: - impl = globals()['do_' + opname.lower()] # do_arraylen_gc etc. - def op(self, descr, *args): - if descr is None: - return impl(*args) - else: - return impl(descr, *args) - except KeyError: - op = cls._make_impl_from_blackhole_interp(opname) - cls.OPHANDLERS[opnum] = op - - @classmethod - def _make_impl_from_blackhole_interp(cls, opname): - from pypy.jit.metainterp.blackhole import BlackholeInterpreter - name = 'bhimpl_' + opname.lower() - func = BlackholeInterpreter.__dict__[name] - for argtype in func.argtypes: - assert argtype in ('i', 'r', 'f') - # - def _op_default_implementation(self, descr, *args): - # for all operations implemented in the blackhole interpreter - return func(*args) - # - return _op_default_implementation - - def op_label(self, _, *args): - op = self.loop.operations[self.opindex] - assert op.opnum == rop.LABEL - assert len(op.args) == len(args) - newenv = {} - for v, value in zip(op.args, args): - newenv[v] = value - self.env = newenv - - def op_debug_merge_point(self, _, *args): - from pypy.jit.metainterp.warmspot import get_stats - try: - stats = get_stats() - except AttributeError: - pass - else: - stats.add_merge_point_location(args[1:]) - pass - - def op_guard_true(self, _, value): - if not value: - raise GuardFailed - - def op_guard_false(self, _, value): - if value: - raise GuardFailed - - op_guard_nonnull = op_guard_true - op_guard_isnull = op_guard_false - - def op_guard_class(self, _, value, expected_class): - value = lltype.cast_opaque_ptr(rclass.OBJECTPTR, value) - expected_class = llmemory.cast_adr_to_ptr( - llmemory.cast_int_to_adr(expected_class), - rclass.CLASSTYPE) - if value.typeptr != expected_class: - raise GuardFailed - - def op_guard_nonnull_class(self, _, value, expected_class): - if not value: - raise GuardFailed - self.op_guard_class(_, value, expected_class) - - def op_guard_value(self, _, value, expected_value): - if value != expected_value: - raise GuardFailed - - def op_guard_no_exception(self, _): - if _last_exception: - raise GuardFailed - - def _check_exception(self, expected_exception): - global _last_exception - expected_exception = self._cast_exception(expected_exception) - assert expected_exception - exc = _last_exception - if exc: - got = exc.args[0] - # exact match! - if got != expected_exception: - return False - return True - else: - return False - - def _cast_exception(self, exception): - return llmemory.cast_adr_to_ptr( - llmemory.cast_int_to_adr(exception), - rclass.CLASSTYPE) - - def _issubclass(self, cls1, cls2): - return rclass.ll_issubclass(cls1, cls2) - - def op_guard_exception(self, _, expected_exception): - global _last_exception - if not self._check_exception(expected_exception): - raise GuardFailed - res = _last_exception[1] - _last_exception = None - return res - - def op_guard_no_overflow(self, _): - flag = self.overflow_flag - del self.overflow_flag - if flag: - raise GuardFailed - - def op_guard_overflow(self, _): - flag = self.overflow_flag - del self.overflow_flag - if not flag: - raise GuardFailed - - def op_int_add_ovf(self, _, x, y): - try: - z = ovfcheck(x + y) - except OverflowError: - ovf = True - z = 0 - else: - ovf = False - self.overflow_flag = ovf - return z - - def op_int_sub_ovf(self, _, x, y): - try: - z = ovfcheck(x - y) - except OverflowError: - ovf = True - z = 0 - else: - ovf = False - self.overflow_flag = ovf - return z - - def op_int_mul_ovf(self, _, x, y): - try: - z = ovfcheck(x * y) - except OverflowError: - ovf = True - z = 0 - else: - ovf = False - self.overflow_flag = ovf - return z - - def op_keepalive(self, _, x): - pass - - # ---------- - # delegating to the builtins do_xxx() (done automatically for simple cases) - - def op_getarrayitem_gc(self, arraydescr, array, index): - if arraydescr.typeinfo == REF: - return do_getarrayitem_gc_ptr(array, index) - elif arraydescr.typeinfo == INT: - return do_getarrayitem_gc_int(array, index) - elif arraydescr.typeinfo == FLOAT: - return do_getarrayitem_gc_float(array, index) - else: - raise NotImplementedError - - op_getarrayitem_gc_pure = op_getarrayitem_gc - - def op_getarrayitem_raw(self, arraydescr, array, index): - if arraydescr.typeinfo == REF: - raise NotImplementedError("getarrayitem_raw -> gcref") - elif arraydescr.typeinfo == INT: - return do_getarrayitem_raw_int(array, index, arraydescr.ofs) - elif arraydescr.typeinfo == FLOAT: - return do_getarrayitem_raw_float(array, index) - else: - raise NotImplementedError - - op_getarrayitem_raw_pure = op_getarrayitem_raw - - def op_getfield_gc(self, fielddescr, struct): - if fielddescr.typeinfo == REF: - return do_getfield_gc_ptr(struct, fielddescr.ofs) - elif fielddescr.typeinfo == INT: From noreply at buildbot.pypy.org Wed Jan 9 23:25:39 2013 From: noreply at buildbot.pypy.org (rguillebert) Date: Wed, 9 Jan 2013 23:25:39 +0100 (CET) Subject: [pypy-commit] pypy default: Fix typo Message-ID: <20130109222539.15BBB1C11F3@cobra.cs.uni-duesseldorf.de> Author: Romain Guillebert Branch: Changeset: r59904:cad5c05cae8e Date: 2013-01-09 23:24 +0100 http://bitbucket.org/pypy/pypy/changeset/cad5c05cae8e/ Log: Fix typo diff --git a/pypy/translator/driver.py b/pypy/translator/driver.py --- a/pypy/translator/driver.py +++ b/pypy/translator/driver.py @@ -362,7 +362,7 @@ @taskdef([RTYPE], "JIT compiler generation") def task_pyjitpl_lltype(self): """ Generate bytecodes for JIT and flow the JIT helper functions - ootype version + lltype version """ get_policy = self.extra['jitpolicy'] self.jitpolicy = get_policy(self) From noreply at buildbot.pypy.org Thu Jan 10 00:08:09 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 10 Jan 2013 00:08:09 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: resize entries like a normal list Message-ID: <20130109230809.8D44D1C0041@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59905:4f2c3a20e4d1 Date: 2013-01-10 01:07 +0200 http://bitbucket.org/pypy/pypy/changeset/4f2c3a20e4d1/ Log: resize entries like a normal list diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py --- a/pypy/rpython/lltypesystem/rdict.py +++ b/pypy/rpython/lltypesystem/rdict.py @@ -55,7 +55,11 @@ 'clear_key': (isinstance(DICTKEY, lltype.Ptr) and DICTKEY._needsgc()), 'clear_value': (isinstance(DICTVALUE, lltype.Ptr) and - DICTVALUE._needsgc())} + DICTVALUE._needsgc()), + 'getitem': ll_entry_getitem, + # no boundary checking versions + 'getitem_clean': ll_entry_getitem_clean, + 'popitem': ll_entry_popitem} # * the key entryfields.append(("key", DICTKEY)) @@ -401,6 +405,31 @@ ll_index_getitem(size, from_indexes, i)) i += 1 +# ---------------------- entries ----------------------- + +def ll_entries_resize_up(d): + lgt = len(d.entries) + if lgt < 9: + some = 4 + else: + some = 7 + new_lgt = lgt + some + (lgt >> 3) + new_entries = lltype.typeOf(d).TO.entries.TO.allocate(new_lgt) + rgc.ll_arraycopy(d.entries, new_entries, 0, 0, lgt) + d.entries = new_entries + return new_entries + +def ll_entry_getitem(entries, d, item): + if len(entries) <= item: + entries = ll_entries_resize_up(d) + return entries[item] + +def ll_entry_getitem_clean(entries, item): + return entries[item] + +def ll_entry_popitem(d): + pass + # ---------------------- hashes ------------------- def ll_hash_from_cache(entries, i): @@ -414,7 +443,7 @@ return ENTRIES.fasthashfn(entries[i].key) def ll_get_value(d, i): - return d.entries[ll_index_getitem(d.size, d.indexes, i)].value + return d.entries.getitem_clean(ll_index_getitem(d.size, d.indexes, i)).value def ll_keyhash_custom(d, key): DICT = lltype.typeOf(d).TO @@ -457,7 +486,7 @@ index = ll_index_getitem(d.size, d.indexes, i) if index == FREE: index = d.num_items - entry = d.entries[index] + entry = d.entries.getitem(d, index) # a new entry that was never used before ll_assert(not valid, "valid but not everused") rc = d.resize_counter - 1 @@ -465,7 +494,7 @@ d.resize() i = ll_dict_lookup_clean(d, hash) # then redo the lookup for 'key' - entry = d.entries[index] + entry = d.entries.getitem(d, index) rc = d.resize_counter - 1 ll_assert(rc > 0, "ll_dict_resize failed?") ll_assert(index < len(d.entries), "invalid insert") @@ -474,12 +503,12 @@ entry.value = value elif index == DELETED: index = d.num_items - entry = d.entries[index] + entry = d.entries.getitem(d, index) ll_index_setitem(d.size, d.indexes, i, index) entry.value = value else: # override an existing or deleted entry - entry = d.entries[index] + entry = d.entries.getitem_clean(index) entry.value = value if valid: return @@ -499,13 +528,13 @@ ENTRIES = lltype.typeOf(d.entries).TO ENTRY = ENTRIES.OF if index != d.num_items - 1: - old_entry = d.entries[d.num_items - 1] + old_entry = d.entries.getitem(d, d.num_items - 1) key = old_entry.key to_insert_i = ll_dict_lookup(d, key, d.keyhash(key)) ll_assert(not to_insert_i & HIGHEST_BIT, "invalid entry") ll_index_setitem(d.size, d.indexes, to_insert_i, index) # copy the value - new_entry = d.entries[index] + new_entry = d.entries.getitem_clean(index) new_entry.key = key new_entry.value = old_entry.value if hasattr(ENTRY, 'f_hash'): @@ -513,11 +542,12 @@ # clear the key and the value if they are GC pointers ll_index_setitem(d.size, d.indexes, i, DELETED) d.num_items -= 1 - entry = d.entries[d.num_items] + entry = d.entries.getitem_clean(d.num_items) if ENTRIES.clear_key: entry.key = lltype.nullptr(ENTRY.key.TO) if ENTRIES.clear_value: entry.value = lltype.nullptr(ENTRY.value.TO) + ll_entry_popitem(d) # # The rest is commented out: like CPython we no longer shrink the # dictionary here. It may shrink later if we try to append a number @@ -558,13 +588,6 @@ pos = ll_dict_lookup_clean(d, old_entries.hash(index)) ll_index_setitem(d.size, indexes, pos, index) i += 1 - if len(old_entries) != new_item_size: - d.entries = lltype.typeOf(old_entries).TO.allocate(new_item_size) - rgc.ll_arraycopy(old_entries, d.entries, 0, 0, min(len(old_entries), - len(d.entries))) - else: - # we just removed deleted items, but we didn't do anything else special - d.entries = old_entries ll_dict_resize.oopspec = 'dict.resize(d)' # ------- a port of CPython's dictobject.c's lookdict implementation ------- @@ -591,7 +614,7 @@ direct_compare = not hasattr(ENTRIES, 'no_direct_compare') index = ll_index_getitem(d.size, indexes, i) if entries.valid(index): - checkingkey = entries[index].key + checkingkey = entries.getitem_clean(index).key if direct_compare and checkingkey == key: return i # found the entry if d.keyeq is not None and entries.hash(index) == hash: @@ -602,7 +625,7 @@ if d.paranoia: if (entries != d.entries or not entries.valid(ll_index_getitem(d.size, indexes, i)) - or entries[index].key != checkingkey): + or entries.getitem_clean(index).key != checkingkey): # the compare did major nasty stuff to the dict: start over return ll_dict_lookup(d, key, hash) if found: @@ -629,7 +652,7 @@ freeslot = i return freeslot | HIGHEST_BIT elif entries.valid(index): - checkingkey = entries[index].key + checkingkey = entries.getitem_clean(index).key if direct_compare and checkingkey == key: return i if d.keyeq is not None and entries.hash(index) == hash: @@ -639,7 +662,7 @@ if d.paranoia: if (entries != d.entries or not entries.valid(ll_index_getitem(d.size, indexes, i)) or - entries[index].key != checkingkey): + entries.getitem_clean(index).key != checkingkey): # the compare did major nasty stuff to the dict: # start over return ll_dict_lookup(d, key, hash) @@ -669,7 +692,8 @@ # Irregular operations. DICT_INITSIZE = 8 -DICT_ITEMS_INITSIZE = 5 +DICT_ITEMS_INITSIZE = 3 +DICT_RESIZE_START = 5 def ll_newdict(DICT): d = DICT.allocate() @@ -677,7 +701,7 @@ d.size = DICT_INITSIZE d.num_items = 0 d.entries = DICT.entries.TO.allocate(DICT_ITEMS_INITSIZE) - d.resize_counter = DICT_ITEMS_INITSIZE + d.resize_counter = DICT_RESIZE_START return d def ll_newdict_size(DICT, length_estimate): @@ -687,7 +711,7 @@ n *= 2 items_size = n // 3 * 2 + 1 d = DICT.allocate() - d.entries = DICT.entries.TO.allocate(items_size) + d.entries = DICT.entries.TO.allocate(length_estimate) d.indexes = _ll_malloc_indexes(n) d.size = n d.num_items = 0 @@ -757,7 +781,7 @@ # clear the reference to the dict and prevent restarts iter.dict = lltype.nullptr(lltype.typeOf(iter).TO.dict.TO) raise StopIteration - entry = dict.entries[iter.index] + entry = dict.entries.getitem_clean(iter.index) iter.index += 1 if RETURNTYPE is lltype.Void: return None @@ -797,7 +821,7 @@ def ll_copy(dict): DICT = lltype.typeOf(dict).TO - dictsize = len(dict.entries) + dictsize = dict.num_items d = DICT.allocate() d.entries = lltype.malloc(DICT.entries.TO, dictsize) d.indexes = _ll_malloc_indexes(dict.size) @@ -808,20 +832,20 @@ if hasattr(DICT, 'fnkeyhash'): d.fnkeyhash = dict.fnkeyhash ll_dict_copy_indexes(d.size, dict.indexes, d.indexes) #rgc.ll_arraycopy(dict.indexes, d.indexes, 0, 0, len(dict.indexes)) - rgc.ll_arraycopy(dict.entries, d.entries, 0, 0, len(dict.entries)) + rgc.ll_arraycopy(dict.entries, d.entries, 0, 0, dict.num_items) return d ll_copy.oopspec = 'dict.copy(dict)' def ll_clear(d): if (d.size == DICT_INITSIZE and - d.resize_counter == DICT_ITEMS_INITSIZE): + d.resize_counter == DICT_RESIZE_START): return old_entries = d.entries d.entries = lltype.typeOf(old_entries).TO.allocate(DICT_ITEMS_INITSIZE) d.indexes = _ll_malloc_indexes(DICT_INITSIZE) d.size = DICT_INITSIZE d.num_items = 0 - d.resize_counter = DICT_ITEMS_INITSIZE + d.resize_counter = DICT_RESIZE_START ll_clear.oopspec = 'dict.clear(d)' def ll_update(dic1, dic2): @@ -829,7 +853,7 @@ d2len = dic2.num_items i = 0 while i < d2len: - entry = entries[i] + entry = entries.getitem_clean(i) hash = entries.hash(i) key = entry.key j = ll_dict_lookup(dic1, key, hash) @@ -858,7 +882,7 @@ while i < dlen: ELEM = lltype.typeOf(items).TO.OF if ELEM is not lltype.Void: - entry = entries[i] + entry = entries.getitem_clean(i) if kind == 'items': r = lltype.malloc(ELEM.TO) r.item0 = recast(ELEM.TO.item0, entry.key) @@ -884,7 +908,7 @@ def ll_popitem(ELEM, dic): if dic.num_items == 0: raise KeyError - entry = dic.entries[dic.num_items - 1] + entry = dic.entries.getitem_clean(dic.num_items - 1) r = lltype.malloc(ELEM.TO) r.item0 = recast(ELEM.TO.item0, entry.key) r.item1 = recast(ELEM.TO.item1, entry.value) diff --git a/pypy/rpython/test/test_rdict.py b/pypy/rpython/test/test_rdict.py --- a/pypy/rpython/test/test_rdict.py +++ b/pypy/rpython/test/test_rdict.py @@ -971,25 +971,6 @@ res = f() assert res == 1 - def test_nonnull_hint(self): - def eq(a, b): - return a == b - def rhash(a): - return 3 - - def func(i): - d = r_dict(eq, rhash) - if not i: - d[None] = i - else: - d[str(i)] = i - return "12" in d, d - - llres = self.interpret(func, [12]) - assert llres.item0 == 1 - DICT = lltype.typeOf(llres.item1) - assert sorted(DICT.TO.entries.TO.OF._flds) == ['f_hash', 'key', 'value'] - def test_memoryerror_should_not_insert(self): # This shows a misbehaviour that also exists in CPython 2.7, but not # any more in CPython 3.3. The behaviour is that even if a dict From noreply at buildbot.pypy.org Thu Jan 10 00:11:26 2013 From: noreply at buildbot.pypy.org (wlav) Date: Thu, 10 Jan 2013 00:11:26 +0100 (CET) Subject: [pypy-commit] pypy reflex-support: minor bugfix for CINT backend Message-ID: <20130109231126.0667A1C0041@cobra.cs.uni-duesseldorf.de> Author: Wim Lavrijsen Branch: reflex-support Changeset: r59906:8b4ea08c6d71 Date: 2013-01-09 15:11 -0800 http://bitbucket.org/pypy/pypy/changeset/8b4ea08c6d71/ Log: minor bugfix for CINT backend diff --git a/pypy/module/cppyy/capi/cint_capi.py b/pypy/module/cppyy/capi/cint_capi.py --- a/pypy/module/cppyy/capi/cint_capi.py +++ b/pypy/module/cppyy/capi/cint_capi.py @@ -200,16 +200,24 @@ else: # builtin data w_leaf = space.call_method(w_self, "GetLeaf", args_w[0]) - w_typename = space.call_method(w_leaf, "GetTypeName" ) - from pypy.module.cppyy import capi - typename = capi.c_resolve_name(space.str_w(w_typename)) + space.call_method(w_branch, "GetEntry", space.wrap(0)) + + # location w_address = space.call_method(w_leaf, "GetValuePointer") buf = space.buffer_w(w_address) from pypy.module._rawffi import buffer assert isinstance(buf, buffer.RawFFIBuffer) address = rffi.cast(rffi.CCHARP, buf.datainstance.ll_buffer) + + # placeholder + w_typename = space.call_method(w_leaf, "GetTypeName" ) + from pypy.module.cppyy import capi + typename = capi.c_resolve_name(space.str_w(w_typename)) + w_address = space.call_method(w_leaf, "GetValuePointer") from pypy.module._cffi_backend import cdataobj, newtype cdata = cdataobj.W_CData(space, address, newtype.new_primitive_type(space, typename)) + + # cache result space.setattr(w_self, space.wrap('_'+attr), space.wrap(cdata)) return space.getattr(w_self, args_w[0]) diff --git a/pypy/module/cppyy/test/test_cint.py b/pypy/module/cppyy/test/test_cint.py --- a/pypy/module/cppyy/test/test_cint.py +++ b/pypy/module/cppyy/test/test_cint.py @@ -336,8 +336,8 @@ mytree.Branch("myd", b, "myd/D") for i in range(self.N): - a[0] = i - b[0] = i/2. + a[0] = i+1 # make sure value is different from default (0) + b[0] = (i+1)/2. # id. 0. mytree.Fill() f.Write() f.Close() @@ -351,12 +351,12 @@ f = TFile(self.fname) mytree = f.Get(self.tname) - i = 0 + i = 1 for event in mytree: assert event.myi == i assert event.myd == i/2. i += 1 - assert i == self.N + assert (i-1) == self.N f.Close() From noreply at buildbot.pypy.org Thu Jan 10 00:18:02 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Thu, 10 Jan 2013 00:18:02 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: "Reasonable may have left the building" Message-ID: <20130109231802.C6B2B1C0041@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59907:e6716eb7d47f Date: 2013-01-10 00:17 +0100 http://bitbucket.org/pypy/pypy/changeset/e6716eb7d47f/ Log: "Reasonable may have left the building" diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py --- a/pypy/interpreter/astcompiler/assemble.py +++ b/pypy/interpreter/astcompiler/assemble.py @@ -5,6 +5,8 @@ from pypy.interpreter.astcompiler import ast, symtable from pypy.interpreter import pycode from pypy.tool import stdlib_opcode as ops +from rpython.tool import stdlib_opcode as ops2 +print dir(ops2) from pypy.interpreter.error import OperationError from rpython.rlib.objectmodel import we_are_translated diff --git a/pypy/tool/stdlib_opcode.py b/pypy/tool/stdlib_opcode.py --- a/pypy/tool/stdlib_opcode.py +++ b/pypy/tool/stdlib_opcode.py @@ -32,7 +32,7 @@ del load_pypy_opcode bytecode_spec = BytecodeSpec('pypy', opmap, HAVE_ARGUMENT) -bytecode_spec.to_globals() +bytecode_spec.to_globals(globals()) opcode_method_names = bytecode_spec.method_names opcodedesc = bytecode_spec.opcodedesc From noreply at buildbot.pypy.org Thu Jan 10 00:21:23 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Thu, 10 Jan 2013 00:21:23 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Forgot to save file. Added globals parameter to BytecodeSpec Message-ID: <20130109232123.76A2E1C0041@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59908:431d3124d089 Date: 2013-01-10 00:20 +0100 http://bitbucket.org/pypy/pypy/changeset/431d3124d089/ Log: Forgot to save file. Added globals parameter to BytecodeSpec diff --git a/rpython/tool/stdlib_opcode.py b/rpython/tool/stdlib_opcode.py --- a/rpython/tool/stdlib_opcode.py +++ b/rpython/tool/stdlib_opcode.py @@ -69,13 +69,12 @@ self.ordered_opdescs = lst = self.opdescmap.values() lst.sort() - def to_globals(self): + def to_globals(self, globals_dict): """NOT_RPYTHON. Add individual opcodes to the module constants.""" - g = globals() - g.update(self.opmap) - g['SLICE'] = self.opmap["SLICE+0"] - g['STORE_SLICE'] = self.opmap["STORE_SLICE+0"] - g['DELETE_SLICE'] = self.opmap["DELETE_SLICE+0"] + globals_dict.update(self.opmap) + globals_dict['SLICE'] = self.opmap["SLICE+0"] + globals_dict['STORE_SLICE'] = self.opmap["STORE_SLICE+0"] + globals_dict['DELETE_SLICE'] = self.opmap["DELETE_SLICE+0"] def __str__(self): return "<%s bytecode>" % (self.name,) From noreply at buildbot.pypy.org Thu Jan 10 10:26:48 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 10 Jan 2013 10:26:48 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: try some luck? Message-ID: <20130110092648.7396F1C0285@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59909:2a8d98927e5e Date: 2013-01-10 11:26 +0200 http://bitbucket.org/pypy/pypy/changeset/2a8d98927e5e/ Log: try some luck? diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py --- a/pypy/rpython/lltypesystem/rdict.py +++ b/pypy/rpython/lltypesystem/rdict.py @@ -352,6 +352,7 @@ MAX_SHORT_MASK = ~(2 ** 16 - 1) MAX_BYTE_MASK = ~(2 ** 8 - 1) + at jit.look_inside_iff(lambda n : jit.isconstant(n)) def _ll_malloc_indexes(n): # XXXX 64 bit only #if n & MAX_INT_MASK: @@ -447,11 +448,17 @@ def ll_keyhash_custom(d, key): DICT = lltype.typeOf(d).TO - return objectmodel.hlinvoke(DICT.r_rdict_hashfn, d.fnkeyhash, key) + if objectmodel.we_are_translated(): + return objectmodel.hlinvoke(DICT.r_rdict_hashfn, d.fnkeyhash, key) + else: + return DICT.r_rdict_hashfn(d.fnkeyhash, key) def ll_keyeq_custom(d, key1, key2): DICT = lltype.typeOf(d).TO - return objectmodel.hlinvoke(DICT.r_rdict_eqfn, d.fnkeyeq, key1, key2) + if objectmodel.we_are_translated(): + return objectmodel.hlinvoke(DICT.r_rdict_eqfn, d.fnkeyeq, key1, key2) + else: + DICT.r_rdict_eqfn(d.fnkeyeq, key1, key2) def ll_dict_len(d): return d.num_items @@ -588,7 +595,6 @@ pos = ll_dict_lookup_clean(d, old_entries.hash(index)) ll_index_setitem(d.size, indexes, pos, index) i += 1 -ll_dict_resize.oopspec = 'dict.resize(d)' # ------- a port of CPython's dictobject.c's lookdict implementation ------- PERTURB_SHIFT = 5 @@ -834,7 +840,6 @@ #rgc.ll_arraycopy(dict.indexes, d.indexes, 0, 0, len(dict.indexes)) rgc.ll_arraycopy(dict.entries, d.entries, 0, 0, dict.num_items) return d -ll_copy.oopspec = 'dict.copy(dict)' def ll_clear(d): if (d.size == DICT_INITSIZE and @@ -846,7 +851,6 @@ d.size = DICT_INITSIZE d.num_items = 0 d.resize_counter = DICT_RESIZE_START -ll_clear.oopspec = 'dict.clear(d)' def ll_update(dic1, dic2): entries = dic2.entries @@ -859,7 +863,6 @@ j = ll_dict_lookup(dic1, key, hash) _ll_dict_setitem_lookup_done(dic1, key, entry.value, hash, j) i += 1 -ll_update.oopspec = 'dict.update(dic1, dic2)' # this is an implementation of keys(), values() and items() # in a single function. @@ -894,7 +897,6 @@ items[i] = recast(ELEM, entry.value) i += 1 return res - ll_kvi.oopspec = 'dict.%s(dic)' % kind return ll_kvi ll_dict_keys = _make_ll_keys_values_items('keys') From noreply at buildbot.pypy.org Thu Jan 10 10:54:08 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 10 Jan 2013 10:54:08 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: eh, what the fuck Message-ID: <20130110095408.521B91C11B7@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59910:52e561f89bbf Date: 2013-01-10 11:52 +0200 http://bitbucket.org/pypy/pypy/changeset/52e561f89bbf/ Log: eh, what the fuck diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py --- a/pypy/rpython/lltypesystem/rdict.py +++ b/pypy/rpython/lltypesystem/rdict.py @@ -448,17 +448,11 @@ def ll_keyhash_custom(d, key): DICT = lltype.typeOf(d).TO - if objectmodel.we_are_translated(): - return objectmodel.hlinvoke(DICT.r_rdict_hashfn, d.fnkeyhash, key) - else: - return DICT.r_rdict_hashfn(d.fnkeyhash, key) + return objectmodel.hlinvoke(DICT.r_rdict_hashfn, d.fnkeyhash, key) def ll_keyeq_custom(d, key1, key2): DICT = lltype.typeOf(d).TO - if objectmodel.we_are_translated(): - return objectmodel.hlinvoke(DICT.r_rdict_eqfn, d.fnkeyeq, key1, key2) - else: - DICT.r_rdict_eqfn(d.fnkeyeq, key1, key2) + return objectmodel.hlinvoke(DICT.r_rdict_eqfn, d.fnkeyeq, key1, key2) def ll_dict_len(d): return d.num_items From noreply at buildbot.pypy.org Thu Jan 10 11:14:16 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 10 Jan 2013 11:14:16 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: I think I got it Message-ID: <20130110101416.5531E1C00E2@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59911:568aae06d7c0 Date: 2013-01-10 12:13 +0200 http://bitbucket.org/pypy/pypy/changeset/568aae06d7c0/ Log: I think I got it diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py --- a/pypy/rpython/lltypesystem/rdict.py +++ b/pypy/rpython/lltypesystem/rdict.py @@ -196,7 +196,11 @@ for dictkeycontainer, dictvalue in dictobj._dict.items(): llkey = r_key.convert_const(dictkeycontainer.key) llvalue = r_value.convert_const(dictvalue) - ll_dict_setitem(l_dict, llkey, llvalue) + # we cannot call normal ll_dict_setitem here + # because calling keyhash might potentially + # be illegal + ll_dict_insertclean(l_dict, llkey, llvalue, + dictkeycontainer.hash) return l_dict else: @@ -471,8 +475,30 @@ def ll_dict_setitem(d, key, value): hash = d.keyhash(key) i = ll_dict_lookup(d, key, hash) - res = _ll_dict_setitem_lookup_done(d, key, value, hash, i) - return res + return _ll_dict_setitem_lookup_done(d, key, value, hash, i) + +def ll_dict_insertclean(d, key, value, hash): + i = ll_dict_lookup_clean(d, key, hash) + return _ll_dict_setitem_lookup_done(d, key, value, hash, i) + +def ll_dict_lookup_clean(d, hash): + # a simplified version of ll_dict_lookup() which assumes that the + # key is new, and the dictionary doesn't contain deleted entries. + # It only finds the next free slot for the given hash. + + # this is crucial during convert_const, where we cannot call keyhash + # directly. Unused otherwise + + indexes = d.indexes + mask = len(indexes) - 1 + i = hash & mask + perturb = r_uint(hash) + while indexes[i] >= 0: + i = r_uint(i) + i = (i << 2) + i + perturb + 1 + i = intmask(i) & mask + perturb >>= PERTURB_SHIFT + return i def _look_inside_setitem(d, key, value, hash, i): return jit.isvirtual(d) and jit.isconstant(key) From noreply at buildbot.pypy.org Thu Jan 10 11:23:40 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 10 Jan 2013 11:23:40 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: fix, I need to write a test for that Message-ID: <20130110102340.59CB81C00E2@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59912:4fe4f7845b00 Date: 2013-01-10 12:23 +0200 http://bitbucket.org/pypy/pypy/changeset/4fe4f7845b00/ Log: fix, I need to write a test for that diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py --- a/pypy/rpython/lltypesystem/rdict.py +++ b/pypy/rpython/lltypesystem/rdict.py @@ -478,7 +478,7 @@ return _ll_dict_setitem_lookup_done(d, key, value, hash, i) def ll_dict_insertclean(d, key, value, hash): - i = ll_dict_lookup_clean(d, key, hash) + i = ll_dict_lookup_clean(d, hash) return _ll_dict_setitem_lookup_done(d, key, value, hash, i) def ll_dict_lookup_clean(d, hash): From noreply at buildbot.pypy.org Thu Jan 10 11:29:38 2013 From: noreply at buildbot.pypy.org (antocuni) Date: Thu, 10 Jan 2013 11:29:38 +0100 (CET) Subject: [pypy-commit] pypy virtual-raw-mallocs: hg merge default. IN-PROGRESS: a couple of things broke because of remove-globals-in-jit, need to fix them Message-ID: <20130110102938.F22671C00E2@cobra.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: virtual-raw-mallocs Changeset: r59913:80b6a6da5ea0 Date: 2013-01-10 11:00 +0100 http://bitbucket.org/pypy/pypy/changeset/80b6a6da5ea0/ Log: hg merge default. IN-PROGRESS: a couple of things broke because of remove-globals-in-jit, need to fix them diff too long, truncating to 2000 out of 13934 lines diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -28,7 +28,7 @@ DEALINGS IN THE SOFTWARE. -PyPy Copyright holders 2003-2012 +PyPy Copyright holders 2003-2013 ----------------------------------- Except when otherwise stated (look for LICENSE files or information at diff --git a/lib-python/2.7/ctypes/test/test_internals.py b/lib-python/2.7/ctypes/test/test_internals.py --- a/lib-python/2.7/ctypes/test/test_internals.py +++ b/lib-python/2.7/ctypes/test/test_internals.py @@ -1,7 +1,10 @@ # This tests the internal _objects attribute import unittest from ctypes import * -from sys import getrefcount as grc +try: + from sys import getrefcount as grc +except ImportError: + grc = None # e.g. PyPy # XXX This test must be reviewed for correctness!!! @@ -22,6 +25,8 @@ self.assertEqual(id(a), id(b)) def test_ints(self): + if grc is None: + return unittest.skip("no sys.getrefcount()") i = 42000123 refcnt = grc(i) ci = c_int(i) @@ -29,6 +34,8 @@ self.assertEqual(ci._objects, None) def test_c_char_p(self): + if grc is None: + return unittest.skip("no sys.getrefcount()") s = "Hello, World" refcnt = grc(s) cs = c_char_p(s) diff --git a/lib-python/2.7/ctypes/test/test_memfunctions.py b/lib-python/2.7/ctypes/test/test_memfunctions.py --- a/lib-python/2.7/ctypes/test/test_memfunctions.py +++ b/lib-python/2.7/ctypes/test/test_memfunctions.py @@ -53,7 +53,8 @@ s = string_at("foo bar") # XXX The following may be wrong, depending on how Python # manages string instances - self.assertEqual(2, sys.getrefcount(s)) + if hasattr(sys, 'getrefcount'): + self.assertEqual(2, sys.getrefcount(s)) self.assertTrue(s, "foo bar") self.assertEqual(string_at("foo bar", 8), "foo bar\0") diff --git a/lib-python/2.7/ctypes/test/test_python_api.py b/lib-python/2.7/ctypes/test/test_python_api.py --- a/lib-python/2.7/ctypes/test/test_python_api.py +++ b/lib-python/2.7/ctypes/test/test_python_api.py @@ -9,7 +9,10 @@ ################################################################ -from sys import getrefcount as grc +try: + from sys import getrefcount as grc +except ImportError: + grc = None # e.g. PyPy if sys.version_info > (2, 4): c_py_ssize_t = c_size_t else: diff --git a/lib-python/2.7/ctypes/test/test_refcounts.py b/lib-python/2.7/ctypes/test/test_refcounts.py --- a/lib-python/2.7/ctypes/test/test_refcounts.py +++ b/lib-python/2.7/ctypes/test/test_refcounts.py @@ -11,7 +11,10 @@ class RefcountTestCase(unittest.TestCase): def test_1(self): - from sys import getrefcount as grc + try: + from sys import getrefcount as grc + except ImportError: + return unittest.skip("no sys.getrefcount()") f = dll._testfunc_callback_i_if f.restype = ctypes.c_int @@ -35,7 +38,10 @@ def test_refcount(self): - from sys import getrefcount as grc + try: + from sys import getrefcount as grc + except ImportError: + return unittest.skip("no sys.getrefcount()") def func(*args): pass # this is the standard refcount for func @@ -84,6 +90,10 @@ class AnotherLeak(unittest.TestCase): def test_callback(self): import sys + try: + from sys import getrefcount + except ImportError: + return unittest.skip("no sys.getrefcount()") proto = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int, ctypes.c_int) def func(a, b): diff --git a/lib-python/2.7/sre_parse.py b/lib-python/2.7/sre_parse.py --- a/lib-python/2.7/sre_parse.py +++ b/lib-python/2.7/sre_parse.py @@ -16,6 +16,12 @@ from sre_constants import * +try: + from __pypy__ import newdict +except ImportError: + def newdict(tp): + return {} + SPECIAL_CHARS = ".\\[{()*+?^$|" REPEAT_CHARS = "*+?{" @@ -68,7 +74,7 @@ self.flags = 0 self.open = [] self.groups = 1 - self.groupdict = {} + self.groupdict = newdict("module") def opengroup(self, name=None): gid = self.groups self.groups = gid + 1 diff --git a/lib_pypy/_ctypes_test.py b/lib_pypy/_ctypes_test.py --- a/lib_pypy/_ctypes_test.py +++ b/lib_pypy/_ctypes_test.py @@ -2,10 +2,6 @@ import tempfile import gc -# Monkeypatch & hacks to let ctypes.tests import. -# This should be removed at some point. -sys.getrefcount = lambda x: len(gc.get_referrers(x)) - 1 - def compile_shared(): """Compile '_ctypes_test.c' into an extension module, and import it """ diff --git a/lib_pypy/numpypy/core/arrayprint.py b/lib_pypy/numpypy/core/arrayprint.py --- a/lib_pypy/numpypy/core/arrayprint.py +++ b/lib_pypy/numpypy/core/arrayprint.py @@ -248,9 +248,9 @@ 'int' : IntegerFormat(data), 'float' : FloatFormat(data, precision, suppress_small), 'longfloat' : LongFloatFormat(precision), - #'complexfloat' : ComplexFormat(data, precision, - # suppress_small), - #'longcomplexfloat' : LongComplexFormat(precision), + 'complexfloat' : ComplexFormat(data, precision, + suppress_small), + 'longcomplexfloat' : LongComplexFormat(precision), 'datetime' : DatetimeFormat(data), 'timedelta' : TimedeltaFormat(data), 'numpystr' : repr_format, @@ -294,19 +294,19 @@ #else: format_function = formatdict['int'] elif issubclass(dtypeobj, _nt.floating): - #if issubclass(dtypeobj, _nt.longfloat): - # format_function = formatdict['longfloat'] - #else: - format_function = formatdict['float'] - #elif issubclass(dtypeobj, _nt.complexfloating): - # if issubclass(dtypeobj, _nt.clongfloat): - # format_function = formatdict['longcomplexfloat'] - # else: - # format_function = formatdict['complexfloat'] + if issubclass(dtypeobj, _nt.longfloat): + format_function = formatdict['longfloat'] + else: + format_function = formatdict['float'] + elif issubclass(dtypeobj, _nt.complexfloating): + if issubclass(dtypeobj, _nt.clongfloat): + format_function = formatdict['longcomplexfloat'] + else: + format_function = formatdict['complexfloat'] elif issubclass(dtypeobj, (_nt.unicode_, _nt.string_)): format_function = formatdict['numpystr'] - elif issubclass(dtypeobj, _nt.datetime64): - format_function = formatdict['datetime'] + #elif issubclass(dtypeobj, _nt.datetime64): + # format_function = formatdict['datetime'] else: format_function = formatdict['str'] diff --git a/pypy/annotation/binaryop.py b/pypy/annotation/binaryop.py --- a/pypy/annotation/binaryop.py +++ b/pypy/annotation/binaryop.py @@ -10,7 +10,7 @@ from pypy.annotation.model import SomeUnicodeCodePoint, SomeUnicodeString from pypy.annotation.model import SomeTuple, SomeImpossibleValue, s_ImpossibleValue from pypy.annotation.model import SomeInstance, SomeBuiltin, SomeIterator -from pypy.annotation.model import SomePBC, SomeFloat, s_None +from pypy.annotation.model import SomePBC, SomeFloat, s_None, SomeByteArray from pypy.annotation.model import SomeExternalObject, SomeWeakRef from pypy.annotation.model import SomeAddress, SomeTypedAddressAccess from pypy.annotation.model import SomeSingleFloat, SomeLongFloat, SomeType @@ -415,6 +415,34 @@ result.const = str1.const + str2.const return result +class __extend__(pairtype(SomeByteArray, SomeByteArray)): + def union((b1, b2)): + can_be_None = b1.can_be_None or b2.can_be_None + return SomeByteArray(can_be_None=can_be_None) + + def add((b1, b2)): + result = SomeByteArray() + if b1.is_immutable_constant() and b2.is_immutable_constant(): + result.const = b1.const + b2.const + return result + +class __extend__(pairtype(SomeByteArray, SomeInteger)): + def getitem((s_b, s_i)): + return SomeInteger() + + def setitem((s_b, s_i), s_i2): + assert isinstance(s_i2, SomeInteger) + +class __extend__(pairtype(SomeString, SomeByteArray), + pairtype(SomeByteArray, SomeString), + pairtype(SomeChar, SomeByteArray), + pairtype(SomeByteArray, SomeChar)): + def add((b1, b2)): + result = SomeByteArray() + if b1.is_immutable_constant() and b2.is_immutable_constant(): + result.const = b1.const + b2.const + return result + class __extend__(pairtype(SomeChar, SomeChar)): def union((chr1, chr2)): diff --git a/pypy/annotation/bookkeeper.py b/pypy/annotation/bookkeeper.py --- a/pypy/annotation/bookkeeper.py +++ b/pypy/annotation/bookkeeper.py @@ -13,7 +13,7 @@ SomeUnicodeCodePoint, SomeOOStaticMeth, s_None, s_ImpossibleValue, \ SomeLLADTMeth, SomeBool, SomeTuple, SomeOOClass, SomeImpossibleValue, \ SomeUnicodeString, SomeList, SomeObject, HarmlesslyBlocked, \ - SomeWeakRef, lltype_to_annotation, SomeType + SomeWeakRef, lltype_to_annotation, SomeType, SomeByteArray from pypy.annotation.classdef import InstanceSource, ClassDef from pypy.annotation.listdef import ListDef, ListItem from pypy.annotation.dictdef import DictDef @@ -349,6 +349,8 @@ result = SomeUnicodeCodePoint() else: result = SomeUnicodeString() + elif tp is bytearray: + result = SomeByteArray() elif tp is tuple: result = SomeTuple(items = [self.immutablevalue(e, need_const) for e in x]) elif tp is float: diff --git a/pypy/annotation/builtin.py b/pypy/annotation/builtin.py --- a/pypy/annotation/builtin.py +++ b/pypy/annotation/builtin.py @@ -9,7 +9,7 @@ from pypy.annotation.model import SomeFloat, unionof, SomeUnicodeString from pypy.annotation.model import SomePBC, SomeInstance, SomeDict, SomeList from pypy.annotation.model import SomeWeakRef, SomeIterator -from pypy.annotation.model import SomeOOObject +from pypy.annotation.model import SomeOOObject, SomeByteArray from pypy.annotation.model import annotation_to_lltype, lltype_to_annotation, ll_to_annotation from pypy.annotation.model import add_knowntypedata from pypy.annotation.model import s_ImpossibleValue @@ -119,6 +119,9 @@ def builtin_unicode(s_unicode): return constpropagate(unicode, [s_unicode], SomeUnicodeString()) +def builtin_bytearray(s_str): + return constpropagate(bytearray, [s_str], SomeByteArray()) + def our_issubclass(cls1, cls2): """ we're going to try to be less silly in the face of old-style classes""" from pypy.annotation.classdef import ClassDef @@ -253,24 +256,6 @@ s = SomeInteger(nonneg=True, knowntype=s.knowntype) return s -def builtin_apply(*stuff): - getbookkeeper().warning("ignoring apply%r" % (stuff,)) - return SomeObject() - -##def builtin_slice(*args): -## bk = getbookkeeper() -## if len(args) == 1: -## return SomeSlice( -## bk.immutablevalue(None), args[0], bk.immutablevalue(None)) -## elif len(args) == 2: -## return SomeSlice( -## args[0], args[1], bk.immutablevalue(None)) -## elif len(args) == 3: -## return SomeSlice( -## args[0], args[1], args[2]) -## else: -## raise Exception, "bogus call to slice()" - def OSError_init(s_self, *args): pass diff --git a/pypy/annotation/model.py b/pypy/annotation/model.py --- a/pypy/annotation/model.py +++ b/pypy/annotation/model.py @@ -241,6 +241,9 @@ "Stands for an object which is known to be an unicode string" knowntype = unicode +class SomeByteArray(SomeStringOrUnicode): + knowntype = bytearray + class SomeChar(SomeString): "Stands for an object known to be a string of length 1." can_be_None = False diff --git a/pypy/annotation/test/test_annrpython.py b/pypy/annotation/test/test_annrpython.py --- a/pypy/annotation/test/test_annrpython.py +++ b/pypy/annotation/test/test_annrpython.py @@ -3819,6 +3819,37 @@ a = self.RPythonAnnotator() a.build_types(f, []) # assert did not explode + def test_bytearray(self): + def f(): + return bytearray("xyz") + + a = self.RPythonAnnotator() + assert isinstance(a.build_types(f, []), annmodel.SomeByteArray) + + def test_bytearray_add(self): + def f(a): + return a + bytearray("xyz") + + a = self.RPythonAnnotator() + assert isinstance(a.build_types(f, [annmodel.SomeByteArray()]), + annmodel.SomeByteArray) + a = self.RPythonAnnotator() + assert isinstance(a.build_types(f, [str]), + annmodel.SomeByteArray) + a = self.RPythonAnnotator() + assert isinstance(a.build_types(f, [annmodel.SomeChar()]), + annmodel.SomeByteArray) + + def test_bytearray_setitem_getitem(self): + def f(b, i, c): + b[i] = c + return b[i + 1] + + a = self.RPythonAnnotator() + assert isinstance(a.build_types(f, [annmodel.SomeByteArray(), + int, int]), + annmodel.SomeInteger) + def g(n): return [0,1,2,n] diff --git a/pypy/doc/Makefile b/pypy/doc/Makefile --- a/pypy/doc/Makefile +++ b/pypy/doc/Makefile @@ -32,38 +32,38 @@ -rm -rf $(BUILDDIR)/* html: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." pickle: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ @@ -73,7 +73,7 @@ @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/PyPy.qhc" latex: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @@ -81,26 +81,26 @@ "run these through (pdf)latex." man: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man" changes: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." doctest: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -38,7 +38,7 @@ # General information about the project. project = u'PyPy' -copyright = u'2011, The PyPy Project' +copyright = u'2013, The PyPy Project' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/pypy/doc/config/translation.no__thread.txt b/pypy/doc/config/translation.no__thread.txt --- a/pypy/doc/config/translation.no__thread.txt +++ b/pypy/doc/config/translation.no__thread.txt @@ -1,3 +1,3 @@ Don't use gcc __thread attribute for fast thread local storage -implementation . Increases the chance that moving the resulting +implementation. Increases the chance that moving the resulting executable to another same processor Linux machine will work. diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst --- a/pypy/doc/index.rst +++ b/pypy/doc/index.rst @@ -67,7 +67,7 @@ .. _Python: http://docs.python.org/index.html .. _`more...`: architecture.html#mission-statement .. _`PyPy blog`: http://morepypy.blogspot.com/ -.. _`development bug/feature tracker`: https://codespeak.net/issue/pypy-dev/ +.. _`development bug/feature tracker`: https://bugs.pypy.org .. _here: http://tismerysoft.de/pypy/irc-logs/pypy .. _`Mercurial commit mailing list`: http://python.org/mailman/listinfo/pypy-commit .. _`development mailing list`: http://python.org/mailman/listinfo/pypy-dev diff --git a/pypy/doc/pypyconfig.py b/pypy/doc/pypyconfig.py --- a/pypy/doc/pypyconfig.py +++ b/pypy/doc/pypyconfig.py @@ -3,6 +3,10 @@ def setup(app): import sys, os sys.path.insert(0, os.path.abspath("../../")) + + #Autmatically calls make_cmdlline_overview + from pypy.doc.config import generate + from pypy.config import makerestdoc import py role = makerestdoc.register_config_role(py.path.local()) diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -5,15 +5,35 @@ .. this is a revision shortly after release-2.0-beta1 .. startrev: 0e6161a009c6 +.. branch: callback-jit +Callbacks from C are now better JITted + +.. branch: remove-globals-in-jit + .. branch: length-hint Implement __lenght_hint__ according to PEP 424 +.. branch: numpypy-longdouble +Long double support for numpypy + +.. branch: signatures +Improved RPython typing + +.. branch: rpython-bytearray +Rudimentary support for bytearray in RPython + .. branches we don't care about .. branch: autoreds +.. branch: reflex-support +.. branch: kill-faking +.. branch: improved_ebnfparse_error +.. branch: task-decorator .. branch: release-2.0-beta1 .. branch: remove-PYPY_NOT_MAIN_FILE +.. branch: missing-jit-operations + .. branch: fix-lookinside-iff-oopspec Fixed the interaction between two internal tools for controlling the JIT. diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -56,11 +56,24 @@ ---------------------------- On Windows, there is no standard place where to download, build and -install third-party libraries. We chose to install them in the parent +install third-party libraries. We recommend installing them in the parent directory of the pypy checkout. For example, if you installed pypy in ``d:\pypy\trunk\`` (This directory contains a README file), the base -directory is ``d:\pypy``. You may choose different values by setting the -INCLUDE, LIB and PATH (for DLLs) +directory is ``d:\pypy``. You must then set the +INCLUDE, LIB and PATH (for DLLs) environment variables appropriately. + +Abridged method (for -Ojit builds using Visual Studio 2008) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Download the versions of all the external packages +from +https://bitbucket.org/pypy/pypy/downloads/local.zip +Then expand it into the base directory (base_dir) and modify your environment to reflect this:: + + set PATH=\bin;%PATH% + set INCLUDE=\include;%INCLUDE% + set LIB=\lib;%LIB% + +Now you should be good to go. Read on for more information. The Boehm garbage collector ~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py --- a/pypy/interpreter/typedef.py +++ b/pypy/interpreter/typedef.py @@ -105,6 +105,13 @@ # typical usage (case 1 is the most common kind of app-level subclasses; # case 2 is the memory-saving kind defined with __slots__). # +# +----------------------------------------------------------------+ +# | NOTE: if withmapdict is enabled, the following doesn't apply! | +# | Map dicts can flexibly allow any slots/__dict__/__weakref__ to | +# | show up only when needed. In particular there is no way with | +# | mapdict to prevent some objects from being weakrefable. | +# +----------------------------------------------------------------+ +# # dict slots del weakrefable # # 1. Y N N Y UserDictWeakref diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py --- a/pypy/jit/backend/arm/assembler.py +++ b/pypy/jit/backend/arm/assembler.py @@ -1,5 +1,6 @@ from __future__ import with_statement import os +from pypy.jit.backend.llsupport import jitframe from pypy.jit.backend.arm.helper.assembler import saved_registers from pypy.jit.backend.arm import conditions as c from pypy.jit.backend.arm import registers as r @@ -8,7 +9,7 @@ from pypy.jit.backend.arm.codebuilder import ARMv7Builder, OverwritingBuilder from pypy.jit.backend.arm.locations import get_fp_offset from pypy.jit.backend.arm.regalloc import (Regalloc, ARMFrameManager, - ARMv7RegisterManager, check_imm_arg, + CoreRegisterManager, check_imm_arg, operations as regalloc_operations, operations_with_guard as regalloc_operations_with_guard) from pypy.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper @@ -25,12 +26,12 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.jit.backend.arm.opassembler import ResOpAssembler from pypy.rlib.debug import (debug_print, debug_start, debug_stop, - have_debug_prints) + have_debug_prints, fatalerror) from pypy.rlib.jit import AsmInfo from pypy.rlib.objectmodel import compute_unique_id # XXX Move to llsupport -from pypy.jit.backend.x86.support import values_array, memcpy_fn +from pypy.jit.backend.x86.support import memcpy_fn DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', ('i', lltype.Signed), ('type', lltype.Char), # 'b'ridge, 'l'abel or @@ -44,14 +45,8 @@ debug = True - def __init__(self, cpu, failargs_limit=1000): + def __init__(self, cpu, translate_support_code=False): self.cpu = cpu - self.fail_boxes_int = values_array(lltype.Signed, failargs_limit) - self.fail_boxes_float = values_array(longlong.FLOATSTORAGE, - failargs_limit) - self.fail_boxes_ptr = values_array(llmemory.GCREF, failargs_limit) - self.fail_boxes_count = 0 - self.fail_force_index = 0 self.setup_failure_recovery() self.mc = None self.memcpy_addr = 0 @@ -68,6 +63,7 @@ self._debug = False self.loop_run_counters = [] self.debug_counter_descr = cpu.fielddescrof(DEBUG_COUNTER, 'i') + self.force_token_to_dead_frame = {} # XXX temporary hack def set_debug(self, v): r = self._debug @@ -112,9 +108,13 @@ gc_ll_descr.initialize() self._build_wb_slowpath(False) self._build_wb_slowpath(True) + self._build_failure_recovery(exc=True, withfloats=False) + self._build_failure_recovery(exc=False, withfloats=False) if self.cpu.supports_floats: self._build_wb_slowpath(False, withfloats=True) self._build_wb_slowpath(True, withfloats=True) + self._build_failure_recovery(exc=True, withfloats=True) + self._build_failure_recovery(exc=False, withfloats=True) self._build_propagate_exception_path() if gc_ll_descr.get_malloc_slowpath_addr is not None: self._build_malloc_slowpath() @@ -122,10 +122,7 @@ if gc_ll_descr.gcrootmap and gc_ll_descr.gcrootmap.is_shadow_stack: self._build_release_gil(gc_ll_descr.gcrootmap) self.memcpy_addr = self.cpu.cast_ptr_to_int(memcpy_fn) - self._exit_code_addr = self._gen_exit_path() - self._leave_jitted_hook_save_exc = \ - self._gen_leave_jitted_hook_code(True) - self._leave_jitted_hook = self._gen_leave_jitted_hook_code(False) + if not self._debug: # if self._debug is already set it means that someone called # set_debug by hand before initializing the assembler. Leave it @@ -220,36 +217,17 @@ self.releasegil_addr = rffi.cast(lltype.Signed, releasegil_func) self.reacqgil_addr = rffi.cast(lltype.Signed, reacqgil_func) - def _gen_leave_jitted_hook_code(self, save_exc): - mc = ARMv7Builder() - if self.cpu.supports_floats: - floats = r.caller_vfp_resp - else: - floats = [] - with saved_registers(mc, r.caller_resp + [r.lr], floats): - addr = self.cpu.get_on_leave_jitted_int(save_exception=save_exc) - mc.BL(addr) - assert self._exit_code_addr != 0 - mc.B(self._exit_code_addr) - return mc.materialize(self.cpu.asmmemmgr, [], - self.cpu.gc_ll_descr.gcrootmap) - def _build_propagate_exception_path(self): if self.cpu.propagate_exception_v < 0: return # not supported (for tests, or non-translated) # mc = ARMv7Builder() - # call on_leave_jitted_save_exc() - if self.cpu.supports_floats: - floats = r.caller_vfp_resp - else: - floats = [] - with saved_registers(mc, r.caller_resp + [r.lr], floats): - addr = self.cpu.get_on_leave_jitted_int(save_exception=True, - default_to_memoryerror=True) - mc.BL(addr) - mc.gen_load_int(r.ip.value, self.cpu.propagate_exception_v) - mc.MOV_rr(r.r0.value, r.ip.value) + # + # Call the helper, which will return a dead frame object with + # the correct exception set, or MemoryError by default + # XXX make sure we return the correct value here + addr = rffi.cast(lltype.Signed, self.cpu.get_propagate_exception()) + mc.BL(addr) self.gen_func_epilog(mc=mc) self.propagate_exception_path = mc.materialize(self.cpu.asmmemmgr, []) @@ -282,12 +260,12 @@ # restore registers and return # We check for c.EQ here, meaning all bits zero in this case mc.POP([reg.value for reg in r.argument_regs] + [r.pc.value], cond=c.EQ) - # call on_leave_jitted_save_exc() - addr = self.cpu.get_on_leave_jitted_int(save_exception=True) + # + # Call the helper, which will return a dead frame object with + # the correct exception set, or MemoryError by default + addr = rffi.cast(lltype.Signed, self.cpu.get_propagate_exception()) mc.BL(addr) # - mc.gen_load_int(r.r0.value, self.cpu.propagate_exception_v) - # # footer -- note the ADD, which skips the return address of this # function, and will instead return to the caller's caller. Note # also that we completely ignore the saved arguments, because we @@ -339,39 +317,82 @@ def setup_failure_recovery(self): - @rgc.no_collect + #@rgc.no_collect -- XXX still true, but hacked gc_set_extra_threshold def failure_recovery_func(mem_loc, frame_pointer, stack_pointer): """mem_loc is a structure in memory describing where the values for the failargs are stored. frame loc is the address of the frame pointer for the frame to be decoded frame """ - vfp_registers = rffi.cast(rffi.LONGLONGP, stack_pointer) - registers = rffi.ptradd(vfp_registers, len(r.all_vfp_regs)) + vfp_registers = rffi.cast(rffi.LONGP, stack_pointer) + registers = rffi.ptradd(vfp_registers, 2*len(r.all_vfp_regs)) registers = rffi.cast(rffi.LONGP, registers) - return self.decode_registers_and_descr(mem_loc, frame_pointer, + bytecode = rffi.cast(rffi.UCHARP, mem_loc) + return self.grab_frame_values(self.cpu, bytecode, frame_pointer, registers, vfp_registers) + self.failure_recovery_code = [0, 0, 0, 0] self.failure_recovery_func = failure_recovery_func - recovery_func_sign = lltype.Ptr(lltype.FuncType([lltype.Signed] * 3, - lltype.Signed)) + _FAILURE_RECOVERY_FUNC = lltype.Ptr(lltype.FuncType([rffi.LONGP] * 3, + llmemory.GCREF)) - @rgc.no_collect - def decode_registers_and_descr(self, mem_loc, frame_pointer, + @staticmethod + #@rgc.no_collect -- XXX still true, but hacked gc_set_extra_threshold + def grab_frame_values(cpu, bytecode, frame_pointer, registers, vfp_registers): - """Decode locations encoded in memory at mem_loc and write the values - to the failboxes. Values for spilled vars and registers are stored on - stack at frame_loc """ - assert frame_pointer & 1 == 0 - self.fail_force_index = frame_pointer - bytecode = rffi.cast(rffi.UCHARP, mem_loc) + # no malloc allowed here!! xxx apart from one, hacking a lot + force_index = rffi.cast(lltype.Signed, frame_pointer) num = 0 - value = 0 - fvalue = 0 + deadframe = lltype.nullptr(jitframe.DEADFRAME) + # step 1: lots of mess just to count the final value of 'num' + bytecode1 = bytecode + while 1: + code = rffi.cast(lltype.Signed, bytecode1[0]) + bytecode1 = rffi.ptradd(bytecode1, 1) + if code >= AssemblerARM.CODE_FROMSTACK: + while code > 0x7F: + code = rffi.cast(lltype.Signed, bytecode1[0]) + bytecode1 = rffi.ptradd(bytecode1, 1) + else: + kind = code & 3 + if kind == AssemblerARM.DESCR_SPECIAL: + if code == AssemblerARM.CODE_HOLE: + num += 1 + continue + if code == AssemblerARM.CODE_INPUTARG: + continue + if code == AssemblerARM.CODE_FORCED: + # resuming from a GUARD_NOT_FORCED + token = force_index + deadframe = ( + cpu.assembler.force_token_to_dead_frame.pop(token)) + deadframe = lltype.cast_opaque_ptr( + jitframe.DEADFRAMEPTR, deadframe) + continue + assert code == AssemblerARM.CODE_STOP + break + num += 1 + + # allocate the deadframe + if not deadframe: + # Remove the "reserve" at the end of the nursery. This means + # that it is guaranteed that the following malloc() works + # without requiring a collect(), but it needs to be re-added + # as soon as possible. + cpu.gc_clear_extra_threshold() + assert num <= cpu.get_failargs_limit() + try: + deadframe = lltype.malloc(jitframe.DEADFRAME, num) + except MemoryError: + fatalerror("memory usage error in grab_frame_values") + # fill it code_inputarg = False - while True: + num = 0 + value_hi = 0 + while 1: + # decode the next instruction from the bytecode code = rffi.cast(lltype.Signed, bytecode[0]) bytecode = rffi.ptradd(bytecode, 1) - if code >= self.CODE_FROMSTACK: + if code >= AssemblerARM.CODE_FROMSTACK: if code > 0x7F: shift = 7 code &= 0x7F @@ -384,54 +405,63 @@ break # load the value from the stack kind = code & 3 - code = int((code - self.CODE_FROMSTACK) >> 2) + code = (code - AssemblerARM.CODE_FROMSTACK) >> 2 if code_inputarg: code = ~code code_inputarg = False - if kind == self.DESCR_FLOAT: - # we use code + 1 to get the hi word of the double worded float - stackloc = frame_pointer - get_fp_offset(int(code) + 1) - assert stackloc & 3 == 0 - fvalue = rffi.cast(rffi.LONGLONGP, stackloc)[0] - else: - stackloc = frame_pointer - get_fp_offset(int(code)) - assert stackloc & 1 == 0 - value = rffi.cast(rffi.LONGP, stackloc)[0] + stackloc = force_index - get_fp_offset(int(code)) + value = rffi.cast(rffi.LONGP, stackloc)[0] + if kind == AssemblerARM.DESCR_FLOAT: + assert WORD == 4 + value_hi = value + value = rffi.cast(rffi.LONGP, stackloc - WORD)[0] else: - # 'code' identifies a register: load its value kind = code & 3 - if kind == self.DESCR_SPECIAL: - if code == self.CODE_HOLE: + if kind == AssemblerARM.DESCR_SPECIAL: + if code == AssemblerARM.CODE_HOLE: num += 1 continue - if code == self.CODE_INPUTARG: + if code == AssemblerARM.CODE_INPUTARG: code_inputarg = True continue - assert code == self.CODE_STOP + if code == AssemblerARM.CODE_FORCED: + continue + assert code == AssemblerARM.CODE_STOP break + # 'code' identifies a register: load its value code >>= 2 - if kind == self.DESCR_FLOAT: - fvalue = vfp_registers[code] + if kind == AssemblerARM.DESCR_FLOAT: + if WORD == 4: + value = vfp_registers[2*code] + value_hi = vfp_registers[2*code + 1] + else: + value = registers[code] else: value = registers[code] # store the loaded value into fail_boxes_ - if kind == self.DESCR_FLOAT: - tgt = self.fail_boxes_float.get_addr_for_num(num) - rffi.cast(rffi.LONGLONGP, tgt)[0] = fvalue + if kind == AssemblerARM.DESCR_INT: + deadframe.jf_values[num].int = value + elif kind == AssemblerARM.DESCR_REF: + deadframe.jf_values[num].ref = rffi.cast(llmemory.GCREF, value) + elif kind == AssemblerARM.DESCR_FLOAT: + assert WORD == 4 + assert not longlong.is_64_bit + floatvalue = rffi.cast(lltype.SignedLongLong, value_hi) + floatvalue <<= 32 + floatvalue |= rffi.cast(lltype.SignedLongLong, + rffi.cast(lltype.Unsigned, value)) + deadframe.jf_values[num].float = floatvalue else: - if kind == self.DESCR_INT: - tgt = self.fail_boxes_int.get_addr_for_num(num) - elif kind == self.DESCR_REF: - assert (value & 3) == 0, "misaligned pointer" - tgt = self.fail_boxes_ptr.get_addr_for_num(num) - else: - assert 0, "bogus kind" - rffi.cast(rffi.LONGP, tgt)[0] = value + assert 0, "bogus kind" num += 1 - self.fail_boxes_count = num + # + assert num == len(deadframe.jf_values) + if not we_are_translated(): + assert bytecode[4] == 0xCC fail_index = rffi.cast(rffi.INTP, bytecode)[0] - fail_index = rffi.cast(lltype.Signed, fail_index) - return fail_index + fail_descr = cpu.get_fail_descr_from_number(fail_index) + deadframe.jf_descr = fail_descr.hide(cpu) + return lltype.cast_opaque_ptr(llmemory.GCREF, deadframe) def decode_inputargs(self, code): descr_to_box_type = [REF, INT, FLOAT] @@ -492,10 +522,10 @@ # are stored in r0 and r1. mc.SUB_rr(r.r0.value, r.r1.value, r.r0.value) addr = self.cpu.gc_ll_descr.get_malloc_slowpath_addr() - for reg, ofs in ARMv7RegisterManager.REGLOC_TO_COPY_AREA_OFS.items(): + for reg, ofs in CoreRegisterManager.REGLOC_TO_COPY_AREA_OFS.items(): mc.STR_ri(reg.value, r.fp.value, imm=ofs) mc.BL(addr) - for reg, ofs in ARMv7RegisterManager.REGLOC_TO_COPY_AREA_OFS.items(): + for reg, ofs in CoreRegisterManager.REGLOC_TO_COPY_AREA_OFS.items(): mc.LDR_ri(reg.value, r.fp.value, imm=ofs) mc.CMP_ri(r.r0.value, 0) @@ -514,12 +544,26 @@ self.mc.CMP_ri(r.r0.value, 0) self.mc.B(self.propagate_exception_path, c=c.EQ) - def _gen_exit_path(self): + def _build_failure_recovery(self, exc, withfloats=False): mc = ARMv7Builder() - decode_registers_addr = llhelper(self.recovery_func_sign, + failure_recovery = llhelper(self._FAILURE_RECOVERY_FUNC, self.failure_recovery_func) self._insert_checks(mc) - with saved_registers(mc, r.all_regs, r.all_vfp_regs): + if withfloats: + f = r.all_vfp_regs + else: + f = [] + with saved_registers(mc, r.all_regs, f): + if exc: + # We might have an exception pending. Load it into r4 + # (this is a register saved across calls) + mc.gen_load_int(r.r5.value, self.cpu.pos_exc_value()) + mc.LDR_ri(r.r4.value, self.cpu.pos_exc_value()) + # clear the exc flags + mc.gen_load_int(r.r6.value, 0) + mc.STR_ri(r.r6.value, r.r5.value) + mc.gen_load_int(r.r5.value, self.cpu.pos_exception()) + mc.STR_ri(r.r6.value, r.r5.value) # move mem block address, to r0 to pass as mc.MOV_rr(r.r0.value, r.lr.value) # pass the current frame pointer as second param @@ -527,23 +571,32 @@ # pass the current stack pointer as third param mc.MOV_rr(r.r2.value, r.sp.value) self._insert_checks(mc) - mc.BL(rffi.cast(lltype.Signed, decode_registers_addr)) + mc.BL(rffi.cast(lltype.Signed, failure_recovery)) + if exc: + # save ebx into 'jf_guard_exc' + from pypy.jit.backend.llsupport.descr import unpack_fielddescr + descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) + offset, size, _ = unpack_fielddescr(descrs.jf_guard_exc) + mc.STR_rr(r.r4.value, r.r0.value, offset, cond=c.AL) mc.MOV_rr(r.ip.value, r.r0.value) mc.MOV_rr(r.r0.value, r.ip.value) self.gen_func_epilog(mc=mc) - return mc.materialize(self.cpu.asmmemmgr, [], + rawstart = mc.materialize(self.cpu.asmmemmgr, [], self.cpu.gc_ll_descr.gcrootmap) + self.failure_recovery_code[exc + 2 * withfloats] = rawstart + self.mc = None DESCR_REF = 0x00 DESCR_INT = 0x01 DESCR_FLOAT = 0x02 DESCR_SPECIAL = 0x03 CODE_FROMSTACK = 64 - CODE_STOP = 0 | DESCR_SPECIAL - CODE_HOLE = 4 | DESCR_SPECIAL - CODE_INPUTARG = 8 | DESCR_SPECIAL + CODE_STOP = 0 | DESCR_SPECIAL + CODE_HOLE = 4 | DESCR_SPECIAL + CODE_INPUTARG = 8 | DESCR_SPECIAL + CODE_FORCED = 12 | DESCR_SPECIAL #XXX where should this be written? - def gen_descr_encoding(self, descr, failargs, locs): + def write_failure_recovery_description(self, descr, failargs, locs): assert self.mc is not None for i in range(len(failargs)): arg = failargs[i] @@ -575,26 +628,33 @@ self.mc.writechar(chr(n)) self.mc.writechar(chr(self.CODE_STOP)) - fdescr = self.cpu.get_fail_descr_number(descr) - self.mc.write32(fdescr) + + def generate_quick_failure(self, guardtok, fcond=c.AL): + assert isinstance(guardtok.save_exc, bool) + fail_index = self.cpu.get_fail_descr_number(guardtok.descr) + startpos = self.mc.currpos() + withfloats = False + for box in guardtok.failargs: + if box is not None and box.type == FLOAT: + withfloats = True + break + exc = guardtok.save_exc + target = self.failure_recovery_code[exc + 2 * withfloats] + assert target != 0 + self.mc.BL(target) + # write tight data that describes the failure recovery + if guardtok.is_guard_not_forced: + self.mc.writechar(chr(self.CODE_FORCED)) + self.write_failure_recovery_description(guardtok.descr, + guardtok.failargs, guardtok.faillocs[1:]) + self.mc.write32(fail_index) + # for testing the decoding, write a final byte 0xCC + if not we_are_translated(): + self.mc.writechar('\xCC') + faillocs = [loc for loc in guardtok.faillocs if loc is not None] + guardtok.descr._arm_debug_faillocs = faillocs self.align() - - # assert that the fail_boxes lists are big enough - assert len(failargs) <= self.fail_boxes_int.SIZE - - def _gen_path_to_exit_path(self, descr, args, arglocs, - save_exc, fcond=c.AL): - assert isinstance(save_exc, bool) - self.gen_exit_code(self.mc, save_exc, fcond) - self.gen_descr_encoding(descr, args, arglocs[1:]) - - def gen_exit_code(self, mc, save_exc, fcond=c.AL): - assert isinstance(save_exc, bool) - if save_exc: - path = self._leave_jitted_hook_save_exc - else: - path = self._leave_jitted_hook - mc.BL(path) + return startpos def align(self): while(self.mc.currpos() % FUNC_ALIGN != 0): @@ -841,15 +901,10 @@ def write_pending_failure_recoveries(self): for tok in self.pending_guards: - descr = tok.descr #generate the exit stub and the encoded representation - pos = self.mc.currpos() - tok.pos_recovery_stub = pos - - self._gen_path_to_exit_path(descr, tok.failargs, - tok.faillocs, save_exc=tok.save_exc) + tok.pos_recovery_stub = self.generate_quick_failure(tok) # store info on the descr - descr._arm_current_frame_depth = tok.faillocs[0].getint() + tok.descr._arm_current_frame_depth = tok.faillocs[0].getint() def process_pending_guards(self, block_start): clt = self.current_clt @@ -860,7 +915,7 @@ descr._arm_failure_recovery_block = failure_recovery_pos relative_offset = tok.pos_recovery_stub - tok.offset guard_pos = block_start + tok.offset - if not tok.is_invalidate: + if not tok.is_guard_not_invalidated: # patch the guard jumpt to the stub # overwrite the generate NOP with a B_offs to the pos of the # stub @@ -1255,11 +1310,6 @@ else: raise AssertionError('Trying to pop to an invalid location') - def leave_jitted_hook(self): - ptrs = self.fail_boxes_ptr.ar - llop.gc_assume_young_pointers(lltype.Void, - llmemory.cast_ptr_to_adr(ptrs)) - def malloc_cond(self, nursery_free_adr, nursery_top_adr, size): assert size & (WORD-1) == 0 # must be correctly aligned diff --git a/pypy/jit/backend/arm/helper/regalloc.py b/pypy/jit/backend/arm/helper/regalloc.py --- a/pypy/jit/backend/arm/helper/regalloc.py +++ b/pypy/jit/backend/arm/helper/regalloc.py @@ -32,14 +32,14 @@ imm_a0 = check_imm_box(a0, imm_size, allow_zero=allow_zero) imm_a1 = check_imm_box(a1, imm_size, allow_zero=allow_zero) if not imm_a0 and imm_a1: - l0 = self._ensure_value_is_boxed(a0) + l0 = self.make_sure_var_in_reg(a0) l1 = self.convert_to_imm(a1) elif commutative and imm_a0 and not imm_a1: l1 = self.convert_to_imm(a0) - l0 = self._ensure_value_is_boxed(a1, boxes) + l0 = self.make_sure_var_in_reg(a1, boxes) else: - l0 = self._ensure_value_is_boxed(a0, boxes) - l1 = self._ensure_value_is_boxed(a1, boxes) + l0 = self.make_sure_var_in_reg(a0, boxes) + l1 = self.make_sure_var_in_reg(a1, boxes) self.possibly_free_vars_for_op(op) self.free_temp_vars() res = self.force_allocate_reg(op.result, boxes) @@ -52,10 +52,10 @@ if guard: def f(self, op, guard_op, fcond): locs = [] - loc1 = self._ensure_value_is_boxed(op.getarg(0)) + loc1 = self.make_sure_var_in_reg(op.getarg(0)) locs.append(loc1) if base: - loc2 = self._ensure_value_is_boxed(op.getarg(1)) + loc2 = self.make_sure_var_in_reg(op.getarg(1)) locs.append(loc2) self.possibly_free_vars_for_op(op) self.free_temp_vars() @@ -70,10 +70,10 @@ else: def f(self, op, fcond): locs = [] - loc1 = self._ensure_value_is_boxed(op.getarg(0)) + loc1 = self.make_sure_var_in_reg(op.getarg(0)) locs.append(loc1) if base: - loc2 = self._ensure_value_is_boxed(op.getarg(1)) + loc2 = self.make_sure_var_in_reg(op.getarg(1)) locs.append(loc2) self.possibly_free_vars_for_op(op) self.free_temp_vars() @@ -111,11 +111,11 @@ arg0, arg1 = boxes imm_a1 = check_imm_box(arg1) - l0 = self._ensure_value_is_boxed(arg0, forbidden_vars=boxes) + l0 = self.make_sure_var_in_reg(arg0, forbidden_vars=boxes) if imm_a1: l1 = self.convert_to_imm(arg1) else: - l1 = self._ensure_value_is_boxed(arg1, forbidden_vars=boxes) + l1 = self.make_sure_var_in_reg(arg1, forbidden_vars=boxes) self.possibly_free_vars_for_op(op) self.free_temp_vars() @@ -134,7 +134,7 @@ assert fcond is not None a0 = op.getarg(0) assert isinstance(a0, Box) - reg = self._ensure_value_is_boxed(a0) + reg = self.make_sure_var_in_reg(a0) self.possibly_free_vars_for_op(op) if guard_op is None: res = self.force_allocate_reg(op.result, [a0]) diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py --- a/pypy/jit/backend/arm/opassembler.py +++ b/pypy/jit/backend/arm/opassembler.py @@ -16,6 +16,7 @@ gen_emit_unary_float_op, saved_registers, count_reg_args) +from pypy.jit.backend.arm.helper.regalloc import check_imm_arg from pypy.jit.backend.arm.codebuilder import ARMv7Builder, OverwritingBuilder from pypy.jit.backend.arm.jump import remap_frame_layout from pypy.jit.backend.arm.regalloc import TempInt, TempPtr @@ -27,18 +28,20 @@ from pypy.jit.metainterp.history import JitCellToken, TargetToken from pypy.jit.metainterp.resoperation import rop from pypy.rlib.objectmodel import we_are_translated -from pypy.rpython.lltypesystem import rstr +from pypy.rlib import rgc +from pypy.rpython.lltypesystem import rstr, rffi, lltype, llmemory NO_FORCE_INDEX = -1 class GuardToken(object): - def __init__(self, descr, failargs, faillocs, offset, - save_exc, fcond=c.AL, is_invalidate=False): + def __init__(self, descr, failargs, faillocs, offset, save_exc, fcond=c.AL, + is_guard_not_invalidated=False, is_guard_not_forced=False): assert isinstance(save_exc, bool) self.descr = descr self.offset = offset - self.is_invalidate = is_invalidate + self.is_guard_not_invalidated = is_guard_not_invalidated + self.is_guard_not_forced = is_guard_not_forced self.failargs = failargs self.faillocs = faillocs self.save_exc = save_exc @@ -190,7 +193,8 @@ return fcond def _emit_guard(self, op, arglocs, fcond, save_exc, - is_guard_not_invalidated=False): + is_guard_not_invalidated=False, + is_guard_not_forced=False): assert isinstance(save_exc, bool) assert isinstance(fcond, int) descr = op.getdescr() @@ -210,7 +214,8 @@ faillocs=arglocs, offset=pos, save_exc=save_exc, - is_invalidate=is_guard_not_invalidated, + is_guard_not_invalidated=is_guard_not_invalidated, + is_guard_not_forced=is_guard_not_forced, fcond=fcond)) return c.AL @@ -312,49 +317,10 @@ return fcond def emit_op_finish(self, op, arglocs, regalloc, fcond): - for i in range(len(arglocs) - 1): - loc = arglocs[i] - box = op.getarg(i) - if loc is None: - continue - if loc.is_reg(): - if box.type == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(i) - elif box.type == INT: - adr = self.fail_boxes_int.get_addr_for_num(i) - else: - assert 0 - self.mc.gen_load_int(r.ip.value, adr) - self.mc.STR_ri(loc.value, r.ip.value) - elif loc.is_vfp_reg(): - assert box.type == FLOAT - adr = self.fail_boxes_float.get_addr_for_num(i) - self.mc.gen_load_int(r.ip.value, adr) - self.mc.VSTR(loc.value, r.ip.value) - elif loc.is_stack() or loc.is_imm() or loc.is_imm_float(): - if box.type == FLOAT: - adr = self.fail_boxes_float.get_addr_for_num(i) - self.mov_loc_loc(loc, r.vfp_ip) - self.mc.gen_load_int(r.ip.value, adr) - self.mc.VSTR(r.vfp_ip.value, r.ip.value) - elif box.type == REF or box.type == INT: - if box.type == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(i) - elif box.type == INT: - adr = self.fail_boxes_int.get_addr_for_num(i) - else: - assert 0 - self.mov_loc_loc(loc, r.ip) - self.mc.gen_load_int(r.lr.value, adr) - self.mc.STR_ri(r.ip.value, r.lr.value) - else: - assert 0 - # note: no exception should currently be set in llop.get_exception_addr - # even if this finish may be an exit_frame_with_exception (in this case - # the exception instance is in arglocs[0]). - addr = self.cpu.get_on_leave_jitted_int(save_exception=False) - self.mc.BL(addr) - self.mc.gen_load_int(r.r0.value, arglocs[-1].value) + [argloc] = arglocs + if argloc is not r.r0: #XXX verify this + self.mov_loc_loc(argloc, r.r0, fcond) + # exit function self.gen_func_epilog() return fcond @@ -988,8 +954,8 @@ def _emit_copystrcontent(self, op, regalloc, fcond, is_unicode): # compute the source address args = op.getarglist() - base_loc = regalloc._ensure_value_is_boxed(args[0], args) - ofs_loc = regalloc._ensure_value_is_boxed(args[2], args) + base_loc = regalloc.make_sure_var_in_reg(args[0], args) + ofs_loc = regalloc.make_sure_var_in_reg(args[2], args) assert args[0] is not args[1] # forbidden case of aliasing regalloc.possibly_free_var(args[0]) regalloc.free_temp_vars() @@ -1009,8 +975,8 @@ dstaddr_loc = regalloc.force_allocate_reg(dstaddr_box, selected_reg=r.r0) forbidden_vars.append(dstaddr_box) - base_loc = regalloc._ensure_value_is_boxed(args[1], forbidden_vars) - ofs_loc = regalloc._ensure_value_is_boxed(args[3], forbidden_vars) + base_loc = regalloc.make_sure_var_in_reg(args[1], forbidden_vars) + ofs_loc = regalloc.make_sure_var_in_reg(args[3], forbidden_vars) assert base_loc.is_reg() assert ofs_loc.is_reg() regalloc.possibly_free_var(args[1]) @@ -1026,7 +992,7 @@ # need the box here if isinstance(args[4], Box): length_box = args[4] - length_loc = regalloc._ensure_value_is_boxed(args[4], + length_loc = regalloc.make_sure_var_in_reg(args[4], forbidden_vars) else: length_box = TempInt() @@ -1152,8 +1118,26 @@ value = self.cpu.done_with_this_frame_float_v else: raise AssertionError(kind) - self.mc.gen_load_int(r.ip.value, value) - self.mc.CMP_rr(tmploc.value, r.ip.value) + from pypy.jit.backend.llsupport.descr import unpack_fielddescr + from pypy.jit.backend.llsupport.descr import unpack_interiorfielddescr + descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) + _offset, _size, _ = unpack_fielddescr(descrs.jf_descr) + fail_descr = self.cpu.get_fail_descr_from_number(value) + value = fail_descr.hide(self.cpu) + rgc._make_sure_does_not_move(value) + value = rffi.cast(lltype.Signed, value) + + if check_imm_arg(_offset): + self.mc.LDR_ri(r.ip.value, tmploc.value, imm=_offset) + else: + self.mc.gen_load_int(r.ip.value, _offset) + self.mc.LDR_rr(r.ip.value, tmploc.value, r.ip.value) + if check_imm_arg(value): + self.mc.CMP_ri(r.ip.value, imm=value) + else: + self.mc.gen_load_int(r.lr.value, value) + self.mc.CMP_rr(r.lr.value, r.ip.value) + #if values are equal we take the fast path # Slow path, calling helper @@ -1180,19 +1164,31 @@ if op.result is not None: # load the return value from fail_boxes_xxx[0] kind = op.result.type - if kind == INT: - adr = self.fail_boxes_int.get_addr_for_num(0) - elif kind == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(0) - elif kind == FLOAT: - adr = self.fail_boxes_float.get_addr_for_num(0) + if kind == FLOAT: + t = unpack_interiorfielddescr(descrs.as_float)[0] + if not check_imm_arg(t): + self.mc.gen_load_int(r.ip.value, t, cond=fast_path_cond) + self.mc.ADD_rr(r.ip.value, r.r0.value, r.ip.value, + cond=fast_path_cond) + t = 0 + base = r.ip + else: + base = r.r0 + self.mc.VLDR(resloc.value, base.value, imm=t, + cond=fast_path_cond) else: - raise AssertionError(kind) - self.mc.gen_load_int(r.ip.value, adr, cond=fast_path_cond) - if op.result.type == FLOAT: - self.mc.VLDR(resloc.value, r.ip.value, cond=fast_path_cond) - else: - self.mc.LDR_ri(resloc.value, r.ip.value, cond=fast_path_cond) + assert resloc is r.r0 + if kind == INT: + t = unpack_interiorfielddescr(descrs.as_int)[0] + else: + t = unpack_interiorfielddescr(descrs.as_ref)[0] + if not check_imm_arg(t): + self.mc.gen_load_int(r.ip.value, t, cond=fast_path_cond) + self.mc.LDR_rr(resloc.value, resloc.value, r.ip.value, + cond=fast_path_cond) + else: + self.mc.LDR_ri(resloc.value, resloc.value, imm=t, + cond=fast_path_cond) # jump to merge point jmp_pos = self.mc.currpos() self.mc.BKPT() @@ -1266,7 +1262,8 @@ self.mc.LDR_ri(r.ip.value, r.fp.value) self.mc.CMP_ri(r.ip.value, 0) - self._emit_guard(guard_op, arglocs[1 + numargs:], c.GE, save_exc=True) + self._emit_guard(guard_op, arglocs[1 + numargs:], c.GE, + save_exc=True, is_guard_not_forced=True) return fcond def emit_guard_call_release_gil(self, op, guard_op, arglocs, regalloc, diff --git a/pypy/jit/backend/arm/regalloc.py b/pypy/jit/backend/arm/regalloc.py --- a/pypy/jit/backend/arm/regalloc.py +++ b/pypy/jit/backend/arm/regalloc.py @@ -84,8 +84,28 @@ def void(self, op, fcond): return [] +class ARMRegisterManager(RegisterManager): + def return_constant(self, v, forbidden_vars=[], selected_reg=None): + self._check_type(v) + if isinstance(v, Const): + if isinstance(v, ConstPtr): + tp = REF + elif isinstance(v, ConstFloat): + tp = FLOAT + else: + tp = INT + loc = self.get_scratch_reg(tp, + self.temp_boxes + forbidden_vars, + selected_reg=selected_reg) + immvalue = self.convert_to_imm(v) + self.assembler.load(loc, immvalue) + return loc + else: + return RegisterManager.return_constant(self, v, + forbidden_vars, selected_reg) -class VFPRegisterManager(RegisterManager): + +class VFPRegisterManager(ARMRegisterManager): all_regs = r.all_vfp_regs box_types = [FLOAT] save_around_call_regs = r.all_vfp_regs @@ -107,20 +127,7 @@ reg = self.force_allocate_reg(v, selected_reg=r.d0) return reg - def ensure_value_is_boxed(self, thing, forbidden_vars=[]): - loc = None - if isinstance(thing, Const): - assert isinstance(thing, ConstFloat) - loc = self.get_scratch_reg(FLOAT, self.temp_boxes + forbidden_vars) - immvalue = self.convert_to_imm(thing) - self.assembler.load(loc, immvalue) - else: - loc = self.make_sure_var_in_reg(thing, - forbidden_vars=self.temp_boxes + forbidden_vars) - return loc - - def get_scratch_reg(self, type=FLOAT, forbidden_vars=[], - selected_reg=None): + def get_scratch_reg(self, type=FLOAT, forbidden_vars=[], selected_reg=None): assert type == FLOAT # for now box = TempFloat() self.temp_boxes.append(box) @@ -129,7 +136,7 @@ return reg -class ARMv7RegisterManager(RegisterManager): +class CoreRegisterManager(ARMRegisterManager): all_regs = r.all_regs box_types = None # or a list of acceptable types no_lower_byte_regs = all_regs @@ -162,22 +169,6 @@ return locations.ImmLocation(rffi.cast(lltype.Signed, c.value)) assert 0 - def ensure_value_is_boxed(self, thing, forbidden_vars=None): - loc = None - if isinstance(thing, Const): - if isinstance(thing, ConstPtr): - tp = REF - else: - tp = INT - loc = self.get_scratch_reg(tp, forbidden_vars=self.temp_boxes - + forbidden_vars) - immvalue = self.convert_to_imm(thing) - self.assembler.load(loc, immvalue) - else: - loc = self.make_sure_var_in_reg(thing, - forbidden_vars=self.temp_boxes + forbidden_vars) - return loc - def get_scratch_reg(self, type=INT, forbidden_vars=[], selected_reg=None): assert type == INT or type == REF box = TempBox() @@ -277,7 +268,12 @@ def make_sure_var_in_reg(self, var, forbidden_vars=[], selected_reg=None, need_lower_byte=False): - assert 0, 'should not be called directly' + if var.type == FLOAT: + return self.vfprm.make_sure_var_in_reg(var, forbidden_vars, + selected_reg, need_lower_byte) + else: + return self.rm.make_sure_var_in_reg(var, forbidden_vars, + selected_reg, need_lower_byte) def convert_to_imm(self, value): if isinstance(value, ConstInt): @@ -294,7 +290,7 @@ fm = self.frame_manager asm = self.assembler self.vfprm = VFPRegisterManager(longevity, fm, asm) - self.rm = ARMv7RegisterManager(longevity, fm, asm) + self.rm = CoreRegisterManager(longevity, fm, asm) def prepare_loop(self, inputargs, operations): self._prepare(inputargs, operations) @@ -426,12 +422,6 @@ self.rm.before_call(force_store, save_all_regs) self.vfprm.before_call(force_store, save_all_regs) - def _ensure_value_is_boxed(self, thing, forbidden_vars=[]): - if thing.type == FLOAT: - return self.vfprm.ensure_value_is_boxed(thing, forbidden_vars) - else: - return self.rm.ensure_value_is_boxed(thing, forbidden_vars) - def _sync_var(self, v): if v.type == FLOAT: self.vfprm._sync_var(v) @@ -444,14 +434,14 @@ imm_a0 = check_imm_box(a0) imm_a1 = check_imm_box(a1) if not imm_a0 and imm_a1: - l0 = self._ensure_value_is_boxed(a0, boxes) + l0 = self.make_sure_var_in_reg(a0, boxes) l1 = self.convert_to_imm(a1) elif imm_a0 and not imm_a1: l0 = self.convert_to_imm(a0) - l1 = self._ensure_value_is_boxed(a1, boxes) + l1 = self.make_sure_var_in_reg(a1, boxes) else: - l0 = self._ensure_value_is_boxed(a0, boxes) - l1 = self._ensure_value_is_boxed(a1, boxes) + l0 = self.make_sure_var_in_reg(a0, boxes) + l1 = self.make_sure_var_in_reg(a1, boxes) return [l0, l1] def prepare_op_int_add(self, op, fcond): @@ -466,14 +456,14 @@ imm_a0 = check_imm_box(a0) imm_a1 = check_imm_box(a1) if not imm_a0 and imm_a1: - l0 = self._ensure_value_is_boxed(a0, boxes) + l0 = self.make_sure_var_in_reg(a0, boxes) l1 = self.convert_to_imm(a1) elif imm_a0 and not imm_a1: l0 = self.convert_to_imm(a0) - l1 = self._ensure_value_is_boxed(a1, boxes) + l1 = self.make_sure_var_in_reg(a1, boxes) else: - l0 = self._ensure_value_is_boxed(a0, boxes) - l1 = self._ensure_value_is_boxed(a1, boxes) + l0 = self.make_sure_var_in_reg(a0, boxes) + l1 = self.make_sure_var_in_reg(a1, boxes) return [l0, l1] def prepare_op_int_sub(self, op, fcond): @@ -487,8 +477,8 @@ boxes = op.getarglist() a0, a1 = boxes - reg1 = self._ensure_value_is_boxed(a0, forbidden_vars=boxes) - reg2 = self._ensure_value_is_boxed(a1, forbidden_vars=boxes) + reg1 = self.make_sure_var_in_reg(a0, forbidden_vars=boxes) + reg2 = self.make_sure_var_in_reg(a1, forbidden_vars=boxes) self.possibly_free_vars(boxes) self.possibly_free_vars_for_op(op) @@ -497,14 +487,14 @@ return [reg1, reg2, res] def prepare_op_int_force_ge_zero(self, op, fcond): - argloc = self._ensure_value_is_boxed(op.getarg(0)) + argloc = self.make_sure_var_in_reg(op.getarg(0)) resloc = self.force_allocate_reg(op.result, [op.getarg(0)]) return [argloc, resloc] def prepare_guard_int_mul_ovf(self, op, guard, fcond): boxes = op.getarglist() - reg1 = self._ensure_value_is_boxed(boxes[0], forbidden_vars=boxes) - reg2 = self._ensure_value_is_boxed(boxes[1], forbidden_vars=boxes) + reg1 = self.make_sure_var_in_reg(boxes[0], forbidden_vars=boxes) + reg2 = self.make_sure_var_in_reg(boxes[1], forbidden_vars=boxes) res = self.force_allocate_reg(op.result) return self._prepare_guard(guard, [reg1, reg2, res]) @@ -576,7 +566,7 @@ prepare_guard_int_is_zero = prepare_op_unary_cmp('int_is_zero') def prepare_op_int_neg(self, op, fcond): - l0 = self._ensure_value_is_boxed(op.getarg(0)) + l0 = self.make_sure_var_in_reg(op.getarg(0)) self.possibly_free_vars_for_op(op) self.free_temp_vars() resloc = self.force_allocate_reg(op.result) @@ -629,15 +619,15 @@ def _prepare_llong_binop_xx(self, op, fcond): # arg 0 is the address of the function - loc0 = self._ensure_value_is_boxed(op.getarg(1)) - loc1 = self._ensure_value_is_boxed(op.getarg(2)) + loc0 = self.make_sure_var_in_reg(op.getarg(1)) + loc1 = self.make_sure_var_in_reg(op.getarg(2)) self.possibly_free_vars_for_op(op) self.free_temp_vars() res = self.vfprm.force_allocate_reg(op.result) return [loc0, loc1, res] def _prepare_llong_to_int(self, op, fcond): - loc0 = self._ensure_value_is_boxed(op.getarg(1)) + loc0 = self.make_sure_var_in_reg(op.getarg(1)) res = self.force_allocate_reg(op.result) return [loc0, res] @@ -654,18 +644,12 @@ return args def prepare_op_finish(self, op, fcond): - args = [None] * (op.numargs() + 1) - for i in range(op.numargs()): - arg = op.getarg(i) - if arg: - args[i] = self.loc(arg) - self.possibly_free_var(arg) - n = self.cpu.get_fail_descr_number(op.getdescr()) - args[-1] = imm(n) - return args + loc = self.loc(op.getarg(0)) + self.possibly_free_var(op.getarg(0)) + return [loc] def prepare_op_guard_true(self, op, fcond): - l0 = self._ensure_value_is_boxed(op.getarg(0)) + l0 = self.make_sure_var_in_reg(op.getarg(0)) args = self._prepare_guard(op, [l0]) return args @@ -677,9 +661,9 @@ boxes = op.getarglist() a0, a1 = boxes imm_a1 = check_imm_box(a1) - l0 = self._ensure_value_is_boxed(a0, boxes) + l0 = self.make_sure_var_in_reg(a0, boxes) if not imm_a1: - l1 = self._ensure_value_is_boxed(a1, boxes) + l1 = self.make_sure_var_in_reg(a1, boxes) else: l1 = self.convert_to_imm(a1) assert op.result is None @@ -699,7 +683,7 @@ def prepare_op_guard_exception(self, op, fcond): boxes = op.getarglist() arg0 = ConstInt(rffi.cast(lltype.Signed, op.getarg(0).getint())) - loc = self._ensure_value_is_boxed(arg0) + loc = self.make_sure_var_in_reg(arg0) loc1 = self.get_scratch_reg(INT, boxes) if op.result in self.longevity: resloc = self.force_allocate_reg(op.result, boxes) @@ -713,7 +697,7 @@ return arglocs def prepare_op_guard_no_exception(self, op, fcond): - loc = self._ensure_value_is_boxed( + loc = self.make_sure_var_in_reg( ConstInt(self.cpu.pos_exception())) arglocs = self._prepare_guard(op, [loc]) return arglocs @@ -727,7 +711,7 @@ assert isinstance(op.getarg(0), Box) boxes = op.getarglist() - x = self._ensure_value_is_boxed(boxes[0], boxes) + x = self.make_sure_var_in_reg(boxes[0], boxes) y_val = rffi.cast(lltype.Signed, op.getarg(1).getint()) arglocs = [x, None, None] @@ -837,8 +821,8 @@ boxes = op.getarglist() a0, a1 = boxes ofs, size, sign = unpack_fielddescr(op.getdescr()) - base_loc = self._ensure_value_is_boxed(a0, boxes) - value_loc = self._ensure_value_is_boxed(a1, boxes) + base_loc = self.make_sure_var_in_reg(a0, boxes) + value_loc = self.make_sure_var_in_reg(a1, boxes) if check_imm_arg(ofs): ofs_loc = imm(ofs) else: @@ -851,7 +835,7 @@ def prepare_op_getfield_gc(self, op, fcond): a0 = op.getarg(0) ofs, size, sign = unpack_fielddescr(op.getdescr()) - base_loc = self._ensure_value_is_boxed(a0) + base_loc = self.make_sure_var_in_reg(a0) immofs = imm(ofs) if check_imm_arg(ofs): ofs_loc = immofs @@ -871,8 +855,8 @@ t = unpack_interiorfielddescr(op.getdescr()) ofs, itemsize, fieldsize, sign = t args = op.getarglist() - base_loc = self._ensure_value_is_boxed(op.getarg(0), args) - index_loc = self._ensure_value_is_boxed(op.getarg(1), args) + base_loc = self.make_sure_var_in_reg(op.getarg(0), args) + index_loc = self.make_sure_var_in_reg(op.getarg(1), args) immofs = imm(ofs) if check_imm_arg(ofs): ofs_loc = immofs @@ -889,9 +873,9 @@ t = unpack_interiorfielddescr(op.getdescr()) ofs, itemsize, fieldsize, sign = t args = op.getarglist() - base_loc = self._ensure_value_is_boxed(op.getarg(0), args) - index_loc = self._ensure_value_is_boxed(op.getarg(1), args) - value_loc = self._ensure_value_is_boxed(op.getarg(2), args) + base_loc = self.make_sure_var_in_reg(op.getarg(0), args) + index_loc = self.make_sure_var_in_reg(op.getarg(1), args) + value_loc = self.make_sure_var_in_reg(op.getarg(2), args) immofs = imm(ofs) if check_imm_arg(ofs): ofs_loc = immofs @@ -907,7 +891,7 @@ assert isinstance(arraydescr, ArrayDescr) ofs = arraydescr.lendescr.offset arg = op.getarg(0) - base_loc = self._ensure_value_is_boxed(arg) + base_loc = self.make_sure_var_in_reg(arg) self.possibly_free_vars_for_op(op) self.free_temp_vars() res = self.force_allocate_reg(op.result) @@ -917,9 +901,9 @@ size, ofs, _ = unpack_arraydescr(op.getdescr()) scale = get_scale(size) args = op.getarglist() - base_loc = self._ensure_value_is_boxed(args[0], args) - ofs_loc = self._ensure_value_is_boxed(args[1], args) - value_loc = self._ensure_value_is_boxed(args[2], args) + base_loc = self.make_sure_var_in_reg(args[0], args) + ofs_loc = self.make_sure_var_in_reg(args[1], args) + value_loc = self.make_sure_var_in_reg(args[2], args) assert check_imm_arg(ofs) return [value_loc, base_loc, ofs_loc, imm(scale), imm(ofs)] prepare_op_setarrayitem_raw = prepare_op_setarrayitem_gc @@ -929,8 +913,8 @@ boxes = op.getarglist() size, ofs, _ = unpack_arraydescr(op.getdescr()) scale = get_scale(size) - base_loc = self._ensure_value_is_boxed(boxes[0], boxes) - ofs_loc = self._ensure_value_is_boxed(boxes[1], boxes) + base_loc = self.make_sure_var_in_reg(boxes[0], boxes) + ofs_loc = self.make_sure_var_in_reg(boxes[1], boxes) self.possibly_free_vars_for_op(op) self.free_temp_vars() res = self.force_allocate_reg(op.result) @@ -944,7 +928,7 @@ def prepare_op_strlen(self, op, fcond): args = op.getarglist() - l0 = self._ensure_value_is_boxed(op.getarg(0)) + l0 = self.make_sure_var_in_reg(op.getarg(0)) basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, self.cpu.translate_support_code) immofs = imm(ofs_length) @@ -963,14 +947,14 @@ def prepare_op_strgetitem(self, op, fcond): boxes = op.getarglist() - base_loc = self._ensure_value_is_boxed(boxes[0]) + base_loc = self.make_sure_var_in_reg(boxes[0]) a1 = boxes[1] imm_a1 = check_imm_box(a1) if imm_a1: ofs_loc = self.convert_to_imm(a1) else: - ofs_loc = self._ensure_value_is_boxed(a1, boxes) + ofs_loc = self.make_sure_var_in_reg(a1, boxes) self.possibly_free_vars_for_op(op) self.free_temp_vars() @@ -983,9 +967,9 @@ def prepare_op_strsetitem(self, op, fcond): boxes = op.getarglist() - base_loc = self._ensure_value_is_boxed(boxes[0], boxes) - ofs_loc = self._ensure_value_is_boxed(boxes[1], boxes) - value_loc = self._ensure_value_is_boxed(boxes[2], boxes) + base_loc = self.make_sure_var_in_reg(boxes[0], boxes) + ofs_loc = self.make_sure_var_in_reg(boxes[1], boxes) + value_loc = self.make_sure_var_in_reg(boxes[2], boxes) basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, self.cpu.translate_support_code) assert itemsize == 1 @@ -995,7 +979,7 @@ prepare_op_copyunicodecontent = void def prepare_op_unicodelen(self, op, fcond): - l0 = self._ensure_value_is_boxed(op.getarg(0)) + l0 = self.make_sure_var_in_reg(op.getarg(0)) basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, self.cpu.translate_support_code) immofs = imm(ofs_length) @@ -1012,8 +996,8 @@ def prepare_op_unicodegetitem(self, op, fcond): boxes = op.getarglist() - base_loc = self._ensure_value_is_boxed(boxes[0], boxes) - ofs_loc = self._ensure_value_is_boxed(boxes[1], boxes) + base_loc = self.make_sure_var_in_reg(boxes[0], boxes) + ofs_loc = self.make_sure_var_in_reg(boxes[1], boxes) self.possibly_free_vars_for_op(op) self.free_temp_vars() @@ -1027,9 +1011,9 @@ def prepare_op_unicodesetitem(self, op, fcond): boxes = op.getarglist() - base_loc = self._ensure_value_is_boxed(boxes[0], boxes) - ofs_loc = self._ensure_value_is_boxed(boxes[1], boxes) - value_loc = self._ensure_value_is_boxed(boxes[2], boxes) + base_loc = self.make_sure_var_in_reg(boxes[0], boxes) + ofs_loc = self.make_sure_var_in_reg(boxes[1], boxes) + value_loc = self.make_sure_var_in_reg(boxes[2], boxes) basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, self.cpu.translate_support_code) scale = itemsize / 2 @@ -1042,7 +1026,7 @@ if imm_arg: argloc = self.convert_to_imm(arg) else: - argloc = self._ensure_value_is_boxed(arg) + argloc = self.make_sure_var_in_reg(arg) self.possibly_free_vars_for_op(op) self.free_temp_vars() resloc = self.force_allocate_reg(op.result) @@ -1093,7 +1077,7 @@ # twice from the memory. N = op.numargs() args = op.getarglist() - arglocs = [self._ensure_value_is_boxed(op.getarg(i), args) + arglocs = [self.make_sure_var_in_reg(op.getarg(i), args) for i in range(N)] tmp = self.get_scratch_reg(INT, args) assert tmp not in arglocs @@ -1215,7 +1199,7 @@ float_result=False, name='prepare_guard_float_ge') def prepare_op_math_sqrt(self, op, fcond): - loc = self._ensure_value_is_boxed(op.getarg(1)) + loc = self.make_sure_var_in_reg(op.getarg(1)) self.possibly_free_vars_for_op(op) self.free_temp_vars() res = self.vfprm.force_allocate_reg(op.result) @@ -1223,12 +1207,12 @@ return [loc, res] def prepare_op_cast_float_to_int(self, op, fcond): - loc1 = self._ensure_value_is_boxed(op.getarg(0)) + loc1 = self.make_sure_var_in_reg(op.getarg(0)) res = self.rm.force_allocate_reg(op.result) return [loc1, res] def prepare_op_cast_int_to_float(self, op, fcond): - loc1 = self._ensure_value_is_boxed(op.getarg(0)) + loc1 = self.make_sure_var_in_reg(op.getarg(0)) res = self.vfprm.force_allocate_reg(op.result) return [loc1, res] @@ -1247,12 +1231,12 @@ return [loc, res] def prepare_op_cast_float_to_singlefloat(self, op, fcond): - loc1 = self._ensure_value_is_boxed(op.getarg(0)) + loc1 = self.make_sure_var_in_reg(op.getarg(0)) res = self.force_allocate_reg(op.result) return [loc1, res] - + def prepare_op_cast_singlefloat_to_float(self, op, fcond): - loc1 = self._ensure_value_is_boxed(op.getarg(0)) + loc1 = self.make_sure_var_in_reg(op.getarg(0)) res = self.force_allocate_reg(op.result) return [loc1, res] diff --git a/pypy/jit/backend/arm/runner.py b/pypy/jit/backend/arm/runner.py --- a/pypy/jit/backend/arm/runner.py +++ b/pypy/jit/backend/arm/runner.py @@ -23,15 +23,21 @@ AbstractLLCPU.__init__(self, rtyper, stats, opts, translate_support_code, gcdescr) + from pypy.jit.backend.llsupport import jitframe + self.deadframe_size_max = llmemory.sizeof(jitframe.DEADFRAME, + self.get_failargs_limit()) + def set_debug(self, flag): return self.assembler.set_debug(flag) + def get_failargs_limit(self): + if self.opts is not None: + return self.opts.failargs_limit + else: + return 1000 + def setup(self): - if self.opts is not None: - failargs_limit = self.opts.failargs_limit - else: - failargs_limit = 1000 - self.assembler = AssemblerARM(self, failargs_limit=failargs_limit) + self.assembler = AssemblerARM(self, self.translate_support_code) def setup_once(self): self.assembler.setup_once() @@ -51,24 +57,6 @@ return self.assembler.assemble_bridge(faildescr, inputargs, operations, original_loop_token, log=log) - def get_latest_value_float(self, index): - return self.assembler.fail_boxes_float.getitem(index) - - def get_latest_value_int(self, index): - return self.assembler.fail_boxes_int.getitem(index) - - def get_latest_value_ref(self, index): - return self.assembler.fail_boxes_ptr.getitem(index) - - def get_latest_value_count(self): - return self.assembler.fail_boxes_count - - def get_latest_force_token(self): - return self.assembler.fail_force_index - - def get_on_leave_jitted_hook(self): - return self.assembler.leave_jitted_hook - def clear_latest_values(self, count): setitem = self.assembler.fail_boxes_ptr.setitem null = lltype.nullptr(llmemory.GCREF.TO) @@ -76,7 +64,7 @@ setitem(index, null) def make_execute_token(self, *ARGS): - FUNCPTR = lltype.Ptr(lltype.FuncType(ARGS, lltype.Signed)) + FUNCPTR = lltype.Ptr(lltype.FuncType(ARGS, llmemory.GCREF)) def execute_token(executable_token, *args): clt = executable_token.compiled_loop_token @@ -91,12 +79,13 @@ prev_interpreter = LLInterpreter.current_interpreter LLInterpreter.current_interpreter = self.debug_ll_interpreter try: - fail_index = func(*args) + deadframe = func(*args) finally: if not self.translate_support_code: LLInterpreter.current_interpreter = prev_interpreter #llop.debug_print(lltype.Void, "<<<< Back") - return self.get_fail_descr_from_number(fail_index) + self.gc_set_extra_threshold() + return deadframe return execute_token def cast_ptr_to_int(x): @@ -115,17 +104,22 @@ assert fail_index >= 0, "already forced!" faildescr = self.get_fail_descr_from_number(fail_index) rffi.cast(TP, addr_of_force_index)[0] = ~fail_index - bytecode = self.assembler._find_failure_recovery_bytecode(faildescr) + frb = self.assembler._find_failure_recovery_bytecode(faildescr) + bytecode = rffi.cast(rffi.UCHARP, frb) addr_all_null_regsiters = rffi.cast(rffi.LONG, self.all_null_registers) - # start of "no gc operation!" block - fail_index_2 = self.assembler.failure_recovery_func( - bytecode, - addr_of_force_index, - addr_all_null_regsiters) - self.assembler.leave_jitted_hook() - # end of "no gc operation!" block - assert fail_index == fail_index_2 - return faildescr + # + assert (rffi.cast(lltype.Signed, bytecode[0]) == + self.assembler.CODE_FORCED) + bytecode = rffi.ptradd(bytecode, 1) + deadframe = self.assembler.grab_frame_values(self, + bytecode, addr_of_force_index, + self.all_null_registers, + self.all_null_registers) + # + assert self.get_latest_descr(deadframe) is faildescr + self.assembler.force_token_to_dead_frame[addr_of_force_index] = ( + deadframe) + return deadframe def redirect_call_assembler(self, oldlooptoken, newlooptoken): self.assembler.redirect_call_assembler(oldlooptoken, newlooptoken) diff --git a/pypy/jit/backend/arm/test/test_gc_integration.py b/pypy/jit/backend/arm/test/test_gc_integration.py --- a/pypy/jit/backend/arm/test/test_gc_integration.py +++ b/pypy/jit/backend/arm/test/test_gc_integration.py @@ -2,25 +2,13 @@ From noreply at buildbot.pypy.org Thu Jan 10 11:29:40 2013 From: noreply at buildbot.pypy.org (antocuni) Date: Thu, 10 Jan 2013 11:29:40 +0100 (CET) Subject: [pypy-commit] pypy virtual-raw-mallocs: fix Message-ID: <20130110102940.237CF1C00E2@cobra.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: virtual-raw-mallocs Changeset: r59914:f7bd1d7587e5 Date: 2013-01-10 11:00 +0100 http://bitbucket.org/pypy/pypy/changeset/f7bd1d7587e5/ Log: fix diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py --- a/pypy/jit/metainterp/warmspot.py +++ b/pypy/jit/metainterp/warmspot.py @@ -1111,7 +1111,7 @@ for graph, block, i in find_force_quasi_immutable(graphs): self.replace_force_quasiimmut_with_direct_call(block.operations[i]) -def hook_for_tests(): +def hook_for_tests(cpu): """ This function is empty and does nothing. Its only role is to be monkey-patched by tests to "fix" the annotator if needed (see From noreply at buildbot.pypy.org Thu Jan 10 11:29:41 2013 From: noreply at buildbot.pypy.org (antocuni) Date: Thu, 10 Jan 2013 11:29:41 +0100 (CET) Subject: [pypy-commit] pypy virtual-raw-mallocs: now we can implement unpack_arraydescr_size for real Message-ID: <20130110102941.5221E1C00E2@cobra.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: virtual-raw-mallocs Changeset: r59915:fea15c6e7ec8 Date: 2013-01-10 11:09 +0100 http://bitbucket.org/pypy/pypy/changeset/fea15c6e7ec8/ Log: now we can implement unpack_arraydescr_size for real diff --git a/pypy/jit/backend/llgraph/runner.py b/pypy/jit/backend/llgraph/runner.py --- a/pypy/jit/backend/llgraph/runner.py +++ b/pypy/jit/backend/llgraph/runner.py @@ -482,12 +482,13 @@ return self.bh_raw_load_i(struct, offset, descr) def unpack_arraydescr_size(self, arraydescr): - # so far this is used only by optimizeopt.virtualize for - # {GET,SET}ARRAYITEM_RAW: for now we just return dummy values for - # basesize and is_signed - assert isinstance(arraydescr, Descr) - XXX # ??? - return 0, arraydescr.width, True + from pypy.jit.backend.llsupport.symbolic import get_array_token + from pypy.jit.backend.llsupport.descr import get_type_flag, FLAG_SIGNED + assert isinstance(arraydescr, ArrayDescr) + basesize, itemsize, _ = get_array_token(arraydescr.A, False) + flag = get_type_flag(arraydescr.A.OF) + is_signed = (flag == FLAG_SIGNED) + return basesize, itemsize, is_signed def bh_raw_store_i(self, struct, offset, newvalue, descr): ll_p = rffi.cast(rffi.CCHARP, struct) From noreply at buildbot.pypy.org Thu Jan 10 11:29:42 2013 From: noreply at buildbot.pypy.org (antocuni) Date: Thu, 10 Jan 2013 11:29:42 +0100 (CET) Subject: [pypy-commit] pypy virtual-raw-mallocs: implement bh_new_raw_buffer Message-ID: <20130110102942.709361C00E2@cobra.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: virtual-raw-mallocs Changeset: r59916:6a7bad1acf4a Date: 2013-01-10 11:11 +0100 http://bitbucket.org/pypy/pypy/changeset/6a7bad1acf4a/ Log: implement bh_new_raw_buffer diff --git a/pypy/jit/backend/llgraph/runner.py b/pypy/jit/backend/llgraph/runner.py --- a/pypy/jit/backend/llgraph/runner.py +++ b/pypy/jit/backend/llgraph/runner.py @@ -573,9 +573,7 @@ return read_timestamp() def bh_new_raw_buffer(self, size): - ## def do_new_raw_buffer(size): - ## return lltype.malloc(rffi.CCHARP.TO, size, flavor='raw') - return llimpl.do_new_raw_buffer(size) + return lltype.malloc(rffi.CCHARP.TO, size, flavor='raw') class LLDeadFrame(object): From noreply at buildbot.pypy.org Thu Jan 10 11:29:43 2013 From: noreply at buildbot.pypy.org (antocuni) Date: Thu, 10 Jan 2013 11:29:43 +0100 (CET) Subject: [pypy-commit] pypy virtual-raw-mallocs: replace all the references to virtual_ref_cache&co. with virtual_ptr_cache&co., to avoid confusions with virtualrefs, which are (obviously :)) another thing Message-ID: <20130110102943.9F29E1C00E2@cobra.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: virtual-raw-mallocs Changeset: r59917:6fb648947ab5 Date: 2013-01-10 11:15 +0100 http://bitbucket.org/pypy/pypy/changeset/6fb648947ab5/ Log: replace all the references to virtual_ref_cache&co. with virtual_ptr_cache&co., to avoid confusions with virtualrefs, which are (obviously :)) another thing diff --git a/pypy/jit/metainterp/compile.py b/pypy/jit/metainterp/compile.py --- a/pypy/jit/metainterp/compile.py +++ b/pypy/jit/metainterp/compile.py @@ -650,7 +650,7 @@ class AllVirtuals: llopaque = True - list = [resume.ResumeDataDirectReader.virtual_ref_default] # annotation hack + list = [resume.ResumeDataDirectReader.virtual_ptr_default] # annotation hack def __init__(self, list): self.list = list def hide(self, cpu): diff --git a/pypy/jit/metainterp/resume.py b/pypy/jit/metainterp/resume.py --- a/pypy/jit/metainterp/resume.py +++ b/pypy/jit/metainterp/resume.py @@ -745,7 +745,7 @@ _mixin_ = True rd_virtuals = None virtuals_cache = None - virtual_ref_default = None + virtual_ptr_default = None virtual_int_default = None @@ -758,7 +758,7 @@ self._prepare_virtuals(storage.rd_virtuals) self._prepare_pendingfields(storage.rd_pendingfields) - def getvirtual_ref(self, index): + def getvirtual_ptr(self, index): # Returns the index'th virtual, building it lazily if needed. # Note that this may be called recursively; that's why the # allocate() methods must fill in the cache as soon as they @@ -786,7 +786,7 @@ rd_virtual = rd_virtuals[i] if rd_virtual is not None: if rd_virtual.kind == REF: - self.getvirtual_ref(i) + self.getvirtual_ptr(i) elif rd_virtual.kind == INT: self.getvirtual_int(i) else: @@ -800,7 +800,7 @@ # for REFs and one for INTs: but for each index, we are using # either one or the other, so we should think of a way to # "compact" them - self.virtuals_cache = self.VirtualCache([self.virtual_ref_default] * len(virtuals), + self.virtuals_cache = self.VirtualCache([self.virtual_ptr_default] * len(virtuals), [self.virtual_int_default] * len(virtuals)) def _prepare_pendingfields(self, pendingfields): @@ -1057,7 +1057,7 @@ if kind == INT: box = self.getvirtual_int(num) else: - box = self.getvirtual_ref(num) + box = self.getvirtual_ptr(num) elif tag == TAGINT: box = ConstInt(num) else: @@ -1152,7 +1152,7 @@ class ResumeDataDirectReader(AbstractResumeDataReader): unique_id = lambda: None - virtual_ref_default = lltype.nullptr(llmemory.GCREF.TO) + virtual_ptr_default = lltype.nullptr(llmemory.GCREF.TO) virtual_int_default = 0 resume_after_guard_not_forced = 0 VirtualCache = get_VirtualCache_class('DirectReader') @@ -1370,7 +1370,7 @@ return self.cpu.ts.NULLREF return self.consts[num].getref_base() elif tag == TAGVIRTUAL: - return self.getvirtual_ref(num) + return self.getvirtual_ptr(num) else: assert tag == TAGBOX if num < 0: From noreply at buildbot.pypy.org Thu Jan 10 11:29:44 2013 From: noreply at buildbot.pypy.org (antocuni) Date: Thu, 10 Jan 2013 11:29:44 +0100 (CET) Subject: [pypy-commit] pypy virtual-raw-mallocs: all_virtuals is no longer a list, but an instance of VirtualCache now Message-ID: <20130110102944.D9C141C00E2@cobra.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: virtual-raw-mallocs Changeset: r59918:8f40ffdee0dd Date: 2013-01-10 11:29 +0100 http://bitbucket.org/pypy/pypy/changeset/8f40ffdee0dd/ Log: all_virtuals is no longer a list, but an instance of VirtualCache now diff --git a/pypy/jit/metainterp/compile.py b/pypy/jit/metainterp/compile.py --- a/pypy/jit/metainterp/compile.py +++ b/pypy/jit/metainterp/compile.py @@ -650,9 +650,9 @@ class AllVirtuals: llopaque = True - list = [resume.ResumeDataDirectReader.virtual_ptr_default] # annotation hack - def __init__(self, list): - self.list = list + cache = None + def __init__(self, cache): + self.cache = cache def hide(self, cpu): ptr = cpu.ts.cast_instance_to_base_ref(self) return cpu.ts.cast_to_ref(ptr) @@ -676,7 +676,7 @@ from pypy.jit.metainterp.blackhole import resume_in_blackhole hidden_all_virtuals = metainterp_sd.cpu.get_savedata_ref(deadframe) obj = AllVirtuals.show(metainterp_sd.cpu, hidden_all_virtuals) - all_virtuals = obj.list + all_virtuals = obj.cache if all_virtuals is None: all_virtuals = ResumeDataDirectReader.VirtualCache([], []) assert jitdriver_sd is self.jitdriver_sd From noreply at buildbot.pypy.org Thu Jan 10 11:45:18 2013 From: noreply at buildbot.pypy.org (cfbolz) Date: Thu, 10 Jan 2013 11:45:18 +0100 (CET) Subject: [pypy-commit] pypy type-specialized-instances: merge default Message-ID: <20130110104518.3C4971C0285@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: type-specialized-instances Changeset: r59919:32dd3c20e8b9 Date: 2013-01-10 11:20 +0100 http://bitbucket.org/pypy/pypy/changeset/32dd3c20e8b9/ Log: merge default diff too long, truncating to 2000 out of 553811 lines diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -1,5 +1,6 @@ syntax: glob *.py[co] +*.sw[po] *~ .*.swp .idea @@ -20,6 +21,16 @@ ^pypy/module/cpyext/test/.+\.obj$ ^pypy/module/cpyext/test/.+\.manifest$ ^pypy/module/test_lib_pypy/ctypes_tests/.+\.o$ +^pypy/module/cppyy/src/.+\.o$ +^pypy/module/cppyy/bench/.+\.so$ +^pypy/module/cppyy/bench/.+\.root$ +^pypy/module/cppyy/bench/.+\.d$ +^pypy/module/cppyy/src/.+\.errors$ +^pypy/module/cppyy/test/.+_rflx\.cpp$ +^pypy/module/cppyy/test/.+\.so$ +^pypy/module/cppyy/test/.+\.rootmap$ +^pypy/module/cppyy/test/.+\.exe$ +^pypy/module/cppyy/test/.+_cint.h$ ^pypy/doc/.+\.html$ ^pypy/doc/config/.+\.rst$ ^pypy/doc/basicblock\.asc$ diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -2,3 +2,4 @@ b48df0bf4e75b81d98f19ce89d4a7dc3e1dab5e5 benchmarked d8ac7d23d3ec5f9a0fa1264972f74a010dbfd07f release-1.6 ff4af8f318821f7f5ca998613a60fca09aa137da release-1.7 +07e08e9c885ca67d89bcc304e45a32346daea2fa release-2.0-beta-1 diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -1,9 +1,10 @@ -License for files in the pypy/ directory -================================================== +License +======= Except when otherwise stated (look for LICENSE files in directories or information at the beginning of each file) all software and -documentation in the 'pypy' directories is licensed as follows: +documentation in the 'pypy', 'ctype_configure', 'dotviewer', 'demo', +and 'lib_pypy' directories is licensed as follows: The MIT License @@ -27,7 +28,7 @@ DEALINGS IN THE SOFTWARE. -PyPy Copyright holders 2003-2012 +PyPy Copyright holders 2003-2013 ----------------------------------- Except when otherwise stated (look for LICENSE files or information at @@ -216,6 +217,7 @@ DFKI GmbH, Germany Impara, Germany Change Maker, Sweden + University of California Berkeley, USA The PyPy Logo as used by http://speed.pypy.org and others was created by Samuel Reis and is distributed on terms of Creative Commons Share Alike diff --git a/README b/README deleted file mode 100644 --- a/README +++ /dev/null @@ -1,24 +0,0 @@ -===================================== -PyPy: Python in Python Implementation -===================================== - -Welcome to PyPy! - -PyPy is both an implementation of the Python programming language, and -an extensive compiler framework for dynamic language implementations. -You can build self-contained Python implementations which execute -independently from CPython. - -The home page is: - - http://pypy.org/ - -The getting-started document will help guide you: - - http://doc.pypy.org/en/latest/getting-started.html - -It will also point you to the rest of the documentation which is generated -from files in the pypy/doc directory within the source repositories. Enjoy -and send us feedback! - - the pypy-dev team diff --git a/README.rst b/README.rst new file mode 100644 --- /dev/null +++ b/README.rst @@ -0,0 +1,24 @@ +===================================== +PyPy: Python in Python Implementation +===================================== + +Welcome to PyPy! + +PyPy is both an implementation of the Python programming language, and +an extensive compiler framework for dynamic language implementations. +You can build self-contained Python implementations which execute +independently from CPython. + +The home page is: + + http://pypy.org/ + +The getting-started document will help guide you: + + http://doc.pypy.org/en/latest/getting-started.html + +It will also point you to the rest of the documentation which is generated +from files in the pypy/doc directory within the source repositories. Enjoy +and send us feedback! + + the pypy-dev team diff --git a/_pytest/__init__.py b/_pytest/__init__.py --- a/_pytest/__init__.py +++ b/_pytest/__init__.py @@ -1,2 +1,2 @@ # -__version__ = '2.1.0.dev4' +__version__ = '2.2.4.dev2' diff --git a/_pytest/assertion/__init__.py b/_pytest/assertion/__init__.py --- a/_pytest/assertion/__init__.py +++ b/_pytest/assertion/__init__.py @@ -2,35 +2,25 @@ support for presenting detailed information in failing assertions. """ import py -import imp -import marshal -import struct import sys import pytest from _pytest.monkeypatch import monkeypatch -from _pytest.assertion import reinterpret, util - -try: - from _pytest.assertion.rewrite import rewrite_asserts -except ImportError: - rewrite_asserts = None -else: - import ast +from _pytest.assertion import util def pytest_addoption(parser): group = parser.getgroup("debugconfig") - group.addoption('--assertmode', action="store", dest="assertmode", - choices=("on", "old", "off", "default"), default="default", - metavar="on|old|off", + group.addoption('--assert', action="store", dest="assertmode", + choices=("rewrite", "reinterp", "plain",), + default="rewrite", metavar="MODE", help="""control assertion debugging tools. -'off' performs no assertion debugging. -'old' reinterprets the expressions in asserts to glean information. -'on' (the default) rewrites the assert statements in test modules to provide -sub-expression results.""") +'plain' performs no assertion debugging. +'reinterp' reinterprets assert statements after they failed to provide assertion expression information. +'rewrite' (the default) rewrites assert statements in test modules on import +to provide assert expression information. """) group.addoption('--no-assert', action="store_true", default=False, - dest="noassert", help="DEPRECATED equivalent to --assertmode=off") + dest="noassert", help="DEPRECATED equivalent to --assert=plain") group.addoption('--nomagic', action="store_true", default=False, - dest="nomagic", help="DEPRECATED equivalent to --assertmode=off") + dest="nomagic", help="DEPRECATED equivalent to --assert=plain") class AssertionState: """State for the assertion plugin.""" @@ -40,89 +30,90 @@ self.trace = config.trace.root.get("assertion") def pytest_configure(config): - warn_about_missing_assertion() mode = config.getvalue("assertmode") if config.getvalue("noassert") or config.getvalue("nomagic"): - if mode not in ("off", "default"): - raise pytest.UsageError("assertion options conflict") - mode = "off" - elif mode == "default": - mode = "on" - if mode != "off": - def callbinrepr(op, left, right): - hook_result = config.hook.pytest_assertrepr_compare( - config=config, op=op, left=left, right=right) - for new_expl in hook_result: - if new_expl: - return '\n~'.join(new_expl) + mode = "plain" + if mode == "rewrite": + try: + import ast + except ImportError: + mode = "reinterp" + else: + if sys.platform.startswith('java'): + mode = "reinterp" + if mode != "plain": + _load_modules(mode) m = monkeypatch() config._cleanup.append(m.undo) m.setattr(py.builtin.builtins, 'AssertionError', reinterpret.AssertionError) - m.setattr(util, '_reprcompare', callbinrepr) - if mode == "on" and rewrite_asserts is None: - mode = "old" + hook = None + if mode == "rewrite": + hook = rewrite.AssertionRewritingHook() + sys.meta_path.append(hook) + warn_about_missing_assertion(mode) config._assertstate = AssertionState(config, mode) + config._assertstate.hook = hook config._assertstate.trace("configured with mode set to %r" % (mode,)) -def _write_pyc(co, source_path): - if hasattr(imp, "cache_from_source"): - # Handle PEP 3147 pycs. - pyc = py.path.local(imp.cache_from_source(str(source_path))) - pyc.ensure() - else: - pyc = source_path + "c" - mtime = int(source_path.mtime()) - fp = pyc.open("wb") - try: - fp.write(imp.get_magic()) - fp.write(struct.pack(" 0 and - item.identifier != "__future__"): + elif (not isinstance(item, ast.ImportFrom) or item.level > 0 or + item.module != "__future__"): lineno = item.lineno break pos += 1 @@ -118,9 +357,9 @@ for alias in aliases] mod.body[pos:pos] = imports # Collect asserts. - nodes = collections.deque([mod]) + nodes = [mod] while nodes: - node = nodes.popleft() + node = nodes.pop() for name, field in ast.iter_fields(node): if isinstance(field, list): new = [] @@ -143,7 +382,7 @@ """Get a new variable.""" # Use a character invalid in python identifiers to avoid clashing. name = "@py_assert" + str(next(self.variable_counter)) - self.variables.add(name) + self.variables.append(name) return name def assign(self, expr): @@ -198,7 +437,8 @@ # There's already a message. Don't mess with it. return [assert_] self.statements = [] - self.variables = set() + self.cond_chain = () + self.variables = [] self.variable_counter = itertools.count() self.stack = [] self.on_failure = [] @@ -220,11 +460,11 @@ else: raise_ = ast.Raise(exc, None, None) body.append(raise_) - # Delete temporary variables. - names = [ast.Name(name, ast.Del()) for name in self.variables] - if names: - delete = ast.Delete(names) - self.statements.append(delete) + # Clear temporary variables by setting them to None. + if self.variables: + variables = [ast.Name(name, ast.Store()) for name in self.variables] + clear = ast.Assign(variables, ast.Name("None", ast.Load())) + self.statements.append(clear) # Fix line numbers. for stmt in self.statements: set_location(stmt, assert_.lineno, assert_.col_offset) @@ -240,21 +480,38 @@ return name, self.explanation_param(expr) def visit_BoolOp(self, boolop): - operands = [] - explanations = [] + res_var = self.variable() + expl_list = self.assign(ast.List([], ast.Load())) + app = ast.Attribute(expl_list, "append", ast.Load()) + is_or = int(isinstance(boolop.op, ast.Or)) + body = save = self.statements + fail_save = self.on_failure + levels = len(boolop.values) - 1 self.push_format_context() - for operand in boolop.values: - res, explanation = self.visit(operand) - operands.append(res) - explanations.append(explanation) - expls = ast.Tuple([ast.Str(expl) for expl in explanations], ast.Load()) - is_or = ast.Num(isinstance(boolop.op, ast.Or)) - expl_template = self.helper("format_boolop", - ast.Tuple(operands, ast.Load()), expls, - is_or) + # Process each operand, short-circuting if needed. + for i, v in enumerate(boolop.values): + if i: + fail_inner = [] + self.on_failure.append(ast.If(cond, fail_inner, [])) + self.on_failure = fail_inner + self.push_format_context() + res, expl = self.visit(v) + body.append(ast.Assign([ast.Name(res_var, ast.Store())], res)) + expl_format = self.pop_format_context(ast.Str(expl)) + call = ast.Call(app, [expl_format], [], None, None) + self.on_failure.append(ast.Expr(call)) + if i < levels: + cond = res + if is_or: + cond = ast.UnaryOp(ast.Not(), cond) + inner = [] + self.statements.append(ast.If(cond, inner, [])) + self.statements = body = inner + self.statements = save + self.on_failure = fail_save + expl_template = self.helper("format_boolop", expl_list, ast.Num(is_or)) expl = self.pop_format_context(expl_template) - res = self.assign(ast.BoolOp(boolop.op, operands)) - return res, self.explanation_param(expl) + return ast.Name(res_var, ast.Load()), self.explanation_param(expl) def visit_UnaryOp(self, unary): pattern = unary_map[unary.op.__class__] @@ -288,7 +545,7 @@ new_star, expl = self.visit(call.starargs) arg_expls.append("*" + expl) if call.kwargs: - new_kwarg, expl = self.visit(call.kwarg) + new_kwarg, expl = self.visit(call.kwargs) arg_expls.append("**" + expl) expl = "%s(%s)" % (func_expl, ', '.join(arg_expls)) new_call = ast.Call(new_func, new_args, new_kwargs, new_star, new_kwarg) diff --git a/_pytest/assertion/util.py b/_pytest/assertion/util.py --- a/_pytest/assertion/util.py +++ b/_pytest/assertion/util.py @@ -2,6 +2,7 @@ import py +BuiltinAssertionError = py.builtin.builtins.AssertionError # The _reprcompare attribute on the util module is used by the new assertion # interpretation code and assertion rewriter to detect this plugin was diff --git a/_pytest/capture.py b/_pytest/capture.py --- a/_pytest/capture.py +++ b/_pytest/capture.py @@ -11,22 +11,22 @@ group._addoption('-s', action="store_const", const="no", dest="capture", help="shortcut for --capture=no.") + at pytest.mark.tryfirst +def pytest_cmdline_parse(pluginmanager, args): + # we want to perform capturing already for plugin/conftest loading + if '-s' in args or "--capture=no" in args: + method = "no" + elif hasattr(os, 'dup') and '--capture=sys' not in args: + method = "fd" + else: + method = "sys" + capman = CaptureManager(method) + pluginmanager.register(capman, "capturemanager") + def addouterr(rep, outerr): - repr = getattr(rep, 'longrepr', None) - if not hasattr(repr, 'addsection'): - return for secname, content in zip(["out", "err"], outerr): if content: - repr.addsection("Captured std%s" % secname, content.rstrip()) - -def pytest_unconfigure(config): - # registered in config.py during early conftest.py loading - capman = config.pluginmanager.getplugin('capturemanager') - while capman._method2capture: - name, cap = capman._method2capture.popitem() - # XXX logging module may wants to close it itself on process exit - # otherwise we could do finalization here and call "reset()". - cap.suspend() + rep.sections.append(("Captured std%s" % secname, content)) class NoCapture: def startall(self): @@ -39,8 +39,9 @@ return "", "" class CaptureManager: - def __init__(self): + def __init__(self, defaultmethod=None): self._method2capture = {} + self._defaultmethod = defaultmethod def _maketempfile(self): f = py.std.tempfile.TemporaryFile() @@ -65,14 +66,6 @@ else: raise ValueError("unknown capturing method: %r" % method) - def _getmethod_preoptionparse(self, args): - if '-s' in args or "--capture=no" in args: - return "no" - elif hasattr(os, 'dup') and '--capture=sys' not in args: - return "fd" - else: - return "sys" - def _getmethod(self, config, fspath): if config.option.capture: method = config.option.capture @@ -85,16 +78,22 @@ method = "sys" return method + def reset_capturings(self): + for name, cap in self._method2capture.items(): + cap.reset() + def resumecapture_item(self, item): method = self._getmethod(item.config, item.fspath) if not hasattr(item, 'outerr'): item.outerr = ('', '') # we accumulate outerr on the item return self.resumecapture(method) - def resumecapture(self, method): + def resumecapture(self, method=None): if hasattr(self, '_capturing'): raise ValueError("cannot resume, already capturing with %r" % (self._capturing,)) + if method is None: + method = self._defaultmethod cap = self._method2capture.get(method) self._capturing = method if cap is None: @@ -164,17 +163,6 @@ def pytest_runtest_teardown(self, item): self.resumecapture_item(item) - def pytest__teardown_final(self, __multicall__, session): - method = self._getmethod(session.config, None) - self.resumecapture(method) - try: - rep = __multicall__.execute() - finally: - outerr = self.suspendcapture() - if rep: - addouterr(rep, outerr) - return rep - def pytest_keyboard_interrupt(self, excinfo): if hasattr(self, '_capturing'): self.suspendcapture() diff --git a/_pytest/config.py b/_pytest/config.py --- a/_pytest/config.py +++ b/_pytest/config.py @@ -8,13 +8,15 @@ def pytest_cmdline_parse(pluginmanager, args): config = Config(pluginmanager) config.parse(args) - if config.option.debug: - config.trace.root.setwriter(sys.stderr.write) return config def pytest_unconfigure(config): - for func in config._cleanup: - func() + while 1: + try: + fin = config._cleanup.pop() + except IndexError: + break + fin() class Parser: """ Parser for command line arguments. """ @@ -81,6 +83,7 @@ self._inidict[name] = (help, type, default) self._ininames.append(name) + class OptionGroup: def __init__(self, name, description="", parser=None): self.name = name @@ -256,11 +259,14 @@ self.hook = self.pluginmanager.hook self._inicache = {} self._cleanup = [] - + @classmethod def fromdictargs(cls, option_dict, args): """ constructor useable for subprocesses. """ config = cls() + # XXX slightly crude way to initialize capturing + import _pytest.capture + _pytest.capture.pytest_cmdline_parse(config.pluginmanager, args) config._preparse(args, addopts=False) config.option.__dict__.update(option_dict) for x in config.option.plugins: @@ -285,11 +291,10 @@ def _setinitialconftest(self, args): # capture output during conftest init (#issue93) - from _pytest.capture import CaptureManager - capman = CaptureManager() - self.pluginmanager.register(capman, 'capturemanager') - # will be unregistered in capture.py's unconfigure() - capman.resumecapture(capman._getmethod_preoptionparse(args)) + # XXX introduce load_conftest hook to avoid needing to know + # about capturing plugin here + capman = self.pluginmanager.getplugin("capturemanager") + capman.resumecapture() try: try: self._conftest.setinitial(args) @@ -334,6 +339,7 @@ # Note that this can only be called once per testing process. assert not hasattr(self, 'args'), ( "can only parse cmdline args at most once per Config object") + self._origargs = args self._preparse(args) self._parser.hints.extend(self.pluginmanager._hints) args = self._parser.parse_setoption(args, self.option) @@ -341,6 +347,14 @@ args.append(py.std.os.getcwd()) self.args = args + def addinivalue_line(self, name, line): + """ add a line to an ini-file option. The option must have been + declared but might not yet be set in which case the line becomes the + the first line in its value. """ + x = self.getini(name) + assert isinstance(x, list) + x.append(line) # modifies the cached list inline + def getini(self, name): """ return configuration value from an ini file. If the specified name hasn't been registered through a prior ``parse.addini`` @@ -422,7 +436,7 @@ def getcfg(args, inibasenames): - args = [x for x in args if str(x)[0] != "-"] + args = [x for x in args if not str(x).startswith("-")] if not args: args = [py.path.local()] for arg in args: diff --git a/_pytest/core.py b/_pytest/core.py --- a/_pytest/core.py +++ b/_pytest/core.py @@ -16,11 +16,10 @@ "junitxml resultlog doctest").split() class TagTracer: - def __init__(self, prefix="[pytest] "): + def __init__(self): self._tag2proc = {} self.writer = None self.indent = 0 - self.prefix = prefix def get(self, name): return TagTracerSub(self, (name,)) @@ -30,7 +29,7 @@ if args: indent = " " * self.indent content = " ".join(map(str, args)) - self.writer("%s%s%s\n" %(self.prefix, indent, content)) + self.writer("%s%s [%s]\n" %(indent, content, ":".join(tags))) try: self._tag2proc[tags](tags, args) except KeyError: @@ -212,6 +211,14 @@ self.register(mod, modname) self.consider_module(mod) + def pytest_configure(self, config): + config.addinivalue_line("markers", + "tryfirst: mark a hook implementation function such that the " + "plugin machinery will try to call it first/as early as possible.") + config.addinivalue_line("markers", + "trylast: mark a hook implementation function such that the " + "plugin machinery will try to call it last/as late as possible.") + def pytest_plugin_registered(self, plugin): import pytest dic = self.call_plugin(plugin, "pytest_namespace", {}) or {} @@ -432,10 +439,7 @@ def _preloadplugins(): _preinit.append(PluginManager(load=True)) -def main(args=None, plugins=None): - """ returned exit code integer, after an in-process testing run - with the given command line arguments, preloading an optional list - of passed in plugin objects. """ +def _prepareconfig(args=None, plugins=None): if args is None: args = sys.argv[1:] elif isinstance(args, py.path.local): @@ -449,13 +453,19 @@ else: # subsequent calls to main will create a fresh instance _pluginmanager = PluginManager(load=True) hook = _pluginmanager.hook + if plugins: + for plugin in plugins: + _pluginmanager.register(plugin) + return hook.pytest_cmdline_parse( + pluginmanager=_pluginmanager, args=args) + +def main(args=None, plugins=None): + """ returned exit code integer, after an in-process testing run + with the given command line arguments, preloading an optional list + of passed in plugin objects. """ try: - if plugins: - for plugin in plugins: - _pluginmanager.register(plugin) - config = hook.pytest_cmdline_parse( - pluginmanager=_pluginmanager, args=args) - exitstatus = hook.pytest_cmdline_main(config=config) + config = _prepareconfig(args, plugins) + exitstatus = config.hook.pytest_cmdline_main(config=config) except UsageError: e = sys.exc_info()[1] sys.stderr.write("ERROR: %s\n" %(e.args[0],)) diff --git a/_pytest/helpconfig.py b/_pytest/helpconfig.py --- a/_pytest/helpconfig.py +++ b/_pytest/helpconfig.py @@ -1,7 +1,7 @@ """ version info, help messages, tracing configuration. """ import py import pytest -import inspect, sys +import os, inspect, sys from _pytest.core import varnames def pytest_addoption(parser): @@ -18,7 +18,29 @@ help="trace considerations of conftest.py files."), group.addoption('--debug', action="store_true", dest="debug", default=False, - help="generate and show internal debugging information.") + help="store internal tracing debug information in 'pytestdebug.log'.") + + +def pytest_cmdline_parse(__multicall__): + config = __multicall__.execute() + if config.option.debug: + path = os.path.abspath("pytestdebug.log") + f = open(path, 'w') + config._debugfile = f + f.write("versions pytest-%s, py-%s, python-%s\ncwd=%s\nargs=%s\n\n" %( + pytest.__version__, py.__version__, ".".join(map(str, sys.version_info)), + os.getcwd(), config._origargs)) + config.trace.root.setwriter(f.write) + sys.stderr.write("writing pytestdebug information to %s\n" % path) + return config + + at pytest.mark.trylast +def pytest_unconfigure(config): + if hasattr(config, '_debugfile'): + config._debugfile.close() + sys.stderr.write("wrote pytestdebug information to %s\n" % + config._debugfile.name) + config.trace.root.setwriter(None) def pytest_cmdline_main(config): @@ -34,6 +56,7 @@ elif config.option.help: config.pluginmanager.do_configure(config) showhelp(config) + config.pluginmanager.do_unconfigure(config) return 0 def showhelp(config): @@ -91,7 +114,7 @@ verinfo = getpluginversioninfo(config) if verinfo: lines.extend(verinfo) - + if config.option.traceconfig: lines.append("active plugins:") plugins = [] diff --git a/_pytest/hookspec.py b/_pytest/hookspec.py --- a/_pytest/hookspec.py +++ b/_pytest/hookspec.py @@ -121,16 +121,23 @@ def pytest_itemstart(item, node=None): """ (deprecated, use pytest_runtest_logstart). """ -def pytest_runtest_protocol(item): - """ implements the standard runtest_setup/call/teardown protocol including - capturing exceptions and calling reporting hooks on the results accordingly. +def pytest_runtest_protocol(item, nextitem): + """ implements the runtest_setup/call/teardown protocol for + the given test item, including capturing exceptions and calling + reporting hooks. + + :arg item: test item for which the runtest protocol is performed. + + :arg nexitem: the scheduled-to-be-next test item (or None if this + is the end my friend). This argument is passed on to + :py:func:`pytest_runtest_teardown`. :return boolean: True if no further hook implementations should be invoked. """ pytest_runtest_protocol.firstresult = True def pytest_runtest_logstart(nodeid, location): - """ signal the start of a test run. """ + """ signal the start of running a single test item. """ def pytest_runtest_setup(item): """ called before ``pytest_runtest_call(item)``. """ @@ -138,8 +145,14 @@ def pytest_runtest_call(item): """ called to execute the test ``item``. """ -def pytest_runtest_teardown(item): - """ called after ``pytest_runtest_call``. """ +def pytest_runtest_teardown(item, nextitem): + """ called after ``pytest_runtest_call``. + + :arg nexitem: the scheduled-to-be-next test item (None if no further + test item is scheduled). This argument can be used to + perform exact teardowns, i.e. calling just enough finalizers + so that nextitem only needs to call setup-functions. + """ def pytest_runtest_makereport(item, call): """ return a :py:class:`_pytest.runner.TestReport` object @@ -149,15 +162,8 @@ pytest_runtest_makereport.firstresult = True def pytest_runtest_logreport(report): - """ process item test report. """ - -# special handling for final teardown - somewhat internal for now -def pytest__teardown_final(session): - """ called before test session finishes. """ -pytest__teardown_final.firstresult = True - -def pytest__teardown_final_logerror(report, session): - """ called if runtest_teardown_final failed. """ + """ process a test setup/call/teardown report relating to + the respective phase of executing a test. """ # ------------------------------------------------------------------------- # test session related hooks diff --git a/_pytest/junitxml.py b/_pytest/junitxml.py --- a/_pytest/junitxml.py +++ b/_pytest/junitxml.py @@ -25,21 +25,39 @@ long = int +class Junit(py.xml.Namespace): + pass + + # We need to get the subset of the invalid unicode ranges according to # XML 1.0 which are valid in this python build. Hence we calculate # this dynamically instead of hardcoding it. The spec range of valid # chars is: Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] # | [#x10000-#x10FFFF] -_illegal_unichrs = [(0x00, 0x08), (0x0B, 0x0C), (0x0E, 0x19), - (0xD800, 0xDFFF), (0xFDD0, 0xFFFF)] -_illegal_ranges = [unicode("%s-%s") % (unichr(low), unichr(high)) - for (low, high) in _illegal_unichrs +_legal_chars = (0x09, 0x0A, 0x0d) +_legal_ranges = ( + (0x20, 0xD7FF), + (0xE000, 0xFFFD), + (0x10000, 0x10FFFF), +) +_legal_xml_re = [unicode("%s-%s") % (unichr(low), unichr(high)) + for (low, high) in _legal_ranges if low < sys.maxunicode] -illegal_xml_re = re.compile(unicode('[%s]') % - unicode('').join(_illegal_ranges)) -del _illegal_unichrs -del _illegal_ranges +_legal_xml_re = [unichr(x) for x in _legal_chars] + _legal_xml_re +illegal_xml_re = re.compile(unicode('[^%s]') % + unicode('').join(_legal_xml_re)) +del _legal_chars +del _legal_ranges +del _legal_xml_re +def bin_xml_escape(arg): + def repl(matchobj): + i = ord(matchobj.group()) + if i <= 0xFF: + return unicode('#x%02X') % i + else: + return unicode('#x%04X') % i + return illegal_xml_re.sub(repl, py.xml.escape(arg)) def pytest_addoption(parser): group = parser.getgroup("terminal reporting") @@ -68,117 +86,97 @@ logfile = os.path.expanduser(os.path.expandvars(logfile)) self.logfile = os.path.normpath(logfile) self.prefix = prefix - self.test_logs = [] + self.tests = [] self.passed = self.skipped = 0 self.failed = self.errors = 0 - self._durations = {} def _opentestcase(self, report): names = report.nodeid.split("::") names[0] = names[0].replace("/", '.') - names = tuple(names) - d = {'time': self._durations.pop(report.nodeid, "0")} names = [x.replace(".py", "") for x in names if x != "()"] classnames = names[:-1] if self.prefix: classnames.insert(0, self.prefix) - d['classname'] = ".".join(classnames) - d['name'] = py.xml.escape(names[-1]) - attrs = ['%s="%s"' % item for item in sorted(d.items())] - self.test_logs.append("\n" % " ".join(attrs)) + self.tests.append(Junit.testcase( + classname=".".join(classnames), + name=names[-1], + time=getattr(report, 'duration', 0) + )) - def _closetestcase(self): - self.test_logs.append("") - - def appendlog(self, fmt, *args): - def repl(matchobj): - i = ord(matchobj.group()) - if i <= 0xFF: - return unicode('#x%02X') % i - else: - return unicode('#x%04X') % i - args = tuple([illegal_xml_re.sub(repl, py.xml.escape(arg)) - for arg in args]) - self.test_logs.append(fmt % args) + def append(self, obj): + self.tests[-1].append(obj) def append_pass(self, report): self.passed += 1 - self._opentestcase(report) - self._closetestcase() def append_failure(self, report): - self._opentestcase(report) #msg = str(report.longrepr.reprtraceback.extraline) if "xfail" in report.keywords: - self.appendlog( - '') + self.append( + Junit.skipped(message="xfail-marked test passes unexpectedly")) self.skipped += 1 else: - self.appendlog('%s', - report.longrepr) + sec = dict(report.sections) + fail = Junit.failure(message="test failure") + fail.append(str(report.longrepr)) + self.append(fail) + for name in ('out', 'err'): + content = sec.get("Captured std%s" % name) + if content: + tag = getattr(Junit, 'system-'+name) + self.append(tag(bin_xml_escape(content))) self.failed += 1 - self._closetestcase() def append_collect_failure(self, report): - self._opentestcase(report) #msg = str(report.longrepr.reprtraceback.extraline) - self.appendlog('%s', - report.longrepr) - self._closetestcase() + self.append(Junit.failure(str(report.longrepr), + message="collection failure")) self.errors += 1 def append_collect_skipped(self, report): - self._opentestcase(report) #msg = str(report.longrepr.reprtraceback.extraline) - self.appendlog('%s', - report.longrepr) - self._closetestcase() + self.append(Junit.skipped(str(report.longrepr), + message="collection skipped")) self.skipped += 1 def append_error(self, report): - self._opentestcase(report) - self.appendlog('%s', - report.longrepr) - self._closetestcase() + self.append(Junit.error(str(report.longrepr), + message="test setup failure")) self.errors += 1 def append_skipped(self, report): - self._opentestcase(report) if "xfail" in report.keywords: - self.appendlog( - '%s', - report.keywords['xfail']) + self.append(Junit.skipped(str(report.keywords['xfail']), + message="expected test failure")) else: filename, lineno, skipreason = report.longrepr if skipreason.startswith("Skipped: "): skipreason = skipreason[9:] - self.appendlog('%s', - skipreason, "%s:%s: %s" % report.longrepr, - ) - self._closetestcase() + self.append( + Junit.skipped("%s:%s: %s" % report.longrepr, + type="pytest.skip", + message=skipreason + )) self.skipped += 1 def pytest_runtest_logreport(self, report): if report.passed: - self.append_pass(report) + if report.when == "call": # ignore setup/teardown + self._opentestcase(report) + self.append_pass(report) elif report.failed: + self._opentestcase(report) if report.when != "call": self.append_error(report) else: self.append_failure(report) elif report.skipped: + self._opentestcase(report) self.append_skipped(report) - def pytest_runtest_call(self, item, __multicall__): - start = time.time() - try: - return __multicall__.execute() - finally: - self._durations[item.nodeid] = time.time() - start - def pytest_collectreport(self, report): if not report.passed: + self._opentestcase(report) if report.failed: self.append_collect_failure(report) else: @@ -187,10 +185,11 @@ def pytest_internalerror(self, excrepr): self.errors += 1 data = py.xml.escape(excrepr) - self.test_logs.append( - '\n' - ' ' - '%s' % data) + self.tests.append( + Junit.testcase( + Junit.error(data, message="internal error"), + classname="pytest", + name="internal")) def pytest_sessionstart(self, session): self.suite_start_time = time.time() @@ -204,17 +203,17 @@ suite_stop_time = time.time() suite_time_delta = suite_stop_time - self.suite_start_time numtests = self.passed + self.failed + logfile.write('') - logfile.write('') - logfile.writelines(self.test_logs) - logfile.write('') + logfile.write(Junit.testsuite( + self.tests, + name="", + errors=self.errors, + failures=self.failed, + skips=self.skipped, + tests=numtests, + time="%.3f" % suite_time_delta, + ).unicode(indent=0)) logfile.close() def pytest_terminal_summary(self, terminalreporter): diff --git a/_pytest/main.py b/_pytest/main.py --- a/_pytest/main.py +++ b/_pytest/main.py @@ -2,7 +2,7 @@ import py import pytest, _pytest -import os, sys +import os, sys, imp tracebackcutdir = py.path.local(_pytest.__file__).dirpath() # exitcodes for the command line @@ -11,6 +11,8 @@ EXIT_INTERRUPTED = 2 EXIT_INTERNALERROR = 3 +name_re = py.std.re.compile("^[a-zA-Z_]\w*$") + def pytest_addoption(parser): parser.addini("norecursedirs", "directory patterns to avoid for recursion", type="args", default=('.*', 'CVS', '_darcs', '{arch}')) @@ -27,6 +29,9 @@ action="store", type="int", dest="maxfail", default=0, help="exit after first num failures or errors.") + group._addoption('--strict', action="store_true", + help="run pytest in strict mode, warnings become errors.") + group = parser.getgroup("collect", "collection") group.addoption('--collectonly', action="store_true", dest="collectonly", @@ -48,7 +53,7 @@ def pytest_namespace(): collect = dict(Item=Item, Collector=Collector, File=File, Session=Session) return dict(collect=collect) - + def pytest_configure(config): py.test.config = config # compatibiltiy if config.option.exitfirst: @@ -77,11 +82,11 @@ session.exitstatus = EXIT_INTERNALERROR if excinfo.errisinstance(SystemExit): sys.stderr.write("mainloop: caught Spurious SystemExit!\n") + if initstate >= 2: + config.hook.pytest_sessionfinish(session=session, + exitstatus=session.exitstatus or (session._testsfailed and 1)) if not session.exitstatus and session._testsfailed: session.exitstatus = EXIT_TESTSFAILED - if initstate >= 2: - config.hook.pytest_sessionfinish(session=session, - exitstatus=session.exitstatus) if initstate >= 1: config.pluginmanager.do_unconfigure(config) return session.exitstatus @@ -101,8 +106,12 @@ def pytest_runtestloop(session): if session.config.option.collectonly: return True - for item in session.session.items: - item.config.hook.pytest_runtest_protocol(item=item) + for i, item in enumerate(session.items): + try: + nextitem = session.items[i+1] + except IndexError: + nextitem = None + item.config.hook.pytest_runtest_protocol(item=item, nextitem=nextitem) if session.shouldstop: raise session.Interrupted(session.shouldstop) return True @@ -132,7 +141,7 @@ return getattr(pytest, name) return property(fget, None, None, "deprecated attribute %r, use pytest.%s" % (name,name)) - + class Node(object): """ base class for all Nodes in the collection tree. Collector subclasses have children, Items are terminal nodes.""" @@ -143,13 +152,13 @@ #: the parent collector node. self.parent = parent - + #: the test config object self.config = config or parent.config #: the collection this node is part of self.session = session or parent.session - + #: filesystem path where this node was collected from self.fspath = getattr(parent, 'fspath', None) self.ihook = self.session.gethookproxy(self.fspath) @@ -224,13 +233,13 @@ def listchain(self): """ return list of all parent collectors up to self, starting from root of collection tree. """ - l = [self] - while 1: - x = l[0] - if x.parent is not None: # and x.parent.parent is not None: - l.insert(0, x.parent) - else: - return l + chain = [] + item = self + while item is not None: + chain.append(item) + item = item.parent + chain.reverse() + return chain def listnames(self): return [x.name for x in self.listchain()] @@ -325,6 +334,8 @@ """ a basic test invocation item. Note that for a single function there might be multiple test invocation items. """ + nextitem = None + def reportinfo(self): return self.fspath, None, "" @@ -399,6 +410,7 @@ self._notfound = [] self._initialpaths = set() self._initialparts = [] + self.items = items = [] for arg in args: parts = self._parsearg(arg) self._initialparts.append(parts) @@ -414,7 +426,6 @@ if not genitems: return rep.result else: - self.items = items = [] if rep.passed: for node in rep.result: self.items.extend(self.genitems(node)) @@ -469,16 +480,29 @@ return True def _tryconvertpyarg(self, x): - try: - mod = __import__(x, None, None, ['__doc__']) - except (ValueError, ImportError): - return x - p = py.path.local(mod.__file__) - if p.purebasename == "__init__": - p = p.dirpath() - else: - p = p.new(basename=p.purebasename+".py") - return str(p) + mod = None + path = [os.path.abspath('.')] + sys.path + for name in x.split('.'): + # ignore anything that's not a proper name here + # else something like --pyargs will mess up '.' + # since imp.find_module will actually sometimes work for it + # but it's supposed to be considered a filesystem path + # not a package + if name_re.match(name) is None: + return x + try: + fd, mod, type_ = imp.find_module(name, path) + except ImportError: + return x + else: + if fd is not None: + fd.close() + + if type_[2] != imp.PKG_DIRECTORY: + path = [os.path.dirname(mod)] + else: + path = [mod] + return mod def _parsearg(self, arg): """ return (fspath, names) tuple after checking the file exists. """ @@ -496,7 +520,7 @@ raise pytest.UsageError(msg + arg) parts[0] = path return parts - + def matchnodes(self, matching, names): self.trace("matchnodes", matching, names) self.trace.root.indent += 1 diff --git a/_pytest/mark.py b/_pytest/mark.py --- a/_pytest/mark.py +++ b/_pytest/mark.py @@ -14,12 +14,37 @@ "Terminate expression with ':' to make the first match match " "all subsequent tests (usually file-order). ") + group._addoption("-m", + action="store", dest="markexpr", default="", metavar="MARKEXPR", + help="only run tests matching given mark expression. " + "example: -m 'mark1 and not mark2'." + ) + + group.addoption("--markers", action="store_true", help= + "show markers (builtin, plugin and per-project ones).") + + parser.addini("markers", "markers for test functions", 'linelist') + +def pytest_cmdline_main(config): + if config.option.markers: + config.pluginmanager.do_configure(config) + tw = py.io.TerminalWriter() + for line in config.getini("markers"): + name, rest = line.split(":", 1) + tw.write("@pytest.mark.%s:" % name, bold=True) + tw.line(rest) + tw.line() + config.pluginmanager.do_unconfigure(config) + return 0 +pytest_cmdline_main.tryfirst = True + def pytest_collection_modifyitems(items, config): keywordexpr = config.option.keyword - if not keywordexpr: + matchexpr = config.option.markexpr + if not keywordexpr and not matchexpr: return selectuntil = False - if keywordexpr[-1] == ":": + if keywordexpr[-1:] == ":": selectuntil = True keywordexpr = keywordexpr[:-1] @@ -29,21 +54,38 @@ if keywordexpr and skipbykeyword(colitem, keywordexpr): deselected.append(colitem) else: - remaining.append(colitem) if selectuntil: keywordexpr = None + if matchexpr: + if not matchmark(colitem, matchexpr): + deselected.append(colitem) + continue + remaining.append(colitem) if deselected: config.hook.pytest_deselected(items=deselected) items[:] = remaining +class BoolDict: + def __init__(self, mydict): + self._mydict = mydict + def __getitem__(self, name): + return name in self._mydict + +def matchmark(colitem, matchexpr): + return eval(matchexpr, {}, BoolDict(colitem.obj.__dict__)) + +def pytest_configure(config): + if config.option.strict: + pytest.mark._config = config + def skipbykeyword(colitem, keywordexpr): """ return True if they given keyword expression means to skip this collector/item. """ if not keywordexpr: return - + itemkeywords = getkeywords(colitem) for key in filter(None, keywordexpr.split()): eor = key[:1] == '-' @@ -77,15 +119,31 @@ @py.test.mark.slowtest def test_function(): pass - + will set a 'slowtest' :class:`MarkInfo` object on the ``test_function`` object. """ def __getattr__(self, name): if name[0] == "_": raise AttributeError(name) + if hasattr(self, '_config'): + self._check(name) return MarkDecorator(name) + def _check(self, name): + try: + if name in self._markers: + return + except AttributeError: + pass + self._markers = l = set() + for line in self._config.getini("markers"): + beginning = line.split(":", 1) + x = beginning[0].split("(", 1)[0] + l.add(x) + if name not in self._markers: + raise AttributeError("%r not a registered marker" % (name,)) + class MarkDecorator: """ A decorator for test functions and test classes. When applied it will create :class:`MarkInfo` objects which may be @@ -133,8 +191,7 @@ holder = MarkInfo(self.markname, self.args, self.kwargs) setattr(func, self.markname, holder) else: - holder.kwargs.update(self.kwargs) - holder.args += self.args + holder.add(self.args, self.kwargs) return func kw = self.kwargs.copy() kw.update(kwargs) @@ -150,27 +207,20 @@ self.args = args #: keyword argument dictionary, empty if nothing specified self.kwargs = kwargs + self._arglist = [(args, kwargs.copy())] def __repr__(self): return "" % ( self.name, self.args, self.kwargs) -def pytest_itemcollected(item): - if not isinstance(item, pytest.Function): - return - try: - func = item.obj.__func__ - except AttributeError: - func = getattr(item.obj, 'im_func', item.obj) - pyclasses = (pytest.Class, pytest.Module) - for node in item.listchain(): - if isinstance(node, pyclasses): - marker = getattr(node.obj, 'pytestmark', None) - if marker is not None: - if isinstance(marker, list): - for mark in marker: - mark(func) - else: - marker(func) - node = node.parent - item.keywords.update(py.builtin._getfuncdict(func)) + def add(self, args, kwargs): + """ add a MarkInfo with the given args and kwargs. """ + self._arglist.append((args, kwargs)) + self.args += args + self.kwargs.update(kwargs) + + def __iter__(self): + """ yield MarkInfo objects each relating to a marking-call. """ + for args, kwargs in self._arglist: + yield MarkInfo(self.name, args, kwargs) + diff --git a/_pytest/monkeypatch.py b/_pytest/monkeypatch.py --- a/_pytest/monkeypatch.py +++ b/_pytest/monkeypatch.py @@ -13,6 +13,7 @@ monkeypatch.setenv(name, value, prepend=False) monkeypatch.delenv(name, value, raising=True) monkeypatch.syspath_prepend(path) + monkeypatch.chdir(path) All modifications will be undone after the requesting test function has finished. The ``raising`` @@ -30,6 +31,7 @@ def __init__(self): self._setattr = [] self._setitem = [] + self._cwd = None def setattr(self, obj, name, value, raising=True): """ set attribute ``name`` on ``obj`` to ``value``, by default @@ -83,6 +85,17 @@ self._savesyspath = sys.path[:] sys.path.insert(0, str(path)) + def chdir(self, path): + """ change the current working directory to the specified path + path can be a string or a py.path.local object + """ + if self._cwd is None: + self._cwd = os.getcwd() + if hasattr(path, "chdir"): + path.chdir() + else: + os.chdir(path) + def undo(self): """ undo previous changes. This call consumes the undo stack. Calling it a second time has no effect unless @@ -95,9 +108,17 @@ self._setattr[:] = [] for dictionary, name, value in self._setitem: if value is notset: - del dictionary[name] + try: + del dictionary[name] + except KeyError: + pass # was already deleted, so we have the desired state else: dictionary[name] = value self._setitem[:] = [] if hasattr(self, '_savesyspath'): sys.path[:] = self._savesyspath + del self._savesyspath + + if self._cwd is not None: + os.chdir(self._cwd) + self._cwd = None diff --git a/_pytest/nose.py b/_pytest/nose.py --- a/_pytest/nose.py +++ b/_pytest/nose.py @@ -13,6 +13,7 @@ call.excinfo = call2.excinfo + at pytest.mark.trylast def pytest_runtest_setup(item): if isinstance(item, (pytest.Function)): if isinstance(item.parent, pytest.Generator): diff --git a/_pytest/pastebin.py b/_pytest/pastebin.py --- a/_pytest/pastebin.py +++ b/_pytest/pastebin.py @@ -38,7 +38,11 @@ del tr._tw.__dict__['write'] def getproxy(): - return py.std.xmlrpclib.ServerProxy(url.xmlrpc).pastes + if sys.version_info < (3, 0): + from xmlrpclib import ServerProxy + else: + from xmlrpc.client import ServerProxy + return ServerProxy(url.xmlrpc).pastes def pytest_terminal_summary(terminalreporter): if terminalreporter.config.option.pastebin != "failed": diff --git a/_pytest/pdb.py b/_pytest/pdb.py --- a/_pytest/pdb.py +++ b/_pytest/pdb.py @@ -19,11 +19,13 @@ class pytestPDB: """ Pseudo PDB that defers to the real pdb. """ item = None + collector = None def set_trace(self): """ invoke PDB set_trace debugging, dropping any IO capturing. """ frame = sys._getframe().f_back - item = getattr(self, 'item', None) + item = self.item or self.collector + if item is not None: capman = item.config.pluginmanager.getplugin("capturemanager") out, err = capman.suspendcapture() @@ -38,6 +40,14 @@ pytestPDB.item = item pytest_runtest_setup = pytest_runtest_call = pytest_runtest_teardown = pdbitem + at pytest.mark.tryfirst +def pytest_make_collect_report(__multicall__, collector): + try: + pytestPDB.collector = collector + return __multicall__.execute() + finally: + pytestPDB.collector = None + def pytest_runtest_makereport(): pytestPDB.item = None @@ -60,7 +70,13 @@ tw.sep(">", "traceback") rep.toterminal(tw) tw.sep(">", "entering PDB") - post_mortem(call.excinfo._excinfo[2]) + # A doctest.UnexpectedException is not useful for post_mortem. + # Use the underlying exception instead: + if isinstance(call.excinfo.value, py.std.doctest.UnexpectedException): + tb = call.excinfo.value.exc_info[2] + else: + tb = call.excinfo._excinfo[2] + post_mortem(tb) rep._pdbshown = True return rep diff --git a/_pytest/pytester.py b/_pytest/pytester.py --- a/_pytest/pytester.py +++ b/_pytest/pytester.py @@ -25,6 +25,7 @@ _pytest_fullpath except NameError: _pytest_fullpath = os.path.abspath(pytest.__file__.rstrip("oc")) + _pytest_fullpath = _pytest_fullpath.replace("$py.class", ".py") def pytest_funcarg___pytest(request): return PytestArg(request) @@ -313,16 +314,6 @@ result.extend(session.genitems(colitem)) return result - def inline_genitems(self, *args): - #config = self.parseconfig(*args) - config = self.parseconfigure(*args) - rec = self.getreportrecorder(config) - session = Session(config) - config.hook.pytest_sessionstart(session=session) - session.perform_collect() - config.hook.pytest_sessionfinish(session=session, exitstatus=EXIT_OK) - return session.items, rec - def runitem(self, source): # used from runner functional tests item = self.getitem(source) @@ -343,64 +334,57 @@ l = list(args) + [p] reprec = self.inline_run(*l) reports = reprec.getreports("pytest_runtest_logreport") - assert len(reports) == 1, reports - return reports[0] + assert len(reports) == 3, reports # setup/call/teardown + return reports[1] + + def inline_genitems(self, *args): + return self.inprocess_run(list(args) + ['--collectonly']) def inline_run(self, *args): - args = ("-s", ) + args # otherwise FD leakage - config = self.parseconfig(*args) - reprec = self.getreportrecorder(config) - #config.pluginmanager.do_configure(config) - config.hook.pytest_cmdline_main(config=config) - #config.pluginmanager.do_unconfigure(config) - return reprec + items, rec = self.inprocess_run(args) + return rec - def config_preparse(self): - config = self.Config() - for plugin in self.plugins: - if isinstance(plugin, str): - config.pluginmanager.import_plugin(plugin) - else: - if isinstance(plugin, dict): From noreply at buildbot.pypy.org Thu Jan 10 11:45:20 2013 From: noreply at buildbot.pypy.org (cfbolz) Date: Thu, 10 Jan 2013 11:45:20 +0100 (CET) Subject: [pypy-commit] pypy type-specialized-instances: adapt to new way of doing stuff Message-ID: <20130110104520.0CD7F1C0285@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: type-specialized-instances Changeset: r59920:3a26e7c6c310 Date: 2013-01-10 11:44 +0100 http://bitbucket.org/pypy/pypy/changeset/3a26e7c6c310/ Log: adapt to new way of doing stuff diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py --- a/pypy/objspace/std/test/test_mapdict.py +++ b/pypy/objspace/std/test/test_mapdict.py @@ -303,8 +303,7 @@ assert c.terminator.size_estimate() in [(i + 10) // 2, (i + 11) // 2] class TestTypeSpecializedAttributes(object): - def setup_class(cls): - cls.space = gettestobjspace(**{"objspace.std.withmapdict": True}) + spaceconfig = {"objspace.std.withmapdict": True} def test_attributes(self): space = self.space From noreply at buildbot.pypy.org Thu Jan 10 16:36:04 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 10 Jan 2013 16:36:04 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: test and a fix Message-ID: <20130110153604.7634D1C00E2@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59921:063e02b6c76c Date: 2013-01-10 17:35 +0200 http://bitbucket.org/pypy/pypy/changeset/063e02b6c76c/ Log: test and a fix diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py --- a/pypy/rpython/lltypesystem/rdict.py +++ b/pypy/rpython/lltypesystem/rdict.py @@ -479,7 +479,7 @@ def ll_dict_insertclean(d, key, value, hash): i = ll_dict_lookup_clean(d, hash) - return _ll_dict_setitem_lookup_done(d, key, value, hash, i) + return _ll_dict_setitem_lookup_done(d, key, value, hash, i | HIGHEST_BIT) def ll_dict_lookup_clean(d, hash): # a simplified version of ll_dict_lookup() which assumes that the @@ -487,13 +487,13 @@ # It only finds the next free slot for the given hash. # this is crucial during convert_const, where we cannot call keyhash - # directly. Unused otherwise + # directly. indexes = d.indexes - mask = len(indexes) - 1 + mask = d.size - 1 i = hash & mask perturb = r_uint(hash) - while indexes[i] >= 0: + while ll_index_getitem(d.size, indexes, i) >= 0: i = r_uint(i) i = (i << 2) + i + perturb + 1 i = intmask(i) & mask @@ -698,21 +698,6 @@ freeslot = i perturb >>= PERTURB_SHIFT -def ll_dict_lookup_clean(d, hash): - # a simplified version of ll_dict_lookup() which assumes that the - # key is new, and the dictionary doesn't contain deleted entries. - # It only finds the next free slot for the given hash. - indexes = d.indexes - mask = d.size - 1 - i = hash & mask - perturb = r_uint(hash) - while ll_index_getitem(d.size, indexes, i) != FREE: - i = r_uint(i) - i = (i << 2) + i + perturb + 1 - i = intmask(i) & mask - perturb >>= PERTURB_SHIFT - return i - # ____________________________________________________________ # # Irregular operations. diff --git a/pypy/rpython/test/test_rdict.py b/pypy/rpython/test/test_rdict.py --- a/pypy/rpython/test/test_rdict.py +++ b/pypy/rpython/test/test_rdict.py @@ -1004,6 +1004,31 @@ finally: lltype._array._check_range = original_check_range + def test_prebuilt_custom_dict(self): + class W(object): + def __init__(self, x): + self.x = x + + def hash(w): + return w.x + + def eq(w1, w2): + return w1.x == w2.x + + d = r_dict(eq, hash) + w1 = W(1) + w2 = W(2) + w3 = W(3) + d[w1] = 3 + d[w2] = 2 + d[w3] = 8 + + def f(i): + return d[W(i)] + + res = self.interpret(f, [2]) + assert res == 2 + # ____________________________________________________________ From noreply at buildbot.pypy.org Thu Jan 10 16:43:37 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 10 Jan 2013 16:43:37 +0100 (CET) Subject: [pypy-commit] pypy numpypy-real-as-view: close branch to be merged Message-ID: <20130110154337.2A1091C00E2@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: numpypy-real-as-view Changeset: r59922:7f34d8362ea8 Date: 2013-01-10 17:42 +0200 http://bitbucket.org/pypy/pypy/changeset/7f34d8362ea8/ Log: close branch to be merged From noreply at buildbot.pypy.org Thu Jan 10 16:43:38 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 10 Jan 2013 16:43:38 +0100 (CET) Subject: [pypy-commit] pypy default: (mattip) merge numpypy-real-as-view, which makes .real and .imag attributes Message-ID: <20130110154338.B528E1C00E2@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r59923:32137bbbfdf8 Date: 2013-01-10 17:42 +0200 http://bitbucket.org/pypy/pypy/changeset/32137bbbfdf8/ Log: (mattip) merge numpypy-real-as-view, which makes .real and .imag attributes views instead of ufuncs, for compatibility diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py --- a/pypy/module/micronumpy/arrayimpl/concrete.py +++ b/pypy/module/micronumpy/arrayimpl/concrete.py @@ -13,6 +13,7 @@ from pypy.rlib.debug import make_sure_not_resized class ConcreteArrayIterator(base.BaseArrayIterator): + _immutable_fields_ = ['dtype', 'skip', 'size'] def __init__(self, array): self.array = array self.offset = 0 @@ -21,10 +22,10 @@ self.size = array.size def setitem(self, elem): - self.array.setitem(self.offset, elem) + self.dtype.setitem(self.array, self.offset, elem) def getitem(self): - return self.array.getitem(self.offset) + return self.dtype.getitem(self.array, self.offset) def getitem_bool(self): return self.dtype.getitem_bool(self.array, self.offset) @@ -42,13 +43,16 @@ self.offset %= self.size class OneDimViewIterator(ConcreteArrayIterator): - def __init__(self, array): + ''' The view iterator dtype can be different from the + array.dtype, this is what makes it a View + ''' + def __init__(self, array, dtype, start, strides, shape): self.array = array - self.offset = array.start - self.skip = array.get_strides()[0] - self.dtype = array.dtype + self.dtype = dtype + self.offset = start + self.skip = strides[0] self.index = 0 - self.size = array.get_shape()[0] + self.size = shape[0] def next(self): self.offset += self.skip @@ -65,9 +69,13 @@ self.offset %= self.size class MultiDimViewIterator(ConcreteArrayIterator): - def __init__(self, array, start, strides, backstrides, shape): + ''' The view iterator dtype can be different from the + array.dtype, this is what makes it a View + ''' + def __init__(self, array, dtype, start, strides, backstrides, shape): self.indexes = [0] * len(shape) self.array = array + self.dtype = dtype self.shape = shape self.offset = start self.shapelen = len(shape) @@ -131,12 +139,13 @@ self.offset = array.start self.dim = dim self.array = array + self.dtype = array.dtype def setitem(self, elem): - self.array.setitem(self.offset, elem) + self.dtype.setitem(self.array, self.offset, elem) def getitem(self): - return self.array.getitem(self.offset) + return self.dtype.getitem(self.array, self.offset) @jit.unroll_safe def next(self): @@ -220,6 +229,32 @@ new_shape, self) else: return None + + def get_real(self): + strides = self.get_strides() + backstrides = self.get_backstrides() + if self.dtype.is_complex_type(): + dtype = self.dtype.float_type + return SliceArray(self.start, strides, backstrides, + self.get_shape(), self, dtype=dtype) + return SliceArray(self.start, strides, backstrides, + self.get_shape(), self) + + def get_imag(self): + strides = self.get_strides() + backstrides = self.get_backstrides() + if self.dtype.is_complex_type(): + dtype = self.dtype.float_type + return SliceArray(self.start + dtype.get_size(), strides, + backstrides, self.get_shape(), self, dtype=dtype) + if self.dtype.is_flexible_type(): + # numpy returns self for self.imag + return SliceArray(self.start, strides, backstrides, + self.get_shape(), self) + impl = NonWritableArray(self.get_shape(), self.dtype, self.order, strides, + backstrides) + impl.fill(self.dtype.box(0)) + return impl # -------------------- applevel get/setitem ----------------------- @@ -377,7 +412,7 @@ def create_dot_iter(self, shape, skip): r = calculate_dot_strides(self.get_strides(), self.get_backstrides(), shape, skip) - return MultiDimViewIterator(self, self.start, r[0], r[1], shape) + return MultiDimViewIterator(self, self.dtype, self.start, r[0], r[1], shape) def swapaxes(self, axis1, axis2): shape = self.get_shape()[:] @@ -411,7 +446,7 @@ r = calculate_broadcast_strides(self.get_strides(), self.get_backstrides(), self.get_shape(), shape) - return MultiDimViewIterator(self, 0, r[0], r[1], shape) + return MultiDimViewIterator(self, self.dtype, 0, r[0], r[1], shape) def fill(self, box): self.dtype.fill(self.storage, box, 0, self.size) @@ -424,6 +459,12 @@ self.order) return SliceArray(0, strides, backstrides, new_shape, self) +class NonWritableArray(ConcreteArray): + def descr_setitem(self, space, w_index, w_value): + raise OperationError(space.w_RuntimeError, space.wrap( + "array is not writable")) + + class SliceArray(BaseConcreteArray): def __init__(self, start, strides, backstrides, shape, parent, dtype=None): self.strides = strides @@ -448,11 +489,12 @@ r = calculate_broadcast_strides(self.get_strides(), self.get_backstrides(), self.get_shape(), shape) - return MultiDimViewIterator(self.parent, + return MultiDimViewIterator(self.parent, self.dtype, self.start, r[0], r[1], shape) if len(self.get_shape()) == 1: - return OneDimViewIterator(self) - return MultiDimViewIterator(self.parent, self.start, + return OneDimViewIterator(self.parent, self.dtype, self.start, + self.get_strides(), self.get_shape()) + return MultiDimViewIterator(self.parent, self.dtype, self.start, self.get_strides(), self.get_backstrides(), self.get_shape()) diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -136,7 +136,7 @@ return self.kind == SIGNEDLTR def is_complex_type(self): - return (self.num == 14 or self.num == 15 or self.num == 16) + return False def is_bool_type(self): return self.kind == BOOLLTR @@ -155,6 +155,19 @@ def get_size(self): return self.itemtype.get_element_size() +class W_ComplexDtype(W_Dtype): + + def __init__(self, itemtype, num, kind, name, char, w_box_type, + alternate_constructors=[], aliases=[], + fields=None, fieldnames=None, native=True, float_type=None): + W_Dtype.__init__(self, itemtype, num, kind, name, char, w_box_type, + alternate_constructors=alternate_constructors, aliases=aliases, + fields=fields, fieldnames=fieldnames, native=native) + self.float_type = float_type + + def is_complex_type(self): + return True + def dtype_from_list(space, w_lst): lst_w = space.listview(w_lst) fields = {} @@ -413,15 +426,16 @@ alternate_constructors=[space.w_float], aliases=["float"], ) - self.w_complex64dtype = W_Dtype( + self.w_complex64dtype = W_ComplexDtype( types.Complex64(), num=14, kind=COMPLEXLTR, name="complex64", char="F", w_box_type = space.gettypefor(interp_boxes.W_Complex64Box), + float_type = self.w_float32dtype, ) - self.w_complex128dtype = W_Dtype( + self.w_complex128dtype = W_ComplexDtype( types.Complex128(), num=15, kind=COMPLEXLTR, @@ -430,6 +444,7 @@ w_box_type = space.gettypefor(interp_boxes.W_Complex128Box), alternate_constructors=[space.w_complex], aliases=["complex"], + float_type = self.w_float64dtype, ) if interp_boxes.long_double_size == 12: self.w_float96dtype = W_Dtype( @@ -443,7 +458,7 @@ ) self.w_longdouble = self.w_float96dtype - self.w_complex192dtype = W_Dtype( + self.w_complex192dtype = W_ComplexDtype( types.Complex192(), num=16, kind=COMPLEXLTR, @@ -452,6 +467,7 @@ w_box_type = space.gettypefor(interp_boxes.W_Complex192Box), alternate_constructors=[space.w_complex], aliases=["clongdouble", "clongfloat"], + float_type = self.w_float96dtype, ) self.w_clongdouble = self.w_complex192dtype @@ -467,7 +483,7 @@ ) self.w_longdouble = self.w_float128dtype - self.w_complex256dtype = W_Dtype( + self.w_complex256dtype = W_ComplexDtype( types.Complex256(), num=16, kind=COMPLEXLTR, @@ -476,6 +492,7 @@ w_box_type = space.gettypefor(interp_boxes.W_Complex256Box), alternate_constructors=[space.w_complex], aliases=["clongdouble", "clongfloat"], + float_type = self.w_float128dtype, ) self.w_clongdouble = self.w_complex256dtype else: diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -235,6 +235,29 @@ def descr_copy(self, space): return W_NDimArray(self.implementation.copy()) + def descr_get_real(self, space): + return W_NDimArray(self.implementation.get_real()) + + def descr_get_imag(self, space): + ret = self.implementation.get_imag() + if ret: + return W_NDimArray(ret) + raise OperationError(space.w_NotImplementedError, + space.wrap('imag not implemented for this dtype')) + + def descr_set_real(self, space, w_value): + # copy (broadcast) values into self + tmp = self.implementation.get_real() + tmp.setslice(space, convert_to_array(space, w_value)) + + def descr_set_imag(self, space, w_value): + # if possible, copy (broadcast) values into self + if not self.get_dtype().is_complex_type(): + raise OperationError(space.w_TypeError, + space.wrap('array does not have imaginary part to set')) + tmp = self.implementation.get_imag() + tmp.setslice(space, convert_to_array(space, w_value)) + def descr_reshape(self, space, args_w): """reshape(...) a.reshape(shape) @@ -387,8 +410,6 @@ descr_neg = _unaryop_impl("negative") descr_abs = _unaryop_impl("absolute") descr_invert = _unaryop_impl("invert") - descr_get_real = _unaryop_impl("real") - descr_get_imag = _unaryop_impl("imag") def descr_nonzero(self, space): if self.get_size() > 1: @@ -635,9 +656,10 @@ swapaxes = interp2app(W_NDimArray.descr_swapaxes), flat = GetSetProperty(W_NDimArray.descr_get_flatiter), item = interp2app(W_NDimArray.descr_item), - real = GetSetProperty(W_NDimArray.descr_get_real), - imag = GetSetProperty(W_NDimArray.descr_get_imag), - + real = GetSetProperty(W_NDimArray.descr_get_real, + W_NDimArray.descr_set_real), + imag = GetSetProperty(W_NDimArray.descr_get_imag, + W_NDimArray.descr_set_imag), __array_interface__ = GetSetProperty(W_NDimArray.descr_array_iface), ) diff --git a/pypy/module/micronumpy/test/test_iter.py b/pypy/module/micronumpy/test/test_iter.py --- a/pypy/module/micronumpy/test/test_iter.py +++ b/pypy/module/micronumpy/test/test_iter.py @@ -12,7 +12,7 @@ strides = [5, 1] backstrides = [x * (y - 1) for x,y in zip(strides, shape)] assert backstrides == [10, 4] - i = MultiDimViewIterator(MockArray, start, strides, backstrides, shape) + i = MultiDimViewIterator(MockArray, None, start, strides, backstrides, shape) i.next() i.next() i.next() @@ -30,7 +30,7 @@ strides = [1, 3] backstrides = [x * (y - 1) for x,y in zip(strides, shape)] assert backstrides == [2, 12] - i = MultiDimViewIterator(MockArray, start, strides, backstrides, shape) + i = MultiDimViewIterator(MockArray, None, start, strides, backstrides, shape) i.next() i.next() i.next() @@ -51,7 +51,7 @@ strides = [5, 1] backstrides = [x * (y - 1) for x,y in zip(strides, shape)] assert backstrides == [10, 4] - i = MultiDimViewIterator(MockArray, start, strides, backstrides, shape) + i = MultiDimViewIterator(MockArray, None, start, strides, backstrides, shape) i.next_skip_x(2) i.next_skip_x(2) i.next_skip_x(2) @@ -74,7 +74,7 @@ strides = [1, 3] backstrides = [x * (y - 1) for x,y in zip(strides, shape)] assert backstrides == [2, 12] - i = MultiDimViewIterator(MockArray, start, strides, backstrides, shape) + i = MultiDimViewIterator(MockArray, None, start, strides, backstrides, shape) i.next_skip_x(2) i.next_skip_x(2) i.next_skip_x(2) diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1169,8 +1169,8 @@ assert (d == [[1, 0, 0], [0, 1, 0], [0, 0, 1]]).all() def test_eye(self): - from _numpypy import eye, array - from _numpypy import int32, float64, dtype + from _numpypy import eye + from _numpypy import int32, dtype a = eye(0) assert len(a) == 0 assert a.dtype == dtype('float64') @@ -1345,6 +1345,42 @@ assert b[0] == 3 assert b[1] == 2 + def test_realimag_views(self): + from _numpypy import arange, array + a = arange(15) + b = a.real + b[5]=50 + assert a[5] == 50 + b = a.imag + assert b[7] == 0 + raises(RuntimeError, 'b[7] = -2') + raises(TypeError, 'a.imag = -2') + a = array(['abc','def'],dtype='S3') + b = a.real + assert a[0] == b[0] + assert a[1] == b[1] + b[1] = 'xyz' + assert a[1] == 'xyz' + assert a.imag[0] == 'abc' + raises(TypeError, 'a.imag = "qop"') + a=array([[1+1j, 2-3j, 4+5j],[-6+7j, 8-9j, -2-1j]]) + assert a.real[0,1] == 2 + a.real[0,1] = -20 + assert a[0,1].real == -20 + b = a.imag + assert b[1,2] == -1 + b[1,2] = 30 + assert a[1,2].imag == 30 + a.real = 13 + assert a[1,1].real == 13 + a=array([1+1j, 2-3j, 4+5j, -6+7j, 8-9j, -2-1j]) + a.real = 13 + assert a[3].real == 13 + a.imag = -5 + a.imag[3] = -10 + assert a[3].imag == -10 + assert a[2].imag == -5 + def test_tolist_scalar(self): from _numpypy import int32, bool_ x = int32(23) @@ -2273,7 +2309,7 @@ assert a[0]['x'] == 'a' def test_stringarray(self): - from _numpypy import array, flexible + from _numpypy import array a = array(['abc'],'S3') assert str(a.dtype) == '|S3' a = array(['abc']) @@ -2293,7 +2329,6 @@ def test_flexible_repr(self): # import overrides str(), repr() for array - from numpypy.core import arrayprint from _numpypy import array a = array(['abc'],'S3') s = repr(a) From noreply at buildbot.pypy.org Thu Jan 10 17:30:23 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 10 Jan 2013 17:30:23 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: play with inlining Message-ID: <20130110163023.0AE381C11B7@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59924:498b4a2742d7 Date: 2013-01-10 18:28 +0200 http://bitbucket.org/pypy/pypy/changeset/498b4a2742d7/ Log: play with inlining diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py --- a/pypy/rpython/lltypesystem/rdict.py +++ b/pypy/rpython/lltypesystem/rdict.py @@ -423,14 +423,17 @@ rgc.ll_arraycopy(d.entries, new_entries, 0, 0, lgt) d.entries = new_entries return new_entries +ll_entries_resize_up._never_inline_ = True def ll_entry_getitem(entries, d, item): if len(entries) <= item: entries = ll_entries_resize_up(d) return entries[item] +ll_entry_getitem._always_inline_ = True def ll_entry_getitem_clean(entries, item): return entries[item] +ll_entry_getitem_clean._always_inline_ = True def ll_entry_popitem(d): pass From noreply at buildbot.pypy.org Thu Jan 10 19:19:23 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 10 Jan 2013 19:19:23 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: hack enough to make those two tests pass again, this time with understanding Message-ID: <20130110181923.823CA1C11B7@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r59925:8c847106dab3 Date: 2013-01-10 20:18 +0200 http://bitbucket.org/pypy/pypy/changeset/8c847106dab3/ Log: hack enough to make those two tests pass again, this time with understanding diff --git a/pypy/jit/backend/llgraph/runner.py b/pypy/jit/backend/llgraph/runner.py --- a/pypy/jit/backend/llgraph/runner.py +++ b/pypy/jit/backend/llgraph/runner.py @@ -49,6 +49,7 @@ class WeakrefDescr(AbstractDescr): def __init__(self, realdescr): self.realdescrref = weakref.ref(realdescr) + self.final_descr = getattr(realdescr, 'final_descr', False) class ExecutionFinished(Exception): def __init__(self, deadframe): diff --git a/pypy/jit/backend/llsupport/jitframe.py b/pypy/jit/backend/llsupport/jitframe.py --- a/pypy/jit/backend/llsupport/jitframe.py +++ b/pypy/jit/backend/llsupport/jitframe.py @@ -32,7 +32,7 @@ # exception is not stored there, but is simply kept as a variable there) ('jf_guard_exc', llmemory.GCREF), # the actual frame - ('jf_frame', lltype.Array(lltype.Signed)) + ('jf_frame', lltype.Array(lltype.Signed, hints={'nolength': True})) ) JITFRAMEPTR = lltype.Ptr(JITFRAME) diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py --- a/pypy/jit/backend/llsupport/llmodel.py +++ b/pypy/jit/backend/llsupport/llmodel.py @@ -274,20 +274,30 @@ descr = deadframe.jf_descr return history.AbstractDescr.show(self, descr) + def _decode_pos(self, deadframe, index): + descr = self.get_latest_descr(deadframe) + if descr.final_descr: + assert index == 0 + return 0 + xxx + def get_int_value(self, deadframe, index): + pos = self._decode_pos(deadframe, index) descr = self.gc_ll_descr.getframedescrs(self).arraydescr ofs = self.unpack_arraydescr(descr) - return self.read_int_at_mem(deadframe, index + ofs, 1, WORD) + return self.read_int_at_mem(deadframe, pos + ofs, 1, WORD) def get_ref_value(self, deadframe, index): + pos = self._decode_pos(deadframe, index) descr = self.gc_ll_descr.getframedescrs(self).arraydescr ofs = self.unpack_arraydescr(descr) - return self.read_ref_at_mem(deadframe, index + ofs) + return self.read_ref_at_mem(deadframe, pos + ofs) def get_float_value(self, deadframe, index): + pos = self._decode_pos(deadframe, index) descr = self.gc_ll_descr.getframedescrs(self).arraydescr ofs = self.unpack_arraydescr(descr) - return self.read_float_at_mem(deadframe, index + ofs) + return self.read_float_at_mem(deadframe, pos + ofs) # ____________________ RAW PRIMITIVES ________________________ diff --git a/pypy/jit/backend/model.py b/pypy/jit/backend/model.py --- a/pypy/jit/backend/model.py +++ b/pypy/jit/backend/model.py @@ -332,6 +332,8 @@ asmmemmgr_blocks = None asmmemmgr_gcroots = 0 + frame_depth = 0 + def __init__(self, cpu, number): cpu.total_compiled_loops += 1 self.cpu = cpu diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py --- a/pypy/jit/backend/test/runner_test.py +++ b/pypy/jit/backend/test/runner_test.py @@ -2,6 +2,7 @@ from pypy.jit.metainterp.history import (AbstractFailDescr, AbstractDescr, BasicFailDescr, + BasicFinalDescr, BoxInt, Box, BoxPtr, JitCellToken, TargetToken, ConstInt, ConstPtr, @@ -94,7 +95,7 @@ results = [result] operations = [ResOperation(opnum, valueboxes, result), ResOperation(rop.FINISH, results, None, - descr=BasicFailDescr(0))] + descr=BasicFinalDescr(0))] if operations[0].is_guard(): operations[0].setfailargs([]) if not descr: @@ -111,24 +112,19 @@ avoid_instances = False - def find_pos_on_faildescr(self, fail): - # overloaded by backends - return 0 - def test_compile_linear_loop(self): i0 = BoxInt() i1 = BoxInt() operations = [ ResOperation(rop.INT_ADD, [i0, ConstInt(1)], i1), - ResOperation(rop.FINISH, [i1], None, descr=BasicFailDescr(1)) + ResOperation(rop.FINISH, [i1], None, descr=BasicFinalDescr(1)) ] inputargs = [i0] looptoken = JitCellToken() self.cpu.compile_loop(inputargs, operations, looptoken) deadframe = self.cpu.execute_token(looptoken, 2) fail = self.cpu.get_latest_descr(deadframe) - res = self.cpu.get_int_value(deadframe, - self.find_pos_on_faildescr(fail)) + res = self.cpu.get_int_value(deadframe, 0) assert res == 3 assert fail.identifier == 1 @@ -137,7 +133,7 @@ i1 = BoxFloat() operations = [ ResOperation(rop.FLOAT_ADD, [i0, constfloat(2.3)], i1), - ResOperation(rop.FINISH, [i1], None, descr=BasicFailDescr(1)) + ResOperation(rop.FINISH, [i1], None, descr=BasicFinalDescr(1)) ] inputargs = [i0] looptoken = JitCellToken() @@ -145,8 +141,7 @@ deadframe = self.cpu.execute_token(looptoken, longlong.getfloatstorage(2.8)) fail = self.cpu.get_latest_descr(deadframe) - res = self.cpu.get_float_value(deadframe, - self.find_pos_on_faildescr(fail)) + res = self.cpu.get_float_value(deadframe, 0) assert longlong.getrealfloat(res) == 5.1 fail = self.cpu.get_latest_descr(deadframe) assert fail.identifier == 1 @@ -313,7 +308,7 @@ looptoken = JitCellToken() operations = [ ResOperation(rop.GUARD_FALSE, [i0], None, descr=faildescr1), - ResOperation(rop.FINISH, [], None, descr=BasicFailDescr(2)), + ResOperation(rop.FINISH, [], None, descr=BasicFinalDescr(2)), ] inputargs = [i0] operations[0].setfailargs([i0]) @@ -328,7 +323,7 @@ bridge.append(ResOperation(rop.GUARD_FALSE, [i0], None, descr=BasicFailDescr(3))) bridge.append(ResOperation(rop.FINISH, [], None, - descr=BasicFailDescr(4))) + descr=BasicFinalDescr(4))) bridge[-2].setfailargs(i1list) self.cpu.compile_bridge(faildescr1, [i0], bridge, looptoken) @@ -343,6 +338,8 @@ def test_finish(self): i0 = BoxInt() class UntouchableFailDescr(AbstractFailDescr): + final_descr = True + def __setattr__(self, name, value): if (name == 'index' or name == '_carry_around_for_tests' or name == '_TYPE'): @@ -468,7 +465,7 @@ ResOperation(rop.GUARD_NO_OVERFLOW, [], None, descr=BasicFailDescr(1)), ResOperation(rop.FINISH, [v_res], None, - descr=BasicFailDescr(2)), + descr=BasicFinalDescr(2)), ] ops[1].setfailargs([]) else: @@ -477,7 +474,7 @@ ResOperation(opnum, [v1, v2], v_res), ResOperation(rop.GUARD_OVERFLOW, [], None, descr=BasicFailDescr(1)), - ResOperation(rop.FINISH, [], None, descr=BasicFailDescr(2)), + ResOperation(rop.FINISH, [], None, descr=BasicFinalDescr(2)), ] ops[1].setfailargs([v_res]) # @@ -1171,7 +1168,7 @@ values.append(longlong.getfloatstorage(r.random())) # looptoken = JitCellToken() - faildescr = BasicFailDescr(42) + faildescr = BasicFinalDescr(42) operations = [] retboxes = [] retvalues = [] @@ -1322,7 +1319,7 @@ i2 = BoxInt() targettoken = TargetToken() faildescr1 = BasicFailDescr(1) - faildescr2 = BasicFailDescr(2) + faildescr2 = BasicFinalDescr(2) operations = [ ResOperation(rop.LABEL, fboxes, None, descr=targettoken), ResOperation(rop.FLOAT_LE, [fboxes[0], constfloat(9.2)], i2), @@ -1360,7 +1357,7 @@ if not self.cpu.supports_floats: py.test.skip("requires floats") fboxes = [BoxFloat() for i in range(3)] - faildescr1 = BasicFailDescr(100) + faildescr1 = BasicFinalDescr(100) loopops = """ [i0,f1, f2] f3 = float_add(f1, f2) @@ -1414,7 +1411,7 @@ box = BoxInt() res = BoxInt() faildescr1 = BasicFailDescr(1) - faildescr2 = BasicFailDescr(2) + faildescr2 = BasicFinalDescr(2) inputargs = [box] operations = [ ResOperation(opname, [box], res), @@ -1458,7 +1455,7 @@ ibox2 = ConstInt(-42) b1 = BoxInt() faildescr1 = BasicFailDescr(1) - faildescr2 = BasicFailDescr(2) + faildescr2 = BasicFinalDescr(2) inputargs = [ib for ib in [ibox1, ibox2] if isinstance(ib, BoxInt)] operations = [ @@ -1510,7 +1507,7 @@ ibox2 = ConstInt(42) b1 = BoxInt() faildescr1 = BasicFailDescr(1) - faildescr2 = BasicFailDescr(2) + faildescr2 = BasicFinalDescr(2) inputargs = [ib for ib in [ibox1, ibox2] if isinstance(ib, BoxInt)] operations = [ @@ -1566,7 +1563,7 @@ fbox2 = constfloat(-4.5) b1 = BoxInt() faildescr1 = BasicFailDescr(1) - faildescr2 = BasicFailDescr(2) + faildescr2 = BasicFinalDescr(2) inputargs = [fb for fb in [fbox1, fbox2] if isinstance(fb, BoxFloat)] operations = [ @@ -1630,7 +1627,7 @@ operations.append(ResOperation(opnum, boxargs, boxres)) # Unique-ify inputargs inputargs = list(set(inputargs)) - faildescr = BasicFailDescr(1) + faildescr = BasicFinalDescr(1) operations.append(ResOperation(rop.FINISH, [], None, descr=faildescr)) looptoken = JitCellToken() @@ -1703,7 +1700,7 @@ ResOperation(guard_opnum, [box], None, descr=BasicFailDescr(4)), ResOperation(rop.FINISH, [], None, - descr=BasicFailDescr(5))] + descr=BasicFinalDescr(5))] operations[1].setfailargs([]) looptoken = JitCellToken() # Use "set" to unique-ify inputargs @@ -2226,7 +2223,7 @@ ResOperation(rop.CALL_MAY_FORCE, [funcbox, tok, i1], None, descr=calldescr), ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr), - ResOperation(rop.FINISH, [i0], None, descr=BasicFailDescr(0)) + ResOperation(rop.FINISH, [i0], None, descr=BasicFinalDescr(0)) ] ops[2].setfailargs([i1, i0]) looptoken = JitCellToken() @@ -2272,7 +2269,7 @@ ResOperation(rop.CALL_MAY_FORCE, [funcbox, tok, i1], i2, descr=calldescr), ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr), - ResOperation(rop.FINISH, [i2], None, descr=BasicFailDescr(0)) + ResOperation(rop.FINISH, [i2], None, descr=BasicFinalDescr(0)) ] ops[2].setfailargs([i1, i2, i0]) looptoken = JitCellToken() @@ -2320,7 +2317,7 @@ ResOperation(rop.CALL_MAY_FORCE, [funcbox, tok, i1], f2, descr=calldescr), ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr), - ResOperation(rop.FINISH, [f2], None, descr=BasicFailDescr(0)) + ResOperation(rop.FINISH, [f2], None, descr=BasicFinalDescr(0)) ] ops[2].setfailargs([i1, f2, i0]) looptoken = JitCellToken() @@ -2362,7 +2359,7 @@ ResOperation(rop.CALL_RELEASE_GIL, [funcbox, i1], i2, descr=calldescr), ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr), - ResOperation(rop.FINISH, [i2], None, descr=BasicFailDescr(0)) + ResOperation(rop.FINISH, [i2], None, descr=BasicFinalDescr(0)) ] ops[1].setfailargs([i1, i2]) looptoken = JitCellToken() @@ -2420,7 +2417,7 @@ ResOperation(rop.CALL_RELEASE_GIL, [funcbox, i0, i1, i2, i3], None, descr=calldescr), ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr), - ResOperation(rop.FINISH, [], None, descr=BasicFailDescr(0)) + ResOperation(rop.FINISH, [], None, descr=BasicFinalDescr(0)) ] ops[1].setfailargs([]) looptoken = JitCellToken() @@ -2478,7 +2475,7 @@ ] ops[-1].setfailargs([]) ops += [ - ResOperation(rop.FINISH, [i3], None, descr=BasicFailDescr(0)) + ResOperation(rop.FINISH, [i3], None, descr=BasicFinalDescr(0)) ] looptoken = JitCellToken() self.cpu.compile_loop([i1, i2], ops, looptoken) @@ -2499,7 +2496,7 @@ faildescr = BasicFailDescr(1) ops = [ ResOperation(rop.GUARD_NOT_INVALIDATED, [], None, descr=faildescr), - ResOperation(rop.FINISH, [i0], None, descr=BasicFailDescr(0)) + ResOperation(rop.FINISH, [i0], None, descr=BasicFinalDescr(0)) ] ops[0].setfailargs([i1]) looptoken = JitCellToken() @@ -2527,7 +2524,7 @@ faildescr2 = BasicFailDescr(2) ops = [ ResOperation(rop.GUARD_NOT_INVALIDATED, [],None, descr=faildescr2), - ResOperation(rop.FINISH, [i2], None, descr=BasicFailDescr(3)) + ResOperation(rop.FINISH, [i2], None, descr=BasicFinalDescr(3)) ] ops[0].setfailargs([]) self.cpu.compile_bridge(faildescr, [i2], ops, looptoken) @@ -2559,7 +2556,7 @@ ops = [ ResOperation(rop.GUARD_NOT_INVALIDATED, [], None, descr=faildescr), ResOperation(rop.LABEL, [i0], None, descr=labeldescr), - ResOperation(rop.FINISH, [i0], None, descr=BasicFailDescr(3)), + ResOperation(rop.FINISH, [i0], None, descr=BasicFinalDescr(3)), ] ops[0].setfailargs([]) looptoken = JitCellToken() @@ -3339,7 +3336,7 @@ p0 = BoxPtr() operations = [ ResOperation(rop.NEWUNICODE, [i0], p0), - ResOperation(rop.FINISH, [p0], None, descr=BasicFailDescr(1)) + ResOperation(rop.FINISH, [p0], None, descr=BasicFinalDescr(1)) ] inputargs = [i0] looptoken = JitCellToken() @@ -3495,7 +3492,7 @@ operations = [ ResOperation(rop.INT_LE, [i0, ConstInt(1)], i1), ResOperation(rop.GUARD_TRUE, [i1], None, descr=faildescr1), - ResOperation(rop.FINISH, [i0], None, descr=BasicFailDescr(1234)), + ResOperation(rop.FINISH, [i0], None, descr=BasicFinalDescr(1234)), ] operations[1].setfailargs([i0]) self.cpu.compile_loop(inputargs, operations, looptoken1) @@ -3572,13 +3569,13 @@ operations = [ ResOperation(rop.GUARD_NONNULL_CLASS, [t_box, T_box], None, descr=faildescr), - ResOperation(rop.FINISH, [], None, descr=BasicFailDescr(1))] + ResOperation(rop.FINISH, [], None, descr=BasicFinalDescr(1))] operations[0].setfailargs([]) looptoken = JitCellToken() inputargs = [t_box] self.cpu.compile_loop(inputargs, operations, looptoken) operations = [ - ResOperation(rop.FINISH, [], None, descr=BasicFailDescr(99)) + ResOperation(rop.FINISH, [], None, descr=BasicFinalDescr(99)) ] self.cpu.compile_bridge(faildescr, [], operations, looptoken) deadframe = self.cpu.execute_token(looptoken, null_box.getref_base()) @@ -3709,7 +3706,7 @@ ResOperation(rop.CALL_MAY_FORCE, [funcbox, tok, i1], i2, descr=calldescr), ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr), - ResOperation(rop.FINISH, [i2], None, descr=BasicFailDescr(0)) + ResOperation(rop.FINISH, [i2], None, descr=BasicFinalDescr(0)) ] ops[2].setfailargs([i2]) looptoken = JitCellToken() diff --git a/pypy/jit/backend/x86/arch.py b/pypy/jit/backend/x86/arch.py --- a/pypy/jit/backend/x86/arch.py +++ b/pypy/jit/backend/x86/arch.py @@ -48,12 +48,14 @@ FORCE_INDEX_OFS = 0 SAVED_REGISTERS = 1 # range(1, 5) MY_COPY_OF_REGS = 5 # range(5, 9) + JITFRAME_FIXED_SIZE = 1 else: # rbp + rbx + r12 + r13 + r14 + r15 + 11 extra words + force_index = 18 FRAME_FIXED_SIZE = 18 # 18 aligned to 16 bytes = 2 * WORD FORCE_INDEX_OFS = 0 SAVED_REGISTERS = 1 # range(1, 7) MY_COPY_OF_REGS = 7 # range(7, 18) + JITFRAME_FIXED_SIZE = 1 # "My copy of regs" has room for almost all registers, apart from eax and edx # which are used in the malloc itself. They are: diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -1951,6 +1951,8 @@ # no malloc allowed here!! xxx apart from one, hacking a lot #self.fail_ebp = allregisters[16 + ebp.value] num = 0 + import pdb + pdb.set_trace() XXX deadframe = lltype.nullptr(jitframe.DEADFRAME) # step 1: lots of mess just to count the final value of 'num' @@ -2156,7 +2158,14 @@ self.mc = None def genop_finish(self, op, arglocs, result_loc): - [argloc] = arglocs + if len(arglocs) == 2: + [return_val, argloc] = arglocs + if op.getarg(0).type == FLOAT: + self.mc.MOVSD_bx(0, return_val.value) + else: + self.mc.MOV_br(0, return_val.value) + else: + [argloc] = arglocs if argloc is not eax: self.mov(argloc, eax) # exit function diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py --- a/pypy/jit/backend/x86/regalloc.py +++ b/pypy/jit/backend/x86/regalloc.py @@ -4,7 +4,7 @@ import os from pypy.jit.metainterp.history import (Box, Const, ConstInt, ConstPtr, - ResOperation, BoxPtr, ConstFloat, + BoxPtr, ConstFloat, BoxFloat, INT, REF, FLOAT, TargetToken, JitCellToken) from pypy.jit.backend.x86.regloc import * @@ -13,18 +13,17 @@ from pypy.rlib import rgc from pypy.jit.backend.llsupport import symbolic from pypy.jit.backend.x86.jump import remap_frame_layout_mixed -from pypy.jit.codewriter import heaptracker, longlong +from pypy.jit.codewriter import longlong from pypy.jit.codewriter.effectinfo import EffectInfo from pypy.jit.metainterp.resoperation import rop -from pypy.jit.backend.llsupport.descr import FieldDescr, ArrayDescr -from pypy.jit.backend.llsupport.descr import CallDescr, SizeDescr -from pypy.jit.backend.llsupport.descr import InteriorFieldDescr +from pypy.jit.backend.llsupport.descr import ArrayDescr +from pypy.jit.backend.llsupport.descr import CallDescr from pypy.jit.backend.llsupport.descr import unpack_arraydescr from pypy.jit.backend.llsupport.descr import unpack_fielddescr from pypy.jit.backend.llsupport.descr import unpack_interiorfielddescr from pypy.jit.backend.llsupport.regalloc import FrameManager, RegisterManager,\ TempBox, compute_vars_longevity, is_comparison_or_ovf_op -from pypy.jit.backend.x86.arch import WORD, FRAME_FIXED_SIZE +from pypy.jit.backend.x86.arch import WORD, JITFRAME_FIXED_SIZE from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64, MY_COPY_OF_REGS from pypy.jit.backend.x86 import rx86 from pypy.rlib.rarithmetic import r_longlong @@ -133,10 +132,7 @@ class X86FrameManager(FrameManager): @staticmethod def frame_pos(i, box_type): - if IS_X86_32 and box_type == FLOAT: - return StackLoc(i, get_ebp_ofs(i+1), box_type) - else: - return StackLoc(i, get_ebp_ofs(i), box_type) + return StackLoc(i, get_ebp_ofs(i), box_type) @staticmethod def frame_size(box_type): if IS_X86_32 and box_type == FLOAT: @@ -498,11 +494,13 @@ # the frame is in ebp, but we have to point where in the frame is # the potential argument to FINISH descr = op.getdescr() - self.force_spill_var(op.getarg(0)) - loc = self.loc(op.getarg(0)) - descr._x86_result_offset = loc.value fail_no = self.assembler.cpu.get_fail_descr_number(descr) - self.Perform(op, [imm(fail_no)], None) + if op.numargs() == 1: + loc = self.make_sure_var_in_reg(op.getarg(0)) + locs = [loc, imm(fail_no)] + else: + locs = [imm(fail_no)] + self.Perform(op, locs, None) self.possibly_free_var(op.getarg(0)) def consider_guard_no_exception(self, op): @@ -1481,9 +1479,9 @@ def get_ebp_ofs(position): # Argument is a frame position (0, 1, 2...). - # Returns (ebp-20), (ebp-24), (ebp-28)... + # Returns (ebp+20), (ebp+24), (ebp+28)... # i.e. the n'th word beyond the fixed frame size. - return WORD * position + return WORD * (position + JITFRAME_FIXED_SIZE) def _valid_addressing_size(size): return size == 1 or size == 2 or size == 4 or size == 8 diff --git a/pypy/jit/backend/x86/runner.py b/pypy/jit/backend/x86/runner.py --- a/pypy/jit/backend/x86/runner.py +++ b/pypy/jit/backend/x86/runner.py @@ -8,7 +8,8 @@ from pypy.jit.codewriter import longlong from pypy.jit.metainterp import history, compile from pypy.jit.backend.x86.assembler import Assembler386 -from pypy.jit.backend.x86.arch import FORCE_INDEX_OFS, IS_X86_32 +from pypy.jit.backend.x86.arch import (FORCE_INDEX_OFS, IS_X86_32, + JITFRAME_FIXED_SIZE) from pypy.jit.backend.x86.profagent import ProfileAgent from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU from pypy.jit.backend.llsupport import jitframe @@ -114,7 +115,8 @@ addr = executable_token._x86_function_addr func = rffi.cast(FUNCPTR, addr) #llop.debug_print(lltype.Void, ">>>> Entering", addr) - frame = lltype.malloc(jitframe.JITFRAME, clt.frame_depth) + frame = lltype.malloc(jitframe.JITFRAME, clt.frame_depth + + JITFRAME_FIXED_SIZE) frame.jf_frame_info = clt.frame_info ll_frame = lltype.cast_opaque_ptr(llmemory.GCREF, frame) prev_interpreter = None # help flow space @@ -123,7 +125,7 @@ LLInterpreter.current_interpreter = self.debug_ll_interpreter try: # XXX RPythonize - num = 0 + num = JITFRAME_FIXED_SIZE * WORD for arg in args: if isinstance(arg, int): self.set_int_value(frame, num, arg) @@ -207,17 +209,17 @@ """ descr = self.gc_ll_descr.getframedescrs(self).arraydescr ofs = self.unpack_arraydescr(descr) - self.write_int_at_mem(newframe, ofs + index * WORD, WORD, 1, value) + self.write_int_at_mem(newframe, ofs + index, WORD, 1, value) def set_ref_value(self, newframe, index, value): descr = self.gc_ll_descr.getframedescrs(self).arraydescr ofs = self.unpack_arraydescr(descr) - self.write_ref_at_mem(newframe, ofs + index * WORD, value) + self.write_ref_at_mem(newframe, ofs + index, value) def set_float_value(self, newframe, index, value): descr = self.gc_ll_descr.getframedescrs(self).arraydescr ofs = self.unpack_arraydescr(descr) - self.write_float_at_mem(newframe, ofs + index * WORD, value) + self.write_float_at_mem(newframe, ofs + index, value) class CPU386(AbstractX86CPU): backend_name = 'x86' diff --git a/pypy/jit/backend/x86/test/test_runner.py b/pypy/jit/backend/x86/test/test_runner.py --- a/pypy/jit/backend/x86/test/test_runner.py +++ b/pypy/jit/backend/x86/test/test_runner.py @@ -33,9 +33,6 @@ # for the individual tests see # ====> ../../test/runner_test.py - def find_pos_on_faildescr(self, descr): - return descr._x86_result_offset - add_loop_instructions = ['mov', 'add', 'test', 'je', 'jmp'] if WORD == 4: bridge_loop_instructions = ['lea', 'jmp'] diff --git a/pypy/jit/metainterp/compile.py b/pypy/jit/metainterp/compile.py --- a/pypy/jit/metainterp/compile.py +++ b/pypy/jit/metainterp/compile.py @@ -412,7 +412,7 @@ # ____________________________________________________________ class _DoneWithThisFrameDescr(AbstractFailDescr): - pass + final_descr = True class DoneWithThisFrameDescrVoid(_DoneWithThisFrameDescr): def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd): diff --git a/pypy/jit/metainterp/history.py b/pypy/jit/metainterp/history.py --- a/pypy/jit/metainterp/history.py +++ b/pypy/jit/metainterp/history.py @@ -166,12 +166,19 @@ class AbstractFailDescr(AbstractDescr): index = -1 + final_descr = False def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd): raise NotImplementedError def compile_and_attach(self, metainterp, new_loop): raise NotImplementedError +class BasicFinalDescr(AbstractFailDescr): + final_descr = True + + def __init__(self, identifier=None): + self.identifier = identifier # for testing + class BasicFailDescr(AbstractFailDescr): def __init__(self, identifier=None): self.identifier = identifier # for testing diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py --- a/pypy/jit/metainterp/resoperation.py +++ b/pypy/jit/metainterp/resoperation.py @@ -6,6 +6,10 @@ op.initarglist(args) if descr is not None: assert isinstance(op, ResOpWithDescr) + if opnum == rop.FINISH: + assert descr.final_descr + elif op.is_guard(): + assert not descr.final_descr op.setdescr(descr) return op diff --git a/pypy/jit/metainterp/resume.py b/pypy/jit/metainterp/resume.py --- a/pypy/jit/metainterp/resume.py +++ b/pypy/jit/metainterp/resume.py @@ -1279,17 +1279,6 @@ def write_a_float(self, index, float): self.blackholeinterp.setarg_f(index, float) -def resume_renum(list_of_positions, storage): - num = storage.rd_numb - while num: - i = 0 - while i < len(num.nums): - pos, flags = untag(num.nums[i]) - if flags == TAGBOX: - num.nums[i] = tag(list_of_positions[pos], TAGBOX) - i += 1 - num = num.prev - # ____________________________________________________________ def dump_storage(storage, liveboxes): diff --git a/pypy/jit/metainterp/test/test_resume.py b/pypy/jit/metainterp/test/test_resume.py --- a/pypy/jit/metainterp/test/test_resume.py +++ b/pypy/jit/metainterp/test/test_resume.py @@ -499,22 +499,6 @@ assert snapshot.prev is fs[2].parent_resumedata_snapshot assert snapshot.boxes == fs[2]._env -def test_renaming(): - b1, b2, b3 = [BoxInt(), BoxPtr(), BoxInt()] - c1, c2, c3 = [ConstInt(1), ConstInt(2), ConstInt(3)] - fs = [FakeFrame("code0", 0, b1, c1, b2)] - - storage = Storage() - capture_resumedata(fs, None, [], storage) - memo = ResumeDataLoopMemo(FakeMetaInterpStaticData()) - modifier = ResumeDataVirtualAdder(storage, memo) - modifier.finish(FakeOptimizer({})) - resume_renum([10, 20], storage) - for num, x in zip(storage.rd_numb.prev.nums, [10, 20]): - pos, tag = untag(num) - if tag == TAGBOX: - assert pos == x - class FakeMetaInterpStaticData: cpu = LLtypeMixin.cpu diff --git a/pypy/jit/tool/oparser.py b/pypy/jit/tool/oparser.py --- a/pypy/jit/tool/oparser.py +++ b/pypy/jit/tool/oparser.py @@ -49,7 +49,9 @@ return newop -def default_fail_descr(model, fail_args=None): +def default_fail_descr(model, opnum, fail_args=None): + if opnum == rop.FINISH: + return model.BasicFinalDescr() return model.BasicFailDescr() @@ -244,14 +246,14 @@ "Unknown var in fail_args: %s" % arg) fail_args.append(fail_arg) if descr is None and self.invent_fail_descr: - descr = self.invent_fail_descr(self.model, fail_args) + descr = self.invent_fail_descr(self.model, opnum, fail_args) if hasattr(descr, '_oparser_uses_descr_of_guard'): descr._oparser_uses_descr_of_guard(self, fail_args) else: fail_args = None if opnum == rop.FINISH: if descr is None and self.invent_fail_descr: - descr = self.invent_fail_descr(self.model, fail_args) + descr = self.invent_fail_descr(self.model, opnum, fail_args) elif opnum == rop.JUMP: if descr is None and self.invent_fail_descr: descr = self.original_jitcell_token diff --git a/pypy/jit/tool/oparser_model.py b/pypy/jit/tool/oparser_model.py --- a/pypy/jit/tool/oparser_model.py +++ b/pypy/jit/tool/oparser_model.py @@ -6,7 +6,7 @@ from pypy.jit.metainterp.history import TreeLoop, JitCellToken from pypy.jit.metainterp.history import Box, BoxInt, BoxFloat from pypy.jit.metainterp.history import ConstInt, ConstObj, ConstPtr, ConstFloat - from pypy.jit.metainterp.history import BasicFailDescr, TargetToken + from pypy.jit.metainterp.history import BasicFailDescr, BasicFinalDescr, TargetToken from pypy.jit.metainterp.typesystem import llhelper from pypy.jit.metainterp.history import get_const_ptr_for_string @@ -49,6 +49,9 @@ class BasicFailDescr(object): I_am_a_descr = True + class BasicFinalDescr(object): + I_am_a_descr = True + class Box(object): _counter = 0 type = 'b' From noreply at buildbot.pypy.org Thu Jan 10 19:29:56 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 10 Jan 2013 19:29:56 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: a more promiscuous resizing strategy Message-ID: <20130110182956.A658D1C11B7@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59926:ea720c684dd7 Date: 2013-01-10 20:29 +0200 http://bitbucket.org/pypy/pypy/changeset/ea720c684dd7/ Log: a more promiscuous resizing strategy diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py --- a/pypy/rpython/lltypesystem/rdict.py +++ b/pypy/rpython/lltypesystem/rdict.py @@ -418,7 +418,7 @@ some = 4 else: some = 7 - new_lgt = lgt + some + (lgt >> 3) + new_lgt = lgt + some + (lgt >> 2) new_entries = lltype.typeOf(d).TO.entries.TO.allocate(new_lgt) rgc.ll_arraycopy(d.entries, new_entries, 0, 0, lgt) d.entries = new_entries From noreply at buildbot.pypy.org Thu Jan 10 19:41:30 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 10 Jan 2013 19:41:30 +0100 (CET) Subject: [pypy-commit] pypy default: Fix the remaining tests that use FINISH with more than one argument. Message-ID: <20130110184130.F13F31C00E2@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r59927:7a74a080cd9c Date: 2013-01-10 19:41 +0100 http://bitbucket.org/pypy/pypy/changeset/7a74a080cd9c/ Log: Fix the remaining tests that use FINISH with more than one argument. diff --git a/pypy/jit/backend/test/calling_convention_test.py b/pypy/jit/backend/test/calling_convention_test.py --- a/pypy/jit/backend/test/calling_convention_test.py +++ b/pypy/jit/backend/test/calling_convention_test.py @@ -105,11 +105,13 @@ ops = '[%s]\n' % arguments ops += '%s\n' % spill_ops ops += 'f99 = call(ConstClass(func_ptr), %s, descr=calldescr)\n' % arguments - ops += 'finish(f99, %s)\n' % arguments + ops += 'i99 = same_as(0)\n' + ops += 'guard_true(i99) [f99, %s]\n' % arguments + ops += 'finish()\n' loop = parse(ops, namespace=locals()) looptoken = JitCellToken() - done_number = self.cpu.get_fail_descr_number(loop.operations[-1].getdescr()) + done_number = self.cpu.get_fail_descr_number(loop.operations[-2].getdescr()) self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) argvals, expected_result = self._prepare_args(args, floats, ints) diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py --- a/pypy/jit/backend/test/runner_test.py +++ b/pypy/jit/backend/test/runner_test.py @@ -1192,7 +1192,8 @@ values.append(longlong.getfloatstorage(r.random())) # looptoken = JitCellToken() - faildescr = BasicFailDescr(42) + guarddescr = BasicFailDescr(42) + faildescr = BasicFailDescr(43) operations = [] retboxes = [] retvalues = [] @@ -1221,9 +1222,13 @@ retboxes.insert(kk, newbox) retvalues.insert(kk, y) # - operations.append( - ResOperation(rop.FINISH, retboxes, None, descr=faildescr) - ) + zero = BoxInt() + operations.extend([ + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=guarddescr), + ResOperation(rop.FINISH, [], None, descr=faildescr) + ]) + operations[-2].setfailargs(retboxes) print inputargs for op in operations: print op @@ -1344,12 +1349,15 @@ targettoken = TargetToken() faildescr1 = BasicFailDescr(1) faildescr2 = BasicFailDescr(2) + faildescr3 = BasicFailDescr(3) operations = [ ResOperation(rop.LABEL, fboxes, None, descr=targettoken), ResOperation(rop.FLOAT_LE, [fboxes[0], constfloat(9.2)], i2), ResOperation(rop.GUARD_TRUE, [i2], None, descr=faildescr1), - ResOperation(rop.FINISH, fboxes, None, descr=faildescr2), + ResOperation(rop.GUARD_FALSE, [i2], None, descr=faildescr2), + ResOperation(rop.FINISH, [], None, descr=faildescr3), ] + operations[-3].setfailargs(fboxes) operations[-2].setfailargs(fboxes) looptoken = JitCellToken() self.cpu.compile_loop(fboxes, operations, looptoken) @@ -1382,6 +1390,7 @@ py.test.skip("requires floats") fboxes = [BoxFloat() for i in range(3)] faildescr1 = BasicFailDescr(100) + faildescr2 = BasicFailDescr(102) loopops = """ [i0,f1, f2] f3 = float_add(f1, f2) @@ -1406,9 +1415,13 @@ assert longlong.getrealfloat(f2) == 0.75 assert longlong.getrealfloat(f3) == 133.0 + zero = BoxInt() bridgeops = [ - ResOperation(rop.FINISH, fboxes, None, descr=faildescr1), + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=faildescr1), + ResOperation(rop.FINISH, [], None, descr=faildescr2), ] + bridgeops[-2].setfailargs(fboxes[:]) self.cpu.compile_bridge(loop.operations[-2].getdescr(), fboxes, bridgeops, looptoken) args = [1, @@ -2002,7 +2015,7 @@ i1 = same_as(1) call(ConstClass(fptr), i0, descr=calldescr) p0 = guard_exception(ConstClass(xtp)) [i1] - finish(0, p0) + finish(p0) ''' FPTR = lltype.Ptr(lltype.FuncType([lltype.Signed], lltype.Void)) fptr = llhelper(FPTR, func) @@ -2023,8 +2036,7 @@ looptoken = JitCellToken() self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) deadframe = self.cpu.execute_token(looptoken, 1) - assert self.cpu.get_latest_value_int(deadframe, 0) == 0 - assert self.cpu.get_latest_value_ref(deadframe, 1) == xptr + assert self.cpu.get_latest_value_ref(deadframe, 0) == xptr excvalue = self.cpu.grab_exc_value(deadframe) assert not excvalue deadframe = self.cpu.execute_token(looptoken, 0) diff --git a/pypy/jit/backend/x86/test/test_recompilation.py b/pypy/jit/backend/x86/test/test_recompilation.py --- a/pypy/jit/backend/x86/test/test_recompilation.py +++ b/pypy/jit/backend/x86/test/test_recompilation.py @@ -45,7 +45,8 @@ force_spill(i5) i8 = int_add(i7, 1) i9 = int_add(i8, 1) - finish(i3, i4, i5, i6, i7, i8, i9, descr=fdescr2) + guard_false(i3, descr=fdescr2) [i3, i4, i5, i6, i7, i8, i9] + finish() ''' bridge = self.attach_bridge(ops, loop, -2) descr = loop.operations[3].getdescr() diff --git a/pypy/jit/backend/x86/test/test_regalloc.py b/pypy/jit/backend/x86/test/test_regalloc.py --- a/pypy/jit/backend/x86/test/test_regalloc.py +++ b/pypy/jit/backend/x86/test/test_regalloc.py @@ -327,7 +327,7 @@ assert self.getint(0) == 0 bridge_ops = ''' [i0, i1] - finish(1, 2) + finish(2) ''' self.attach_bridge(bridge_ops, loop, 0) self.run(loop, 0, 1) @@ -428,7 +428,7 @@ ops = ''' [i0, i1, i2, i3, i4, i5, i6, i7] guard_value(i6, i1) [i0, i2, i3, i4, i5, i6] - finish(i0, i2, i3, i4, i5, i6) + finish(i0) ''' self.interpret(ops, [0, 0, 0, 0, 0, 0, 0, 0]) assert self.getint(0) == 0 @@ -438,14 +438,15 @@ [i0, i1, i2, i3, i4, i5, i6, i7, i8] i9 = same_as(0) guard_true(i0) [i9, i0, i1, i2, i3, i4, i5, i6, i7, i8] - finish(1, i0, i1, i2, i3, i4, i5, i6, i7, i8) + finish(1) ''' loop = self.interpret(ops, [0, 1, 2, 3, 4, 5, 6, 7, 8]) assert self.getint(0) == 0 bridge_ops = ''' [i9, i0, i1, i2, i3, i4, i5, i6, i7, i8] call(ConstClass(raising_fptr), 0, descr=raising_calldescr) - finish(i0, i1, i2, i3, i4, i5, i6, i7, i8) + guard_true(i9) [i0, i1, i2, i3, i4, i5, i6, i7, i8] + finish() ''' self.attach_bridge(bridge_ops, loop, 1) self.run(loop, 0, 1, 2, 3, 4, 5, 6, 7, 8) @@ -474,7 +475,8 @@ i1 = same_as(1) i2 = int_lt(i0, 100) guard_true(i3) [i1, i2] - finish(0, i2) + i4 = int_neg(i2) + finish(0) ''' self.interpret(ops, [0, 1]) assert self.getint(0) == 0 @@ -503,7 +505,8 @@ i15 = int_is_true(i5) i16 = int_is_true(i6) i17 = int_is_true(i7) - finish(i10, i11, i12, i13, i14, i15, i16, i17) + guard_true(i0) [i10, i11, i12, i13, i14, i15, i16, i17] + finish() ''' self.interpret(ops, [0, 42, 12, 0, 13, 0, 0, 3333]) assert self.getints(8) == [0, 1, 1, 0, 1, 0, 0, 1] @@ -517,7 +520,8 @@ i13 = int_eq(i5, i6) i14 = int_gt(i6, i2) i15 = int_ne(i2, i6) - finish(i10, i11, i12, i13, i14, i15) + guard_true(i0) [i10, i11, i12, i13, i14, i15] + finish() ''' self.interpret(ops, [0, 1, 2, 3, 4, 5, 6]) assert self.getints(6) == [1, 1, 0, 0, 1, 1] @@ -574,7 +578,9 @@ ops = ''' [f0, f1] f2 = float_add(f0, f1) - finish(f2, f0, f1) + i0 = same_as(0) + guard_true(i0) [f2, f0, f1] + finish() ''' self.interpret(ops, [3.0, 1.5]) assert self.getfloats(3) == [4.5, 3.0, 1.5] @@ -584,7 +590,9 @@ [f0, f1, f2, f3, f4, f5, f6, f7, f8] f9 = float_add(f0, f1) f10 = float_add(f8, 3.5) - finish(f9, f10, f2, f3, f4, f5, f6, f7, f8) + i0 = same_as(0) + guard_true(i0) [f9, f10, f2, f3, f4, f5, f6, f7, f8] + finish() ''' self.interpret(ops, [0.1, .2, .3, .4, .5, .6, .7, .8, .9]) assert self.getfloats(9) == [.1+.2, .9+3.5, .3, .4, .5, .6, .7, .8, .9] @@ -613,7 +621,8 @@ i7 = float_ne(f7, 0.0) i8 = float_ne(f8, 0.0) i9 = float_ne(f9, 0.0) - finish(i0, i1, i2, i3, i4, i5, i6, i7, i8, i9) + guard_true(i0) [i0, i1, i2, i3, i4, i5, i6, i7, i8, i9] + finish() ''' loop = self.interpret(ops, [0.0, .1, .2, .3, .4, .5, .6, .7, .8, .9]) assert self.getints(9) == [0, 1, 1, 1, 1, 1, 1, 1, 1] diff --git a/pypy/jit/backend/x86/test/test_regalloc2.py b/pypy/jit/backend/x86/test/test_regalloc2.py --- a/pypy/jit/backend/x86/test/test_regalloc2.py +++ b/pypy/jit/backend/x86/test/test_regalloc2.py @@ -11,13 +11,17 @@ v2 = BoxInt() v3 = BoxInt() v4 = BoxInt() + zero = BoxInt() inputargs = [v1] operations = [ ResOperation(rop.INT_ADD, [v1, v1], v2), ResOperation(rop.INT_INVERT, [v2], v3), ResOperation(rop.UINT_RSHIFT, [v1, ConstInt(3)], v4), - ResOperation(rop.FINISH, [v4, v3], None, descr=BasicFailDescr()), + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=BasicFailDescr()), + ResOperation(rop.FINISH, [], None, descr=BasicFailDescr()) ] + operations[-2].setfailargs([v4, v3]) cpu = CPU(None, None) cpu.setup_once() looptoken = JitCellToken() @@ -31,6 +35,7 @@ v2 = BoxInt() v3 = BoxInt() v4 = BoxInt() + zero = BoxInt() tmp5 = BoxInt() inputargs = [v1] operations = [ @@ -38,8 +43,11 @@ ResOperation(rop.INT_MUL, [v2, v1], v3), ResOperation(rop.INT_IS_TRUE, [v2], tmp5), ResOperation(rop.INT_IS_ZERO, [tmp5], v4), - ResOperation(rop.FINISH, [v4, v3, tmp5], None, descr=BasicFailDescr()), + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=BasicFailDescr()), + ResOperation(rop.FINISH, [], None, descr=BasicFailDescr()) ] + operations[-2].setfailargs([v4, v3, tmp5]) cpu = CPU(None, None) cpu.setup_once() looptoken = JitCellToken() @@ -90,6 +98,7 @@ v38 = BoxInt() v39 = BoxInt() v40 = BoxInt() + zero = BoxInt() tmp41 = BoxInt() tmp42 = BoxInt() tmp43 = BoxInt() @@ -134,8 +143,12 @@ ResOperation(rop.UINT_GT, [v33, ConstInt(-11)], v38), ResOperation(rop.INT_NEG, [v7], v39), ResOperation(rop.INT_GT, [v24, v32], v40), - ResOperation(rop.FINISH, [v40, v36, v37, v31, v16, v34, v35, v23, v22, v29, v14, v39, v30, v38], None, descr=BasicFailDescr()), + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=BasicFailDescr()), + ResOperation(rop.FINISH, [], None, descr=BasicFailDescr()) ] + operations[-2].setfailargs([v40, v36, v37, v31, v16, v34, v35, v23, + v22, v29, v14, v39, v30, v38]) cpu = CPU(None, None) cpu.setup_once() looptoken = JitCellToken() @@ -198,6 +211,7 @@ v38 = BoxInt() v39 = BoxInt() v40 = BoxInt() + zero = BoxInt() tmp41 = BoxInt() tmp42 = BoxInt() tmp43 = BoxInt() @@ -240,8 +254,13 @@ ResOperation(rop.INT_GT, [v4, v11], v38), ResOperation(rop.INT_LT, [v27, v22], v39), ResOperation(rop.INT_NEG, [v27], v40), - ResOperation(rop.FINISH, [v40, v10, v36, v26, v13, v30, v21, v33, v18, v25, v31, v32, v28, v29, v35, v38, v20, v39, v34, v23, v37], None, descr=BasicFailDescr()), + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=BasicFailDescr()), + ResOperation(rop.FINISH, [], None, descr=BasicFailDescr()) ] + operations[-2].setfailargs([v40, v10, v36, v26, v13, v30, v21, v33, + v18, v25, v31, v32, v28, v29, v35, v38, + v20, v39, v34, v23, v37]) cpu = CPU(None, None) cpu.setup_once() looptoken = JitCellToken() diff --git a/pypy/jit/backend/x86/test/test_runner.py b/pypy/jit/backend/x86/test/test_runner.py --- a/pypy/jit/backend/x86/test/test_runner.py +++ b/pypy/jit/backend/x86/test/test_runner.py @@ -499,9 +499,12 @@ i6, descr=calldescr), ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr), - ResOperation(rop.FINISH, [i3, i4, i5, i6], None, - descr=BasicFailDescr(0)) + ResOperation(rop.GUARD_FALSE, [i3], None, + descr=BasicFailDescr(0)), + ResOperation(rop.FINISH, [], None, + descr=BasicFailDescr(1)) ] + ops[-2].setfailargs([i3, i4, i5, i6]) ops[1].setfailargs([]) ops[3].setfailargs([]) ops[5].setfailargs([]) diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py --- a/pypy/jit/metainterp/resoperation.py +++ b/pypy/jit/metainterp/resoperation.py @@ -353,6 +353,9 @@ def initarglist(self, args): self._args = args + if not we_are_translated() and \ + self.__class__.__name__.startswith('FINISH'): # XXX remove me + assert len(args) <= 1 # FINISH operations take 0 or 1 arg now def getarglist(self): return self._args From noreply at buildbot.pypy.org Thu Jan 10 21:16:21 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Thu, 10 Jan 2013 21:16:21 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Oops. Removed debug code Message-ID: <20130110201621.CB6B91C00E2@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59928:155a4893645f Date: 2013-01-10 05:11 +0100 http://bitbucket.org/pypy/pypy/changeset/155a4893645f/ Log: Oops. Removed debug code diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py --- a/pypy/interpreter/astcompiler/assemble.py +++ b/pypy/interpreter/astcompiler/assemble.py @@ -5,8 +5,6 @@ from pypy.interpreter.astcompiler import ast, symtable from pypy.interpreter import pycode from pypy.tool import stdlib_opcode as ops -from rpython.tool import stdlib_opcode as ops2 -print dir(ops2) from pypy.interpreter.error import OperationError from rpython.rlib.objectmodel import we_are_translated From noreply at buildbot.pypy.org Thu Jan 10 21:16:23 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Thu, 10 Jan 2013 21:16:23 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: split up interp_signal, added rlib.rsignal Message-ID: <20130110201623.8B4841C00E2@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59929:355ee8f6486f Date: 2013-01-10 21:15 +0100 http://bitbucket.org/pypy/pypy/changeset/355ee8f6486f/ Log: split up interp_signal, added rlib.rsignal diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py --- a/pypy/module/signal/interp_signal.py +++ b/pypy/module/signal/interp_signal.py @@ -12,99 +12,7 @@ import sys from rpython.rlib import jit, rposix from rpython.rlib.rarithmetic import intmask, is_valid_int - -def setup(): - for key, value in cpy_signal.__dict__.items(): - if (key.startswith('SIG') or key.startswith('CTRL_')) and \ - is_valid_int(value) and \ - key != 'SIG_DFL' and key != 'SIG_IGN': - globals()[key] = value - yield key - -NSIG = cpy_signal.NSIG -SIG_DFL = cpy_signal.SIG_DFL -SIG_IGN = cpy_signal.SIG_IGN -signal_names = list(setup()) -signal_values = {} -for key in signal_names: - signal_values[globals()[key]] = None -if sys.platform == 'win32' and not hasattr(cpy_signal,'CTRL_C_EVENT'): - # XXX Hack to revive values that went missing, - # Remove this once we are sure the host cpy module has them. - signal_values[0] = None - signal_values[1] = None - signal_names.append('CTRL_C_EVENT') - signal_names.append('CTRL_BREAK_EVENT') - CTRL_C_EVENT = 0 - CTRL_BREAK_EVENT = 1 -includes = ['stdlib.h', 'src/signals.h'] -if sys.platform != 'win32': - includes.append('sys/time.h') - -cdir = py.path.local(cdir) - -eci = ExternalCompilationInfo( - includes = includes, - separate_module_files = [cdir / 'src' / 'signals.c'], - include_dirs = [str(cdir)], - export_symbols = ['pypysig_poll', 'pypysig_default', - 'pypysig_ignore', 'pypysig_setflag', - 'pypysig_reinstall', - 'pypysig_set_wakeup_fd', - 'pypysig_getaddr_occurred'], -) - -class CConfig: - _compilation_info_ = eci - -if sys.platform != 'win32': - for name in """ITIMER_REAL ITIMER_VIRTUAL ITIMER_PROF""".split(): - setattr(CConfig, name, rffi_platform.DefinedConstantInteger(name)) - - CConfig.timeval = rffi_platform.Struct( - 'struct timeval', - [('tv_sec', rffi.LONG), - ('tv_usec', rffi.LONG)]) - - CConfig.itimerval = rffi_platform.Struct( - 'struct itimerval', - [('it_value', CConfig.timeval), - ('it_interval', CConfig.timeval)]) - -for k, v in rffi_platform.configure(CConfig).items(): - globals()[k] = v - -def external(name, args, result, **kwds): - return rffi.llexternal(name, args, result, compilation_info=eci, - sandboxsafe=True, **kwds) - -pypysig_ignore = external('pypysig_ignore', [rffi.INT], lltype.Void) -pypysig_default = external('pypysig_default', [rffi.INT], lltype.Void) -pypysig_setflag = external('pypysig_setflag', [rffi.INT], lltype.Void) -pypysig_reinstall = external('pypysig_reinstall', [rffi.INT], lltype.Void) -pypysig_set_wakeup_fd = external('pypysig_set_wakeup_fd', [rffi.INT], rffi.INT) -pypysig_poll = external('pypysig_poll', [], rffi.INT, threadsafe=False) -# don't bother releasing the GIL around a call to pypysig_poll: it's -# pointless and a performance issue - -# don't use rffi.LONGP because the JIT doesn't support raw arrays so far -struct_name = 'pypysig_long_struct' -LONG_STRUCT = lltype.Struct(struct_name, ('c_value', lltype.Signed), - hints={'c_name' : struct_name, 'external' : 'C'}) -del struct_name - -pypysig_getaddr_occurred = external('pypysig_getaddr_occurred', [], - lltype.Ptr(LONG_STRUCT), _nowrapper=True, - elidable_function=True) -c_alarm = external('alarm', [rffi.INT], rffi.INT) -c_pause = external('pause', [], rffi.INT, threadsafe=True) -c_siginterrupt = external('siginterrupt', [rffi.INT, rffi.INT], rffi.INT) - -if sys.platform != 'win32': - itimervalP = rffi.CArrayPtr(itimerval) - c_setitimer = external('setitimer', - [rffi.INT, itimervalP, itimervalP], rffi.INT) - c_getitimer = external('getitimer', [rffi.INT, itimervalP], rffi.INT) +from rpython.rlib.rsignal import * class SignalActionFlag(AbstractActionFlag): diff --git a/rpython/jit/metainterp/test/test_del.py b/rpython/jit/metainterp/test/test_del.py --- a/rpython/jit/metainterp/test/test_del.py +++ b/rpython/jit/metainterp/test/test_del.py @@ -123,46 +123,8 @@ res = self.meta_interp(main, [20]) assert res == 1001 -# Minimal copy of pypy.module.signal.interp_signal.SignalActionFlag for -# TestLLtype -from rpython.rtyper.lltypesystem import lltype, rffi - -class Ticker(object): - def __init__(self): - self.ticker = rffi.llexternal('ticker', [], - lltype.Ptr(LONG_STRUCT), - compilation_info=eci, - sandboxsafe=True, _nowrapper=True, - elidable_function=True) - - def reset_ticker(self, value): - self.ticker().c_value = value - - def decrement_ticker(self, by): - self.ticker().c_value -= by - return self.ticker().c_value - class TestLLtype(DelTests, LLJitMixin): - def test_signal_action(self): - action = Ticker() - # - myjitdriver = JitDriver(greens = [], reds = ['n', 'x']) - class X: - pass - # - def f(n): - x = X() - action.reset_ticker(n) - while True: - myjitdriver.can_enter_jit(n=n, x=x) - myjitdriver.jit_merge_point(n=n, x=x) - x.foo = n - n -= 1 - if action.decrement_ticker(1) < 0: - break - return 42 - self.meta_interp(f, [20]) - self.check_resops(call_pure=0, setfield_raw=2, call=0, getfield_raw=2) + pass class TestOOtype(DelTests, OOJitMixin): def setup_class(cls): diff --git a/rpython/rlib/rsignal.py b/rpython/rlib/rsignal.py new file mode 100644 --- /dev/null +++ b/rpython/rlib/rsignal.py @@ -0,0 +1,100 @@ +import signal as cpy_signal +import sys +import py +from rpython.conftest import cdir +from rpython.rtyper.tool import rffi_platform +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.translator.tool.cbuild import ExternalCompilationInfo + +def setup(): + for key, value in cpy_signal.__dict__.items(): + if (key.startswith('SIG') or key.startswith('CTRL_')) and \ + is_valid_int(value) and \ + key != 'SIG_DFL' and key != 'SIG_IGN': + globals()[key] = value + yield key + +NSIG = cpy_signal.NSIG +SIG_DFL = cpy_signal.SIG_DFL +SIG_IGN = cpy_signal.SIG_IGN +signal_names = list(setup()) +signal_values = {} +for key in signal_names: + signal_values[globals()[key]] = None +if sys.platform == 'win32' and not hasattr(cpy_signal,'CTRL_C_EVENT'): + # XXX Hack to revive values that went missing, + # Remove this once we are sure the host cpy module has them. + signal_values[0] = None + signal_values[1] = None + signal_names.append('CTRL_C_EVENT') + signal_names.append('CTRL_BREAK_EVENT') + CTRL_C_EVENT = 0 + CTRL_BREAK_EVENT = 1 +includes = ['stdlib.h', 'src/signals.h'] +if sys.platform != 'win32': + includes.append('sys/time.h') + +cdir = py.path.local(cdir) + +eci = ExternalCompilationInfo( + includes = includes, + separate_module_files = [cdir / 'src' / 'signals.c'], + include_dirs = [str(cdir)], + export_symbols = ['pypysig_poll', 'pypysig_default', + 'pypysig_ignore', 'pypysig_setflag', + 'pypysig_reinstall', + 'pypysig_set_wakeup_fd', + 'pypysig_getaddr_occurred'], +) + +class CConfig: + _compilation_info_ = eci + +if sys.platform != 'win32': + for name in """ITIMER_REAL ITIMER_VIRTUAL ITIMER_PROF""".split(): + setattr(CConfig, name, rffi_platform.DefinedConstantInteger(name)) + + CConfig.timeval = rffi_platform.Struct( + 'struct timeval', + [('tv_sec', rffi.LONG), + ('tv_usec', rffi.LONG)]) + + CConfig.itimerval = rffi_platform.Struct( + 'struct itimerval', + [('it_value', CConfig.timeval), + ('it_interval', CConfig.timeval)]) + +for k, v in rffi_platform.configure(CConfig).items(): + globals()[k] = v + +def external(name, args, result, **kwds): + return rffi.llexternal(name, args, result, compilation_info=eci, + sandboxsafe=True, **kwds) + +pypysig_ignore = external('pypysig_ignore', [rffi.INT], lltype.Void) +pypysig_default = external('pypysig_default', [rffi.INT], lltype.Void) +pypysig_setflag = external('pypysig_setflag', [rffi.INT], lltype.Void) +pypysig_reinstall = external('pypysig_reinstall', [rffi.INT], lltype.Void) +pypysig_set_wakeup_fd = external('pypysig_set_wakeup_fd', [rffi.INT], rffi.INT) +pypysig_poll = external('pypysig_poll', [], rffi.INT, threadsafe=False) +# don't bother releasing the GIL around a call to pypysig_poll: it's +# pointless and a performance issue + +# don't use rffi.LONGP because the JIT doesn't support raw arrays so far +struct_name = 'pypysig_long_struct' +LONG_STRUCT = lltype.Struct(struct_name, ('c_value', lltype.Signed), + hints={'c_name' : struct_name, 'external' : 'C'}) +del struct_name + +pypysig_getaddr_occurred = external('pypysig_getaddr_occurred', [], + lltype.Ptr(LONG_STRUCT), _nowrapper=True, + elidable_function=True) +c_alarm = external('alarm', [rffi.INT], rffi.INT) +c_pause = external('pause', [], rffi.INT, threadsafe=True) +c_siginterrupt = external('siginterrupt', [rffi.INT, rffi.INT], rffi.INT) + +if sys.platform != 'win32': + itimervalP = rffi.CArrayPtr(itimerval) + c_setitimer = external('setitimer', + [rffi.INT, itimervalP, itimervalP], rffi.INT) + c_getitimer = external('getitimer', [rffi.INT, itimervalP], rffi.INT) diff --git a/pypy/module/signal/test/test_interp_signal.py b/rpython/rlib/test/test_rsignal.py rename from pypy/module/signal/test/test_interp_signal.py rename to rpython/rlib/test/test_rsignal.py --- a/pypy/module/signal/test/test_interp_signal.py +++ b/rpython/rlib/test/test_rsignal.py @@ -1,16 +1,16 @@ import os, py from rpython.translator.c.test.test_genc import compile -from pypy.module.signal import interp_signal +from rpython.rlib import rsignal def setup_module(mod): if not hasattr(os, 'kill') or not hasattr(os, 'getpid'): py.test.skip("requires os.kill() and os.getpid()") - if not hasattr(interp_signal, 'SIGUSR1'): + if not hasattr(signals, 'SIGUSR1'): py.test.skip("requires SIGUSR1 in signal") def check(expected): - res = interp_signal.pypysig_poll() + res = rsignal.pypysig_poll() os.write(1, "poll() => %d, expected %d\n" % (res, expected)) assert res == expected @@ -19,18 +19,18 @@ check(-1) check(-1) for i in range(3): - interp_signal.pypysig_setflag(interp_signal.SIGUSR1) - os.kill(os.getpid(), interp_signal.SIGUSR1) - check(interp_signal.SIGUSR1) + rsignal.pypysig_setflag(rsignal.SIGUSR1) + os.kill(os.getpid(), rsignal.SIGUSR1) + check(rsignal.SIGUSR1) check(-1) check(-1) - interp_signal.pypysig_ignore(interp_signal.SIGUSR1) - os.kill(os.getpid(), interp_signal.SIGUSR1) + rsignal.pypysig_ignore(rsignal.SIGUSR1) + os.kill(os.getpid(), rsignal.SIGUSR1) check(-1) check(-1) - interp_signal.pypysig_default(interp_signal.SIGUSR1) + rsignal.pypysig_default(rsignal.SIGUSR1) check(-1) diff --git a/rpython/translator/c/src/signals.h b/rpython/translator/c/src/signals.h --- a/rpython/translator/c/src/signals.h +++ b/rpython/translator/c/src/signals.h @@ -13,7 +13,7 @@ int pypysig_poll(void); /* => signum or -1 */ /* When a signal is received, pypysig_counter is set to -1. */ -/* This is a struct for the JIT. See interp_signal.py. */ +/* This is a struct for the JIT. See rsignal.py. */ struct pypysig_long_struct { long value; }; diff --git a/rpython/translator/c/test/test_extfunc.py b/rpython/translator/c/test/test_extfunc.py --- a/rpython/translator/c/test/test_extfunc.py +++ b/rpython/translator/c/test/test_extfunc.py @@ -547,13 +547,13 @@ if hasattr(os, 'kill'): def test_kill_to_send_sigusr1(): import signal - from pypy.module.signal import interp_signal + from rpython.rlib import rsignal def does_stuff(): - interp_signal.pypysig_setflag(signal.SIGUSR1) + rsignal.pypysig_setflag(signal.SIGUSR1) os.kill(os.getpid(), signal.SIGUSR1) - interp_signal.pypysig_ignore(signal.SIGUSR1) + rsignal.pypysig_ignore(signal.SIGUSR1) while True: - n = interp_signal.pypysig_poll() + n = rsignal.pypysig_poll() if n < 0 or n == signal.SIGUSR1: break return n @@ -564,14 +564,14 @@ if hasattr(os, 'killpg'): def test_killpg(): import signal - from pypy.module.signal import interp_signal + from rpython.rlib import rsignal def does_stuff(): os.setpgid(0, 0) # become its own separated process group - interp_signal.pypysig_setflag(signal.SIGUSR1) + rsignal.pypysig_setflag(signal.SIGUSR1) os.killpg(os.getpgrp(), signal.SIGUSR1) - interp_signal.pypysig_ignore(signal.SIGUSR1) + rsignal.pypysig_ignore(signal.SIGUSR1) while True: - n = interp_signal.pypysig_poll() + n = rsignal.pypysig_poll() if n < 0 or n == signal.SIGUSR1: break return n From noreply at buildbot.pypy.org Thu Jan 10 21:20:31 2013 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 10 Jan 2013 21:20:31 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: merge from default Message-ID: <20130110202031.438471C00E2@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: missing-ndarray-attributes Changeset: r59930:54d295249a0a Date: 2013-01-10 20:03 +0200 http://bitbucket.org/pypy/pypy/changeset/54d295249a0a/ Log: merge from default diff too long, truncating to 2000 out of 12632 lines diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -28,7 +28,7 @@ DEALINGS IN THE SOFTWARE. -PyPy Copyright holders 2003-2012 +PyPy Copyright holders 2003-2013 ----------------------------------- Except when otherwise stated (look for LICENSE files or information at diff --git a/lib-python/2.7/sre_parse.py b/lib-python/2.7/sre_parse.py --- a/lib-python/2.7/sre_parse.py +++ b/lib-python/2.7/sre_parse.py @@ -16,6 +16,12 @@ from sre_constants import * +try: + from __pypy__ import newdict +except ImportError: + def newdict(tp): + return {} + SPECIAL_CHARS = ".\\[{()*+?^$|" REPEAT_CHARS = "*+?{" @@ -68,7 +74,7 @@ self.flags = 0 self.open = [] self.groups = 1 - self.groupdict = {} + self.groupdict = newdict("module") def opengroup(self, name=None): gid = self.groups self.groups = gid + 1 diff --git a/lib_pypy/numpypy/core/arrayprint.py b/lib_pypy/numpypy/core/arrayprint.py --- a/lib_pypy/numpypy/core/arrayprint.py +++ b/lib_pypy/numpypy/core/arrayprint.py @@ -248,9 +248,9 @@ 'int' : IntegerFormat(data), 'float' : FloatFormat(data, precision, suppress_small), 'longfloat' : LongFloatFormat(precision), - #'complexfloat' : ComplexFormat(data, precision, - # suppress_small), - #'longcomplexfloat' : LongComplexFormat(precision), + 'complexfloat' : ComplexFormat(data, precision, + suppress_small), + 'longcomplexfloat' : LongComplexFormat(precision), 'datetime' : DatetimeFormat(data), 'timedelta' : TimedeltaFormat(data), 'numpystr' : repr_format, @@ -294,19 +294,19 @@ #else: format_function = formatdict['int'] elif issubclass(dtypeobj, _nt.floating): - #if issubclass(dtypeobj, _nt.longfloat): - # format_function = formatdict['longfloat'] - #else: - format_function = formatdict['float'] - #elif issubclass(dtypeobj, _nt.complexfloating): - # if issubclass(dtypeobj, _nt.clongfloat): - # format_function = formatdict['longcomplexfloat'] - # else: - # format_function = formatdict['complexfloat'] + if issubclass(dtypeobj, _nt.longfloat): + format_function = formatdict['longfloat'] + else: + format_function = formatdict['float'] + elif issubclass(dtypeobj, _nt.complexfloating): + if issubclass(dtypeobj, _nt.clongfloat): + format_function = formatdict['longcomplexfloat'] + else: + format_function = formatdict['complexfloat'] elif issubclass(dtypeobj, (_nt.unicode_, _nt.string_)): format_function = formatdict['numpystr'] - elif issubclass(dtypeobj, _nt.datetime64): - format_function = formatdict['datetime'] + #elif issubclass(dtypeobj, _nt.datetime64): + # format_function = formatdict['datetime'] else: format_function = formatdict['str'] diff --git a/pypy/doc/Makefile b/pypy/doc/Makefile --- a/pypy/doc/Makefile +++ b/pypy/doc/Makefile @@ -32,38 +32,38 @@ -rm -rf $(BUILDDIR)/* html: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." pickle: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ @@ -73,7 +73,7 @@ @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/PyPy.qhc" latex: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @@ -81,26 +81,26 @@ "run these through (pdf)latex." man: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man" changes: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." doctest: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -38,7 +38,7 @@ # General information about the project. project = u'PyPy' -copyright = u'2011, The PyPy Project' +copyright = u'2013, The PyPy Project' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/pypy/doc/config/translation.no__thread.txt b/pypy/doc/config/translation.no__thread.txt --- a/pypy/doc/config/translation.no__thread.txt +++ b/pypy/doc/config/translation.no__thread.txt @@ -1,3 +1,3 @@ Don't use gcc __thread attribute for fast thread local storage -implementation . Increases the chance that moving the resulting +implementation. Increases the chance that moving the resulting executable to another same processor Linux machine will work. diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst --- a/pypy/doc/index.rst +++ b/pypy/doc/index.rst @@ -67,7 +67,7 @@ .. _Python: http://docs.python.org/index.html .. _`more...`: architecture.html#mission-statement .. _`PyPy blog`: http://morepypy.blogspot.com/ -.. _`development bug/feature tracker`: https://codespeak.net/issue/pypy-dev/ +.. _`development bug/feature tracker`: https://bugs.pypy.org .. _here: http://tismerysoft.de/pypy/irc-logs/pypy .. _`Mercurial commit mailing list`: http://python.org/mailman/listinfo/pypy-commit .. _`development mailing list`: http://python.org/mailman/listinfo/pypy-dev diff --git a/pypy/doc/pypyconfig.py b/pypy/doc/pypyconfig.py --- a/pypy/doc/pypyconfig.py +++ b/pypy/doc/pypyconfig.py @@ -3,6 +3,10 @@ def setup(app): import sys, os sys.path.insert(0, os.path.abspath("../../")) + + #Autmatically calls make_cmdlline_overview + from pypy.doc.config import generate + from pypy.config import makerestdoc import py role = makerestdoc.register_config_role(py.path.local()) diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -5,15 +5,35 @@ .. this is a revision shortly after release-2.0-beta1 .. startrev: 0e6161a009c6 +.. branch: callback-jit +Callbacks from C are now better JITted + +.. branch: remove-globals-in-jit + .. branch: length-hint Implement __lenght_hint__ according to PEP 424 +.. branch: numpypy-longdouble +Long double support for numpypy + +.. branch: signatures +Improved RPython typing + +.. branch: rpython-bytearray +Rudimentary support for bytearray in RPython + .. branches we don't care about .. branch: autoreds +.. branch: reflex-support +.. branch: kill-faking +.. branch: improved_ebnfparse_error +.. branch: task-decorator .. branch: release-2.0-beta1 .. branch: remove-PYPY_NOT_MAIN_FILE +.. branch: missing-jit-operations + .. branch: fix-lookinside-iff-oopspec Fixed the interaction between two internal tools for controlling the JIT. diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -56,11 +56,24 @@ ---------------------------- On Windows, there is no standard place where to download, build and -install third-party libraries. We chose to install them in the parent +install third-party libraries. We recommend installing them in the parent directory of the pypy checkout. For example, if you installed pypy in ``d:\pypy\trunk\`` (This directory contains a README file), the base -directory is ``d:\pypy``. You may choose different values by setting the -INCLUDE, LIB and PATH (for DLLs) +directory is ``d:\pypy``. You must then set the +INCLUDE, LIB and PATH (for DLLs) environment variables appropriately. + +Abridged method (for -Ojit builds using Visual Studio 2008) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Download the versions of all the external packages +from +https://bitbucket.org/pypy/pypy/downloads/local.zip +Then expand it into the base directory (base_dir) and modify your environment to reflect this:: + + set PATH=\bin;%PATH% + set INCLUDE=\include;%INCLUDE% + set LIB=\lib;%LIB% + +Now you should be good to go. Read on for more information. The Boehm garbage collector ~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py --- a/pypy/interpreter/typedef.py +++ b/pypy/interpreter/typedef.py @@ -105,6 +105,13 @@ # typical usage (case 1 is the most common kind of app-level subclasses; # case 2 is the memory-saving kind defined with __slots__). # +# +----------------------------------------------------------------+ +# | NOTE: if withmapdict is enabled, the following doesn't apply! | +# | Map dicts can flexibly allow any slots/__dict__/__weakref__ to | +# | show up only when needed. In particular there is no way with | +# | mapdict to prevent some objects from being weakrefable. | +# +----------------------------------------------------------------+ +# # dict slots del weakrefable # # 1. Y N N Y UserDictWeakref diff --git a/pypy/jit/backend/arm/assembler.py b/pypy/jit/backend/arm/assembler.py --- a/pypy/jit/backend/arm/assembler.py +++ b/pypy/jit/backend/arm/assembler.py @@ -1,5 +1,6 @@ from __future__ import with_statement import os +from pypy.jit.backend.llsupport import jitframe from pypy.jit.backend.arm.helper.assembler import saved_registers from pypy.jit.backend.arm import conditions as c from pypy.jit.backend.arm import registers as r @@ -8,7 +9,7 @@ from pypy.jit.backend.arm.codebuilder import ARMv7Builder, OverwritingBuilder from pypy.jit.backend.arm.locations import get_fp_offset from pypy.jit.backend.arm.regalloc import (Regalloc, ARMFrameManager, - ARMv7RegisterManager, check_imm_arg, + CoreRegisterManager, check_imm_arg, operations as regalloc_operations, operations_with_guard as regalloc_operations_with_guard) from pypy.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper @@ -25,12 +26,12 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.jit.backend.arm.opassembler import ResOpAssembler from pypy.rlib.debug import (debug_print, debug_start, debug_stop, - have_debug_prints) + have_debug_prints, fatalerror) from pypy.rlib.jit import AsmInfo from pypy.rlib.objectmodel import compute_unique_id # XXX Move to llsupport -from pypy.jit.backend.x86.support import values_array, memcpy_fn +from pypy.jit.backend.x86.support import memcpy_fn DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', ('i', lltype.Signed), ('type', lltype.Char), # 'b'ridge, 'l'abel or @@ -44,14 +45,8 @@ debug = True - def __init__(self, cpu, failargs_limit=1000): + def __init__(self, cpu, translate_support_code=False): self.cpu = cpu - self.fail_boxes_int = values_array(lltype.Signed, failargs_limit) - self.fail_boxes_float = values_array(longlong.FLOATSTORAGE, - failargs_limit) - self.fail_boxes_ptr = values_array(llmemory.GCREF, failargs_limit) - self.fail_boxes_count = 0 - self.fail_force_index = 0 self.setup_failure_recovery() self.mc = None self.memcpy_addr = 0 @@ -68,6 +63,7 @@ self._debug = False self.loop_run_counters = [] self.debug_counter_descr = cpu.fielddescrof(DEBUG_COUNTER, 'i') + self.force_token_to_dead_frame = {} # XXX temporary hack def set_debug(self, v): r = self._debug @@ -112,9 +108,13 @@ gc_ll_descr.initialize() self._build_wb_slowpath(False) self._build_wb_slowpath(True) + self._build_failure_recovery(exc=True, withfloats=False) + self._build_failure_recovery(exc=False, withfloats=False) if self.cpu.supports_floats: self._build_wb_slowpath(False, withfloats=True) self._build_wb_slowpath(True, withfloats=True) + self._build_failure_recovery(exc=True, withfloats=True) + self._build_failure_recovery(exc=False, withfloats=True) self._build_propagate_exception_path() if gc_ll_descr.get_malloc_slowpath_addr is not None: self._build_malloc_slowpath() @@ -122,10 +122,7 @@ if gc_ll_descr.gcrootmap and gc_ll_descr.gcrootmap.is_shadow_stack: self._build_release_gil(gc_ll_descr.gcrootmap) self.memcpy_addr = self.cpu.cast_ptr_to_int(memcpy_fn) - self._exit_code_addr = self._gen_exit_path() - self._leave_jitted_hook_save_exc = \ - self._gen_leave_jitted_hook_code(True) - self._leave_jitted_hook = self._gen_leave_jitted_hook_code(False) + if not self._debug: # if self._debug is already set it means that someone called # set_debug by hand before initializing the assembler. Leave it @@ -220,36 +217,17 @@ self.releasegil_addr = rffi.cast(lltype.Signed, releasegil_func) self.reacqgil_addr = rffi.cast(lltype.Signed, reacqgil_func) - def _gen_leave_jitted_hook_code(self, save_exc): - mc = ARMv7Builder() - if self.cpu.supports_floats: - floats = r.caller_vfp_resp - else: - floats = [] - with saved_registers(mc, r.caller_resp + [r.lr], floats): - addr = self.cpu.get_on_leave_jitted_int(save_exception=save_exc) - mc.BL(addr) - assert self._exit_code_addr != 0 - mc.B(self._exit_code_addr) - return mc.materialize(self.cpu.asmmemmgr, [], - self.cpu.gc_ll_descr.gcrootmap) - def _build_propagate_exception_path(self): if self.cpu.propagate_exception_v < 0: return # not supported (for tests, or non-translated) # mc = ARMv7Builder() - # call on_leave_jitted_save_exc() - if self.cpu.supports_floats: - floats = r.caller_vfp_resp - else: - floats = [] - with saved_registers(mc, r.caller_resp + [r.lr], floats): - addr = self.cpu.get_on_leave_jitted_int(save_exception=True, - default_to_memoryerror=True) - mc.BL(addr) - mc.gen_load_int(r.ip.value, self.cpu.propagate_exception_v) - mc.MOV_rr(r.r0.value, r.ip.value) + # + # Call the helper, which will return a dead frame object with + # the correct exception set, or MemoryError by default + # XXX make sure we return the correct value here + addr = rffi.cast(lltype.Signed, self.cpu.get_propagate_exception()) + mc.BL(addr) self.gen_func_epilog(mc=mc) self.propagate_exception_path = mc.materialize(self.cpu.asmmemmgr, []) @@ -282,12 +260,12 @@ # restore registers and return # We check for c.EQ here, meaning all bits zero in this case mc.POP([reg.value for reg in r.argument_regs] + [r.pc.value], cond=c.EQ) - # call on_leave_jitted_save_exc() - addr = self.cpu.get_on_leave_jitted_int(save_exception=True) + # + # Call the helper, which will return a dead frame object with + # the correct exception set, or MemoryError by default + addr = rffi.cast(lltype.Signed, self.cpu.get_propagate_exception()) mc.BL(addr) # - mc.gen_load_int(r.r0.value, self.cpu.propagate_exception_v) - # # footer -- note the ADD, which skips the return address of this # function, and will instead return to the caller's caller. Note # also that we completely ignore the saved arguments, because we @@ -339,39 +317,82 @@ def setup_failure_recovery(self): - @rgc.no_collect + #@rgc.no_collect -- XXX still true, but hacked gc_set_extra_threshold def failure_recovery_func(mem_loc, frame_pointer, stack_pointer): """mem_loc is a structure in memory describing where the values for the failargs are stored. frame loc is the address of the frame pointer for the frame to be decoded frame """ - vfp_registers = rffi.cast(rffi.LONGLONGP, stack_pointer) - registers = rffi.ptradd(vfp_registers, len(r.all_vfp_regs)) + vfp_registers = rffi.cast(rffi.LONGP, stack_pointer) + registers = rffi.ptradd(vfp_registers, 2*len(r.all_vfp_regs)) registers = rffi.cast(rffi.LONGP, registers) - return self.decode_registers_and_descr(mem_loc, frame_pointer, + bytecode = rffi.cast(rffi.UCHARP, mem_loc) + return self.grab_frame_values(self.cpu, bytecode, frame_pointer, registers, vfp_registers) + self.failure_recovery_code = [0, 0, 0, 0] self.failure_recovery_func = failure_recovery_func - recovery_func_sign = lltype.Ptr(lltype.FuncType([lltype.Signed] * 3, - lltype.Signed)) + _FAILURE_RECOVERY_FUNC = lltype.Ptr(lltype.FuncType([rffi.LONGP] * 3, + llmemory.GCREF)) - @rgc.no_collect - def decode_registers_and_descr(self, mem_loc, frame_pointer, + @staticmethod + #@rgc.no_collect -- XXX still true, but hacked gc_set_extra_threshold + def grab_frame_values(cpu, bytecode, frame_pointer, registers, vfp_registers): - """Decode locations encoded in memory at mem_loc and write the values - to the failboxes. Values for spilled vars and registers are stored on - stack at frame_loc """ - assert frame_pointer & 1 == 0 - self.fail_force_index = frame_pointer - bytecode = rffi.cast(rffi.UCHARP, mem_loc) + # no malloc allowed here!! xxx apart from one, hacking a lot + force_index = rffi.cast(lltype.Signed, frame_pointer) num = 0 - value = 0 - fvalue = 0 + deadframe = lltype.nullptr(jitframe.DEADFRAME) + # step 1: lots of mess just to count the final value of 'num' + bytecode1 = bytecode + while 1: + code = rffi.cast(lltype.Signed, bytecode1[0]) + bytecode1 = rffi.ptradd(bytecode1, 1) + if code >= AssemblerARM.CODE_FROMSTACK: + while code > 0x7F: + code = rffi.cast(lltype.Signed, bytecode1[0]) + bytecode1 = rffi.ptradd(bytecode1, 1) + else: + kind = code & 3 + if kind == AssemblerARM.DESCR_SPECIAL: + if code == AssemblerARM.CODE_HOLE: + num += 1 + continue + if code == AssemblerARM.CODE_INPUTARG: + continue + if code == AssemblerARM.CODE_FORCED: + # resuming from a GUARD_NOT_FORCED + token = force_index + deadframe = ( + cpu.assembler.force_token_to_dead_frame.pop(token)) + deadframe = lltype.cast_opaque_ptr( + jitframe.DEADFRAMEPTR, deadframe) + continue + assert code == AssemblerARM.CODE_STOP + break + num += 1 + + # allocate the deadframe + if not deadframe: + # Remove the "reserve" at the end of the nursery. This means + # that it is guaranteed that the following malloc() works + # without requiring a collect(), but it needs to be re-added + # as soon as possible. + cpu.gc_clear_extra_threshold() + assert num <= cpu.get_failargs_limit() + try: + deadframe = lltype.malloc(jitframe.DEADFRAME, num) + except MemoryError: + fatalerror("memory usage error in grab_frame_values") + # fill it code_inputarg = False - while True: + num = 0 + value_hi = 0 + while 1: + # decode the next instruction from the bytecode code = rffi.cast(lltype.Signed, bytecode[0]) bytecode = rffi.ptradd(bytecode, 1) - if code >= self.CODE_FROMSTACK: + if code >= AssemblerARM.CODE_FROMSTACK: if code > 0x7F: shift = 7 code &= 0x7F @@ -384,54 +405,63 @@ break # load the value from the stack kind = code & 3 - code = int((code - self.CODE_FROMSTACK) >> 2) + code = (code - AssemblerARM.CODE_FROMSTACK) >> 2 if code_inputarg: code = ~code code_inputarg = False - if kind == self.DESCR_FLOAT: - # we use code + 1 to get the hi word of the double worded float - stackloc = frame_pointer - get_fp_offset(int(code) + 1) - assert stackloc & 3 == 0 - fvalue = rffi.cast(rffi.LONGLONGP, stackloc)[0] - else: - stackloc = frame_pointer - get_fp_offset(int(code)) - assert stackloc & 1 == 0 - value = rffi.cast(rffi.LONGP, stackloc)[0] + stackloc = force_index - get_fp_offset(int(code)) + value = rffi.cast(rffi.LONGP, stackloc)[0] + if kind == AssemblerARM.DESCR_FLOAT: + assert WORD == 4 + value_hi = value + value = rffi.cast(rffi.LONGP, stackloc - WORD)[0] else: - # 'code' identifies a register: load its value kind = code & 3 - if kind == self.DESCR_SPECIAL: - if code == self.CODE_HOLE: + if kind == AssemblerARM.DESCR_SPECIAL: + if code == AssemblerARM.CODE_HOLE: num += 1 continue - if code == self.CODE_INPUTARG: + if code == AssemblerARM.CODE_INPUTARG: code_inputarg = True continue - assert code == self.CODE_STOP + if code == AssemblerARM.CODE_FORCED: + continue + assert code == AssemblerARM.CODE_STOP break + # 'code' identifies a register: load its value code >>= 2 - if kind == self.DESCR_FLOAT: - fvalue = vfp_registers[code] + if kind == AssemblerARM.DESCR_FLOAT: + if WORD == 4: + value = vfp_registers[2*code] + value_hi = vfp_registers[2*code + 1] + else: + value = registers[code] else: value = registers[code] # store the loaded value into fail_boxes_ - if kind == self.DESCR_FLOAT: - tgt = self.fail_boxes_float.get_addr_for_num(num) - rffi.cast(rffi.LONGLONGP, tgt)[0] = fvalue + if kind == AssemblerARM.DESCR_INT: + deadframe.jf_values[num].int = value + elif kind == AssemblerARM.DESCR_REF: + deadframe.jf_values[num].ref = rffi.cast(llmemory.GCREF, value) + elif kind == AssemblerARM.DESCR_FLOAT: + assert WORD == 4 + assert not longlong.is_64_bit + floatvalue = rffi.cast(lltype.SignedLongLong, value_hi) + floatvalue <<= 32 + floatvalue |= rffi.cast(lltype.SignedLongLong, + rffi.cast(lltype.Unsigned, value)) + deadframe.jf_values[num].float = floatvalue else: - if kind == self.DESCR_INT: - tgt = self.fail_boxes_int.get_addr_for_num(num) - elif kind == self.DESCR_REF: - assert (value & 3) == 0, "misaligned pointer" - tgt = self.fail_boxes_ptr.get_addr_for_num(num) - else: - assert 0, "bogus kind" - rffi.cast(rffi.LONGP, tgt)[0] = value + assert 0, "bogus kind" num += 1 - self.fail_boxes_count = num + # + assert num == len(deadframe.jf_values) + if not we_are_translated(): + assert bytecode[4] == 0xCC fail_index = rffi.cast(rffi.INTP, bytecode)[0] - fail_index = rffi.cast(lltype.Signed, fail_index) - return fail_index + fail_descr = cpu.get_fail_descr_from_number(fail_index) + deadframe.jf_descr = fail_descr.hide(cpu) + return lltype.cast_opaque_ptr(llmemory.GCREF, deadframe) def decode_inputargs(self, code): descr_to_box_type = [REF, INT, FLOAT] @@ -492,10 +522,10 @@ # are stored in r0 and r1. mc.SUB_rr(r.r0.value, r.r1.value, r.r0.value) addr = self.cpu.gc_ll_descr.get_malloc_slowpath_addr() - for reg, ofs in ARMv7RegisterManager.REGLOC_TO_COPY_AREA_OFS.items(): + for reg, ofs in CoreRegisterManager.REGLOC_TO_COPY_AREA_OFS.items(): mc.STR_ri(reg.value, r.fp.value, imm=ofs) mc.BL(addr) - for reg, ofs in ARMv7RegisterManager.REGLOC_TO_COPY_AREA_OFS.items(): + for reg, ofs in CoreRegisterManager.REGLOC_TO_COPY_AREA_OFS.items(): mc.LDR_ri(reg.value, r.fp.value, imm=ofs) mc.CMP_ri(r.r0.value, 0) @@ -514,12 +544,26 @@ self.mc.CMP_ri(r.r0.value, 0) self.mc.B(self.propagate_exception_path, c=c.EQ) - def _gen_exit_path(self): + def _build_failure_recovery(self, exc, withfloats=False): mc = ARMv7Builder() - decode_registers_addr = llhelper(self.recovery_func_sign, + failure_recovery = llhelper(self._FAILURE_RECOVERY_FUNC, self.failure_recovery_func) self._insert_checks(mc) - with saved_registers(mc, r.all_regs, r.all_vfp_regs): + if withfloats: + f = r.all_vfp_regs + else: + f = [] + with saved_registers(mc, r.all_regs, f): + if exc: + # We might have an exception pending. Load it into r4 + # (this is a register saved across calls) + mc.gen_load_int(r.r5.value, self.cpu.pos_exc_value()) + mc.LDR_ri(r.r4.value, self.cpu.pos_exc_value()) + # clear the exc flags + mc.gen_load_int(r.r6.value, 0) + mc.STR_ri(r.r6.value, r.r5.value) + mc.gen_load_int(r.r5.value, self.cpu.pos_exception()) + mc.STR_ri(r.r6.value, r.r5.value) # move mem block address, to r0 to pass as mc.MOV_rr(r.r0.value, r.lr.value) # pass the current frame pointer as second param @@ -527,23 +571,32 @@ # pass the current stack pointer as third param mc.MOV_rr(r.r2.value, r.sp.value) self._insert_checks(mc) - mc.BL(rffi.cast(lltype.Signed, decode_registers_addr)) + mc.BL(rffi.cast(lltype.Signed, failure_recovery)) + if exc: + # save ebx into 'jf_guard_exc' + from pypy.jit.backend.llsupport.descr import unpack_fielddescr + descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) + offset, size, _ = unpack_fielddescr(descrs.jf_guard_exc) + mc.STR_rr(r.r4.value, r.r0.value, offset, cond=c.AL) mc.MOV_rr(r.ip.value, r.r0.value) mc.MOV_rr(r.r0.value, r.ip.value) self.gen_func_epilog(mc=mc) - return mc.materialize(self.cpu.asmmemmgr, [], + rawstart = mc.materialize(self.cpu.asmmemmgr, [], self.cpu.gc_ll_descr.gcrootmap) + self.failure_recovery_code[exc + 2 * withfloats] = rawstart + self.mc = None DESCR_REF = 0x00 DESCR_INT = 0x01 DESCR_FLOAT = 0x02 DESCR_SPECIAL = 0x03 CODE_FROMSTACK = 64 - CODE_STOP = 0 | DESCR_SPECIAL - CODE_HOLE = 4 | DESCR_SPECIAL - CODE_INPUTARG = 8 | DESCR_SPECIAL + CODE_STOP = 0 | DESCR_SPECIAL + CODE_HOLE = 4 | DESCR_SPECIAL + CODE_INPUTARG = 8 | DESCR_SPECIAL + CODE_FORCED = 12 | DESCR_SPECIAL #XXX where should this be written? - def gen_descr_encoding(self, descr, failargs, locs): + def write_failure_recovery_description(self, descr, failargs, locs): assert self.mc is not None for i in range(len(failargs)): arg = failargs[i] @@ -575,26 +628,33 @@ self.mc.writechar(chr(n)) self.mc.writechar(chr(self.CODE_STOP)) - fdescr = self.cpu.get_fail_descr_number(descr) - self.mc.write32(fdescr) + + def generate_quick_failure(self, guardtok, fcond=c.AL): + assert isinstance(guardtok.save_exc, bool) + fail_index = self.cpu.get_fail_descr_number(guardtok.descr) + startpos = self.mc.currpos() + withfloats = False + for box in guardtok.failargs: + if box is not None and box.type == FLOAT: + withfloats = True + break + exc = guardtok.save_exc + target = self.failure_recovery_code[exc + 2 * withfloats] + assert target != 0 + self.mc.BL(target) + # write tight data that describes the failure recovery + if guardtok.is_guard_not_forced: + self.mc.writechar(chr(self.CODE_FORCED)) + self.write_failure_recovery_description(guardtok.descr, + guardtok.failargs, guardtok.faillocs[1:]) + self.mc.write32(fail_index) + # for testing the decoding, write a final byte 0xCC + if not we_are_translated(): + self.mc.writechar('\xCC') + faillocs = [loc for loc in guardtok.faillocs if loc is not None] + guardtok.descr._arm_debug_faillocs = faillocs self.align() - - # assert that the fail_boxes lists are big enough - assert len(failargs) <= self.fail_boxes_int.SIZE - - def _gen_path_to_exit_path(self, descr, args, arglocs, - save_exc, fcond=c.AL): - assert isinstance(save_exc, bool) - self.gen_exit_code(self.mc, save_exc, fcond) - self.gen_descr_encoding(descr, args, arglocs[1:]) - - def gen_exit_code(self, mc, save_exc, fcond=c.AL): - assert isinstance(save_exc, bool) - if save_exc: - path = self._leave_jitted_hook_save_exc - else: - path = self._leave_jitted_hook - mc.BL(path) + return startpos def align(self): while(self.mc.currpos() % FUNC_ALIGN != 0): @@ -841,15 +901,10 @@ def write_pending_failure_recoveries(self): for tok in self.pending_guards: - descr = tok.descr #generate the exit stub and the encoded representation - pos = self.mc.currpos() - tok.pos_recovery_stub = pos - - self._gen_path_to_exit_path(descr, tok.failargs, - tok.faillocs, save_exc=tok.save_exc) + tok.pos_recovery_stub = self.generate_quick_failure(tok) # store info on the descr - descr._arm_current_frame_depth = tok.faillocs[0].getint() + tok.descr._arm_current_frame_depth = tok.faillocs[0].getint() def process_pending_guards(self, block_start): clt = self.current_clt @@ -860,7 +915,7 @@ descr._arm_failure_recovery_block = failure_recovery_pos relative_offset = tok.pos_recovery_stub - tok.offset guard_pos = block_start + tok.offset - if not tok.is_invalidate: + if not tok.is_guard_not_invalidated: # patch the guard jumpt to the stub # overwrite the generate NOP with a B_offs to the pos of the # stub @@ -1255,11 +1310,6 @@ else: raise AssertionError('Trying to pop to an invalid location') - def leave_jitted_hook(self): - ptrs = self.fail_boxes_ptr.ar - llop.gc_assume_young_pointers(lltype.Void, - llmemory.cast_ptr_to_adr(ptrs)) - def malloc_cond(self, nursery_free_adr, nursery_top_adr, size): assert size & (WORD-1) == 0 # must be correctly aligned diff --git a/pypy/jit/backend/arm/helper/regalloc.py b/pypy/jit/backend/arm/helper/regalloc.py --- a/pypy/jit/backend/arm/helper/regalloc.py +++ b/pypy/jit/backend/arm/helper/regalloc.py @@ -32,14 +32,14 @@ imm_a0 = check_imm_box(a0, imm_size, allow_zero=allow_zero) imm_a1 = check_imm_box(a1, imm_size, allow_zero=allow_zero) if not imm_a0 and imm_a1: - l0 = self._ensure_value_is_boxed(a0) + l0 = self.make_sure_var_in_reg(a0) l1 = self.convert_to_imm(a1) elif commutative and imm_a0 and not imm_a1: l1 = self.convert_to_imm(a0) - l0 = self._ensure_value_is_boxed(a1, boxes) + l0 = self.make_sure_var_in_reg(a1, boxes) else: - l0 = self._ensure_value_is_boxed(a0, boxes) - l1 = self._ensure_value_is_boxed(a1, boxes) + l0 = self.make_sure_var_in_reg(a0, boxes) + l1 = self.make_sure_var_in_reg(a1, boxes) self.possibly_free_vars_for_op(op) self.free_temp_vars() res = self.force_allocate_reg(op.result, boxes) @@ -52,10 +52,10 @@ if guard: def f(self, op, guard_op, fcond): locs = [] - loc1 = self._ensure_value_is_boxed(op.getarg(0)) + loc1 = self.make_sure_var_in_reg(op.getarg(0)) locs.append(loc1) if base: - loc2 = self._ensure_value_is_boxed(op.getarg(1)) + loc2 = self.make_sure_var_in_reg(op.getarg(1)) locs.append(loc2) self.possibly_free_vars_for_op(op) self.free_temp_vars() @@ -70,10 +70,10 @@ else: def f(self, op, fcond): locs = [] - loc1 = self._ensure_value_is_boxed(op.getarg(0)) + loc1 = self.make_sure_var_in_reg(op.getarg(0)) locs.append(loc1) if base: - loc2 = self._ensure_value_is_boxed(op.getarg(1)) + loc2 = self.make_sure_var_in_reg(op.getarg(1)) locs.append(loc2) self.possibly_free_vars_for_op(op) self.free_temp_vars() @@ -111,11 +111,11 @@ arg0, arg1 = boxes imm_a1 = check_imm_box(arg1) - l0 = self._ensure_value_is_boxed(arg0, forbidden_vars=boxes) + l0 = self.make_sure_var_in_reg(arg0, forbidden_vars=boxes) if imm_a1: l1 = self.convert_to_imm(arg1) else: - l1 = self._ensure_value_is_boxed(arg1, forbidden_vars=boxes) + l1 = self.make_sure_var_in_reg(arg1, forbidden_vars=boxes) self.possibly_free_vars_for_op(op) self.free_temp_vars() @@ -134,7 +134,7 @@ assert fcond is not None a0 = op.getarg(0) assert isinstance(a0, Box) - reg = self._ensure_value_is_boxed(a0) + reg = self.make_sure_var_in_reg(a0) self.possibly_free_vars_for_op(op) if guard_op is None: res = self.force_allocate_reg(op.result, [a0]) diff --git a/pypy/jit/backend/arm/opassembler.py b/pypy/jit/backend/arm/opassembler.py --- a/pypy/jit/backend/arm/opassembler.py +++ b/pypy/jit/backend/arm/opassembler.py @@ -16,6 +16,7 @@ gen_emit_unary_float_op, saved_registers, count_reg_args) +from pypy.jit.backend.arm.helper.regalloc import check_imm_arg from pypy.jit.backend.arm.codebuilder import ARMv7Builder, OverwritingBuilder from pypy.jit.backend.arm.jump import remap_frame_layout from pypy.jit.backend.arm.regalloc import TempInt, TempPtr @@ -27,18 +28,20 @@ from pypy.jit.metainterp.history import JitCellToken, TargetToken from pypy.jit.metainterp.resoperation import rop from pypy.rlib.objectmodel import we_are_translated -from pypy.rpython.lltypesystem import rstr +from pypy.rlib import rgc +from pypy.rpython.lltypesystem import rstr, rffi, lltype, llmemory NO_FORCE_INDEX = -1 class GuardToken(object): - def __init__(self, descr, failargs, faillocs, offset, - save_exc, fcond=c.AL, is_invalidate=False): + def __init__(self, descr, failargs, faillocs, offset, save_exc, fcond=c.AL, + is_guard_not_invalidated=False, is_guard_not_forced=False): assert isinstance(save_exc, bool) self.descr = descr self.offset = offset - self.is_invalidate = is_invalidate + self.is_guard_not_invalidated = is_guard_not_invalidated + self.is_guard_not_forced = is_guard_not_forced self.failargs = failargs self.faillocs = faillocs self.save_exc = save_exc @@ -190,7 +193,8 @@ return fcond def _emit_guard(self, op, arglocs, fcond, save_exc, - is_guard_not_invalidated=False): + is_guard_not_invalidated=False, + is_guard_not_forced=False): assert isinstance(save_exc, bool) assert isinstance(fcond, int) descr = op.getdescr() @@ -210,7 +214,8 @@ faillocs=arglocs, offset=pos, save_exc=save_exc, - is_invalidate=is_guard_not_invalidated, + is_guard_not_invalidated=is_guard_not_invalidated, + is_guard_not_forced=is_guard_not_forced, fcond=fcond)) return c.AL @@ -312,49 +317,10 @@ return fcond def emit_op_finish(self, op, arglocs, regalloc, fcond): - for i in range(len(arglocs) - 1): - loc = arglocs[i] - box = op.getarg(i) - if loc is None: - continue - if loc.is_reg(): - if box.type == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(i) - elif box.type == INT: - adr = self.fail_boxes_int.get_addr_for_num(i) - else: - assert 0 - self.mc.gen_load_int(r.ip.value, adr) - self.mc.STR_ri(loc.value, r.ip.value) - elif loc.is_vfp_reg(): - assert box.type == FLOAT - adr = self.fail_boxes_float.get_addr_for_num(i) - self.mc.gen_load_int(r.ip.value, adr) - self.mc.VSTR(loc.value, r.ip.value) - elif loc.is_stack() or loc.is_imm() or loc.is_imm_float(): - if box.type == FLOAT: - adr = self.fail_boxes_float.get_addr_for_num(i) - self.mov_loc_loc(loc, r.vfp_ip) - self.mc.gen_load_int(r.ip.value, adr) - self.mc.VSTR(r.vfp_ip.value, r.ip.value) - elif box.type == REF or box.type == INT: - if box.type == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(i) - elif box.type == INT: - adr = self.fail_boxes_int.get_addr_for_num(i) - else: - assert 0 - self.mov_loc_loc(loc, r.ip) - self.mc.gen_load_int(r.lr.value, adr) - self.mc.STR_ri(r.ip.value, r.lr.value) - else: - assert 0 - # note: no exception should currently be set in llop.get_exception_addr - # even if this finish may be an exit_frame_with_exception (in this case - # the exception instance is in arglocs[0]). - addr = self.cpu.get_on_leave_jitted_int(save_exception=False) - self.mc.BL(addr) - self.mc.gen_load_int(r.r0.value, arglocs[-1].value) + [argloc] = arglocs + if argloc is not r.r0: #XXX verify this + self.mov_loc_loc(argloc, r.r0, fcond) + # exit function self.gen_func_epilog() return fcond @@ -988,8 +954,8 @@ def _emit_copystrcontent(self, op, regalloc, fcond, is_unicode): # compute the source address args = op.getarglist() - base_loc = regalloc._ensure_value_is_boxed(args[0], args) - ofs_loc = regalloc._ensure_value_is_boxed(args[2], args) + base_loc = regalloc.make_sure_var_in_reg(args[0], args) + ofs_loc = regalloc.make_sure_var_in_reg(args[2], args) assert args[0] is not args[1] # forbidden case of aliasing regalloc.possibly_free_var(args[0]) regalloc.free_temp_vars() @@ -1009,8 +975,8 @@ dstaddr_loc = regalloc.force_allocate_reg(dstaddr_box, selected_reg=r.r0) forbidden_vars.append(dstaddr_box) - base_loc = regalloc._ensure_value_is_boxed(args[1], forbidden_vars) - ofs_loc = regalloc._ensure_value_is_boxed(args[3], forbidden_vars) + base_loc = regalloc.make_sure_var_in_reg(args[1], forbidden_vars) + ofs_loc = regalloc.make_sure_var_in_reg(args[3], forbidden_vars) assert base_loc.is_reg() assert ofs_loc.is_reg() regalloc.possibly_free_var(args[1]) @@ -1026,7 +992,7 @@ # need the box here if isinstance(args[4], Box): length_box = args[4] - length_loc = regalloc._ensure_value_is_boxed(args[4], + length_loc = regalloc.make_sure_var_in_reg(args[4], forbidden_vars) else: length_box = TempInt() @@ -1152,8 +1118,26 @@ value = self.cpu.done_with_this_frame_float_v else: raise AssertionError(kind) - self.mc.gen_load_int(r.ip.value, value) - self.mc.CMP_rr(tmploc.value, r.ip.value) + from pypy.jit.backend.llsupport.descr import unpack_fielddescr + from pypy.jit.backend.llsupport.descr import unpack_interiorfielddescr + descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) + _offset, _size, _ = unpack_fielddescr(descrs.jf_descr) + fail_descr = self.cpu.get_fail_descr_from_number(value) + value = fail_descr.hide(self.cpu) + rgc._make_sure_does_not_move(value) + value = rffi.cast(lltype.Signed, value) + + if check_imm_arg(_offset): + self.mc.LDR_ri(r.ip.value, tmploc.value, imm=_offset) + else: + self.mc.gen_load_int(r.ip.value, _offset) + self.mc.LDR_rr(r.ip.value, tmploc.value, r.ip.value) + if check_imm_arg(value): + self.mc.CMP_ri(r.ip.value, imm=value) + else: + self.mc.gen_load_int(r.lr.value, value) + self.mc.CMP_rr(r.lr.value, r.ip.value) + #if values are equal we take the fast path # Slow path, calling helper @@ -1180,19 +1164,31 @@ if op.result is not None: # load the return value from fail_boxes_xxx[0] kind = op.result.type - if kind == INT: - adr = self.fail_boxes_int.get_addr_for_num(0) - elif kind == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(0) - elif kind == FLOAT: - adr = self.fail_boxes_float.get_addr_for_num(0) + if kind == FLOAT: + t = unpack_interiorfielddescr(descrs.as_float)[0] + if not check_imm_arg(t): + self.mc.gen_load_int(r.ip.value, t, cond=fast_path_cond) + self.mc.ADD_rr(r.ip.value, r.r0.value, r.ip.value, + cond=fast_path_cond) + t = 0 + base = r.ip + else: + base = r.r0 + self.mc.VLDR(resloc.value, base.value, imm=t, + cond=fast_path_cond) else: - raise AssertionError(kind) - self.mc.gen_load_int(r.ip.value, adr, cond=fast_path_cond) - if op.result.type == FLOAT: - self.mc.VLDR(resloc.value, r.ip.value, cond=fast_path_cond) - else: - self.mc.LDR_ri(resloc.value, r.ip.value, cond=fast_path_cond) + assert resloc is r.r0 + if kind == INT: + t = unpack_interiorfielddescr(descrs.as_int)[0] + else: + t = unpack_interiorfielddescr(descrs.as_ref)[0] + if not check_imm_arg(t): + self.mc.gen_load_int(r.ip.value, t, cond=fast_path_cond) + self.mc.LDR_rr(resloc.value, resloc.value, r.ip.value, + cond=fast_path_cond) + else: + self.mc.LDR_ri(resloc.value, resloc.value, imm=t, + cond=fast_path_cond) # jump to merge point jmp_pos = self.mc.currpos() self.mc.BKPT() @@ -1266,7 +1262,8 @@ self.mc.LDR_ri(r.ip.value, r.fp.value) self.mc.CMP_ri(r.ip.value, 0) - self._emit_guard(guard_op, arglocs[1 + numargs:], c.GE, save_exc=True) + self._emit_guard(guard_op, arglocs[1 + numargs:], c.GE, + save_exc=True, is_guard_not_forced=True) return fcond def emit_guard_call_release_gil(self, op, guard_op, arglocs, regalloc, diff --git a/pypy/jit/backend/arm/regalloc.py b/pypy/jit/backend/arm/regalloc.py --- a/pypy/jit/backend/arm/regalloc.py +++ b/pypy/jit/backend/arm/regalloc.py @@ -84,8 +84,28 @@ def void(self, op, fcond): return [] +class ARMRegisterManager(RegisterManager): + def return_constant(self, v, forbidden_vars=[], selected_reg=None): + self._check_type(v) + if isinstance(v, Const): + if isinstance(v, ConstPtr): + tp = REF + elif isinstance(v, ConstFloat): + tp = FLOAT + else: + tp = INT + loc = self.get_scratch_reg(tp, + self.temp_boxes + forbidden_vars, + selected_reg=selected_reg) + immvalue = self.convert_to_imm(v) + self.assembler.load(loc, immvalue) + return loc + else: + return RegisterManager.return_constant(self, v, + forbidden_vars, selected_reg) -class VFPRegisterManager(RegisterManager): + +class VFPRegisterManager(ARMRegisterManager): all_regs = r.all_vfp_regs box_types = [FLOAT] save_around_call_regs = r.all_vfp_regs @@ -107,20 +127,7 @@ reg = self.force_allocate_reg(v, selected_reg=r.d0) return reg - def ensure_value_is_boxed(self, thing, forbidden_vars=[]): - loc = None - if isinstance(thing, Const): - assert isinstance(thing, ConstFloat) - loc = self.get_scratch_reg(FLOAT, self.temp_boxes + forbidden_vars) - immvalue = self.convert_to_imm(thing) - self.assembler.load(loc, immvalue) - else: - loc = self.make_sure_var_in_reg(thing, - forbidden_vars=self.temp_boxes + forbidden_vars) - return loc - - def get_scratch_reg(self, type=FLOAT, forbidden_vars=[], - selected_reg=None): + def get_scratch_reg(self, type=FLOAT, forbidden_vars=[], selected_reg=None): assert type == FLOAT # for now box = TempFloat() self.temp_boxes.append(box) @@ -129,7 +136,7 @@ return reg -class ARMv7RegisterManager(RegisterManager): +class CoreRegisterManager(ARMRegisterManager): all_regs = r.all_regs box_types = None # or a list of acceptable types no_lower_byte_regs = all_regs @@ -162,22 +169,6 @@ return locations.ImmLocation(rffi.cast(lltype.Signed, c.value)) assert 0 - def ensure_value_is_boxed(self, thing, forbidden_vars=None): - loc = None - if isinstance(thing, Const): - if isinstance(thing, ConstPtr): - tp = REF - else: - tp = INT - loc = self.get_scratch_reg(tp, forbidden_vars=self.temp_boxes - + forbidden_vars) - immvalue = self.convert_to_imm(thing) - self.assembler.load(loc, immvalue) - else: - loc = self.make_sure_var_in_reg(thing, - forbidden_vars=self.temp_boxes + forbidden_vars) - return loc - def get_scratch_reg(self, type=INT, forbidden_vars=[], selected_reg=None): assert type == INT or type == REF box = TempBox() @@ -277,7 +268,12 @@ def make_sure_var_in_reg(self, var, forbidden_vars=[], selected_reg=None, need_lower_byte=False): - assert 0, 'should not be called directly' + if var.type == FLOAT: + return self.vfprm.make_sure_var_in_reg(var, forbidden_vars, + selected_reg, need_lower_byte) + else: + return self.rm.make_sure_var_in_reg(var, forbidden_vars, + selected_reg, need_lower_byte) def convert_to_imm(self, value): if isinstance(value, ConstInt): @@ -294,7 +290,7 @@ fm = self.frame_manager asm = self.assembler self.vfprm = VFPRegisterManager(longevity, fm, asm) - self.rm = ARMv7RegisterManager(longevity, fm, asm) + self.rm = CoreRegisterManager(longevity, fm, asm) def prepare_loop(self, inputargs, operations): self._prepare(inputargs, operations) @@ -426,12 +422,6 @@ self.rm.before_call(force_store, save_all_regs) self.vfprm.before_call(force_store, save_all_regs) - def _ensure_value_is_boxed(self, thing, forbidden_vars=[]): - if thing.type == FLOAT: - return self.vfprm.ensure_value_is_boxed(thing, forbidden_vars) - else: - return self.rm.ensure_value_is_boxed(thing, forbidden_vars) - def _sync_var(self, v): if v.type == FLOAT: self.vfprm._sync_var(v) @@ -444,14 +434,14 @@ imm_a0 = check_imm_box(a0) imm_a1 = check_imm_box(a1) if not imm_a0 and imm_a1: - l0 = self._ensure_value_is_boxed(a0, boxes) + l0 = self.make_sure_var_in_reg(a0, boxes) l1 = self.convert_to_imm(a1) elif imm_a0 and not imm_a1: l0 = self.convert_to_imm(a0) - l1 = self._ensure_value_is_boxed(a1, boxes) + l1 = self.make_sure_var_in_reg(a1, boxes) else: - l0 = self._ensure_value_is_boxed(a0, boxes) - l1 = self._ensure_value_is_boxed(a1, boxes) + l0 = self.make_sure_var_in_reg(a0, boxes) + l1 = self.make_sure_var_in_reg(a1, boxes) return [l0, l1] def prepare_op_int_add(self, op, fcond): @@ -466,14 +456,14 @@ imm_a0 = check_imm_box(a0) imm_a1 = check_imm_box(a1) if not imm_a0 and imm_a1: - l0 = self._ensure_value_is_boxed(a0, boxes) + l0 = self.make_sure_var_in_reg(a0, boxes) l1 = self.convert_to_imm(a1) elif imm_a0 and not imm_a1: l0 = self.convert_to_imm(a0) - l1 = self._ensure_value_is_boxed(a1, boxes) + l1 = self.make_sure_var_in_reg(a1, boxes) else: - l0 = self._ensure_value_is_boxed(a0, boxes) - l1 = self._ensure_value_is_boxed(a1, boxes) + l0 = self.make_sure_var_in_reg(a0, boxes) + l1 = self.make_sure_var_in_reg(a1, boxes) return [l0, l1] def prepare_op_int_sub(self, op, fcond): @@ -487,8 +477,8 @@ boxes = op.getarglist() a0, a1 = boxes - reg1 = self._ensure_value_is_boxed(a0, forbidden_vars=boxes) - reg2 = self._ensure_value_is_boxed(a1, forbidden_vars=boxes) + reg1 = self.make_sure_var_in_reg(a0, forbidden_vars=boxes) + reg2 = self.make_sure_var_in_reg(a1, forbidden_vars=boxes) self.possibly_free_vars(boxes) self.possibly_free_vars_for_op(op) @@ -497,14 +487,14 @@ return [reg1, reg2, res] def prepare_op_int_force_ge_zero(self, op, fcond): - argloc = self._ensure_value_is_boxed(op.getarg(0)) + argloc = self.make_sure_var_in_reg(op.getarg(0)) resloc = self.force_allocate_reg(op.result, [op.getarg(0)]) return [argloc, resloc] def prepare_guard_int_mul_ovf(self, op, guard, fcond): boxes = op.getarglist() - reg1 = self._ensure_value_is_boxed(boxes[0], forbidden_vars=boxes) - reg2 = self._ensure_value_is_boxed(boxes[1], forbidden_vars=boxes) + reg1 = self.make_sure_var_in_reg(boxes[0], forbidden_vars=boxes) + reg2 = self.make_sure_var_in_reg(boxes[1], forbidden_vars=boxes) res = self.force_allocate_reg(op.result) return self._prepare_guard(guard, [reg1, reg2, res]) @@ -576,7 +566,7 @@ prepare_guard_int_is_zero = prepare_op_unary_cmp('int_is_zero') def prepare_op_int_neg(self, op, fcond): - l0 = self._ensure_value_is_boxed(op.getarg(0)) + l0 = self.make_sure_var_in_reg(op.getarg(0)) self.possibly_free_vars_for_op(op) self.free_temp_vars() resloc = self.force_allocate_reg(op.result) @@ -629,15 +619,15 @@ def _prepare_llong_binop_xx(self, op, fcond): # arg 0 is the address of the function - loc0 = self._ensure_value_is_boxed(op.getarg(1)) - loc1 = self._ensure_value_is_boxed(op.getarg(2)) + loc0 = self.make_sure_var_in_reg(op.getarg(1)) + loc1 = self.make_sure_var_in_reg(op.getarg(2)) self.possibly_free_vars_for_op(op) self.free_temp_vars() res = self.vfprm.force_allocate_reg(op.result) return [loc0, loc1, res] def _prepare_llong_to_int(self, op, fcond): - loc0 = self._ensure_value_is_boxed(op.getarg(1)) + loc0 = self.make_sure_var_in_reg(op.getarg(1)) res = self.force_allocate_reg(op.result) return [loc0, res] @@ -654,18 +644,12 @@ return args def prepare_op_finish(self, op, fcond): - args = [None] * (op.numargs() + 1) - for i in range(op.numargs()): - arg = op.getarg(i) - if arg: - args[i] = self.loc(arg) - self.possibly_free_var(arg) - n = self.cpu.get_fail_descr_number(op.getdescr()) - args[-1] = imm(n) - return args + loc = self.loc(op.getarg(0)) + self.possibly_free_var(op.getarg(0)) + return [loc] def prepare_op_guard_true(self, op, fcond): - l0 = self._ensure_value_is_boxed(op.getarg(0)) + l0 = self.make_sure_var_in_reg(op.getarg(0)) args = self._prepare_guard(op, [l0]) return args @@ -677,9 +661,9 @@ boxes = op.getarglist() a0, a1 = boxes imm_a1 = check_imm_box(a1) - l0 = self._ensure_value_is_boxed(a0, boxes) + l0 = self.make_sure_var_in_reg(a0, boxes) if not imm_a1: - l1 = self._ensure_value_is_boxed(a1, boxes) + l1 = self.make_sure_var_in_reg(a1, boxes) else: l1 = self.convert_to_imm(a1) assert op.result is None @@ -699,7 +683,7 @@ def prepare_op_guard_exception(self, op, fcond): boxes = op.getarglist() arg0 = ConstInt(rffi.cast(lltype.Signed, op.getarg(0).getint())) - loc = self._ensure_value_is_boxed(arg0) + loc = self.make_sure_var_in_reg(arg0) loc1 = self.get_scratch_reg(INT, boxes) if op.result in self.longevity: resloc = self.force_allocate_reg(op.result, boxes) @@ -713,7 +697,7 @@ return arglocs def prepare_op_guard_no_exception(self, op, fcond): - loc = self._ensure_value_is_boxed( + loc = self.make_sure_var_in_reg( ConstInt(self.cpu.pos_exception())) arglocs = self._prepare_guard(op, [loc]) return arglocs @@ -727,7 +711,7 @@ assert isinstance(op.getarg(0), Box) boxes = op.getarglist() - x = self._ensure_value_is_boxed(boxes[0], boxes) + x = self.make_sure_var_in_reg(boxes[0], boxes) y_val = rffi.cast(lltype.Signed, op.getarg(1).getint()) arglocs = [x, None, None] @@ -837,8 +821,8 @@ boxes = op.getarglist() a0, a1 = boxes ofs, size, sign = unpack_fielddescr(op.getdescr()) - base_loc = self._ensure_value_is_boxed(a0, boxes) - value_loc = self._ensure_value_is_boxed(a1, boxes) + base_loc = self.make_sure_var_in_reg(a0, boxes) + value_loc = self.make_sure_var_in_reg(a1, boxes) if check_imm_arg(ofs): ofs_loc = imm(ofs) else: @@ -851,7 +835,7 @@ def prepare_op_getfield_gc(self, op, fcond): a0 = op.getarg(0) ofs, size, sign = unpack_fielddescr(op.getdescr()) - base_loc = self._ensure_value_is_boxed(a0) + base_loc = self.make_sure_var_in_reg(a0) immofs = imm(ofs) if check_imm_arg(ofs): ofs_loc = immofs @@ -871,8 +855,8 @@ t = unpack_interiorfielddescr(op.getdescr()) ofs, itemsize, fieldsize, sign = t args = op.getarglist() - base_loc = self._ensure_value_is_boxed(op.getarg(0), args) - index_loc = self._ensure_value_is_boxed(op.getarg(1), args) + base_loc = self.make_sure_var_in_reg(op.getarg(0), args) + index_loc = self.make_sure_var_in_reg(op.getarg(1), args) immofs = imm(ofs) if check_imm_arg(ofs): ofs_loc = immofs @@ -889,9 +873,9 @@ t = unpack_interiorfielddescr(op.getdescr()) ofs, itemsize, fieldsize, sign = t args = op.getarglist() - base_loc = self._ensure_value_is_boxed(op.getarg(0), args) - index_loc = self._ensure_value_is_boxed(op.getarg(1), args) - value_loc = self._ensure_value_is_boxed(op.getarg(2), args) + base_loc = self.make_sure_var_in_reg(op.getarg(0), args) + index_loc = self.make_sure_var_in_reg(op.getarg(1), args) + value_loc = self.make_sure_var_in_reg(op.getarg(2), args) immofs = imm(ofs) if check_imm_arg(ofs): ofs_loc = immofs @@ -907,7 +891,7 @@ assert isinstance(arraydescr, ArrayDescr) ofs = arraydescr.lendescr.offset arg = op.getarg(0) - base_loc = self._ensure_value_is_boxed(arg) + base_loc = self.make_sure_var_in_reg(arg) self.possibly_free_vars_for_op(op) self.free_temp_vars() res = self.force_allocate_reg(op.result) @@ -917,9 +901,9 @@ size, ofs, _ = unpack_arraydescr(op.getdescr()) scale = get_scale(size) args = op.getarglist() - base_loc = self._ensure_value_is_boxed(args[0], args) - ofs_loc = self._ensure_value_is_boxed(args[1], args) - value_loc = self._ensure_value_is_boxed(args[2], args) + base_loc = self.make_sure_var_in_reg(args[0], args) + ofs_loc = self.make_sure_var_in_reg(args[1], args) + value_loc = self.make_sure_var_in_reg(args[2], args) assert check_imm_arg(ofs) return [value_loc, base_loc, ofs_loc, imm(scale), imm(ofs)] prepare_op_setarrayitem_raw = prepare_op_setarrayitem_gc @@ -929,8 +913,8 @@ boxes = op.getarglist() size, ofs, _ = unpack_arraydescr(op.getdescr()) scale = get_scale(size) - base_loc = self._ensure_value_is_boxed(boxes[0], boxes) - ofs_loc = self._ensure_value_is_boxed(boxes[1], boxes) + base_loc = self.make_sure_var_in_reg(boxes[0], boxes) + ofs_loc = self.make_sure_var_in_reg(boxes[1], boxes) self.possibly_free_vars_for_op(op) self.free_temp_vars() res = self.force_allocate_reg(op.result) @@ -944,7 +928,7 @@ def prepare_op_strlen(self, op, fcond): args = op.getarglist() - l0 = self._ensure_value_is_boxed(op.getarg(0)) + l0 = self.make_sure_var_in_reg(op.getarg(0)) basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, self.cpu.translate_support_code) immofs = imm(ofs_length) @@ -963,14 +947,14 @@ def prepare_op_strgetitem(self, op, fcond): boxes = op.getarglist() - base_loc = self._ensure_value_is_boxed(boxes[0]) + base_loc = self.make_sure_var_in_reg(boxes[0]) a1 = boxes[1] imm_a1 = check_imm_box(a1) if imm_a1: ofs_loc = self.convert_to_imm(a1) else: - ofs_loc = self._ensure_value_is_boxed(a1, boxes) + ofs_loc = self.make_sure_var_in_reg(a1, boxes) self.possibly_free_vars_for_op(op) self.free_temp_vars() @@ -983,9 +967,9 @@ def prepare_op_strsetitem(self, op, fcond): boxes = op.getarglist() - base_loc = self._ensure_value_is_boxed(boxes[0], boxes) - ofs_loc = self._ensure_value_is_boxed(boxes[1], boxes) - value_loc = self._ensure_value_is_boxed(boxes[2], boxes) + base_loc = self.make_sure_var_in_reg(boxes[0], boxes) + ofs_loc = self.make_sure_var_in_reg(boxes[1], boxes) + value_loc = self.make_sure_var_in_reg(boxes[2], boxes) basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.STR, self.cpu.translate_support_code) assert itemsize == 1 @@ -995,7 +979,7 @@ prepare_op_copyunicodecontent = void def prepare_op_unicodelen(self, op, fcond): - l0 = self._ensure_value_is_boxed(op.getarg(0)) + l0 = self.make_sure_var_in_reg(op.getarg(0)) basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, self.cpu.translate_support_code) immofs = imm(ofs_length) @@ -1012,8 +996,8 @@ def prepare_op_unicodegetitem(self, op, fcond): boxes = op.getarglist() - base_loc = self._ensure_value_is_boxed(boxes[0], boxes) - ofs_loc = self._ensure_value_is_boxed(boxes[1], boxes) + base_loc = self.make_sure_var_in_reg(boxes[0], boxes) + ofs_loc = self.make_sure_var_in_reg(boxes[1], boxes) self.possibly_free_vars_for_op(op) self.free_temp_vars() @@ -1027,9 +1011,9 @@ def prepare_op_unicodesetitem(self, op, fcond): boxes = op.getarglist() - base_loc = self._ensure_value_is_boxed(boxes[0], boxes) - ofs_loc = self._ensure_value_is_boxed(boxes[1], boxes) - value_loc = self._ensure_value_is_boxed(boxes[2], boxes) + base_loc = self.make_sure_var_in_reg(boxes[0], boxes) + ofs_loc = self.make_sure_var_in_reg(boxes[1], boxes) + value_loc = self.make_sure_var_in_reg(boxes[2], boxes) basesize, itemsize, ofs_length = symbolic.get_array_token(rstr.UNICODE, self.cpu.translate_support_code) scale = itemsize / 2 @@ -1042,7 +1026,7 @@ if imm_arg: argloc = self.convert_to_imm(arg) else: - argloc = self._ensure_value_is_boxed(arg) + argloc = self.make_sure_var_in_reg(arg) self.possibly_free_vars_for_op(op) self.free_temp_vars() resloc = self.force_allocate_reg(op.result) @@ -1093,7 +1077,7 @@ # twice from the memory. N = op.numargs() args = op.getarglist() - arglocs = [self._ensure_value_is_boxed(op.getarg(i), args) + arglocs = [self.make_sure_var_in_reg(op.getarg(i), args) for i in range(N)] tmp = self.get_scratch_reg(INT, args) assert tmp not in arglocs @@ -1215,7 +1199,7 @@ float_result=False, name='prepare_guard_float_ge') def prepare_op_math_sqrt(self, op, fcond): - loc = self._ensure_value_is_boxed(op.getarg(1)) + loc = self.make_sure_var_in_reg(op.getarg(1)) self.possibly_free_vars_for_op(op) self.free_temp_vars() res = self.vfprm.force_allocate_reg(op.result) @@ -1223,12 +1207,12 @@ return [loc, res] def prepare_op_cast_float_to_int(self, op, fcond): - loc1 = self._ensure_value_is_boxed(op.getarg(0)) + loc1 = self.make_sure_var_in_reg(op.getarg(0)) res = self.rm.force_allocate_reg(op.result) return [loc1, res] def prepare_op_cast_int_to_float(self, op, fcond): - loc1 = self._ensure_value_is_boxed(op.getarg(0)) + loc1 = self.make_sure_var_in_reg(op.getarg(0)) res = self.vfprm.force_allocate_reg(op.result) return [loc1, res] @@ -1247,12 +1231,12 @@ return [loc, res] def prepare_op_cast_float_to_singlefloat(self, op, fcond): - loc1 = self._ensure_value_is_boxed(op.getarg(0)) + loc1 = self.make_sure_var_in_reg(op.getarg(0)) res = self.force_allocate_reg(op.result) return [loc1, res] - + def prepare_op_cast_singlefloat_to_float(self, op, fcond): - loc1 = self._ensure_value_is_boxed(op.getarg(0)) + loc1 = self.make_sure_var_in_reg(op.getarg(0)) res = self.force_allocate_reg(op.result) return [loc1, res] diff --git a/pypy/jit/backend/arm/runner.py b/pypy/jit/backend/arm/runner.py --- a/pypy/jit/backend/arm/runner.py +++ b/pypy/jit/backend/arm/runner.py @@ -23,15 +23,21 @@ AbstractLLCPU.__init__(self, rtyper, stats, opts, translate_support_code, gcdescr) + from pypy.jit.backend.llsupport import jitframe + self.deadframe_size_max = llmemory.sizeof(jitframe.DEADFRAME, + self.get_failargs_limit()) + def set_debug(self, flag): return self.assembler.set_debug(flag) + def get_failargs_limit(self): + if self.opts is not None: + return self.opts.failargs_limit + else: + return 1000 + def setup(self): - if self.opts is not None: - failargs_limit = self.opts.failargs_limit - else: - failargs_limit = 1000 - self.assembler = AssemblerARM(self, failargs_limit=failargs_limit) + self.assembler = AssemblerARM(self, self.translate_support_code) def setup_once(self): self.assembler.setup_once() @@ -51,24 +57,6 @@ return self.assembler.assemble_bridge(faildescr, inputargs, operations, original_loop_token, log=log) - def get_latest_value_float(self, index): - return self.assembler.fail_boxes_float.getitem(index) - - def get_latest_value_int(self, index): - return self.assembler.fail_boxes_int.getitem(index) - - def get_latest_value_ref(self, index): - return self.assembler.fail_boxes_ptr.getitem(index) - - def get_latest_value_count(self): - return self.assembler.fail_boxes_count - - def get_latest_force_token(self): - return self.assembler.fail_force_index - - def get_on_leave_jitted_hook(self): - return self.assembler.leave_jitted_hook - def clear_latest_values(self, count): setitem = self.assembler.fail_boxes_ptr.setitem null = lltype.nullptr(llmemory.GCREF.TO) @@ -76,7 +64,7 @@ setitem(index, null) def make_execute_token(self, *ARGS): - FUNCPTR = lltype.Ptr(lltype.FuncType(ARGS, lltype.Signed)) + FUNCPTR = lltype.Ptr(lltype.FuncType(ARGS, llmemory.GCREF)) def execute_token(executable_token, *args): clt = executable_token.compiled_loop_token @@ -91,12 +79,13 @@ prev_interpreter = LLInterpreter.current_interpreter LLInterpreter.current_interpreter = self.debug_ll_interpreter try: - fail_index = func(*args) + deadframe = func(*args) finally: if not self.translate_support_code: LLInterpreter.current_interpreter = prev_interpreter #llop.debug_print(lltype.Void, "<<<< Back") - return self.get_fail_descr_from_number(fail_index) + self.gc_set_extra_threshold() + return deadframe return execute_token def cast_ptr_to_int(x): @@ -115,17 +104,22 @@ assert fail_index >= 0, "already forced!" faildescr = self.get_fail_descr_from_number(fail_index) rffi.cast(TP, addr_of_force_index)[0] = ~fail_index - bytecode = self.assembler._find_failure_recovery_bytecode(faildescr) + frb = self.assembler._find_failure_recovery_bytecode(faildescr) + bytecode = rffi.cast(rffi.UCHARP, frb) addr_all_null_regsiters = rffi.cast(rffi.LONG, self.all_null_registers) - # start of "no gc operation!" block - fail_index_2 = self.assembler.failure_recovery_func( - bytecode, - addr_of_force_index, - addr_all_null_regsiters) - self.assembler.leave_jitted_hook() - # end of "no gc operation!" block - assert fail_index == fail_index_2 - return faildescr + # + assert (rffi.cast(lltype.Signed, bytecode[0]) == + self.assembler.CODE_FORCED) + bytecode = rffi.ptradd(bytecode, 1) + deadframe = self.assembler.grab_frame_values(self, + bytecode, addr_of_force_index, + self.all_null_registers, + self.all_null_registers) + # + assert self.get_latest_descr(deadframe) is faildescr + self.assembler.force_token_to_dead_frame[addr_of_force_index] = ( + deadframe) + return deadframe def redirect_call_assembler(self, oldlooptoken, newlooptoken): self.assembler.redirect_call_assembler(oldlooptoken, newlooptoken) diff --git a/pypy/jit/backend/arm/test/test_gc_integration.py b/pypy/jit/backend/arm/test/test_gc_integration.py --- a/pypy/jit/backend/arm/test/test_gc_integration.py +++ b/pypy/jit/backend/arm/test/test_gc_integration.py @@ -2,25 +2,13 @@ """ Tests for register allocation for common constructs """ -import py -from pypy.jit.metainterp.history import BoxInt, \ - BoxPtr, TreeLoop, TargetToken -from pypy.jit.metainterp.resoperation import rop, ResOperation -from pypy.jit.codewriter import heaptracker -from pypy.jit.backend.llsupport.descr import GcCache -from pypy.jit.backend.llsupport.gc import GcLLDescription +from pypy.jit.metainterp.history import TargetToken +from pypy.jit.backend.llsupport.gc import GcLLDescription, GcLLDescr_boehm from pypy.jit.backend.detect_cpu import getcpuclass from pypy.jit.backend.arm.arch import WORD from pypy.rpython.lltypesystem import lltype, llmemory, rffi -from pypy.rpython.annlowlevel import llhelper -from pypy.rpython.lltypesystem import rclass -from pypy.jit.backend.llsupport.gc import GcLLDescr_framework -from pypy.jit.backend.arm.test.test_regalloc import MockAssembler from pypy.jit.backend.arm.test.test_regalloc import BaseTestRegalloc -from pypy.jit.backend.arm.regalloc import ARMFrameManager, VFPRegisterManager -from pypy.jit.codewriter.effectinfo import EffectInfo -from pypy.jit.backend.arm.regalloc import Regalloc, ARMv7RegisterManager CPU = getcpuclass() @@ -44,23 +32,14 @@ return ['compressed'] + shape[1:] -class MockGcDescr(GcCache): - get_malloc_slowpath_addr = None - write_barrier_descr = None - moving_gc = True +class MockGcDescr(GcLLDescr_boehm): gcrootmap = MockGcRootMap() - def initialize(self): - pass - - _record_constptrs = GcLLDescr_framework._record_constptrs.im_func - rewrite_assembler = GcLLDescr_framework.rewrite_assembler.im_func - class TestRegallocGcIntegration(BaseTestRegalloc): cpu = CPU(None, None) - cpu.gc_ll_descr = MockGcDescr(False) + cpu.gc_ll_descr = MockGcDescr(None, None, None) cpu.setup_once() S = lltype.GcForwardReference() diff --git a/pypy/jit/backend/arm/test/test_loop_unroll.py b/pypy/jit/backend/arm/test/test_loop_unroll.py --- a/pypy/jit/backend/arm/test/test_loop_unroll.py +++ b/pypy/jit/backend/arm/test/test_loop_unroll.py @@ -1,8 +1,8 @@ import py -from pypy.jit.backend.x86.test.test_basic import Jit386Mixin +from pypy.jit.backend.arm.test.support import JitARMMixin from pypy.jit.metainterp.test import test_loop_unroll -class TestLoopSpec(Jit386Mixin, test_loop_unroll.LoopUnrollTest): +class TestLoopSpec(JitARMMixin, test_loop_unroll.LoopUnrollTest): # for the individual tests see # ====> ../../../metainterp/test/test_loop.py pass diff --git a/pypy/jit/backend/arm/test/test_runner.py b/pypy/jit/backend/arm/test/test_runner.py --- a/pypy/jit/backend/arm/test/test_runner.py +++ b/pypy/jit/backend/arm/test/test_runner.py @@ -60,16 +60,18 @@ ] cpu.compile_loop(inp, operations, looptoken) args = [i for i in range(1, 15)] - self.cpu.execute_token(looptoken, *args) - output = [self.cpu.get_latest_value_int(i - 1) for i in range(1, 15)] + deadframe = self.cpu.execute_token(looptoken, *args) + output = [self.cpu.get_latest_value_int(deadframe, i - 1) for i in range(1, 15)] expected = [3, 7, 11, 15, 19, 23, 27, 3, 7, 11, 15, 19, 23, 27] assert output == expected - def test_redirect_call_assember2(self): - def assembler_helper(failindex, virtualizable): - return self.cpu.get_latest_value_int(0) + def test_redirect_call_assembler2(self): + def assembler_helper(deadframe, virtualizable): + x = self.cpu.get_latest_value_int(deadframe, 0) + assert x == 11 + return 7 - FUNCPTR = lltype.Ptr(lltype.FuncType([lltype.Signed, llmemory.GCREF], + FUNCPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF, llmemory.GCREF], lltype.Signed)) class FakeJitDriverSD: @@ -101,12 +103,12 @@ self.cpu.compile_loop(loop2.inputargs, loop2.operations, lt2) self.cpu.compile_loop(loop3.inputargs, loop3.operations, lt3) self.cpu.compile_loop(loop1.inputargs, loop1.operations, lt1) - self.cpu.execute_token(lt1, 11) - assert self.cpu.get_latest_value_int(0) == 12 + df = self.cpu.execute_token(lt1, 10) + assert self.cpu.get_latest_value_int(df, 0) == 7 self.cpu.redirect_call_assembler(lt2, lt3) - self.cpu.execute_token(lt1, 11) - assert self.cpu.get_latest_value_int(0) == 10 + df = self.cpu.execute_token(lt1, 12) + assert self.cpu.get_latest_value_int(df, 0) == 7 SFloat = lltype.GcForwardReference() SFloat.become(lltype.GcStruct('SFloat', ('parent', rclass.OBJECT), @@ -202,8 +204,8 @@ ARGS = [lltype.Signed] * numargs RES = lltype.Signed args = [i+1 for i in range(numargs)] - res = self.cpu.execute_token(looptoken, *args) - assert self.cpu.get_latest_value_int(0) == sum(args) + deadframe = self.cpu.execute_token(looptoken, *args) + assert self.cpu.get_latest_value_int(deadframe, 0) == sum(args) def test_debugger_on(self): from pypy.rlib import debug diff --git a/pypy/jit/backend/llgraph/llimpl.py b/pypy/jit/backend/llgraph/llimpl.py deleted file mode 100644 --- a/pypy/jit/backend/llgraph/llimpl.py +++ /dev/null @@ -1,1968 +0,0 @@ -""" -The non-RPythonic part of the llgraph backend. -This contains all the code that is directly run -when executing on top of the llinterpreter. -""" - -import weakref -from pypy.objspace.flow.model import Variable, Constant -from pypy.annotation import model as annmodel -from pypy.jit.metainterp.history import REF, INT, FLOAT -from pypy.jit.metainterp import history -from pypy.jit.codewriter import heaptracker -from pypy.rpython.lltypesystem import lltype, llmemory, rclass, rstr, rffi -from pypy.rpython.ootypesystem import ootype -from pypy.rpython.module.support import LLSupport, OOSupport -from pypy.rpython.llinterp import LLException -from pypy.rpython.extregistry import ExtRegistryEntry - -from pypy.jit.metainterp import resoperation -from pypy.jit.metainterp.resoperation import rop -from pypy.jit.backend.llgraph import symbolic -from pypy.jit.codewriter import longlong -from pypy.jit.codewriter.effectinfo import EffectInfo - -from pypy.rlib.objectmodel import ComputedIntSymbolic, we_are_translated -from pypy.rlib.rarithmetic import ovfcheck -from pypy.rlib.rarithmetic import r_longlong, r_ulonglong, r_uint -from pypy.rlib.rtimer import read_timestamp - -import py -from pypy.tool.ansi_print import ansi_log -log = py.log.Producer('runner') -py.log.setconsumer('runner', ansi_log) - -IS_32_BIT = r_ulonglong is not r_uint - - -def _from_opaque(opq): - return opq._obj.externalobj - -_TO_OPAQUE = {} - -def _to_opaque(value): - try: - return value._the_opaque_pointer - except AttributeError: - op = lltype.opaqueptr(_TO_OPAQUE[value.__class__], 'opaque', - externalobj=value) - value._the_opaque_pointer = op - return op - -def _normalize(value): - if isinstance(value, lltype._ptr): - value = lltype.top_container(value._obj) - return value - -def from_opaque_string(s): - if isinstance(s, str): - return s - elif isinstance(s, ootype._string): - return OOSupport.from_rstr(s) - else: - return LLSupport.from_rstr(s) - -FLOAT_ARRAY_TP = lltype.Ptr(lltype.Array(lltype.Float, hints={"nolength": True})) -def maybe_uncast(TP, array): - if array._TYPE.TO.OF != lltype.Float: - # array._TYPE.TO._hints.get("uncast_on_llgraph"): - array = rffi.cast(TP, array) - return array - -# a list of argtypes of all operations - couldn't find any and it's -# very useful. Note however that the table is half-broken here and -# there, in ways that are sometimes a bit hard to fix; that's why -# it is not "official". -TYPES = { - 'int_add' : (('int', 'int'), 'int'), - 'int_sub' : (('int', 'int'), 'int'), - 'int_mul' : (('int', 'int'), 'int'), - 'int_floordiv' : (('int', 'int'), 'int'), - 'int_mod' : (('int', 'int'), 'int'), - 'int_and' : (('int', 'int'), 'int'), - 'int_or' : (('int', 'int'), 'int'), - 'int_xor' : (('int', 'int'), 'int'), - 'int_lshift' : (('int', 'int'), 'int'), - 'int_rshift' : (('int', 'int'), 'int'), - 'int_lt' : (('int', 'int'), 'bool'), - 'int_gt' : (('int', 'int'), 'bool'), - 'int_ge' : (('int', 'int'), 'bool'), - 'int_le' : (('int', 'int'), 'bool'), - 'int_eq' : (('int', 'int'), 'bool'), - 'int_ne' : (('int', 'int'), 'bool'), - 'int_is_true' : (('int',), 'bool'), - 'int_is_zero' : (('int',), 'bool'), - 'int_neg' : (('int',), 'int'), - 'int_invert' : (('int',), 'int'), - 'int_add_ovf' : (('int', 'int'), 'int'), - 'int_sub_ovf' : (('int', 'int'), 'int'), - 'int_mul_ovf' : (('int', 'int'), 'int'), - 'int_force_ge_zero':(('int',), 'int'), - 'uint_add' : (('int', 'int'), 'int'), - 'uint_sub' : (('int', 'int'), 'int'), - 'uint_mul' : (('int', 'int'), 'int'), - 'uint_lt' : (('int', 'int'), 'bool'), - 'uint_le' : (('int', 'int'), 'bool'), - 'uint_eq' : (('int', 'int'), 'bool'), - 'uint_ne' : (('int', 'int'), 'bool'), - 'uint_gt' : (('int', 'int'), 'bool'), - 'uint_ge' : (('int', 'int'), 'bool'), - 'uint_xor' : (('int', 'int'), 'int'), - 'uint_rshift' : (('int', 'int'), 'int'), - 'uint_floordiv' : (('int', 'int'), 'int'), - 'float_add' : (('float', 'float'), 'float'), - 'float_sub' : (('float', 'float'), 'float'), - 'float_mul' : (('float', 'float'), 'float'), - 'float_truediv' : (('float', 'float'), 'float'), - 'float_lt' : (('float', 'float'), 'bool'), - 'float_le' : (('float', 'float'), 'bool'), - 'float_eq' : (('float', 'float'), 'bool'), - 'float_ne' : (('float', 'float'), 'bool'), - 'float_gt' : (('float', 'float'), 'bool'), - 'float_ge' : (('float', 'float'), 'bool'), - 'float_neg' : (('float',), 'float'), - 'float_abs' : (('float',), 'float'), - 'cast_float_to_int':(('float',), 'int'), - 'cast_int_to_float':(('int',), 'float'), - 'same_as' : (('int',), 'int'), # could also be ptr=>ptr - 'new_with_vtable' : (('ref',), 'ref'), - 'new' : ((), 'ref'), - 'new_array' : (('int',), 'ref'), - 'oois' : (('ref', 'ref'), 'bool'), - 'ooisnot' : (('ref', 'ref'), 'bool'), - 'instanceof' : (('ref',), 'bool'), - 'subclassof' : (('ref', 'ref'), 'bool'), - 'runtimenew' : (('ref',), 'ref'), - 'setfield_gc' : (('ref', 'intorptr'), None), - 'getfield_gc' : (('ref',), 'intorptr'), - 'getfield_gc_pure': (('ref',), 'intorptr'), - 'setfield_raw' : (('ref', 'intorptr'), None), - 'getfield_raw' : (('ref',), 'intorptr'), - 'getfield_raw_pure': (('ref',), 'intorptr'), - 'setarrayitem_gc' : (('ref', 'int', 'intorptr'), None), - 'getarrayitem_gc' : (('ref', 'int'), 'intorptr'), - 'getarrayitem_gc_pure' : (('ref', 'int'), 'intorptr'), - 'setarrayitem_raw' : (('ref', 'int', 'intorptr'), None), - 'getarrayitem_raw' : (('ref', 'int'), 'intorptr'), - 'getarrayitem_raw_pure' : (('ref', 'int'), 'intorptr'), - 'arraylen_gc' : (('ref',), 'int'), - 'call' : (('ref', 'varargs'), 'intorptr'), - 'call_assembler' : (('varargs',), 'intorptr'), - 'cond_call_gc_wb' : (('ptr', 'ptr'), None), - 'cond_call_gc_wb_array': (('ptr', 'int', 'ptr'), None), - 'oosend' : (('varargs',), 'intorptr'), - 'oosend_pure' : (('varargs',), 'intorptr'), From noreply at buildbot.pypy.org Thu Jan 10 21:20:32 2013 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 10 Jan 2013 21:20:32 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: fix merge problems Message-ID: <20130110202032.9E1081C00E2@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: missing-ndarray-attributes Changeset: r59931:69e846f17fae Date: 2013-01-10 21:48 +0200 http://bitbucket.org/pypy/pypy/changeset/69e846f17fae/ Log: fix merge problems diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py --- a/pypy/module/micronumpy/arrayimpl/concrete.py +++ b/pypy/module/micronumpy/arrayimpl/concrete.py @@ -78,9 +78,9 @@ if self.dtype.is_complex_type(): dtype = self.dtype.float_type return SliceArray(self.start, strides, backstrides, - self.get_shape(), self, dtype=dtype) + self.get_shape(), self, self, dtype=dtype) return SliceArray(self.start, strides, backstrides, - self.get_shape(), self) + self.get_shape(), self, self) def get_imag(self): strides = self.get_strides() @@ -88,11 +88,11 @@ if self.dtype.is_complex_type(): dtype = self.dtype.float_type return SliceArray(self.start + dtype.get_size(), strides, - backstrides, self.get_shape(), self, dtype=dtype) + backstrides, self.get_shape(), self, self, dtype=dtype) if self.dtype.is_flexible_type(): # numpy returns self for self.imag return SliceArray(self.start, strides, backstrides, - self.get_shape(), self) + self.get_shape(), self, self) impl = NonWritableArray(self.get_shape(), self.dtype, self.order, strides, backstrides) impl.fill(self.dtype.box(0)) @@ -320,7 +320,7 @@ return None class NonWritableArray(ConcreteArray): - def descr_setitem(self, space, w_index, w_value): + def descr_setitem(self, space, orig_array, w_index, w_value): raise OperationError(space.w_RuntimeError, space.wrap( "array is not writable")) diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -2448,8 +2448,8 @@ assert a.shape == () def test_flexible_repr(self): - # import overrides str(), repr() for array - from _numpypy import array + # numpypy overrides _numpypy repr with pure python one + from numpypy import array a = array(['abc'],'S3') s = repr(a) # simplify test for \n in repr From noreply at buildbot.pypy.org Thu Jan 10 21:20:33 2013 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 10 Jan 2013 21:20:33 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: translation fixes Message-ID: <20130110202033.E46F71C00E2@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: missing-ndarray-attributes Changeset: r59932:99c58c500571 Date: 2013-01-10 22:20 +0200 http://bitbucket.org/pypy/pypy/changeset/99c58c500571/ Log: translation fixes diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -368,8 +368,8 @@ def str_format(self, box): return str(self.for_computation(self.unbox(box))) - @staticmethod - def for_computation(v): + #@staticmethod #v can be all kinds of int + def for_computation(self, v): return widen(v) def default_fromstring(self, space): @@ -634,8 +634,8 @@ return float2string(self.for_computation(self.unbox(box)), "g", rfloat.DTSF_STR_PRECISION) - @staticmethod - def for_computation(v): + #@staticmethod #v can be a longfloat + def for_computation(self, v): return float(v) def default_fromstring(self, space): From noreply at buildbot.pypy.org Thu Jan 10 21:34:49 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Thu, 10 Jan 2013 21:34:49 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed missing import in rsignla.py Message-ID: <20130110203449.5A0571C00E2@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59933:202f1cf5fcc0 Date: 2013-01-10 21:33 +0100 http://bitbucket.org/pypy/pypy/changeset/202f1cf5fcc0/ Log: Fixed missing import in rsignla.py diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py --- a/pypy/module/signal/interp_signal.py +++ b/pypy/module/signal/interp_signal.py @@ -11,7 +11,7 @@ import py import sys from rpython.rlib import jit, rposix -from rpython.rlib.rarithmetic import intmask, is_valid_int +from rpython.rlib.rarithmetic import intmask from rpython.rlib.rsignal import * diff --git a/rpython/rlib/rsignal.py b/rpython/rlib/rsignal.py --- a/rpython/rlib/rsignal.py +++ b/rpython/rlib/rsignal.py @@ -5,6 +5,7 @@ from rpython.rtyper.tool import rffi_platform from rpython.rtyper.lltypesystem import lltype, rffi from rpython.translator.tool.cbuild import ExternalCompilationInfo +from rpython.rlib.rarithmetic import is_valid_int def setup(): for key, value in cpy_signal.__dict__.items(): From noreply at buildbot.pypy.org Thu Jan 10 22:26:35 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Thu, 10 Jan 2013 22:26:35 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Moved need_sparse_files to test_extfunc Message-ID: <20130110212635.99ADF1C00E2@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59934:a71aef0e6309 Date: 2013-01-10 22:26 +0100 http://bitbucket.org/pypy/pypy/changeset/a71aef0e6309/ Log: Moved need_sparse_files to test_extfunc diff --git a/pypy/module/_file/test/test_large_file.py b/pypy/module/_file/test/test_large_file.py --- a/pypy/module/_file/test/test_large_file.py +++ b/pypy/module/_file/test/test_large_file.py @@ -12,7 +12,7 @@ def setup_method(self, meth): if getattr(meth, 'need_sparse_files', False): - from pypy.module.posix.test.test_posix2 import need_sparse_files + from rpython.translator.c.test.test_extfunc import need_sparse_files need_sparse_files() def test_large_seek_offsets(self): diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -7,6 +7,7 @@ from pypy.tool.pytest.objspace import gettestobjspace from pypy.conftest import pypydir from rpython.rtyper.module.ll_os import RegisterOs +from rpython.translator.c.test.test_extfunc import need_sparse_files import os import py import sys @@ -42,11 +43,7 @@ # Initialize sys.filesystemencoding # space.call_method(space.getbuiltinmodule('sys'), 'getfilesystemencoding') -def need_sparse_files(): - if sys.platform == 'darwin': - py.test.skip("no sparse files on default Mac OS X file system") - if os.name == 'nt': - py.test.skip("no sparse files on Windows") + GET_POSIX = "(): import %s as m ; return m" % os.name diff --git a/rpython/translator/c/test/test_extfunc.py b/rpython/translator/c/test/test_extfunc.py --- a/rpython/translator/c/test/test_extfunc.py +++ b/rpython/translator/c/test/test_extfunc.py @@ -101,10 +101,15 @@ f1() os.unlink(filename) +def need_sparse_files(): + if sys.platform == 'darwin': + py.test.skip("no sparse files on default Mac OS X file system") + if os.name == 'nt': + py.test.skip("no sparse files on Windows") + def test_largefile(): if not hasattr(os, 'ftruncate'): py.test.skip("this os has no ftruncate :-(") - from pypy.module.posix.test.test_posix2 import need_sparse_files need_sparse_files() filename = str(udir.join('test_largefile')) r4800000000 = r_longlong(4800000000L) From noreply at buildbot.pypy.org Thu Jan 10 23:02:26 2013 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 10 Jan 2013 23:02:26 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: progress towards translation Message-ID: <20130110220226.E6C621C0285@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: missing-ndarray-attributes Changeset: r59935:81224797e9f1 Date: 2013-01-11 00:02 +0200 http://bitbucket.org/pypy/pypy/changeset/81224797e9f1/ Log: progress towards translation diff --git a/pypy/module/micronumpy/arrayimpl/base.py b/pypy/module/micronumpy/arrayimpl/base.py --- a/pypy/module/micronumpy/arrayimpl/base.py +++ b/pypy/module/micronumpy/arrayimpl/base.py @@ -3,6 +3,12 @@ def is_scalar(self): return False + def base(self): + raise NotImplementedError + + def create_iter(self, shape=None): + raise NotImplementedError + class BaseArrayIterator(object): def next(self): raise NotImplementedError # purely abstract base class diff --git a/pypy/module/micronumpy/arrayimpl/scalar.py b/pypy/module/micronumpy/arrayimpl/scalar.py --- a/pypy/module/micronumpy/arrayimpl/scalar.py +++ b/pypy/module/micronumpy/arrayimpl/scalar.py @@ -86,7 +86,7 @@ def create_axis_iter(self, shape, dim, cum): raise Exception("axis iter should not happen on scalar") - def swapaxes(self, axis1, axis2): + def swapaxes(self, orig_array, axis1, axis2): raise Exception("should not be called") def fill(self, w_value): diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py --- a/pypy/module/micronumpy/base.py +++ b/pypy/module/micronumpy/base.py @@ -2,6 +2,7 @@ from pypy.interpreter.baseobjspace import Wrappable from pypy.tool.pairtype import extendabletype from pypy.module.micronumpy.support import calc_strides +from pypy.module.micronumpy.arrayimpl.base import BaseArrayImplementation def issequence_w(space, w_obj): return (space.isinstance_w(w_obj, space.w_tuple) or @@ -15,6 +16,7 @@ __metaclass__ = extendabletype def __init__(self, implementation): + assert isinstance(implementation, BaseArrayImplementation) self.implementation = implementation @staticmethod diff --git a/pypy/module/micronumpy/interp_flatiter.py b/pypy/module/micronumpy/interp_flatiter.py --- a/pypy/module/micronumpy/interp_flatiter.py +++ b/pypy/module/micronumpy/interp_flatiter.py @@ -9,6 +9,7 @@ like a real array for descr_eq and friends """ def __init__(self, base): + assert isinstance(base, BaseArrayImplementation) self.base = base self.dtype = base.get_dtype() self.shape = [base.get_size()] @@ -17,6 +18,7 @@ return self.shape def create_iter(self, shape=None): + assert isinstance(self.base, BaseArrayImplementation) return self.base.create_iter() class W_FlatIterator(W_NDimArray): diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -18,6 +18,7 @@ from pypy.tool.sourcetools import func_with_new_name from pypy.rlib import jit from pypy.rlib.rstring import StringBuilder +from pypy.module.micronumpy.arrayimpl.base import BaseArrayImplementation def _find_shape(space, w_size): if space.is_none(w_size): @@ -208,6 +209,7 @@ return s.build() def create_iter(self, shape=None): + assert isinstance(self.implementation, BaseArrayImplementation) return self.implementation.create_iter(shape) def create_axis_iter(self, shape, dim, cum): @@ -414,7 +416,11 @@ return self.implementation.astype(space, dtype) def descr_get_base(self, space): - return self.implementation.base() + impl = self.implementation + ret = impl.base() + if ret is None: + return space.w_None + return ret @unwrap_spec(inplace=bool) def descr_byteswap(self, space, inplace=False): diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py --- a/pypy/module/micronumpy/loop.py +++ b/pypy/module/micronumpy/loop.py @@ -602,17 +602,27 @@ def diagonal_array(space, arr, out, offset, axis1, axis2, shape): out_iter = out.create_iter() iter = PureShapeIterator(shape, []) - shapelen = len(shape) + shapelen_minus_1 = len(shape) - 1 + assert shapelen_minus_1 >= 0 + if axis1 < axis2: + a = axis1 + b = axis2 - 1 + else: + a = axis2 + b = axis1 - 1 + assert a >= 0 + assert b >= 0 while not iter.done(): last_index = iter.indexes[-1] if axis1 < axis2: - indexes = (iter.indexes[:axis1] + [last_index] + - iter.indexes[axis1:axis2 - 1] + [last_index + offset] + - iter.indexes[axis2 - 1:shapelen - 1]) + indexes = (iter.indexes[:a] + [last_index] + + iter.indexes[a:b] + [last_index + offset] + + iter.indexes[b:shapelen_minus_1]) else: - indexes = (iter.indexes[:axis2] + [last_index + offset] + - iter.indexes[axis2:axis1 - 1] + [last_index] + - iter.indexes[axis1 - 1:shapelen - 1]) + indexes = (iter.indexes[:a] + [last_index + offset] + + iter.indexes[a:b] + [last_index] + + iter.indexes[b:shapelen_minus_1]) out_iter.setitem(arr.getitem_index(space, indexes)) iter.next() out_iter.next() + From noreply at buildbot.pypy.org Thu Jan 10 23:43:45 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Thu, 10 Jan 2013 23:43:45 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Removed get_repo_info from targetrpystonedalone Message-ID: <20130110224345.EF1911C00E2@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59936:ff7fd376623a Date: 2013-01-10 23:06 +0100 http://bitbucket.org/pypy/pypy/changeset/ff7fd376623a/ Log: Removed get_repo_info from targetrpystonedalone diff --git a/rpython/translator/goal/targetrpystonedalone.py b/rpython/translator/goal/targetrpystonedalone.py --- a/rpython/translator/goal/targetrpystonedalone.py +++ b/rpython/translator/goal/targetrpystonedalone.py @@ -1,21 +1,17 @@ import os, sys from rpython.translator.test import rpystone from rpython.translator.goal import richards -import pypy.interpreter.gateway # needed before sys, order of imports !!! -from pypy.tool.version import get_repo_version_info # __________ Entry point __________ -VERSION = get_repo_version_info()[2] - # note that we have %f but no length specifiers in RPython def pystones_main(loops): benchtime, stones = rpystone.pystones(abs(loops)) s = '' # annotator happiness if loops >= 0: - s = ("RPystone(%s) time for %d passes = %f" % - (VERSION, loops, benchtime) + '\n' + ( + s = ("RPystone time for %d passes = %f" % + (loops, benchtime) + '\n' + ( "This machine benchmarks at %f pystones/second\n" % stones)) os.write(1, s) if loops == 12345: From noreply at buildbot.pypy.org Thu Jan 10 23:43:47 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Thu, 10 Jan 2013 23:43:47 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Moved HASH_MALLOC_SIZE to ropenssl Message-ID: <20130110224347.3B2561C00E2@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59937:90e32a0b1c06 Date: 2013-01-10 23:18 +0100 http://bitbucket.org/pypy/pypy/changeset/90e32a0b1c06/ Log: Moved HASH_MALLOC_SIZE to ropenssl diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py --- a/pypy/module/_hashlib/interp_hashlib.py +++ b/pypy/module/_hashlib/interp_hashlib.py @@ -12,13 +12,6 @@ algorithms = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512') -# HASH_MALLOC_SIZE is the size of EVP_MD, EVP_MD_CTX plus their points -# Used for adding memory pressure. Last number is an (under?)estimate of -# EVP_PKEY_CTX's size. -# XXX: Make a better estimate here -HASH_MALLOC_SIZE = ropenssl.EVP_MD_SIZE + ropenssl.EVP_MD_CTX_SIZE \ - + rffi.sizeof(ropenssl.EVP_MD) * 2 + 208 - class W_Hash(Wrappable): NULL_CTX = lltype.nullptr(ropenssl.EVP_MD_CTX.TO) ctx = NULL_CTX diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py --- a/rpython/rlib/ropenssl.py +++ b/rpython/rlib/ropenssl.py @@ -316,6 +316,13 @@ EVP_MD_CTX_cleanup = external( 'EVP_MD_CTX_cleanup', [EVP_MD_CTX], rffi.INT, threadsafe=False) +# HASH_MALLOC_SIZE is the size of EVP_MD, EVP_MD_CTX plus their points +# Used for adding memory pressure. Last number is an (under?)estimate of +# EVP_PKEY_CTX's size. +# XXX: Make a better estimate here +HASH_MALLOC_SIZE = EVP_MD_SIZE + EVP_MD_CTX_SIZE \ + + rffi.sizeof(EVP_MD) * 2 + 208 + def init_ssl(): libssl_SSL_load_error_strings() libssl_SSL_library_init() diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py --- a/rpython/translator/c/test/test_newgc.py +++ b/rpython/translator/c/test/test_newgc.py @@ -1440,7 +1440,6 @@ assert res == -99997 def define_nongc_opaque_attached_to_gc(cls): - from pypy.module._hashlib.interp_hashlib import HASH_MALLOC_SIZE from rpython.rlib import rgc, ropenssl class A: @@ -1449,7 +1448,7 @@ flavor='raw') digest = ropenssl.EVP_get_digestbyname('sha1') ropenssl.EVP_DigestInit(self.ctx, digest) - rgc.add_memory_pressure(HASH_MALLOC_SIZE + 64) + rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + 64) def __del__(self): ropenssl.EVP_MD_CTX_cleanup(self.ctx) From noreply at buildbot.pypy.org Thu Jan 10 23:43:48 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Thu, 10 Jan 2013 23:43:48 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Killed pypy.tool.debug_print in favor of rpython.rlib.debug Message-ID: <20130110224348.707171C00E2@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59938:258992ac0308 Date: 2013-01-10 23:43 +0100 http://bitbucket.org/pypy/pypy/changeset/258992ac0308/ Log: Killed pypy.tool.debug_print in favor of rpython.rlib.debug diff --git a/pypy/tool/debug_print.py b/pypy/tool/debug_print.py deleted file mode 100644 --- a/pypy/tool/debug_print.py +++ /dev/null @@ -1,20 +0,0 @@ -""" -If we are on pypy, use the real debug_* functions defined in __pypy__. - -Else, use empty functions -""" - -try: - from __pypy__ import debug_start, debug_stop, debug_print, debug_print_once -except ImportError: - def debug_start(*args): - pass - - def debug_stop(*args): - pass - - def debug_print(*args): - pass - - def debug_print_once(*args): - pass diff --git a/rpython/translator/driver.py b/rpython/translator/driver.py --- a/rpython/translator/driver.py +++ b/rpython/translator/driver.py @@ -9,7 +9,7 @@ from rpython.annotator.listdef import s_list_of_strings from rpython.annotator import policy as annpolicy from rpython.tool.udir import udir -from pypy.tool.debug_print import debug_start, debug_print, debug_stop +from rpython.rlib.debug import debug_start, debug_print, debug_stop from rpython.rlib.entrypoint import secondary_entrypoints import py From noreply at buildbot.pypy.org Thu Jan 10 23:59:29 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Thu, 10 Jan 2013 23:59:29 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed missing HASH_MALLOC_SIZE -> ropenssl.HASH_MALLOC_SIZE Message-ID: <20130110225929.E065E1C0285@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59939:e3fc26859697 Date: 2013-01-10 23:57 +0100 http://bitbucket.org/pypy/pypy/changeset/e3fc26859697/ Log: Fixed missing HASH_MALLOC_SIZE -> ropenssl.HASH_MALLOC_SIZE diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py --- a/pypy/module/_hashlib/interp_hashlib.py +++ b/pypy/module/_hashlib/interp_hashlib.py @@ -27,7 +27,7 @@ self.lock = Lock(space) ctx = lltype.malloc(ropenssl.EVP_MD_CTX.TO, flavor='raw') - rgc.add_memory_pressure(HASH_MALLOC_SIZE + self.digest_size) + rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + self.digest_size) try: if copy_from: ropenssl.EVP_MD_CTX_copy(ctx, copy_from) From noreply at buildbot.pypy.org Fri Jan 11 02:21:58 2013 From: noreply at buildbot.pypy.org (wlav) Date: Fri, 11 Jan 2013 02:21:58 +0100 (CET) Subject: [pypy-commit] pypy reflex-support: ability to read-back TTree branches that are of bool type Message-ID: <20130111012158.D69681C00E2@cobra.cs.uni-duesseldorf.de> Author: Wim Lavrijsen Branch: reflex-support Changeset: r59940:99c1280ae1cd Date: 2013-01-10 17:21 -0800 http://bitbucket.org/pypy/pypy/changeset/99c1280ae1cd/ Log: ability to read-back TTree branches that are of bool type diff --git a/pypy/module/cppyy/capi/cint_capi.py b/pypy/module/cppyy/capi/cint_capi.py --- a/pypy/module/cppyy/capi/cint_capi.py +++ b/pypy/module/cppyy/capi/cint_capi.py @@ -213,6 +213,7 @@ w_typename = space.call_method(w_leaf, "GetTypeName" ) from pypy.module.cppyy import capi typename = capi.c_resolve_name(space.str_w(w_typename)) + if typename == 'bool': typename = '_Bool' w_address = space.call_method(w_leaf, "GetValuePointer") from pypy.module._cffi_backend import cdataobj, newtype cdata = cdataobj.W_CData(space, address, newtype.new_primitive_type(space, typename)) diff --git a/pypy/module/cppyy/test/test_cint.py b/pypy/module/cppyy/test/test_cint.py --- a/pypy/module/cppyy/test/test_cint.py +++ b/pypy/module/cppyy/test/test_cint.py @@ -318,7 +318,7 @@ assert i == self.N def test07_write_builtin(self): - """Test writing of a builtins""" + """Test writing of builtins""" from cppyy import gbl # bootstraps, only needed for tests from cppyy.gbl import TFile, TTree @@ -329,21 +329,25 @@ mytree._python_owns = False import array - a = array.array('i', [0]) - b = array.array('d', [0.]) + ba = array.array('c', [chr(0)]) + ia = array.array('i', [0]) + da = array.array('d', [0.]) - mytree.Branch("myi", a, "myi/I") - mytree.Branch("myd", b, "myd/D") + mytree.Branch("my_bool", ba, "my_bool/O") + mytree.Branch("my_int", ia, "my_int/I") + mytree.Branch("my_double", da, "my_double/D") for i in range(self.N): - a[0] = i+1 # make sure value is different from default (0) - b[0] = (i+1)/2. # id. 0. + # make sure value is different from default (0) + ba[0] = i%2 and chr(0) or chr(1) + ia[0] = i+1 + da[0] = (i+1)/2. mytree.Fill() f.Write() f.Close() def test08_read_builtin(self): - """Test reading of a single branched TTree with an std::vector""" + """Test reading of builtins""" from cppyy import gbl from cppyy.gbl import TFile @@ -351,10 +355,13 @@ f = TFile(self.fname) mytree = f.Get(self.tname) + raises(AttributeError, getattr, mytree, "does_not_exist") + i = 1 for event in mytree: - assert event.myi == i - assert event.myd == i/2. + assert event.my_bool == (i-1)%2 and 0 or 1 + assert event.my_int == i + assert event.my_double == i/2. i += 1 assert (i-1) == self.N From noreply at buildbot.pypy.org Fri Jan 11 07:31:58 2013 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 11 Jan 2013 07:31:58 +0100 (CET) Subject: [pypy-commit] pypy default: switch ' and " for windows Message-ID: <20130111063158.C9B9A1C0285@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r59941:4765dafa998c Date: 2013-01-11 08:30 +0200 http://bitbucket.org/pypy/pypy/changeset/4765dafa998c/ Log: switch ' and " for windows diff --git a/pypy/doc/test/test_whatsnew.py b/pypy/doc/test/test_whatsnew.py --- a/pypy/doc/test/test_whatsnew.py +++ b/pypy/doc/test/test_whatsnew.py @@ -29,7 +29,7 @@ merge() and \ branch(default)) and \ not branch(default)' % (startrev, endrev) - cmd = r"hg log -R '%s' -r '%s' --template '{branches}\n'" % (path, revset) + cmd = r'hg log -R "%s" -r "%s" --template "{branches}\n"' % (path, revset) out = getoutput(cmd) branches = set(map(str.strip, out.splitlines())) return branches From noreply at buildbot.pypy.org Fri Jan 11 07:32:00 2013 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 11 Jan 2013 07:32:00 +0100 (CET) Subject: [pypy-commit] pypy default: document merged branch Message-ID: <20130111063200.0DE561C0285@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r59942:12f996bee262 Date: 2013-01-11 08:31 +0200 http://bitbucket.org/pypy/pypy/changeset/12f996bee262/ Log: document merged branch diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -15,6 +15,9 @@ .. branch: numpypy-longdouble Long double support for numpypy +.. branch: numpypy-real-as-view +Convert real, imag from ufuncs to views. This involves the beginning of +view() functionality .. branch: signatures Improved RPython typing From noreply at buildbot.pypy.org Fri Jan 11 08:00:58 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Fri, 11 Jan 2013 08:00:58 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Refactored rsocket and interp_socket for the split. Hacky! Message-ID: <20130111070058.2485E1C0285@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59943:898e7335e13b Date: 2013-01-11 08:00 +0100 http://bitbucket.org/pypy/pypy/changeset/898e7335e13b/ Log: Refactored rsocket and interp_socket for the split. Hacky! diff --git a/pypy/module/_socket/interp_func.py b/pypy/module/_socket/interp_func.py --- a/pypy/module/_socket/interp_func.py +++ b/pypy/module/_socket/interp_func.py @@ -1,5 +1,5 @@ from pypy.interpreter.gateway import unwrap_spec, WrappedDefault -from pypy.module._socket.interp_socket import converted_error, W_RSocket +from pypy.module._socket.interp_socket import converted_error, W_RSocket, addr_as_object, ipaddr_from_object from rpython.rlib import rsocket from rpython.rlib.rsocket import SocketError, INVALID_SOCKET from pypy.interpreter.error import OperationError @@ -120,7 +120,7 @@ Get host and port for a sockaddr.""" try: - addr = rsocket.ipaddr_from_object(space, w_sockaddr) + addr = ipaddr_from_object(space, w_sockaddr) host, servport = rsocket.getnameinfo(addr, flags) except SocketError, e: raise converted_error(space, e) @@ -284,7 +284,7 @@ space.wrap(socktype), space.wrap(protocol), space.wrap(canonname), - addr.as_object(INVALID_SOCKET, space)]) # -1 as per cpython + addr_as_object(addr, INVALID_SOCKET, space)]) # -1 as per cpython for (family, socktype, protocol, canonname, addr) in lst] return space.newlist(lst1) diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -5,7 +5,8 @@ from rpython.rlib.rarithmetic import intmask from rpython.rlib import rsocket from rpython.rlib.rsocket import RSocket, AF_INET, SOCK_STREAM -from rpython.rlib.rsocket import SocketError, SocketErrorWithErrno +from rpython.rlib.rsocket import SocketError, SocketErrorWithErrno, RSocketError +from rpython.rlib.socket import INETAddress, INET6Address from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter import gateway @@ -16,6 +17,124 @@ def check(self): self.space.getexecutioncontext().checksignals() + +# XXX Hack to seperate rpython and pypy +def addr_as_object(addr, fd, space): + if addr.family == rsocket.AF_INET: + return space.newtuple([space.wrap(addr.get_host()), + space.wrap(addr.get_port())]) + if addr.family == rsocket.AF_INET6: + return space.newtuple([space.wrap(addr.get_host()), + space.wrap(addr.get_port()), + space.wrap(addr.get_flowinfo()), + space.wrap(addr.get_scope_id())]) + if 'AF_PACKET' in rsocket.constants and addr.family == rsocket.AF_PACKET: + return space.newtuple([space.wrap(addr.get_ifname(fd)), + space.wrap(addr.get_protocol()), + space.wrap(addr.get_pkttype()), + space.wrap(addr.get_hatype()), + space.wrap(addr.get_addr())]) + if 'AF_UNIX' in rsocket.constants and addr.family == rsocket.AF_UNIX: + return space.wrap(addr.get_path()) + if 'AF_NETLINK' in rsocket.constants and addr.family == rsocket.AF_NETLINK: + return space.newtuple([space.wrap(addr.get_pid()), + space.wrap(addr.get_groups())]) + # If we don't know the address family, don't raise an + # exception -- return it as a tuple. + a = addr.lock() + family = rffi.cast(lltype.Signed, a.c_sa_family) + datalen = addr.addrlen - offsetof(_c.sockaddr, 'c_sa_data') + rawdata = ''.join([a.c_sa_data[i] for i in range(datalen)]) + addr.unlock() + return space.newtuple([space.wrap(family), + space.wrap(rawdata)]) + +# XXX Hack to seperate rpython and pypy +# XXX a bit of code duplication +def fill_from_object(addr, space, w_address): + if addr.family == rsocket.AF_INET: + from pypy.interpreter.error import OperationError + _, w_port = space.unpackiterable(w_address, 2) + port = space.int_w(w_port) + port = make_ushort_port(space, port) + a = addr.lock(_c.sockaddr_in) + rffi.setintfield(a, 'c_sin_port', htons(port)) + addr.unlock() + elif addr.family == rsocket.AF_INET6: + from pypy.interpreter.error import OperationError + pieces_w = space.unpackiterable(w_address) + if not (2 <= len(pieces_w) <= 4): + raise RSocketError("AF_INET6 address must be a tuple of length 2 " + "to 4, not %d" % len(pieces_w)) + port = space.int_w(pieces_w[1]) + port = make_ushort_port(space, port) + if len(pieces_w) > 2: flowinfo = space.int_w(pieces_w[2]) + else: flowinfo = 0 + if len(pieces_w) > 3: scope_id = space.uint_w(pieces_w[3]) + else: scope_id = 0 + if flowinfo < 0 or flowinfo > 0xfffff: + raise OperationError(space.w_OverflowError, space.wrap( + "flowinfo must be 0-1048575.")) + flowinfo = rffi.cast(lltype.Unsigned, flowinfo) + a = addr.lock(_c.sockaddr_in6) + rffi.setintfield(a, 'c_sin6_port', htons(port)) + rffi.setintfield(a, 'c_sin6_flowinfo', htonl(flowinfo)) + rffi.setintfield(a, 'c_sin6_scope_id', scope_id) + addr.unlock() + else: + raise NotImplementedError + +# XXX Hack to seperate rpython and pypy +def addr_from_object(family, space, w_address): + if family == rsocket.AF_INET: + w_host, w_port = space.unpackiterable(w_address, 2) + host = space.str_w(w_host) + port = space.int_w(w_port) + port = make_ushort_port(space, port) + return INETAddress(host, port) + if family == rsocket.AF_INET6: + from pypy.interpreter.error import OperationError + pieces_w = space.unpackiterable(w_address) + if not (2 <= len(pieces_w) <= 4): + raise TypeError("AF_INET6 address must be a tuple of length 2 " + "to 4, not %d" % len(pieces_w)) + host = space.str_w(pieces_w[0]) + port = space.int_w(pieces_w[1]) + port = make_ushort_port(space, port) + if len(pieces_w) > 2: flowinfo = space.int_w(pieces_w[2]) + else: flowinfo = 0 + if len(pieces_w) > 3: scope_id = space.uint_w(pieces_w[3]) + else: scope_id = 0 + if flowinfo < 0 or flowinfo > 0xfffff: + raise OperationError(space.w_OverflowError, space.wrap( + "flowinfo must be 0-1048575.")) + flowinfo = rffi.cast(lltype.Unsigned, flowinfo) + return INET6Address(host, port, flowinfo, scope_id) + if 'AF_UNIX' in rsocket.constants and family == rsocket.AF_UNIX: + from rpython.rlib.rsocket import UNIXAddress + return UNIXAddress(space.str_w(w_address)) + if 'AF_NETLINK' in rsocket.constants and family == rsocket.AF_NETLINK: + from rpython.rlib.rsocket import NETLINKAddress + w_pid, w_groups = space.unpackiterable(w_address, 2) + return NETLINKAddress(space.uint_w(w_pid), space.uint_w(w_groups)) + raise RSocketError("unknown address family") + +# XXX Hack to seperate rpython and pypy +def make_ushort_port(space, port): + from pypy.interpreter.error import OperationError + if port < 0 or port > 0xffff: + raise OperationError(space.w_ValueError, space.wrap( + "port must be 0-65535.")) + return rffi.cast(rffi.USHORT, port) + +# XXX Hack to seperate rpython and pypy +def ipaddr_from_object(space, w_sockaddr): + host = space.str_w(space.getitem(w_sockaddr, space.wrap(0))) + addr = makeipaddr(host) + addr_fill_from_object(addr, space, w_sockaddr) + return addr + + class W_RSocket(Wrappable, RSocket): def __del__(self): self.clear_all_weakrefs() @@ -33,10 +152,19 @@ sock = rsocket.make_socket( fd, self.family, self.type, self.proto, W_RSocket) return space.newtuple([space.wrap(sock), - addr.as_object(sock.fd, space)]) + addr_as_object(addr, sock.fd, space)]) except SocketError, e: raise converted_error(space, e) + # convert an Address into an app-level object + def addr_as_object(self, space, address): + return addr_as_object(address, self.fd, space) + + # convert an app-level object into an Address + # based on the current socket's family + def addr_from_object(self, space, w_address): + return addr_from_object(self.family, space, w_address) + def bind_w(self, space, w_addr): """bind(address) @@ -104,7 +232,7 @@ """ try: addr = self.getpeername() - return addr.as_object(self.fd, space) + return addr_as_object(addr, self.fd, space) except SocketError, e: raise converted_error(space, e) @@ -116,7 +244,7 @@ """ try: addr = self.getsockname() - return addr.as_object(self.fd, space) + return addr_as_object(addr, self.fd, space) except SocketError, e: raise converted_error(space, e) @@ -194,7 +322,7 @@ try: data, addr = self.recvfrom(buffersize, flags) if addr: - w_addr = addr.as_object(self.fd, space) + w_addr = addr_as_object(addr, self.fd, space) else: w_addr = space.w_None return space.newtuple([space.wrap(data), w_addr]) @@ -319,7 +447,7 @@ try: readlgt, addr = self.recvfrom_into(rwbuffer, nbytes, flags) if addr: - w_addr = addr.as_object(self.fd, space) + w_addr = addr_as_object(addr, self.fd, space) else: w_addr = space.w_None return space.newtuple([space.wrap(readlgt), w_addr]) diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -229,13 +229,14 @@ assert space.unwrap(w_l) == info def test_unknown_addr_as_object(): + from pypy.module._socket.interp_socket import addr_as_object c_addr = lltype.malloc(rsocket._c.sockaddr, flavor='raw') c_addr.c_sa_data[0] = 'c' rffi.setintfield(c_addr, 'c_sa_family', 15) # XXX what size to pass here? for the purpose of this test it has # to be short enough so we have some data, 1 sounds good enough # + sizeof USHORT - w_obj = rsocket.Address(c_addr, 1 + 2).as_object(-1, space) + w_obj = addr_as_object(rsocket.Address(c_addr, 1 + 2), t(-1, space)) assert space.is_true(space.isinstance(w_obj, space.w_tuple)) assert space.int_w(space.getitem(w_obj, space.wrap(0))) == 15 assert space.str_w(space.getitem(w_obj, space.wrap(1))) == 'c' diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py --- a/rpython/rlib/rsocket.py +++ b/rpython/rlib/rsocket.py @@ -112,38 +112,6 @@ """ keepalive_until_here(self) - def as_object(self, fd, space): - """Convert the address to an app-level object.""" - # If we don't know the address family, don't raise an - # exception -- return it as a tuple. - addr = self.lock() - family = rffi.cast(lltype.Signed, addr.c_sa_family) - datalen = self.addrlen - offsetof(_c.sockaddr, 'c_sa_data') - rawdata = ''.join([addr.c_sa_data[i] for i in range(datalen)]) - self.unlock() - return space.newtuple([space.wrap(family), - space.wrap(rawdata)]) - - def from_object(space, w_address): - """Convert an app-level object to an Address.""" - # It's a static method but it's overridden and must be called - # on the correct subclass. - raise RSocketError("unknown address family") - from_object = staticmethod(from_object) - - @staticmethod - def make_ushort_port(space, port): - from pypy.interpreter.error import OperationError - if port < 0 or port > 0xffff: - raise OperationError(space.w_ValueError, space.wrap( - "port must be 0-65535.")) - return rffi.cast(rffi.USHORT, port) - - def fill_from_object(self, space, w_address): - """ Purely abstract - """ - raise NotImplementedError - # ____________________________________________________________ def makeipaddr(name, result=None): @@ -268,12 +236,6 @@ self.unlock() return res - def as_object(self, fd, space): - return space.newtuple([space.wrap(self.get_ifname(fd)), - space.wrap(self.get_protocol()), - space.wrap(self.get_pkttype()), - space.wrap(self.get_hatype()), - space.wrap(self.get_addr())]) class INETAddress(IPAddress): family = AF_INET @@ -304,29 +266,6 @@ self.get_host() == other.get_host() and self.get_port() == other.get_port()) - def as_object(self, fd, space): - return space.newtuple([space.wrap(self.get_host()), - space.wrap(self.get_port())]) - - def from_object(space, w_address): - # Parse an app-level object representing an AF_INET address - w_host, w_port = space.unpackiterable(w_address, 2) - host = space.str_w(w_host) - port = space.int_w(w_port) - port = Address.make_ushort_port(space, port) - return INETAddress(host, port) - from_object = staticmethod(from_object) - - def fill_from_object(self, space, w_address): - # XXX a bit of code duplication - from pypy.interpreter.error import OperationError - _, w_port = space.unpackiterable(w_address, 2) - port = space.int_w(w_port) - port = self.make_ushort_port(space, port) - a = self.lock(_c.sockaddr_in) - rffi.setintfield(a, 'c_sin_port', htons(port)) - self.unlock() - def from_in_addr(in_addr): result = instantiate(INETAddress) # store the malloc'ed data into 'result' as soon as possible @@ -393,55 +332,6 @@ self.get_flowinfo() == other.get_flowinfo() and self.get_scope_id() == other.get_scope_id()) - def as_object(self, fd, space): - return space.newtuple([space.wrap(self.get_host()), - space.wrap(self.get_port()), - space.wrap(self.get_flowinfo()), - space.wrap(self.get_scope_id())]) - - def from_object(space, w_address): - from pypy.interpreter.error import OperationError - pieces_w = space.unpackiterable(w_address) - if not (2 <= len(pieces_w) <= 4): - raise TypeError("AF_INET6 address must be a tuple of length 2 " - "to 4, not %d" % len(pieces_w)) - host = space.str_w(pieces_w[0]) - port = space.int_w(pieces_w[1]) - port = Address.make_ushort_port(space, port) - if len(pieces_w) > 2: flowinfo = space.int_w(pieces_w[2]) - else: flowinfo = 0 - if len(pieces_w) > 3: scope_id = space.uint_w(pieces_w[3]) - else: scope_id = 0 - if flowinfo < 0 or flowinfo > 0xfffff: - raise OperationError(space.w_OverflowError, space.wrap( - "flowinfo must be 0-1048575.")) - flowinfo = rffi.cast(lltype.Unsigned, flowinfo) - return INET6Address(host, port, flowinfo, scope_id) - from_object = staticmethod(from_object) - - def fill_from_object(self, space, w_address): - # XXX a bit of code duplication - from pypy.interpreter.error import OperationError - pieces_w = space.unpackiterable(w_address) - if not (2 <= len(pieces_w) <= 4): - raise RSocketError("AF_INET6 address must be a tuple of length 2 " - "to 4, not %d" % len(pieces_w)) - port = space.int_w(pieces_w[1]) - port = self.make_ushort_port(space, port) - if len(pieces_w) > 2: flowinfo = space.int_w(pieces_w[2]) - else: flowinfo = 0 - if len(pieces_w) > 3: scope_id = space.uint_w(pieces_w[3]) - else: scope_id = 0 - if flowinfo < 0 or flowinfo > 0xfffff: - raise OperationError(space.w_OverflowError, space.wrap( - "flowinfo must be 0-1048575.")) - flowinfo = rffi.cast(lltype.Unsigned, flowinfo) - a = self.lock(_c.sockaddr_in6) - rffi.setintfield(a, 'c_sin6_port', htons(port)) - rffi.setintfield(a, 'c_sin6_flowinfo', htonl(flowinfo)) - rffi.setintfield(a, 'c_sin6_scope_id', scope_id) - self.unlock() - def from_in6_addr(in6_addr): result = instantiate(INET6Address) # store the malloc'ed data into 'result' as soon as possible @@ -509,13 +399,6 @@ return (isinstance(other, UNIXAddress) and self.get_path() == other.get_path()) - def as_object(self, fd, space): - return space.wrap(self.get_path()) - - def from_object(space, w_address): - return UNIXAddress(space.str_w(w_address)) - from_object = staticmethod(from_object) - if 'AF_NETLINK' in constants: class NETLINKAddress(Address): family = AF_NETLINK @@ -543,15 +426,6 @@ def __repr__(self): return '' % (self.get_pid(), self.get_groups()) - - def as_object(self, fd, space): - return space.newtuple([space.wrap(self.get_pid()), - space.wrap(self.get_groups())]) - - def from_object(space, w_address): - w_pid, w_groups = space.unpackiterable(w_address, 2) - return NETLINKAddress(space.uint_w(w_pid), space.uint_w(w_groups)) - from_object = staticmethod(from_object) # ____________________________________________________________ @@ -596,12 +470,6 @@ result.setdata(buf, 0) return result, klass.maxlen -def ipaddr_from_object(space, w_sockaddr): - host = space.str_w(space.getitem(w_sockaddr, space.wrap(0))) - addr = makeipaddr(host) - addr.fill_from_object(space, w_sockaddr) - return addr - # ____________________________________________________________ class RSocket(object): @@ -701,15 +569,6 @@ def error_handler(self): return last_error() - # convert an Address into an app-level object - def addr_as_object(self, space, address): - return address.as_object(self.fd, space) - - # convert an app-level object into an Address - # based on the current socket's family - def addr_from_object(self, space, w_address): - return af_get(self.family).from_object(space, w_address) - # build a null address object, ready to be used as output argument to # C functions that return an address. It must be unlock()ed after you # are done using addr_p. From noreply at buildbot.pypy.org Fri Jan 11 08:16:20 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Fri, 11 Jan 2013 08:16:20 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed typo in import Message-ID: <20130111071620.B0A801C00E2@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59944:3354e7cfa070 Date: 2013-01-11 08:16 +0100 http://bitbucket.org/pypy/pypy/changeset/3354e7cfa070/ Log: Fixed typo in import diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -6,7 +6,7 @@ from rpython.rlib import rsocket from rpython.rlib.rsocket import RSocket, AF_INET, SOCK_STREAM from rpython.rlib.rsocket import SocketError, SocketErrorWithErrno, RSocketError -from rpython.rlib.socket import INETAddress, INET6Address +from rpython.rlib.rsocket import INETAddress, INET6Address from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter import gateway From noreply at buildbot.pypy.org Fri Jan 11 08:32:37 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Fri, 11 Jan 2013 08:32:37 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed a few more socket things Message-ID: <20130111073237.092B21C00E2@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59945:6e461b5f80d0 Date: 2013-01-11 08:32 +0100 http://bitbucket.org/pypy/pypy/changeset/6e461b5f80d0/ Log: Fixed a few more socket things diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -3,6 +3,7 @@ interp_attrproperty from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault from rpython.rlib.rarithmetic import intmask +from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rlib import rsocket from rpython.rlib.rsocket import RSocket, AF_INET, SOCK_STREAM from rpython.rlib.rsocket import SocketError, SocketErrorWithErrno, RSocketError @@ -129,6 +130,7 @@ # XXX Hack to seperate rpython and pypy def ipaddr_from_object(space, w_sockaddr): + from rpython.rlib.rsocket import makeipaddr host = space.str_w(space.getitem(w_sockaddr, space.wrap(0))) addr = makeipaddr(host) addr_fill_from_object(addr, space, w_sockaddr) diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -242,6 +242,7 @@ assert space.str_w(space.getitem(w_obj, space.wrap(1))) == 'c' def test_addr_raw_packet(): + from pypy.module._socket.interp_socket import addr_as_object if not hasattr(rsocket._c, 'sockaddr_ll'): py.test.skip("posix specific test") # HACK: To get the correct interface numer of lo, which in most cases is 1, @@ -266,7 +267,7 @@ # fd needs to be somehow valid s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) fd = s.fileno() - w_obj = rsocket.make_address(c_addr, addrlen).as_object(fd, space) + w_obj = addr_as_object(rsocket.make_address(c_addr, addrlen), (fd, space)) lltype.free(c_addr_ll, flavor='raw') assert space.is_true(space.eq(w_obj, space.newtuple([ space.wrap('lo'), From noreply at buildbot.pypy.org Fri Jan 11 08:48:34 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Fri, 11 Jan 2013 08:48:34 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed a few more typos regarding sockets Message-ID: <20130111074834.C5A121C0343@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59946:af4c9ae740c1 Date: 2013-01-11 08:48 +0100 http://bitbucket.org/pypy/pypy/changeset/af4c9ae740c1/ Log: Fixed a few more typos regarding sockets diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -133,7 +133,7 @@ from rpython.rlib.rsocket import makeipaddr host = space.str_w(space.getitem(w_sockaddr, space.wrap(0))) addr = makeipaddr(host) - addr_fill_from_object(addr, space, w_sockaddr) + fill_from_object(addr, space, w_sockaddr) return addr diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -236,7 +236,7 @@ # XXX what size to pass here? for the purpose of this test it has # to be short enough so we have some data, 1 sounds good enough # + sizeof USHORT - w_obj = addr_as_object(rsocket.Address(c_addr, 1 + 2), t(-1, space)) + w_obj = addr_as_object(rsocket.Address(c_addr, 1 + 2), -1, space) assert space.is_true(space.isinstance(w_obj, space.w_tuple)) assert space.int_w(space.getitem(w_obj, space.wrap(0))) == 15 assert space.str_w(space.getitem(w_obj, space.wrap(1))) == 'c' @@ -267,7 +267,7 @@ # fd needs to be somehow valid s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) fd = s.fileno() - w_obj = addr_as_object(rsocket.make_address(c_addr, addrlen), (fd, space)) + w_obj = addr_as_object(rsocket.make_address(c_addr, addrlen), fd, space) lltype.free(c_addr_ll, flavor='raw') assert space.is_true(space.eq(w_obj, space.newtuple([ space.wrap('lo'), From noreply at buildbot.pypy.org Fri Jan 11 08:49:31 2013 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 11 Jan 2013 08:49:31 +0100 (CET) Subject: [pypy-commit] pypy default: Add comments (from stm-thread-2). Message-ID: <20130111074931.412F01C0343@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r59947:067f06080d2e Date: 2013-01-11 08:48 +0100 http://bitbucket.org/pypy/pypy/changeset/067f06080d2e/ Log: Add comments (from stm-thread-2). diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -178,6 +178,11 @@ else: oparg = 0 + # note: the structure of the code here is such that it makes + # (after translation) a big "if/elif" chain, which is then + # turned into a switch(). It starts here: even if the first + # one is not an "if" but a "while" the effect is the same. + while opcode == self.opcodedesc.EXTENDED_ARG.index: opcode = ord(co_code[next_instr]) if opcode < self.HAVE_ARGUMENT: @@ -226,6 +231,8 @@ 'END_FINALLY', 'JUMP_ABSOLUTE'): continue # opcodes implemented above + # the following "if" is part of the big switch described + # above. if opcode == opdesc.index: # dispatch to the opcode method meth = getattr(self, opdesc.methodname) From noreply at buildbot.pypy.org Fri Jan 11 11:02:51 2013 From: noreply at buildbot.pypy.org (cfbolz) Date: Fri, 11 Jan 2013 11:02:51 +0100 (CET) Subject: [pypy-commit] pypy type-specialized-instances: the code as it was before used instantiate within an elidable function, which Message-ID: <20130111100251.87AA71C0285@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: type-specialized-instances Changeset: r59948:6bf5e8e07db2 Date: 2013-01-11 11:02 +0100 http://bitbucket.org/pypy/pypy/changeset/6bf5e8e07db2/ Log: the code as it was before used instantiate within an elidable function, which makes it not analyzable. hack differently. diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py --- a/pypy/objspace/std/mapdict.py +++ b/pypy/objspace/std/mapdict.py @@ -131,14 +131,19 @@ return None @jit.elidable - def _get_new_attr(self, name, index, attrclass): - key = name, index, attrclass.key_for_attr_cache + def _get_new_attr(self, name, index, attrclass_key): + key = name, index, attrclass_key cache = self.cache_attrs if cache is None: cache = self.cache_attrs = {} attr = cache.get(key, None) if attr is None: - attr = attrclass((name, index), self) + # XXX not so nice that the classes have to be listed + if attrclass_key == PlainAttribute.attrclass_key: + attr = PlainAttribute((name, index), self) + else: + assert attrclass_key == IntAttribute.attrclass_key + attr = IntAttribute((name, index), self) cache[key] = attr return attr @@ -149,7 +154,8 @@ def add_attr(self, obj, selector, w_value): attrclass = get_attrclass_from_value(self.space, w_value) # grumble, jit needs this - attr = self._get_new_attr(selector[0], selector[1], attrclass) + attr = self._get_new_attr(selector[0], selector[1], + attrclass.attrclass_key) oldattr = obj._get_mapdict_map() if not jit.we_are_jitted(): size_est = (oldattr._size_estimate + attr.size_estimate() @@ -332,7 +338,7 @@ return "" % (self.selector, self.position, self.back) class PlainAttribute(AbstractStoredAttribute): - key_for_attr_cache = 0 + attrclass_key = 0 erase_item, unerase_item = rerased.new_erasing_pair("mapdict storage object item") erase_item = staticmethod(erase_item) @@ -348,7 +354,7 @@ obj._mapdict_write_storage(self.position, erased) class IntAttribute(AbstractStoredAttribute): - key_for_attr_cache = 1 + attrclass_key = 1 erase_item, unerase_item = rerased.erase_int, rerased.unerase_int erase_item = staticmethod(erase_item) From noreply at buildbot.pypy.org Fri Jan 11 11:08:46 2013 From: noreply at buildbot.pypy.org (cfbolz) Date: Fri, 11 Jan 2013 11:08:46 +0100 (CET) Subject: [pypy-commit] pypy type-specialized-instances: on the branch mapdict needs tagged pointers Message-ID: <20130111100846.BC9671C0285@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: type-specialized-instances Changeset: r59949:a6da532ea8b0 Date: 2013-01-11 11:05 +0100 http://bitbucket.org/pypy/pypy/changeset/a6da532ea8b0/ Log: on the branch mapdict needs tagged pointers diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -238,6 +238,7 @@ default=False, requires=[("objspace.std.getattributeshortcut", True), ("objspace.std.withmethodcache", True), + ("translation.taggedpointers", True), ]), BoolOption("withrangelist", From noreply at buildbot.pypy.org Fri Jan 11 15:01:26 2013 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 11 Jan 2013 15:01:26 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: Add a TODO. Message-ID: <20130111140126.D7D071C114A@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r59950:c336149c39bb Date: 2013-01-11 15:00 +0100 http://bitbucket.org/pypy/pypy/changeset/c336149c39bb/ Log: Add a TODO. diff --git a/TODO b/TODO new file mode 100644 --- /dev/null +++ b/TODO @@ -0,0 +1,46 @@ + +RPyAssert(i < len(lst)): if lst is global this turns into tons of code + +------------------------------------------------------------ + +signal module: plan: + + signal handler: + + d = main thread's thread_descriptor + d->reads_size_limit = GOT_SIGNAL ( = -1) + this makes stm_should_break_transaction(d) return 1 + + + def _runs_normal_handler(): + if rstm.should_break_transaction(): + if d->reads_size_limit == GOT_SIGNAL: + checksignals() + return not rstm.should_break_transaction() + return False + return True + + + def checksignals(): + if d->reads_size_limit == GOT_SIGNAL: + turn inevitable + reset d->reads_size_limit + handle the signal + +------------------------------------------------------------ + +after an abort, keep old global_to_local and access it using special code + +------------------------------------------------------------ + +GC: major collections + +------------------------------------------------------------ + +JIT: finish (missing: the call in execute_token(), reorganize pypy source, ?) + +------------------------------------------------------------ + +investigate abusing jitdriver to do rstm.perform_transaction() + +------------------------------------------------------------ From noreply at buildbot.pypy.org Fri Jan 11 15:36:41 2013 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 11 Jan 2013 15:36:41 +0100 (CET) Subject: [pypy-commit] pypy default: jitdriver(reds='auto'): fix it to only include the red vars that are Message-ID: <20130111143641.7539D1C114A@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r59951:02f2e74659ab Date: 2013-01-11 15:35 +0100 http://bitbucket.org/pypy/pypy/changeset/02f2e74659ab/ Log: jitdriver(reds='auto'): fix it to only include the red vars that are alive across the jitdriver. Test. Move the logic to support, too. diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py --- a/pypy/jit/codewriter/support.py +++ b/pypy/jit/codewriter/support.py @@ -10,7 +10,7 @@ from pypy.rpython.extregistry import ExtRegistryEntry from pypy.translator.simplify import get_funcobj from pypy.translator.unsimplify import split_block -from pypy.objspace.flow.model import Constant +from pypy.objspace.flow.model import Variable, Constant from pypy.translator.translator import TranslationContext from pypy.annotation.policy import AnnotatorPolicy from pypy.annotation import model as annmodel @@ -59,6 +59,45 @@ rtyper = annotate(func, values) return rtyper.annotator.translator.graphs[0] +def autodetect_jit_markers_redvars(graph): + # the idea is to find all the jit_merge_point and + # add all the variables across the links to the reds. + for block, op in graph.iterblockops(): + if op.opname == 'jit_marker': + jitdriver = op.args[1].value + if not jitdriver.autoreds: + continue + # if we want to support also can_enter_jit, we should find a + # way to detect a consistent set of red vars to pass *both* to + # jit_merge_point and can_enter_jit. The current simple + # solution doesn't work because can_enter_jit might be in + # another block, so the set of alive_v will be different. + methname = op.args[0].value + assert methname == 'jit_merge_point', ( + "reds='auto' is supported only for jit drivers which " + "calls only jit_merge_point. Found a call to %s" % methname) + # + # compute the set of live variables across the jit_marker + alive_v = set() + for link in block.exits: + alive_v.update(link.args) + alive_v.difference_update(link.getextravars()) + for op1 in block.operations[::-1]: + if op1 is op: + break # stop when the meet the jit_marker + alive_v.discard(op1.result) + alive_v.update(op1.args) + greens_v = op.args[2:] + reds_v = alive_v - set(greens_v) + reds_v = [v for v in reds_v if isinstance(v, Variable) and + v.concretetype is not lltype.Void] + reds_v = sort_vars(reds_v) + op.args.extend(reds_v) + if jitdriver.numreds is None: + jitdriver.numreds = len(reds_v) + else: + assert jitdriver.numreds == len(reds_v), 'inconsistent number of reds_v' + def split_before_jit_merge_point(graph, portalblock, portalopindex): """Split the block just before the 'jit_merge_point', making sure the input args are in the canonical order. diff --git a/pypy/jit/metainterp/test/test_warmspot.py b/pypy/jit/metainterp/test/test_warmspot.py --- a/pypy/jit/metainterp/test/test_warmspot.py +++ b/pypy/jit/metainterp/test/test_warmspot.py @@ -383,6 +383,23 @@ assert res == expected self.check_resops(int_sub=2, int_mul=0, int_add=2) + def test_loop_automatic_reds_not_too_many_redvars(self): + myjitdriver = JitDriver(greens = ['m'], reds = 'auto') + def one(): + return 1 + def f(n, m): + res = 0 + while n > 0: + n -= one() + myjitdriver.jit_merge_point(m=m) + res += m*2 + return res + expected = f(21, 5) + res = self.meta_interp(f, [21, 5]) + assert res == expected + oplabel = get_stats().loops[0].operations[0] + assert len(oplabel.getarglist()) == 2 # 'n', 'res' in some order + def test_inline_jit_merge_point(self): # test that the machinery to inline jit_merge_points in callers # works. The final user does not need to mess manually with the diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py --- a/pypy/jit/metainterp/warmspot.py +++ b/pypy/jit/metainterp/warmspot.py @@ -347,49 +347,13 @@ self.jitdrivers_sd = [] graphs = self.translator.graphs for graph, block, pos in find_jit_merge_points(graphs): - self.autodetect_jit_markers_redvars(graph) + support.autodetect_jit_markers_redvars(graph) self.split_graph_and_record_jitdriver(graph, block, pos) # assert (len(set([jd.jitdriver for jd in self.jitdrivers_sd])) == len(self.jitdrivers_sd)), \ "there are multiple jit_merge_points with the same jitdriver" - def autodetect_jit_markers_redvars(self, graph): - # the idea is to find all the jit_merge_point and can_enter_jit and - # add all the variables across the links to the reds. - for block, op in graph.iterblockops(): - if op.opname == 'jit_marker': - jitdriver = op.args[1].value - if not jitdriver.autoreds: - continue - # if we want to support also can_enter_jit, we should find a - # way to detect a consistent set of red vars to pass *both* to - # jit_merge_point and can_enter_jit. The current simple - # solution doesn't work because can_enter_jit might be in - # another block, so the set of alive_v will be different. - methname = op.args[0].value - assert methname == 'jit_merge_point', ( - "reds='auto' is supported only for jit drivers which " - "calls only jit_merge_point. Found a call to %s" % methname) - # - # compute the set of live variables before the jit_marker - alive_v = set(block.inputargs) - for op1 in block.operations: - if op1 is op: - break # stop when the meet the jit_marker - if op1.result.concretetype != lltype.Void: - alive_v.add(op1.result) - greens_v = op.args[2:] - reds_v = alive_v - set(greens_v) - reds_v = [v for v in reds_v if v.concretetype is not lltype.Void] - reds_v = support.sort_vars(reds_v) - op.args.extend(reds_v) - if jitdriver.numreds is None: - jitdriver.numreds = len(reds_v) - else: - assert jitdriver.numreds == len(reds_v), 'inconsistent number of reds_v' - - def split_graph_and_record_jitdriver(self, graph, block, pos): op = block.operations[pos] jd = JitDriverStaticData() @@ -717,6 +681,9 @@ jitdriver = op.args[1].value assert jitdriver in sublists, \ "can_enter_jit with no matching jit_merge_point" + assert not jitdriver.autoreds, ( + "can_enter_jit not supported with a jitdriver that " + "has reds='auto'") jd, sublist = sublists[jitdriver] origportalgraph = jd._jit_merge_point_in if graph is not origportalgraph: From noreply at buildbot.pypy.org Fri Jan 11 15:45:06 2013 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 11 Jan 2013 15:45:06 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: hg merge default Message-ID: <20130111144506.E76F61C05DD@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r59952:22f849260e70 Date: 2013-01-11 15:43 +0100 http://bitbucket.org/pypy/pypy/changeset/22f849260e70/ Log: hg merge default diff too long, truncating to 2000 out of 2076 lines diff --git a/pypy/doc/Makefile b/pypy/doc/Makefile --- a/pypy/doc/Makefile +++ b/pypy/doc/Makefile @@ -32,38 +32,38 @@ -rm -rf $(BUILDDIR)/* html: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." pickle: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ @@ -73,7 +73,7 @@ @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/PyPy.qhc" latex: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @@ -81,26 +81,26 @@ "run these through (pdf)latex." man: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man" changes: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." doctest: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -38,7 +38,7 @@ # General information about the project. project = u'PyPy' -copyright = u'2011, The PyPy Project' +copyright = u'2013, The PyPy Project' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/pypy/doc/config/translation.no__thread.txt b/pypy/doc/config/translation.no__thread.txt --- a/pypy/doc/config/translation.no__thread.txt +++ b/pypy/doc/config/translation.no__thread.txt @@ -1,3 +1,3 @@ Don't use gcc __thread attribute for fast thread local storage -implementation . Increases the chance that moving the resulting +implementation. Increases the chance that moving the resulting executable to another same processor Linux machine will work. diff --git a/pypy/doc/pypyconfig.py b/pypy/doc/pypyconfig.py --- a/pypy/doc/pypyconfig.py +++ b/pypy/doc/pypyconfig.py @@ -3,6 +3,10 @@ def setup(app): import sys, os sys.path.insert(0, os.path.abspath("../../")) + + #Autmatically calls make_cmdlline_overview + from pypy.doc.config import generate + from pypy.config import makerestdoc import py role = makerestdoc.register_config_role(py.path.local()) diff --git a/pypy/doc/test/test_whatsnew.py b/pypy/doc/test/test_whatsnew.py --- a/pypy/doc/test/test_whatsnew.py +++ b/pypy/doc/test/test_whatsnew.py @@ -29,7 +29,7 @@ merge() and \ branch(default)) and \ not branch(default)' % (startrev, endrev) - cmd = r"hg log -R '%s' -r '%s' --template '{branches}\n'" % (path, revset) + cmd = r'hg log -R "%s" -r "%s" --template "{branches}\n"' % (path, revset) out = getoutput(cmd) branches = set(map(str.strip, out.splitlines())) return branches diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -5,11 +5,19 @@ .. this is a revision shortly after release-2.0-beta1 .. startrev: 0e6161a009c6 +.. branch: callback-jit +Callbacks from C are now better JITted + +.. branch: remove-globals-in-jit + .. branch: length-hint Implement __lenght_hint__ according to PEP 424 .. branch: numpypy-longdouble Long double support for numpypy +.. branch: numpypy-real-as-view +Convert real, imag from ufuncs to views. This involves the beginning of +view() functionality .. branch: signatures Improved RPython typing diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -56,11 +56,24 @@ ---------------------------- On Windows, there is no standard place where to download, build and -install third-party libraries. We chose to install them in the parent +install third-party libraries. We recommend installing them in the parent directory of the pypy checkout. For example, if you installed pypy in ``d:\pypy\trunk\`` (This directory contains a README file), the base -directory is ``d:\pypy``. You may choose different values by setting the -INCLUDE, LIB and PATH (for DLLs) +directory is ``d:\pypy``. You must then set the +INCLUDE, LIB and PATH (for DLLs) environment variables appropriately. + +Abridged method (for -Ojit builds using Visual Studio 2008) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Download the versions of all the external packages +from +https://bitbucket.org/pypy/pypy/downloads/local.zip +Then expand it into the base directory (base_dir) and modify your environment to reflect this:: + + set PATH=\bin;%PATH% + set INCLUDE=\include;%INCLUDE% + set LIB=\lib;%LIB% + +Now you should be good to go. Read on for more information. The Boehm garbage collector ~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py --- a/pypy/interpreter/typedef.py +++ b/pypy/interpreter/typedef.py @@ -105,6 +105,13 @@ # typical usage (case 1 is the most common kind of app-level subclasses; # case 2 is the memory-saving kind defined with __slots__). # +# +----------------------------------------------------------------+ +# | NOTE: if withmapdict is enabled, the following doesn't apply! | +# | Map dicts can flexibly allow any slots/__dict__/__weakref__ to | +# | show up only when needed. In particular there is no way with | +# | mapdict to prevent some objects from being weakrefable. | +# +----------------------------------------------------------------+ +# # dict slots del weakrefable # # 1. Y N N Y UserDictWeakref diff --git a/pypy/jit/backend/arm/test/test_gc_integration.py b/pypy/jit/backend/arm/test/test_gc_integration.py --- a/pypy/jit/backend/arm/test/test_gc_integration.py +++ b/pypy/jit/backend/arm/test/test_gc_integration.py @@ -2,25 +2,13 @@ """ Tests for register allocation for common constructs """ -import py -from pypy.jit.metainterp.history import BoxInt, \ - BoxPtr, TreeLoop, TargetToken -from pypy.jit.metainterp.resoperation import rop, ResOperation -from pypy.jit.codewriter import heaptracker -from pypy.jit.backend.llsupport.descr import GcCache -from pypy.jit.backend.llsupport.gc import GcLLDescription +from pypy.jit.metainterp.history import TargetToken +from pypy.jit.backend.llsupport.gc import GcLLDescription, GcLLDescr_boehm from pypy.jit.backend.detect_cpu import getcpuclass from pypy.jit.backend.arm.arch import WORD from pypy.rpython.lltypesystem import lltype, llmemory, rffi -from pypy.rpython.annlowlevel import llhelper -from pypy.rpython.lltypesystem import rclass -from pypy.jit.backend.llsupport.gc import GcLLDescr_framework -from pypy.jit.backend.arm.test.test_regalloc import MockAssembler from pypy.jit.backend.arm.test.test_regalloc import BaseTestRegalloc -from pypy.jit.backend.arm.regalloc import ARMFrameManager, VFPRegisterManager -from pypy.jit.codewriter.effectinfo import EffectInfo -from pypy.jit.backend.arm.regalloc import Regalloc CPU = getcpuclass() @@ -44,23 +32,14 @@ return ['compressed'] + shape[1:] -class MockGcDescr(GcCache): - get_malloc_slowpath_addr = None - write_barrier_descr = None - moving_gc = True +class MockGcDescr(GcLLDescr_boehm): gcrootmap = MockGcRootMap() - def initialize(self): - pass - - _record_constptrs = GcLLDescr_framework._record_constptrs.im_func - rewrite_assembler = GcLLDescr_framework.rewrite_assembler.im_func - class TestRegallocGcIntegration(BaseTestRegalloc): cpu = CPU(None, None) - cpu.gc_ll_descr = MockGcDescr(False) + cpu.gc_ll_descr = MockGcDescr(None, None, None) cpu.setup_once() S = lltype.GcForwardReference() diff --git a/pypy/jit/backend/llgraph/runner.py b/pypy/jit/backend/llgraph/runner.py --- a/pypy/jit/backend/llgraph/runner.py +++ b/pypy/jit/backend/llgraph/runner.py @@ -877,13 +877,16 @@ # assembler_helper_ptr = jd.assembler_helper_adr.ptr # fish try: - return assembler_helper_ptr(pframe, vable) + result = assembler_helper_ptr(pframe, vable) except LLException, lle: assert self.last_exception is None, "exception left behind" self.last_exception = lle # fish op op = self.current_op return op.result and op.result.value + if isinstance(result, float): + result = support.cast_to_floatstorage(result) + return result def execute_same_as(self, _, x): return x diff --git a/pypy/jit/backend/test/calling_convention_test.py b/pypy/jit/backend/test/calling_convention_test.py --- a/pypy/jit/backend/test/calling_convention_test.py +++ b/pypy/jit/backend/test/calling_convention_test.py @@ -105,11 +105,13 @@ ops = '[%s]\n' % arguments ops += '%s\n' % spill_ops ops += 'f99 = call(ConstClass(func_ptr), %s, descr=calldescr)\n' % arguments - ops += 'finish(f99, %s)\n' % arguments + ops += 'i99 = same_as(0)\n' + ops += 'guard_true(i99) [f99, %s]\n' % arguments + ops += 'finish()\n' loop = parse(ops, namespace=locals()) looptoken = JitCellToken() - done_number = self.cpu.get_fail_descr_number(loop.operations[-1].getdescr()) + done_number = self.cpu.get_fail_descr_number(loop.operations[-2].getdescr()) self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) argvals, expected_result = self._prepare_args(args, floats, ints) diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py --- a/pypy/jit/backend/test/runner_test.py +++ b/pypy/jit/backend/test/runner_test.py @@ -1192,7 +1192,8 @@ values.append(longlong.getfloatstorage(r.random())) # looptoken = JitCellToken() - faildescr = BasicFailDescr(42) + guarddescr = BasicFailDescr(42) + faildescr = BasicFailDescr(43) operations = [] retboxes = [] retvalues = [] @@ -1221,9 +1222,13 @@ retboxes.insert(kk, newbox) retvalues.insert(kk, y) # - operations.append( - ResOperation(rop.FINISH, retboxes, None, descr=faildescr) - ) + zero = BoxInt() + operations.extend([ + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=guarddescr), + ResOperation(rop.FINISH, [], None, descr=faildescr) + ]) + operations[-2].setfailargs(retboxes) print inputargs for op in operations: print op @@ -1344,12 +1349,15 @@ targettoken = TargetToken() faildescr1 = BasicFailDescr(1) faildescr2 = BasicFailDescr(2) + faildescr3 = BasicFailDescr(3) operations = [ ResOperation(rop.LABEL, fboxes, None, descr=targettoken), ResOperation(rop.FLOAT_LE, [fboxes[0], constfloat(9.2)], i2), ResOperation(rop.GUARD_TRUE, [i2], None, descr=faildescr1), - ResOperation(rop.FINISH, fboxes, None, descr=faildescr2), + ResOperation(rop.GUARD_FALSE, [i2], None, descr=faildescr2), + ResOperation(rop.FINISH, [], None, descr=faildescr3), ] + operations[-3].setfailargs(fboxes) operations[-2].setfailargs(fboxes) looptoken = JitCellToken() self.cpu.compile_loop(fboxes, operations, looptoken) @@ -1382,6 +1390,7 @@ py.test.skip("requires floats") fboxes = [BoxFloat() for i in range(3)] faildescr1 = BasicFailDescr(100) + faildescr2 = BasicFailDescr(102) loopops = """ [i0,f1, f2] f3 = float_add(f1, f2) @@ -1406,9 +1415,13 @@ assert longlong.getrealfloat(f2) == 0.75 assert longlong.getrealfloat(f3) == 133.0 + zero = BoxInt() bridgeops = [ - ResOperation(rop.FINISH, fboxes, None, descr=faildescr1), + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=faildescr1), + ResOperation(rop.FINISH, [], None, descr=faildescr2), ] + bridgeops[-2].setfailargs(fboxes[:]) self.cpu.compile_bridge(loop.operations[-2].getdescr(), fboxes, bridgeops, looptoken) args = [1, @@ -2002,7 +2015,7 @@ i1 = same_as(1) call(ConstClass(fptr), i0, descr=calldescr) p0 = guard_exception(ConstClass(xtp)) [i1] - finish(0, p0) + finish(p0) ''' FPTR = lltype.Ptr(lltype.FuncType([lltype.Signed], lltype.Void)) fptr = llhelper(FPTR, func) @@ -2023,8 +2036,7 @@ looptoken = JitCellToken() self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) deadframe = self.cpu.execute_token(looptoken, 1) - assert self.cpu.get_latest_value_int(deadframe, 0) == 0 - assert self.cpu.get_latest_value_ref(deadframe, 1) == xptr + assert self.cpu.get_latest_value_ref(deadframe, 0) == xptr excvalue = self.cpu.grab_exc_value(deadframe) assert not excvalue deadframe = self.cpu.execute_token(looptoken, 0) diff --git a/pypy/jit/backend/x86/test/test_recompilation.py b/pypy/jit/backend/x86/test/test_recompilation.py --- a/pypy/jit/backend/x86/test/test_recompilation.py +++ b/pypy/jit/backend/x86/test/test_recompilation.py @@ -45,7 +45,8 @@ force_spill(i5) i8 = int_add(i7, 1) i9 = int_add(i8, 1) - finish(i3, i4, i5, i6, i7, i8, i9, descr=fdescr2) + guard_false(i3, descr=fdescr2) [i3, i4, i5, i6, i7, i8, i9] + finish() ''' bridge = self.attach_bridge(ops, loop, -2) descr = loop.operations[3].getdescr() diff --git a/pypy/jit/backend/x86/test/test_regalloc.py b/pypy/jit/backend/x86/test/test_regalloc.py --- a/pypy/jit/backend/x86/test/test_regalloc.py +++ b/pypy/jit/backend/x86/test/test_regalloc.py @@ -327,7 +327,7 @@ assert self.getint(0) == 0 bridge_ops = ''' [i0, i1] - finish(1, 2) + finish(2) ''' self.attach_bridge(bridge_ops, loop, 0) self.run(loop, 0, 1) @@ -428,7 +428,7 @@ ops = ''' [i0, i1, i2, i3, i4, i5, i6, i7] guard_value(i6, i1) [i0, i2, i3, i4, i5, i6] - finish(i0, i2, i3, i4, i5, i6) + finish(i0) ''' self.interpret(ops, [0, 0, 0, 0, 0, 0, 0, 0]) assert self.getint(0) == 0 @@ -438,14 +438,15 @@ [i0, i1, i2, i3, i4, i5, i6, i7, i8] i9 = same_as(0) guard_true(i0) [i9, i0, i1, i2, i3, i4, i5, i6, i7, i8] - finish(1, i0, i1, i2, i3, i4, i5, i6, i7, i8) + finish(1) ''' loop = self.interpret(ops, [0, 1, 2, 3, 4, 5, 6, 7, 8]) assert self.getint(0) == 0 bridge_ops = ''' [i9, i0, i1, i2, i3, i4, i5, i6, i7, i8] call(ConstClass(raising_fptr), 0, descr=raising_calldescr) - finish(i0, i1, i2, i3, i4, i5, i6, i7, i8) + guard_true(i9) [i0, i1, i2, i3, i4, i5, i6, i7, i8] + finish() ''' self.attach_bridge(bridge_ops, loop, 1) self.run(loop, 0, 1, 2, 3, 4, 5, 6, 7, 8) @@ -474,7 +475,8 @@ i1 = same_as(1) i2 = int_lt(i0, 100) guard_true(i3) [i1, i2] - finish(0, i2) + i4 = int_neg(i2) + finish(0) ''' self.interpret(ops, [0, 1]) assert self.getint(0) == 0 @@ -503,7 +505,8 @@ i15 = int_is_true(i5) i16 = int_is_true(i6) i17 = int_is_true(i7) - finish(i10, i11, i12, i13, i14, i15, i16, i17) + guard_true(i0) [i10, i11, i12, i13, i14, i15, i16, i17] + finish() ''' self.interpret(ops, [0, 42, 12, 0, 13, 0, 0, 3333]) assert self.getints(8) == [0, 1, 1, 0, 1, 0, 0, 1] @@ -517,7 +520,8 @@ i13 = int_eq(i5, i6) i14 = int_gt(i6, i2) i15 = int_ne(i2, i6) - finish(i10, i11, i12, i13, i14, i15) + guard_true(i0) [i10, i11, i12, i13, i14, i15] + finish() ''' self.interpret(ops, [0, 1, 2, 3, 4, 5, 6]) assert self.getints(6) == [1, 1, 0, 0, 1, 1] @@ -574,7 +578,9 @@ ops = ''' [f0, f1] f2 = float_add(f0, f1) - finish(f2, f0, f1) + i0 = same_as(0) + guard_true(i0) [f2, f0, f1] + finish() ''' self.interpret(ops, [3.0, 1.5]) assert self.getfloats(3) == [4.5, 3.0, 1.5] @@ -584,7 +590,9 @@ [f0, f1, f2, f3, f4, f5, f6, f7, f8] f9 = float_add(f0, f1) f10 = float_add(f8, 3.5) - finish(f9, f10, f2, f3, f4, f5, f6, f7, f8) + i0 = same_as(0) + guard_true(i0) [f9, f10, f2, f3, f4, f5, f6, f7, f8] + finish() ''' self.interpret(ops, [0.1, .2, .3, .4, .5, .6, .7, .8, .9]) assert self.getfloats(9) == [.1+.2, .9+3.5, .3, .4, .5, .6, .7, .8, .9] @@ -613,7 +621,8 @@ i7 = float_ne(f7, 0.0) i8 = float_ne(f8, 0.0) i9 = float_ne(f9, 0.0) - finish(i0, i1, i2, i3, i4, i5, i6, i7, i8, i9) + guard_true(i0) [i0, i1, i2, i3, i4, i5, i6, i7, i8, i9] + finish() ''' loop = self.interpret(ops, [0.0, .1, .2, .3, .4, .5, .6, .7, .8, .9]) assert self.getints(9) == [0, 1, 1, 1, 1, 1, 1, 1, 1] diff --git a/pypy/jit/backend/x86/test/test_regalloc2.py b/pypy/jit/backend/x86/test/test_regalloc2.py --- a/pypy/jit/backend/x86/test/test_regalloc2.py +++ b/pypy/jit/backend/x86/test/test_regalloc2.py @@ -11,13 +11,17 @@ v2 = BoxInt() v3 = BoxInt() v4 = BoxInt() + zero = BoxInt() inputargs = [v1] operations = [ ResOperation(rop.INT_ADD, [v1, v1], v2), ResOperation(rop.INT_INVERT, [v2], v3), ResOperation(rop.UINT_RSHIFT, [v1, ConstInt(3)], v4), - ResOperation(rop.FINISH, [v4, v3], None, descr=BasicFailDescr()), + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=BasicFailDescr()), + ResOperation(rop.FINISH, [], None, descr=BasicFailDescr()) ] + operations[-2].setfailargs([v4, v3]) cpu = CPU(None, None) cpu.setup_once() looptoken = JitCellToken() @@ -31,6 +35,7 @@ v2 = BoxInt() v3 = BoxInt() v4 = BoxInt() + zero = BoxInt() tmp5 = BoxInt() inputargs = [v1] operations = [ @@ -38,8 +43,11 @@ ResOperation(rop.INT_MUL, [v2, v1], v3), ResOperation(rop.INT_IS_TRUE, [v2], tmp5), ResOperation(rop.INT_IS_ZERO, [tmp5], v4), - ResOperation(rop.FINISH, [v4, v3, tmp5], None, descr=BasicFailDescr()), + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=BasicFailDescr()), + ResOperation(rop.FINISH, [], None, descr=BasicFailDescr()) ] + operations[-2].setfailargs([v4, v3, tmp5]) cpu = CPU(None, None) cpu.setup_once() looptoken = JitCellToken() @@ -90,6 +98,7 @@ v38 = BoxInt() v39 = BoxInt() v40 = BoxInt() + zero = BoxInt() tmp41 = BoxInt() tmp42 = BoxInt() tmp43 = BoxInt() @@ -134,8 +143,12 @@ ResOperation(rop.UINT_GT, [v33, ConstInt(-11)], v38), ResOperation(rop.INT_NEG, [v7], v39), ResOperation(rop.INT_GT, [v24, v32], v40), - ResOperation(rop.FINISH, [v40, v36, v37, v31, v16, v34, v35, v23, v22, v29, v14, v39, v30, v38], None, descr=BasicFailDescr()), + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=BasicFailDescr()), + ResOperation(rop.FINISH, [], None, descr=BasicFailDescr()) ] + operations[-2].setfailargs([v40, v36, v37, v31, v16, v34, v35, v23, + v22, v29, v14, v39, v30, v38]) cpu = CPU(None, None) cpu.setup_once() looptoken = JitCellToken() @@ -198,6 +211,7 @@ v38 = BoxInt() v39 = BoxInt() v40 = BoxInt() + zero = BoxInt() tmp41 = BoxInt() tmp42 = BoxInt() tmp43 = BoxInt() @@ -240,8 +254,13 @@ ResOperation(rop.INT_GT, [v4, v11], v38), ResOperation(rop.INT_LT, [v27, v22], v39), ResOperation(rop.INT_NEG, [v27], v40), - ResOperation(rop.FINISH, [v40, v10, v36, v26, v13, v30, v21, v33, v18, v25, v31, v32, v28, v29, v35, v38, v20, v39, v34, v23, v37], None, descr=BasicFailDescr()), + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=BasicFailDescr()), + ResOperation(rop.FINISH, [], None, descr=BasicFailDescr()) ] + operations[-2].setfailargs([v40, v10, v36, v26, v13, v30, v21, v33, + v18, v25, v31, v32, v28, v29, v35, v38, + v20, v39, v34, v23, v37]) cpu = CPU(None, None) cpu.setup_once() looptoken = JitCellToken() diff --git a/pypy/jit/backend/x86/test/test_runner.py b/pypy/jit/backend/x86/test/test_runner.py --- a/pypy/jit/backend/x86/test/test_runner.py +++ b/pypy/jit/backend/x86/test/test_runner.py @@ -499,9 +499,12 @@ i6, descr=calldescr), ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr), - ResOperation(rop.FINISH, [i3, i4, i5, i6], None, - descr=BasicFailDescr(0)) + ResOperation(rop.GUARD_FALSE, [i3], None, + descr=BasicFailDescr(0)), + ResOperation(rop.FINISH, [], None, + descr=BasicFailDescr(1)) ] + ops[-2].setfailargs([i3, i4, i5, i6]) ops[1].setfailargs([]) ops[3].setfailargs([]) ops[5].setfailargs([]) diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py --- a/pypy/jit/codewriter/support.py +++ b/pypy/jit/codewriter/support.py @@ -10,7 +10,7 @@ from pypy.rpython.extregistry import ExtRegistryEntry from pypy.translator.simplify import get_funcobj from pypy.translator.unsimplify import split_block -from pypy.objspace.flow.model import Constant +from pypy.objspace.flow.model import Variable, Constant from pypy.translator.translator import TranslationContext from pypy.annotation.policy import AnnotatorPolicy from pypy.annotation import model as annmodel @@ -59,6 +59,45 @@ rtyper = annotate(func, values) return rtyper.annotator.translator.graphs[0] +def autodetect_jit_markers_redvars(graph): + # the idea is to find all the jit_merge_point and + # add all the variables across the links to the reds. + for block, op in graph.iterblockops(): + if op.opname == 'jit_marker': + jitdriver = op.args[1].value + if not jitdriver.autoreds: + continue + # if we want to support also can_enter_jit, we should find a + # way to detect a consistent set of red vars to pass *both* to + # jit_merge_point and can_enter_jit. The current simple + # solution doesn't work because can_enter_jit might be in + # another block, so the set of alive_v will be different. + methname = op.args[0].value + assert methname == 'jit_merge_point', ( + "reds='auto' is supported only for jit drivers which " + "calls only jit_merge_point. Found a call to %s" % methname) + # + # compute the set of live variables across the jit_marker + alive_v = set() + for link in block.exits: + alive_v.update(link.args) + alive_v.difference_update(link.getextravars()) + for op1 in block.operations[::-1]: + if op1 is op: + break # stop when the meet the jit_marker + alive_v.discard(op1.result) + alive_v.update(op1.args) + greens_v = op.args[2:] + reds_v = alive_v - set(greens_v) + reds_v = [v for v in reds_v if isinstance(v, Variable) and + v.concretetype is not lltype.Void] + reds_v = sort_vars(reds_v) + op.args.extend(reds_v) + if jitdriver.numreds is None: + jitdriver.numreds = len(reds_v) + else: + assert jitdriver.numreds == len(reds_v), 'inconsistent number of reds_v' + def split_before_jit_merge_point(graph, portalblock, portalopindex): """Split the block just before the 'jit_merge_point', making sure the input args are in the canonical order. diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py --- a/pypy/jit/metainterp/pyjitpl.py +++ b/pypy/jit/metainterp/pyjitpl.py @@ -1379,16 +1379,15 @@ """The 'residual_call' operation is emitted in two cases: when we have to generate a residual CALL operation, but also to handle an indirect_call that may need to be inlined.""" - assert isinstance(funcbox, Const) - sd = self.metainterp.staticdata - key = sd.cpu.ts.getaddr_for_box(funcbox) - jitcode = sd.bytecode_for_address(key) - if jitcode is not None: - # we should follow calls to this graph - return self.metainterp.perform_call(jitcode, argboxes) - else: - # but we should not follow calls to that graph - return self.do_residual_call(funcbox, argboxes, calldescr) + if isinstance(funcbox, Const): + sd = self.metainterp.staticdata + key = sd.cpu.ts.getaddr_for_box(funcbox) + jitcode = sd.bytecode_for_address(key) + if jitcode is not None: + # we should follow calls to this graph + return self.metainterp.perform_call(jitcode, argboxes) + # but we should not follow calls to that graph + return self.do_residual_call(funcbox, argboxes, calldescr) # ____________________________________________________________ diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py --- a/pypy/jit/metainterp/resoperation.py +++ b/pypy/jit/metainterp/resoperation.py @@ -349,6 +349,9 @@ def initarglist(self, args): self._args = args + if not we_are_translated() and \ + self.__class__.__name__.startswith('FINISH'): # XXX remove me + assert len(args) <= 1 # FINISH operations take 0 or 1 arg now def getarglist(self): return self._args diff --git a/pypy/jit/metainterp/test/test_ajit.py b/pypy/jit/metainterp/test/test_ajit.py --- a/pypy/jit/metainterp/test/test_ajit.py +++ b/pypy/jit/metainterp/test/test_ajit.py @@ -3979,6 +3979,8 @@ rgc.add_memory_pressure(1234) return 3 + self.interp_operations(f, []) + def test_external_call(self): from pypy.rlib.objectmodel import invoke_around_extcall diff --git a/pypy/jit/metainterp/test/test_call.py b/pypy/jit/metainterp/test/test_call.py new file mode 100644 --- /dev/null +++ b/pypy/jit/metainterp/test/test_call.py @@ -0,0 +1,27 @@ + +from pypy.jit.metainterp.test.support import LLJitMixin +from pypy.rlib import jit + +class TestCall(LLJitMixin): + def test_indirect_call(self): + @jit.dont_look_inside + def f1(x): + return x + 1 + + @jit.dont_look_inside + def f2(x): + return x + 2 + + @jit.dont_look_inside + def choice(i): + if i: + return f1 + return f2 + + def f(i): + func = choice(i) + return func(i) + + res = self.interp_operations(f, [3]) + assert res == f(3) + diff --git a/pypy/jit/metainterp/test/test_fficall.py b/pypy/jit/metainterp/test/test_fficall.py --- a/pypy/jit/metainterp/test/test_fficall.py +++ b/pypy/jit/metainterp/test/test_fficall.py @@ -1,9 +1,11 @@ -import py + +import ctypes, math from pypy.rpython.lltypesystem import lltype, rffi from pypy.jit.metainterp.test.support import LLJitMixin from pypy.rlib import jit from pypy.rlib.jit_libffi import types, CIF_DESCRIPTION, FFI_TYPE_PP from pypy.rlib.unroll import unrolling_iterable +from pypy.rlib.rarithmetic import intmask def get_description(atypes, rtype): @@ -103,4 +105,40 @@ class TestFfiCall(FfiCallTests, LLJitMixin): - pass + def test_jit_fii_vref(self): + from pypy.rlib import clibffi + from pypy.rlib.jit_libffi import jit_ffi_prep_cif, jit_ffi_call + + math_sin = intmask(ctypes.cast(ctypes.CDLL(None).sin, + ctypes.c_void_p).value) + math_sin = rffi.cast(rffi.VOIDP, math_sin) + + cd = lltype.malloc(CIF_DESCRIPTION, 1, flavor='raw') + cd.abi = clibffi.FFI_DEFAULT_ABI + cd.nargs = 1 + cd.rtype = clibffi.cast_type_to_ffitype(rffi.DOUBLE) + atypes = lltype.malloc(clibffi.FFI_TYPE_PP.TO, 1, flavor='raw') + atypes[0] = clibffi.cast_type_to_ffitype(rffi.DOUBLE) + cd.atypes = atypes + cd.exchange_size = 64 # 64 bytes of exchange data + cd.exchange_result = 24 + cd.exchange_result_libffi = 24 + cd.exchange_args[0] = 16 + + def f(): + # + jit_ffi_prep_cif(cd) + # + assert rffi.sizeof(rffi.DOUBLE) == 8 + exb = lltype.malloc(rffi.DOUBLEP.TO, 8, flavor='raw') + exb[2] = 1.23 + jit_ffi_call(cd, math_sin, rffi.cast(rffi.CCHARP, exb)) + res = exb[3] + lltype.free(exb, flavor='raw') + # + lltype.free(atypes, flavor='raw') + return res + # + res = self.interp_operations(f, []) + lltype.free(cd, flavor='raw') + assert res == math.sin(1.23) diff --git a/pypy/jit/metainterp/test/test_warmspot.py b/pypy/jit/metainterp/test/test_warmspot.py --- a/pypy/jit/metainterp/test/test_warmspot.py +++ b/pypy/jit/metainterp/test/test_warmspot.py @@ -383,6 +383,23 @@ assert res == expected self.check_resops(int_sub=2, int_mul=0, int_add=2) + def test_loop_automatic_reds_not_too_many_redvars(self): + myjitdriver = JitDriver(greens = ['m'], reds = 'auto') + def one(): + return 1 + def f(n, m): + res = 0 + while n > 0: + n -= one() + myjitdriver.jit_merge_point(m=m) + res += m*2 + return res + expected = f(21, 5) + res = self.meta_interp(f, [21, 5]) + assert res == expected + oplabel = get_stats().loops[0].operations[0] + assert len(oplabel.getarglist()) == 2 # 'n', 'res' in some order + def test_inline_jit_merge_point(self): # test that the machinery to inline jit_merge_points in callers # works. The final user does not need to mess manually with the @@ -519,6 +536,44 @@ self.check_trace_count(1) + def test_callback_jit_merge_point(self): + from pypy.rlib.objectmodel import register_around_callback_hook + from pypy.rpython.lltypesystem import lltype, rffi + from pypy.translator.tool.cbuild import ExternalCompilationInfo + + callback_jit_driver = JitDriver(greens = ['name'], reds = 'auto') + + def callback_merge_point(name): + callback_jit_driver.jit_merge_point(name=name) + + @callback_jit_driver.inline(callback_merge_point) + def callback_hook(name): + pass + + def callback(a, b): + if a > b: + return 1 + return -1 + + CB_TP = rffi.CCallback([lltype.Signed, lltype.Signed], lltype.Signed) + eci = ExternalCompilationInfo(includes=['stdlib.h']) + qsort = rffi.llexternal('qsort', + [rffi.VOIDP, lltype.Signed, lltype.Signed, + CB_TP], lltype.Void, compilation_info=eci) + ARR = rffi.CArray(lltype.Signed) + + def main(): + register_around_callback_hook(callback_hook) + raw = lltype.malloc(ARR, 10, flavor='raw') + for i in range(10): + raw[i] = 10 - i + qsort(raw, 10, rffi.sizeof(lltype.Signed), callback) + lltype.free(raw, flavor='raw') + + self.meta_interp(main, []) + self.check_trace_count(1) + + class TestLLWarmspot(WarmspotTests, LLJitMixin): CPUClass = runner.LLGraphCPU type_system = 'lltype' diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py --- a/pypy/jit/metainterp/warmspot.py +++ b/pypy/jit/metainterp/warmspot.py @@ -290,11 +290,13 @@ callgraph = inlinable_static_callers(self.translator.graphs, store_calls=True) new_callgraph = [] new_portals = set() + inlined_jit_merge_points = set() for caller, block, op_call, callee in callgraph: func = getattr(callee, 'func', None) _inline_jit_merge_point_ = getattr(func, '_inline_jit_merge_point_', None) if _inline_jit_merge_point_: _inline_jit_merge_point_._always_inline_ = True + inlined_jit_merge_points.add(_inline_jit_merge_point_) op_jmp_call, jmp_graph = get_jmp_call(callee, _inline_jit_merge_point_) # # now we move the op_jmp_call from callee to caller, just @@ -315,6 +317,9 @@ # inline them! inline_threshold = 0.1 # we rely on the _always_inline_ set above auto_inlining(self.translator, inline_threshold, new_callgraph) + # clean up _always_inline_ = True, it can explode later + for item in inlined_jit_merge_points: + del item._always_inline_ # make a fresh copy of the JitDriver in all newly created # jit_merge_points @@ -342,49 +347,13 @@ self.jitdrivers_sd = [] graphs = self.translator.graphs for graph, block, pos in find_jit_merge_points(graphs): - self.autodetect_jit_markers_redvars(graph) + support.autodetect_jit_markers_redvars(graph) self.split_graph_and_record_jitdriver(graph, block, pos) # assert (len(set([jd.jitdriver for jd in self.jitdrivers_sd])) == len(self.jitdrivers_sd)), \ "there are multiple jit_merge_points with the same jitdriver" - def autodetect_jit_markers_redvars(self, graph): - # the idea is to find all the jit_merge_point and can_enter_jit and - # add all the variables across the links to the reds. - for block, op in graph.iterblockops(): - if op.opname == 'jit_marker': - jitdriver = op.args[1].value - if not jitdriver.autoreds: - continue - # if we want to support also can_enter_jit, we should find a - # way to detect a consistent set of red vars to pass *both* to - # jit_merge_point and can_enter_jit. The current simple - # solution doesn't work because can_enter_jit might be in - # another block, so the set of alive_v will be different. - methname = op.args[0].value - assert methname == 'jit_merge_point', ( - "reds='auto' is supported only for jit drivers which " - "calls only jit_merge_point. Found a call to %s" % methname) - # - # compute the set of live variables before the jit_marker - alive_v = set(block.inputargs) - for op1 in block.operations: - if op1 is op: - break # stop when the meet the jit_marker - if op1.result.concretetype != lltype.Void: - alive_v.add(op1.result) - greens_v = op.args[2:] - reds_v = alive_v - set(greens_v) - reds_v = [v for v in reds_v if v.concretetype is not lltype.Void] - reds_v = support.sort_vars(reds_v) - op.args.extend(reds_v) - if jitdriver.numreds is None: - jitdriver.numreds = len(reds_v) - else: - assert jitdriver.numreds == len(reds_v), 'inconsistent number of reds_v' - - def split_graph_and_record_jitdriver(self, graph, block, pos): op = block.operations[pos] jd = JitDriverStaticData() @@ -712,6 +681,9 @@ jitdriver = op.args[1].value assert jitdriver in sublists, \ "can_enter_jit with no matching jit_merge_point" + assert not jitdriver.autoreds, ( + "can_enter_jit not supported with a jitdriver that " + "has reds='auto'") jd, sublist = sublists[jitdriver] origportalgraph = jd._jit_merge_point_in if graph is not origportalgraph: @@ -1011,6 +983,9 @@ origblock.operations.append(newop) origblock.exitswitch = None origblock.recloseblock(Link([v_result], origportalgraph.returnblock)) + # the origportal now can raise (even if it did not raise before), + # which means that we cannot inline it anywhere any more, but that's + # fine since any forced inlining has been done before # checkgraph(origportalgraph) diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py --- a/pypy/module/_cffi_backend/ccallback.py +++ b/pypy/module/_cffi_backend/ccallback.py @@ -3,10 +3,10 @@ """ import os from pypy.interpreter.error import OperationError, operationerrfmt -from pypy.rpython.lltypesystem import lltype, llmemory, rffi +from pypy.rpython.lltypesystem import lltype, rffi from pypy.rlib.objectmodel import compute_unique_id, keepalive_until_here -from pypy.rlib import clibffi, rweakref, rgc -from pypy.rlib.rarithmetic import r_ulonglong +from pypy.rlib import clibffi, rweakref +from pypy.rlib import jit from pypy.module._cffi_backend.cdataobj import W_CData from pypy.module._cffi_backend.ctypefunc import SIZE_OF_FFI_ARG, BIG_ENDIAN @@ -77,6 +77,7 @@ space.wrap("expected a function ctype")) return ctype + @jit.unroll_safe def invoke(self, ll_args): space = self.space ctype = self.getfunctype() diff --git a/pypy/module/_weakref/test/test_weakref.py b/pypy/module/_weakref/test/test_weakref.py --- a/pypy/module/_weakref/test/test_weakref.py +++ b/pypy/module/_weakref/test/test_weakref.py @@ -114,19 +114,28 @@ class A(object): def __eq__(self, other): return True + def __ne__(self, other): + return False a1 = A() a2 = A() ref1 = _weakref.ref(a1) ref2 = _weakref.ref(a2) assert ref1 == ref2 + assert not (ref1 != ref2) + assert not (ref1 == []) + assert ref1 != [] del a1 gc.collect() assert not ref1 == ref2 assert ref1 != ref2 + assert not (ref1 == []) + assert ref1 != [] del a2 gc.collect() assert not ref1 == ref2 assert ref1 != ref2 + assert not (ref1 == []) + assert ref1 != [] def test_getweakrefs(self): import _weakref, gc @@ -298,6 +307,13 @@ if seen_callback: assert seen_callback == [True, True, True] + def test_type_weakrefable(self): + import _weakref, gc + w = _weakref.ref(list) + assert w() is list + gc.collect() + assert w() is list + class AppTestProxy(object): spaceconfig = dict(usemodules=('_weakref',)) @@ -435,6 +451,8 @@ class A(object): def __eq__(self, other): return True + def __ne__(self, other): + return False a = A() assert _weakref.ref(a) == a diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py --- a/pypy/module/micronumpy/arrayimpl/concrete.py +++ b/pypy/module/micronumpy/arrayimpl/concrete.py @@ -13,6 +13,7 @@ from pypy.rlib.debug import make_sure_not_resized class ConcreteArrayIterator(base.BaseArrayIterator): + _immutable_fields_ = ['dtype', 'skip', 'size'] def __init__(self, array): self.array = array self.offset = 0 @@ -21,10 +22,10 @@ self.size = array.size def setitem(self, elem): - self.array.setitem(self.offset, elem) + self.dtype.setitem(self.array, self.offset, elem) def getitem(self): - return self.array.getitem(self.offset) + return self.dtype.getitem(self.array, self.offset) def getitem_bool(self): return self.dtype.getitem_bool(self.array, self.offset) @@ -42,13 +43,16 @@ self.offset %= self.size class OneDimViewIterator(ConcreteArrayIterator): - def __init__(self, array): + ''' The view iterator dtype can be different from the + array.dtype, this is what makes it a View + ''' + def __init__(self, array, dtype, start, strides, shape): self.array = array - self.offset = array.start - self.skip = array.get_strides()[0] - self.dtype = array.dtype + self.dtype = dtype + self.offset = start + self.skip = strides[0] self.index = 0 - self.size = array.get_shape()[0] + self.size = shape[0] def next(self): self.offset += self.skip @@ -65,9 +69,13 @@ self.offset %= self.size class MultiDimViewIterator(ConcreteArrayIterator): - def __init__(self, array, start, strides, backstrides, shape): + ''' The view iterator dtype can be different from the + array.dtype, this is what makes it a View + ''' + def __init__(self, array, dtype, start, strides, backstrides, shape): self.indexes = [0] * len(shape) self.array = array + self.dtype = dtype self.shape = shape self.offset = start self.shapelen = len(shape) @@ -131,12 +139,13 @@ self.offset = array.start self.dim = dim self.array = array + self.dtype = array.dtype def setitem(self, elem): - self.array.setitem(self.offset, elem) + self.dtype.setitem(self.array, self.offset, elem) def getitem(self): - return self.array.getitem(self.offset) + return self.dtype.getitem(self.array, self.offset) @jit.unroll_safe def next(self): @@ -220,6 +229,32 @@ new_shape, self) else: return None + + def get_real(self): + strides = self.get_strides() + backstrides = self.get_backstrides() + if self.dtype.is_complex_type(): + dtype = self.dtype.float_type + return SliceArray(self.start, strides, backstrides, + self.get_shape(), self, dtype=dtype) + return SliceArray(self.start, strides, backstrides, + self.get_shape(), self) + + def get_imag(self): + strides = self.get_strides() + backstrides = self.get_backstrides() + if self.dtype.is_complex_type(): + dtype = self.dtype.float_type + return SliceArray(self.start + dtype.get_size(), strides, + backstrides, self.get_shape(), self, dtype=dtype) + if self.dtype.is_flexible_type(): + # numpy returns self for self.imag + return SliceArray(self.start, strides, backstrides, + self.get_shape(), self) + impl = NonWritableArray(self.get_shape(), self.dtype, self.order, strides, + backstrides) + impl.fill(self.dtype.box(0)) + return impl # -------------------- applevel get/setitem ----------------------- @@ -377,7 +412,7 @@ def create_dot_iter(self, shape, skip): r = calculate_dot_strides(self.get_strides(), self.get_backstrides(), shape, skip) - return MultiDimViewIterator(self, self.start, r[0], r[1], shape) + return MultiDimViewIterator(self, self.dtype, self.start, r[0], r[1], shape) def swapaxes(self, axis1, axis2): shape = self.get_shape()[:] @@ -411,7 +446,7 @@ r = calculate_broadcast_strides(self.get_strides(), self.get_backstrides(), self.get_shape(), shape) - return MultiDimViewIterator(self, 0, r[0], r[1], shape) + return MultiDimViewIterator(self, self.dtype, 0, r[0], r[1], shape) def fill(self, box): self.dtype.fill(self.storage, box, 0, self.size) @@ -424,6 +459,12 @@ self.order) return SliceArray(0, strides, backstrides, new_shape, self) +class NonWritableArray(ConcreteArray): + def descr_setitem(self, space, w_index, w_value): + raise OperationError(space.w_RuntimeError, space.wrap( + "array is not writable")) + + class SliceArray(BaseConcreteArray): def __init__(self, start, strides, backstrides, shape, parent, dtype=None): self.strides = strides @@ -448,11 +489,12 @@ r = calculate_broadcast_strides(self.get_strides(), self.get_backstrides(), self.get_shape(), shape) - return MultiDimViewIterator(self.parent, + return MultiDimViewIterator(self.parent, self.dtype, self.start, r[0], r[1], shape) if len(self.get_shape()) == 1: - return OneDimViewIterator(self) - return MultiDimViewIterator(self.parent, self.start, + return OneDimViewIterator(self.parent, self.dtype, self.start, + self.get_strides(), self.get_shape()) + return MultiDimViewIterator(self.parent, self.dtype, self.start, self.get_strides(), self.get_backstrides(), self.get_shape()) diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -136,7 +136,7 @@ return self.kind == SIGNEDLTR def is_complex_type(self): - return (self.num == 14 or self.num == 15 or self.num == 16) + return False def is_bool_type(self): return self.kind == BOOLLTR @@ -155,6 +155,19 @@ def get_size(self): return self.itemtype.get_element_size() +class W_ComplexDtype(W_Dtype): + + def __init__(self, itemtype, num, kind, name, char, w_box_type, + alternate_constructors=[], aliases=[], + fields=None, fieldnames=None, native=True, float_type=None): + W_Dtype.__init__(self, itemtype, num, kind, name, char, w_box_type, + alternate_constructors=alternate_constructors, aliases=aliases, + fields=fields, fieldnames=fieldnames, native=native) + self.float_type = float_type + + def is_complex_type(self): + return True + def dtype_from_list(space, w_lst): lst_w = space.listview(w_lst) fields = {} @@ -413,15 +426,16 @@ alternate_constructors=[space.w_float], aliases=["float"], ) - self.w_complex64dtype = W_Dtype( + self.w_complex64dtype = W_ComplexDtype( types.Complex64(), num=14, kind=COMPLEXLTR, name="complex64", char="F", w_box_type = space.gettypefor(interp_boxes.W_Complex64Box), + float_type = self.w_float32dtype, ) - self.w_complex128dtype = W_Dtype( + self.w_complex128dtype = W_ComplexDtype( types.Complex128(), num=15, kind=COMPLEXLTR, @@ -430,6 +444,7 @@ w_box_type = space.gettypefor(interp_boxes.W_Complex128Box), alternate_constructors=[space.w_complex], aliases=["complex"], + float_type = self.w_float64dtype, ) if interp_boxes.long_double_size == 12: self.w_float96dtype = W_Dtype( @@ -443,7 +458,7 @@ ) self.w_longdouble = self.w_float96dtype - self.w_complex192dtype = W_Dtype( + self.w_complex192dtype = W_ComplexDtype( types.Complex192(), num=16, kind=COMPLEXLTR, @@ -452,6 +467,7 @@ w_box_type = space.gettypefor(interp_boxes.W_Complex192Box), alternate_constructors=[space.w_complex], aliases=["clongdouble", "clongfloat"], + float_type = self.w_float96dtype, ) self.w_clongdouble = self.w_complex192dtype @@ -467,7 +483,7 @@ ) self.w_longdouble = self.w_float128dtype - self.w_complex256dtype = W_Dtype( + self.w_complex256dtype = W_ComplexDtype( types.Complex256(), num=16, kind=COMPLEXLTR, @@ -476,6 +492,7 @@ w_box_type = space.gettypefor(interp_boxes.W_Complex256Box), alternate_constructors=[space.w_complex], aliases=["clongdouble", "clongfloat"], + float_type = self.w_float128dtype, ) self.w_clongdouble = self.w_complex256dtype else: diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -235,6 +235,29 @@ def descr_copy(self, space): return W_NDimArray(self.implementation.copy()) + def descr_get_real(self, space): + return W_NDimArray(self.implementation.get_real()) + + def descr_get_imag(self, space): + ret = self.implementation.get_imag() + if ret: + return W_NDimArray(ret) + raise OperationError(space.w_NotImplementedError, + space.wrap('imag not implemented for this dtype')) + + def descr_set_real(self, space, w_value): + # copy (broadcast) values into self + tmp = self.implementation.get_real() + tmp.setslice(space, convert_to_array(space, w_value)) + + def descr_set_imag(self, space, w_value): + # if possible, copy (broadcast) values into self + if not self.get_dtype().is_complex_type(): + raise OperationError(space.w_TypeError, + space.wrap('array does not have imaginary part to set')) + tmp = self.implementation.get_imag() + tmp.setslice(space, convert_to_array(space, w_value)) + def descr_reshape(self, space, args_w): """reshape(...) a.reshape(shape) @@ -387,8 +410,6 @@ descr_neg = _unaryop_impl("negative") descr_abs = _unaryop_impl("absolute") descr_invert = _unaryop_impl("invert") - descr_get_real = _unaryop_impl("real") - descr_get_imag = _unaryop_impl("imag") def descr_nonzero(self, space): if self.get_size() > 1: @@ -635,9 +656,10 @@ swapaxes = interp2app(W_NDimArray.descr_swapaxes), flat = GetSetProperty(W_NDimArray.descr_get_flatiter), item = interp2app(W_NDimArray.descr_item), - real = GetSetProperty(W_NDimArray.descr_get_real), - imag = GetSetProperty(W_NDimArray.descr_get_imag), - + real = GetSetProperty(W_NDimArray.descr_get_real, + W_NDimArray.descr_set_real), + imag = GetSetProperty(W_NDimArray.descr_get_imag, + W_NDimArray.descr_set_imag), __array_interface__ = GetSetProperty(W_NDimArray.descr_array_iface), ) diff --git a/pypy/module/micronumpy/test/test_iter.py b/pypy/module/micronumpy/test/test_iter.py --- a/pypy/module/micronumpy/test/test_iter.py +++ b/pypy/module/micronumpy/test/test_iter.py @@ -12,7 +12,7 @@ strides = [5, 1] backstrides = [x * (y - 1) for x,y in zip(strides, shape)] assert backstrides == [10, 4] - i = MultiDimViewIterator(MockArray, start, strides, backstrides, shape) + i = MultiDimViewIterator(MockArray, None, start, strides, backstrides, shape) i.next() i.next() i.next() @@ -30,7 +30,7 @@ strides = [1, 3] backstrides = [x * (y - 1) for x,y in zip(strides, shape)] assert backstrides == [2, 12] - i = MultiDimViewIterator(MockArray, start, strides, backstrides, shape) + i = MultiDimViewIterator(MockArray, None, start, strides, backstrides, shape) i.next() i.next() i.next() @@ -51,7 +51,7 @@ strides = [5, 1] backstrides = [x * (y - 1) for x,y in zip(strides, shape)] assert backstrides == [10, 4] - i = MultiDimViewIterator(MockArray, start, strides, backstrides, shape) + i = MultiDimViewIterator(MockArray, None, start, strides, backstrides, shape) i.next_skip_x(2) i.next_skip_x(2) i.next_skip_x(2) @@ -74,7 +74,7 @@ strides = [1, 3] backstrides = [x * (y - 1) for x,y in zip(strides, shape)] assert backstrides == [2, 12] - i = MultiDimViewIterator(MockArray, start, strides, backstrides, shape) + i = MultiDimViewIterator(MockArray, None, start, strides, backstrides, shape) i.next_skip_x(2) i.next_skip_x(2) i.next_skip_x(2) diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1169,8 +1169,8 @@ assert (d == [[1, 0, 0], [0, 1, 0], [0, 0, 1]]).all() def test_eye(self): - from _numpypy import eye, array - from _numpypy import int32, float64, dtype + from _numpypy import eye + from _numpypy import int32, dtype a = eye(0) assert len(a) == 0 assert a.dtype == dtype('float64') @@ -1345,6 +1345,42 @@ assert b[0] == 3 assert b[1] == 2 + def test_realimag_views(self): + from _numpypy import arange, array + a = arange(15) + b = a.real + b[5]=50 + assert a[5] == 50 + b = a.imag + assert b[7] == 0 + raises(RuntimeError, 'b[7] = -2') + raises(TypeError, 'a.imag = -2') + a = array(['abc','def'],dtype='S3') + b = a.real + assert a[0] == b[0] + assert a[1] == b[1] + b[1] = 'xyz' + assert a[1] == 'xyz' + assert a.imag[0] == 'abc' + raises(TypeError, 'a.imag = "qop"') + a=array([[1+1j, 2-3j, 4+5j],[-6+7j, 8-9j, -2-1j]]) + assert a.real[0,1] == 2 + a.real[0,1] = -20 + assert a[0,1].real == -20 + b = a.imag + assert b[1,2] == -1 + b[1,2] = 30 + assert a[1,2].imag == 30 + a.real = 13 + assert a[1,1].real == 13 + a=array([1+1j, 2-3j, 4+5j, -6+7j, 8-9j, -2-1j]) + a.real = 13 + assert a[3].real == 13 + a.imag = -5 + a.imag[3] = -10 + assert a[3].imag == -10 + assert a[2].imag == -5 + def test_tolist_scalar(self): from _numpypy import int32, bool_ x = int32(23) @@ -2273,7 +2309,7 @@ assert a[0]['x'] == 'a' def test_stringarray(self): - from _numpypy import array, flexible + from _numpypy import array a = array(['abc'],'S3') assert str(a.dtype) == '|S3' a = array(['abc']) @@ -2293,7 +2329,6 @@ def test_flexible_repr(self): # import overrides str(), repr() for array - from numpypy.core import arrayprint from _numpypy import array a = array(['abc'],'S3') s = repr(a) diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py --- a/pypy/module/pyexpat/interp_pyexpat.py +++ b/pypy/module/pyexpat/interp_pyexpat.py @@ -182,6 +182,8 @@ def clear(self): self.next_id = 0 + self._last_object_id = -1 + self._last_object = None self.storage = {} @staticmethod @@ -194,10 +196,18 @@ @staticmethod def get_object(id): - return global_storage.storage[id] + if id == global_storage._last_object_id: + return global_storage._last_object + result = global_storage.storage[id] + global_storage._last_object_id = id + global_storage._last_object = result + return result @staticmethod def free_nonmoving_id(id): + if id == global_storage._last_object_id: + global_storage._last_object = None + global_storage._last_object_id = -1 del global_storage.storage[id] global_storage = Storage() diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py --- a/pypy/module/pypyjit/interp_jit.py +++ b/pypy/module/pypyjit/interp_jit.py @@ -6,7 +6,7 @@ from pypy.tool.pairtype import extendabletype from pypy.rlib.rarithmetic import r_uint, intmask from pypy.rlib.jit import JitDriver, hint, we_are_jitted, dont_look_inside -from pypy.rlib import jit +from pypy.rlib import jit, objectmodel from pypy.rlib.jit import current_trace_length, unroll_parameters import pypy.interpreter.pyopcode # for side-effects from pypy.interpreter.error import OperationError, operationerrfmt @@ -97,6 +97,15 @@ is_being_profiled=self.is_being_profiled) return jumpto +callback_jit_driver = JitDriver(greens = ['name'], reds = 'auto') + +def callback_merge_point(name): + callback_jit_driver.jit_merge_point(name=name) + + at callback_jit_driver.inline(callback_merge_point) +def callback_hook(name): + pass + def _get_adapted_tick_counter(): # Normally, the tick counter is decremented by 100 for every # Python opcode. Here, to better support JIT compilation of diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py --- a/pypy/module/pypyjit/policy.py +++ b/pypy/module/pypyjit/policy.py @@ -106,7 +106,7 @@ 'posix', '_socket', '_sre', '_lsprof', '_weakref', '__pypy__', 'cStringIO', '_collections', 'struct', 'mmap', 'marshal', '_codecs', 'rctime', 'cppyy', - '_cffi_backend']: + '_cffi_backend', 'pyexpat']: if modname == 'pypyjit' and 'interp_resop' in rest: return False return True diff --git a/pypy/objspace/flow/test/test_objspace.py b/pypy/objspace/flow/test/test_objspace.py --- a/pypy/objspace/flow/test/test_objspace.py +++ b/pypy/objspace/flow/test/test_objspace.py @@ -8,18 +8,17 @@ from pypy.objspace.flow.objspace import FlowObjSpace from pypy.objspace.flow.flowcontext import FlowingError, FlowSpaceFrame from pypy import conftest -from pypy.tool.stdlib_opcode import bytecode_spec, host_bytecode_spec +from pypy.tool.stdlib_opcode import host_bytecode_spec import os import operator is_operator = getattr(operator, 'is_', operator.eq) # it's not there 2.2 @contextmanager -def patching_opcodes(*opcodes): +def patching_opcodes(**opcodes): meth_names = host_bytecode_spec.method_names - opnums = [bytecode_spec.opmap[name] for name in opcodes] old_name = {} - for name, num in zip(opcodes, opnums): + for name, num in opcodes.items(): old_name[num] = meth_names[num] meth_names[num] = name yield @@ -898,7 +897,7 @@ """ Tests code generated by pypy-c compiled with CALL_METHOD bytecode """ - with patching_opcodes('CALL_METHOD', 'LOOKUP_METHOD'): + with patching_opcodes(CALL_METHOD=202, LOOKUP_METHOD=201): class X: def m(self): return 3 @@ -922,7 +921,7 @@ """ Tests code generated by pypy-c compiled with BUILD_LIST_FROM_ARG bytecode """ - with patching_opcodes('BUILD_LIST_FROM_ARG'): + with patching_opcodes(BUILD_LIST_FROM_ARG=203): def f(): return [i for i in "abc"] diff --git a/pypy/rlib/jit_libffi.py b/pypy/rlib/jit_libffi.py --- a/pypy/rlib/jit_libffi.py +++ b/pypy/rlib/jit_libffi.py @@ -1,4 +1,4 @@ -import sys + from pypy.rpython.lltypesystem import lltype, rffi from pypy.rlib import clibffi, jit diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py --- a/pypy/rlib/objectmodel.py +++ b/pypy/rlib/objectmodel.py @@ -603,6 +603,16 @@ rffi.aroundstate.leave_callback = leave_callback llhelper(rffi.LeaveCallbackFnPtr, leave_callback) +def register_around_callback_hook(hook): + """ Register a hook that's called before a callback from C calls RPython. + Primary usage is for JIT to have 'started from' hook. + """ + from pypy.rpython.lltypesystem import rffi + from pypy.rpython.annlowlevel import llhelper + + rffi.aroundstate.callback_hook = hook + llhelper(rffi.CallbackHookPtr, hook) + def is_in_callback(): from pypy.rpython.lltypesystem import rffi return rffi.stackcounter.stacks_counter > 1 diff --git a/pypy/rlib/rgc.py b/pypy/rlib/rgc.py --- a/pypy/rlib/rgc.py +++ b/pypy/rlib/rgc.py @@ -158,6 +158,52 @@ hop.exception_cannot_occur() return hop.genop(opname, vlist, resulttype = hop.r_result.lowleveltype) +def copy_struct_item(source, dest, si, di): + TP = lltype.typeOf(source).TO.OF + i = 0 + while i < len(TP._names): + setattr(dest[di], TP._names[i], getattr(source[si], TP._names[i])) + i += 1 + +class CopyStructEntry(ExtRegistryEntry): + _about_ = copy_struct_item + + def compute_result_annotation(self, s_source, s_dest, si, di): + pass + + def specialize_call(self, hop): + v_source, v_dest, v_si, v_di = hop.inputargs(hop.args_r[0], + hop.args_r[1], + lltype.Signed, + lltype.Signed) + hop.exception_cannot_occur() + TP = v_source.concretetype.TO.OF + for name, TP in TP._flds.iteritems(): + c_name = hop.inputconst(lltype.Void, name) + v_fld = hop.genop('getinteriorfield', [v_source, v_si, c_name], + resulttype=TP) + hop.genop('setinteriorfield', [v_dest, v_di, c_name, v_fld]) + + + at specialize.ll() +def copy_item(source, dest, si, di): + TP = lltype.typeOf(source) + if isinstance(TP.TO.OF, lltype.Struct): + copy_struct_item(source, dest, si, di) + else: + dest[di] = source[si] + + at specialize.memo() +def _contains_gcptr(TP): + if not isinstance(TP, lltype.Struct): + if isinstance(TP, lltype.Ptr) and TP.TO._gckind == 'gc': + return True + return False + for TP in TP._flds.itervalues(): + if _contains_gcptr(TP): + return True + return False + @jit.oopspec('list.ll_arraycopy(source, dest, source_start, dest_start, length)') @enforceargs(None, None, int, int, int) @specialize.ll() @@ -169,7 +215,7 @@ # and also, maybe, speed up very small cases if length <= 1: if length == 1: - dest[dest_start] = source[source_start] + copy_item(source, dest, source_start, dest_start) return # supports non-overlapping copies only @@ -185,7 +231,7 @@ if stm_is_enabled(): slowpath = True # - elif isinstance(TP.OF, lltype.Ptr) and TP.OF.TO._gckind == 'gc': + elif _contains_gcptr(TP.OF): # perform a write barrier that copies necessary flags from # source to dest if not llop.gc_writebarrier_before_copy(lltype.Bool, source, dest, @@ -197,6 +243,7 @@ # enabled, copy by hand i = 0 while i < length: + copy_item(source, dest, i + source_start, i + dest_start) dest[i + dest_start] = source[i + source_start] i += 1 return diff --git a/pypy/rlib/test/test_rgc.py b/pypy/rlib/test/test_rgc.py --- a/pypy/rlib/test/test_rgc.py +++ b/pypy/rlib/test/test_rgc.py @@ -134,6 +134,46 @@ assert check.called +def test_ll_arraycopy_array_of_structs(): + TP = lltype.GcArray(lltype.Struct('x', ('x', lltype.Signed), + ('y', lltype.Signed))) + def f(): + a1 = lltype.malloc(TP, 3) + a2 = lltype.malloc(TP, 3) + for i in range(3): + a1[i].x = 2 * i + a1[i].y = 2 * i + 1 + rgc.ll_arraycopy(a1, a2, 0, 0, 3) + for i in range(3): + assert a2[i].x == 2 * i + assert a2[i].y == 2 * i + 1 + + + interpret(f, []) + a1 = lltype.malloc(TP, 3) + a2 = lltype.malloc(TP, 3) + a1[1].x = 3 + a1[1].y = 15 + rgc.copy_struct_item(a1, a2, 1, 2) + assert a2[2].x == 3 + assert a2[2].y == 15 + +def test__contains_gcptr(): + assert not rgc._contains_gcptr(lltype.Signed) + assert not rgc._contains_gcptr( + lltype.Struct('x', ('x', lltype.Signed))) + assert rgc._contains_gcptr( + lltype.Struct('x', ('x', lltype.Signed), + ('y', lltype.Ptr(lltype.GcArray(lltype.Signed))))) + assert rgc._contains_gcptr( + lltype.Struct('x', ('x', lltype.Signed), + ('y', llmemory.GCREF))) + assert rgc._contains_gcptr(lltype.Ptr(lltype.GcStruct('x'))) + assert not rgc._contains_gcptr(lltype.Ptr(lltype.Struct('x'))) + GCPTR = lltype.Ptr(lltype.GcStruct('x')) + assert rgc._contains_gcptr( + lltype.Struct('FOO', ('s', lltype.Struct('BAR', ('y', GCPTR))))) + def test_ll_arraycopy_small(): TYPE = lltype.GcArray(lltype.Signed) for length in range(5): diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py --- a/pypy/rpython/lltypesystem/rffi.py +++ b/pypy/rpython/lltypesystem/rffi.py @@ -1,6 +1,6 @@ import py from pypy.annotation import model as annmodel -from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem import lltype, rstr from pypy.rpython.lltypesystem import ll2ctypes from pypy.rpython.lltypesystem.llmemory import cast_adr_to_ptr, cast_ptr_to_adr from pypy.rpython.lltypesystem.llmemory import itemoffsetof, raw_memcopy @@ -13,7 +13,7 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.rpython.tool.rfficache import platform, sizeof_c_type from pypy.translator.tool.cbuild import ExternalCompilationInfo -from pypy.rpython.annlowlevel import llhelper +from pypy.rpython.annlowlevel import llhelper, llstr from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.rstring import StringBuilder, UnicodeBuilder, assert_str0 from pypy.rlib import jit @@ -281,9 +281,18 @@ callable_name = getattr(callable, '__name__', '?') if callbackholder is not None: callbackholder.callbacks[callable] = True + callable_name_descr = str(callable).replace('"', '\\"') args = ', '.join(['a%d' % i for i in range(len(TP.TO.ARGS))]) source = py.code.Source(r""" - def wrapper(%s): # no *args - no GIL for mallocing the tuple + def inner_wrapper(%(args)s): + if aroundstate is not None: + callback_hook = aroundstate.callback_hook + if callback_hook: + callback_hook(llstr("%(callable_name_descr)s")) + return callable(%(args)s) + inner_wrapper._never_inline_ = True + + def wrapper(%(args)s): # no *args - no GIL for mallocing the tuple llop.gc_stack_bottom(lltype.Void) # marker for trackgcroot.py token = 0 if aroundstate is not None: @@ -294,7 +303,7 @@ # from now on we hold the GIL stackcounter.stacks_counter += 1 try: - result = callable(%s) + result = inner_wrapper(%(args)s) except Exception, e: os.write(2, "Warning: uncaught exception in callback: %%s %%s\n" %% @@ -313,10 +322,11 @@ # by llexternal, it is essential that no exception checking occurs # after the call to before(). return result - """ % (args, args)) + """ % locals()) miniglobals = locals().copy() miniglobals['Exception'] = Exception miniglobals['os'] = os + miniglobals['llstr'] = llstr miniglobals['we_are_translated'] = we_are_translated miniglobals['stackcounter'] = stackcounter exec source.compile() in miniglobals @@ -326,11 +336,15 @@ AroundFnPtr = lltype.Ptr(lltype.FuncType([], lltype.Void)) EnterCallbackFnPtr = lltype.Ptr(lltype.FuncType([], lltype.Signed)) LeaveCallbackFnPtr = lltype.Ptr(lltype.FuncType([lltype.Signed], lltype.Void)) +CallbackHookPtr = lltype.Ptr(lltype.FuncType([lltype.Ptr(rstr.STR)], lltype.Void)) + class AroundState: _alloc_flavor_ = "raw" + callback_hook = None + def _cleanup_(self): - self.before = None # or a regular RPython function - self.after = None # or a regular RPython function + self.before = None # or a regular RPython function + self.after = None # or a regular RPython function self.enter_callback = None self.leave_callback = None aroundstate = AroundState() diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py --- a/pypy/rpython/memory/gc/minimark.py +++ b/pypy/rpython/memory/gc/minimark.py @@ -2001,6 +2001,17 @@ (obj + offset).address[0] = llmemory.NULL continue # no need to remember this weakref any longer # + elif self.header(pointing_to).tid & GCFLAG_NO_HEAP_PTRS: + # see test_weakref_to_prebuilt: it's not useful to put + # weakrefs into 'old_objects_with_weakrefs' if they point + # to a prebuilt object (they are immortal). If moreover + # the 'pointing_to' prebuilt object still has the + # GCFLAG_NO_HEAP_PTRS flag, then it's even wrong, because + # 'pointing_to' will not get the GCFLAG_VISITED during + # the next major collection. Solve this by not registering + # the weakref into 'old_objects_with_weakrefs'. + continue + # self.old_objects_with_weakrefs.append(obj) def invalidate_old_weakrefs(self): @@ -2014,6 +2025,9 @@ continue # weakref itself dies offset = self.weakpointer_offset(self.get_type_id(obj)) pointing_to = (obj + offset).address[0] + ll_assert((self.header(pointing_to).tid & GCFLAG_NO_HEAP_PTRS) + == 0, "registered old weakref should not " + "point to a NO_HEAP_PTRS obj") if self.header(pointing_to).tid & GCFLAG_VISITED: new_with_weakref.append(obj) else: diff --git a/pypy/rpython/memory/test/test_gc.py b/pypy/rpython/memory/test/test_gc.py --- a/pypy/rpython/memory/test/test_gc.py +++ b/pypy/rpython/memory/test/test_gc.py @@ -557,6 +557,19 @@ res = self.interpret(f, [20]) # for GenerationGC, enough for a minor collection assert res == True + def test_weakref_to_prebuilt(self): + import weakref + class A: + pass + a = A() + def f(x): + ref = weakref.ref(a) + assert ref() is a + llop.gc__collect(lltype.Void) + return ref() is a + res = self.interpret(f, [20]) # for GenerationGC, enough for a minor collection + assert res == True + def test_many_weakrefs(self): # test for the case where allocating the weakref itself triggers # a collection diff --git a/pypy/translator/driver.py b/pypy/translator/driver.py --- a/pypy/translator/driver.py +++ b/pypy/translator/driver.py @@ -362,7 +362,7 @@ @taskdef([RTYPE], "JIT compiler generation") def task_pyjitpl_lltype(self): """ Generate bytecodes for JIT and flow the JIT helper functions - ootype version + lltype version """ get_policy = self.extra['jitpolicy'] self.jitpolicy = get_policy(self) diff --git a/pypy/translator/goal/app_main.py b/pypy/translator/goal/app_main.py --- a/pypy/translator/goal/app_main.py +++ b/pypy/translator/goal/app_main.py @@ -584,7 +584,12 @@ python_startup, 'exec') exec co_python_startup in mainmodule.__dict__ + mainmodule.__file__ = python_startup run_toplevel(run_it) + try: + del mainmodule.__file__ + except (AttributeError, TypeError): + pass # Then we need a prompt. inspect = True else: diff --git a/pypy/translator/goal/targetpypystandalone.py b/pypy/translator/goal/targetpypystandalone.py --- a/pypy/translator/goal/targetpypystandalone.py +++ b/pypy/translator/goal/targetpypystandalone.py @@ -28,6 +28,11 @@ w_call_startup_gateway = space.wrap(gateway.interp2app(call_startup)) withjit = space.config.objspace.usemodules.pypyjit + if withjit: + from pypy.module.pypyjit.interp_jit import callback_hook + from pypy.rlib import objectmodel + objectmodel.register_around_callback_hook(callback_hook) + def entry_point(argv): if withjit: from pypy.jit.backend.hlinfo import highleveljitinfo diff --git a/pypy/translator/goal/test2/test_app_main.py b/pypy/translator/goal/test2/test_app_main.py --- a/pypy/translator/goal/test2/test_app_main.py +++ b/pypy/translator/goal/test2/test_app_main.py @@ -376,6 +376,30 @@ child.expect('Traceback') child.expect('NameError') + def test_pythonstartup_file1(self, monkeypatch): + monkeypatch.setenv('PYTHONPATH', None) + monkeypatch.setenv('PYTHONSTARTUP', demo_script) + child = self.spawn([]) + child.expect('File: [^\n]+\.py') + child.expect('goodbye') + child.expect('>>> ') + child.sendline('[myvalue]') + child.expect(re.escape('[42]')) + child.expect('>>> ') + child.sendline('__file__') + child.expect('Traceback') + child.expect('NameError') + + def test_pythonstartup_file2(self, monkeypatch): + monkeypatch.setenv('PYTHONPATH', None) + monkeypatch.setenv('PYTHONSTARTUP', crashing_demo_script) + child = self.spawn([]) + child.expect('Traceback') + child.expect('>>> ') + child.sendline('__file__') + child.expect('Traceback') + child.expect('NameError') + def test_ignore_python_startup(self): old = os.environ.get('PYTHONSTARTUP', '') try: diff --git a/pypy/translator/platform/linux.py b/pypy/translator/platform/linux.py --- a/pypy/translator/platform/linux.py +++ b/pypy/translator/platform/linux.py @@ -36,7 +36,13 @@ # places where we need to look for libffi.a # XXX obscuuure! only look for libffi.a if run with translate.py if 'translate' in sys.modules: - return self.library_dirs_for_libffi() + ['/usr/lib'] + if sys.maxint > 2**32: + host = 'x86_64' + else: + host = 'x86' + return self.library_dirs_for_libffi() + [ + '/usr/lib', + '/usr/lib/%s-linux-gnu/' % host] else: return [] diff --git a/pypy/translator/platform/test/test_distutils.py b/pypy/translator/platform/test/test_distutils.py --- a/pypy/translator/platform/test/test_distutils.py +++ b/pypy/translator/platform/test/test_distutils.py @@ -8,3 +8,6 @@ def test_nice_errors(self): py.test.skip("Unsupported") + + def test_900_files(self): + py.test.skip('Makefiles not suppoerted') diff --git a/pypy/translator/platform/test/test_platform.py b/pypy/translator/platform/test/test_platform.py --- a/pypy/translator/platform/test/test_platform.py +++ b/pypy/translator/platform/test/test_platform.py From noreply at buildbot.pypy.org Fri Jan 11 16:28:02 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 11 Jan 2013 16:28:02 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: shift stuff a little so we don't have to use negative numbers Message-ID: <20130111152802.ECCE81C1046@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59953:1f3c4036d05e Date: 2013-01-11 17:27 +0200 http://bitbucket.org/pypy/pypy/changeset/1f3c4036d05e/ Log: shift stuff a little so we don't have to use negative numbers diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py --- a/pypy/rpython/lltypesystem/rdict.py +++ b/pypy/rpython/lltypesystem/rdict.py @@ -356,12 +356,12 @@ MAX_SHORT_MASK = ~(2 ** 16 - 1) MAX_BYTE_MASK = ~(2 ** 8 - 1) - at jit.look_inside_iff(lambda n : jit.isconstant(n)) def _ll_malloc_indexes(n): # XXXX 64 bit only #if n & MAX_INT_MASK: res = lltype.cast_opaque_ptr(llmemory.GCREF, - lltype.malloc(DICTINDEX_SIGNED.TO, n)) + lltype.malloc(DICTINDEX_SIGNED.TO, n, + zero=True)) #elif n & MAX_SHORT_MASK: # res = lltype.cast_opaque_ptr(llmemory.GCREF, # lltype.malloc(DICTINDEX_INT.TO, n)) @@ -371,15 +371,11 @@ #else: # res = lltype.cast_opaque_ptr(llmemory.GCREF, # lltype.malloc(DICTINDEX_BYTE.TO, n)) - i = 0 - while i < n: - ll_index_setitem(n, res, i, FREE) - i += 1 return res def ll_index_getitem(size, indexes, i): #if size & MAX_INT_MASK: - return lltype.cast_opaque_ptr(DICTINDEX_SIGNED, indexes)[i] + return lltype.cast_opaque_ptr(DICTINDEX_SIGNED, indexes)[i] - 2 #elif size & MAX_SHORT_MASK: # return rffi.cast(lltype.Signed, # lltype.cast_opaque_ptr(DICTINDEX_INT, indexes)[i]) @@ -392,7 +388,7 @@ def ll_index_setitem(size, indexes, i, v): #if size & MAX_INT_MASK: - lltype.cast_opaque_ptr(DICTINDEX_SIGNED, indexes)[i] = v + lltype.cast_opaque_ptr(DICTINDEX_SIGNED, indexes)[i] = v + 2 #elif size & MAX_SHORT_MASK: # lltype.cast_opaque_ptr(DICTINDEX_INT, indexes)[i] = rffi.cast( # rffi.INT_real, v) From noreply at buildbot.pypy.org Fri Jan 11 17:06:00 2013 From: noreply at buildbot.pypy.org (cfbolz) Date: Fri, 11 Jan 2013 17:06:00 +0100 (CET) Subject: [pypy-commit] pypy type-specialized-instances: make sure that if you tag the same thing twice you reuse the result Message-ID: <20130111160600.534AD1C0DCB@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: type-specialized-instances Changeset: r59954:3eb981d2713e Date: 2013-01-11 17:05 +0100 http://bitbucket.org/pypy/pypy/changeset/3eb981d2713e/ Log: make sure that if you tag the same thing twice you reuse the result diff --git a/pypy/jit/metainterp/optimizeopt/intbounds.py b/pypy/jit/metainterp/optimizeopt/intbounds.py --- a/pypy/jit/metainterp/optimizeopt/intbounds.py +++ b/pypy/jit/metainterp/optimizeopt/intbounds.py @@ -195,6 +195,7 @@ maxbounds = IntBound((-sys.maxint-1) >> 1, sys.maxint >> 1) v1.intbound.intersect(maxbounds) self.pure(rop.INT_UNTAG, [result], args[0]) + self.pure(rop.INT_TAG, args[:], result) self.emit_operation(op) def optimize_GUARD_OVERFLOW(self, op): diff --git a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/pypy/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -5335,6 +5335,28 @@ """ self.optimize_loop(ops, expected, preamble) + def test_int_tag_ovf_not_repeated(self): + ops = """ + [i0] + i1 = int_tag_ovf(i0) + guard_no_overflow() [] + i2 = int_tag_ovf(i0) + guard_no_overflow() [] + i5 = int_add(i2, 1) + escape(i5) + jump(i5) + """ + expected = """ + [i0] + i1 = int_tag_ovf(i0) + guard_no_overflow() [] + i5 = int_add(i1, 1) + escape(i5) + jump(i5) + """ + self.optimize_loop(ops, expected) + + def test_mul_ovf(self): ops = """ [i0, i1] From noreply at buildbot.pypy.org Fri Jan 11 18:22:14 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Fri, 11 Jan 2013 18:22:14 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: More socket fixes Message-ID: <20130111172214.AF2A21C11B7@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59955:c13b3c57c762 Date: 2013-01-11 18:21 +0100 http://bitbucket.org/pypy/pypy/changeset/c13b3c57c762/ Log: More socket fixes diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -21,23 +21,23 @@ # XXX Hack to seperate rpython and pypy def addr_as_object(addr, fd, space): - if addr.family == rsocket.AF_INET: + if hasattr(addr, 'family') and addr.family == rsocket.AF_INET: return space.newtuple([space.wrap(addr.get_host()), space.wrap(addr.get_port())]) - if addr.family == rsocket.AF_INET6: + if hasattr(addr, 'family') and addr.family == rsocket.AF_INET6: return space.newtuple([space.wrap(addr.get_host()), space.wrap(addr.get_port()), space.wrap(addr.get_flowinfo()), space.wrap(addr.get_scope_id())]) - if 'AF_PACKET' in rsocket.constants and addr.family == rsocket.AF_PACKET: + if hasattr(addr, 'family') and hasattr(rsocket, 'AF_PACKET') and addr.family == rsocket.AF_PACKET: return space.newtuple([space.wrap(addr.get_ifname(fd)), space.wrap(addr.get_protocol()), space.wrap(addr.get_pkttype()), space.wrap(addr.get_hatype()), space.wrap(addr.get_addr())]) - if 'AF_UNIX' in rsocket.constants and addr.family == rsocket.AF_UNIX: + if hasattr(addr, 'family') and hasattr(rsocket, 'AF_UNIX') and addr.family == rsocket.AF_UNIX: return space.wrap(addr.get_path()) - if 'AF_NETLINK' in rsocket.constants and addr.family == rsocket.AF_NETLINK: + if hasattr(addr, 'family') and hasattr(rsocket, 'AF_NETLINK') and addr.family == rsocket.AF_NETLINK: return space.newtuple([space.wrap(addr.get_pid()), space.wrap(addr.get_groups())]) # If we don't know the address family, don't raise an @@ -53,16 +53,16 @@ # XXX Hack to seperate rpython and pypy # XXX a bit of code duplication def fill_from_object(addr, space, w_address): - if addr.family == rsocket.AF_INET: - from pypy.interpreter.error import OperationError + from rpython.rlib import _rsocket_rffi as _c + from pypy.interpreter.error import OperationError + if hasattr(addr, 'family') and addr.family == rsocket.AF_INET: _, w_port = space.unpackiterable(w_address, 2) port = space.int_w(w_port) port = make_ushort_port(space, port) a = addr.lock(_c.sockaddr_in) rffi.setintfield(a, 'c_sin_port', htons(port)) addr.unlock() - elif addr.family == rsocket.AF_INET6: - from pypy.interpreter.error import OperationError + elif hasattr(addr, 'family') and addr.family == rsocket.AF_INET6: pieces_w = space.unpackiterable(w_address) if not (2 <= len(pieces_w) <= 4): raise RSocketError("AF_INET6 address must be a tuple of length 2 " From noreply at buildbot.pypy.org Fri Jan 11 18:25:33 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Fri, 11 Jan 2013 18:25:33 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Less imports Message-ID: <20130111172533.2D7AE1C1245@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59956:9cf167573440 Date: 2013-01-11 18:25 +0100 http://bitbucket.org/pypy/pypy/changeset/9cf167573440/ Log: Less imports diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -7,7 +7,6 @@ from rpython.rlib import rsocket from rpython.rlib.rsocket import RSocket, AF_INET, SOCK_STREAM from rpython.rlib.rsocket import SocketError, SocketErrorWithErrno, RSocketError -from rpython.rlib.rsocket import INETAddress, INET6Address from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter import gateway @@ -54,7 +53,6 @@ # XXX a bit of code duplication def fill_from_object(addr, space, w_address): from rpython.rlib import _rsocket_rffi as _c - from pypy.interpreter.error import OperationError if hasattr(addr, 'family') and addr.family == rsocket.AF_INET: _, w_port = space.unpackiterable(w_address, 2) port = space.int_w(w_port) @@ -92,9 +90,8 @@ host = space.str_w(w_host) port = space.int_w(w_port) port = make_ushort_port(space, port) - return INETAddress(host, port) + return rsocket.INETAddress(host, port) if family == rsocket.AF_INET6: - from pypy.interpreter.error import OperationError pieces_w = space.unpackiterable(w_address) if not (2 <= len(pieces_w) <= 4): raise TypeError("AF_INET6 address must be a tuple of length 2 " @@ -110,19 +107,16 @@ raise OperationError(space.w_OverflowError, space.wrap( "flowinfo must be 0-1048575.")) flowinfo = rffi.cast(lltype.Unsigned, flowinfo) - return INET6Address(host, port, flowinfo, scope_id) + return rsocket.INET6Address(host, port, flowinfo, scope_id) if 'AF_UNIX' in rsocket.constants and family == rsocket.AF_UNIX: - from rpython.rlib.rsocket import UNIXAddress - return UNIXAddress(space.str_w(w_address)) + return rsocket.UNIXAddress(space.str_w(w_address)) if 'AF_NETLINK' in rsocket.constants and family == rsocket.AF_NETLINK: - from rpython.rlib.rsocket import NETLINKAddress w_pid, w_groups = space.unpackiterable(w_address, 2) - return NETLINKAddress(space.uint_w(w_pid), space.uint_w(w_groups)) + return rsocket.NETLINKAddress(space.uint_w(w_pid), space.uint_w(w_groups)) raise RSocketError("unknown address family") # XXX Hack to seperate rpython and pypy def make_ushort_port(space, port): - from pypy.interpreter.error import OperationError if port < 0 or port > 0xffff: raise OperationError(space.w_ValueError, space.wrap( "port must be 0-65535.")) @@ -130,9 +124,8 @@ # XXX Hack to seperate rpython and pypy def ipaddr_from_object(space, w_sockaddr): - from rpython.rlib.rsocket import makeipaddr host = space.str_w(space.getitem(w_sockaddr, space.wrap(0))) - addr = makeipaddr(host) + addr = rsocket.makeipaddr(host) fill_from_object(addr, space, w_sockaddr) return addr From noreply at buildbot.pypy.org Fri Jan 11 18:31:01 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 11 Jan 2013 18:31:01 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: Try to use different index sizes in a smarter way. a bit too much mess for Message-ID: <20130111173101.3BEBD1C12F5@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59957:67c1f89d912c Date: 2013-01-11 19:29 +0200 http://bitbucket.org/pypy/pypy/changeset/67c1f89d912c/ Log: Try to use different index sizes in a smarter way. a bit too much mess for my liking, but a bit no clue what to do diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py --- a/pypy/rpython/lltypesystem/rdict.py +++ b/pypy/rpython/lltypesystem/rdict.py @@ -1,3 +1,5 @@ + +import sys from pypy.tool.pairtype import pairtype from pypy.objspace.flow.model import Constant from pypy.rpython.rdict import (AbstractDictRepr, AbstractDictIteratorRepr, @@ -6,6 +8,7 @@ from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib import objectmodel, jit, rgc from pypy.rlib.debug import ll_assert +from pypy.rlib.objectmodel import specialize from pypy.rlib.rarithmetic import r_uint, intmask, LONG_BIT from pypy.rpython import rmodel from pypy.rpython.error import TyperError @@ -352,58 +355,58 @@ DICTINDEX_INT = lltype.Ptr(lltype.GcArray(rffi.INT_real)) DICTINDEX_SHORT = lltype.Ptr(lltype.GcArray(rffi.SHORT)) DICTINDEX_BYTE = lltype.Ptr(lltype.GcArray(rffi.SIGNEDCHAR)) -MAX_INT_MASK = ~(2 ** 31 - 1) -MAX_SHORT_MASK = ~(2 ** 16 - 1) -MAX_BYTE_MASK = ~(2 ** 8 - 1) def _ll_malloc_indexes(n): - # XXXX 64 bit only - #if n & MAX_INT_MASK: - res = lltype.cast_opaque_ptr(llmemory.GCREF, - lltype.malloc(DICTINDEX_SIGNED.TO, n, - zero=True)) - #elif n & MAX_SHORT_MASK: - # res = lltype.cast_opaque_ptr(llmemory.GCREF, - # lltype.malloc(DICTINDEX_INT.TO, n)) - #elif n & MAX_BYTE_MASK: - # res = lltype.cast_opaque_ptr(llmemory.GCREF, - # lltype.malloc(DICTINDEX_SHORT.TO, n)) - #else: - # res = lltype.cast_opaque_ptr(llmemory.GCREF, - # lltype.malloc(DICTINDEX_BYTE.TO, n)) + if IS_64BIT and n < 2 ** 31: + res = lltype.cast_opaque_ptr(llmemory.GCREF, + lltype.malloc(DICTINDEX_INT.TO, n, + zero=True)) + else: + res = lltype.cast_opaque_ptr(llmemory.GCREF, + lltype.malloc(DICTINDEX_SIGNED.TO, n, + zero=True)) return res -def ll_index_getitem(size, indexes, i): - #if size & MAX_INT_MASK: +def ll_index_getitem_signed(size, indexes, i): return lltype.cast_opaque_ptr(DICTINDEX_SIGNED, indexes)[i] - 2 - #elif size & MAX_SHORT_MASK: - # return rffi.cast(lltype.Signed, - # lltype.cast_opaque_ptr(DICTINDEX_INT, indexes)[i]) - #elif size & MAX_BYTE_MASK: - # return rffi.cast(lltype.Signed, - # lltype.cast_opaque_ptr(DICTINDEX_SHORT, indexes)[i]) - #else: - # return rffi.cast(lltype.Signed, - # lltype.cast_opaque_ptr(DICTINDEX_BYTE, indexes)[i]) -def ll_index_setitem(size, indexes, i, v): - #if size & MAX_INT_MASK: +def ll_index_getitem_int(size, indexes, i): + res = lltype.cast_opaque_ptr(DICTINDEX_INT, indexes)[i] + return rffi.cast(lltype.Signed, res) - 2 + +def ll_index_setitem_signed(size, indexes, i, v): lltype.cast_opaque_ptr(DICTINDEX_SIGNED, indexes)[i] = v + 2 - #elif size & MAX_SHORT_MASK: - # lltype.cast_opaque_ptr(DICTINDEX_INT, indexes)[i] = rffi.cast( - # rffi.INT_real, v) - #elif size & MAX_BYTE_MASK: - # lltype.cast_opaque_ptr(DICTINDEX_SHORT, indexes)[i] = rffi.cast( - # rffi.SHORT, v) - #else: - # lltype.cast_opaque_ptr(DICTINDEX_BYTE, indexes)[i] = rffi.cast( - # rffi.SIGNEDCHAR, v) -def ll_dict_copy_indexes(size, from_indexes, to_indexes): +def ll_index_setitem_int(size, indexes, i, v): + v = rffi.cast(rffi.INT_real, v + 2) + lltype.cast_opaque_ptr(DICTINDEX_INT, indexes)[i] = v + +IS_64BIT = sys.maxint != 2 ** 31 - 1 + +def _pick_index_setitem(d): + if IS_64BIT and d.size < 2 ** 31: + return ll_index_setitem_int + return ll_index_setitem_signed + +def _pick_index_getitem(d): + if IS_64BIT and d.size < 2 ** 31: + return ll_index_getitem_int + return ll_index_getitem_signed + +def d_signed_indexes(d): + if IS_64BIT and d.size < 2 ** 31: + return False + return True + +def ll_dict_copy_indexes(size, from_d, to_d): + ll_index_getitem = _pick_index_getitem(from_d) + ll_index_setitem = _pick_index_setitem(to_d) + from_indexes = from_d.indexes + to_indexes = to_d.indexes i = 0 while i < size: - ll_index_setitem(size, to_indexes, i, - ll_index_getitem(size, from_indexes, i)) + v = ll_index_getitem(size, from_indexes, i) + ll_index_setitem(size, to_indexes, i, v) i += 1 # ---------------------- entries ----------------------- @@ -446,7 +449,8 @@ ENTRIES = lltype.typeOf(entries).TO return ENTRIES.fasthashfn(entries[i].key) -def ll_get_value(d, i): + at specialize.ll_and_arg(2) +def ll_get_value(d, i, ll_index_getitem): return d.entries.getitem_clean(ll_index_getitem(d.size, d.indexes, i)).value def ll_keyhash_custom(d, key): @@ -465,22 +469,45 @@ return bool(d) and d.num_items != 0 def ll_dict_getitem(d, key): - i = ll_dict_lookup(d, key, d.keyhash(key)) - if not i & HIGHEST_BIT: - return ll_get_value(d, i) + if d_signed_indexes(d): + i = ll_dict_lookup(d, key, d.keyhash(key), ll_index_getitem_signed) + if not i & HIGHEST_BIT: + return ll_get_value(d, i, ll_index_getitem_signed) + else: + raise KeyError else: - raise KeyError + i = ll_dict_lookup(d, key, d.keyhash(key), ll_index_getitem_int) + if not i & HIGHEST_BIT: + return ll_get_value(d, i, ll_index_getitem_int) + else: + raise KeyError def ll_dict_setitem(d, key, value): - hash = d.keyhash(key) - i = ll_dict_lookup(d, key, hash) - return _ll_dict_setitem_lookup_done(d, key, value, hash, i) + if d_signed_indexes(d): + hash = d.keyhash(key) + i = ll_dict_lookup(d, key, hash, ll_index_getitem_signed) + _ll_dict_setitem_lookup_done(d, key, value, hash, i, + ll_index_getitem_signed, + ll_index_setitem_signed) + else: + hash = d.keyhash(key) + i = ll_dict_lookup(d, key, hash, ll_index_getitem_int) + _ll_dict_setitem_lookup_done(d, key, value, hash, i, + ll_index_getitem_int, + ll_index_setitem_int) def ll_dict_insertclean(d, key, value, hash): - i = ll_dict_lookup_clean(d, hash) - return _ll_dict_setitem_lookup_done(d, key, value, hash, i | HIGHEST_BIT) + if d_signed_indexes(d): + i = ll_dict_lookup_clean(d, hash, ll_index_getitem_signed) + _ll_dict_setitem_lookup_done(d, key, value, hash, i | HIGHEST_BIT, + ll_index_getitem_signed) + else: + i = ll_dict_lookup_clean(d, hash, ll_index_getitem_int) + _ll_dict_setitem_lookup_done(d, key, value, hash, i | HIGHEST_BIT, + ll_index_getitem_int) -def ll_dict_lookup_clean(d, hash): + at specialize.ll_and_arg(2) +def ll_dict_lookup_clean(d, hash, ll_index_getitem): # a simplified version of ll_dict_lookup() which assumes that the # key is new, and the dictionary doesn't contain deleted entries. # It only finds the next free slot for the given hash. @@ -499,13 +526,15 @@ perturb >>= PERTURB_SHIFT return i -def _look_inside_setitem(d, key, value, hash, i): +def _look_inside_setitem(d, key, value, hash, i, _1, _2): return jit.isvirtual(d) and jit.isconstant(key) # It may be safe to look inside always, it has a few branches though, and their # frequencies needs to be investigated. @jit.look_inside_iff(_look_inside_setitem) -def _ll_dict_setitem_lookup_done(d, key, value, hash, i): + at specialize.ll_and_arg(5, 6) +def _ll_dict_setitem_lookup_done(d, key, value, hash, i, ll_index_getitem, + ll_index_setitem): valid = (i & HIGHEST_BIT) == 0 i = i & MASK ENTRY = lltype.typeOf(d.entries).TO.OF @@ -518,7 +547,10 @@ rc = d.resize_counter - 1 if rc <= 0: # if needed, resize the dict -- before the insertion d.resize() - i = ll_dict_lookup_clean(d, hash) + if d_signed_indexes(d): + i = ll_dict_lookup_clean(d, hash, ll_index_getitem_signed) + else: + i = ll_dict_lookup_clean(d, hash, ll_index_getitem_int) # then redo the lookup for 'key' entry = d.entries.getitem(d, index) rc = d.resize_counter - 1 @@ -543,20 +575,28 @@ d.num_items += 1 def ll_dict_delitem(d, key): - i = ll_dict_lookup(d, key, d.keyhash(key)) - if i & HIGHEST_BIT: - raise KeyError - _ll_dict_del(d, i) + if d_signed_indexes(d): + i = ll_dict_lookup(d, key, d.keyhash(key), ll_index_getitem_signed) + if i & HIGHEST_BIT: + raise KeyError + _ll_dict_del(d, i, ll_index_getitem_signed, ll_index_setitem_signed) + else: + i = ll_dict_lookup(d, key, d.keyhash(key), ll_index_getitem_int) + if i & HIGHEST_BIT: + raise KeyError + _ll_dict_del(d, i, ll_index_getitem_int, ll_index_setitem_int) - at jit.look_inside_iff(lambda d, i: jit.isvirtual(d) and jit.isconstant(i)) -def _ll_dict_del(d, i): + at jit.look_inside_iff(lambda d, i, _1, _2: jit.isvirtual(d) and + jit.isconstant(i)) + at specialize.ll_and_arg(2, 3) +def _ll_dict_del(d, i, ll_index_getitem, ll_index_setitem): index = ll_index_getitem(d.size, d.indexes, i) ENTRIES = lltype.typeOf(d.entries).TO ENTRY = ENTRIES.OF if index != d.num_items - 1: old_entry = d.entries.getitem(d, d.num_items - 1) key = old_entry.key - to_insert_i = ll_dict_lookup(d, key, d.keyhash(key)) + to_insert_i = ll_dict_lookup(d, key, d.keyhash(key), ll_index_getitem) ll_assert(not to_insert_i & HIGHEST_BIT, "invalid entry") ll_index_setitem(d.size, d.indexes, to_insert_i, index) # copy the value @@ -603,22 +643,31 @@ new_size *= 2 # new_item_size = new_size // 3 * 2 + 1 + ll_index_getitem = _pick_index_getitem(d) d.indexes = _ll_malloc_indexes(new_size) d.size = new_size d.resize_counter = new_item_size - d.num_items i = 0 indexes = d.indexes + ll_index_setitem = _pick_index_setitem(d) while i < old_size: index = ll_index_getitem(old_size, old_indexes, i) if old_entries.valid(index): - pos = ll_dict_lookup_clean(d, old_entries.hash(index)) + # XXX we can do better here, but we have to remember about + # paranoia + if d_signed_indexes(d): + pos = ll_dict_lookup_clean(d, old_entries.hash(index), + ll_index_getitem_signed) + else: + pos = ll_dict_lookup_clean(d, old_entries.hash(index), + ll_index_getitem_int) ll_index_setitem(d.size, indexes, pos, index) i += 1 # ------- a port of CPython's dictobject.c's lookdict implementation ------- PERTURB_SHIFT = 5 -_look_inside_lookup = lambda d, key, hash: jit.isvirtual(d) and jit.isconstant(key) +_look_inside_lookup = lambda d, key, hash, _: jit.isvirtual(d) and jit.isconstant(key) from pypy.rlib.objectmodel import compute_identity_hash def ll_debugrepr(x): @@ -629,7 +678,8 @@ return '<%x>' % (h,) @jit.look_inside_iff(_look_inside_lookup) -def ll_dict_lookup(d, key, hash): + at specialize.ll_and_arg(3) +def ll_dict_lookup(d, key, hash, ll_index_getitem): entries = d.entries indexes = d.indexes mask = d.size - 1 @@ -648,11 +698,16 @@ found = d.keyeq(checkingkey, key) #llop.debug_print(lltype.Void, "comparing keys", ll_debugrepr(checkingkey), ll_debugrepr(key), found) if d.paranoia: - if (entries != d.entries or + if (entries != d.entries or indexes != d.indexes or not entries.valid(ll_index_getitem(d.size, indexes, i)) or entries.getitem_clean(index).key != checkingkey): # the compare did major nasty stuff to the dict: start over - return ll_dict_lookup(d, key, hash) + if d_signed_indexes(d): + return ll_dict_lookup(d, key, hash, + ll_index_getitem_signed) + else: + return ll_dict_lookup(d, key, hash, + ll_index_getitem_int) if found: return i # found the entry freeslot = -1 @@ -685,12 +740,17 @@ # an equal object found = d.keyeq(checkingkey, key) if d.paranoia: - if (entries != d.entries or + if (entries != d.entries or indexes != d.indexes or not entries.valid(ll_index_getitem(d.size, indexes, i)) or entries.getitem_clean(index).key != checkingkey): # the compare did major nasty stuff to the dict: # start over - return ll_dict_lookup(d, key, hash) + if d_signed_indexes(d): + return ll_dict_lookup(d, key, hash, + ll_index_getitem_signed) + else: + return ll_dict_lookup(d, key, hash, + ll_index_getitem_int) if found: return i # found the entry elif freeslot == -1: @@ -814,20 +874,43 @@ # methods def ll_get(dict, key, default): - i = ll_dict_lookup(dict, key, dict.keyhash(key)) - if not i & HIGHEST_BIT: - return ll_get_value(dict, i) + if d_signed_indexes(dict): + i = ll_dict_lookup(dict, key, dict.keyhash(key), + ll_index_getitem_signed) + if not i & HIGHEST_BIT: + return ll_get_value(dict, i, ll_index_getitem_signed) + else: + return default else: - return default + i = ll_dict_lookup(dict, key, dict.keyhash(key), + ll_index_getitem_int) + if not i & HIGHEST_BIT: + return ll_get_value(dict, i, ll_index_getitem_int) + else: + return default def ll_setdefault(dict, key, default): - hash = dict.keyhash(key) - i = ll_dict_lookup(dict, key, hash) - if not i & HIGHEST_BIT: - return ll_get_value(dict, i) + if d_signed_indexes(dict): + hash = dict.keyhash(key) + i = ll_dict_lookup(dict, key, hash, ll_index_getitem_signed) + if not i & HIGHEST_BIT: + return ll_get_value(dict, i, ll_index_getitem_signed) + else: + _ll_dict_setitem_lookup_done(dict, key, default, hash, i, + ll_index_getitem_signed, + ll_index_setitem_signed) + return default else: - _ll_dict_setitem_lookup_done(dict, key, default, hash, i) - return default + hash = dict.keyhash(key) + i = ll_dict_lookup(dict, key, hash, ll_index_getitem_int) + if not i & HIGHEST_BIT: + return ll_get_value(dict, i, ll_index_getitem_int) + else: + _ll_dict_setitem_lookup_done(dict, key, default, hash, i, + ll_index_getitem_int, + ll_index_setitem_int) + return default + def ll_copy(dict): DICT = lltype.typeOf(dict).TO @@ -840,8 +923,7 @@ d.resize_counter = dict.resize_counter if hasattr(DICT, 'fnkeyeq'): d.fnkeyeq = dict.fnkeyeq if hasattr(DICT, 'fnkeyhash'): d.fnkeyhash = dict.fnkeyhash - ll_dict_copy_indexes(d.size, dict.indexes, d.indexes) - #rgc.ll_arraycopy(dict.indexes, d.indexes, 0, 0, len(dict.indexes)) + ll_dict_copy_indexes(d.size, dict, d) rgc.ll_arraycopy(dict.entries, d.entries, 0, 0, dict.num_items) return d @@ -864,8 +946,19 @@ entry = entries.getitem_clean(i) hash = entries.hash(i) key = entry.key - j = ll_dict_lookup(dic1, key, hash) - _ll_dict_setitem_lookup_done(dic1, key, entry.value, hash, j) + if d_signed_indexes(dic1): + j = ll_dict_lookup(dic1, key, hash, ll_index_getitem_signed) + else: + j = ll_dict_lookup(dic1, key, hash, ll_index_getitem_int) + # a separate lookup since it might have changed + if d_signed_indexes(dic1): + _ll_dict_setitem_lookup_done(dic1, key, entry.value, hash, j, + ll_index_getitem_signed, + ll_index_setitem_signed) + else: + _ll_dict_setitem_lookup_done(dic1, key, entry.value, hash, j, + ll_index_getitem_int, + ll_index_setitem_int) i += 1 # this is an implementation of keys(), values() and items() @@ -908,7 +1001,10 @@ ll_dict_items = _make_ll_keys_values_items('items') def ll_contains(d, key): - i = ll_dict_lookup(d, key, d.keyhash(key)) + if d_signed_indexes(d): + i = ll_dict_lookup(d, key, d.keyhash(key), ll_index_getitem_signed) + else: + i = ll_dict_lookup(d, key, d.keyhash(key), ll_index_getitem_int) return not i & HIGHEST_BIT def ll_popitem(ELEM, dic): @@ -922,13 +1018,24 @@ return r def ll_pop(dic, key): - i = ll_dict_lookup(dic, key, dic.keyhash(key)) - if not i & HIGHEST_BIT: - value = ll_get_value(dic, i) - _ll_dict_del(dic, i) - return value + if d_signed_indexes(dic): + i = ll_dict_lookup(dic, key, dic.keyhash(key), ll_index_getitem_signed) + if not i & HIGHEST_BIT: + value = ll_get_value(dic, i, ll_index_getitem_signed) + _ll_dict_del(dic, i, ll_index_getitem_signed, + ll_index_setitem_signed) + return value + else: + raise KeyError else: - raise KeyError + i = ll_dict_lookup(dic, key, dic.keyhash(key), ll_index_getitem_int) + if not i & HIGHEST_BIT: + value = ll_get_value(dic, i, ll_index_getitem_int) + _ll_dict_del(dic, i, ll_index_getitem_int, + ll_index_setitem_int) + return value + else: + raise KeyError def ll_pop_default(dic, key, dfl): try: diff --git a/pypy/rpython/test/test_rdict.py b/pypy/rpython/test/test_rdict.py --- a/pypy/rpython/test/test_rdict.py +++ b/pypy/rpython/test/test_rdict.py @@ -24,8 +24,9 @@ yield x def foreach_index(ll_d): + ll_index_getitem = rdict._pick_index_getitem(ll_d) for i in range(ll_d.size): - yield rdict.ll_index_getitem(ll_d.size, ll_d.indexes, i) + yield ll_index_getitem(ll_d.size, ll_d.indexes, i) def count_items(ll_d, ITEM): c = 0 From noreply at buildbot.pypy.org Fri Jan 11 18:31:02 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 11 Jan 2013 18:31:02 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: fix Message-ID: <20130111173102.AD5D01C12F5@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59958:c5d822f8a34d Date: 2013-01-11 19:30 +0200 http://bitbucket.org/pypy/pypy/changeset/c5d822f8a34d/ Log: fix diff --git a/pypy/rpython/lltypesystem/rdict.py b/pypy/rpython/lltypesystem/rdict.py --- a/pypy/rpython/lltypesystem/rdict.py +++ b/pypy/rpython/lltypesystem/rdict.py @@ -500,11 +500,13 @@ if d_signed_indexes(d): i = ll_dict_lookup_clean(d, hash, ll_index_getitem_signed) _ll_dict_setitem_lookup_done(d, key, value, hash, i | HIGHEST_BIT, - ll_index_getitem_signed) + ll_index_getitem_signed, + ll_index_setitem_signed) else: i = ll_dict_lookup_clean(d, hash, ll_index_getitem_int) _ll_dict_setitem_lookup_done(d, key, value, hash, i | HIGHEST_BIT, - ll_index_getitem_int) + ll_index_getitem_int, + ll_index_setitem_int) @specialize.ll_and_arg(2) def ll_dict_lookup_clean(d, hash, ll_index_getitem): From noreply at buildbot.pypy.org Fri Jan 11 18:50:55 2013 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Fri, 11 Jan 2013 18:50:55 +0100 (CET) Subject: [pypy-commit] pypy link-old-glibc-abi: disable glibc 2.15 source fortify to avoid a symbol Message-ID: <20130111175056.015221C05DD@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: link-old-glibc-abi Changeset: r59959:c8511419a805 Date: 2013-01-11 13:57 +0100 http://bitbucket.org/pypy/pypy/changeset/c8511419a805/ Log: disable glibc 2.15 source fortify to avoid a symbol diff --git a/pypy/translator/platform/posix.py b/pypy/translator/platform/posix.py --- a/pypy/translator/platform/posix.py +++ b/pypy/translator/platform/posix.py @@ -112,6 +112,10 @@ else: cflags = self.cflags + self.standalone_only + # disables glibc source fortify because + # it requires glibc 2.15 for the __fdel_chk symbol + cflags.append('-D_FORTIFY_SOURCE=0') + m = GnuMakefile(path) m.exe_name = exe_name m.eci = eci From noreply at buildbot.pypy.org Fri Jan 11 18:50:57 2013 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Fri, 11 Jan 2013 18:50:57 +0100 (CET) Subject: [pypy-commit] pypy link-old-glibc-abi: cflags is a tuple Message-ID: <20130111175057.297AC1C05DD@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: link-old-glibc-abi Changeset: r59960:7afca3399960 Date: 2013-01-11 14:33 +0100 http://bitbucket.org/pypy/pypy/changeset/7afca3399960/ Log: cflags is a tuple diff --git a/pypy/translator/platform/posix.py b/pypy/translator/platform/posix.py --- a/pypy/translator/platform/posix.py +++ b/pypy/translator/platform/posix.py @@ -114,7 +114,7 @@ # disables glibc source fortify because # it requires glibc 2.15 for the __fdel_chk symbol - cflags.append('-D_FORTIFY_SOURCE=0') + cflags += '-D_FORTIFY_SOURCE=0', m = GnuMakefile(path) m.exe_name = exe_name From noreply at buildbot.pypy.org Fri Jan 11 18:50:58 2013 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Fri, 11 Jan 2013 18:50:58 +0100 (CET) Subject: [pypy-commit] pypy link-old-glibc-abi: add a symver for memcpy guarded by the define PYPY_FORCE_OLD_GLIBC Message-ID: <20130111175058.53FA31C05DD@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: link-old-glibc-abi Changeset: r59961:855e1c2e6df0 Date: 2013-01-11 18:48 +0100 http://bitbucket.org/pypy/pypy/changeset/855e1c2e6df0/ Log: add a symver for memcpy guarded by the define PYPY_FORCE_OLD_GLIBC diff --git a/pypy/module/cpyext/src/mysnprintf.c b/pypy/module/cpyext/src/mysnprintf.c --- a/pypy/module/cpyext/src/mysnprintf.c +++ b/pypy/module/cpyext/src/mysnprintf.c @@ -38,6 +38,11 @@ CAUTION: Unlike C99, str != NULL and size > 0 are required. */ +#ifdef PYPY_FORCE_OLD_GLIBC +__asm__(".symver memcpy,memcpy at GLIBC_2.2.5"); +#endif + + int PyOS_snprintf(char *str, size_t size, const char *format, ...) { diff --git a/pypy/module/cpyext/src/structseq.c b/pypy/module/cpyext/src/structseq.c --- a/pypy/module/cpyext/src/structseq.c +++ b/pypy/module/cpyext/src/structseq.c @@ -5,6 +5,10 @@ #include "structmember.h" #include "structseq.h" +#ifdef PYPY_FORCE_OLD_GLIBC +__asm__(".symver memcpy,memcpy at GLIBC_2.2.5"); +#endif + static char visible_length_key[] = "n_sequence_fields"; static char real_length_key[] = "n_fields"; static char unnamed_fields_key[] = "n_unnamed_fields"; diff --git a/pypy/translator/c/src/g_prerequisite.h b/pypy/translator/c/src/g_prerequisite.h --- a/pypy/translator/c/src/g_prerequisite.h +++ b/pypy/translator/c/src/g_prerequisite.h @@ -18,5 +18,8 @@ typedef unsigned char bool_t; #endif +#ifdef PYPY_FORCE_OLD_GLIBC +__asm__(".symver memcpy,memcpy at GLIBC_2.2.5"); +#endif #include "src/align.h" diff --git a/pypy/translator/c/src/stacklet/stacklet.c b/pypy/translator/c/src/stacklet/stacklet.c --- a/pypy/translator/c/src/stacklet/stacklet.c +++ b/pypy/translator/c/src/stacklet/stacklet.c @@ -31,6 +31,10 @@ #include #endif +#ifdef PYPY_FORCE_OLD_GLIBC +__asm__(".symver memcpy,memcpy at GLIBC_2.2.5"); +#endif + /************************************************************/ struct stacklet_s { From noreply at buildbot.pypy.org Fri Jan 11 18:54:03 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Fri, 11 Jan 2013 18:54:03 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Moved most parts of unicodedb to rpython Message-ID: <20130111175403.E60161C05DD@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59962:9c4b2ece9cd4 Date: 2013-01-11 18:52 +0100 http://bitbucket.org/pypy/pypy/changeset/9c4b2ece9cd4/ Log: Moved most parts of unicodedb to rpython diff --git a/pypy/module/unicodedata/__init__.py b/pypy/module/unicodedata/__init__.py --- a/pypy/module/unicodedata/__init__.py +++ b/pypy/module/unicodedata/__init__.py @@ -2,7 +2,7 @@ # This is the default unicodedb used in various places: # - the unicode type # - the regular expression engine -from pypy.module.unicodedata import unicodedb_5_2_0 as unicodedb +from rpython.rlib.unicodedata import unicodedb_5_2_0 as unicodedb # to get information about individual unicode chars look at: # http://www.fileformat.info/info/unicode/char/search.htm diff --git a/pypy/module/unicodedata/interp_ucd.py b/pypy/module/unicodedata/interp_ucd.py --- a/pypy/module/unicodedata/interp_ucd.py +++ b/pypy/module/unicodedata/interp_ucd.py @@ -8,9 +8,9 @@ from rpython.rlib.rarithmetic import r_longlong from rpython.rlib.objectmodel import we_are_translated from rpython.rlib.runicode import MAXUNICODE +from rpython.rlib.unicodedata import unicodedb_5_2_0, unicodedb_3_2_0 import sys -from pypy.module.unicodedata import unicodedb_5_2_0, unicodedb_3_2_0 # Contants for Hangul characters SBase = 0xAC00 diff --git a/pypy/module/unicodedata/test_interp_ucd.py b/pypy/module/unicodedata/test_interp_ucd.py new file mode 100644 --- /dev/null +++ b/pypy/module/unicodedata/test_interp_ucd.py @@ -0,0 +1,22 @@ +from rpython.rtyper.test.tool import BaseRtypingTest, LLRtypeMixin +from rpython.rlib.unicodedata import unicodedb_5_2_0 +from pypy.module.unicodedata.interp_ucd import code_to_unichr + +class TestTranslated(BaseRtypingTest, LLRtypeMixin): + + def test_translated(self): + def f(n): + if n == 0: + return -1 + else: + u = unicodedb_5_2_0.lookup("GOTHIC LETTER FAIHU") + return u + res = self.interpret(f, [1]) + print hex(res) + assert res == f(1) + + def test_code_to_unichr(self): + def f(c): + return code_to_unichr(c) + u'' + res = self.ll_to_unicode(self.interpret(f, [0x10346])) + assert res == u'\U00010346' diff --git a/rpython/rlib/rsre/rsre_re.py b/rpython/rlib/rsre/rsre_re.py --- a/rpython/rlib/rsre/rsre_re.py +++ b/rpython/rlib/rsre/rsre_re.py @@ -5,7 +5,7 @@ import re, sys from rpython.rlib.rsre import rsre_core, rsre_char from rpython.rlib.rsre.test.test_match import get_code as _get_code -from pypy.module.unicodedata import unicodedb +from rpython.rlib.unicodedata import unicodedb from rpython.rlib.objectmodel import specialize rsre_char.set_unicode_db(unicodedb) diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py --- a/rpython/rlib/runicode.py +++ b/rpython/rlib/runicode.py @@ -4,7 +4,7 @@ from rpython.rlib.objectmodel import we_are_translated, specialize, enforceargs from rpython.rlib.rstring import StringBuilder, UnicodeBuilder from rpython.rlib.rarithmetic import r_uint, intmask -from pypy.module.unicodedata import unicodedb +from rpython.rlib.unicodedata import unicodedb if rffi.sizeof(lltype.UniChar) == 4: MAXUNICODE = 0x10ffff diff --git a/pypy/module/unicodedata/CompositionExclusions-3.2.0.txt b/rpython/rlib/unicodedata/CompositionExclusions-3.2.0.txt rename from pypy/module/unicodedata/CompositionExclusions-3.2.0.txt rename to rpython/rlib/unicodedata/CompositionExclusions-3.2.0.txt diff --git a/pypy/module/unicodedata/CompositionExclusions-5.2.0.txt b/rpython/rlib/unicodedata/CompositionExclusions-5.2.0.txt rename from pypy/module/unicodedata/CompositionExclusions-5.2.0.txt rename to rpython/rlib/unicodedata/CompositionExclusions-5.2.0.txt diff --git a/pypy/module/unicodedata/EastAsianWidth-3.2.0.txt b/rpython/rlib/unicodedata/EastAsianWidth-3.2.0.txt rename from pypy/module/unicodedata/EastAsianWidth-3.2.0.txt rename to rpython/rlib/unicodedata/EastAsianWidth-3.2.0.txt diff --git a/pypy/module/unicodedata/EastAsianWidth-5.2.0.txt b/rpython/rlib/unicodedata/EastAsianWidth-5.2.0.txt rename from pypy/module/unicodedata/EastAsianWidth-5.2.0.txt rename to rpython/rlib/unicodedata/EastAsianWidth-5.2.0.txt diff --git a/pypy/module/unicodedata/LineBreak-3.2.0.txt b/rpython/rlib/unicodedata/LineBreak-3.2.0.txt rename from pypy/module/unicodedata/LineBreak-3.2.0.txt rename to rpython/rlib/unicodedata/LineBreak-3.2.0.txt diff --git a/pypy/module/unicodedata/LineBreak-5.2.0.txt b/rpython/rlib/unicodedata/LineBreak-5.2.0.txt rename from pypy/module/unicodedata/LineBreak-5.2.0.txt rename to rpython/rlib/unicodedata/LineBreak-5.2.0.txt diff --git a/pypy/module/unicodedata/UnicodeData-3.2.0.txt b/rpython/rlib/unicodedata/UnicodeData-3.2.0.txt rename from pypy/module/unicodedata/UnicodeData-3.2.0.txt rename to rpython/rlib/unicodedata/UnicodeData-3.2.0.txt diff --git a/pypy/module/unicodedata/UnicodeData-5.2.0.txt b/rpython/rlib/unicodedata/UnicodeData-5.2.0.txt rename from pypy/module/unicodedata/UnicodeData-5.2.0.txt rename to rpython/rlib/unicodedata/UnicodeData-5.2.0.txt diff --git a/pypy/module/unicodedata/UnihanNumeric-3.2.0.txt b/rpython/rlib/unicodedata/UnihanNumeric-3.2.0.txt rename from pypy/module/unicodedata/UnihanNumeric-3.2.0.txt rename to rpython/rlib/unicodedata/UnihanNumeric-3.2.0.txt diff --git a/pypy/module/unicodedata/UnihanNumeric-5.2.0.txt b/rpython/rlib/unicodedata/UnihanNumeric-5.2.0.txt rename from pypy/module/unicodedata/UnihanNumeric-5.2.0.txt rename to rpython/rlib/unicodedata/UnihanNumeric-5.2.0.txt diff --git a/pypy/module/unicodedata/__init__.py b/rpython/rlib/unicodedata/__init__.py copy from pypy/module/unicodedata/__init__.py copy to rpython/rlib/unicodedata/__init__.py --- a/pypy/module/unicodedata/__init__.py +++ b/rpython/rlib/unicodedata/__init__.py @@ -1,24 +1,7 @@ -from pypy.interpreter.mixedmodule import MixedModule # This is the default unicodedb used in various places: # - the unicode type # - the regular expression engine -from pypy.module.unicodedata import unicodedb_5_2_0 as unicodedb +from rpython.rlib.unicodedata import unicodedb_5_2_0 as unicodedb # to get information about individual unicode chars look at: # http://www.fileformat.info/info/unicode/char/search.htm - -class Module(MixedModule): - appleveldefs = { - } - interpleveldefs = { - 'unidata_version' : 'space.wrap(interp_ucd.ucd.version)', - 'ucd_3_2_0' : 'space.wrap(interp_ucd.ucd_3_2_0)', - 'ucd_5_2_0' : 'space.wrap(interp_ucd.ucd_5_2_0)', - 'ucd' : 'space.wrap(interp_ucd.ucd)', - '__doc__' : "space.wrap('unicode character database')", - } - for name in '''lookup name decimal digit numeric category bidirectional - east_asian_width combining mirrored decomposition - normalize _get_code'''.split(): - interpleveldefs[name] = '''space.getattr(space.wrap(interp_ucd.ucd), - space.wrap("%s"))''' % name diff --git a/pypy/module/unicodedata/generate_unicodedb.py b/rpython/rlib/unicodedata/generate_unicodedb.py rename from pypy/module/unicodedata/generate_unicodedb.py rename to rpython/rlib/unicodedata/generate_unicodedb.py diff --git a/pypy/module/unicodedata/test/test_trie.py b/rpython/rlib/unicodedata/test/test_trie.py rename from pypy/module/unicodedata/test/test_trie.py rename to rpython/rlib/unicodedata/test/test_trie.py --- a/pypy/module/unicodedata/test/test_trie.py +++ b/rpython/rlib/unicodedata/test/test_trie.py @@ -1,7 +1,7 @@ import py import StringIO -from pypy.module.unicodedata import triegenerator +from rpython.rlib.unicodedata import triegenerator def setup_module(mod): mod.tmpdir = py.test.ensuretemp(mod.__name__) diff --git a/pypy/module/unicodedata/test/test_unicodedata.py b/rpython/rlib/unicodedata/test/test_unicodedata.py rename from pypy/module/unicodedata/test/test_unicodedata.py rename to rpython/rlib/unicodedata/test/test_unicodedata.py --- a/pypy/module/unicodedata/test/test_unicodedata.py +++ b/rpython/rlib/unicodedata/test/test_unicodedata.py @@ -1,5 +1,4 @@ -from rpython.rtyper.test.tool import BaseRtypingTest, LLRtypeMixin -from pypy.module.unicodedata import unicodedb_3_2_0, unicodedb_5_2_0 +from rpython.rlib.unicodedata import unicodedb_3_2_0, unicodedb_5_2_0 class AppTestUnicodeData: spaceconfig = dict(usemodules=('unicodedata',)) @@ -177,25 +176,4 @@ raises(KeyError, unicodedb_3_2_0.lookup, 'BENZENE RING WITH CIRCLE') raises(KeyError, unicodedb_3_2_0.name, 9187) -class TestTranslated(BaseRtypingTest, LLRtypeMixin): - def test_translated(self): - def f(n): - if n == 0: - return -1 - else: - u = unicodedb_5_2_0.lookup("GOTHIC LETTER FAIHU") - return u - res = self.interpret(f, [1]) - print hex(res) - assert res == f(1) - - def test_code_to_unichr(self): - from pypy.module.unicodedata.interp_ucd import code_to_unichr - def f(c): - return code_to_unichr(c) + u'' - res = self.ll_to_unicode(self.interpret(f, [0x10346])) - assert res == u'\U00010346' - - - diff --git a/pypy/module/unicodedata/triegenerator.py b/rpython/rlib/unicodedata/triegenerator.py rename from pypy/module/unicodedata/triegenerator.py rename to rpython/rlib/unicodedata/triegenerator.py diff --git a/pypy/module/unicodedata/unicodedb_3_2_0.py b/rpython/rlib/unicodedata/unicodedb_3_2_0.py rename from pypy/module/unicodedata/unicodedb_3_2_0.py rename to rpython/rlib/unicodedata/unicodedb_3_2_0.py diff --git a/pypy/module/unicodedata/unicodedb_5_2_0.py b/rpython/rlib/unicodedata/unicodedb_5_2_0.py rename from pypy/module/unicodedata/unicodedb_5_2_0.py rename to rpython/rlib/unicodedata/unicodedb_5_2_0.py From noreply at buildbot.pypy.org Fri Jan 11 18:58:35 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Fri, 11 Jan 2013 18:58:35 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed more regarding sockets Message-ID: <20130111175835.0B8471C05DD@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59963:62100b576c9a Date: 2013-01-11 18:58 +0100 http://bitbucket.org/pypy/pypy/changeset/62100b576c9a/ Log: Fixed more regarding sockets diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -43,7 +43,7 @@ # exception -- return it as a tuple. a = addr.lock() family = rffi.cast(lltype.Signed, a.c_sa_family) - datalen = addr.addrlen - offsetof(_c.sockaddr, 'c_sa_data') + datalen = addr.addrlen - rsocket.offsetof(_c.sockaddr, 'c_sa_data') rawdata = ''.join([a.c_sa_data[i] for i in range(datalen)]) addr.unlock() return space.newtuple([space.wrap(family), @@ -58,7 +58,7 @@ port = space.int_w(w_port) port = make_ushort_port(space, port) a = addr.lock(_c.sockaddr_in) - rffi.setintfield(a, 'c_sin_port', htons(port)) + rffi.setintfield(a, 'c_sin_port', rsocket.htons(port)) addr.unlock() elif hasattr(addr, 'family') and addr.family == rsocket.AF_INET6: pieces_w = space.unpackiterable(w_address) @@ -76,8 +76,8 @@ "flowinfo must be 0-1048575.")) flowinfo = rffi.cast(lltype.Unsigned, flowinfo) a = addr.lock(_c.sockaddr_in6) - rffi.setintfield(a, 'c_sin6_port', htons(port)) - rffi.setintfield(a, 'c_sin6_flowinfo', htonl(flowinfo)) + rffi.setintfield(a, 'c_sin6_port', rsocket.htons(port)) + rffi.setintfield(a, 'c_sin6_flowinfo', rsocket.htonl(flowinfo)) rffi.setintfield(a, 'c_sin6_scope_id', scope_id) addr.unlock() else: From noreply at buildbot.pypy.org Fri Jan 11 19:01:29 2013 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 11 Jan 2013 19:01:29 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: In-progress: rewrite the graphs around jit_merge_points Message-ID: <20130111180129.54B531C05DD@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r59964:9b9ee314d16e Date: 2013-01-11 19:01 +0100 http://bitbucket.org/pypy/pypy/changeset/9b9ee314d16e/ Log: In-progress: rewrite the graphs around jit_merge_points into calls to perform_transaction(). diff --git a/pypy/translator/stm/jitdriver.py b/pypy/translator/stm/jitdriver.py new file mode 100644 --- /dev/null +++ b/pypy/translator/stm/jitdriver.py @@ -0,0 +1,218 @@ +from pypy.rpython.lltypesystem import lltype +from pypy.objspace.flow.model import checkgraph, copygraph +from pypy.objspace.flow.model import Block, Link, SpaceOperation, Constant +from pypy.translator.unsimplify import split_block, varoftype +from pypy.translator.stm.stmgcintf import StmOperations +from pypy.translator.backendopt.ssa import SSA_to_SSI +from pypy.annotation.model import lltype_to_annotation, s_Int +from pypy.rpython.annlowlevel import MixLevelHelperAnnotator +from pypy.rlib import rstm + + +def find_jit_merge_point(graph): + found = [] + for block in graph.iterblocks(): + for i in range(len(block.operations)): + op = block.operations[i] + if (op.opname == 'jit_marker' and + op.args[0].value == 'jit_merge_point'): + found.append((block, i)) + if found: + assert len(found) == 1, "several jit_merge_point's in %r" % (graph,) + return found[0] + else: + return None + +def reorganize_around_jit_driver(stmtransformer, graph): + location = find_jit_merge_point(graph) + if location is not None: + JitDriverSplitter(stmtransformer, graph).split(location) + +# ____________________________________________________________ + + +class JitDriverSplitter(object): + # + # def graph(..): | def graph(..): + # stuff_before | stuff_before + # while 1: ====> while 1: + # jit_merge_point() | if should_break_transaction(): + # stuff_after | return invoke_stm(..) + # ----------------------------+ stuff_after + # + # def invoke_stm(..): + # p = new container object + # store (green args, red args) into p + # perform_transaction(callback, p) + # if p.got_exception: raise p.got_exception + # return p.result_value + # + # def callback(p): + # try: + # return run_callback(p) + # except e: + # p.got_exception = e + # return 0 # stop perform_tr() and returns + # + # def run_callback(p): + # fish (green args, red args) from p + # while 1: + # stuff_after + # if should_break_transaction(): + # store (green args, red args) into p + # return 1 # causes perform_tr() to loop and call us again + # p.result_value = result_value + # p.got_exception = NULL + # return 0 # stop perform_tr() and returns + + def __init__(self, stmtransformer, graph): + self.stmtransformer = stmtransformer + self.main_graph = graph + self.RESTYPE = graph.getreturnvar().concretetype + + def split(self, portal_location): + self.check_jitdriver(portal_location) + self.split_after_jit_merge_point(portal_location) + self.make_container_type() + # + rtyper = self.stmtransformer.translator.rtyper + self.mixlevelannotator = MixLevelHelperAnnotator(rtyper) + self.make_run_callback_function() + self.make_callback_function() + self.make_invoke_stm_function() + self.rewrite_main_graph() + self.mixlevelannotator.finish() + + def check_jitdriver(self, (portalblock, portalopindex)): + op_jitmarker = portalblock.operations[portalopindex] + assert op_jitmarker.opname == 'jit_marker' + assert op_jitmarker.args[0].value == 'jit_merge_point' + jitdriver = op_jitmarker.args[1].value + + assert not jitdriver.greens and not jitdriver.reds # XXX + assert not jitdriver.autoreds # XXX + + def split_after_jit_merge_point(self, (portalblock, portalopindex)): + split_block(None, portalblock, portalopindex + 1) + + def make_container_type(self): + self.CONTAINER = lltype.GcStruct('StmArgs', + ('result_value', self.RESTYPE)) + self.CONTAINERP = lltype.Ptr(self.CONTAINER) + + def add_call_should_break_transaction(self, block): + # add a should_break_transaction() call at the end of the block, + # turn the following link into an "if False" link, add a new + # "if True" link going to a fresh new block, and return this new + # block. + funcptr = StmOperations.should_break_transaction + c = Constant(funcptr, lltype.typeOf(funcptr)) + v1 = varoftype(lltype.Signed) + block.operations.append(SpaceOperation('direct_call', [c], v1)) + v2 = varoftype(lltype.Bool) + block.operations.append(SpaceOperation('int_is_true', [v1], v2)) + # + assert block.exitswitch is None + [link] = block.exits + block.exitswitch = v2 + link.exitcase = False + link.llexitcase = False + newblock = Block([varoftype(v.concretetype) for v in link.args]) + otherlink = Link(link.args[:], newblock) + otherlink.exitcase = True + otherlink.llexitcase = True + block.recloseblock(link, otherlink) + return newblock + + def rewrite_main_graph(self): + # add 'should_break_transaction()' + main_graph = self.main_graph + block1, i = find_jit_merge_point(main_graph) + assert i == len(block1.operations) - 1 + del block1.operations[i] + blockf = self.add_call_should_break_transaction(block1) + # + # fill in blockf with a call to invoke_stm() + v = varoftype(self.RESTYPE) + op = SpaceOperation('direct_call', [self.c_invoke_stm_func], v) + blockf.operations.append(op) + blockf.closeblock(Link([v], main_graph.returnblock)) + # + checkgraph(main_graph) + + def make_invoke_stm_function(self): + CONTAINER = self.CONTAINER + callback = self.callback_function + # + def ll_invoke_stm(): + p = lltype.malloc(CONTAINER) + rstm.perform_transaction(callback, p) + #if p.got_exception: + # raise p.got_exception + return p.result_value + # + mix = self.mixlevelannotator + c_func = mix.constfunc(ll_invoke_stm, [], + lltype_to_annotation(self.RESTYPE)) + self.c_invoke_stm_func = c_func + + def make_callback_function(self): + run_callback = self.run_callback_function + # + def ll_callback(p): + #try: + return run_callback(p) + #except Exception, e: + # p.got_exception = e + # return 0 # stop perform_tr() and returns + # + mix = self.mixlevelannotator + args_s = [lltype_to_annotation(self.CONTAINERP)] + self.callback_function = mix.delayedfunction(ll_callback, + args_s, s_Int) + + def make_run_callback_function(self): + # make a copy of the 'main_graph' + run_callback_graph = copygraph(self.main_graph) + self.run_callback_graph = run_callback_graph + # + # make a new startblock + v_p = varoftype(self.CONTAINERP) + blockst = Block([v_p]) + # + # change the startblock of callback_graph to point just after the + # jit_merge_point + block1, i = find_jit_merge_point(run_callback_graph) + assert i == len(block1.operations) - 1 + del block1.operations[i] + [link] = block1.exits + run_callback_graph.startblock = blockst + blockst.closeblock(Link([], link.target)) + # + # hack at the regular return block, to set the result into + # 'p.result_value', clear 'p.got_exception', and return 0 + blockr = run_callback_graph.returnblock + c_result_value = Constant('result_value', lltype.Void) + blockr.operations = [ + SpaceOperation('setfield', + [v_p, c_result_value, blockr.inputargs[0]], + varoftype(lltype.Void)), + #... + ] + v = varoftype(self.RESTYPE) + newblockr = Block([v]) + newblockr.operations = () + newblockr.closeblock() + blockr.recloseblock(Link([Constant(0, lltype.Signed)], newblockr)) + run_callback_graph.returnblock = newblockr + # + # add 'should_break_transaction()' at the end of the loop + blockf = self.add_call_should_break_transaction(block1) + # ...store stuff... + blockf.closeblock(Link([Constant(1, lltype.Signed)], newblockr)) + # + SSA_to_SSI(run_callback_graph) # to pass 'p' everywhere + checkgraph(run_callback_graph) + # + mix = self.mixlevelannotator + self.run_callback_function = mix.graph2delayed(run_callback_graph) diff --git a/pypy/translator/stm/test/test_jitdriver.py b/pypy/translator/stm/test/test_jitdriver.py new file mode 100644 --- /dev/null +++ b/pypy/translator/stm/test/test_jitdriver.py @@ -0,0 +1,22 @@ +from pypy.rpython.lltypesystem import lltype, rffi +from pypy.translator.stm.test.transform2_support import BaseTestTransform +from pypy.rlib.jit import JitDriver + + +class TestJitDriver(BaseTestTransform): + do_jit_driver = True + + def test_loop_no_arg(self): + class X: + counter = 10 + x = X() + myjitdriver = JitDriver(greens=[], reds=[]) + + def f1(): + while x.counter > 0: + myjitdriver.jit_merge_point() + x.counter -= 1 + return 'X' + + res = self.interpret(f1, []) + assert res == 'X' diff --git a/pypy/translator/stm/test/test_writebarrier.py b/pypy/translator/stm/test/test_writebarrier.py --- a/pypy/translator/stm/test/test_writebarrier.py +++ b/pypy/translator/stm/test/test_writebarrier.py @@ -4,7 +4,6 @@ class TestTransform(BaseTestTransform): do_write_barrier = True - do_turn_inevitable = False def test_simple_read(self): X = lltype.GcStruct('X', ('foo', lltype.Signed)) diff --git a/pypy/translator/stm/test/transform2_support.py b/pypy/translator/stm/test/transform2_support.py --- a/pypy/translator/stm/test/transform2_support.py +++ b/pypy/translator/stm/test/transform2_support.py @@ -21,6 +21,9 @@ class BaseTestTransform(object): + do_write_barrier = False + do_turn_inevitable = False + do_jit_driver = False def build_state(self): self.writemode = set() @@ -44,6 +47,8 @@ # self.translator = interp.typer.annotator.translator self.stmtransformer = STMTransformer(self.translator) + if self.do_jit_driver: + self.stmtransformer.transform_jit_driver() if self.do_write_barrier: self.stmtransformer.transform_write_barrier() if self.do_turn_inevitable: diff --git a/pypy/translator/stm/transform2.py b/pypy/translator/stm/transform2.py --- a/pypy/translator/stm/transform2.py +++ b/pypy/translator/stm/transform2.py @@ -7,6 +7,7 @@ def transform(self): assert not hasattr(self.translator, 'stm_transformation_applied') self.start_log() + self.transform_jit_driver() self.transform_write_barrier() self.transform_turn_inevitable() self.print_logs() @@ -27,6 +28,12 @@ for graph in self.translator.graphs: insert_turn_inevitable(graph) + def transform_jit_driver(self): + from pypy.translator.stm.jitdriver import reorganize_around_jit_driver + # + for graph in self.translator.graphs: + reorganize_around_jit_driver(self, graph) + def start_log(self): from pypy.translator.c.support import log log.info("Software Transactional Memory transformation") From noreply at buildbot.pypy.org Fri Jan 11 19:31:51 2013 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 11 Jan 2013 19:31:51 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: In-progress Message-ID: <20130111183151.A12041C05DD@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r59965:57c8b6db810e Date: 2013-01-11 19:31 +0100 http://bitbucket.org/pypy/pypy/changeset/57c8b6db810e/ Log: In-progress diff --git a/pypy/rlib/rstm.py b/pypy/rlib/rstm.py --- a/pypy/rlib/rstm.py +++ b/pypy/rlib/rstm.py @@ -6,8 +6,7 @@ from pypy.rlib.rposix import get_errno, set_errno from pypy.rpython.lltypesystem import lltype, llmemory, rffi, rclass from pypy.rpython.lltypesystem.lloperation import llop -from pypy.rpython.annlowlevel import (cast_base_ptr_to_instance, - cast_instance_to_base_ptr, +from pypy.rpython.annlowlevel import (cast_instance_to_base_ptr, llhelper) def is_inevitable(): @@ -75,33 +74,32 @@ # ____________________________________________________________ - at specialize.memo() -def _get_stm_callback(func, argcls): - def _stm_callback(llarg, retry_counter): +def make_perform_transaction(func, CONTAINERP): + # + def _stm_callback(llcontainer, retry_counter): if not is_atomic(): llop.stm_start_transaction(lltype.Void) - llarg = rffi.cast(rclass.OBJECTPTR, llarg) - arg = cast_base_ptr_to_instance(argcls, llarg) + llcontainer = rffi.cast(CONTAINERP, llcontainer) try: - res = func(arg, retry_counter) - except: - fatalerror("no exception allowed in stm_callback") - assert 0 + res = func(llcontainer, retry_counter) + llcontainer.got_exception = lltype.nullptr(rclass.OBJECT) + except Exception, e: + res = 0 # stop perform_transaction() and returns + lle = cast_instance_to_base_ptr(e) + llcontainer.got_exception = lle if not is_atomic(): llop.stm_stop_transaction(lltype.Void) return res - return _stm_callback - - at specialize.arg(0, 1) -def perform_transaction(func, argcls, arg): - ll_assert(arg is None or isinstance(arg, argcls), - "perform_transaction: wrong class") - before_external_call() - llarg = cast_instance_to_base_ptr(arg) - adr_of_top = llop.gc_adr_of_root_stack_top(llmemory.Address) - callback = _get_stm_callback(func, argcls) - llcallback = llhelper(stmgcintf.StmOperations.CALLBACK_TX, callback) - stmgcintf.StmOperations.perform_transaction(llcallback, llarg, adr_of_top) - after_external_call() - keepalive_until_here(arg) -perform_transaction._transaction_break_ = True + # + def perform_transaction(llcontainer): + before_external_call() + adr_of_top = llop.gc_adr_of_root_stack_top(llmemory.Address) + llcallback = llhelper(stmgcintf.StmOperations.CALLBACK_TX, + _stm_callback) + stmgcintf.StmOperations.perform_transaction(llcallback, llcontainer, + adr_of_top) + after_external_call() + keepalive_until_here(llcontainer) + perform_transaction._transaction_break_ = True + # + return perform_transaction diff --git a/pypy/translator/stm/jitdriver.py b/pypy/translator/stm/jitdriver.py --- a/pypy/translator/stm/jitdriver.py +++ b/pypy/translator/stm/jitdriver.py @@ -1,11 +1,12 @@ -from pypy.rpython.lltypesystem import lltype +from pypy.rpython.lltypesystem import lltype, rclass from pypy.objspace.flow.model import checkgraph, copygraph from pypy.objspace.flow.model import Block, Link, SpaceOperation, Constant from pypy.translator.unsimplify import split_block, varoftype from pypy.translator.stm.stmgcintf import StmOperations from pypy.translator.backendopt.ssa import SSA_to_SSI from pypy.annotation.model import lltype_to_annotation, s_Int -from pypy.rpython.annlowlevel import MixLevelHelperAnnotator +from pypy.rpython.annlowlevel import (MixLevelHelperAnnotator, + cast_base_ptr_to_instance) from pypy.rlib import rstm @@ -47,14 +48,9 @@ # if p.got_exception: raise p.got_exception # return p.result_value # - # def callback(p): - # try: - # return run_callback(p) - # except e: - # p.got_exception = e - # return 0 # stop perform_tr() and returns + # (note that perform_transaction() itself will fill p.got_exception) # - # def run_callback(p): + # def callback(p, retry_counter): # fish (green args, red args) from p # while 1: # stuff_after @@ -77,7 +73,6 @@ # rtyper = self.stmtransformer.translator.rtyper self.mixlevelannotator = MixLevelHelperAnnotator(rtyper) - self.make_run_callback_function() self.make_callback_function() self.make_invoke_stm_function() self.rewrite_main_graph() @@ -97,7 +92,8 @@ def make_container_type(self): self.CONTAINER = lltype.GcStruct('StmArgs', - ('result_value', self.RESTYPE)) + ('result_value', self.RESTYPE), + ('got_exception', rclass.OBJECTPTR)) self.CONTAINERP = lltype.Ptr(self.CONTAINER) def add_call_should_break_transaction(self, block): @@ -143,12 +139,14 @@ def make_invoke_stm_function(self): CONTAINER = self.CONTAINER callback = self.callback_function + perform_transaction = rstm.make_perform_transaction(callback, + self.CONTAINERP) # def ll_invoke_stm(): p = lltype.malloc(CONTAINER) - rstm.perform_transaction(callback, p) - #if p.got_exception: - # raise p.got_exception + perform_transaction(p) + if p.got_exception: + raise cast_base_ptr_to_instance(Exception, p.got_exception) return p.result_value # mix = self.mixlevelannotator @@ -157,41 +155,35 @@ self.c_invoke_stm_func = c_func def make_callback_function(self): - run_callback = self.run_callback_function - # - def ll_callback(p): - #try: - return run_callback(p) - #except Exception, e: - # p.got_exception = e - # return 0 # stop perform_tr() and returns - # - mix = self.mixlevelannotator - args_s = [lltype_to_annotation(self.CONTAINERP)] - self.callback_function = mix.delayedfunction(ll_callback, - args_s, s_Int) - - def make_run_callback_function(self): # make a copy of the 'main_graph' - run_callback_graph = copygraph(self.main_graph) - self.run_callback_graph = run_callback_graph + callback_graph = copygraph(self.main_graph) + callback_graph.name += '_stm' + self.callback_graph = callback_graph + #for v1, v2 in zip( + # self.main_graph.getargs() + [self.main_graph.getreturnvar()], + # callback_graph.getargs() + [callback_graph.getreturnvar()]): + # self.stmtransformer.translator.annotator.transfer_binding(v2, v1) # # make a new startblock v_p = varoftype(self.CONTAINERP) - blockst = Block([v_p]) + v_retry_counter = varoftype(lltype.Signed) + blockst = Block([v_p, v_retry_counter]) + annotator = self.stmtransformer.translator.annotator + annotator.setbinding(v_p, lltype_to_annotation(self.CONTAINERP)) + annotator.setbinding(v_retry_counter, s_Int) # # change the startblock of callback_graph to point just after the # jit_merge_point - block1, i = find_jit_merge_point(run_callback_graph) + block1, i = find_jit_merge_point(callback_graph) assert i == len(block1.operations) - 1 del block1.operations[i] [link] = block1.exits - run_callback_graph.startblock = blockst + callback_graph.startblock = blockst blockst.closeblock(Link([], link.target)) # # hack at the regular return block, to set the result into # 'p.result_value', clear 'p.got_exception', and return 0 - blockr = run_callback_graph.returnblock + blockr = callback_graph.returnblock c_result_value = Constant('result_value', lltype.Void) blockr.operations = [ SpaceOperation('setfield', @@ -199,20 +191,24 @@ varoftype(lltype.Void)), #... ] - v = varoftype(self.RESTYPE) + v = varoftype(lltype.Signed) + annotator.setbinding(v, s_Int) newblockr = Block([v]) newblockr.operations = () newblockr.closeblock() blockr.recloseblock(Link([Constant(0, lltype.Signed)], newblockr)) - run_callback_graph.returnblock = newblockr + callback_graph.returnblock = newblockr # # add 'should_break_transaction()' at the end of the loop blockf = self.add_call_should_break_transaction(block1) # ...store stuff... blockf.closeblock(Link([Constant(1, lltype.Signed)], newblockr)) # - SSA_to_SSI(run_callback_graph) # to pass 'p' everywhere - checkgraph(run_callback_graph) + SSA_to_SSI(callback_graph) # to pass 'p' everywhere + checkgraph(callback_graph) # + FUNCTYPE = lltype.FuncType([self.CONTAINERP, lltype.Signed], + lltype.Signed) mix = self.mixlevelannotator - self.run_callback_function = mix.graph2delayed(run_callback_graph) + self.callback_function = mix.graph2delayed(callback_graph, + FUNCTYPE=FUNCTYPE) From noreply at buildbot.pypy.org Fri Jan 11 19:38:12 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Fri, 11 Jan 2013 19:38:12 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed more regarding sockets Message-ID: <20130111183812.ACC3F1C0DCB@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59966:9f897963d1f0 Date: 2013-01-11 19:37 +0100 http://bitbucket.org/pypy/pypy/changeset/9f897963d1f0/ Log: Fixed more regarding sockets diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -41,6 +41,7 @@ space.wrap(addr.get_groups())]) # If we don't know the address family, don't raise an # exception -- return it as a tuple. + from rpython.rlib import _rsocket_rffi as _c a = addr.lock() family = rffi.cast(lltype.Signed, a.c_sa_family) datalen = addr.addrlen - rsocket.offsetof(_c.sockaddr, 'c_sa_data') From noreply at buildbot.pypy.org Fri Jan 11 20:06:40 2013 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 11 Jan 2013 20:06:40 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: Finish it, maybe. Hard to test! Tested so far by staring at the graphs Message-ID: <20130111190641.1D0AF1C0DCB@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r59967:20fe85f53d29 Date: 2013-01-11 20:06 +0100 http://bitbucket.org/pypy/pypy/changeset/20fe85f53d29/ Log: Finish it, maybe. Hard to test! Tested so far by staring at the graphs :-( diff --git a/pypy/rlib/rstm.py b/pypy/rlib/rstm.py --- a/pypy/rlib/rstm.py +++ b/pypy/rlib/rstm.py @@ -82,7 +82,6 @@ llcontainer = rffi.cast(CONTAINERP, llcontainer) try: res = func(llcontainer, retry_counter) - llcontainer.got_exception = lltype.nullptr(rclass.OBJECT) except Exception, e: res = 0 # stop perform_transaction() and returns lle = cast_instance_to_base_ptr(e) diff --git a/pypy/translator/stm/jitdriver.py b/pypy/translator/stm/jitdriver.py --- a/pypy/translator/stm/jitdriver.py +++ b/pypy/translator/stm/jitdriver.py @@ -8,6 +8,7 @@ from pypy.rpython.annlowlevel import (MixLevelHelperAnnotator, cast_base_ptr_to_instance) from pypy.rlib import rstm +from pypy.tool.sourcetools import compile2 def find_jit_merge_point(graph): @@ -83,17 +84,18 @@ assert op_jitmarker.opname == 'jit_marker' assert op_jitmarker.args[0].value == 'jit_merge_point' jitdriver = op_jitmarker.args[1].value - - assert not jitdriver.greens and not jitdriver.reds # XXX assert not jitdriver.autoreds # XXX def split_after_jit_merge_point(self, (portalblock, portalopindex)): - split_block(None, portalblock, portalopindex + 1) + link = split_block(None, portalblock, portalopindex + 1) + self.TYPES = [v.concretetype for v in link.args] def make_container_type(self): + args = [('a%d' % i, self.TYPES[i]) for i in range(len(self.TYPES))] self.CONTAINER = lltype.GcStruct('StmArgs', ('result_value', self.RESTYPE), - ('got_exception', rclass.OBJECTPTR)) + ('got_exception', rclass.OBJECTPTR), + *args) self.CONTAINERP = lltype.Ptr(self.CONTAINER) def add_call_should_break_transaction(self, block): @@ -130,7 +132,8 @@ # # fill in blockf with a call to invoke_stm() v = varoftype(self.RESTYPE) - op = SpaceOperation('direct_call', [self.c_invoke_stm_func], v) + op = SpaceOperation('direct_call', + [self.c_invoke_stm_func] + blockf.inputargs, v) blockf.operations.append(op) blockf.closeblock(Link([v], main_graph.returnblock)) # @@ -141,16 +144,28 @@ callback = self.callback_function perform_transaction = rstm.make_perform_transaction(callback, self.CONTAINERP) - # - def ll_invoke_stm(): + irange = range(len(self.TYPES)) + source = """if 1: + def ll_invoke_stm(%s): p = lltype.malloc(CONTAINER) + %s perform_transaction(p) if p.got_exception: raise cast_base_ptr_to_instance(Exception, p.got_exception) return p.result_value +""" % (', '.join(['a%d' % i for i in irange]), + '\n '.join(['p.a%d = a%d' % (i, i) for i in irange])) + d = {'CONTAINER': CONTAINER, + 'lltype': lltype, + 'perform_transaction': perform_transaction, + 'cast_base_ptr_to_instance': cast_base_ptr_to_instance, + } + exec compile2(source) in d + ll_invoke_stm = d['ll_invoke_stm'] # mix = self.mixlevelannotator - c_func = mix.constfunc(ll_invoke_stm, [], + c_func = mix.constfunc(ll_invoke_stm, + map(lltype_to_annotation, self.TYPES), lltype_to_annotation(self.RESTYPE)) self.c_invoke_stm_func = c_func @@ -159,6 +174,7 @@ callback_graph = copygraph(self.main_graph) callback_graph.name += '_stm' self.callback_graph = callback_graph + self.stmtransformer.translator.graphs.append(callback_graph) #for v1, v2 in zip( # self.main_graph.getargs() + [self.main_graph.getreturnvar()], # callback_graph.getargs() + [callback_graph.getreturnvar()]): @@ -179,17 +195,31 @@ del block1.operations[i] [link] = block1.exits callback_graph.startblock = blockst - blockst.closeblock(Link([], link.target)) + # + # fill in the operations of blockst: getfields reading all live vars + a_vars = [] + for i in range(len(self.TYPES)): + c_a_i = Constant('a%d' % i, lltype.Void) + v_a_i = varoftype(self.TYPES[i]) + blockst.operations.append( + SpaceOperation('getfield', [v_p, c_a_i], v_a_i)) + a_vars.append(v_a_i) + blockst.closeblock(Link(a_vars, link.target)) # # hack at the regular return block, to set the result into # 'p.result_value', clear 'p.got_exception', and return 0 blockr = callback_graph.returnblock c_result_value = Constant('result_value', lltype.Void) + c_got_exception = Constant('got_exception', lltype.Void) + c_null = Constant(lltype.nullptr(self.CONTAINER.got_exception.TO), + self.CONTAINER.got_exception) blockr.operations = [ SpaceOperation('setfield', [v_p, c_result_value, blockr.inputargs[0]], varoftype(lltype.Void)), - #... + SpaceOperation('setfield', + [v_p, c_got_exception, c_null], + varoftype(lltype.Void)), ] v = varoftype(lltype.Signed) annotator.setbinding(v, s_Int) @@ -201,7 +231,14 @@ # # add 'should_break_transaction()' at the end of the loop blockf = self.add_call_should_break_transaction(block1) - # ...store stuff... + # store the variables again into v_p + for i in range(len(self.TYPES)): + c_a_i = Constant('a%d' % i, lltype.Void) + v_a_i = blockf.inputargs[i] + assert v_a_i.concretetype == self.TYPES[i] + blockf.operations.append( + SpaceOperation('setfield', [v_p, c_a_i, v_a_i], + varoftype(lltype.Void))) blockf.closeblock(Link([Constant(1, lltype.Signed)], newblockr)) # SSA_to_SSI(callback_graph) # to pass 'p' everywhere diff --git a/pypy/translator/stm/test/test_jitdriver.py b/pypy/translator/stm/test/test_jitdriver.py --- a/pypy/translator/stm/test/test_jitdriver.py +++ b/pypy/translator/stm/test/test_jitdriver.py @@ -20,3 +20,18 @@ res = self.interpret(f1, []) assert res == 'X' + + def test_loop_args(self): + class X: + counter = 100 + x = X() + myjitdriver = JitDriver(greens=['a'], reds=['b', 'c']) + + def f1(a, b, c): + while x.counter > 0: + myjitdriver.jit_merge_point(a=a, b=b, c=c) + x.counter -= (ord(a) + rffi.cast(lltype.Signed, b) + c) + return 'X' + + res = self.interpret(f1, ['\x03', rffi.cast(rffi.SHORT, 4), 2]) + assert res == 'X' diff --git a/pypy/translator/stm/test/transform2_support.py b/pypy/translator/stm/test/transform2_support.py --- a/pypy/translator/stm/test/transform2_support.py +++ b/pypy/translator/stm/test/transform2_support.py @@ -56,6 +56,9 @@ if option.view: self.translator.view() # + if self.do_jit_driver: + import py + py.test.skip("XXX how to test?") result = interp.eval_graph(self.graph, args) return result From noreply at buildbot.pypy.org Fri Jan 11 20:45:41 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Fri, 11 Jan 2013 20:45:41 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed missing import rename Message-ID: <20130111194541.075601C0343@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59968:7e45f3c1e8f6 Date: 2013-01-11 20:16 +0100 http://bitbucket.org/pypy/pypy/changeset/7e45f3c1e8f6/ Log: Fixed missing import rename diff --git a/rpython/rlib/rsre/test/test_char.py b/rpython/rlib/rsre/test/test_char.py --- a/rpython/rlib/rsre/test/test_char.py +++ b/rpython/rlib/rsre/test/test_char.py @@ -2,7 +2,7 @@ from rpython.rlib.rsre.rsre_char import SRE_FLAG_LOCALE, SRE_FLAG_UNICODE def setup_module(mod): - from pypy.module.unicodedata import unicodedb + from rpython.rlib.unicodedata import unicodedb rsre_char.set_unicode_db(unicodedb) UPPER_PI = 0x3a0 From noreply at buildbot.pypy.org Fri Jan 11 20:45:42 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Fri, 11 Jan 2013 20:45:42 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed rstruct for the split Message-ID: <20130111194542.737321C0343@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59969:b60e1c94763c Date: 2013-01-11 20:44 +0100 http://bitbucket.org/pypy/pypy/changeset/b60e1c94763c/ Log: Fixed rstruct for the split diff --git a/pypy/module/struct/interp_struct.py b/pypy/module/struct/interp_struct.py --- a/pypy/module/struct/interp_struct.py +++ b/pypy/module/struct/interp_struct.py @@ -1,4 +1,5 @@ from pypy.interpreter.gateway import unwrap_spec +from pypy.interpreter.error import OperationError from pypy.module.struct.formatiterator import PackFormatIterator, UnpackFormatIterator from rpython.rlib import jit from rpython.rlib.rstruct.error import StructError @@ -13,8 +14,12 @@ fmtiter = CalcSizeFormatIterator() try: fmtiter.interpret(format) + except StructOverflowError, e: + raise OperationError(space.w_OverflowError, space.wrap(self.msg)) except StructError, e: - raise e.at_applevel(space) + w_module = space.getbuiltinmodule('struct') + w_error = space.getattr(w_module, space.wrap('error')) + raise OperationError(w_error, space.wrap(e.msg)) return fmtiter.totalsize @unwrap_spec(format=str) @@ -26,8 +31,12 @@ fmtiter = PackFormatIterator(space, args_w, size) try: fmtiter.interpret(format) + except StructOverflowError, e: + raise OperationError(space.w_OverflowError, space.wrap(self.msg)) except StructError, e: - raise e.at_applevel(space) + w_module = space.getbuiltinmodule('struct') + w_error = space.getattr(w_module, space.wrap('error')) + raise OperationError(w_error, space.wrap(e.msg)) return space.wrap(fmtiter.result.build()) @@ -36,6 +45,10 @@ fmtiter = UnpackFormatIterator(space, input) try: fmtiter.interpret(format) + except StructOverflowError, e: + raise OperationError(space.w_OverflowError, space.wrap(self.msg)) except StructError, e: - raise e.at_applevel(space) + w_module = space.getbuiltinmodule('struct') + w_error = space.getattr(w_module, space.wrap('error')) + raise OperationError(w_error, space.wrap(e.msg)) return space.newtuple(fmtiter.result_w[:]) diff --git a/rpython/rlib/rstruct/error.py b/rpython/rlib/rstruct/error.py --- a/rpython/rlib/rstruct/error.py +++ b/rpython/rlib/rstruct/error.py @@ -1,22 +1,11 @@ class StructError(Exception): - "Interp-level error that gets mapped to an app-level struct.error." - def __init__(self, msg): self.msg = msg def __str__(self): return self.msg - def at_applevel(self, space): - from pypy.interpreter.error import OperationError - w_module = space.getbuiltinmodule('struct') - w_error = space.getattr(w_module, space.wrap('error')) - return OperationError(w_error, space.wrap(self.msg)) - class StructOverflowError(StructError): - - def at_applevel(self, space): - from pypy.interpreter.error import OperationError - return OperationError(space.w_OverflowError, space.wrap(self.msg)) + pass From noreply at buildbot.pypy.org Fri Jan 11 20:46:24 2013 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 11 Jan 2013 20:46:24 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: Start to adapt the targetdemo2 test. Message-ID: <20130111194624.1D7CD1C0343@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r59970:7a6514ddc5de Date: 2013-01-11 20:32 +0100 http://bitbucket.org/pypy/pypy/changeset/7a6514ddc5de/ Log: Start to adapt the targetdemo2 test. diff --git a/pypy/translator/stm/test/targetdemo2.py b/pypy/translator/stm/test/targetdemo2.py --- a/pypy/translator/stm/test/targetdemo2.py +++ b/pypy/translator/stm/test/targetdemo2.py @@ -1,10 +1,10 @@ import time from pypy.module.thread import ll_thread -from pypy.rlib import rstm +from pypy.rlib import rstm, jit from pypy.rlib.objectmodel import invoke_around_extcall, we_are_translated from pypy.rlib.objectmodel import compute_identity_hash from pypy.rlib.debug import ll_assert -from pypy.rpython.lltypesystem import lltype, rffi +from pypy.rpython.lltypesystem import lltype, rffi, rclass class Node: @@ -65,6 +65,12 @@ print "check ok!" +jitdriver_hash = jit.JitDriver(greens=[], reds=['self']) +jitdriver_inev = jit.JitDriver(greens=[], reds=['self']) +jitdriver_ptreq = jit.JitDriver(greens=[], reds=['self']) +jitdriver_really = jit.JitDriver(greens=[], reds=['self']) + + class ThreadRunner(object): arg = None @@ -77,21 +83,23 @@ try: self.value = 0 self.lst = [] - rstm.perform_transaction(ThreadRunner.check_hash, - ThreadRunner, self) + self.do_check_hash() self.value = 0 - rstm.perform_transaction(ThreadRunner.check_inev, - ThreadRunner, self) + self.do_check_inev() self.value = 0 self.arg = Arg() self.glob_p = lltype.malloc(STRUCT) - rstm.perform_transaction(ThreadRunner.check_ptr_equality, - ThreadRunner, self) - rstm.perform_transaction(ThreadRunner.run_really, - ThreadRunner, self) + self.do_check_ptr_equality() + self.do_run_really() finally: self.finished_lock.release() + def do_run_really(self): + while True: + jitdriver_really.jit_merge_point(self=self) + if not self.run_really(0): + break + def run_really(self, retry_counter): if self.value == glob.LENGTH // 2: print "atomic!" @@ -106,7 +114,11 @@ # add_at_end_of_chained_list(glob.anchor, self.value, self.index) self.value += 1 - return int(self.value < glob.LENGTH) + return self.value < glob.LENGTH + + def do_check_ptr_equality(self): + jitdriver_ptreq.jit_merge_point(self=self) + self.check_ptr_equality(0) def check_ptr_equality(self, retry_counter): assert self.glob_p != lltype.nullptr(STRUCT) @@ -115,7 +127,12 @@ raw1 = rffi.cast(rffi.CCHARP, retry_counter) raw2 = rffi.cast(rffi.CCHARP, -1) ll_assert(raw1 != raw2, "ERROR: retry_counter == -1") - return 0 + + def do_check_inev(self): + while True: + jitdriver_inev.jit_merge_point(self=self) + if not self.check_inev(0): + break def _check_content(self, content): ll_assert(glob.othernode2.value == content, "bogus value after inev") @@ -135,7 +152,13 @@ for n in glob.othernodes: # lots of unrelated writes in-between n.value = new_value glob.othernode2.value = new_value - return int(self.value < glob.LENGTH) + return self.value < glob.LENGTH + + def do_check_hash(self): + while True: + jitdriver_hash.jit_merge_point(self=self) + if not self.check_hash(0): + break def check_hash(self, retry_counter): if self.value == 0: @@ -152,7 +175,7 @@ x.value += 1 assert compute_identity_hash(x) == expected_hash self.value += 20 - return int(self.value < glob.LENGTH) + return self.value < glob.LENGTH class Arg: foobar = 42 @@ -180,9 +203,8 @@ bootstrapper.lock = None bootstrapper.args = None - def _freeze_(self): + def _cleanup_(self): self.reinit() - return False @staticmethod def bootstrap(): diff --git a/pypy/translator/stm/test/test_jitdriver.py b/pypy/translator/stm/test/test_jitdriver.py --- a/pypy/translator/stm/test/test_jitdriver.py +++ b/pypy/translator/stm/test/test_jitdriver.py @@ -35,3 +35,17 @@ res = self.interpret(f1, ['\x03', rffi.cast(rffi.SHORT, 4), 2]) assert res == 'X' + + def test_loop_void_result(self): + class X: + counter = 10 + x = X() + myjitdriver = JitDriver(greens=[], reds=[]) + + def f1(): + while x.counter > 0: + myjitdriver.jit_merge_point() + x.counter -= 1 + + res = self.interpret(f1, []) + assert res == None From noreply at buildbot.pypy.org Fri Jan 11 20:46:25 2013 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 11 Jan 2013 20:46:25 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: Don't use SSA_to_SSI; it's not used by other code and seems not really Message-ID: <20130111194625.5FE9D1C0343@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r59971:eb6ae03dec5d Date: 2013-01-11 20:45 +0100 http://bitbucket.org/pypy/pypy/changeset/eb6ae03dec5d/ Log: Don't use SSA_to_SSI; it's not used by other code and seems not really well-tested enough :-( diff --git a/pypy/translator/stm/jitdriver.py b/pypy/translator/stm/jitdriver.py --- a/pypy/translator/stm/jitdriver.py +++ b/pypy/translator/stm/jitdriver.py @@ -3,7 +3,6 @@ from pypy.objspace.flow.model import Block, Link, SpaceOperation, Constant from pypy.translator.unsimplify import split_block, varoftype from pypy.translator.stm.stmgcintf import StmOperations -from pypy.translator.backendopt.ssa import SSA_to_SSI from pypy.annotation.model import lltype_to_annotation, s_Int from pypy.rpython.annlowlevel import (MixLevelHelperAnnotator, cast_base_ptr_to_instance) @@ -131,7 +130,7 @@ blockf = self.add_call_should_break_transaction(block1) # # fill in blockf with a call to invoke_stm() - v = varoftype(self.RESTYPE) + v = varoftype(self.RESTYPE, 'result') op = SpaceOperation('direct_call', [self.c_invoke_stm_func] + blockf.inputargs, v) blockf.operations.append(op) @@ -169,6 +168,9 @@ lltype_to_annotation(self.RESTYPE)) self.c_invoke_stm_func = c_func + def container_var(self): + return varoftype(self.CONTAINERP, 'stmargs') + def make_callback_function(self): # make a copy of the 'main_graph' callback_graph = copygraph(self.main_graph) @@ -181,9 +183,10 @@ # self.stmtransformer.translator.annotator.transfer_binding(v2, v1) # # make a new startblock - v_p = varoftype(self.CONTAINERP) - v_retry_counter = varoftype(lltype.Signed) - blockst = Block([v_p, v_retry_counter]) + v_p = self.container_var() + v_retry_counter = varoftype(lltype.Signed, 'retry_counter') + blockst = Block([v_retry_counter]) # 'v_p' inserted below + renamed_p = {blockst: v_p} annotator = self.stmtransformer.translator.annotator annotator.setbinding(v_p, lltype_to_annotation(self.CONTAINERP)) annotator.setbinding(v_retry_counter, s_Int) @@ -213,6 +216,8 @@ c_got_exception = Constant('got_exception', lltype.Void) c_null = Constant(lltype.nullptr(self.CONTAINER.got_exception.TO), self.CONTAINER.got_exception) + v_p = self.container_var() + renamed_p[blockr] = v_p blockr.operations = [ SpaceOperation('setfield', [v_p, c_result_value, blockr.inputargs[0]], @@ -232,6 +237,8 @@ # add 'should_break_transaction()' at the end of the loop blockf = self.add_call_should_break_transaction(block1) # store the variables again into v_p + v_p = self.container_var() + renamed_p[blockf] = v_p for i in range(len(self.TYPES)): c_a_i = Constant('a%d' % i, lltype.Void) v_a_i = blockf.inputargs[i] @@ -241,7 +248,16 @@ varoftype(lltype.Void))) blockf.closeblock(Link([Constant(1, lltype.Signed)], newblockr)) # - SSA_to_SSI(callback_graph) # to pass 'p' everywhere + # now pass the original 'v_p' everywhere + for block in callback_graph.iterblocks(): + if block.operations == (): # skip return and except blocks + continue + v_p = renamed_p.get(block, self.container_var()) + block.inputargs = [v_p] + block.inputargs + for link in block.exits: + if link.target.operations != (): # to return or except block + link.args = [v_p] + link.args + # checkgraph(callback_graph) # FUNCTYPE = lltype.FuncType([self.CONTAINERP, lltype.Signed], From noreply at buildbot.pypy.org Fri Jan 11 20:53:37 2013 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 11 Jan 2013 20:53:37 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: Untested: revert some of the changes to pyopcode, relying instead on Message-ID: <20130111195337.0209A1C0DCB@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r59972:9594d565f5c7 Date: 2013-01-11 20:53 +0100 http://bitbucket.org/pypy/pypy/changeset/9594d565f5c7/ Log: Untested: revert some of the changes to pyopcode, relying instead on a new jitdriver to trigger stm. diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -58,6 +58,9 @@ unrolling_compare_dispatch_table = unrolling_iterable( enumerate(compare_dispatch_table)) +stmonly_jitdriver = jit.JitDriver(greens=[], reds=['self', 'co_code', + 'next_instr', 'ec']) + class __extend__(pyframe.PyFrame): """A PyFrame that knows about interpretation of standard Python opcodes @@ -77,56 +80,15 @@ co_code = pycode.co_code try: - while self._runs_normal_handler(): + while True: + if self.space.config.translation.stm: + stmonly_jitdriver.jit_merge_point( + self=self, co_code=co_code, + next_instr=next_instr, ec=ec) + self = self._hints_for_stm() next_instr = self.handle_bytecode(co_code, next_instr, ec) except ExitFrame: return self.popvalue() - # - # we only get here if _runs_normal_handler() returned False - return self._dispatch_stm_breaking_transaction(next_instr) - - def _runs_normal_handler(self): - if self.space.config.translation.stm and we_are_translated(): - from pypy.rlib import rstm - return not rstm.should_break_transaction() - return True - - def _dispatch_stm_breaking_transaction(self, next_instr): - # STM: entered the first time this frame is asked to introduce - # a transaction break. This is done by invoking the C function - # rstm.perform_transaction(), which calls back - # _dispatch_new_stm_transaction(). - from pypy.rlib import rstm - self.last_instr = intmask(next_instr) # save this value - rstm.perform_transaction(pyframe.PyFrame._dispatch_new_stm_transaction, - self.space.FrameClass, self) - e = self.__reraise - if e is None: - return self.popvalue() # normal exit path - else: - self.__reraise = None - raise e # re-raise the exception we got - - def _dispatch_new_stm_transaction(self, retry_counter): - self = self._hints_for_stm() - co_code = self.pycode.co_code - next_instr = r_uint(self.last_instr) # restore this value - ec = self.space.getexecutioncontext() - - # a loop similar to the one in dispatch() - try: - while self._runs_normal_handler(): - next_instr = self.handle_bytecode(co_code, next_instr, ec) - except ExitFrame: - self.__reraise = None - return 0 # stop perform_transaction() and returns - except Exception, e: - self.__reraise = e - return 0 # stop perform_transaction() and returns - # - # we get there if _runs_normal_handler() return False again - self.last_instr = intmask(next_instr) # save this value - return 1 # causes perform_transaction() to loop and call us again def handle_bytecode(self, co_code, next_instr, ec): try: From noreply at buildbot.pypy.org Fri Jan 11 21:16:38 2013 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 11 Jan 2013 21:16:38 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: Reorder (ints first, refs next). Message-ID: <20130111201638.0DE011C1338@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r59973:e43c998a2147 Date: 2013-01-11 21:19 +0100 http://bitbucket.org/pypy/pypy/changeset/e43c998a2147/ Log: Reorder (ints first, refs next). diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -58,8 +58,8 @@ unrolling_compare_dispatch_table = unrolling_iterable( enumerate(compare_dispatch_table)) -stmonly_jitdriver = jit.JitDriver(greens=[], reds=['self', 'co_code', - 'next_instr', 'ec']) +stmonly_jitdriver = jit.JitDriver(greens=[], reds=['next_instr', 'ec', + 'self', 'co_code']) class __extend__(pyframe.PyFrame): From noreply at buildbot.pypy.org Fri Jan 11 21:25:56 2013 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 11 Jan 2013 21:25:56 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: Kill 'self.value' and use a regular local variable instead, now that we Message-ID: <20130111202556.70AED1C1338@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r59974:8e0354b5aabe Date: 2013-01-11 21:25 +0100 http://bitbucket.org/pypy/pypy/changeset/8e0354b5aabe/ Log: Kill 'self.value' and use a regular local variable instead, now that we can. It improves the testing. diff --git a/pypy/translator/stm/test/targetdemo2.py b/pypy/translator/stm/test/targetdemo2.py --- a/pypy/translator/stm/test/targetdemo2.py +++ b/pypy/translator/stm/test/targetdemo2.py @@ -65,10 +65,10 @@ print "check ok!" -jitdriver_hash = jit.JitDriver(greens=[], reds=['self']) -jitdriver_inev = jit.JitDriver(greens=[], reds=['self']) +jitdriver_hash = jit.JitDriver(greens=[], reds=['value', 'self']) +jitdriver_inev = jit.JitDriver(greens=[], reds=['value', 'self']) jitdriver_ptreq = jit.JitDriver(greens=[], reds=['self']) -jitdriver_really = jit.JitDriver(greens=[], reds=['self']) +jitdriver_really = jit.JitDriver(greens=[], reds=['value', 'self']) class ThreadRunner(object): @@ -81,12 +81,9 @@ def run(self): try: - self.value = 0 self.lst = [] self.do_check_hash() - self.value = 0 self.do_check_inev() - self.value = 0 self.arg = Arg() self.glob_p = lltype.malloc(STRUCT) self.do_check_ptr_equality() @@ -95,44 +92,47 @@ self.finished_lock.release() def do_run_really(self): + value = 0 while True: - jitdriver_really.jit_merge_point(self=self) - if not self.run_really(0): + jitdriver_really.jit_merge_point(self=self, value=value) + if not self.run_really(value): break + value += 1 - def run_really(self, retry_counter): - if self.value == glob.LENGTH // 2: + def run_really(self, value): + if value == glob.LENGTH // 2: print "atomic!" assert not rstm.is_atomic() rstm.increment_atomic() assert rstm.is_atomic() - if self.value == glob.LENGTH * 2 // 3: + if value == glob.LENGTH * 2 // 3: print "--------------- done atomic" assert rstm.is_atomic() rstm.decrement_atomic() assert not rstm.is_atomic() # - add_at_end_of_chained_list(glob.anchor, self.value, self.index) - self.value += 1 - return self.value < glob.LENGTH + add_at_end_of_chained_list(glob.anchor, value, self.index) + return (value+1) < glob.LENGTH def do_check_ptr_equality(self): jitdriver_ptreq.jit_merge_point(self=self) self.check_ptr_equality(0) - def check_ptr_equality(self, retry_counter): + def check_ptr_equality(self, foo): assert self.glob_p != lltype.nullptr(STRUCT) res = _check_pointer(self.arg) # 'self.arg' reads a GLOBAL object ll_assert(res is self.arg, "ERROR: bogus pointer equality") - raw1 = rffi.cast(rffi.CCHARP, retry_counter) + raw1 = rffi.cast(rffi.CCHARP, foo) raw2 = rffi.cast(rffi.CCHARP, -1) - ll_assert(raw1 != raw2, "ERROR: retry_counter == -1") + ll_assert(raw1 != raw2, "ERROR: foo == -1") def do_check_inev(self): + value = 0 while True: - jitdriver_inev.jit_merge_point(self=self) - if not self.check_inev(0): + jitdriver_inev.jit_merge_point(self=self, value=value) + if not self.check_inev(value): break + value += 1 def _check_content(self, content): ll_assert(glob.othernode2.value == content, "bogus value after inev") @@ -144,38 +144,40 @@ self._check_content(read_value) _check_inev._dont_inline_ = True - def check_inev(self, retry_counter): - self.value += 1 - new_value = self.index * 1000000 + self.value + def check_inev(self, value): + value += 1 + new_value = self.index * 1000000 + value self._check_inev() glob.othernode1.value = new_value for n in glob.othernodes: # lots of unrelated writes in-between n.value = new_value glob.othernode2.value = new_value - return self.value < glob.LENGTH + return value < glob.LENGTH def do_check_hash(self): + value = 0 while True: - jitdriver_hash.jit_merge_point(self=self) - if not self.check_hash(0): + jitdriver_hash.jit_merge_point(self=self, value=value) + value = self.check_hash(value) + if value >= glob.LENGTH: break - def check_hash(self, retry_counter): - if self.value == 0: + def check_hash(self, value): + if value == 0: glob.othernode2hash = compute_identity_hash(glob.othernode2) assert glob.othernode1hash == compute_identity_hash(glob.othernode1) assert glob.othernode2hash == compute_identity_hash(glob.othernode2) - x = Node(retry_counter) + x = Node(0) lst = self.lst lst.append((x, compute_identity_hash(x))) for i in range(len(lst)): x, expected_hash = lst[i] assert compute_identity_hash(x) == expected_hash - if i % 7 == retry_counter: + if i % 7 == 0: x.value += 1 assert compute_identity_hash(x) == expected_hash - self.value += 20 - return self.value < glob.LENGTH + value += 20 + return value class Arg: foobar = 42 From noreply at buildbot.pypy.org Fri Jan 11 21:39:08 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Fri, 11 Jan 2013 21:39:08 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Moved pypy.interpreter.argument.Signature to rpython.rtyper.signature Message-ID: <20130111203908.99ECA1C0343@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59975:f7bae0b0a270 Date: 2013-01-11 21:38 +0100 http://bitbucket.org/pypy/pypy/changeset/f7bae0b0a270/ Log: Moved pypy.interpreter.argument.Signature to rpython.rtyper.signature diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py --- a/pypy/interpreter/gateway.py +++ b/pypy/interpreter/gateway.py @@ -14,7 +14,8 @@ import py from pypy.interpreter.eval import Code -from pypy.interpreter.argument import Arguments, Signature +from pypy.interpreter.argument import Arguments +from rpython.rtyper.signature import Signature from pypy.interpreter.baseobjspace import (W_Root, ObjSpace, Wrappable, SpaceCache, DescrMismatch) from pypy.interpreter.error import OperationError diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -7,7 +7,7 @@ import dis, imp, struct, types, new, sys from pypy.interpreter import eval -from pypy.interpreter.argument import Signature +from rpython.rtyper.signature import Signature from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import unwrap_spec from pypy.interpreter.astcompiler.consts import ( diff --git a/pypy/module/oracle/interp_pool.py b/pypy/module/oracle/interp_pool.py --- a/pypy/module/oracle/interp_pool.py +++ b/pypy/module/oracle/interp_pool.py @@ -1,10 +1,11 @@ from pypy.interpreter.baseobjspace import Wrappable -from pypy.interpreter.argument import Arguments, Signature +from pypy.interpreter.argument import Arguments from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.interpreter.typedef import interp_attrproperty, interp_attrproperty_w from pypy.interpreter.error import OperationError from rpython.rtyper.lltypesystem import rffi, lltype +from rpython.rtyper.signature import Signature from pypy.module.oracle import roci, config from pypy.module.oracle import interp_error, interp_environ diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py --- a/pypy/objspace/std/bytearrayobject.py +++ b/pypy/objspace/std/bytearrayobject.py @@ -19,13 +19,13 @@ from pypy.objspace.std.unicodeobject import W_UnicodeObject from pypy.objspace.std import slicetype from pypy.interpreter import gateway -from pypy.interpreter.argument import Signature from pypy.interpreter.buffer import RWBuffer from pypy.objspace.std.bytearraytype import ( makebytearraydata_w, getbytevalue, new_bytearray ) from rpython.tool.sourcetools import func_with_new_name +from rpython.rtyper.signature import Signature class W_BytearrayObject(W_Object): diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py --- a/pypy/objspace/std/dictmultiobject.py +++ b/pypy/objspace/std/dictmultiobject.py @@ -4,13 +4,13 @@ from pypy.objspace.std.settype import set_typedef as settypedef from pypy.objspace.std.frozensettype import frozenset_typedef as frozensettypedef from pypy.interpreter import gateway -from pypy.interpreter.argument import Signature from pypy.interpreter.error import OperationError, operationerrfmt from rpython.rlib.objectmodel import r_dict, we_are_translated, specialize,\ newlist_hint from rpython.rlib.debug import mark_dict_non_null from rpython.tool.sourcetools import func_with_new_name +from rpython.rtyper.signature import Signature from rpython.rlib import rerased from rpython.rlib import jit diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py --- a/pypy/objspace/std/listobject.py +++ b/pypy/objspace/std/listobject.py @@ -12,7 +12,7 @@ resizelist_hint) from rpython.rlib.listsort import make_timsort_class from rpython.rlib import rerased, jit, debug -from pypy.interpreter.argument import Signature +from rpython.rtyper.signature import Signature from rpython.tool.sourcetools import func_with_new_name UNROLL_CUTOFF = 5 diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py --- a/pypy/objspace/std/setobject.py +++ b/pypy/objspace/std/setobject.py @@ -4,11 +4,11 @@ from rpython.rlib.rarithmetic import intmask, r_uint from pypy.interpreter.error import OperationError from pypy.interpreter import gateway -from pypy.interpreter.argument import Signature from pypy.objspace.std.settype import set_typedef as settypedef from pypy.objspace.std.frozensettype import frozenset_typedef as frozensettypedef from rpython.rlib import rerased from rpython.rlib.objectmodel import instantiate +from rpython.rtyper.signature import Signature from pypy.interpreter.generator import GeneratorIterator from pypy.objspace.std.listobject import W_ListObject from pypy.objspace.std.intobject import W_IntObject diff --git a/rpython/rtyper/normalizecalls.py b/rpython/rtyper/normalizecalls.py --- a/rpython/rtyper/normalizecalls.py +++ b/rpython/rtyper/normalizecalls.py @@ -1,5 +1,5 @@ from rpython.annotator import model as annmodel, description -from pypy.interpreter.argument import Signature +from rpython.rtyper.signature import Signature from rpython.flowspace.model import (Variable, Constant, Block, Link, checkgraph, FunctionGraph, SpaceOperation) from rpython.rlib.objectmodel import ComputedIntSymbolic diff --git a/rpython/rtyper/signature.py b/rpython/rtyper/signature.py new file mode 100644 --- /dev/null +++ b/rpython/rtyper/signature.py @@ -0,0 +1,72 @@ +from rpython.rlib import jit + +class Signature(object): + _immutable_ = True + _immutable_fields_ = ["argnames[*]"] + __slots__ = ("argnames", "varargname", "kwargname") + + def __init__(self, argnames, varargname=None, kwargname=None): + self.argnames = argnames + self.varargname = varargname + self.kwargname = kwargname + + @jit.elidable + def find_argname(self, name): + try: + return self.argnames.index(name) + except ValueError: + return -1 + + def num_argnames(self): + return len(self.argnames) + + def has_vararg(self): + return self.varargname is not None + + def has_kwarg(self): + return self.kwargname is not None + + def scope_length(self): + scopelen = len(self.argnames) + scopelen += self.has_vararg() + scopelen += self.has_kwarg() + return scopelen + + def getallvarnames(self): + argnames = self.argnames + if self.varargname is not None: + argnames = argnames + [self.varargname] + if self.kwargname is not None: + argnames = argnames + [self.kwargname] + return argnames + + def __repr__(self): + return "Signature(%r, %r, %r)" % ( + self.argnames, self.varargname, self.kwargname) + + def __eq__(self, other): + if not isinstance(other, Signature): + return NotImplemented + return (self.argnames == other.argnames and + self.varargname == other.varargname and + self.kwargname == other.kwargname) + + def __ne__(self, other): + if not isinstance(other, Signature): + return NotImplemented + return not self == other + + + # make it look tuply for its use in the annotator + + def __len__(self): + return 3 + + def __getitem__(self, i): + if i == 0: + return self.argnames + if i == 1: + return self.varargname + if i == 2: + return self.kwargname + raise IndexError \ No newline at end of file From noreply at buildbot.pypy.org Fri Jan 11 21:55:05 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Fri, 11 Jan 2013 21:55:05 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed missing import in interp_struct Message-ID: <20130111205505.324951C0343@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59976:599114782639 Date: 2013-01-11 21:54 +0100 http://bitbucket.org/pypy/pypy/changeset/599114782639/ Log: Fixed missing import in interp_struct diff --git a/pypy/module/struct/interp_struct.py b/pypy/module/struct/interp_struct.py --- a/pypy/module/struct/interp_struct.py +++ b/pypy/module/struct/interp_struct.py @@ -2,7 +2,7 @@ from pypy.interpreter.error import OperationError from pypy.module.struct.formatiterator import PackFormatIterator, UnpackFormatIterator from rpython.rlib import jit -from rpython.rlib.rstruct.error import StructError +from rpython.rlib.rstruct.error import StructError, StructOverflowError from rpython.rlib.rstruct.formatiterator import CalcSizeFormatIterator From noreply at buildbot.pypy.org Fri Jan 11 22:13:07 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Fri, 11 Jan 2013 22:13:07 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed interp_struct Message-ID: <20130111211307.19D4C1C0343@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59977:bc67ae89d391 Date: 2013-01-11 22:08 +0100 http://bitbucket.org/pypy/pypy/changeset/bc67ae89d391/ Log: Fixed interp_struct diff --git a/pypy/module/struct/interp_struct.py b/pypy/module/struct/interp_struct.py --- a/pypy/module/struct/interp_struct.py +++ b/pypy/module/struct/interp_struct.py @@ -15,7 +15,7 @@ try: fmtiter.interpret(format) except StructOverflowError, e: - raise OperationError(space.w_OverflowError, space.wrap(self.msg)) + raise OperationError(space.w_OverflowError, space.wrap(e.msg)) except StructError, e: w_module = space.getbuiltinmodule('struct') w_error = space.getattr(w_module, space.wrap('error')) @@ -32,7 +32,7 @@ try: fmtiter.interpret(format) except StructOverflowError, e: - raise OperationError(space.w_OverflowError, space.wrap(self.msg)) + raise OperationError(space.w_OverflowError, space.wrap(e.msg)) except StructError, e: w_module = space.getbuiltinmodule('struct') w_error = space.getattr(w_module, space.wrap('error')) @@ -46,7 +46,7 @@ try: fmtiter.interpret(format) except StructOverflowError, e: - raise OperationError(space.w_OverflowError, space.wrap(self.msg)) + raise OperationError(space.w_OverflowError, space.wrap(e.msg)) except StructError, e: w_module = space.getbuiltinmodule('struct') w_error = space.getattr(w_module, space.wrap('error')) From noreply at buildbot.pypy.org Fri Jan 11 22:13:08 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Fri, 11 Jan 2013 22:13:08 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed missing imports Message-ID: <20130111211308.869181C0343@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59978:b1d1c363030a Date: 2013-01-11 22:12 +0100 http://bitbucket.org/pypy/pypy/changeset/b1d1c363030a/ Log: Fixed missing imports diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py --- a/pypy/interpreter/test/test_gateway.py +++ b/pypy/interpreter/test/test_gateway.py @@ -3,6 +3,7 @@ from pypy.interpreter import gateway, argument from pypy.interpreter.gateway import ObjSpace, W_Root, WrappedDefault +from rpython.rtyper.signature import Signature import py import sys @@ -20,24 +21,24 @@ gateway.W_Root, gateway.W_Root, 'args_w']) - assert code.signature() == argument.Signature(['x', 'y'], 'hello', None) + assert code.signature() == Signature(['x', 'y'], 'hello', None) def d(self, w_boo): pass code = gateway.BuiltinCode(d, unwrap_spec= ['self', gateway.W_Root], self_type=gateway.Wrappable) - assert code.signature() == argument.Signature(['self', 'boo'], None, None) + assert code.signature() == Signature(['self', 'boo'], None, None) def e(space, w_x, w_y, __args__): pass code = gateway.BuiltinCode(e, unwrap_spec=[gateway.ObjSpace, gateway.W_Root, gateway.W_Root, gateway.Arguments]) - assert code.signature() == argument.Signature(['x', 'y'], 'args', 'keywords') + assert code.signature() == Signature(['x', 'y'], 'args', 'keywords') def f(space, index): pass code = gateway.BuiltinCode(f, unwrap_spec=[gateway.ObjSpace, "index"]) - assert code.signature() == argument.Signature(["index"], None, None) + assert code.signature() == Signature(["index"], None, None) def test_call(self): diff --git a/rpython/rtyper/lltypesystem/module/test/test_ll_math.py b/rpython/rtyper/lltypesystem/module/test/test_ll_math.py --- a/rpython/rtyper/lltypesystem/module/test/test_ll_math.py +++ b/rpython/rtyper/lltypesystem/module/test/test_ll_math.py @@ -1,7 +1,7 @@ """ Try to test systematically all cases of ll_math.py. """ -import math +import sys, math from rpython.rtyper.lltypesystem.module import ll_math from rpython.translator.c.test.test_genc import compile from rpython.rlib.rfloat import isinf, isnan, INFINITY, NAN From noreply at buildbot.pypy.org Fri Jan 11 22:28:20 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Fri, 11 Jan 2013 22:28:20 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed import in test_direct Message-ID: <20130111212820.5F51E1C0DCB@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59979:5689da86ce23 Date: 2013-01-11 22:27 +0100 http://bitbucket.org/pypy/pypy/changeset/5689da86ce23/ Log: Fixed import in test_direct diff --git a/pypy/module/math/test/test_direct.py b/pypy/module/math/test/test_direct.py --- a/pypy/module/math/test/test_direct.py +++ b/pypy/module/math/test/test_direct.py @@ -4,7 +4,7 @@ import py, sys, math from rpython.rlib import rfloat from rpython.rtyper.lltypesystem.module.test.test_ll_math import (MathTests, - getTester) + get_tester) consistent_host = True if '__pypy__' not in sys.builtin_module_names: From noreply at buildbot.pypy.org Sat Jan 12 00:11:36 2013 From: noreply at buildbot.pypy.org (wlav) Date: Sat, 12 Jan 2013 00:11:36 +0100 (CET) Subject: [pypy-commit] pypy reflex-support: allow user steering of entry getting Message-ID: <20130111231136.5C7501C0343@cobra.cs.uni-duesseldorf.de> Author: Wim Lavrijsen Branch: reflex-support Changeset: r59980:731ac16e881f Date: 2013-01-11 15:11 -0800 http://bitbucket.org/pypy/pypy/changeset/731ac16e881f/ Log: allow user steering of entry getting diff --git a/pypy/module/cppyy/capi/cint_capi.py b/pypy/module/cppyy/capi/cint_capi.py --- a/pypy/module/cppyy/capi/cint_capi.py +++ b/pypy/module/cppyy/capi/cint_capi.py @@ -188,19 +188,26 @@ if not space.is_true(w_branch): raise OperationError(space.w_AttributeError, args_w[0]) activate_branch(space, w_branch) + + # figure out from where we're reading + entry = space.int_w(space.call_method(w_self, "GetReadEntry")) + if entry == -1: + entry = 0 + + # setup cache structure w_klassname = space.call_method(w_branch, "GetClassName") if space.is_true(w_klassname): # some instance klass = interp_cppyy.scope_byname(space, space.str_w(w_klassname)) w_obj = klass.construct() space.call_method(w_branch, "SetObject", w_obj) - space.call_method(w_branch, "GetEntry", space.wrap(0)) + space.call_method(w_branch, "GetEntry", space.wrap(entry)) space.setattr(w_self, args_w[0], w_obj) return w_obj else: # builtin data w_leaf = space.call_method(w_self, "GetLeaf", args_w[0]) - space.call_method(w_branch, "GetEntry", space.wrap(0)) + space.call_method(w_branch, "GetEntry", space.wrap(entry)) # location w_address = space.call_method(w_leaf, "GetValuePointer") diff --git a/pypy/module/cppyy/test/test_cint.py b/pypy/module/cppyy/test/test_cint.py --- a/pypy/module/cppyy/test/test_cint.py +++ b/pypy/module/cppyy/test/test_cint.py @@ -335,6 +335,7 @@ mytree.Branch("my_bool", ba, "my_bool/O") mytree.Branch("my_int", ia, "my_int/I") + mytree.Branch("my_int2", ia, "my_int2/I") mytree.Branch("my_double", da, "my_double/D") for i in range(self.N): @@ -367,6 +368,21 @@ f.Close() + def test09_user_read_builtin(self): + """Test user-directed reading of builtins""" + + from cppyy import gbl + from cppyy.gbl import TFile + + f = TFile(self.fname) + mytree = f.Get(self.tname) + + # note, this is an old, annoted tree from test08 + for i in range(3, mytree.GetEntriesFast()): + mytree.GetEntry(i) + assert mytree.my_int == i+1 + assert mytree.my_int2 == i+1 + class AppTestRegression: spaceconfig = dict(usemodules=['cppyy']) From noreply at buildbot.pypy.org Sat Jan 12 02:17:09 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sat, 12 Jan 2013 02:17:09 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Forgot to save argument.py Message-ID: <20130112011709.84F071C05DD@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59981:490e3dabd510 Date: 2013-01-12 02:16 +0100 http://bitbucket.org/pypy/pypy/changeset/490e3dabd510/ Log: Forgot to save argument.py diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py --- a/pypy/interpreter/argument.py +++ b/pypy/interpreter/argument.py @@ -7,79 +7,6 @@ from rpython.rlib import jit -class Signature(object): - _immutable_ = True - _immutable_fields_ = ["argnames[*]"] - __slots__ = ("argnames", "varargname", "kwargname") - - def __init__(self, argnames, varargname=None, kwargname=None): - self.argnames = argnames - self.varargname = varargname - self.kwargname = kwargname - - @jit.elidable - def find_argname(self, name): - try: - return self.argnames.index(name) - except ValueError: - return -1 - - def num_argnames(self): - return len(self.argnames) - - def has_vararg(self): - return self.varargname is not None - - def has_kwarg(self): - return self.kwargname is not None - - def scope_length(self): - scopelen = len(self.argnames) - scopelen += self.has_vararg() - scopelen += self.has_kwarg() - return scopelen - - def getallvarnames(self): - argnames = self.argnames - if self.varargname is not None: - argnames = argnames + [self.varargname] - if self.kwargname is not None: - argnames = argnames + [self.kwargname] - return argnames - - def __repr__(self): - return "Signature(%r, %r, %r)" % ( - self.argnames, self.varargname, self.kwargname) - - def __eq__(self, other): - if not isinstance(other, Signature): - return NotImplemented - return (self.argnames == other.argnames and - self.varargname == other.varargname and - self.kwargname == other.kwargname) - - def __ne__(self, other): - if not isinstance(other, Signature): - return NotImplemented - return not self == other - - - # make it look tuply for its use in the annotator - - def __len__(self): - return 3 - - def __getitem__(self, i): - if i == 0: - return self.argnames - if i == 1: - return self.varargname - if i == 2: - return self.kwargname - raise IndexError - - - class Arguments(object): """ Collects the arguments of a function call. From noreply at buildbot.pypy.org Sat Jan 12 10:31:44 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 12 Jan 2013 10:31:44 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: fix enforceargs on some rffi helpers Message-ID: <20130112093144.20B6A1C11FD@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59982:924293f74f7b Date: 2013-01-12 11:28 +0200 http://bitbucket.org/pypy/pypy/changeset/924293f74f7b/ Log: fix enforceargs on some rffi helpers diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py --- a/pypy/rpython/lltypesystem/rffi.py +++ b/pypy/rpython/lltypesystem/rffi.py @@ -2,12 +2,12 @@ from pypy.annotation import model as annmodel from pypy.rpython.lltypesystem import lltype, rstr from pypy.rpython.lltypesystem import ll2ctypes -from pypy.rpython.lltypesystem.llmemory import cast_adr_to_ptr, cast_ptr_to_adr +from pypy.rpython.lltypesystem.llmemory import cast_ptr_to_adr from pypy.rpython.lltypesystem.llmemory import itemoffsetof, raw_memcopy from pypy.annotation.model import lltype_to_annotation from pypy.tool.sourcetools import func_with_new_name -from pypy.rlib.objectmodel import Symbolic, CDefinedIntSymbolic -from pypy.rlib.objectmodel import keepalive_until_here +from pypy.rlib.objectmodel import Symbolic +from pypy.rlib.objectmodel import keepalive_until_here, enforceargs from pypy.rlib import rarithmetic, rgc from pypy.rpython.extregistry import ExtRegistryEntry from pypy.rlib.unroll import unrolling_iterable @@ -795,6 +795,7 @@ # (char*, str, int, int) -> None @jit.dont_look_inside + @enforceargs(None, None, int, int) def str_from_buffer(raw_buf, gc_buf, allocated_size, needed_size): """ Converts from a pair returned by alloc_buffer to a high-level string. @@ -833,6 +834,7 @@ lltype.free(raw_buf, flavor='raw') # char* -> str, with an upper bound on the length in case there is no \x00 + @enforceargs(None, int) def charp2strn(cp, maxlen): b = builder_class(maxlen) i = 0 From noreply at buildbot.pypy.org Sat Jan 12 10:31:45 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 12 Jan 2013 10:31:45 +0100 (CET) Subject: [pypy-commit] pypy default: fix enforceargs on some rffi helpers Message-ID: <20130112093145.762C31C11FD@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r59983:89456005d20e Date: 2013-01-12 11:28 +0200 http://bitbucket.org/pypy/pypy/changeset/89456005d20e/ Log: fix enforceargs on some rffi helpers diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py --- a/pypy/rpython/lltypesystem/rffi.py +++ b/pypy/rpython/lltypesystem/rffi.py @@ -2,12 +2,12 @@ from pypy.annotation import model as annmodel from pypy.rpython.lltypesystem import lltype, rstr from pypy.rpython.lltypesystem import ll2ctypes -from pypy.rpython.lltypesystem.llmemory import cast_adr_to_ptr, cast_ptr_to_adr +from pypy.rpython.lltypesystem.llmemory import cast_ptr_to_adr from pypy.rpython.lltypesystem.llmemory import itemoffsetof, raw_memcopy from pypy.annotation.model import lltype_to_annotation from pypy.tool.sourcetools import func_with_new_name -from pypy.rlib.objectmodel import Symbolic, CDefinedIntSymbolic -from pypy.rlib.objectmodel import keepalive_until_here +from pypy.rlib.objectmodel import Symbolic +from pypy.rlib.objectmodel import keepalive_until_here, enforceargs from pypy.rlib import rarithmetic, rgc from pypy.rpython.extregistry import ExtRegistryEntry from pypy.rlib.unroll import unrolling_iterable @@ -795,6 +795,7 @@ # (char*, str, int, int) -> None @jit.dont_look_inside + @enforceargs(None, None, int, int) def str_from_buffer(raw_buf, gc_buf, allocated_size, needed_size): """ Converts from a pair returned by alloc_buffer to a high-level string. @@ -833,6 +834,7 @@ lltype.free(raw_buf, flavor='raw') # char* -> str, with an upper bound on the length in case there is no \x00 + @enforceargs(None, int) def charp2strn(cp, maxlen): b = builder_class(maxlen) i = 0 From noreply at buildbot.pypy.org Sat Jan 12 10:31:47 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 12 Jan 2013 10:31:47 +0100 (CET) Subject: [pypy-commit] pypy default: merge Message-ID: <20130112093147.2B4A71C11FD@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r59984:092d0cd721de Date: 2013-01-12 11:31 +0200 http://bitbucket.org/pypy/pypy/changeset/092d0cd721de/ Log: merge diff --git a/pypy/doc/test/test_whatsnew.py b/pypy/doc/test/test_whatsnew.py --- a/pypy/doc/test/test_whatsnew.py +++ b/pypy/doc/test/test_whatsnew.py @@ -29,7 +29,7 @@ merge() and \ branch(default)) and \ not branch(default)' % (startrev, endrev) - cmd = r"hg log -R '%s' -r '%s' --template '{branches}\n'" % (path, revset) + cmd = r'hg log -R "%s" -r "%s" --template "{branches}\n"' % (path, revset) out = getoutput(cmd) branches = set(map(str.strip, out.splitlines())) return branches diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -15,6 +15,9 @@ .. branch: numpypy-longdouble Long double support for numpypy +.. branch: numpypy-real-as-view +Convert real, imag from ufuncs to views. This involves the beginning of +view() functionality .. branch: signatures Improved RPython typing diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -178,6 +178,11 @@ else: oparg = 0 + # note: the structure of the code here is such that it makes + # (after translation) a big "if/elif" chain, which is then + # turned into a switch(). It starts here: even if the first + # one is not an "if" but a "while" the effect is the same. + while opcode == self.opcodedesc.EXTENDED_ARG.index: opcode = ord(co_code[next_instr]) if opcode < self.HAVE_ARGUMENT: @@ -226,6 +231,8 @@ 'END_FINALLY', 'JUMP_ABSOLUTE'): continue # opcodes implemented above + # the following "if" is part of the big switch described + # above. if opcode == opdesc.index: # dispatch to the opcode method meth = getattr(self, opdesc.methodname) diff --git a/pypy/jit/backend/test/calling_convention_test.py b/pypy/jit/backend/test/calling_convention_test.py --- a/pypy/jit/backend/test/calling_convention_test.py +++ b/pypy/jit/backend/test/calling_convention_test.py @@ -105,11 +105,13 @@ ops = '[%s]\n' % arguments ops += '%s\n' % spill_ops ops += 'f99 = call(ConstClass(func_ptr), %s, descr=calldescr)\n' % arguments - ops += 'finish(f99, %s)\n' % arguments + ops += 'i99 = same_as(0)\n' + ops += 'guard_true(i99) [f99, %s]\n' % arguments + ops += 'finish()\n' loop = parse(ops, namespace=locals()) looptoken = JitCellToken() - done_number = self.cpu.get_fail_descr_number(loop.operations[-1].getdescr()) + done_number = self.cpu.get_fail_descr_number(loop.operations[-2].getdescr()) self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) argvals, expected_result = self._prepare_args(args, floats, ints) diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py --- a/pypy/jit/backend/test/runner_test.py +++ b/pypy/jit/backend/test/runner_test.py @@ -1192,7 +1192,8 @@ values.append(longlong.getfloatstorage(r.random())) # looptoken = JitCellToken() - faildescr = BasicFailDescr(42) + guarddescr = BasicFailDescr(42) + faildescr = BasicFailDescr(43) operations = [] retboxes = [] retvalues = [] @@ -1221,9 +1222,13 @@ retboxes.insert(kk, newbox) retvalues.insert(kk, y) # - operations.append( - ResOperation(rop.FINISH, retboxes, None, descr=faildescr) - ) + zero = BoxInt() + operations.extend([ + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=guarddescr), + ResOperation(rop.FINISH, [], None, descr=faildescr) + ]) + operations[-2].setfailargs(retboxes) print inputargs for op in operations: print op @@ -1344,12 +1349,15 @@ targettoken = TargetToken() faildescr1 = BasicFailDescr(1) faildescr2 = BasicFailDescr(2) + faildescr3 = BasicFailDescr(3) operations = [ ResOperation(rop.LABEL, fboxes, None, descr=targettoken), ResOperation(rop.FLOAT_LE, [fboxes[0], constfloat(9.2)], i2), ResOperation(rop.GUARD_TRUE, [i2], None, descr=faildescr1), - ResOperation(rop.FINISH, fboxes, None, descr=faildescr2), + ResOperation(rop.GUARD_FALSE, [i2], None, descr=faildescr2), + ResOperation(rop.FINISH, [], None, descr=faildescr3), ] + operations[-3].setfailargs(fboxes) operations[-2].setfailargs(fboxes) looptoken = JitCellToken() self.cpu.compile_loop(fboxes, operations, looptoken) @@ -1382,6 +1390,7 @@ py.test.skip("requires floats") fboxes = [BoxFloat() for i in range(3)] faildescr1 = BasicFailDescr(100) + faildescr2 = BasicFailDescr(102) loopops = """ [i0,f1, f2] f3 = float_add(f1, f2) @@ -1406,9 +1415,13 @@ assert longlong.getrealfloat(f2) == 0.75 assert longlong.getrealfloat(f3) == 133.0 + zero = BoxInt() bridgeops = [ - ResOperation(rop.FINISH, fboxes, None, descr=faildescr1), + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=faildescr1), + ResOperation(rop.FINISH, [], None, descr=faildescr2), ] + bridgeops[-2].setfailargs(fboxes[:]) self.cpu.compile_bridge(loop.operations[-2].getdescr(), fboxes, bridgeops, looptoken) args = [1, @@ -2002,7 +2015,7 @@ i1 = same_as(1) call(ConstClass(fptr), i0, descr=calldescr) p0 = guard_exception(ConstClass(xtp)) [i1] - finish(0, p0) + finish(p0) ''' FPTR = lltype.Ptr(lltype.FuncType([lltype.Signed], lltype.Void)) fptr = llhelper(FPTR, func) @@ -2023,8 +2036,7 @@ looptoken = JitCellToken() self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) deadframe = self.cpu.execute_token(looptoken, 1) - assert self.cpu.get_latest_value_int(deadframe, 0) == 0 - assert self.cpu.get_latest_value_ref(deadframe, 1) == xptr + assert self.cpu.get_latest_value_ref(deadframe, 0) == xptr excvalue = self.cpu.grab_exc_value(deadframe) assert not excvalue deadframe = self.cpu.execute_token(looptoken, 0) diff --git a/pypy/jit/backend/x86/test/test_recompilation.py b/pypy/jit/backend/x86/test/test_recompilation.py --- a/pypy/jit/backend/x86/test/test_recompilation.py +++ b/pypy/jit/backend/x86/test/test_recompilation.py @@ -45,7 +45,8 @@ force_spill(i5) i8 = int_add(i7, 1) i9 = int_add(i8, 1) - finish(i3, i4, i5, i6, i7, i8, i9, descr=fdescr2) + guard_false(i3, descr=fdescr2) [i3, i4, i5, i6, i7, i8, i9] + finish() ''' bridge = self.attach_bridge(ops, loop, -2) descr = loop.operations[3].getdescr() diff --git a/pypy/jit/backend/x86/test/test_regalloc.py b/pypy/jit/backend/x86/test/test_regalloc.py --- a/pypy/jit/backend/x86/test/test_regalloc.py +++ b/pypy/jit/backend/x86/test/test_regalloc.py @@ -327,7 +327,7 @@ assert self.getint(0) == 0 bridge_ops = ''' [i0, i1] - finish(1, 2) + finish(2) ''' self.attach_bridge(bridge_ops, loop, 0) self.run(loop, 0, 1) @@ -428,7 +428,7 @@ ops = ''' [i0, i1, i2, i3, i4, i5, i6, i7] guard_value(i6, i1) [i0, i2, i3, i4, i5, i6] - finish(i0, i2, i3, i4, i5, i6) + finish(i0) ''' self.interpret(ops, [0, 0, 0, 0, 0, 0, 0, 0]) assert self.getint(0) == 0 @@ -438,14 +438,15 @@ [i0, i1, i2, i3, i4, i5, i6, i7, i8] i9 = same_as(0) guard_true(i0) [i9, i0, i1, i2, i3, i4, i5, i6, i7, i8] - finish(1, i0, i1, i2, i3, i4, i5, i6, i7, i8) + finish(1) ''' loop = self.interpret(ops, [0, 1, 2, 3, 4, 5, 6, 7, 8]) assert self.getint(0) == 0 bridge_ops = ''' [i9, i0, i1, i2, i3, i4, i5, i6, i7, i8] call(ConstClass(raising_fptr), 0, descr=raising_calldescr) - finish(i0, i1, i2, i3, i4, i5, i6, i7, i8) + guard_true(i9) [i0, i1, i2, i3, i4, i5, i6, i7, i8] + finish() ''' self.attach_bridge(bridge_ops, loop, 1) self.run(loop, 0, 1, 2, 3, 4, 5, 6, 7, 8) @@ -474,7 +475,8 @@ i1 = same_as(1) i2 = int_lt(i0, 100) guard_true(i3) [i1, i2] - finish(0, i2) + i4 = int_neg(i2) + finish(0) ''' self.interpret(ops, [0, 1]) assert self.getint(0) == 0 @@ -503,7 +505,8 @@ i15 = int_is_true(i5) i16 = int_is_true(i6) i17 = int_is_true(i7) - finish(i10, i11, i12, i13, i14, i15, i16, i17) + guard_true(i0) [i10, i11, i12, i13, i14, i15, i16, i17] + finish() ''' self.interpret(ops, [0, 42, 12, 0, 13, 0, 0, 3333]) assert self.getints(8) == [0, 1, 1, 0, 1, 0, 0, 1] @@ -517,7 +520,8 @@ i13 = int_eq(i5, i6) i14 = int_gt(i6, i2) i15 = int_ne(i2, i6) - finish(i10, i11, i12, i13, i14, i15) + guard_true(i0) [i10, i11, i12, i13, i14, i15] + finish() ''' self.interpret(ops, [0, 1, 2, 3, 4, 5, 6]) assert self.getints(6) == [1, 1, 0, 0, 1, 1] @@ -574,7 +578,9 @@ ops = ''' [f0, f1] f2 = float_add(f0, f1) - finish(f2, f0, f1) + i0 = same_as(0) + guard_true(i0) [f2, f0, f1] + finish() ''' self.interpret(ops, [3.0, 1.5]) assert self.getfloats(3) == [4.5, 3.0, 1.5] @@ -584,7 +590,9 @@ [f0, f1, f2, f3, f4, f5, f6, f7, f8] f9 = float_add(f0, f1) f10 = float_add(f8, 3.5) - finish(f9, f10, f2, f3, f4, f5, f6, f7, f8) + i0 = same_as(0) + guard_true(i0) [f9, f10, f2, f3, f4, f5, f6, f7, f8] + finish() ''' self.interpret(ops, [0.1, .2, .3, .4, .5, .6, .7, .8, .9]) assert self.getfloats(9) == [.1+.2, .9+3.5, .3, .4, .5, .6, .7, .8, .9] @@ -613,7 +621,8 @@ i7 = float_ne(f7, 0.0) i8 = float_ne(f8, 0.0) i9 = float_ne(f9, 0.0) - finish(i0, i1, i2, i3, i4, i5, i6, i7, i8, i9) + guard_true(i0) [i0, i1, i2, i3, i4, i5, i6, i7, i8, i9] + finish() ''' loop = self.interpret(ops, [0.0, .1, .2, .3, .4, .5, .6, .7, .8, .9]) assert self.getints(9) == [0, 1, 1, 1, 1, 1, 1, 1, 1] diff --git a/pypy/jit/backend/x86/test/test_regalloc2.py b/pypy/jit/backend/x86/test/test_regalloc2.py --- a/pypy/jit/backend/x86/test/test_regalloc2.py +++ b/pypy/jit/backend/x86/test/test_regalloc2.py @@ -11,13 +11,17 @@ v2 = BoxInt() v3 = BoxInt() v4 = BoxInt() + zero = BoxInt() inputargs = [v1] operations = [ ResOperation(rop.INT_ADD, [v1, v1], v2), ResOperation(rop.INT_INVERT, [v2], v3), ResOperation(rop.UINT_RSHIFT, [v1, ConstInt(3)], v4), - ResOperation(rop.FINISH, [v4, v3], None, descr=BasicFailDescr()), + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=BasicFailDescr()), + ResOperation(rop.FINISH, [], None, descr=BasicFailDescr()) ] + operations[-2].setfailargs([v4, v3]) cpu = CPU(None, None) cpu.setup_once() looptoken = JitCellToken() @@ -31,6 +35,7 @@ v2 = BoxInt() v3 = BoxInt() v4 = BoxInt() + zero = BoxInt() tmp5 = BoxInt() inputargs = [v1] operations = [ @@ -38,8 +43,11 @@ ResOperation(rop.INT_MUL, [v2, v1], v3), ResOperation(rop.INT_IS_TRUE, [v2], tmp5), ResOperation(rop.INT_IS_ZERO, [tmp5], v4), - ResOperation(rop.FINISH, [v4, v3, tmp5], None, descr=BasicFailDescr()), + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=BasicFailDescr()), + ResOperation(rop.FINISH, [], None, descr=BasicFailDescr()) ] + operations[-2].setfailargs([v4, v3, tmp5]) cpu = CPU(None, None) cpu.setup_once() looptoken = JitCellToken() @@ -90,6 +98,7 @@ v38 = BoxInt() v39 = BoxInt() v40 = BoxInt() + zero = BoxInt() tmp41 = BoxInt() tmp42 = BoxInt() tmp43 = BoxInt() @@ -134,8 +143,12 @@ ResOperation(rop.UINT_GT, [v33, ConstInt(-11)], v38), ResOperation(rop.INT_NEG, [v7], v39), ResOperation(rop.INT_GT, [v24, v32], v40), - ResOperation(rop.FINISH, [v40, v36, v37, v31, v16, v34, v35, v23, v22, v29, v14, v39, v30, v38], None, descr=BasicFailDescr()), + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=BasicFailDescr()), + ResOperation(rop.FINISH, [], None, descr=BasicFailDescr()) ] + operations[-2].setfailargs([v40, v36, v37, v31, v16, v34, v35, v23, + v22, v29, v14, v39, v30, v38]) cpu = CPU(None, None) cpu.setup_once() looptoken = JitCellToken() @@ -198,6 +211,7 @@ v38 = BoxInt() v39 = BoxInt() v40 = BoxInt() + zero = BoxInt() tmp41 = BoxInt() tmp42 = BoxInt() tmp43 = BoxInt() @@ -240,8 +254,13 @@ ResOperation(rop.INT_GT, [v4, v11], v38), ResOperation(rop.INT_LT, [v27, v22], v39), ResOperation(rop.INT_NEG, [v27], v40), - ResOperation(rop.FINISH, [v40, v10, v36, v26, v13, v30, v21, v33, v18, v25, v31, v32, v28, v29, v35, v38, v20, v39, v34, v23, v37], None, descr=BasicFailDescr()), + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=BasicFailDescr()), + ResOperation(rop.FINISH, [], None, descr=BasicFailDescr()) ] + operations[-2].setfailargs([v40, v10, v36, v26, v13, v30, v21, v33, + v18, v25, v31, v32, v28, v29, v35, v38, + v20, v39, v34, v23, v37]) cpu = CPU(None, None) cpu.setup_once() looptoken = JitCellToken() diff --git a/pypy/jit/backend/x86/test/test_runner.py b/pypy/jit/backend/x86/test/test_runner.py --- a/pypy/jit/backend/x86/test/test_runner.py +++ b/pypy/jit/backend/x86/test/test_runner.py @@ -499,9 +499,12 @@ i6, descr=calldescr), ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr), - ResOperation(rop.FINISH, [i3, i4, i5, i6], None, - descr=BasicFailDescr(0)) + ResOperation(rop.GUARD_FALSE, [i3], None, + descr=BasicFailDescr(0)), + ResOperation(rop.FINISH, [], None, + descr=BasicFailDescr(1)) ] + ops[-2].setfailargs([i3, i4, i5, i6]) ops[1].setfailargs([]) ops[3].setfailargs([]) ops[5].setfailargs([]) diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py --- a/pypy/jit/codewriter/support.py +++ b/pypy/jit/codewriter/support.py @@ -10,7 +10,7 @@ from pypy.rpython.extregistry import ExtRegistryEntry from pypy.translator.simplify import get_funcobj from pypy.translator.unsimplify import split_block -from pypy.objspace.flow.model import Constant +from pypy.objspace.flow.model import Variable, Constant from pypy.translator.translator import TranslationContext from pypy.annotation.policy import AnnotatorPolicy from pypy.annotation import model as annmodel @@ -59,6 +59,45 @@ rtyper = annotate(func, values) return rtyper.annotator.translator.graphs[0] +def autodetect_jit_markers_redvars(graph): + # the idea is to find all the jit_merge_point and + # add all the variables across the links to the reds. + for block, op in graph.iterblockops(): + if op.opname == 'jit_marker': + jitdriver = op.args[1].value + if not jitdriver.autoreds: + continue + # if we want to support also can_enter_jit, we should find a + # way to detect a consistent set of red vars to pass *both* to + # jit_merge_point and can_enter_jit. The current simple + # solution doesn't work because can_enter_jit might be in + # another block, so the set of alive_v will be different. + methname = op.args[0].value + assert methname == 'jit_merge_point', ( + "reds='auto' is supported only for jit drivers which " + "calls only jit_merge_point. Found a call to %s" % methname) + # + # compute the set of live variables across the jit_marker + alive_v = set() + for link in block.exits: + alive_v.update(link.args) + alive_v.difference_update(link.getextravars()) + for op1 in block.operations[::-1]: + if op1 is op: + break # stop when the meet the jit_marker + alive_v.discard(op1.result) + alive_v.update(op1.args) + greens_v = op.args[2:] + reds_v = alive_v - set(greens_v) + reds_v = [v for v in reds_v if isinstance(v, Variable) and + v.concretetype is not lltype.Void] + reds_v = sort_vars(reds_v) + op.args.extend(reds_v) + if jitdriver.numreds is None: + jitdriver.numreds = len(reds_v) + else: + assert jitdriver.numreds == len(reds_v), 'inconsistent number of reds_v' + def split_before_jit_merge_point(graph, portalblock, portalopindex): """Split the block just before the 'jit_merge_point', making sure the input args are in the canonical order. diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py --- a/pypy/jit/metainterp/resoperation.py +++ b/pypy/jit/metainterp/resoperation.py @@ -353,6 +353,9 @@ def initarglist(self, args): self._args = args + if not we_are_translated() and \ + self.__class__.__name__.startswith('FINISH'): # XXX remove me + assert len(args) <= 1 # FINISH operations take 0 or 1 arg now def getarglist(self): return self._args diff --git a/pypy/jit/metainterp/test/test_warmspot.py b/pypy/jit/metainterp/test/test_warmspot.py --- a/pypy/jit/metainterp/test/test_warmspot.py +++ b/pypy/jit/metainterp/test/test_warmspot.py @@ -383,6 +383,23 @@ assert res == expected self.check_resops(int_sub=2, int_mul=0, int_add=2) + def test_loop_automatic_reds_not_too_many_redvars(self): + myjitdriver = JitDriver(greens = ['m'], reds = 'auto') + def one(): + return 1 + def f(n, m): + res = 0 + while n > 0: + n -= one() + myjitdriver.jit_merge_point(m=m) + res += m*2 + return res + expected = f(21, 5) + res = self.meta_interp(f, [21, 5]) + assert res == expected + oplabel = get_stats().loops[0].operations[0] + assert len(oplabel.getarglist()) == 2 # 'n', 'res' in some order + def test_inline_jit_merge_point(self): # test that the machinery to inline jit_merge_points in callers # works. The final user does not need to mess manually with the diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py --- a/pypy/jit/metainterp/warmspot.py +++ b/pypy/jit/metainterp/warmspot.py @@ -347,49 +347,13 @@ self.jitdrivers_sd = [] graphs = self.translator.graphs for graph, block, pos in find_jit_merge_points(graphs): - self.autodetect_jit_markers_redvars(graph) + support.autodetect_jit_markers_redvars(graph) self.split_graph_and_record_jitdriver(graph, block, pos) # assert (len(set([jd.jitdriver for jd in self.jitdrivers_sd])) == len(self.jitdrivers_sd)), \ "there are multiple jit_merge_points with the same jitdriver" - def autodetect_jit_markers_redvars(self, graph): - # the idea is to find all the jit_merge_point and can_enter_jit and - # add all the variables across the links to the reds. - for block, op in graph.iterblockops(): - if op.opname == 'jit_marker': - jitdriver = op.args[1].value - if not jitdriver.autoreds: - continue - # if we want to support also can_enter_jit, we should find a - # way to detect a consistent set of red vars to pass *both* to - # jit_merge_point and can_enter_jit. The current simple - # solution doesn't work because can_enter_jit might be in - # another block, so the set of alive_v will be different. - methname = op.args[0].value - assert methname == 'jit_merge_point', ( - "reds='auto' is supported only for jit drivers which " - "calls only jit_merge_point. Found a call to %s" % methname) - # - # compute the set of live variables before the jit_marker - alive_v = set(block.inputargs) - for op1 in block.operations: - if op1 is op: - break # stop when the meet the jit_marker - if op1.result.concretetype != lltype.Void: - alive_v.add(op1.result) - greens_v = op.args[2:] - reds_v = alive_v - set(greens_v) - reds_v = [v for v in reds_v if v.concretetype is not lltype.Void] - reds_v = support.sort_vars(reds_v) - op.args.extend(reds_v) - if jitdriver.numreds is None: - jitdriver.numreds = len(reds_v) - else: - assert jitdriver.numreds == len(reds_v), 'inconsistent number of reds_v' - - def split_graph_and_record_jitdriver(self, graph, block, pos): op = block.operations[pos] jd = JitDriverStaticData() @@ -717,6 +681,9 @@ jitdriver = op.args[1].value assert jitdriver in sublists, \ "can_enter_jit with no matching jit_merge_point" + assert not jitdriver.autoreds, ( + "can_enter_jit not supported with a jitdriver that " + "has reds='auto'") jd, sublist = sublists[jitdriver] origportalgraph = jd._jit_merge_point_in if graph is not origportalgraph: From noreply at buildbot.pypy.org Sat Jan 12 10:59:30 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 12 Jan 2013 10:59:30 +0100 (CET) Subject: [pypy-commit] pypy rdict-experiments-2: two more things that are needed these days Message-ID: <20130112095930.72CF21C11E8@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: rdict-experiments-2 Changeset: r59985:867b49505d4b Date: 2013-01-12 11:58 +0200 http://bitbucket.org/pypy/pypy/changeset/867b49505d4b/ Log: two more things that are needed these days diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -42,7 +42,7 @@ translation_modules.update(dict.fromkeys( ["fcntl", "rctime", "select", "signal", "_rawffi", "zlib", "struct", "_md5", "cStringIO", "array", "_ffi", - "binascii", + "binascii", "itertools", "_collections", # the following are needed for pyrepl (and hence for the # interactive prompt/pdb) "termios", "_minimal_curses", From noreply at buildbot.pypy.org Sat Jan 12 14:43:24 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 12 Jan 2013 14:43:24 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: kill kill kill - enough progress to pass call_loop Message-ID: <20130112134324.B12491C0343@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r59986:3a2e21e6fe99 Date: 2013-01-12 15:41 +0200 http://bitbucket.org/pypy/pypy/changeset/3a2e21e6fe99/ Log: kill kill kill - enough progress to pass call_loop diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py --- a/pypy/jit/backend/llsupport/llmodel.py +++ b/pypy/jit/backend/llsupport/llmodel.py @@ -279,7 +279,7 @@ if descr.final_descr: assert index == 0 return 0 - xxx + return descr.rd_locs[index] def get_int_value(self, deadframe, index): pos = self._decode_pos(deadframe, index) diff --git a/pypy/jit/backend/x86/arch.py b/pypy/jit/backend/x86/arch.py --- a/pypy/jit/backend/x86/arch.py +++ b/pypy/jit/backend/x86/arch.py @@ -48,14 +48,15 @@ FORCE_INDEX_OFS = 0 SAVED_REGISTERS = 1 # range(1, 5) MY_COPY_OF_REGS = 5 # range(5, 9) - JITFRAME_FIXED_SIZE = 1 + XXX else: # rbp + rbx + r12 + r13 + r14 + r15 + 11 extra words + force_index = 18 FRAME_FIXED_SIZE = 18 # 18 aligned to 16 bytes = 2 * WORD FORCE_INDEX_OFS = 0 SAVED_REGISTERS = 1 # range(1, 7) MY_COPY_OF_REGS = 7 # range(7, 18) - JITFRAME_FIXED_SIZE = 1 + JITFRAME_FIXED_SIZE = 32 # 1 for the number and 32 for all the registers, + # but they're never used together # "My copy of regs" has room for almost all registers, apart from eax and edx # which are used in the malloc itself. They are: diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -14,7 +14,8 @@ gpr_reg_mgr_cls, xmm_reg_mgr_cls, _valid_addressing_size) from pypy.jit.backend.x86.arch import (FRAME_FIXED_SIZE, FORCE_INDEX_OFS, WORD, - IS_X86_32, IS_X86_64) + IS_X86_32, IS_X86_64, + JITFRAME_FIXED_SIZE) from pypy.jit.backend.x86.regloc import (eax, ecx, edx, ebx, esp, ebp, esi, edi, @@ -698,6 +699,7 @@ p = rffi.cast(rffi.INTP, adr_jump_offset) adr_target = adr_jump_offset + 4 + rffi.cast(lltype.Signed, p[0]) # skip the CALL + XXX if WORD == 4: adr_target += 5 # CALL imm else: @@ -1834,279 +1836,41 @@ exc = guardtok.exc target = self.failure_recovery_code[exc + 2 * withfloats] if WORD == 4: - mc.CALL(imm(target)) + mc.PUSH(imm(fail_index)) + mc.JMP(imm(target)) else: - # Generate exactly 13 bytes: - # MOV r11, target-as-8-bytes - # CALL *r11 - # Keep the number 13 in sync with _find_failure_recovery_bytecode. - start = mc.get_relative_pos() mc.MOV_ri64(X86_64_SCRATCH_REG.value, target) - mc.CALL_r(X86_64_SCRATCH_REG.value) - assert mc.get_relative_pos() == start + 13 + mc.PUSH(imm(fail_index)) + mc.JMP_r(X86_64_SCRATCH_REG.value) # write tight data that describes the failure recovery if guardtok.is_guard_not_forced: + XXX mc.writechar(chr(self.CODE_FORCED)) - self.write_failure_recovery_description(mc, guardtok.failargs, - guardtok.fail_locs) - # write the fail_index too - mc.writeimm32(fail_index) + positions = [0] * len(guardtok.fail_locs) + assert IS_X86_64 + for i, loc in enumerate(guardtok.fail_locs): + if loc is None: + positions[i] = -1 + elif isinstance(loc, StackLoc): + xxx + else: + assert isinstance(loc, RegLoc) + positions[i] = (loc.value + loc.is_xmm * 16) * WORD + guardtok.faildescr.rd_locs = positions + # write fail_index too # for testing the decoding, write a final byte 0xCC - if not we_are_translated(): - mc.writechar('\xCC') - faillocs = [loc for loc in guardtok.fail_locs if loc is not None] - guardtok.faildescr._x86_debug_faillocs = faillocs return startpos - DESCR_REF = 0x00 - DESCR_INT = 0x01 - DESCR_FLOAT = 0x02 - DESCR_SPECIAL = 0x03 - CODE_FROMSTACK = 4 * (8 + 8*IS_X86_64) - CODE_STOP = 0 | DESCR_SPECIAL - CODE_HOLE = 4 | DESCR_SPECIAL - CODE_INPUTARG = 8 | DESCR_SPECIAL - CODE_FORCED = 12 | DESCR_SPECIAL - - def write_failure_recovery_description(self, mc, failargs, locs): - for i in range(len(failargs)): - arg = failargs[i] - if arg is not None: - if arg.type == REF: - kind = self.DESCR_REF - elif arg.type == INT: - kind = self.DESCR_INT - elif arg.type == FLOAT: - kind = self.DESCR_FLOAT - else: - raise AssertionError("bogus kind") - loc = locs[i] - if isinstance(loc, StackLoc): - pos = loc.position - if pos < 0: - mc.writechar(chr(self.CODE_INPUTARG)) - pos = ~pos - n = self.CODE_FROMSTACK//4 + pos - else: - assert isinstance(loc, RegLoc) - n = loc.value - n = kind + 4*n - while n > 0x7F: - mc.writechar(chr((n & 0x7F) | 0x80)) - n >>= 7 - else: - n = self.CODE_HOLE - mc.writechar(chr(n)) - mc.writechar(chr(self.CODE_STOP)) - - def rebuild_faillocs_from_descr(self, bytecode): - from pypy.jit.backend.x86.regalloc import X86FrameManager - descr_to_box_type = [REF, INT, FLOAT] - bytecode = rffi.cast(rffi.UCHARP, bytecode) - arglocs = [] - code_inputarg = False - while 1: - # decode the next instruction from the bytecode - code = rffi.cast(lltype.Signed, bytecode[0]) - bytecode = rffi.ptradd(bytecode, 1) - if code >= self.CODE_FROMSTACK: - # 'code' identifies a stack location - if code > 0x7F: - shift = 7 - code &= 0x7F - while True: - nextcode = rffi.cast(lltype.Signed, bytecode[0]) - bytecode = rffi.ptradd(bytecode, 1) - code |= (nextcode & 0x7F) << shift - shift += 7 - if nextcode <= 0x7F: - break - kind = code & 3 - code = (code - self.CODE_FROMSTACK) >> 2 - if code_inputarg: - code = ~code - code_inputarg = False - loc = X86FrameManager.frame_pos(code, descr_to_box_type[kind]) - elif code == self.CODE_STOP: - break - elif code == self.CODE_HOLE: - continue - elif code == self.CODE_INPUTARG: - code_inputarg = True - continue - else: - # 'code' identifies a register - kind = code & 3 - code >>= 2 - if kind == self.DESCR_FLOAT: - loc = regloc.XMMREGLOCS[code] - else: - loc = regloc.REGLOCS[code] - arglocs.append(loc) - return arglocs[:] - - @staticmethod - @rgc.no_collect - def grab_frame_values(cpu, bytecode, frame_addr, allregisters): - # no malloc allowed here!! xxx apart from one, hacking a lot - #self.fail_ebp = allregisters[16 + ebp.value] - num = 0 - import pdb - pdb.set_trace() - XXX - deadframe = lltype.nullptr(jitframe.DEADFRAME) - # step 1: lots of mess just to count the final value of 'num' - bytecode1 = bytecode - while 1: - code = rffi.cast(lltype.Signed, bytecode1[0]) - bytecode1 = rffi.ptradd(bytecode1, 1) - if code >= Assembler386.CODE_FROMSTACK: - while code > 0x7F: - code = rffi.cast(lltype.Signed, bytecode1[0]) - bytecode1 = rffi.ptradd(bytecode1, 1) - else: - kind = code & 3 - if kind == Assembler386.DESCR_SPECIAL: - if code == Assembler386.CODE_HOLE: - num += 1 - continue - if code == Assembler386.CODE_INPUTARG: - continue - if code == Assembler386.CODE_FORCED: - # resuming from a GUARD_NOT_FORCED - token = allregisters[16 + ebp.value] - deadframe = ( - cpu.assembler.force_token_to_dead_frame.pop(token)) - deadframe = lltype.cast_opaque_ptr( - jitframe.DEADFRAMEPTR, deadframe) - continue - assert code == Assembler386.CODE_STOP - break - num += 1 - # allocate the deadframe - if not deadframe: - # Remove the "reserve" at the end of the nursery. This means - # that it is guaranteed that the following malloc() works - # without requiring a collect(), but it needs to be re-added - # as soon as possible. - assert num <= cpu.get_failargs_limit() - try: - deadframe = lltype.malloc(jitframe.DEADFRAME, num) - except MemoryError: - fatalerror("memory usage error in grab_frame_values") - # fill it - code_inputarg = False - num = 0 - value_hi = 0 - while 1: - # decode the next instruction from the bytecode - code = rffi.cast(lltype.Signed, bytecode[0]) - bytecode = rffi.ptradd(bytecode, 1) - if code >= Assembler386.CODE_FROMSTACK: - if code > 0x7F: - shift = 7 - code &= 0x7F - while True: - nextcode = rffi.cast(lltype.Signed, bytecode[0]) - bytecode = rffi.ptradd(bytecode, 1) - code |= (nextcode & 0x7F) << shift - shift += 7 - if nextcode <= 0x7F: - break - # load the value from the stack - kind = code & 3 - code = (code - Assembler386.CODE_FROMSTACK) >> 2 - if code_inputarg: - code = ~code - code_inputarg = False - stackloc = frame_addr + get_ebp_ofs(code) - value = rffi.cast(rffi.LONGP, stackloc)[0] - if kind == Assembler386.DESCR_FLOAT and WORD == 4: - value_hi = value - value = rffi.cast(rffi.LONGP, stackloc - 4)[0] - else: - kind = code & 3 - if kind == Assembler386.DESCR_SPECIAL: - if code == Assembler386.CODE_HOLE: - num += 1 - continue - if code == Assembler386.CODE_INPUTARG: - code_inputarg = True - continue - if code == Assembler386.CODE_FORCED: - continue - assert code == Assembler386.CODE_STOP - break - # 'code' identifies a register: load its value - code >>= 2 - if kind == Assembler386.DESCR_FLOAT: - if WORD == 4: - value = allregisters[2*code] - value_hi = allregisters[2*code + 1] - else: - value = allregisters[code] - else: - value = allregisters[16 + code] - - # store the loaded value into fail_boxes_ - if kind == Assembler386.DESCR_INT: - deadframe.jf_values[num].int = value - elif kind == Assembler386.DESCR_REF: - deadframe.jf_values[num].ref = rffi.cast(llmemory.GCREF, value) - elif kind == Assembler386.DESCR_FLOAT: - if WORD == 4: - assert not longlong.is_64_bit - floatvalue = rffi.cast(lltype.SignedLongLong, value_hi) - floatvalue <<= 32 - floatvalue |= rffi.cast(lltype.SignedLongLong, - rffi.cast(lltype.Unsigned, value)) - else: - assert longlong.is_64_bit - floatvalue = longlong2float.longlong2float(value) - deadframe.jf_values[num].float = floatvalue - else: - assert 0, "bogus kind" - num += 1 - # - assert num == len(deadframe.jf_values) - if not we_are_translated(): - assert bytecode[4] == 0xCC - #self.fail_boxes_count = num - fail_index = rffi.cast(rffi.INTP, bytecode)[0] - fail_descr = cpu.get_fail_descr_from_number(fail_index) - deadframe.jf_descr = fail_descr.hide(cpu) - return lltype.cast_opaque_ptr(llmemory.GCREF, deadframe) - def setup_failure_recovery(self): - @rgc.no_collect - def failure_recovery_func(registers): - # 'registers' is a pointer to a structure containing the - # original value of the registers, optionally the original - # value of XMM registers, and finally a reference to the - # recovery bytecode. See _build_failure_recovery() for details. - stack_at_ebp = registers[ebp.value] - bytecode = rffi.cast(rffi.UCHARP, registers[self.cpu.NUM_REGS]) - allregisters = rffi.ptradd(registers, -16) - return self.grab_frame_values(self.cpu, bytecode, stack_at_ebp, - allregisters) - - self.failure_recovery_func = failure_recovery_func self.failure_recovery_code = [0, 0, 0, 0] - _FAILURE_RECOVERY_FUNC = lltype.Ptr(lltype.FuncType([rffi.LONGP], - llmemory.GCREF)) - def _build_failure_recovery(self, exc, withfloats=False): - failure_recovery_func = llhelper(self._FAILURE_RECOVERY_FUNC, - self.failure_recovery_func) - failure_recovery_func = rffi.cast(lltype.Signed, - failure_recovery_func) mc = codebuf.MachineCodeBlockWrapper() self.mc = mc # Push all general purpose registers - for gpr in range(self.cpu.NUM_REGS-1, -1, -1): - mc.PUSH_r(gpr) + for gpr in range(self.cpu.NUM_REGS): + mc.MOV_br(gpr * WORD, gpr) if exc: # We might have an exception pending. Load it into ebx @@ -2115,42 +1879,23 @@ mc.MOV(heap(self.cpu.pos_exception()), imm0) mc.MOV(heap(self.cpu.pos_exc_value()), imm0) - # Load the current esp value into edi. On 64-bit, this is the - # argument. On 32-bit, it will be pushed as argument below. - mc.MOV_rr(edi.value, esp.value) - if withfloats: - # Push all float registers - mc.SUB_ri(esp.value, self.cpu.NUM_REGS*8) for i in range(self.cpu.NUM_REGS): - mc.MOVSD_sx(8*i, i) - - # the following call saves all values from the stack and from - # registers to a fresh new deadframe object. - # Note that the registers are saved so far in esi[0] to esi[7], - # as pushed above, plus optionally in esi[-16] to esi[-1] for - # the XMM registers. Moreover, esi[8] is a pointer to the recovery - # bytecode, pushed just before by the CALL instruction written by - # generate_quick_failure(). - - if IS_X86_32: - mc.SUB_ri(esp.value, 3*WORD) # for stack alignment - mc.PUSH_r(edi.value) - - mc.CALL(imm(failure_recovery_func)) - # returns in eax the deadframe object + mc.MOVSD_bx((16 + i) * WORD, i) if exc: # save ebx into 'jf_guard_exc' from pypy.jit.backend.llsupport.descr import unpack_fielddescr descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) offset, size, _ = unpack_fielddescr(descrs.jf_guard_exc) + # XXX ebp relative, not eax relative mc.MOV_mr((eax.value, offset), ebx.value) # now we return from the complete frame, which starts from # _call_header_with_stack_check(). The LEA in _call_footer below # throws away most of the frame, including all the PUSHes that we # did just above. + mc.POP(eax) self._call_footer() rawstart = mc.materialize(self.cpu.asmmemmgr, []) diff --git a/pypy/jit/backend/x86/runner.py b/pypy/jit/backend/x86/runner.py --- a/pypy/jit/backend/x86/runner.py +++ b/pypy/jit/backend/x86/runner.py @@ -116,7 +116,7 @@ func = rffi.cast(FUNCPTR, addr) #llop.debug_print(lltype.Void, ">>>> Entering", addr) frame = lltype.malloc(jitframe.JITFRAME, clt.frame_depth + - JITFRAME_FIXED_SIZE) + JITFRAME_FIXED_SIZE, zero=True) frame.jf_frame_info = clt.frame_info ll_frame = lltype.cast_opaque_ptr(llmemory.GCREF, frame) prev_interpreter = None # help flow space From noreply at buildbot.pypy.org Sat Jan 12 14:43:26 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 12 Jan 2013 14:43:26 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: seems to be fine, pass some more, now on bridges Message-ID: <20130112134326.146E21C054C@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r59987:c8e8d49b84f7 Date: 2013-01-12 15:42 +0200 http://bitbucket.org/pypy/pypy/changeset/c8e8d49b84f7/ Log: seems to be fine, pass some more, now on bridges diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -699,7 +699,6 @@ p = rffi.cast(rffi.INTP, adr_jump_offset) adr_target = adr_jump_offset + 4 + rffi.cast(lltype.Signed, p[0]) # skip the CALL - XXX if WORD == 4: adr_target += 5 # CALL imm else: From noreply at buildbot.pypy.org Sat Jan 12 16:03:07 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 12 Jan 2013 16:03:07 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: sort out the indexing and compress the frame Message-ID: <20130112150307.21C9C1C0343@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r59988:1dc6fdd4000d Date: 2013-01-12 16:59 +0200 http://bitbucket.org/pypy/pypy/changeset/1dc6fdd4000d/ Log: sort out the indexing and compress the frame diff --git a/pypy/jit/backend/x86/arch.py b/pypy/jit/backend/x86/arch.py --- a/pypy/jit/backend/x86/arch.py +++ b/pypy/jit/backend/x86/arch.py @@ -49,15 +49,17 @@ SAVED_REGISTERS = 1 # range(1, 5) MY_COPY_OF_REGS = 5 # range(5, 9) XXX + JITFRAME_FIXED_SIZE = 29 # 13 GPR + 16 XMM + # reg, we don't save it else: # rbp + rbx + r12 + r13 + r14 + r15 + 11 extra words + force_index = 18 FRAME_FIXED_SIZE = 18 # 18 aligned to 16 bytes = 2 * WORD FORCE_INDEX_OFS = 0 SAVED_REGISTERS = 1 # range(1, 7) MY_COPY_OF_REGS = 7 # range(7, 18) - JITFRAME_FIXED_SIZE = 32 # 1 for the number and 32 for all the registers, - # but they're never used together - + JITFRAME_FIXED_SIZE = 29 # 13 GPR + 16 XMM + # reg, we don't save it + # "My copy of regs" has room for almost all registers, apart from eax and edx # which are used in the malloc itself. They are: # ecx, ebx, esi, edi [32 and 64 bits] diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -563,17 +563,14 @@ assert len(set(inputargs)) == len(inputargs) descr_number = self.cpu.get_fail_descr_number(faildescr) - failure_recovery = self._find_failure_recovery_bytecode(faildescr) self.setup(original_loop_token) if log: operations = self._inject_debugging_code(faildescr, operations, 'b', descr_number) - arglocs = self.rebuild_faillocs_from_descr(failure_recovery) - if not we_are_translated(): - assert ([loc.assembler() for loc in arglocs] == - [loc.assembler() for loc in faildescr._x86_debug_faillocs]) + descr = self.cpu.get_fail_descr_from_number(descr_number) + arglocs = self.rebuild_faillocs_from_descr(descr) regalloc = RegAlloc(self, self.cpu.translate_support_code) startpos = self.mc.get_relative_pos() operations = regalloc.prepare_bridge(inputargs, arglocs, @@ -687,24 +684,7 @@ struct.number = compute_unique_id(token) self.loop_run_counters.append(struct) return struct - - def _find_failure_recovery_bytecode(self, faildescr): - adr_jump_offset = faildescr._x86_adr_jump_offset - if adr_jump_offset == 0: - # This case should be prevented by the logic in compile.py: - # look for CNT_BUSY_FLAG, which disables tracing from a guard - # when another tracing from the same guard is already in progress. - raise BridgeAlreadyCompiled - # follow the JMP/Jcond - p = rffi.cast(rffi.INTP, adr_jump_offset) - adr_target = adr_jump_offset + 4 + rffi.cast(lltype.Signed, p[0]) - # skip the CALL - if WORD == 4: - adr_target += 5 # CALL imm - else: - adr_target += 13 # MOV r11, imm-as-8-bytes; CALL *r11 xxxxxxxxxx - return adr_target - + def patch_jump_for_descr(self, faildescr, adr_new_target): adr_jump_offset = faildescr._x86_adr_jump_offset assert adr_jump_offset != 0 @@ -1846,7 +1826,6 @@ XXX mc.writechar(chr(self.CODE_FORCED)) positions = [0] * len(guardtok.fail_locs) - assert IS_X86_64 for i, loc in enumerate(guardtok.fail_locs): if loc is None: positions[i] = -1 @@ -1854,12 +1833,29 @@ xxx else: assert isinstance(loc, RegLoc) - positions[i] = (loc.value + loc.is_xmm * 16) * WORD + v = (gpr_reg_mgr_cls.all_reg_indexes[loc.value] + + loc.is_xmm * len(gpr_reg_mgr_cls.all_regs)) + positions[i] = v * WORD guardtok.faildescr.rd_locs = positions # write fail_index too # for testing the decoding, write a final byte 0xCC return startpos + def rebuild_faillocs_from_descr(self, descr): + locs = [] + for pos in descr.rd_locs: + if pos == -1: + pass + elif pos < self.cpu.NUM_REGS * WORD: + locs.append(gpr_reg_mgr_cls.all_regs[pos // WORD]) + elif pos < self.cpu.NUM_REGS * 2 * WORD: + locs.append(xmm_reg_mgr_cls.xrm.all_regs[pos // WORD]) + else: + i = pos // WORD - 2 * self.cpu.NUM_REGS + tp = xxx + locs.append(StackLoc(i, pos, tp)) + return locs + def setup_failure_recovery(self): self.failure_recovery_code = [0, 0, 0, 0] @@ -1868,8 +1864,8 @@ self.mc = mc # Push all general purpose registers - for gpr in range(self.cpu.NUM_REGS): - mc.MOV_br(gpr * WORD, gpr) + for i, gpr in enumerate(gpr_reg_mgr_cls.all_regs): + mc.MOV_br(i * WORD, gpr.value) if exc: # We might have an exception pending. Load it into ebx @@ -1879,8 +1875,9 @@ mc.MOV(heap(self.cpu.pos_exc_value()), imm0) if withfloats: + ofs = len(gpr_reg_mgr_cls.all_regs) for i in range(self.cpu.NUM_REGS): - mc.MOVSD_bx((16 + i) * WORD, i) + mc.MOVSD_bx((ofs + i) * WORD, i) if exc: # save ebx into 'jf_guard_exc' diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py --- a/pypy/jit/backend/x86/regalloc.py +++ b/pypy/jit/backend/x86/regalloc.py @@ -64,6 +64,7 @@ class X86_64_RegisterManager(X86RegisterManager): # r11 omitted because it's used as scratch all_regs = [ecx, eax, edx, ebx, esi, edi, r8, r9, r10, r12, r13, r14, r15] + no_lower_byte_regs = [] save_around_call_regs = [eax, ecx, edx, esi, edi, r8, r9, r10] @@ -153,6 +154,9 @@ else: raise AssertionError("Word size should be 4 or 8") +gpr_reg_mgr_cls.all_reg_indexes = [-1] * WORD * 2 # eh, happens to be true +for _i, _reg in enumerate(gpr_reg_mgr_cls.all_regs): + gpr_reg_mgr_cls.all_reg_indexes[_reg.value] = _i class RegAlloc(object): @@ -501,7 +505,8 @@ else: locs = [imm(fail_no)] self.Perform(op, locs, None) - self.possibly_free_var(op.getarg(0)) + if op.numargs() == 1: + self.possibly_free_var(op.getarg(0)) def consider_guard_no_exception(self, op): self.perform_guard(op, [], None) From noreply at buildbot.pypy.org Sat Jan 12 16:03:08 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 12 Jan 2013 16:03:08 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: merge default Message-ID: <20130112150308.C97F01C0343@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r59989:f3cfd5e2c1e3 Date: 2013-01-12 17:02 +0200 http://bitbucket.org/pypy/pypy/changeset/f3cfd5e2c1e3/ Log: merge default diff --git a/pypy/doc/test/test_whatsnew.py b/pypy/doc/test/test_whatsnew.py --- a/pypy/doc/test/test_whatsnew.py +++ b/pypy/doc/test/test_whatsnew.py @@ -29,7 +29,7 @@ merge() and \ branch(default)) and \ not branch(default)' % (startrev, endrev) - cmd = r"hg log -R '%s' -r '%s' --template '{branches}\n'" % (path, revset) + cmd = r'hg log -R "%s" -r "%s" --template "{branches}\n"' % (path, revset) out = getoutput(cmd) branches = set(map(str.strip, out.splitlines())) return branches diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -15,6 +15,9 @@ .. branch: numpypy-longdouble Long double support for numpypy +.. branch: numpypy-real-as-view +Convert real, imag from ufuncs to views. This involves the beginning of +view() functionality .. branch: signatures Improved RPython typing diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -178,6 +178,11 @@ else: oparg = 0 + # note: the structure of the code here is such that it makes + # (after translation) a big "if/elif" chain, which is then + # turned into a switch(). It starts here: even if the first + # one is not an "if" but a "while" the effect is the same. + while opcode == self.opcodedesc.EXTENDED_ARG.index: opcode = ord(co_code[next_instr]) if opcode < self.HAVE_ARGUMENT: @@ -226,6 +231,8 @@ 'END_FINALLY', 'JUMP_ABSOLUTE'): continue # opcodes implemented above + # the following "if" is part of the big switch described + # above. if opcode == opdesc.index: # dispatch to the opcode method meth = getattr(self, opdesc.methodname) diff --git a/pypy/jit/backend/test/calling_convention_test.py b/pypy/jit/backend/test/calling_convention_test.py --- a/pypy/jit/backend/test/calling_convention_test.py +++ b/pypy/jit/backend/test/calling_convention_test.py @@ -105,11 +105,13 @@ ops = '[%s]\n' % arguments ops += '%s\n' % spill_ops ops += 'f99 = call(ConstClass(func_ptr), %s, descr=calldescr)\n' % arguments - ops += 'finish(f99, %s)\n' % arguments + ops += 'i99 = same_as(0)\n' + ops += 'guard_true(i99) [f99, %s]\n' % arguments + ops += 'finish()\n' loop = parse(ops, namespace=locals()) looptoken = JitCellToken() - done_number = self.cpu.get_fail_descr_number(loop.operations[-1].getdescr()) + done_number = self.cpu.get_fail_descr_number(loop.operations[-2].getdescr()) self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) argvals, expected_result = self._prepare_args(args, floats, ints) diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py --- a/pypy/jit/backend/test/runner_test.py +++ b/pypy/jit/backend/test/runner_test.py @@ -1168,7 +1168,8 @@ values.append(longlong.getfloatstorage(r.random())) # looptoken = JitCellToken() - faildescr = BasicFinalDescr(42) + guarddescr = BasicFailDescr(42) + faildescr = BasicFinalDescr(43) operations = [] retboxes = [] retvalues = [] @@ -1197,9 +1198,13 @@ retboxes.insert(kk, newbox) retvalues.insert(kk, y) # - operations.append( - ResOperation(rop.FINISH, retboxes, None, descr=faildescr) - ) + zero = BoxInt() + operations.extend([ + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=guarddescr), + ResOperation(rop.FINISH, [], None, descr=faildescr) + ]) + operations[-2].setfailargs(retboxes) print inputargs for op in operations: print op @@ -1319,13 +1324,16 @@ i2 = BoxInt() targettoken = TargetToken() faildescr1 = BasicFailDescr(1) - faildescr2 = BasicFinalDescr(2) + faildescr2 = BasicFailDescr(2) + faildescr3 = BasicFinalDescr(3) operations = [ ResOperation(rop.LABEL, fboxes, None, descr=targettoken), ResOperation(rop.FLOAT_LE, [fboxes[0], constfloat(9.2)], i2), ResOperation(rop.GUARD_TRUE, [i2], None, descr=faildescr1), - ResOperation(rop.FINISH, fboxes, None, descr=faildescr2), + ResOperation(rop.GUARD_FALSE, [i2], None, descr=faildescr2), + ResOperation(rop.FINISH, [], None, descr=faildescr3), ] + operations[-3].setfailargs(fboxes) operations[-2].setfailargs(fboxes) looptoken = JitCellToken() self.cpu.compile_loop(fboxes, operations, looptoken) @@ -1357,7 +1365,8 @@ if not self.cpu.supports_floats: py.test.skip("requires floats") fboxes = [BoxFloat() for i in range(3)] - faildescr1 = BasicFinalDescr(100) + faildescr1 = BasicFailDescr(100) + faildescr2 = BasicFinalDescr(102) loopops = """ [i0,f1, f2] f3 = float_add(f1, f2) @@ -1382,9 +1391,13 @@ assert longlong.getrealfloat(f2) == 0.75 assert longlong.getrealfloat(f3) == 133.0 + zero = BoxInt() bridgeops = [ - ResOperation(rop.FINISH, fboxes, None, descr=faildescr1), + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=faildescr1), + ResOperation(rop.FINISH, [], None, descr=faildescr2), ] + bridgeops[-2].setfailargs(fboxes[:]) self.cpu.compile_bridge(loop.operations[-2].getdescr(), fboxes, bridgeops, looptoken) args = [1, @@ -1978,7 +1991,7 @@ i1 = same_as(1) call(ConstClass(fptr), i0, descr=calldescr) p0 = guard_exception(ConstClass(xtp)) [i1] - finish(0, p0) + finish(p0) ''' FPTR = lltype.Ptr(lltype.FuncType([lltype.Signed], lltype.Void)) fptr = llhelper(FPTR, func) @@ -1999,8 +2012,7 @@ looptoken = JitCellToken() self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) deadframe = self.cpu.execute_token(looptoken, 1) - assert self.cpu.get_int_value(deadframe, 0) == 0 - assert self.cpu.get_ref_value(deadframe, 1) == xptr + assert self.cpu.get_ref_value(deadframe, 0) == xptr excvalue = self.cpu.grab_exc_value(deadframe) assert not excvalue deadframe = self.cpu.execute_token(looptoken, 0) diff --git a/pypy/jit/backend/x86/test/test_recompilation.py b/pypy/jit/backend/x86/test/test_recompilation.py --- a/pypy/jit/backend/x86/test/test_recompilation.py +++ b/pypy/jit/backend/x86/test/test_recompilation.py @@ -45,7 +45,8 @@ force_spill(i5) i8 = int_add(i7, 1) i9 = int_add(i8, 1) - finish(i3, i4, i5, i6, i7, i8, i9, descr=fdescr2) + guard_false(i3, descr=fdescr2) [i3, i4, i5, i6, i7, i8, i9] + finish() ''' bridge = self.attach_bridge(ops, loop, -2) descr = loop.operations[3].getdescr() diff --git a/pypy/jit/backend/x86/test/test_regalloc.py b/pypy/jit/backend/x86/test/test_regalloc.py --- a/pypy/jit/backend/x86/test/test_regalloc.py +++ b/pypy/jit/backend/x86/test/test_regalloc.py @@ -327,7 +327,7 @@ assert self.getint(0) == 0 bridge_ops = ''' [i0, i1] - finish(1, 2) + finish(2) ''' self.attach_bridge(bridge_ops, loop, 0) self.run(loop, 0, 1) @@ -428,7 +428,7 @@ ops = ''' [i0, i1, i2, i3, i4, i5, i6, i7] guard_value(i6, i1) [i0, i2, i3, i4, i5, i6] - finish(i0, i2, i3, i4, i5, i6) + finish(i0) ''' self.interpret(ops, [0, 0, 0, 0, 0, 0, 0, 0]) assert self.getint(0) == 0 @@ -438,14 +438,15 @@ [i0, i1, i2, i3, i4, i5, i6, i7, i8] i9 = same_as(0) guard_true(i0) [i9, i0, i1, i2, i3, i4, i5, i6, i7, i8] - finish(1, i0, i1, i2, i3, i4, i5, i6, i7, i8) + finish(1) ''' loop = self.interpret(ops, [0, 1, 2, 3, 4, 5, 6, 7, 8]) assert self.getint(0) == 0 bridge_ops = ''' [i9, i0, i1, i2, i3, i4, i5, i6, i7, i8] call(ConstClass(raising_fptr), 0, descr=raising_calldescr) - finish(i0, i1, i2, i3, i4, i5, i6, i7, i8) + guard_true(i9) [i0, i1, i2, i3, i4, i5, i6, i7, i8] + finish() ''' self.attach_bridge(bridge_ops, loop, 1) self.run(loop, 0, 1, 2, 3, 4, 5, 6, 7, 8) @@ -474,7 +475,8 @@ i1 = same_as(1) i2 = int_lt(i0, 100) guard_true(i3) [i1, i2] - finish(0, i2) + i4 = int_neg(i2) + finish(0) ''' self.interpret(ops, [0, 1]) assert self.getint(0) == 0 @@ -503,7 +505,8 @@ i15 = int_is_true(i5) i16 = int_is_true(i6) i17 = int_is_true(i7) - finish(i10, i11, i12, i13, i14, i15, i16, i17) + guard_true(i0) [i10, i11, i12, i13, i14, i15, i16, i17] + finish() ''' self.interpret(ops, [0, 42, 12, 0, 13, 0, 0, 3333]) assert self.getints(8) == [0, 1, 1, 0, 1, 0, 0, 1] @@ -517,7 +520,8 @@ i13 = int_eq(i5, i6) i14 = int_gt(i6, i2) i15 = int_ne(i2, i6) - finish(i10, i11, i12, i13, i14, i15) + guard_true(i0) [i10, i11, i12, i13, i14, i15] + finish() ''' self.interpret(ops, [0, 1, 2, 3, 4, 5, 6]) assert self.getints(6) == [1, 1, 0, 0, 1, 1] @@ -574,7 +578,9 @@ ops = ''' [f0, f1] f2 = float_add(f0, f1) - finish(f2, f0, f1) + i0 = same_as(0) + guard_true(i0) [f2, f0, f1] + finish() ''' self.interpret(ops, [3.0, 1.5]) assert self.getfloats(3) == [4.5, 3.0, 1.5] @@ -584,7 +590,9 @@ [f0, f1, f2, f3, f4, f5, f6, f7, f8] f9 = float_add(f0, f1) f10 = float_add(f8, 3.5) - finish(f9, f10, f2, f3, f4, f5, f6, f7, f8) + i0 = same_as(0) + guard_true(i0) [f9, f10, f2, f3, f4, f5, f6, f7, f8] + finish() ''' self.interpret(ops, [0.1, .2, .3, .4, .5, .6, .7, .8, .9]) assert self.getfloats(9) == [.1+.2, .9+3.5, .3, .4, .5, .6, .7, .8, .9] @@ -613,7 +621,8 @@ i7 = float_ne(f7, 0.0) i8 = float_ne(f8, 0.0) i9 = float_ne(f9, 0.0) - finish(i0, i1, i2, i3, i4, i5, i6, i7, i8, i9) + guard_true(i0) [i0, i1, i2, i3, i4, i5, i6, i7, i8, i9] + finish() ''' loop = self.interpret(ops, [0.0, .1, .2, .3, .4, .5, .6, .7, .8, .9]) assert self.getints(9) == [0, 1, 1, 1, 1, 1, 1, 1, 1] diff --git a/pypy/jit/backend/x86/test/test_regalloc2.py b/pypy/jit/backend/x86/test/test_regalloc2.py --- a/pypy/jit/backend/x86/test/test_regalloc2.py +++ b/pypy/jit/backend/x86/test/test_regalloc2.py @@ -11,13 +11,17 @@ v2 = BoxInt() v3 = BoxInt() v4 = BoxInt() + zero = BoxInt() inputargs = [v1] operations = [ ResOperation(rop.INT_ADD, [v1, v1], v2), ResOperation(rop.INT_INVERT, [v2], v3), ResOperation(rop.UINT_RSHIFT, [v1, ConstInt(3)], v4), - ResOperation(rop.FINISH, [v4, v3], None, descr=BasicFailDescr()), + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=BasicFailDescr()), + ResOperation(rop.FINISH, [], None, descr=BasicFailDescr()) ] + operations[-2].setfailargs([v4, v3]) cpu = CPU(None, None) cpu.setup_once() looptoken = JitCellToken() @@ -31,6 +35,7 @@ v2 = BoxInt() v3 = BoxInt() v4 = BoxInt() + zero = BoxInt() tmp5 = BoxInt() inputargs = [v1] operations = [ @@ -38,8 +43,11 @@ ResOperation(rop.INT_MUL, [v2, v1], v3), ResOperation(rop.INT_IS_TRUE, [v2], tmp5), ResOperation(rop.INT_IS_ZERO, [tmp5], v4), - ResOperation(rop.FINISH, [v4, v3, tmp5], None, descr=BasicFailDescr()), + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=BasicFailDescr()), + ResOperation(rop.FINISH, [], None, descr=BasicFailDescr()) ] + operations[-2].setfailargs([v4, v3, tmp5]) cpu = CPU(None, None) cpu.setup_once() looptoken = JitCellToken() @@ -90,6 +98,7 @@ v38 = BoxInt() v39 = BoxInt() v40 = BoxInt() + zero = BoxInt() tmp41 = BoxInt() tmp42 = BoxInt() tmp43 = BoxInt() @@ -134,8 +143,12 @@ ResOperation(rop.UINT_GT, [v33, ConstInt(-11)], v38), ResOperation(rop.INT_NEG, [v7], v39), ResOperation(rop.INT_GT, [v24, v32], v40), - ResOperation(rop.FINISH, [v40, v36, v37, v31, v16, v34, v35, v23, v22, v29, v14, v39, v30, v38], None, descr=BasicFailDescr()), + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=BasicFailDescr()), + ResOperation(rop.FINISH, [], None, descr=BasicFailDescr()) ] + operations[-2].setfailargs([v40, v36, v37, v31, v16, v34, v35, v23, + v22, v29, v14, v39, v30, v38]) cpu = CPU(None, None) cpu.setup_once() looptoken = JitCellToken() @@ -198,6 +211,7 @@ v38 = BoxInt() v39 = BoxInt() v40 = BoxInt() + zero = BoxInt() tmp41 = BoxInt() tmp42 = BoxInt() tmp43 = BoxInt() @@ -240,8 +254,13 @@ ResOperation(rop.INT_GT, [v4, v11], v38), ResOperation(rop.INT_LT, [v27, v22], v39), ResOperation(rop.INT_NEG, [v27], v40), - ResOperation(rop.FINISH, [v40, v10, v36, v26, v13, v30, v21, v33, v18, v25, v31, v32, v28, v29, v35, v38, v20, v39, v34, v23, v37], None, descr=BasicFailDescr()), + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=BasicFailDescr()), + ResOperation(rop.FINISH, [], None, descr=BasicFailDescr()) ] + operations[-2].setfailargs([v40, v10, v36, v26, v13, v30, v21, v33, + v18, v25, v31, v32, v28, v29, v35, v38, + v20, v39, v34, v23, v37]) cpu = CPU(None, None) cpu.setup_once() looptoken = JitCellToken() diff --git a/pypy/jit/backend/x86/test/test_runner.py b/pypy/jit/backend/x86/test/test_runner.py --- a/pypy/jit/backend/x86/test/test_runner.py +++ b/pypy/jit/backend/x86/test/test_runner.py @@ -499,9 +499,12 @@ i6, descr=calldescr), ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr), - ResOperation(rop.FINISH, [i3, i4, i5, i6], None, - descr=BasicFailDescr(0)) + ResOperation(rop.GUARD_FALSE, [i3], None, + descr=BasicFailDescr(0)), + ResOperation(rop.FINISH, [], None, + descr=BasicFailDescr(1)) ] + ops[-2].setfailargs([i3, i4, i5, i6]) ops[1].setfailargs([]) ops[3].setfailargs([]) ops[5].setfailargs([]) diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py --- a/pypy/jit/codewriter/support.py +++ b/pypy/jit/codewriter/support.py @@ -10,7 +10,7 @@ from pypy.rpython.extregistry import ExtRegistryEntry from pypy.translator.simplify import get_funcobj from pypy.translator.unsimplify import split_block -from pypy.objspace.flow.model import Constant +from pypy.objspace.flow.model import Variable, Constant from pypy.translator.translator import TranslationContext from pypy.annotation.policy import AnnotatorPolicy from pypy.annotation import model as annmodel @@ -59,6 +59,45 @@ rtyper = annotate(func, values) return rtyper.annotator.translator.graphs[0] +def autodetect_jit_markers_redvars(graph): + # the idea is to find all the jit_merge_point and + # add all the variables across the links to the reds. + for block, op in graph.iterblockops(): + if op.opname == 'jit_marker': + jitdriver = op.args[1].value + if not jitdriver.autoreds: + continue + # if we want to support also can_enter_jit, we should find a + # way to detect a consistent set of red vars to pass *both* to + # jit_merge_point and can_enter_jit. The current simple + # solution doesn't work because can_enter_jit might be in + # another block, so the set of alive_v will be different. + methname = op.args[0].value + assert methname == 'jit_merge_point', ( + "reds='auto' is supported only for jit drivers which " + "calls only jit_merge_point. Found a call to %s" % methname) + # + # compute the set of live variables across the jit_marker + alive_v = set() + for link in block.exits: + alive_v.update(link.args) + alive_v.difference_update(link.getextravars()) + for op1 in block.operations[::-1]: + if op1 is op: + break # stop when the meet the jit_marker + alive_v.discard(op1.result) + alive_v.update(op1.args) + greens_v = op.args[2:] + reds_v = alive_v - set(greens_v) + reds_v = [v for v in reds_v if isinstance(v, Variable) and + v.concretetype is not lltype.Void] + reds_v = sort_vars(reds_v) + op.args.extend(reds_v) + if jitdriver.numreds is None: + jitdriver.numreds = len(reds_v) + else: + assert jitdriver.numreds == len(reds_v), 'inconsistent number of reds_v' + def split_before_jit_merge_point(graph, portalblock, portalopindex): """Split the block just before the 'jit_merge_point', making sure the input args are in the canonical order. diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py --- a/pypy/jit/metainterp/resoperation.py +++ b/pypy/jit/metainterp/resoperation.py @@ -357,6 +357,9 @@ def initarglist(self, args): self._args = args + if not we_are_translated() and \ + self.__class__.__name__.startswith('FINISH'): # XXX remove me + assert len(args) <= 1 # FINISH operations take 0 or 1 arg now def getarglist(self): return self._args diff --git a/pypy/jit/metainterp/test/test_warmspot.py b/pypy/jit/metainterp/test/test_warmspot.py --- a/pypy/jit/metainterp/test/test_warmspot.py +++ b/pypy/jit/metainterp/test/test_warmspot.py @@ -383,6 +383,23 @@ assert res == expected self.check_resops(int_sub=2, int_mul=0, int_add=2) + def test_loop_automatic_reds_not_too_many_redvars(self): + myjitdriver = JitDriver(greens = ['m'], reds = 'auto') + def one(): + return 1 + def f(n, m): + res = 0 + while n > 0: + n -= one() + myjitdriver.jit_merge_point(m=m) + res += m*2 + return res + expected = f(21, 5) + res = self.meta_interp(f, [21, 5]) + assert res == expected + oplabel = get_stats().loops[0].operations[0] + assert len(oplabel.getarglist()) == 2 # 'n', 'res' in some order + def test_inline_jit_merge_point(self): # test that the machinery to inline jit_merge_points in callers # works. The final user does not need to mess manually with the diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py --- a/pypy/jit/metainterp/warmspot.py +++ b/pypy/jit/metainterp/warmspot.py @@ -347,49 +347,13 @@ self.jitdrivers_sd = [] graphs = self.translator.graphs for graph, block, pos in find_jit_merge_points(graphs): - self.autodetect_jit_markers_redvars(graph) + support.autodetect_jit_markers_redvars(graph) self.split_graph_and_record_jitdriver(graph, block, pos) # assert (len(set([jd.jitdriver for jd in self.jitdrivers_sd])) == len(self.jitdrivers_sd)), \ "there are multiple jit_merge_points with the same jitdriver" - def autodetect_jit_markers_redvars(self, graph): - # the idea is to find all the jit_merge_point and can_enter_jit and - # add all the variables across the links to the reds. - for block, op in graph.iterblockops(): - if op.opname == 'jit_marker': - jitdriver = op.args[1].value - if not jitdriver.autoreds: - continue - # if we want to support also can_enter_jit, we should find a - # way to detect a consistent set of red vars to pass *both* to - # jit_merge_point and can_enter_jit. The current simple - # solution doesn't work because can_enter_jit might be in - # another block, so the set of alive_v will be different. - methname = op.args[0].value - assert methname == 'jit_merge_point', ( - "reds='auto' is supported only for jit drivers which " - "calls only jit_merge_point. Found a call to %s" % methname) - # - # compute the set of live variables before the jit_marker - alive_v = set(block.inputargs) - for op1 in block.operations: - if op1 is op: - break # stop when the meet the jit_marker - if op1.result.concretetype != lltype.Void: - alive_v.add(op1.result) - greens_v = op.args[2:] - reds_v = alive_v - set(greens_v) - reds_v = [v for v in reds_v if v.concretetype is not lltype.Void] - reds_v = support.sort_vars(reds_v) - op.args.extend(reds_v) - if jitdriver.numreds is None: - jitdriver.numreds = len(reds_v) - else: - assert jitdriver.numreds == len(reds_v), 'inconsistent number of reds_v' - - def split_graph_and_record_jitdriver(self, graph, block, pos): op = block.operations[pos] jd = JitDriverStaticData() @@ -717,6 +681,9 @@ jitdriver = op.args[1].value assert jitdriver in sublists, \ "can_enter_jit with no matching jit_merge_point" + assert not jitdriver.autoreds, ( + "can_enter_jit not supported with a jitdriver that " + "has reds='auto'") jd, sublist = sublists[jitdriver] origportalgraph = jd._jit_merge_point_in if graph is not origportalgraph: diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py --- a/pypy/module/micronumpy/arrayimpl/concrete.py +++ b/pypy/module/micronumpy/arrayimpl/concrete.py @@ -13,6 +13,7 @@ from pypy.rlib.debug import make_sure_not_resized class ConcreteArrayIterator(base.BaseArrayIterator): + _immutable_fields_ = ['dtype', 'skip', 'size'] def __init__(self, array): self.array = array self.offset = 0 @@ -21,10 +22,10 @@ self.size = array.size def setitem(self, elem): - self.array.setitem(self.offset, elem) + self.dtype.setitem(self.array, self.offset, elem) def getitem(self): - return self.array.getitem(self.offset) + return self.dtype.getitem(self.array, self.offset) def getitem_bool(self): return self.dtype.getitem_bool(self.array, self.offset) @@ -42,13 +43,16 @@ self.offset %= self.size class OneDimViewIterator(ConcreteArrayIterator): - def __init__(self, array): + ''' The view iterator dtype can be different from the + array.dtype, this is what makes it a View + ''' + def __init__(self, array, dtype, start, strides, shape): self.array = array - self.offset = array.start - self.skip = array.get_strides()[0] - self.dtype = array.dtype + self.dtype = dtype + self.offset = start + self.skip = strides[0] self.index = 0 - self.size = array.get_shape()[0] + self.size = shape[0] def next(self): self.offset += self.skip @@ -65,9 +69,13 @@ self.offset %= self.size class MultiDimViewIterator(ConcreteArrayIterator): - def __init__(self, array, start, strides, backstrides, shape): + ''' The view iterator dtype can be different from the + array.dtype, this is what makes it a View + ''' + def __init__(self, array, dtype, start, strides, backstrides, shape): self.indexes = [0] * len(shape) self.array = array + self.dtype = dtype self.shape = shape self.offset = start self.shapelen = len(shape) @@ -131,12 +139,13 @@ self.offset = array.start self.dim = dim self.array = array + self.dtype = array.dtype def setitem(self, elem): - self.array.setitem(self.offset, elem) + self.dtype.setitem(self.array, self.offset, elem) def getitem(self): - return self.array.getitem(self.offset) + return self.dtype.getitem(self.array, self.offset) @jit.unroll_safe def next(self): @@ -220,6 +229,32 @@ new_shape, self) else: return None + + def get_real(self): + strides = self.get_strides() + backstrides = self.get_backstrides() + if self.dtype.is_complex_type(): + dtype = self.dtype.float_type + return SliceArray(self.start, strides, backstrides, + self.get_shape(), self, dtype=dtype) + return SliceArray(self.start, strides, backstrides, + self.get_shape(), self) + + def get_imag(self): + strides = self.get_strides() + backstrides = self.get_backstrides() + if self.dtype.is_complex_type(): + dtype = self.dtype.float_type + return SliceArray(self.start + dtype.get_size(), strides, + backstrides, self.get_shape(), self, dtype=dtype) + if self.dtype.is_flexible_type(): + # numpy returns self for self.imag + return SliceArray(self.start, strides, backstrides, + self.get_shape(), self) + impl = NonWritableArray(self.get_shape(), self.dtype, self.order, strides, + backstrides) + impl.fill(self.dtype.box(0)) + return impl # -------------------- applevel get/setitem ----------------------- @@ -377,7 +412,7 @@ def create_dot_iter(self, shape, skip): r = calculate_dot_strides(self.get_strides(), self.get_backstrides(), shape, skip) - return MultiDimViewIterator(self, self.start, r[0], r[1], shape) + return MultiDimViewIterator(self, self.dtype, self.start, r[0], r[1], shape) def swapaxes(self, axis1, axis2): shape = self.get_shape()[:] @@ -411,7 +446,7 @@ r = calculate_broadcast_strides(self.get_strides(), self.get_backstrides(), self.get_shape(), shape) - return MultiDimViewIterator(self, 0, r[0], r[1], shape) + return MultiDimViewIterator(self, self.dtype, 0, r[0], r[1], shape) def fill(self, box): self.dtype.fill(self.storage, box, 0, self.size) @@ -424,6 +459,12 @@ self.order) return SliceArray(0, strides, backstrides, new_shape, self) +class NonWritableArray(ConcreteArray): + def descr_setitem(self, space, w_index, w_value): + raise OperationError(space.w_RuntimeError, space.wrap( + "array is not writable")) + + class SliceArray(BaseConcreteArray): def __init__(self, start, strides, backstrides, shape, parent, dtype=None): self.strides = strides @@ -448,11 +489,12 @@ r = calculate_broadcast_strides(self.get_strides(), self.get_backstrides(), self.get_shape(), shape) - return MultiDimViewIterator(self.parent, + return MultiDimViewIterator(self.parent, self.dtype, self.start, r[0], r[1], shape) if len(self.get_shape()) == 1: - return OneDimViewIterator(self) - return MultiDimViewIterator(self.parent, self.start, + return OneDimViewIterator(self.parent, self.dtype, self.start, + self.get_strides(), self.get_shape()) + return MultiDimViewIterator(self.parent, self.dtype, self.start, self.get_strides(), self.get_backstrides(), self.get_shape()) diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -136,7 +136,7 @@ return self.kind == SIGNEDLTR def is_complex_type(self): - return (self.num == 14 or self.num == 15 or self.num == 16) + return False def is_bool_type(self): return self.kind == BOOLLTR @@ -155,6 +155,19 @@ def get_size(self): return self.itemtype.get_element_size() +class W_ComplexDtype(W_Dtype): + + def __init__(self, itemtype, num, kind, name, char, w_box_type, + alternate_constructors=[], aliases=[], + fields=None, fieldnames=None, native=True, float_type=None): + W_Dtype.__init__(self, itemtype, num, kind, name, char, w_box_type, + alternate_constructors=alternate_constructors, aliases=aliases, + fields=fields, fieldnames=fieldnames, native=native) + self.float_type = float_type + + def is_complex_type(self): + return True + def dtype_from_list(space, w_lst): lst_w = space.listview(w_lst) fields = {} @@ -413,15 +426,16 @@ alternate_constructors=[space.w_float], aliases=["float"], ) - self.w_complex64dtype = W_Dtype( + self.w_complex64dtype = W_ComplexDtype( types.Complex64(), num=14, kind=COMPLEXLTR, name="complex64", char="F", w_box_type = space.gettypefor(interp_boxes.W_Complex64Box), + float_type = self.w_float32dtype, ) - self.w_complex128dtype = W_Dtype( + self.w_complex128dtype = W_ComplexDtype( types.Complex128(), num=15, kind=COMPLEXLTR, @@ -430,6 +444,7 @@ w_box_type = space.gettypefor(interp_boxes.W_Complex128Box), alternate_constructors=[space.w_complex], aliases=["complex"], + float_type = self.w_float64dtype, ) if interp_boxes.long_double_size == 12: self.w_float96dtype = W_Dtype( @@ -443,7 +458,7 @@ ) self.w_longdouble = self.w_float96dtype - self.w_complex192dtype = W_Dtype( + self.w_complex192dtype = W_ComplexDtype( types.Complex192(), num=16, kind=COMPLEXLTR, @@ -452,6 +467,7 @@ w_box_type = space.gettypefor(interp_boxes.W_Complex192Box), alternate_constructors=[space.w_complex], aliases=["clongdouble", "clongfloat"], + float_type = self.w_float96dtype, ) self.w_clongdouble = self.w_complex192dtype @@ -467,7 +483,7 @@ ) self.w_longdouble = self.w_float128dtype - self.w_complex256dtype = W_Dtype( + self.w_complex256dtype = W_ComplexDtype( types.Complex256(), num=16, kind=COMPLEXLTR, @@ -476,6 +492,7 @@ w_box_type = space.gettypefor(interp_boxes.W_Complex256Box), alternate_constructors=[space.w_complex], aliases=["clongdouble", "clongfloat"], + float_type = self.w_float128dtype, ) self.w_clongdouble = self.w_complex256dtype else: diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -235,6 +235,29 @@ def descr_copy(self, space): return W_NDimArray(self.implementation.copy()) + def descr_get_real(self, space): + return W_NDimArray(self.implementation.get_real()) + + def descr_get_imag(self, space): + ret = self.implementation.get_imag() + if ret: + return W_NDimArray(ret) + raise OperationError(space.w_NotImplementedError, + space.wrap('imag not implemented for this dtype')) + + def descr_set_real(self, space, w_value): + # copy (broadcast) values into self + tmp = self.implementation.get_real() + tmp.setslice(space, convert_to_array(space, w_value)) + + def descr_set_imag(self, space, w_value): + # if possible, copy (broadcast) values into self + if not self.get_dtype().is_complex_type(): + raise OperationError(space.w_TypeError, + space.wrap('array does not have imaginary part to set')) + tmp = self.implementation.get_imag() + tmp.setslice(space, convert_to_array(space, w_value)) + def descr_reshape(self, space, args_w): """reshape(...) a.reshape(shape) @@ -387,8 +410,6 @@ descr_neg = _unaryop_impl("negative") descr_abs = _unaryop_impl("absolute") descr_invert = _unaryop_impl("invert") - descr_get_real = _unaryop_impl("real") - descr_get_imag = _unaryop_impl("imag") def descr_nonzero(self, space): if self.get_size() > 1: @@ -635,9 +656,10 @@ swapaxes = interp2app(W_NDimArray.descr_swapaxes), flat = GetSetProperty(W_NDimArray.descr_get_flatiter), item = interp2app(W_NDimArray.descr_item), - real = GetSetProperty(W_NDimArray.descr_get_real), - imag = GetSetProperty(W_NDimArray.descr_get_imag), - + real = GetSetProperty(W_NDimArray.descr_get_real, + W_NDimArray.descr_set_real), + imag = GetSetProperty(W_NDimArray.descr_get_imag, + W_NDimArray.descr_set_imag), __array_interface__ = GetSetProperty(W_NDimArray.descr_array_iface), ) diff --git a/pypy/module/micronumpy/test/test_iter.py b/pypy/module/micronumpy/test/test_iter.py --- a/pypy/module/micronumpy/test/test_iter.py +++ b/pypy/module/micronumpy/test/test_iter.py @@ -12,7 +12,7 @@ strides = [5, 1] backstrides = [x * (y - 1) for x,y in zip(strides, shape)] assert backstrides == [10, 4] - i = MultiDimViewIterator(MockArray, start, strides, backstrides, shape) + i = MultiDimViewIterator(MockArray, None, start, strides, backstrides, shape) i.next() i.next() i.next() @@ -30,7 +30,7 @@ strides = [1, 3] backstrides = [x * (y - 1) for x,y in zip(strides, shape)] assert backstrides == [2, 12] - i = MultiDimViewIterator(MockArray, start, strides, backstrides, shape) + i = MultiDimViewIterator(MockArray, None, start, strides, backstrides, shape) i.next() i.next() i.next() @@ -51,7 +51,7 @@ strides = [5, 1] backstrides = [x * (y - 1) for x,y in zip(strides, shape)] assert backstrides == [10, 4] - i = MultiDimViewIterator(MockArray, start, strides, backstrides, shape) + i = MultiDimViewIterator(MockArray, None, start, strides, backstrides, shape) i.next_skip_x(2) i.next_skip_x(2) i.next_skip_x(2) @@ -74,7 +74,7 @@ strides = [1, 3] backstrides = [x * (y - 1) for x,y in zip(strides, shape)] assert backstrides == [2, 12] - i = MultiDimViewIterator(MockArray, start, strides, backstrides, shape) + i = MultiDimViewIterator(MockArray, None, start, strides, backstrides, shape) i.next_skip_x(2) i.next_skip_x(2) i.next_skip_x(2) diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1169,8 +1169,8 @@ assert (d == [[1, 0, 0], [0, 1, 0], [0, 0, 1]]).all() def test_eye(self): - from _numpypy import eye, array - from _numpypy import int32, float64, dtype + from _numpypy import eye + from _numpypy import int32, dtype a = eye(0) assert len(a) == 0 assert a.dtype == dtype('float64') @@ -1345,6 +1345,42 @@ assert b[0] == 3 assert b[1] == 2 + def test_realimag_views(self): + from _numpypy import arange, array + a = arange(15) + b = a.real + b[5]=50 + assert a[5] == 50 + b = a.imag + assert b[7] == 0 + raises(RuntimeError, 'b[7] = -2') + raises(TypeError, 'a.imag = -2') + a = array(['abc','def'],dtype='S3') + b = a.real + assert a[0] == b[0] + assert a[1] == b[1] + b[1] = 'xyz' + assert a[1] == 'xyz' + assert a.imag[0] == 'abc' + raises(TypeError, 'a.imag = "qop"') + a=array([[1+1j, 2-3j, 4+5j],[-6+7j, 8-9j, -2-1j]]) + assert a.real[0,1] == 2 + a.real[0,1] = -20 + assert a[0,1].real == -20 + b = a.imag + assert b[1,2] == -1 + b[1,2] = 30 + assert a[1,2].imag == 30 + a.real = 13 + assert a[1,1].real == 13 + a=array([1+1j, 2-3j, 4+5j, -6+7j, 8-9j, -2-1j]) + a.real = 13 + assert a[3].real == 13 + a.imag = -5 + a.imag[3] = -10 + assert a[3].imag == -10 + assert a[2].imag == -5 + def test_tolist_scalar(self): from _numpypy import int32, bool_ x = int32(23) @@ -2273,7 +2309,7 @@ assert a[0]['x'] == 'a' def test_stringarray(self): - from _numpypy import array, flexible + from _numpypy import array a = array(['abc'],'S3') assert str(a.dtype) == '|S3' a = array(['abc']) @@ -2293,7 +2329,6 @@ def test_flexible_repr(self): # import overrides str(), repr() for array - from numpypy.core import arrayprint from _numpypy import array a = array(['abc'],'S3') s = repr(a) diff --git a/pypy/objspace/flow/test/test_objspace.py b/pypy/objspace/flow/test/test_objspace.py --- a/pypy/objspace/flow/test/test_objspace.py +++ b/pypy/objspace/flow/test/test_objspace.py @@ -8,18 +8,17 @@ from pypy.objspace.flow.objspace import FlowObjSpace from pypy.objspace.flow.flowcontext import FlowingError, FlowSpaceFrame from pypy import conftest -from pypy.tool.stdlib_opcode import bytecode_spec, host_bytecode_spec +from pypy.tool.stdlib_opcode import host_bytecode_spec import os import operator is_operator = getattr(operator, 'is_', operator.eq) # it's not there 2.2 @contextmanager -def patching_opcodes(*opcodes): +def patching_opcodes(**opcodes): meth_names = host_bytecode_spec.method_names - opnums = [bytecode_spec.opmap[name] for name in opcodes] old_name = {} - for name, num in zip(opcodes, opnums): + for name, num in opcodes.items(): old_name[num] = meth_names[num] meth_names[num] = name yield @@ -898,7 +897,7 @@ """ Tests code generated by pypy-c compiled with CALL_METHOD bytecode """ - with patching_opcodes('CALL_METHOD', 'LOOKUP_METHOD'): + with patching_opcodes(CALL_METHOD=202, LOOKUP_METHOD=201): class X: def m(self): return 3 @@ -922,7 +921,7 @@ """ Tests code generated by pypy-c compiled with BUILD_LIST_FROM_ARG bytecode """ - with patching_opcodes('BUILD_LIST_FROM_ARG'): + with patching_opcodes(BUILD_LIST_FROM_ARG=203): def f(): return [i for i in "abc"] diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py --- a/pypy/rpython/lltypesystem/rffi.py +++ b/pypy/rpython/lltypesystem/rffi.py @@ -2,12 +2,12 @@ from pypy.annotation import model as annmodel from pypy.rpython.lltypesystem import lltype, rstr from pypy.rpython.lltypesystem import ll2ctypes -from pypy.rpython.lltypesystem.llmemory import cast_adr_to_ptr, cast_ptr_to_adr +from pypy.rpython.lltypesystem.llmemory import cast_ptr_to_adr from pypy.rpython.lltypesystem.llmemory import itemoffsetof, raw_memcopy from pypy.annotation.model import lltype_to_annotation from pypy.tool.sourcetools import func_with_new_name -from pypy.rlib.objectmodel import Symbolic, CDefinedIntSymbolic -from pypy.rlib.objectmodel import keepalive_until_here +from pypy.rlib.objectmodel import Symbolic +from pypy.rlib.objectmodel import keepalive_until_here, enforceargs from pypy.rlib import rarithmetic, rgc from pypy.rpython.extregistry import ExtRegistryEntry from pypy.rlib.unroll import unrolling_iterable @@ -795,6 +795,7 @@ # (char*, str, int, int) -> None @jit.dont_look_inside + @enforceargs(None, None, int, int) def str_from_buffer(raw_buf, gc_buf, allocated_size, needed_size): """ Converts from a pair returned by alloc_buffer to a high-level string. @@ -833,6 +834,7 @@ lltype.free(raw_buf, flavor='raw') # char* -> str, with an upper bound on the length in case there is no \x00 + @enforceargs(None, int) def charp2strn(cp, maxlen): b = builder_class(maxlen) i = 0 diff --git a/pypy/translator/driver.py b/pypy/translator/driver.py --- a/pypy/translator/driver.py +++ b/pypy/translator/driver.py @@ -362,7 +362,7 @@ @taskdef([RTYPE], "JIT compiler generation") def task_pyjitpl_lltype(self): """ Generate bytecodes for JIT and flow the JIT helper functions - ootype version + lltype version """ get_policy = self.extra['jitpolicy'] self.jitpolicy = get_policy(self) From noreply at buildbot.pypy.org Sat Jan 12 16:05:52 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 12 Jan 2013 16:05:52 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: one fix Message-ID: <20130112150552.8B0031C0343@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r59990:e81153e4154a Date: 2013-01-12 17:05 +0200 http://bitbucket.org/pypy/pypy/changeset/e81153e4154a/ Log: one fix diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -1830,7 +1830,7 @@ if loc is None: positions[i] = -1 elif isinstance(loc, StackLoc): - xxx + positions[i] = (loc.value + JITFRAME_FIXED_SIZE) * WORD else: assert isinstance(loc, RegLoc) v = (gpr_reg_mgr_cls.all_reg_indexes[loc.value] + @@ -1843,12 +1843,14 @@ def rebuild_faillocs_from_descr(self, descr): locs = [] + GPR_REGS = len(gpr_reg_mgr_cls.all_regs) + XMM_REGS = len(xmm_reg_mgr_cls.all_regs) for pos in descr.rd_locs: if pos == -1: pass - elif pos < self.cpu.NUM_REGS * WORD: + elif pos < GPR_REGS * WORD: locs.append(gpr_reg_mgr_cls.all_regs[pos // WORD]) - elif pos < self.cpu.NUM_REGS * 2 * WORD: + elif pos < (GPR_REGS + XMM_REGS) * WORD: locs.append(xmm_reg_mgr_cls.xrm.all_regs[pos // WORD]) else: i = pos // WORD - 2 * self.cpu.NUM_REGS From noreply at buildbot.pypy.org Sat Jan 12 18:42:37 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sat, 12 Jan 2013 18:42:37 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Moved pypy.config parts to rpython Message-ID: <20130112174237.26C221C015D@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59991:a879d5890037 Date: 2013-01-12 18:42 +0100 http://bitbucket.org/pypy/pypy/changeset/a879d5890037/ Log: Moved pypy.config parts to rpython diff --git a/pypy/bin/pyinteractive.py b/pypy/bin/pyinteractive.py --- a/pypy/bin/pyinteractive.py +++ b/pypy/bin/pyinteractive.py @@ -12,9 +12,9 @@ import pypy from pypy.tool import option from optparse import make_option -from pypy.interpreter import main, interactive, error, gateway -from pypy.config.config import OptionDescription, BoolOption, StrOption -from pypy.config.config import Config, to_optparse +from rpython.interpreter import main, interactive, error, gateway +from rpython.config.config import OptionDescription, BoolOption, StrOption +from rpython.config.config import Config, to_optparse from pypy.config import pypyoption diff --git a/pypy/config/makerestdoc.py b/pypy/config/makerestdoc.py --- a/pypy/config/makerestdoc.py +++ b/pypy/config/makerestdoc.py @@ -2,10 +2,10 @@ from pypy.tool.rest.rst import Rest, Paragraph, Strong, ListItem, Title, Link from pypy.tool.rest.rst import Directive, Em, Quote, Text -from pypy.config.config import ChoiceOption, BoolOption, StrOption, IntOption -from pypy.config.config import FloatOption, OptionDescription, Option, Config -from pypy.config.config import ArbitraryOption, DEFAULT_OPTION_NAME -from pypy.config.config import _getnegation +from rpython.config.config import ChoiceOption, BoolOption, StrOption, IntOption +from rpython.config.config import FloatOption, OptionDescription, Option, Config +from rpython.config.config import ArbitraryOption, DEFAULT_OPTION_NAME +from rpython.config.config import _getnegation configdocdir = py.path.local(__file__).dirpath().dirpath().join("doc", "config") diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -2,9 +2,9 @@ import py -from pypy.config.config import (OptionDescription, BoolOption, IntOption, +from rpython.config.config import (OptionDescription, BoolOption, IntOption, ChoiceOption, StrOption, to_optparse, ConflictConfigError) -from pypy.config.translationoption import IS_64_BITS +from rpython.config.translationoption import IS_64_BITS modulepath = py.path.local(__file__).dirpath().dirpath().join("module") diff --git a/pypy/config/test/test_makerestdoc.py b/pypy/config/test/test_makerestdoc.py --- a/pypy/config/test/test_makerestdoc.py +++ b/pypy/config/test/test_makerestdoc.py @@ -1,4 +1,4 @@ -from pypy.config.config import * +from rpython.config.config import * from pypy.config.makerestdoc import make_cmdline_overview from pypy.tool.rest.rest import process as restcheck diff --git a/pypy/config/test/test_pypyoption.py b/pypy/config/test/test_pypyoption.py --- a/pypy/config/test/test_pypyoption.py +++ b/pypy/config/test/test_pypyoption.py @@ -1,7 +1,7 @@ import py from pypy.config.pypyoption import get_pypy_config, set_pypy_opt_level -from pypy.config.config import Config, ConfigError -from pypy.config.translationoption import set_opt_level +from rpython.config.config import Config, ConfigError +from rpython.config.translationoption import set_opt_level thisdir = py.path.local(__file__).dirpath() diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -36,7 +36,7 @@ option = config.option def _set_platform(opt, opt_str, value, parser): - from pypy.config.translationoption import PLATFORMS + from rpython.config.translationoption import PLATFORMS from rpython.translator.platform import set_platform if value not in PLATFORMS: raise ValueError("%s not in %s" % (value, PLATFORMS)) diff --git a/pypy/doc/config/confrest.py b/pypy/doc/config/confrest.py --- a/pypy/doc/config/confrest.py +++ b/pypy/doc/config/confrest.py @@ -1,7 +1,8 @@ from pypy.doc.confrest import * from pypy.config.makerestdoc import make_cmdline_overview -from pypy.config.config import Config -from pypy.config import pypyoption, translationoption +from pypy.config import pypyoption +from rpython.config.config import Config +from rpython.config import translationoption all_optiondescrs = [pypyoption.pypy_optiondescription, diff --git a/pypy/doc/config/generate.py b/pypy/doc/config/generate.py --- a/pypy/doc/config/generate.py +++ b/pypy/doc/config/generate.py @@ -1,6 +1,7 @@ import py -from pypy.config import pypyoption, translationoption, config, makerestdoc +from pypy.config import pypyoption, makerestdoc from pypy.doc.config.confrest import all_optiondescrs +from rpython.config import translationoption, config all_optiondescrs = [pypyoption.pypy_optiondescription, translationoption.translation_optiondescription, diff --git a/pypy/doc/config/makemodules.py b/pypy/doc/config/makemodules.py --- a/pypy/doc/config/makemodules.py +++ b/pypy/doc/config/makemodules.py @@ -1,5 +1,6 @@ import py -from pypy.config import pypyoption, translationoption, config +from pypy.config import pypyoption +from rpython.config import translationoption, config thisdir = py.path.local(__file__).dirpath() diff --git a/pypy/doc/configuration.rst b/pypy/doc/configuration.rst --- a/pypy/doc/configuration.rst +++ b/pypy/doc/configuration.rst @@ -133,7 +133,7 @@ ``OptionDescription`` instance. The attributes of the ``Config`` objects are the names of the children of the ``OptionDescription``. Example:: - >>> from pypy.config.config import OptionDescription, Config, BoolOption + >>> from rpython.config.config import OptionDescription, Config, BoolOption >>> descr = OptionDescription("options", "", [ ... BoolOption("bool", "", default=False)]) >>> diff --git a/pypy/tool/option.py b/pypy/tool/option.py --- a/pypy/tool/option.py +++ b/pypy/tool/option.py @@ -2,7 +2,7 @@ import os from pypy.config.pypyoption import get_pypy_config -from pypy.config.config import Config, OptionDescription, to_optparse +from rpython.config.config import Config, OptionDescription, to_optparse import optparse extra_useage = """For detailed descriptions of all the options see diff --git a/pypy/tool/pypyjit.py b/pypy/tool/pypyjit.py --- a/pypy/tool/pypyjit.py +++ b/pypy/tool/pypyjit.py @@ -8,7 +8,7 @@ import py, os from pypy.objspace.std import Space -from pypy.config.translationoption import set_opt_level +from rpython.config.translationoption import set_opt_level from pypy.config.pypyoption import get_pypy_config, set_pypy_opt_level from pypy.objspace.std import multimethod from rpython.rtyper.annlowlevel import llhelper, llstr, oostr, hlstr diff --git a/pypy/tool/pytest/objspace.py b/pypy/tool/pytest/objspace.py --- a/pypy/tool/pytest/objspace.py +++ b/pypy/tool/pytest/objspace.py @@ -1,6 +1,6 @@ import py import sys -from pypy.config.config import ConflictConfigError +from rpython.config.config import ConflictConfigError from pypy.tool.option import make_config, make_objspace from pypy.tool.pytest import appsupport from pypy.conftest import option diff --git a/pypy/config/__init__.py b/rpython/config/__init__.py copy from pypy/config/__init__.py copy to rpython/config/__init__.py diff --git a/pypy/config/config.py b/rpython/config/config.py rename from pypy/config/config.py rename to rpython/config/config.py diff --git a/pypy/config/parse.py b/rpython/config/parse.py rename from pypy/config/parse.py rename to rpython/config/parse.py diff --git a/pypy/config/support.py b/rpython/config/support.py rename from pypy/config/support.py rename to rpython/config/support.py diff --git a/pypy/config/test/test_config.py b/rpython/config/test/test_config.py rename from pypy/config/test/test_config.py rename to rpython/config/test/test_config.py --- a/pypy/config/test/test_config.py +++ b/rpython/config/test/test_config.py @@ -1,4 +1,4 @@ -from pypy.config.config import * +from rpython.config.config import * import py, sys def make_description(): diff --git a/pypy/config/test/test_parse.py b/rpython/config/test/test_parse.py rename from pypy/config/test/test_parse.py rename to rpython/config/test/test_parse.py --- a/pypy/config/test/test_parse.py +++ b/rpython/config/test/test_parse.py @@ -1,4 +1,4 @@ -from pypy.config.parse import parse_info +from rpython.config.parse import parse_info def test_parse_new_format(): diff --git a/pypy/config/test/test_support.py b/rpython/config/test/test_support.py rename from pypy/config/test/test_support.py rename to rpython/config/test/test_support.py --- a/pypy/config/test/test_support.py +++ b/rpython/config/test/test_support.py @@ -1,6 +1,6 @@ from cStringIO import StringIO -from pypy.config import support +from rpython.config import support import os, sys, py cpuinfo = """ diff --git a/pypy/config/test/test_translationoption.py b/rpython/config/test/test_translationoption.py rename from pypy/config/test/test_translationoption.py rename to rpython/config/test/test_translationoption.py --- a/pypy/config/test/test_translationoption.py +++ b/rpython/config/test/test_translationoption.py @@ -1,7 +1,7 @@ import py -from pypy.config.translationoption import get_combined_translation_config -from pypy.config.translationoption import set_opt_level -from pypy.config.config import ConflictConfigError +from rpython.config.translationoption import get_combined_translation_config +from rpython.config.translationoption import set_opt_level +from rpython.config.config import ConflictConfigError def test_no_gcrootfinder_with_boehm(): diff --git a/pypy/config/translationoption.py b/rpython/config/translationoption.py rename from pypy/config/translationoption.py rename to rpython/config/translationoption.py --- a/pypy/config/translationoption.py +++ b/rpython/config/translationoption.py @@ -1,8 +1,8 @@ import py, os, sys -from pypy.config.config import OptionDescription, BoolOption, IntOption, ArbitraryOption, FloatOption -from pypy.config.config import ChoiceOption, StrOption, to_optparse, Config -from pypy.config.config import ConfigError -from pypy.config.support import detect_number_of_processors +from rpython.config.config import OptionDescription, BoolOption, IntOption, ArbitraryOption, FloatOption +from rpython.config.config import ChoiceOption, StrOption, to_optparse, Config +from rpython.config.config import ConfigError +from rpython.config.support import detect_number_of_processors DEFL_INLINE_THRESHOLD = 32.4 # just enough to inline add__Int_Int() # and just small enough to prevend inlining of some rlist functions. diff --git a/rpython/jit/backend/arm/test/test_zrpy_gc.py b/rpython/jit/backend/arm/test/test_zrpy_gc.py --- a/rpython/jit/backend/arm/test/test_zrpy_gc.py +++ b/rpython/jit/backend/arm/test/test_zrpy_gc.py @@ -13,7 +13,7 @@ from rpython.rlib.jit import elidable, unroll_safe from rpython.jit.backend.llsupport.gc import GcLLDescr_framework from rpython.tool.udir import udir -from pypy.config.translationoption import DEFL_GC +from rpython.config.translationoption import DEFL_GC from rpython.jit.backend.arm.test.support import skip_unless_run_slow_tests skip_unless_run_slow_tests() diff --git a/rpython/jit/backend/arm/test/test_ztranslation.py b/rpython/jit/backend/arm/test/test_ztranslation.py --- a/rpython/jit/backend/arm/test/test_ztranslation.py +++ b/rpython/jit/backend/arm/test/test_ztranslation.py @@ -9,7 +9,7 @@ from rpython.jit.backend.test.support import CCompiledMixin from rpython.jit.codewriter.policy import StopAtXPolicy from rpython.translator.translator import TranslationContext -from pypy.config.translationoption import DEFL_GC +from rpython.config.translationoption import DEFL_GC from rpython.rlib import rgc from rpython.jit.backend.arm.test.support import skip_unless_run_slow_tests skip_unless_run_slow_tests() diff --git a/rpython/jit/backend/x86/test/test_zrpy_gc.py b/rpython/jit/backend/x86/test/test_zrpy_gc.py --- a/rpython/jit/backend/x86/test/test_zrpy_gc.py +++ b/rpython/jit/backend/x86/test/test_zrpy_gc.py @@ -13,7 +13,7 @@ from rpython.rlib.jit import elidable, unroll_safe from rpython.jit.backend.llsupport.gc import GcLLDescr_framework from rpython.tool.udir import udir -from pypy.config.translationoption import DEFL_GC +from rpython.config.translationoption import DEFL_GC class X(object): def __init__(self, x=0): diff --git a/rpython/jit/backend/x86/test/test_ztranslation.py b/rpython/jit/backend/x86/test/test_ztranslation.py --- a/rpython/jit/backend/x86/test/test_ztranslation.py +++ b/rpython/jit/backend/x86/test/test_ztranslation.py @@ -10,7 +10,7 @@ from rpython.jit.codewriter.policy import StopAtXPolicy from rpython.translator.translator import TranslationContext from rpython.jit.backend.x86.arch import IS_X86_32, IS_X86_64 -from pypy.config.translationoption import DEFL_GC +from rpython.config.translationoption import DEFL_GC from rpython.rlib import rgc class TestTranslationX86(CCompiledMixin): diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py --- a/rpython/jit/metainterp/warmspot.py +++ b/rpython/jit/metainterp/warmspot.py @@ -70,7 +70,7 @@ inline=False, loop_longevity=0, retrace_limit=5, function_threshold=4, enable_opts=ALL_OPTS_NAMES, max_retrace_guards=15, **kwds): - from pypy.config.config import ConfigError + from rpython.config.config import ConfigError translator = interp.typer.annotator.translator try: translator.config.translation.gc = "boehm" diff --git a/rpython/translator/cli/gencli.py b/rpython/translator/cli/gencli.py --- a/rpython/translator/cli/gencli.py +++ b/rpython/translator/cli/gencli.py @@ -3,7 +3,7 @@ import py import subprocess -from pypy.config.config import Config +from rpython.config.config import Config from rpython.translator.oosupport.genoo import GenOO from rpython.translator.cli import conftest from rpython.translator.cli.ilgenerator import IlasmGenerator diff --git a/rpython/translator/goal/translate.py b/rpython/translator/goal/translate.py --- a/rpython/translator/goal/translate.py +++ b/rpython/translator/goal/translate.py @@ -16,9 +16,9 @@ except Exception: pass -from pypy.config.config import (to_optparse, OptionDescription, BoolOption, +from rpython.config.config import (to_optparse, OptionDescription, BoolOption, ArbitraryOption, StrOption, IntOption, Config, ChoiceOption, OptHelpFormatter) -from pypy.config.translationoption import (get_combined_translation_config, +from rpython.config.translationoption import (get_combined_translation_config, set_opt_level, final_check_config, OPT_LEVELS, DEFAULT_OPT_LEVEL, set_platform) diff --git a/rpython/translator/translator.py b/rpython/translator/translator.py --- a/rpython/translator/translator.py +++ b/rpython/translator/translator.py @@ -12,8 +12,8 @@ from rpython.tool.ansi_print import ansi_log from rpython.tool.sourcetools import nice_repr_for_func from pypy.config.pypyoption import pypy_optiondescription -from pypy.config.translationoption import get_combined_translation_config -from pypy.config.translationoption import get_platform +from rpython.config.translationoption import get_combined_translation_config +from rpython.config.translationoption import get_platform import py log = py.log.Producer("flowgraph") py.log.setconsumer("flowgraph", ansi_log) From noreply at buildbot.pypy.org Sat Jan 12 19:28:08 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sat, 12 Jan 2013 19:28:08 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed missing import rename Message-ID: <20130112182808.E51A41C0343@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59992:61de7af26ed1 Date: 2013-01-12 19:27 +0100 http://bitbucket.org/pypy/pypy/changeset/61de7af26ed1/ Log: Fixed missing import rename diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -304,7 +304,7 @@ ]) def get_pypy_config(overrides=None, translating=False): - from pypy.config.translationoption import get_combined_translation_config + from rpython.config.translationoption import get_combined_translation_config return get_combined_translation_config( pypy_optiondescription, overrides=overrides, translating=translating) From noreply at buildbot.pypy.org Sat Jan 12 20:11:03 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sat, 12 Jan 2013 20:11:03 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed import Message-ID: <20130112191103.3712D1C015D@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59993:36b137f03142 Date: 2013-01-12 19:43 +0100 http://bitbucket.org/pypy/pypy/changeset/36b137f03142/ Log: Fixed import diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py --- a/pypy/interpreter/test/test_argument.py +++ b/pypy/interpreter/test/test_argument.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- import py from pypy.interpreter.argument import (Arguments, ArgErr, ArgErrUnknownKwds, - ArgErrMultipleValues, ArgErrCount, Signature) + ArgErrMultipleValues, ArgErrCount) +from rpython.rtyper.signature import Signature from pypy.interpreter.error import OperationError From noreply at buildbot.pypy.org Sat Jan 12 20:11:04 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sat, 12 Jan 2013 20:11:04 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Removed unused import Message-ID: <20130112191104.A18371C015D@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59994:67dcbb675175 Date: 2013-01-12 19:59 +0100 http://bitbucket.org/pypy/pypy/changeset/67dcbb675175/ Log: Removed unused import diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -14,7 +14,6 @@ from rpython.jit.tool.oparser import pure_parse from rpython.jit.metainterp.optimizeopt.util import args_dict from rpython.jit.metainterp.optimizeopt.test.test_optimizebasic import FakeMetaInterpStaticData -from pypy.config.pypyoption import get_pypy_config from rpython.jit.metainterp.optimizeopt.unroll import Inliner def test_build_opt_chain(): From noreply at buildbot.pypy.org Sat Jan 12 20:11:05 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sat, 12 Jan 2013 20:11:05 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: get_pypy_config -> get_combined_translation_config Message-ID: <20130112191105.EBE741C015D@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r59995:11426dcd2a68 Date: 2013-01-12 20:10 +0100 http://bitbucket.org/pypy/pypy/changeset/11426dcd2a68/ Log: get_pypy_config -> get_combined_translation_config diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py --- a/rpython/jit/metainterp/optimizeopt/test/test_util.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py @@ -18,7 +18,7 @@ from rpython.jit.metainterp.quasiimmut import QuasiImmutDescr from rpython.jit.metainterp import compile, resume, history from rpython.jit.metainterp.jitprof import EmptyProfiler -from pypy.config.pypyoption import get_pypy_config +from rpython.config.translationoption import get_combined_translation_config from rpython.jit.metainterp.resoperation import rop, opname, ResOperation from rpython.jit.metainterp.optimizeopt.unroll import Inliner @@ -345,7 +345,7 @@ self.profiler = EmptyProfiler() self.options = Fake() self.globaldata = Fake() - self.config = get_pypy_config(translating=True) + self.config = get_combined_translation_config(translating=True) class logger_noopt: @classmethod diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -1407,8 +1407,8 @@ if warmrunnerdesc: self.config = warmrunnerdesc.translator.config else: - from pypy.config.pypyoption import get_pypy_config - self.config = get_pypy_config(translating=True) + from rpython.config.translationoption import get_combined_translation_config + self.config = get_combined_translation_config(translating=True) backendmodule = self.cpu.__module__ backendmodule = backendmodule.split('.')[-2] diff --git a/rpython/jit/metainterp/test/test_compile.py b/rpython/jit/metainterp/test/test_compile.py --- a/rpython/jit/metainterp/test/test_compile.py +++ b/rpython/jit/metainterp/test/test_compile.py @@ -1,4 +1,4 @@ -from pypy.config.pypyoption import get_pypy_config +from rpython.config.translationoption import get_combined_translation_config from rpython.jit.metainterp.history import TargetToken, ConstInt, History, Stats from rpython.jit.metainterp.history import BoxInt, INT from rpython.jit.metainterp.compile import compile_loop @@ -41,7 +41,7 @@ logger_noopt = FakeLogger() logger_ops = FakeLogger() - config = get_pypy_config(translating=True) + config = get_combined_translation_config(translating=True) stats = Stats() profiler = jitprof.EmptyProfiler() diff --git a/rpython/rlib/test/test_rstacklet.py b/rpython/rlib/test/test_rstacklet.py --- a/rpython/rlib/test/test_rstacklet.py +++ b/rpython/rlib/test/test_rstacklet.py @@ -226,8 +226,8 @@ class BaseTestStacklet(StandaloneTests): def setup_class(cls): - from pypy.config.pypyoption import get_pypy_config - config = get_pypy_config(translating=True) + from rpython.config.translationoption import get_combined_translation_config + config = get_combined_translation_config(translating=True) config.translation.gc = cls.gc if cls.gcrootfinder is not None: config.translation.continuation = True diff --git a/rpython/rtyper/memory/gc/test/test_direct.py b/rpython/rtyper/memory/gc/test/test_direct.py --- a/rpython/rtyper/memory/gc/test/test_direct.py +++ b/rpython/rtyper/memory/gc/test/test_direct.py @@ -64,8 +64,8 @@ GC_PARAMS = {} def setup_method(self, meth): - from pypy.config.pypyoption import get_pypy_config - config = get_pypy_config(translating=True).translation + from rpython.config.translationoption import get_combined_translation_config + config = get_combined_translation_config(translating=True).translation self.stackroots = [] GC_PARAMS = self.GC_PARAMS.copy() if hasattr(meth, 'GC_PARAMS'): diff --git a/rpython/rtyper/test/test_rpbc.py b/rpython/rtyper/test/test_rpbc.py --- a/rpython/rtyper/test/test_rpbc.py +++ b/rpython/rtyper/test/test_rpbc.py @@ -1963,8 +1963,8 @@ class TestLLtypeSmallFuncSets(TestLLtype): def setup_class(cls): - from pypy.config.pypyoption import get_pypy_config - cls.config = get_pypy_config(translating=True) + from rpython.config.translationoption import get_combined_translation_config + cls.config = get_combined_translation_config(translating=True) cls.config.translation.withsmallfuncsets = 3 def interpret(self, fn, args, **kwds): @@ -1973,9 +1973,9 @@ def test_smallfuncsets_basic(): from rpython.translator.translator import TranslationContext, graphof - from pypy.config.pypyoption import get_pypy_config + from rpython.config.translationoption import get_combined_translation_config from rpython.rtyper.llinterp import LLInterpreter - config = get_pypy_config(translating=True) + config = get_combined_translation_config(translating=True) config.translation.withsmallfuncsets = 10 def g(x): diff --git a/rpython/translator/c/gcc/test/test_asmgcroot.py b/rpython/translator/c/gcc/test/test_asmgcroot.py --- a/rpython/translator/c/gcc/test/test_asmgcroot.py +++ b/rpython/translator/c/gcc/test/test_asmgcroot.py @@ -27,8 +27,8 @@ def make_config(cls): if _MSVC and _WIN64: py.test.skip("all asmgcroot tests disabled for MSVC X64") - from pypy.config.pypyoption import get_pypy_config - config = get_pypy_config(translating=True) + from rpython.config.translationoption import get_combined_translation_config + config = get_combined_translation_config(translating=True) config.translation.gc = cls.gcpolicy config.translation.gcrootfinder = "asmgcc" config.translation.taggedpointers = getattr(cls, "taggedpointers", False) diff --git a/rpython/translator/c/gcc/test/test_thread.py b/rpython/translator/c/gcc/test/test_thread.py --- a/rpython/translator/c/gcc/test/test_thread.py +++ b/rpython/translator/c/gcc/test/test_thread.py @@ -13,6 +13,6 @@ def setup_class(cls): if sys.platform == 'win32': - from pypy.config.pypyoption import get_pypy_config - cls.config = get_pypy_config(translating=True) + from rpython.config.translationoption import get_combined_translation_config + cls.config = get_combined_translation_config(translating=True) cls.config.translation.cc = 'mingw32' diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py --- a/rpython/translator/c/test/test_standalone.py +++ b/rpython/translator/c/test/test_standalone.py @@ -409,8 +409,8 @@ assert 'bok' in data # # finally, check compiling with logging disabled - from pypy.config.pypyoption import get_pypy_config - config = get_pypy_config(translating=True) + from rpython.config.translationoption import get_combined_translation_config + config = get_combined_translation_config(translating=True) config.translation.log = False self.config = config t, cbuilder = self.compile(entry_point) @@ -819,8 +819,8 @@ py.test.skip("TestMaemo: tests skipped for now") from rpython.translator.platform.maemo import check_scratchbox check_scratchbox() - from pypy.config.pypyoption import get_pypy_config - config = get_pypy_config(translating=True) + from rpython.config.translationoption import get_combined_translation_config + config = get_combined_translation_config(translating=True) config.translation.platform = 'maemo' cls.config = config diff --git a/rpython/translator/driver.py b/rpython/translator/driver.py --- a/rpython/translator/driver.py +++ b/rpython/translator/driver.py @@ -79,8 +79,8 @@ self.log = log if config is None: - from pypy.config.pypyoption import get_pypy_config - config = get_pypy_config(translating=True) + from rpython.config.translationoption import get_combined_translation_config + config = get_combined_translation_config(translating=True) self.config = config if overrides is not None: self.config.override(overrides) diff --git a/rpython/translator/oosupport/genoo.py b/rpython/translator/oosupport/genoo.py --- a/rpython/translator/oosupport/genoo.py +++ b/rpython/translator/oosupport/genoo.py @@ -34,8 +34,8 @@ self.entrypoint = entrypoint self.db = self.Database(self) if config is None: - from pypy.config.pypyoption import get_pypy_config - config = get_pypy_config(translating=True) + from rpython.config.translationoption import get_combined_translation_config + config = get_combined_translation_config(translating=True) self.config = config # XXX: move this option out of the 'cli' section diff --git a/rpython/translator/translator.py b/rpython/translator/translator.py --- a/rpython/translator/translator.py +++ b/rpython/translator/translator.py @@ -27,8 +27,8 @@ def __init__(self, config=None, **flowing_flags): if config is None: - from pypy.config.pypyoption import get_pypy_config - config = get_pypy_config(translating=True) + from rpython.config.translationoption import get_combined_translation_config + config = get_combined_translation_config(translating=True) # ZZZ should go away in the end for attr in ['verbose', 'simplifying', 'list_comprehension_operations']: From noreply at buildbot.pypy.org Sat Jan 12 20:37:43 2013 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 12 Jan 2013 20:37:43 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: fix asserts Message-ID: <20130112193743.BFE501C015D@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: missing-ndarray-attributes Changeset: r59996:dff76a3cb4f9 Date: 2013-01-12 20:06 +0200 http://bitbucket.org/pypy/pypy/changeset/dff76a3cb4f9/ Log: fix asserts diff --git a/pypy/module/micronumpy/interp_flatiter.py b/pypy/module/micronumpy/interp_flatiter.py --- a/pypy/module/micronumpy/interp_flatiter.py +++ b/pypy/module/micronumpy/interp_flatiter.py @@ -9,7 +9,7 @@ like a real array for descr_eq and friends """ def __init__(self, base): - assert isinstance(base, BaseArrayImplementation) + assert isinstance(base, W_NDimArray) self.base = base self.dtype = base.get_dtype() self.shape = [base.get_size()] @@ -18,7 +18,7 @@ return self.shape def create_iter(self, shape=None): - assert isinstance(self.base, BaseArrayImplementation) + assert isinstance(self.base, W_NDimArray) return self.base.create_iter() class W_FlatIterator(W_NDimArray): From noreply at buildbot.pypy.org Sat Jan 12 20:37:45 2013 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 12 Jan 2013 20:37:45 +0100 (CET) Subject: [pypy-commit] pypy default: windows does not like invalid signums Message-ID: <20130112193745.2823E1C015D@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r59997:b052932261c2 Date: 2013-01-12 21:36 +0200 http://bitbucket.org/pypy/pypy/changeset/b052932261c2/ Log: windows does not like invalid signums diff --git a/lib-python/2.7/heapq.py b/lib-python/2.7/heapq.py --- a/lib-python/2.7/heapq.py +++ b/lib-python/2.7/heapq.py @@ -129,7 +129,11 @@ __all__ = ['heappush', 'heappop', 'heapify', 'heapreplace', 'merge', 'nlargest', 'nsmallest', 'heappushpop'] -from itertools import islice, repeat, count, imap, izip, tee, chain +try: + from itertools import islice, repeat, count, imap, izip, tee, chain +except: + import sys + print sys.path from operator import itemgetter import bisect diff --git a/lib-python/2.7/test/test_signal.py b/lib-python/2.7/test/test_signal.py --- a/lib-python/2.7/test/test_signal.py +++ b/lib-python/2.7/test/test_signal.py @@ -224,7 +224,10 @@ signal.signal(-1, handler) with self.assertRaises(ValueError): - signal.signal(7, handler) + pass + #signal.signal(7, handler) + + print 'ok' @unittest.skipIf(sys.platform == "win32", "Not valid on Windows") diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py --- a/pypy/module/signal/interp_signal.py +++ b/pypy/module/signal/interp_signal.py @@ -236,7 +236,10 @@ None -- if an unknown handler is in effect (XXX UNIMPLEMENTED) anything else -- the callable Python object used as a handler """ - check_signum_in_range(space, signum) + if sys.platform == 'win32': + check_signum_exists(space, signum) + else: + check_signum_in_range(space, signum) action = space.check_signal_action if signum in action.handlers_w: return action.handlers_w[signum] diff --git a/pypy/module/signal/test/test_signal.py b/pypy/module/signal/test/test_signal.py --- a/pypy/module/signal/test/test_signal.py +++ b/pypy/module/signal/test/test_signal.py @@ -55,7 +55,7 @@ skip("requires os.kill() and os.getpid()") signal = self.signal # the signal module to test if not hasattr(signal, 'SIGUSR1'): - py.test.skip("requires SIGUSR1 in signal") + skip("requires SIGUSR1 in signal") signum = signal.SIGUSR1 received = [] @@ -156,6 +156,7 @@ import sys if sys.platform == 'win32': raises(ValueError, signal, 42, lambda *args: None) + raises(ValueError, signal, 7, lambda *args: None) else: signal(42, lambda *args: None) signal(42, SIG_DFL) From noreply at buildbot.pypy.org Sat Jan 12 20:40:26 2013 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 12 Jan 2013 20:40:26 +0100 (CET) Subject: [pypy-commit] pypy default: backout half of previous uncareful commit Message-ID: <20130112194026.F39C51C015D@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r59998:794896284f78 Date: 2013-01-12 21:40 +0200 http://bitbucket.org/pypy/pypy/changeset/794896284f78/ Log: backout half of previous uncareful commit diff --git a/lib-python/2.7/heapq.py b/lib-python/2.7/heapq.py --- a/lib-python/2.7/heapq.py +++ b/lib-python/2.7/heapq.py @@ -129,11 +129,7 @@ __all__ = ['heappush', 'heappop', 'heapify', 'heapreplace', 'merge', 'nlargest', 'nsmallest', 'heappushpop'] -try: - from itertools import islice, repeat, count, imap, izip, tee, chain -except: - import sys - print sys.path +from itertools import islice, repeat, count, imap, izip, tee, chain from operator import itemgetter import bisect diff --git a/lib-python/2.7/test/test_signal.py b/lib-python/2.7/test/test_signal.py --- a/lib-python/2.7/test/test_signal.py +++ b/lib-python/2.7/test/test_signal.py @@ -224,8 +224,7 @@ signal.signal(-1, handler) with self.assertRaises(ValueError): - pass - #signal.signal(7, handler) + signal.signal(7, handler) print 'ok' From noreply at buildbot.pypy.org Sat Jan 12 20:41:15 2013 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 12 Jan 2013 20:41:15 +0100 (CET) Subject: [pypy-commit] pypy default: remove debug cruft Message-ID: <20130112194115.D99F91C015D@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r59999:de79f80b06ff Date: 2013-01-12 21:41 +0200 http://bitbucket.org/pypy/pypy/changeset/de79f80b06ff/ Log: remove debug cruft diff --git a/lib-python/2.7/test/test_signal.py b/lib-python/2.7/test/test_signal.py --- a/lib-python/2.7/test/test_signal.py +++ b/lib-python/2.7/test/test_signal.py @@ -225,8 +225,6 @@ with self.assertRaises(ValueError): signal.signal(7, handler) - - print 'ok' @unittest.skipIf(sys.platform == "win32", "Not valid on Windows") From noreply at buildbot.pypy.org Sat Jan 12 20:43:07 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 12 Jan 2013 20:43:07 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: oops Message-ID: <20130112194307.EDAC41C015D@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60000:5b4f2cb04aad Date: 2013-01-12 17:51 +0200 http://bitbucket.org/pypy/pypy/changeset/5b4f2cb04aad/ Log: oops diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py --- a/pypy/jit/backend/llsupport/llmodel.py +++ b/pypy/jit/backend/llsupport/llmodel.py @@ -285,7 +285,7 @@ pos = self._decode_pos(deadframe, index) descr = self.gc_ll_descr.getframedescrs(self).arraydescr ofs = self.unpack_arraydescr(descr) - return self.read_int_at_mem(deadframe, pos + ofs, 1, WORD) + return self.read_int_at_mem(deadframe, pos + ofs, WORD, 1) def get_ref_value(self, deadframe, index): pos = self._decode_pos(deadframe, index) diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py --- a/pypy/jit/backend/test/runner_test.py +++ b/pypy/jit/backend/test/runner_test.py @@ -314,7 +314,7 @@ operations[0].setfailargs([i0]) self.cpu.compile_loop(inputargs, operations, looptoken) - i1list = [BoxInt() for i in range(1000)] + i1list = [BoxInt() for i in range(150)] bridge = [] iprev = i0 for i1 in i1list: @@ -331,7 +331,7 @@ deadframe = self.cpu.execute_token(looptoken, 1) fail = self.cpu.get_latest_descr(deadframe) assert fail.identifier == 3 - for i in range(1000): + for i in range(len(i1list)): res = self.cpu.get_int_value(deadframe, i) assert res == 2 + i diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -1830,7 +1830,7 @@ if loc is None: positions[i] = -1 elif isinstance(loc, StackLoc): - positions[i] = (loc.value + JITFRAME_FIXED_SIZE) * WORD + positions[i] = loc.value else: assert isinstance(loc, RegLoc) v = (gpr_reg_mgr_cls.all_reg_indexes[loc.value] + From noreply at buildbot.pypy.org Sat Jan 12 20:43:09 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 12 Jan 2013 20:43:09 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fix fix fix Message-ID: <20130112194309.5E8C71C015D@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60001:5af5c9bc65e6 Date: 2013-01-12 21:42 +0200 http://bitbucket.org/pypy/pypy/changeset/5af5c9bc65e6/ Log: fix fix fix diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py --- a/pypy/jit/backend/test/runner_test.py +++ b/pypy/jit/backend/test/runner_test.py @@ -342,7 +342,7 @@ def __setattr__(self, name, value): if (name == 'index' or name == '_carry_around_for_tests' - or name == '_TYPE'): + or name == '_TYPE' or name == '_cpu'): return AbstractFailDescr.__setattr__(self, name, value) py.test.fail("finish descrs should not be touched") faildescr = UntouchableFailDescr() # to check that is not touched diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -12,36 +12,25 @@ from pypy.jit.backend.model import CompiledLoopToken from pypy.jit.backend.x86.regalloc import (RegAlloc, get_ebp_ofs, _get_scale, gpr_reg_mgr_cls, xmm_reg_mgr_cls, _valid_addressing_size) - from pypy.jit.backend.x86.arch import (FRAME_FIXED_SIZE, FORCE_INDEX_OFS, WORD, - IS_X86_32, IS_X86_64, - JITFRAME_FIXED_SIZE) - -from pypy.jit.backend.x86.regloc import (eax, ecx, edx, ebx, - esp, ebp, esi, edi, - xmm0, xmm1, xmm2, xmm3, - xmm4, xmm5, xmm6, xmm7, - r8, r9, r10, r11, - r12, r13, r14, r15, - X86_64_SCRATCH_REG, - X86_64_XMM_SCRATCH_REG, - RegLoc, StackLoc, ConstFloatLoc, - ImmedLoc, AddressLoc, imm, - imm0, imm1, FloatImmedLoc) - + IS_X86_32, IS_X86_64) +from pypy.jit.backend.x86.regloc import (eax, ecx, edx, ebx, esp, ebp, esi, edi, + xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, r8, r9, r10, r11, + r12, r13, r14, r15, X86_64_SCRATCH_REG, X86_64_XMM_SCRATCH_REG, + RegLoc, StackLoc, ConstFloatLoc, ImmedLoc, AddressLoc, imm, + imm0, imm1, FloatImmedLoc, RawStackLoc) from pypy.rlib.objectmodel import we_are_translated, specialize -from pypy.jit.backend.x86 import rx86, regloc, codebuf +from pypy.jit.backend.x86 import rx86, codebuf from pypy.jit.metainterp.resoperation import rop, ResOperation from pypy.jit.backend.x86 import support from pypy.rlib.debug import (debug_print, debug_start, debug_stop, - have_debug_prints, fatalerror) + have_debug_prints) from pypy.rlib import rgc from pypy.rlib.clibffi import FFI_DEFAULT_ABI from pypy.jit.backend.x86.jump import remap_frame_layout from pypy.jit.codewriter.effectinfo import EffectInfo from pypy.jit.codewriter import longlong from pypy.rlib.rarithmetic import intmask -from pypy.rlib import longlong2float from pypy.rlib.objectmodel import compute_unique_id # darwin requires the stack to be 16 bytes aligned on calls. Same for gcc 4.5.0, @@ -1903,10 +1892,11 @@ def genop_finish(self, op, arglocs, result_loc): if len(arglocs) == 2: [return_val, argloc] = arglocs - if op.getarg(0).type == FLOAT: - self.mc.MOVSD_bx(0, return_val.value) + if op.getarg(0).type == FLOAT and not IS_X86_64: + size = WORD * 2 else: - self.mc.MOV_br(0, return_val.value) + size = WORD + self.save_into_mem(raw_stack(0), return_val, imm(size)) else: [argloc] = arglocs if argloc is not eax: @@ -2438,6 +2428,9 @@ def mem(loc, offset): return AddressLoc(loc, imm0, 0, offset) +def raw_stack(offset, type=INT): + return RawStackLoc(offset, type) + def heap(addr): return AddressLoc(ImmedLoc(addr), imm0, 0, 0) diff --git a/pypy/jit/backend/x86/regloc.py b/pypy/jit/backend/x86/regloc.py --- a/pypy/jit/backend/x86/regloc.py +++ b/pypy/jit/backend/x86/regloc.py @@ -41,19 +41,15 @@ def find_unused_reg(self): return eax -class StackLoc(AssemblerLocation): +class RawStackLoc(AssemblerLocation): + """ The same as stack location, but does not know it's position. + Mostly usable for raw frame access + """ _immutable_ = True _location_code = 'b' - def __init__(self, position, ebp_offset, type): - # _getregkey() returns self.value; the value returned must not - # conflict with RegLoc._getregkey(). It doesn't a bit by chance, - # so let it fail the following assert if it no longer does. - assert ebp_offset >= 0 - #assert not (0 <= ebp_offset < 8 + 8 * IS_X86_64) - self.position = position - self.value = ebp_offset - # One of INT, REF, FLOAT + def __init__(self, value, type): + self.value = value self.type = type def _getregkey(self): @@ -70,6 +66,18 @@ def assembler(self): return repr(self) +class StackLoc(RawStackLoc): + def __init__(self, position, ebp_offset, type): + # _getregkey() returns self.value; the value returned must not + # conflict with RegLoc._getregkey(). It doesn't a bit by chance, + # so let it fail the following assert if it no longer does. + assert ebp_offset >= 0 + #assert not (0 <= ebp_offset < 8 + 8 * IS_X86_64) + self.position = position + self.value = ebp_offset + # One of INT, REF, FLOAT + self.type = type + class RegLoc(AssemblerLocation): _immutable_ = True def __init__(self, regnum, is_xmm): From noreply at buildbot.pypy.org Sat Jan 12 20:51:33 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 12 Jan 2013 20:51:33 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: base exception handling Message-ID: <20130112195133.9123F1C015D@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60002:6944a86418c9 Date: 2013-01-12 21:51 +0200 http://bitbucket.org/pypy/pypy/changeset/6944a86418c9/ Log: base exception handling diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py --- a/pypy/jit/backend/llsupport/llmodel.py +++ b/pypy/jit/backend/llsupport/llmodel.py @@ -77,11 +77,6 @@ return (rffi.cast(lltype.Signed, _exception_emulator) + rffi.sizeof(lltype.Signed)) - # XXX I think we don't need it any more - #self._memoryerror_emulated = rffi.cast(llmemory.GCREF, -123) - #self.deadframe_memoryerror = lltype.malloc(jitframe.DEADFRAME, 0) - #self.deadframe_memoryerror.jf_guard_exc = self._memoryerror_emulated - def propagate_exception(): exc = _exception_emulator[1] _exception_emulator[0] = 0 @@ -128,8 +123,6 @@ slowpathaddr = rffi.cast(lltype.Signed, f) return endaddr, lengthaddr, slowpathaddr - #self.deadframe_memoryerror = lltype.malloc(jitframe.DEADFRAME, 0) - def propagate_exception(): addr = llop.get_exception_addr(llmemory.Address) addr.address[0] = llmemory.NULL @@ -172,10 +165,7 @@ return llhelper(self.PROPAGATE_EXCEPTION, self._propagate_exception) def grab_exc_value(self, deadframe): - XXX - deadframe = lltype.cast_opaque_ptr(jitframe.DEADFRAMEPTR, deadframe) - if not we_are_translated() and deadframe == self.deadframe_memoryerror: - return "memoryerror!" # for tests + deadframe = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, deadframe) return deadframe.jf_guard_exc def set_savedata_ref(self, deadframe, data): diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -1875,8 +1875,8 @@ from pypy.jit.backend.llsupport.descr import unpack_fielddescr descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) offset, size, _ = unpack_fielddescr(descrs.jf_guard_exc) - # XXX ebp relative, not eax relative - mc.MOV_mr((eax.value, offset), ebx.value) + _, base_offset, _ = unpack_arraydescr(descrs.arraydescr) + mc.MOV_br(offset - base_offset, ebx.value) # now we return from the complete frame, which starts from # _call_header_with_stack_check(). The LEA in _call_footer below From noreply at buildbot.pypy.org Sat Jan 12 21:40:24 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sat, 12 Jan 2013 21:40:24 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Split up conftest Message-ID: <20130112204024.BF0DF1C015D@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60003:9e9c157d7f62 Date: 2013-01-12 21:39 +0100 http://bitbucket.org/pypy/pypy/changeset/9e9c157d7f62/ Log: Split up conftest diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -35,37 +35,14 @@ global option option = config.option -def _set_platform(opt, opt_str, value, parser): - from rpython.config.translationoption import PLATFORMS - from rpython.translator.platform import set_platform - if value not in PLATFORMS: - raise ValueError("%s not in %s" % (value, PLATFORMS)) - set_platform(value, None) - def pytest_addoption(parser): group = parser.getgroup("pypy options") - group.addoption('--view', action="store_true", dest="view", default=False, - help="view translation tests' flow graphs with Pygame") group.addoption('-A', '--runappdirect', action="store_true", default=False, dest="runappdirect", help="run applevel tests directly on python interpreter (not through PyPy)") group.addoption('--direct', action="store_true", default=False, dest="rundirect", help="run pexpect tests directly") - group.addoption('-P', '--platform', action="callback", type="string", - default="host", callback=_set_platform, - help="set up tests to use specified platform as compile/run target") - group = parser.getgroup("JIT options") - group.addoption('--viewloops', action="store_true", - default=False, dest="viewloops", - help="show only the compiled loops") - -def pytest_sessionstart(): - # have python subprocesses avoid startup customizations by default - try: - del os.environ['PYTHONSTARTUP'] - except KeyError: - pass def pytest_funcarg__space(request): from pypy.tool.pytest.objspace import gettestobjspace diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py --- a/rpython/annotator/test/test_annrpython.py +++ b/rpython/annotator/test/test_annrpython.py @@ -1,7 +1,7 @@ from __future__ import with_statement import py.test import sys -from pypy import conftest +from rpython.conftest import option from rpython.annotator import model as annmodel from rpython.annotator.annrpython import RPythonAnnotator as _RPythonAnnotator @@ -49,7 +49,7 @@ class RPythonAnnotator(_RPythonAnnotator): def build_types(self, *args): s = _RPythonAnnotator.build_types(self, *args) - if conftest.option.view: + if option.view: self.translator.view() return s diff --git a/rpython/conftest.py b/rpython/conftest.py --- a/rpython/conftest.py +++ b/rpython/conftest.py @@ -1,4 +1,42 @@ from os.path import * +import py, pytest cdir = realpath(join(dirname(__file__), 'translator', 'c')) cache_dir = realpath(join(dirname(__file__), '_cache')) +option = None + +def braindead_deindent(self): + """monkeypatch that wont end up doing stupid in the python tokenizer""" + text = '\n'.join(self.lines) + short = py.std.textwrap.dedent(text) + newsource = py.code.Source() + newsource.lines[:] = short.splitlines() + return newsource + +py.code.Source.deindent = braindead_deindent + +def pytest_report_header(): + return "pytest-%s from %s" %(pytest.__version__, pytest.__file__) + +def pytest_configure(config): + global option + option = config.option + +def _set_platform(opt, opt_str, value, parser): + from rpython.config.translationoption import PLATFORMS + from rpython.translator.platform import set_platform + if value not in PLATFORMS: + raise ValueError("%s not in %s" % (value, PLATFORMS)) + set_platform(value, None) + +def pytest_addoption(parser): + group = parser.getgroup("rpython options") + group.addoption('--view', action="store_true", dest="view", default=False, + help="view translation tests' flow graphs with Pygame") + group.addoption('-P', '--platform', action="callback", type="string", + default="host", callback=_set_platform, + help="set up tests to use specified platform as compile/run target") + group = parser.getgroup("JIT options") + group.addoption('--viewloops', action="store_true", + default=False, dest="viewloops", + help="show only the compiled loops") diff --git a/rpython/flowspace/test/test_generator.py b/rpython/flowspace/test/test_generator.py --- a/rpython/flowspace/test/test_generator.py +++ b/rpython/flowspace/test/test_generator.py @@ -1,4 +1,4 @@ -from pypy.conftest import option +from rpython.conftest import option from rpython.flowspace.objspace import FlowObjSpace from rpython.flowspace.model import Variable from rpython.flowspace.generator import (make_generatoriterator_class, diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py --- a/rpython/flowspace/test/test_objspace.py +++ b/rpython/flowspace/test/test_objspace.py @@ -7,7 +7,7 @@ from rpython.translator.simplify import simplify_graph from rpython.flowspace.objspace import FlowObjSpace from rpython.flowspace.flowcontext import FlowingError, FlowSpaceFrame -from pypy import conftest +from rpython.conftest import option from rpython.tool.stdlib_opcode import bytecode_spec, host_bytecode_spec import os @@ -41,7 +41,7 @@ return graph def show(self, graph): - if conftest.option.view: + if option.view: graph.show() def setup_class(cls): diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -6,7 +6,7 @@ from rpython.rlib.debug import debug_start, debug_stop, debug_print from rpython.rlib import rstack from rpython.rlib.jit import JitDebugInfo, Counters -from pypy.conftest import option +from rpython.conftest import option from rpython.tool.sourcetools import func_with_new_name from rpython.jit.metainterp.resoperation import ResOperation, rop, get_deep_immutable_oplist diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py --- a/rpython/jit/metainterp/history.py +++ b/rpython/jit/metainterp/history.py @@ -6,7 +6,7 @@ from rpython.rlib.objectmodel import compute_unique_id from rpython.rlib.rarithmetic import r_int64, is_valid_int -from pypy.conftest import option +from rpython.conftest import option from rpython.jit.metainterp.resoperation import ResOperation, rop from rpython.jit.codewriter import heaptracker, longlong diff --git a/rpython/jit/metainterp/test/support.py b/rpython/jit/metainterp/test/support.py --- a/rpython/jit/metainterp/test/support.py +++ b/rpython/jit/metainterp/test/support.py @@ -129,7 +129,7 @@ try: metainterp.compile_and_run_once(jitdriver_sd, *args) except DoneWithThisFrame, e: - #if conftest.option.view: + #if option.view: # metainterp.stats.view() return e.args[0] else: diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py --- a/rpython/jit/metainterp/test/test_ajit.py +++ b/rpython/jit/metainterp/test/test_ajit.py @@ -3,7 +3,6 @@ import py -from pypy import conftest from rpython.jit.codewriter import longlong from rpython.jit.codewriter.policy import JitPolicy, StopAtXPolicy from rpython.jit.metainterp import pyjitpl, history diff --git a/rpython/jit/metainterp/test/test_memmgr.py b/rpython/jit/metainterp/test/test_memmgr.py --- a/rpython/jit/metainterp/test/test_memmgr.py +++ b/rpython/jit/metainterp/test/test_memmgr.py @@ -4,11 +4,11 @@ # XXX we don't invokve py.test machinery but try to make sure # pypy support code sees the test options from the invoking # process - import pypy.conftest + import rpython.conftest class opt: pass - pypy.conftest.option = opt() - pypy.conftest.option.__dict__.update(eval(sys.argv[3])) + rpython.conftest.option = opt() + rpython.conftest.option.__dict__.update(eval(sys.argv[3])) import py from rpython.jit.metainterp.memmgr import MemoryManager @@ -231,7 +231,7 @@ py.test.skip( "passing repr() to subprocess.Popen probably doesn't work") import os, subprocess - from pypy.conftest import option + from rpython.conftest import option thisfile = os.path.abspath(__file__) p = subprocess.Popen([sys.executable, thisfile, '--sub', repr(sys.path), repr(option.__dict__)]) diff --git a/rpython/jit/tl/jittest.py b/rpython/jit/tl/jittest.py --- a/rpython/jit/tl/jittest.py +++ b/rpython/jit/tl/jittest.py @@ -4,7 +4,7 @@ only after the '---> Checkpoint' fork. """ -from pypy.conftest import option +from rpython.conftest import option from rpython.rtyper.lltypesystem import lltype from rpython.rtyper.llinterp import LLInterpreter from rpython.rtyper.annlowlevel import llstr diff --git a/rpython/rlib/test/test_jit.py b/rpython/rlib/test/test_jit.py --- a/rpython/rlib/test/test_jit.py +++ b/rpython/rlib/test/test_jit.py @@ -1,6 +1,6 @@ import py -from pypy.conftest import option +from rpython.conftest import option from rpython.annotator.model import UnionError from rpython.rlib.jit import (hint, we_are_jitted, JitDriver, elidable_promote, JitHintError, oopspec, isconstant) diff --git a/rpython/rlib/test/test_nonconst.py b/rpython/rlib/test/test_nonconst.py --- a/rpython/rlib/test/test_nonconst.py +++ b/rpython/rlib/test/test_nonconst.py @@ -6,7 +6,7 @@ from rpython.flowspace.objspace import FlowObjSpace from rpython.annotator.annrpython import RPythonAnnotator -from pypy.conftest import option +from rpython.conftest import option from rpython.annotator.model import SomeInstance def test_nonconst(): diff --git a/rpython/rlib/test/test_objectmodel.py b/rpython/rlib/test/test_objectmodel.py --- a/rpython/rlib/test/test_objectmodel.py +++ b/rpython/rlib/test/test_objectmodel.py @@ -5,7 +5,7 @@ from rpython.translator.translator import TranslationContext, graphof from rpython.rtyper.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin from rpython.rtyper.test.test_llinterp import interpret -from pypy.conftest import option +from rpython.conftest import option def strange_key_eq(key1, key2): return key1[0] == key2[0] # only the 1st character is relevant diff --git a/rpython/rtyper/lltypesystem/ll2ctypes.py b/rpython/rtyper/lltypesystem/ll2ctypes.py --- a/rpython/rtyper/lltypesystem/ll2ctypes.py +++ b/rpython/rtyper/lltypesystem/ll2ctypes.py @@ -15,7 +15,6 @@ load_library_kwargs = {} import os, platform as host_platform -from pypy import conftest from rpython.rtyper.lltypesystem import lltype, llmemory from rpython.rtyper.extfunc import ExtRegistryEntry from rpython.rlib.objectmodel import Symbolic, ComputedIntSymbolic diff --git a/rpython/rtyper/lltypesystem/test/test_rffi.py b/rpython/rtyper/lltypesystem/test/test_rffi.py --- a/rpython/rtyper/lltypesystem/test/test_rffi.py +++ b/rpython/rtyper/lltypesystem/test/test_rffi.py @@ -15,7 +15,7 @@ from rpython.rtyper.rtyper import RPythonTyper from rpython.translator.backendopt.all import backend_optimizations from rpython.translator.translator import graphof -from pypy.conftest import option +from rpython.conftest import option from rpython.flowspace.model import summary from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.rlib.rarithmetic import r_singlefloat diff --git a/rpython/rtyper/lltypesystem/test/test_rtagged.py b/rpython/rtyper/lltypesystem/test/test_rtagged.py --- a/rpython/rtyper/lltypesystem/test/test_rtagged.py +++ b/rpython/rtyper/lltypesystem/test/test_rtagged.py @@ -5,7 +5,7 @@ from rpython.translator.translator import graphof from rpython.flowspace.model import summary from rpython.translator.backendopt.all import backend_optimizations -from pypy import conftest +from rpython.conftest import option class A(object): @@ -205,7 +205,7 @@ t = interp.typer.annotator.translator t.config.translation.backendopt.constfold = True backend_optimizations(t) - if conftest.option.view: + if option.view: t.view() LLFrame = interp.frame_class diff --git a/rpython/rtyper/memory/gctransform/test/test_refcounting.py b/rpython/rtyper/memory/gctransform/test/test_refcounting.py --- a/rpython/rtyper/memory/gctransform/test/test_refcounting.py +++ b/rpython/rtyper/memory/gctransform/test/test_refcounting.py @@ -5,7 +5,7 @@ from rpython.rtyper.lltypesystem import lltype from rpython.translator.translator import TranslationContext, graphof from rpython.translator.c.gc import RefcountingGcPolicy -from pypy import conftest +from rpython.conftest import option class TestLLInterpedRefcounting(LLInterpedTranformerTests): gcpolicy = RefcountingGcPolicy @@ -128,7 +128,7 @@ fptr = getattr(transformer, attr)(TYPE) transformer.transform_graph(graphof(t, f)) transformer.finish(backendopt=False) - if conftest.option.view: + if option.view: t.view() if fptr: return fptr._obj.graph, t diff --git a/rpython/rtyper/memory/gctransform/test/test_transform.py b/rpython/rtyper/memory/gctransform/test/test_transform.py --- a/rpython/rtyper/memory/gctransform/test/test_transform.py +++ b/rpython/rtyper/memory/gctransform/test/test_transform.py @@ -4,7 +4,7 @@ from rpython.translator.translator import TranslationContext, graphof from rpython.translator.exceptiontransform import ExceptionTransformer from rpython.rtyper.lltypesystem import lltype -from pypy import conftest +from rpython.conftest import option class LLInterpedTranformerTests: @@ -24,7 +24,7 @@ if isinstance(v.concretetype, lltype.Ptr): assert v.concretetype.TO._gckind != 'gc', "fix the test!" llinterp = LLInterpreter(t.rtyper) - if conftest.option.view: + if option.view: t.view() return llinterp, graph @@ -133,7 +133,7 @@ t.buildannotator().build_types(func, inputtypes) if specialize: t.buildrtyper().specialize() - if conftest.option.view: + if option.view: t.view() return t @@ -145,7 +145,7 @@ graphs_borrowed = {} for graph in t.graphs[:]: graphs_borrowed[graph] = transformer.transform_graph(graph) - if conftest.option.view: + if option.view: t.view() t.checkgraphs() if check: diff --git a/rpython/rtyper/memory/test/test_transformed_gc.py b/rpython/rtyper/memory/test/test_transformed_gc.py --- a/rpython/rtyper/memory/test/test_transformed_gc.py +++ b/rpython/rtyper/memory/test/test_transformed_gc.py @@ -9,7 +9,7 @@ from rpython.rlib.objectmodel import compute_unique_id, we_are_translated from rpython.rlib.debug import ll_assert from rpython.rlib import rgc -from pypy import conftest +from rpython.conftest import option from rpython.rlib.rstring import StringBuilder from rpython.rlib.rarithmetic import LONG_BIT @@ -32,7 +32,7 @@ if backendopt: from rpython.translator.backendopt.all import backend_optimizations backend_optimizations(t) - if conftest.option.view: + if option.view: t.viewcg() return t @@ -109,7 +109,7 @@ db = cbuild.generate_graphs_for_llinterp() entrypointptr = cbuild.getentrypointptr() entrygraph = entrypointptr._obj.graph - if conftest.option.view: + if option.view: t.viewcg() cls.name_to_func = name_to_func diff --git a/rpython/rtyper/ootypesystem/test/test_oortype.py b/rpython/rtyper/ootypesystem/test/test_oortype.py --- a/rpython/rtyper/ootypesystem/test/test_oortype.py +++ b/rpython/rtyper/ootypesystem/test/test_oortype.py @@ -1,5 +1,5 @@ import py -from pypy import conftest +from rpython.conftest import option from rpython.rtyper.ootypesystem.ootype import * from rpython.rtyper.ootypesystem import ootype from rpython.rtyper.ootypesystem.rlist import ListRepr @@ -16,10 +16,10 @@ t = TranslationContext() t.config.translation.ootype.mangle = mangle t.buildannotator().build_types(f, args) - if viewBefore or conftest.option.view: + if viewBefore or option.view: t.view() t.buildrtyper(type_system="ootype").specialize() - if viewAfter or conftest.option.view: + if viewAfter or option.view: t.view() return graphof(t, f) @@ -184,7 +184,7 @@ t = TranslationContext() t.buildannotator().build_types(f, []) - if conftest.option.view: + if option.view: t.view() rtyper = t.buildrtyper(type_system="ootype") rtyper.specialize() diff --git a/rpython/rtyper/test/test_llann.py b/rpython/rtyper/test/test_llann.py --- a/rpython/rtyper/test/test_llann.py +++ b/rpython/rtyper/test/test_llann.py @@ -1,7 +1,7 @@ import py from rpython.annotator import model as annmodel -from pypy.conftest import option +from rpython.conftest import option from rpython.flowspace.objspace import FlowObjSpace from rpython.rtyper.annlowlevel import (annotate_lowlevel_helper, MixLevelHelperAnnotator, PseudoHighLevelCallable, llhelper, diff --git a/rpython/rtyper/test/test_llinterp.py b/rpython/rtyper/test/test_llinterp.py --- a/rpython/rtyper/test/test_llinterp.py +++ b/rpython/rtyper/test/test_llinterp.py @@ -12,7 +12,7 @@ from rpython.annotator.model import lltype_to_annotation from rpython.rlib.rarithmetic import r_uint, ovfcheck from rpython.tool import leakfinder -from pypy import conftest +from rpython.conftest import option # switch on logging of interp to show more info on failing tests diff --git a/rpython/rtyper/test/test_rstr.py b/rpython/rtyper/test/test_rstr.py --- a/rpython/rtyper/test/test_rstr.py +++ b/rpython/rtyper/test/test_rstr.py @@ -957,10 +957,10 @@ return const("head")+tail def f(): return g(const("tail")) - from pypy import conftest + from rpython.conftest import option t, typer, fgraph = self.gengraph(f, [], backendopt=True) - if conftest.option.view: + if option.view: t.view() assert summary(fgraph) == {} diff --git a/rpython/rtyper/test/test_rvirtualizable2.py b/rpython/rtyper/test/test_rvirtualizable2.py --- a/rpython/rtyper/test/test_rvirtualizable2.py +++ b/rpython/rtyper/test/test_rvirtualizable2.py @@ -6,7 +6,7 @@ from rpython.flowspace.model import summary from rpython.rtyper.llinterp import LLInterpreter from rpython.rtyper.rclass import IR_IMMUTABLE, IR_IMMUTABLE_ARRAY -from pypy import conftest +from rpython.conftest import option class V(object): diff --git a/rpython/translator/backendopt/test/test_all.py b/rpython/translator/backendopt/test/test_all.py --- a/rpython/translator/backendopt/test/test_all.py +++ b/rpython/translator/backendopt/test/test_all.py @@ -9,7 +9,7 @@ from rpython.annotator import model as annmodel from rpython.rtyper.llinterp import LLInterpreter from rpython.rlib.rarithmetic import intmask -from pypy import conftest +from rpython.conftest import option class A: def __init__(self, x, y): @@ -50,10 +50,10 @@ t = TranslationContext() t.buildannotator().build_types(func, sig) t.buildrtyper(type_system=self.type_system).specialize() - if conftest.option.view: + if option.view: t.view() backend_optimizations(t, **optflags) - if conftest.option.view: + if option.view: t.view() return t @@ -282,7 +282,7 @@ annhelper.backend_optimize() # ^^^ as the inliner can't handle exception-transformed graphs, # this should *not* inline common() into later(). - if conftest.option.view: + if option.view: later_graph.show() common_graph = graphof(t, common) found = False diff --git a/rpython/translator/backendopt/test/test_canraise.py b/rpython/translator/backendopt/test/test_canraise.py --- a/rpython/translator/backendopt/test/test_canraise.py +++ b/rpython/translator/backendopt/test/test_canraise.py @@ -4,7 +4,7 @@ from rpython.translator.backendopt.canraise import RaiseAnalyzer from rpython.translator.backendopt.all import backend_optimizations from rpython.rtyper.test.tool import LLRtypeMixin, OORtypeMixin -from pypy.conftest import option +from rpython.conftest import option class BaseTestCanRaise(object): type_system = None diff --git a/rpython/translator/backendopt/test/test_constfold.py b/rpython/translator/backendopt/test/test_constfold.py --- a/rpython/translator/backendopt/test/test_constfold.py +++ b/rpython/translator/backendopt/test/test_constfold.py @@ -7,19 +7,19 @@ from rpython.rtyper import rclass from rpython.rlib import objectmodel from rpython.translator.backendopt.constfold import constant_fold_graph -from pypy import conftest +from rpython.conftest import option def get_graph(fn, signature): t = TranslationContext() t.buildannotator().build_types(fn, signature) t.buildrtyper().specialize() graph = graphof(t, fn) - if conftest.option.view: + if option.view: t.view() return graph, t def check_graph(graph, args, expected_result, t): - if conftest.option.view: + if option.view: t.view() checkgraph(graph) interp = LLInterpreter(t.rtyper) @@ -254,7 +254,7 @@ inline.auto_inline_graphs(t, t.graphs, threshold=999) constant_fold_graph(graph) removenoops.remove_same_as(graph) - if conftest.option.view: + if option.view: t.view() # check that the graph starts with a condition (which should be 'n==42') # and that if this condition is true, it goes directly to 'return 100'. @@ -276,7 +276,7 @@ from rpython.translator.backendopt import removenoops removenoops.remove_same_as(graph) constant_fold_graph(graph) - if conftest.option.view: + if option.view: t.view() assert summary(graph) == {'int_gt': 1} check_graph(graph, [2], 17, t) @@ -296,7 +296,7 @@ inline.auto_inline_graphs(t, t.graphs, threshold=999) removenoops.remove_same_as(graph) constant_fold_graph(graph) - if conftest.option.view: + if option.view: t.view() # check that the graph starts with a condition (which should be 'n > 5') # and that if this condition is false, it goes directly to 'return 0'. diff --git a/rpython/translator/backendopt/test/test_escape.py b/rpython/translator/backendopt/test/test_escape.py --- a/rpython/translator/backendopt/test/test_escape.py +++ b/rpython/translator/backendopt/test/test_escape.py @@ -3,12 +3,12 @@ from rpython.translator.backendopt.escape import AbstractDataFlowInterpreter from rpython.translator.backendopt.escape import malloc_like_graphs from rpython.rlib.objectmodel import instantiate -from pypy import conftest +from rpython.conftest import option def build_adi(function, types): t = Translation(function, types) t.rtype() - if conftest.option.view: + if option.view: t.view() adi = AbstractDataFlowInterpreter(t.context) graph = graphof(t.context, function) diff --git a/rpython/translator/backendopt/test/test_finalizer.py b/rpython/translator/backendopt/test/test_finalizer.py --- a/rpython/translator/backendopt/test/test_finalizer.py +++ b/rpython/translator/backendopt/test/test_finalizer.py @@ -6,7 +6,7 @@ from rpython.translator.backendopt.all import backend_optimizations from rpython.translator.unsimplify import varoftype from rpython.rtyper.lltypesystem import lltype, rffi -from pypy.conftest import option +from rpython.conftest import option from rpython.rlib import rgc diff --git a/rpython/translator/backendopt/test/test_inline.py b/rpython/translator/backendopt/test/test_inline.py --- a/rpython/translator/backendopt/test/test_inline.py +++ b/rpython/translator/backendopt/test/test_inline.py @@ -16,7 +16,7 @@ from rpython.rlib.rarithmetic import ovfcheck from rpython.translator.test.snippet import is_perfect_number from rpython.translator.backendopt.all import INLINE_THRESHOLD_FOR_TEST -from pypy.conftest import option +from rpython.conftest import option from rpython.translator.backendopt import removenoops from rpython.flowspace.model import summary diff --git a/rpython/translator/backendopt/test/test_innerloop.py b/rpython/translator/backendopt/test/test_innerloop.py --- a/rpython/translator/backendopt/test/test_innerloop.py +++ b/rpython/translator/backendopt/test/test_innerloop.py @@ -1,6 +1,6 @@ from rpython.translator.translator import TranslationContext from rpython.translator.backendopt.innerloop import find_inner_loops -from pypy.conftest import option +from rpython.conftest import option def test_simple_loop(): def snippet_fn(x, y): diff --git a/rpython/translator/backendopt/test/test_malloc.py b/rpython/translator/backendopt/test/test_malloc.py --- a/rpython/translator/backendopt/test/test_malloc.py +++ b/rpython/translator/backendopt/test/test_malloc.py @@ -8,7 +8,7 @@ from rpython.rtyper.lltypesystem import lltype, llmemory from rpython.rtyper.ootypesystem import ootype from rpython.rlib import objectmodel -from pypy.conftest import option +from rpython.conftest import option class BaseMallocRemovalTest(object): type_system = None diff --git a/rpython/translator/backendopt/test/test_mallocprediction.py b/rpython/translator/backendopt/test/test_mallocprediction.py --- a/rpython/translator/backendopt/test/test_mallocprediction.py +++ b/rpython/translator/backendopt/test/test_mallocprediction.py @@ -5,7 +5,7 @@ from rpython.translator.translator import TranslationContext, graphof from rpython.rtyper.llinterp import LLInterpreter from rpython.flowspace.model import checkgraph, Block -from pypy.conftest import option +from rpython.conftest import option import sys from rpython.translator.backendopt.mallocprediction import * diff --git a/rpython/translator/backendopt/test/test_mallocv.py b/rpython/translator/backendopt/test/test_mallocv.py --- a/rpython/translator/backendopt/test/test_mallocv.py +++ b/rpython/translator/backendopt/test/test_mallocv.py @@ -13,7 +13,7 @@ from rpython.rtyper.annlowlevel import llhelper from rpython.rlib import objectmodel from rpython.rlib.rarithmetic import ovfcheck -from pypy.conftest import option +from rpython.conftest import option DONT_CHECK_RESULT = object() class CHECK_RAISES: diff --git a/rpython/translator/backendopt/test/test_removenoops.py b/rpython/translator/backendopt/test/test_removenoops.py --- a/rpython/translator/backendopt/test/test_removenoops.py +++ b/rpython/translator/backendopt/test/test_removenoops.py @@ -10,7 +10,7 @@ from rpython.rtyper.lltypesystem import lltype from rpython.rtyper.lltypesystem.lloperation import llop from rpython.rtyper.llinterp import LLInterpreter -from pypy import conftest +from rpython.conftest import option import py log = py.log.Producer('test_backendoptimization') @@ -24,7 +24,7 @@ constfold=False, raisingop2direct_call=False) graph = graphof(t, fn) - if conftest.option.view: + if option.view: t.view() return graph, t @@ -148,7 +148,7 @@ num_cast_pointer = len(getops(graph_getsum)['cast_pointer']) changed = remove_duplicate_casts(graph_getsum, t) assert changed - if conftest.option.view: + if option.view: t.view() check_graph(graph, [10, True], 75, t) ops = getops(graph_getsum) diff --git a/rpython/translator/backendopt/test/test_storesink.py b/rpython/translator/backendopt/test/test_storesink.py --- a/rpython/translator/backendopt/test/test_storesink.py +++ b/rpython/translator/backendopt/test/test_storesink.py @@ -4,7 +4,7 @@ from rpython.translator.backendopt.storesink import storesink_graph from rpython.translator.backendopt import removenoops from rpython.flowspace.model import last_exception, checkgraph -from pypy.conftest import option +from rpython.conftest import option class TestStoreSink(object): # not sure if it makes any sense on ootype, maybe worth trying diff --git a/rpython/translator/backendopt/test/test_writeanalyze.py b/rpython/translator/backendopt/test/test_writeanalyze.py --- a/rpython/translator/backendopt/test/test_writeanalyze.py +++ b/rpython/translator/backendopt/test/test_writeanalyze.py @@ -6,7 +6,7 @@ from rpython.translator.backendopt.writeanalyze import WriteAnalyzer, top_set from rpython.translator.backendopt.writeanalyze import ReadWriteAnalyzer from rpython.translator.backendopt.all import backend_optimizations -from pypy.conftest import option +from rpython.conftest import option class BaseTest(object): diff --git a/rpython/translator/c/gcc/test/test_asmgcroot.py b/rpython/translator/c/gcc/test/test_asmgcroot.py --- a/rpython/translator/c/gcc/test/test_asmgcroot.py +++ b/rpython/translator/c/gcc/test/test_asmgcroot.py @@ -4,7 +4,7 @@ from rpython.translator.translator import TranslationContext from rpython.translator.c.genc import CStandaloneBuilder from rpython.annotator.listdef import s_list_of_strings -from pypy import conftest +from rpython.conftest import option from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.translator.platform import platform as compiler from rpython.rlib.rarithmetic import is_emulated_long @@ -61,7 +61,7 @@ c_source_filename = cbuilder.generate_source( defines = cbuilder.DEBUG_DEFINES) cls._patch_makefile(cbuilder.targetdir) - if conftest.option.view: + if option.view: t.view() exe_name = cbuilder.compile() diff --git a/rpython/translator/c/test/test_backendoptimized.py b/rpython/translator/c/test/test_backendoptimized.py --- a/rpython/translator/c/test/test_backendoptimized.py +++ b/rpython/translator/c/test/test_backendoptimized.py @@ -1,4 +1,4 @@ -from pypy import conftest +from rpython.conftest import option from rpython.rlib.rarithmetic import r_uint, r_longlong, r_ulonglong from rpython.translator.backendopt.all import backend_optimizations from rpython.translator.c.test.test_typed import TestTypedTestCase as _TestTypedTestCase diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py --- a/rpython/translator/c/test/test_newgc.py +++ b/rpython/translator/c/test/test_newgc.py @@ -5,7 +5,7 @@ import py -from pypy import conftest +from rpython.conftest import option from rpython.rlib import rgc from rpython.rlib.objectmodel import keepalive_until_here, compute_hash, compute_identity_hash from rpython.rlib.rstring import StringBuilder @@ -47,7 +47,7 @@ t.disable(['backendopt']) t.set_backend_extra_options(c_debug_defines=True) t.rtype() - if conftest.option.view: + if option.view: t.viewcg() exename = t.compile() diff --git a/rpython/translator/c/test/test_refcount.py b/rpython/translator/c/test/test_refcount.py --- a/rpython/translator/c/test/test_refcount.py +++ b/rpython/translator/c/test/test_refcount.py @@ -5,7 +5,7 @@ from rpython.translator.c import genc from rpython.translator.c.test.test_genc import compile from rpython.rtyper.lltypesystem import lltype -from pypy import conftest +from rpython.conftest import option def compile_func(func, args): return compile(func, args, gcpolicy='ref') diff --git a/rpython/translator/c/test/test_rtagged.py b/rpython/translator/c/test/test_rtagged.py --- a/rpython/translator/c/test/test_rtagged.py +++ b/rpython/translator/c/test/test_rtagged.py @@ -64,14 +64,14 @@ # only with Boehm so far from rpython.translator.interactive import Translation -from pypy import conftest +from rpython.conftest import option def test_tagged_boehm(): t = Translation(entry_point, gc='boehm', taggedpointers=True) try: exename = str(t.compile_c()) finally: - if conftest.option.view: + if option.view: t.view() g = os.popen(exename, 'r') data = g.read() diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py --- a/rpython/translator/c/test/test_standalone.py +++ b/rpython/translator/c/test/test_standalone.py @@ -11,7 +11,7 @@ from rpython.annotator.listdef import s_list_of_strings from rpython.tool.udir import udir from rpython.conftest import cdir -from pypy.conftest import option +from rpython.conftest import option class StandaloneTests(object): diff --git a/rpython/translator/c/test/test_symbolic.py b/rpython/translator/c/test/test_symbolic.py --- a/rpython/translator/c/test/test_symbolic.py +++ b/rpython/translator/c/test/test_symbolic.py @@ -1,6 +1,6 @@ from rpython.translator.interactive import Translation from rpython.translator.c.test.test_genc import compile -from pypy import conftest +from rpython.conftest import option from rpython.rtyper.lltypesystem import llmemory, lltype from rpython.rlib.objectmodel import ComputedIntSymbolic @@ -97,7 +97,7 @@ t = Translation(f) t.rtype() - if conftest.option.view: + if option.view: t.view() too_early = False fn = compile(f, [int]) diff --git a/rpython/translator/cli/option.py b/rpython/translator/cli/option.py --- a/rpython/translator/cli/option.py +++ b/rpython/translator/cli/option.py @@ -1,4 +1,4 @@ -from pypy.conftest import option +from rpython.conftest import option _defaultopt = dict(wd = False, source = False, nostop = False, stdout = False) diff --git a/rpython/translator/jvm/option.py b/rpython/translator/jvm/option.py --- a/rpython/translator/jvm/option.py +++ b/rpython/translator/jvm/option.py @@ -1,4 +1,4 @@ -from pypy.conftest import option +from rpython.conftest import option # Not sure why this is needed. Sure that it shouldn't be, even. _default_values = { diff --git a/rpython/translator/oosupport/test/test_treebuilder.py b/rpython/translator/oosupport/test/test_treebuilder.py --- a/rpython/translator/oosupport/test/test_treebuilder.py +++ b/rpython/translator/oosupport/test/test_treebuilder.py @@ -2,7 +2,7 @@ from rpython.rtyper.llinterp import LLInterpreter from rpython.translator.translator import TranslationContext, graphof from rpython.translator.oosupport.treebuilder import build_trees, SubOperation -from pypy.conftest import option +from rpython.conftest import option from rpython.rtyper.test.test_rlist import BaseTestRlist from rpython.rtyper.test.tool import BaseRtypingTest, OORtypeMixin from rpython.rtyper.test.test_llinterp import get_interpreter diff --git a/rpython/translator/test/test_exceptiontransform.py b/rpython/translator/test/test_exceptiontransform.py --- a/rpython/translator/test/test_exceptiontransform.py +++ b/rpython/translator/test/test_exceptiontransform.py @@ -5,14 +5,14 @@ from rpython.flowspace.model import summary from rpython.rtyper.test.test_llinterp import get_interpreter from rpython.translator.backendopt.all import backend_optimizations -from pypy import conftest +from rpython.conftest import option import sys def check_debug_build(): - # the 'not conftest.option.view' is because debug builds rarely + # the 'not option.view' is because debug builds rarely # have pygame, so if you want to see the graphs pass --view and # don't be surprised when the test then passes when it shouldn't. - if not hasattr(sys, 'gettotalrefcount') and not conftest.option.view: + if not hasattr(sys, 'gettotalrefcount') and not option.view: py.test.skip("test needs a debug build of Python") _already_transformed = {} @@ -33,7 +33,7 @@ t = TranslationContext() t.buildannotator().build_types(fn, inputtypes) t.buildrtyper(type_system=self.type_system).specialize() - if conftest.option.view: + if option.view: t.view() if backendopt: backend_optimizations(t) @@ -41,7 +41,7 @@ etrafo = exceptiontransform.ExceptionTransformer(t) etrafo.create_exception_handling(g) join_blocks(g) - if conftest.option.view: + if option.view: t.view() return t, g diff --git a/rpython/translator/test/test_simplify.py b/rpython/translator/test/test_simplify.py --- a/rpython/translator/test/test_simplify.py +++ b/rpython/translator/test/test_simplify.py @@ -4,7 +4,7 @@ from rpython.translator.simplify import (get_graph, transform_dead_op_vars, desugar_isinstance) from rpython.flowspace.model import Block, Constant, summary -from pypy import conftest +from rpython.conftest import option def translate(func, argtypes, backend_optimize=True): t = TranslationContext() @@ -12,7 +12,7 @@ t.buildrtyper().specialize() if backend_optimize: backend_optimizations(t) - if conftest.option.view: + if option.view: t.view() return graphof(t, func), t @@ -251,7 +251,7 @@ def check(self, f1, expected): t = TranslationContext(list_comprehension_operations=True) graph = t.buildflowgraph(f1) - if conftest.option.view: + if option.view: graph.show() assert summary(graph) == expected @@ -333,11 +333,11 @@ from rpython.rtyper.llinterp import LLInterpreter t = TranslationContext(list_comprehension_operations=True) t.buildannotator().build_types(func, argtypes) - if conftest.option.view: + if option.view: t.view() t.buildrtyper(self.typesystem).specialize() backend_optimizations(t) - if conftest.option.view: + if option.view: t.view() graph = graphof(t, func) interp = LLInterpreter(t.rtyper) diff --git a/rpython/translator/test/test_stackcheck.py b/rpython/translator/test/test_stackcheck.py --- a/rpython/translator/test/test_stackcheck.py +++ b/rpython/translator/test/test_stackcheck.py @@ -1,4 +1,4 @@ -from pypy import conftest +from rpython.conftest import option from rpython.translator.translator import TranslationContext, graphof from rpython.translator.backendopt.all import backend_optimizations from rpython.translator.transform import insert_ll_stackcheck @@ -72,7 +72,7 @@ n = insert_ll_stackcheck(t) t.checkgraphs() assert n == 1 - if conftest.option.view: + if option.view: t.view() check(graphof(t, f), 'f') @@ -90,7 +90,7 @@ exctransf = t.getexceptiontransformer() f_graph = graphof(t, f) exctransf.create_exception_handling(f_graph) - if conftest.option.view: + if option.view: f_graph.show() check(f_graph, 'f') @@ -101,7 +101,7 @@ gctransf = GCTransform(t) gctransf.transform_graph(f_graph) - if conftest.option.view: + if option.view: f_graph.show() relevant = check(f_graph, 'f') for p in relevant: From noreply at buildbot.pypy.org Sat Jan 12 21:50:17 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sat, 12 Jan 2013 21:50:17 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed a few missing renames Message-ID: <20130112205017.DD2CA1C0343@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60004:47c0f36f9b99 Date: 2013-01-12 21:49 +0100 http://bitbucket.org/pypy/pypy/changeset/47c0f36f9b99/ Log: Fixed a few missing renames diff --git a/rpython/rtyper/lltypesystem/ll2ctypes.py b/rpython/rtyper/lltypesystem/ll2ctypes.py --- a/rpython/rtyper/lltypesystem/ll2ctypes.py +++ b/rpython/rtyper/lltypesystem/ll2ctypes.py @@ -785,7 +785,7 @@ return callback_internal(*cargs) except: import sys - #if conftest.option.usepdb: + #if option.usepdb: # import pdb; pdb.post_mortem(sys.exc_traceback) global _callback_exc_info _callback_exc_info = sys.exc_info() diff --git a/rpython/rtyper/test/test_llinterp.py b/rpython/rtyper/test/test_llinterp.py --- a/rpython/rtyper/test/test_llinterp.py +++ b/rpython/rtyper/test/test_llinterp.py @@ -43,7 +43,7 @@ a = t.buildannotator(policy=policy) timelog("annotating", a.build_types, func, argtypes, main_entry_point=True) if viewbefore == 'auto': - viewbefore = getattr(conftest.option, 'view', False) + viewbefore = getattr(option, 'view', False) if viewbefore: a.simplify() t.view() @@ -98,7 +98,7 @@ if len(_lastinterpreted) >= 4: del _tcache[_lastinterpreted.pop(0)] if view == 'auto': - view = getattr(conftest.option, 'view', False) + view = getattr(option, 'view', False) if view: t.view() return interp, graph diff --git a/rpython/rtyper/test/test_rvirtualizable2.py b/rpython/rtyper/test/test_rvirtualizable2.py --- a/rpython/rtyper/test/test_rvirtualizable2.py +++ b/rpython/rtyper/test/test_rvirtualizable2.py @@ -163,7 +163,7 @@ op_getfield = block.operations[-1] assert op_getfield.opname in ('getfield', 'oogetfield') funcptr = self.replace_force_virtualizable(rtyper, [graph]) - if getattr(conftest.option, 'view', False): + if getattr(option, 'view', False): graph.show() op_promote = block.operations[-2] op_getfield = block.operations[-1] diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py --- a/rpython/translator/c/test/test_newgc.py +++ b/rpython/translator/c/test/test_newgc.py @@ -68,7 +68,7 @@ for fullname in dir(cls): if not fullname.startswith('define'): continue - keyword = conftest.option.keyword + keyword = option.keyword if keyword.startswith('test_'): keyword = keyword[len('test_'):] if keyword not in fullname: From noreply at buildbot.pypy.org Sat Jan 12 23:44:04 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 12 Jan 2013 23:44:04 +0100 (CET) Subject: [pypy-commit] pypy default: Translation fix Message-ID: <20130112224404.7BA481C015D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r60005:0f1e91da6cb2 Date: 2013-01-12 23:47 +0100 http://bitbucket.org/pypy/pypy/changeset/0f1e91da6cb2/ Log: Translation fix diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py --- a/pypy/module/signal/interp_signal.py +++ b/pypy/module/signal/interp_signal.py @@ -40,6 +40,7 @@ includes = ['stdlib.h', 'src/signals.h'] if sys.platform != 'win32': includes.append('sys/time.h') +WIN32 = sys.platform == 'win32' cdir = py.path.local(autopath.pypydir).join('translator', 'c') @@ -236,7 +237,7 @@ None -- if an unknown handler is in effect (XXX UNIMPLEMENTED) anything else -- the callable Python object used as a handler """ - if sys.platform == 'win32': + if WIN32: check_signum_exists(space, signum) else: check_signum_in_range(space, signum) From noreply at buildbot.pypy.org Sun Jan 13 01:37:39 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sun, 13 Jan 2013 01:37:39 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: No more pypy imports. Also I hate mercurial now. Message-ID: <20130113003739.D9DD41C11FD@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60006:1fb77e052760 Date: 2013-01-13 01:37 +0100 http://bitbucket.org/pypy/pypy/changeset/1fb77e052760/ Log: No more pypy imports. Also I hate mercurial now. diff --git a/rpython/rtyper/lltypesystem/ll2ctypes.py b/rpython/rtyper/lltypesystem/ll2ctypes.py --- a/rpython/rtyper/lltypesystem/ll2ctypes.py +++ b/rpython/rtyper/lltypesystem/ll2ctypes.py @@ -785,7 +785,7 @@ return callback_internal(*cargs) except: import sys - #if option.usepdb: + #if conftest.option.usepdb: # import pdb; pdb.post_mortem(sys.exc_traceback) global _callback_exc_info _callback_exc_info = sys.exc_info() diff --git a/rpython/rtyper/test/test_llinterp.py b/rpython/rtyper/test/test_llinterp.py --- a/rpython/rtyper/test/test_llinterp.py +++ b/rpython/rtyper/test/test_llinterp.py @@ -43,7 +43,7 @@ a = t.buildannotator(policy=policy) timelog("annotating", a.build_types, func, argtypes, main_entry_point=True) if viewbefore == 'auto': - viewbefore = getattr(option, 'view', False) + viewbefore = getattr(conftest.option, 'view', False) if viewbefore: a.simplify() t.view() @@ -98,7 +98,7 @@ if len(_lastinterpreted) >= 4: del _tcache[_lastinterpreted.pop(0)] if view == 'auto': - view = getattr(option, 'view', False) + view = getattr(conftest.option, 'view', False) if view: t.view() return interp, graph diff --git a/rpython/rtyper/test/test_rvirtualizable2.py b/rpython/rtyper/test/test_rvirtualizable2.py --- a/rpython/rtyper/test/test_rvirtualizable2.py +++ b/rpython/rtyper/test/test_rvirtualizable2.py @@ -163,7 +163,7 @@ op_getfield = block.operations[-1] assert op_getfield.opname in ('getfield', 'oogetfield') funcptr = self.replace_force_virtualizable(rtyper, [graph]) - if getattr(option, 'view', False): + if getattr(conftest.option, 'view', False): graph.show() op_promote = block.operations[-2] op_getfield = block.operations[-1] diff --git a/rpython/rtyper/tool/rffi_platform.py b/rpython/rtyper/tool/rffi_platform.py --- a/rpython/rtyper/tool/rffi_platform.py +++ b/rpython/rtyper/tool/rffi_platform.py @@ -11,7 +11,6 @@ from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.translator.platform import CompilationError from rpython.tool.udir import udir -from pypy.conftest import pypydir from rpython.rlib.rarithmetic import r_uint, r_longlong, r_ulonglong, intmask # ____________________________________________________________ @@ -738,7 +737,10 @@ # ____________________________________________________________ -PYPY_EXTERNAL_DIR = py.path.local(pypydir).join('..', '..') +from os.path import dirname +import rpython + +PYPY_EXTERNAL_DIR = py.path.local(dirname(rpython.__file__)).join('..', '..') # XXX make this configurable if sys.platform == 'win32': for libdir in [ diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py --- a/rpython/translator/c/genc.py +++ b/rpython/translator/c/genc.py @@ -401,7 +401,7 @@ ('profopt', '', [ '$(MAKENOPROF)', '$(MAKE) CFLAGS="-fprofile-generate $(CFLAGS)" LDFLAGS="-fprofile-generate $(LDFLAGS)" $(TARGET)', - 'cd $(PYPYDIR)/translator/goal && $(ABS_TARGET) $(PROFOPT)', + 'cd $(RPYDIR)/translator/goal && $(ABS_TARGET) $(PROFOPT)', '$(MAKE) clean_noprof', '$(MAKE) CFLAGS="-fprofile-use $(CFLAGS)" LDFLAGS="-fprofile-use $(LDFLAGS)" $(TARGET)'])) for rule in rules: @@ -452,22 +452,22 @@ 'cmd /c $(MASM) /nologo /Cx /Cp /Zm /coff /Fo$@ /c $< $(INCLUDEDIRS)') mk.rule('.c.gcmap', '', ['$(CC) /nologo $(ASM_CFLAGS) /c /FAs /Fa$*.s $< $(INCLUDEDIRS)', - 'cmd /c $(PYTHON) $(PYPYDIR)/translator/c/gcc/trackgcroot.py -fmsvc -t $*.s > $@'] + 'cmd /c $(PYTHON) $(RPYDIR)/translator/c/gcc/trackgcroot.py -fmsvc -t $*.s > $@'] ) mk.rule('gcmaptable.c', '$(GCMAPFILES)', - 'cmd /c $(PYTHON) $(PYPYDIR)/translator/c/gcc/trackgcroot.py -fmsvc $(GCMAPFILES) > $@') + 'cmd /c $(PYTHON) $(RPYDIR)/translator/c/gcc/trackgcroot.py -fmsvc $(GCMAPFILES) > $@') else: mk.definition('OBJECTS', '$(ASMLBLFILES) gcmaptable.s') mk.rule('%.s', '%.c', '$(CC) $(CFLAGS) $(CFLAGSEXTRA) -frandom-seed=$< -o $@ -S $< $(INCLUDEDIRS)') mk.rule('%.lbl.s %.gcmap', '%.s', [ - '$(PYTHON) $(PYPYDIR)/translator/c/gcc/trackgcroot.py ' + '$(PYTHON) $(RPYDIR)/translator/c/gcc/trackgcroot.py ' '-t $< > $*.gctmp', 'mv $*.gctmp $*.gcmap']) mk.rule('gcmaptable.s', '$(GCMAPFILES)', [ - '$(PYTHON) $(PYPYDIR)/translator/c/gcc/trackgcroot.py ' + '$(PYTHON) $(RPYDIR)/translator/c/gcc/trackgcroot.py ' '$(GCMAPFILES) > $@.tmp', 'mv $@.tmp $@']) mk.rule('.PRECIOUS', '%.s', "# don't remove .s files if Ctrl-C'ed") diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py --- a/rpython/translator/c/test/test_newgc.py +++ b/rpython/translator/c/test/test_newgc.py @@ -68,7 +68,7 @@ for fullname in dir(cls): if not fullname.startswith('define'): continue - keyword = option.keyword + keyword = conftest.option.keyword if keyword.startswith('test_'): keyword = keyword[len('test_'):] if keyword not in fullname: diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py --- a/rpython/translator/platform/posix.py +++ b/rpython/translator/platform/posix.py @@ -2,9 +2,11 @@ import py, os, sys -from pypy.conftest import pypydir from rpython.translator.platform import Platform, log, _run_subprocess +import rpython +rpydir = os.path.dirname(rpython.__file__) + class BasePosix(Platform): exe_ext = '' make_cmd = 'make' @@ -88,7 +90,7 @@ if path is None: path = cfiles[0].dirpath() - pypypath = py.path.local(pypydir) + rpypath = py.path.local(rpydir) if exe_name is None: exe_name = cfiles[0].new(ext=self.exe_ext) @@ -116,11 +118,11 @@ m.exe_name = exe_name m.eci = eci - def pypyrel(fpath): + def rpyrel(fpath): lpath = py.path.local(fpath) - rel = lpath.relto(pypypath) + rel = lpath.relto(rpypath) if rel: - return os.path.join('$(PYPYDIR)', rel) + return os.path.join('$(RPYDIR)', rel) m_dir = m.makefile_dir if m_dir == lpath: return '.' @@ -132,14 +134,14 @@ rel_ofiles = [rel_cfile[:rel_cfile.rfind('.')]+'.o' for rel_cfile in rel_cfiles] m.cfiles = rel_cfiles - rel_includedirs = [pypyrel(incldir) for incldir in + rel_includedirs = [rpyrel(incldir) for incldir in self.preprocess_include_dirs(eci.include_dirs)] - rel_libdirs = [pypyrel(libdir) for libdir in + rel_libdirs = [rpyrel(libdir) for libdir in self.preprocess_library_dirs(eci.library_dirs)] m.comment('automatically generated makefile') definitions = [ - ('PYPYDIR', '"%s"' % pypydir), + ('RPYDIR', '"%s"' % rpydir), ('TARGET', target_name), ('DEFAULT_TARGET', exe_name.basename), ('SOURCES', rel_cfiles), diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -7,6 +7,9 @@ from rpython.translator.platform import log, _run_subprocess from rpython.translator.platform import Platform, posix +import rpython +rpydir = os.path.dirname(rpython.__file__) + def _get_compiler_type(cc, x64_flag): import subprocess if not cc: @@ -252,7 +255,7 @@ if path is None: path = cfiles[0].dirpath() - pypypath = py.path.local(pypydir) + rpypath = py.path.local(rpydir) if exe_name is None: exe_name = cfiles[0].new(ext=self.exe_ext) @@ -279,10 +282,10 @@ else: target_name = exe_name.basename - def pypyrel(fpath): - rel = py.path.local(fpath).relto(pypypath) + def rpyrel(fpath): + rel = py.path.local(fpath).relto(rpypath) if rel: - return os.path.join('$(PYPYDIR)', rel) + return os.path.join('$(RPYDIR)', rel) else: return fpath @@ -290,11 +293,11 @@ rel_ofiles = [rel_cfile[:rel_cfile.rfind('.')]+'.obj' for rel_cfile in rel_cfiles] m.cfiles = rel_cfiles - rel_includedirs = [pypyrel(incldir) for incldir in eci.include_dirs] + rel_includedirs = [rpyrel(incldir) for incldir in eci.include_dirs] m.comment('automatically generated makefile') definitions = [ - ('PYPYDIR', pypydir), + ('RPYDIR', rpydir), ('TARGET', target_name), ('DEFAULT_TARGET', exe_name.basename), ('SOURCES', rel_cfiles), diff --git a/lib_pypy/_marshal.py b/rpython/translator/sandbox/_marshal.py copy from lib_pypy/_marshal.py copy to rpython/translator/sandbox/_marshal.py --- a/lib_pypy/_marshal.py +++ b/rpython/translator/sandbox/_marshal.py @@ -1,3 +1,4 @@ +# Copy of lib_pypy/_marshal.py needed by sandlib """Internal Python object serialization This module contains functions that can read and write Python values in a binary format. The format is specific to Python, but independent of machine architecture issues (e.g., you can write a Python value to a file on a PC, transport the file to a Sun, and read it back there). Details of the format may change between Python versions. diff --git a/rpython/translator/sandbox/sandlib.py b/rpython/translator/sandbox/sandlib.py --- a/rpython/translator/sandbox/sandlib.py +++ b/rpython/translator/sandbox/sandlib.py @@ -34,9 +34,7 @@ # load(). Also, marshal.load(f) blocks with the GIL held when # f is a pipe with no data immediately avaialble, preventing the # _waiting_thread to run. -import pypy -marshal = py.path.local(pypy.__file__).join('..', '..', 'lib_pypy', - 'marshal.py').pyimport() +from rpython.translator.sandbox import _marshal as marshal # Non-marshal result types RESULTTYPE_STATRESULT = object() diff --git a/rpython/translator/translator.py b/rpython/translator/translator.py --- a/rpython/translator/translator.py +++ b/rpython/translator/translator.py @@ -11,7 +11,6 @@ from rpython.flowspace.objspace import FlowObjSpace from rpython.tool.ansi_print import ansi_log from rpython.tool.sourcetools import nice_repr_for_func -from pypy.config.pypyoption import pypy_optiondescription from rpython.config.translationoption import get_combined_translation_config from rpython.config.translationoption import get_platform import py From noreply at buildbot.pypy.org Sun Jan 13 01:44:08 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sun, 13 Jan 2013 01:44:08 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Removed unused import Message-ID: <20130113004408.805D41C015D@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60007:458ff95d794d Date: 2013-01-13 01:43 +0100 http://bitbucket.org/pypy/pypy/changeset/458ff95d794d/ Log: Removed unused import diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -2,7 +2,6 @@ import py, os, sys, re -from pypy.conftest import pypydir from rpython.translator.platform import CompilationError from rpython.translator.platform import log, _run_subprocess from rpython.translator.platform import Platform, posix From noreply at buildbot.pypy.org Sun Jan 13 02:00:53 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sun, 13 Jan 2013 02:00:53 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed stuff again Message-ID: <20130113010053.813711C015D@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60008:dc531ba31f7f Date: 2013-01-13 02:00 +0100 http://bitbucket.org/pypy/pypy/changeset/dc531ba31f7f/ Log: Fixed stuff again diff --git a/rpython/rtyper/lltypesystem/ll2ctypes.py b/rpython/rtyper/lltypesystem/ll2ctypes.py --- a/rpython/rtyper/lltypesystem/ll2ctypes.py +++ b/rpython/rtyper/lltypesystem/ll2ctypes.py @@ -785,7 +785,7 @@ return callback_internal(*cargs) except: import sys - #if conftest.option.usepdb: + #if option.usepdb: # import pdb; pdb.post_mortem(sys.exc_traceback) global _callback_exc_info _callback_exc_info = sys.exc_info() diff --git a/rpython/rtyper/test/test_llinterp.py b/rpython/rtyper/test/test_llinterp.py --- a/rpython/rtyper/test/test_llinterp.py +++ b/rpython/rtyper/test/test_llinterp.py @@ -43,7 +43,7 @@ a = t.buildannotator(policy=policy) timelog("annotating", a.build_types, func, argtypes, main_entry_point=True) if viewbefore == 'auto': - viewbefore = getattr(conftest.option, 'view', False) + viewbefore = getattr(option, 'view', False) if viewbefore: a.simplify() t.view() @@ -98,7 +98,7 @@ if len(_lastinterpreted) >= 4: del _tcache[_lastinterpreted.pop(0)] if view == 'auto': - view = getattr(conftest.option, 'view', False) + view = getattr(option, 'view', False) if view: t.view() return interp, graph diff --git a/rpython/rtyper/test/test_rvirtualizable2.py b/rpython/rtyper/test/test_rvirtualizable2.py --- a/rpython/rtyper/test/test_rvirtualizable2.py +++ b/rpython/rtyper/test/test_rvirtualizable2.py @@ -163,7 +163,7 @@ op_getfield = block.operations[-1] assert op_getfield.opname in ('getfield', 'oogetfield') funcptr = self.replace_force_virtualizable(rtyper, [graph]) - if getattr(conftest.option, 'view', False): + if getattr(option, 'view', False): graph.show() op_promote = block.operations[-2] op_getfield = block.operations[-1] diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py --- a/rpython/translator/c/test/test_newgc.py +++ b/rpython/translator/c/test/test_newgc.py @@ -68,7 +68,7 @@ for fullname in dir(cls): if not fullname.startswith('define'): continue - keyword = conftest.option.keyword + keyword = option.keyword if keyword.startswith('test_'): keyword = keyword[len('test_'):] if keyword not in fullname: From noreply at buildbot.pypy.org Sun Jan 13 04:35:25 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sun, 13 Jan 2013 04:35:25 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed targetpypystandalone Message-ID: <20130113033525.0972A1C054C@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60009:ecb6277f781f Date: 2013-01-13 03:23 +0100 http://bitbucket.org/pypy/pypy/changeset/ecb6277f781f/ Log: Fixed targetpypystandalone diff --git a/goal/targetpypystandalone.py b/goal/targetpypystandalone.py --- a/goal/targetpypystandalone.py +++ b/goal/targetpypystandalone.py @@ -4,9 +4,9 @@ from pypy.interpreter import gateway from pypy.interpreter.error import OperationError -from rpython.translator.goal.ann_override import PyPyAnnotatorPolicy -from pypy.config.config import Config, to_optparse, make_dict, SUPPRESS_USAGE -from pypy.config.config import ConflictConfigError +from pypy.tool.ann_override import PyPyAnnotatorPolicy +from rpython.config.config import Config, to_optparse, make_dict, SUPPRESS_USAGE +from rpython.config.config import ConflictConfigError from pypy.tool.option import make_objspace from pypy.conftest import pypydir diff --git a/goal/ann_override.py b/pypy/tool/ann_override.py rename from goal/ann_override.py rename to pypy/tool/ann_override.py diff --git a/pypy/tool/pypyjit.py b/pypy/tool/pypyjit.py --- a/pypy/tool/pypyjit.py +++ b/pypy/tool/pypyjit.py @@ -98,7 +98,7 @@ code.exec_code(space, w_dict, w_dict) def test_run_translation(): - from rpython.translator.goal.ann_override import PyPyAnnotatorPolicy + from pypy.tool.ann_override import PyPyAnnotatorPolicy from rpython.rtyper.test.test_llinterp import get_interpreter # first annotate and rtype From noreply at buildbot.pypy.org Sun Jan 13 04:38:28 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Sun, 13 Jan 2013 04:38:28 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: try refactoring this to be rpython Message-ID: <20130113033828.D93A81C054C@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: split-rpython Changeset: r60010:c4827b57dab7 Date: 2013-01-12 19:38 -0800 http://bitbucket.org/pypy/pypy/changeset/c4827b57dab7/ Log: try refactoring this to be rpython diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -20,23 +20,23 @@ # XXX Hack to seperate rpython and pypy def addr_as_object(addr, fd, space): - if hasattr(addr, 'family') and addr.family == rsocket.AF_INET: + if isinstance(addr, rsocket.INETAddress): return space.newtuple([space.wrap(addr.get_host()), space.wrap(addr.get_port())]) - if hasattr(addr, 'family') and addr.family == rsocket.AF_INET6: + elif isinstance(addr, rsocket.INET6Address): return space.newtuple([space.wrap(addr.get_host()), space.wrap(addr.get_port()), space.wrap(addr.get_flowinfo()), space.wrap(addr.get_scope_id())]) - if hasattr(addr, 'family') and hasattr(rsocket, 'AF_PACKET') and addr.family == rsocket.AF_PACKET: + elif rsocket.HAS_AF_PACKET and isinstance(addr, rsocket.PacketAddress): return space.newtuple([space.wrap(addr.get_ifname(fd)), space.wrap(addr.get_protocol()), space.wrap(addr.get_pkttype()), space.wrap(addr.get_hatype()), space.wrap(addr.get_addr())]) - if hasattr(addr, 'family') and hasattr(rsocket, 'AF_UNIX') and addr.family == rsocket.AF_UNIX: + elif rsocket.HAS_AF_UNIX and isinstance(addr, rsocket.UNIXAddress): return space.wrap(addr.get_path()) - if hasattr(addr, 'family') and hasattr(rsocket, 'AF_NETLINK') and addr.family == rsocket.AF_NETLINK: + elif rsocket.HAS_AF_NETLINK and isinstance(addr, rsocket.NETLINKAddress): return space.newtuple([space.wrap(addr.get_pid()), space.wrap(addr.get_groups())]) # If we don't know the address family, don't raise an diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py --- a/rpython/rlib/rsocket.py +++ b/rpython/rlib/rsocket.py @@ -181,7 +181,8 @@ # ____________________________________________________________ -if 'AF_PACKET' in constants: +HAS_AF_PACKET = 'AF_PACKET' in constants +if HAS_AF_PACKET: class PacketAddress(Address): family = AF_PACKET struct = _c.sockaddr_ll @@ -350,7 +351,8 @@ # ____________________________________________________________ -if 'AF_UNIX' in constants: +HAS_AF_UNIX = 'AF_UNIX' in constants +if HAS_AF_UNIX: class UNIXAddress(Address): family = AF_UNIX struct = _c.sockaddr_un @@ -399,7 +401,8 @@ return (isinstance(other, UNIXAddress) and self.get_path() == other.get_path()) -if 'AF_NETLINK' in constants: +HAS_AF_NETLINK = 'AF_NETLINK' in constants +if HAS_AF_NETLINK: class NETLINKAddress(Address): family = AF_NETLINK struct = _c.sockaddr_nl From noreply at buildbot.pypy.org Sun Jan 13 04:38:30 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Sun, 13 Jan 2013 04:38:30 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: merged upstream Message-ID: <20130113033830.292841C054C@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: split-rpython Changeset: r60011:3e57ac34033b Date: 2013-01-12 19:38 -0800 http://bitbucket.org/pypy/pypy/changeset/3e57ac34033b/ Log: merged upstream diff --git a/goal/targetpypystandalone.py b/goal/targetpypystandalone.py --- a/goal/targetpypystandalone.py +++ b/goal/targetpypystandalone.py @@ -4,9 +4,9 @@ from pypy.interpreter import gateway from pypy.interpreter.error import OperationError -from rpython.translator.goal.ann_override import PyPyAnnotatorPolicy -from pypy.config.config import Config, to_optparse, make_dict, SUPPRESS_USAGE -from pypy.config.config import ConflictConfigError +from pypy.tool.ann_override import PyPyAnnotatorPolicy +from rpython.config.config import Config, to_optparse, make_dict, SUPPRESS_USAGE +from rpython.config.config import ConflictConfigError from pypy.tool.option import make_objspace from pypy.conftest import pypydir diff --git a/goal/ann_override.py b/pypy/tool/ann_override.py rename from goal/ann_override.py rename to pypy/tool/ann_override.py diff --git a/pypy/tool/pypyjit.py b/pypy/tool/pypyjit.py --- a/pypy/tool/pypyjit.py +++ b/pypy/tool/pypyjit.py @@ -98,7 +98,7 @@ code.exec_code(space, w_dict, w_dict) def test_run_translation(): - from rpython.translator.goal.ann_override import PyPyAnnotatorPolicy + from pypy.tool.ann_override import PyPyAnnotatorPolicy from rpython.rtyper.test.test_llinterp import get_interpreter # first annotate and rtype From noreply at buildbot.pypy.org Sun Jan 13 06:50:18 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sun, 13 Jan 2013 06:50:18 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed translation error in interp_socket Message-ID: <20130113055018.C6B281C029E@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60012:c39ffac85542 Date: 2013-01-13 06:32 +0100 http://bitbucket.org/pypy/pypy/changeset/c39ffac85542/ Log: Fixed translation error in interp_socket diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -54,14 +54,14 @@ # XXX a bit of code duplication def fill_from_object(addr, space, w_address): from rpython.rlib import _rsocket_rffi as _c - if hasattr(addr, 'family') and addr.family == rsocket.AF_INET: + if isinstance(addr, rsocket.INETAddress): _, w_port = space.unpackiterable(w_address, 2) port = space.int_w(w_port) port = make_ushort_port(space, port) a = addr.lock(_c.sockaddr_in) rffi.setintfield(a, 'c_sin_port', rsocket.htons(port)) addr.unlock() - elif hasattr(addr, 'family') and addr.family == rsocket.AF_INET6: + elif isinstance(addr, rsocket.INET6Address): pieces_w = space.unpackiterable(w_address) if not (2 <= len(pieces_w) <= 4): raise RSocketError("AF_INET6 address must be a tuple of length 2 " @@ -109,9 +109,9 @@ "flowinfo must be 0-1048575.")) flowinfo = rffi.cast(lltype.Unsigned, flowinfo) return rsocket.INET6Address(host, port, flowinfo, scope_id) - if 'AF_UNIX' in rsocket.constants and family == rsocket.AF_UNIX: + if rsocket.HAS_AF_UNIX and family == rsocket.AF_UNIX: return rsocket.UNIXAddress(space.str_w(w_address)) - if 'AF_NETLINK' in rsocket.constants and family == rsocket.AF_NETLINK: + if rsocket.HAS_AF_NETLINK and family == rsocket.AF_NETLINK: w_pid, w_groups = space.unpackiterable(w_address, 2) return rsocket.NETLINKAddress(space.uint_w(w_pid), space.uint_w(w_groups)) raise RSocketError("unknown address family") From noreply at buildbot.pypy.org Sun Jan 13 07:52:09 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sun, 13 Jan 2013 07:52:09 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Merge with default Message-ID: <20130113065209.784EF1C1322@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60013:9f3e3485ed27 Date: 2013-01-13 07:49 +0100 http://bitbucket.org/pypy/pypy/changeset/9f3e3485ed27/ Log: Merge with default diff too long, truncating to 2000 out of 10188 lines diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -28,7 +28,7 @@ DEALINGS IN THE SOFTWARE. -PyPy Copyright holders 2003-2012 +PyPy Copyright holders 2003-2013 ----------------------------------- Except when otherwise stated (look for LICENSE files or information at diff --git a/goal/targetpypystandalone.py b/goal/targetpypystandalone.py --- a/goal/targetpypystandalone.py +++ b/goal/targetpypystandalone.py @@ -29,6 +29,11 @@ w_call_startup_gateway = space.wrap(gateway.interp2app(call_startup)) withjit = space.config.objspace.usemodules.pypyjit + if withjit: + from pypy.module.pypyjit.interp_jit import callback_hook + from pypy.rlib import objectmodel + objectmodel.register_around_callback_hook(callback_hook) + def entry_point(argv): if withjit: from rpython.jit.backend.hlinfo import highleveljitinfo diff --git a/goal/test2/test_app_main.py b/goal/test2/test_app_main.py --- a/goal/test2/test_app_main.py +++ b/goal/test2/test_app_main.py @@ -376,6 +376,30 @@ child.expect('Traceback') child.expect('NameError') + def test_pythonstartup_file1(self, monkeypatch): + monkeypatch.setenv('PYTHONPATH', None) + monkeypatch.setenv('PYTHONSTARTUP', demo_script) + child = self.spawn([]) + child.expect('File: [^\n]+\.py') + child.expect('goodbye') + child.expect('>>> ') + child.sendline('[myvalue]') + child.expect(re.escape('[42]')) + child.expect('>>> ') + child.sendline('__file__') + child.expect('Traceback') + child.expect('NameError') + + def test_pythonstartup_file2(self, monkeypatch): + monkeypatch.setenv('PYTHONPATH', None) + monkeypatch.setenv('PYTHONSTARTUP', crashing_demo_script) + child = self.spawn([]) + child.expect('Traceback') + child.expect('>>> ') + child.sendline('__file__') + child.expect('Traceback') + child.expect('NameError') + def test_ignore_python_startup(self): old = os.environ.get('PYTHONSTARTUP', '') try: diff --git a/lib-python/2.7/sre_parse.py b/lib-python/2.7/sre_parse.py --- a/lib-python/2.7/sre_parse.py +++ b/lib-python/2.7/sre_parse.py @@ -16,6 +16,12 @@ from sre_constants import * +try: + from __pypy__ import newdict +except ImportError: + def newdict(tp): + return {} + SPECIAL_CHARS = ".\\[{()*+?^$|" REPEAT_CHARS = "*+?{" @@ -68,7 +74,7 @@ self.flags = 0 self.open = [] self.groups = 1 - self.groupdict = {} + self.groupdict = newdict("module") def opengroup(self, name=None): gid = self.groups self.groups = gid + 1 diff --git a/pypy/doc/Makefile b/pypy/doc/Makefile --- a/pypy/doc/Makefile +++ b/pypy/doc/Makefile @@ -32,38 +32,38 @@ -rm -rf $(BUILDDIR)/* html: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." pickle: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ @@ -73,7 +73,7 @@ @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/PyPy.qhc" latex: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @@ -81,26 +81,26 @@ "run these through (pdf)latex." man: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man" changes: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." doctest: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -38,7 +38,7 @@ # General information about the project. project = u'PyPy' -copyright = u'2011, The PyPy Project' +copyright = u'2013, The PyPy Project' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/pypy/doc/config/translation.no__thread.txt b/pypy/doc/config/translation.no__thread.txt --- a/pypy/doc/config/translation.no__thread.txt +++ b/pypy/doc/config/translation.no__thread.txt @@ -1,3 +1,3 @@ Don't use gcc __thread attribute for fast thread local storage -implementation . Increases the chance that moving the resulting +implementation. Increases the chance that moving the resulting executable to another same processor Linux machine will work. diff --git a/pypy/doc/index.rst b/pypy/doc/index.rst --- a/pypy/doc/index.rst +++ b/pypy/doc/index.rst @@ -67,7 +67,7 @@ .. _Python: http://docs.python.org/index.html .. _`more...`: architecture.html#mission-statement .. _`PyPy blog`: http://morepypy.blogspot.com/ -.. _`development bug/feature tracker`: https://codespeak.net/issue/pypy-dev/ +.. _`development bug/feature tracker`: https://bugs.pypy.org .. _here: http://tismerysoft.de/pypy/irc-logs/pypy .. _`Mercurial commit mailing list`: http://python.org/mailman/listinfo/pypy-commit .. _`development mailing list`: http://python.org/mailman/listinfo/pypy-dev diff --git a/pypy/doc/pypyconfig.py b/pypy/doc/pypyconfig.py --- a/pypy/doc/pypyconfig.py +++ b/pypy/doc/pypyconfig.py @@ -3,6 +3,10 @@ def setup(app): import sys, os sys.path.insert(0, os.path.abspath("../../")) + + #Autmatically calls make_cmdlline_overview + from pypy.doc.config import generate + from pypy.config import makerestdoc import py role = makerestdoc.register_config_role(py.path.local()) diff --git a/pypy/doc/test/test_whatsnew.py b/pypy/doc/test/test_whatsnew.py --- a/pypy/doc/test/test_whatsnew.py +++ b/pypy/doc/test/test_whatsnew.py @@ -29,7 +29,7 @@ merge() and \ branch(default)) and \ not branch(default)' % (startrev, endrev) - cmd = r"hg log -R '%s' -r '%s' --template '{branches}\n'" % (path, revset) + cmd = r'hg log -R "%s" -r "%s" --template "{branches}\n"' % (path, revset) out = getoutput(cmd) branches = set(map(str.strip, out.splitlines())) return branches diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -5,15 +5,38 @@ .. this is a revision shortly after release-2.0-beta1 .. startrev: 0e6161a009c6 +.. branch: callback-jit +Callbacks from C are now better JITted + +.. branch: remove-globals-in-jit + .. branch: length-hint Implement __lenght_hint__ according to PEP 424 +.. branch: numpypy-longdouble +Long double support for numpypy +.. branch: numpypy-real-as-view +Convert real, imag from ufuncs to views. This involves the beginning of +view() functionality + +.. branch: signatures +Improved RPython typing + +.. branch: rpython-bytearray +Rudimentary support for bytearray in RPython + .. branches we don't care about .. branch: autoreds +.. branch: reflex-support +.. branch: kill-faking +.. branch: improved_ebnfparse_error +.. branch: task-decorator .. branch: release-2.0-beta1 .. branch: remove-PYPY_NOT_MAIN_FILE +.. branch: missing-jit-operations + .. branch: fix-lookinside-iff-oopspec Fixed the interaction between two internal tools for controlling the JIT. diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -56,11 +56,24 @@ ---------------------------- On Windows, there is no standard place where to download, build and -install third-party libraries. We chose to install them in the parent +install third-party libraries. We recommend installing them in the parent directory of the pypy checkout. For example, if you installed pypy in ``d:\pypy\trunk\`` (This directory contains a README file), the base -directory is ``d:\pypy``. You may choose different values by setting the -INCLUDE, LIB and PATH (for DLLs) +directory is ``d:\pypy``. You must then set the +INCLUDE, LIB and PATH (for DLLs) environment variables appropriately. + +Abridged method (for -Ojit builds using Visual Studio 2008) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Download the versions of all the external packages +from +https://bitbucket.org/pypy/pypy/downloads/local.zip +Then expand it into the base directory (base_dir) and modify your environment to reflect this:: + + set PATH=\bin;%PATH% + set INCLUDE=\include;%INCLUDE% + set LIB=\lib;%LIB% + +Now you should be good to go. Read on for more information. The Boehm garbage collector ~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -584,7 +584,12 @@ python_startup, 'exec') exec co_python_startup in mainmodule.__dict__ + mainmodule.__file__ = python_startup run_toplevel(run_it) + try: + del mainmodule.__file__ + except (AttributeError, TypeError): + pass # Then we need a prompt. inspect = True else: diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -178,6 +178,11 @@ else: oparg = 0 + # note: the structure of the code here is such that it makes + # (after translation) a big "if/elif" chain, which is then + # turned into a switch(). It starts here: even if the first + # one is not an "if" but a "while" the effect is the same. + while opcode == self.opcodedesc.EXTENDED_ARG.index: opcode = ord(co_code[next_instr]) if opcode < self.HAVE_ARGUMENT: @@ -226,6 +231,8 @@ 'END_FINALLY', 'JUMP_ABSOLUTE'): continue # opcodes implemented above + # the following "if" is part of the big switch described + # above. if opcode == opdesc.index: # dispatch to the opcode method meth = getattr(self, opdesc.methodname) diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py --- a/pypy/interpreter/typedef.py +++ b/pypy/interpreter/typedef.py @@ -105,6 +105,13 @@ # typical usage (case 1 is the most common kind of app-level subclasses; # case 2 is the memory-saving kind defined with __slots__). # +# +----------------------------------------------------------------+ +# | NOTE: if withmapdict is enabled, the following doesn't apply! | +# | Map dicts can flexibly allow any slots/__dict__/__weakref__ to | +# | show up only when needed. In particular there is no way with | +# | mapdict to prevent some objects from being weakrefable. | +# +----------------------------------------------------------------+ +# # dict slots del weakrefable # # 1. Y N N Y UserDictWeakref diff --git a/pypy/module/README.txt b/pypy/module/README.txt --- a/pypy/module/README.txt +++ b/pypy/module/README.txt @@ -3,7 +3,7 @@ that require access to interpreter level. See here for more information: - http://codespeak.net/pypy/dist/pypy/doc/coding-guide.html#modules-in-pypy + http://doc.pypy.org/en/latest/coding-guide.html#modules-in-pypy ATTENTION: don't put any '.py' files directly into pypy/module because you can easily get import mixups on e.g. "import sys" diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py --- a/pypy/module/_cffi_backend/ccallback.py +++ b/pypy/module/_cffi_backend/ccallback.py @@ -5,8 +5,8 @@ from pypy.interpreter.error import OperationError, operationerrfmt from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.rlib.objectmodel import compute_unique_id, keepalive_until_here -from rpython.rlib import clibffi, rweakref, rgc -from rpython.rlib.rarithmetic import r_ulonglong +from rpython.rlib import clibffi, rweakref +from rpython.rlib import jit from pypy.module._cffi_backend.cdataobj import W_CData from pypy.module._cffi_backend.ctypefunc import SIZE_OF_FFI_ARG, BIG_ENDIAN @@ -77,6 +77,7 @@ space.wrap("expected a function ctype")) return ctype + @jit.unroll_safe def invoke(self, ll_args): space = self.space ctype = self.getfunctype() diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py --- a/pypy/module/_sre/interp_sre.py +++ b/pypy/module/_sre/interp_sre.py @@ -54,7 +54,7 @@ result = [-1] * (2 * num_groups) mark = ctx.match_marks while mark is not None: - index = mark.gid + index = jit.promote(mark.gid) if result[index] == -1: result[index] = mark.position mark = mark.prev @@ -90,7 +90,7 @@ # SRE_Pattern class class W_SRE_Pattern(Wrappable): - _immutable_fields_ = ["code", "flags", "num_groups"] + _immutable_fields_ = ["code", "flags", "num_groups", "w_groupindex"] def cannot_copy_w(self): space = self.space diff --git a/pypy/module/_weakref/test/test_weakref.py b/pypy/module/_weakref/test/test_weakref.py --- a/pypy/module/_weakref/test/test_weakref.py +++ b/pypy/module/_weakref/test/test_weakref.py @@ -114,19 +114,28 @@ class A(object): def __eq__(self, other): return True + def __ne__(self, other): + return False a1 = A() a2 = A() ref1 = _weakref.ref(a1) ref2 = _weakref.ref(a2) assert ref1 == ref2 + assert not (ref1 != ref2) + assert not (ref1 == []) + assert ref1 != [] del a1 gc.collect() assert not ref1 == ref2 assert ref1 != ref2 + assert not (ref1 == []) + assert ref1 != [] del a2 gc.collect() assert not ref1 == ref2 assert ref1 != ref2 + assert not (ref1 == []) + assert ref1 != [] def test_getweakrefs(self): import _weakref, gc @@ -298,6 +307,13 @@ if seen_callback: assert seen_callback == [True, True, True] + def test_type_weakrefable(self): + import _weakref, gc + w = _weakref.ref(list) + assert w() is list + gc.collect() + assert w() is list + class AppTestProxy(object): spaceconfig = dict(usemodules=('_weakref',)) @@ -435,6 +451,8 @@ class A(object): def __eq__(self, other): return True + def __ne__(self, other): + return False a = A() assert _weakref.ref(a) == a diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py --- a/pypy/module/micronumpy/arrayimpl/concrete.py +++ b/pypy/module/micronumpy/arrayimpl/concrete.py @@ -13,6 +13,7 @@ from rpython.rlib.debug import make_sure_not_resized class ConcreteArrayIterator(base.BaseArrayIterator): + _immutable_fields_ = ['dtype', 'skip', 'size'] def __init__(self, array): self.array = array self.offset = 0 @@ -21,10 +22,10 @@ self.size = array.size def setitem(self, elem): - self.array.setitem(self.offset, elem) + self.dtype.setitem(self.array, self.offset, elem) def getitem(self): - return self.array.getitem(self.offset) + return self.dtype.getitem(self.array, self.offset) def getitem_bool(self): return self.dtype.getitem_bool(self.array, self.offset) @@ -42,13 +43,16 @@ self.offset %= self.size class OneDimViewIterator(ConcreteArrayIterator): - def __init__(self, array): + ''' The view iterator dtype can be different from the + array.dtype, this is what makes it a View + ''' + def __init__(self, array, dtype, start, strides, shape): self.array = array - self.offset = array.start - self.skip = array.get_strides()[0] - self.dtype = array.dtype + self.dtype = dtype + self.offset = start + self.skip = strides[0] self.index = 0 - self.size = array.get_shape()[0] + self.size = shape[0] def next(self): self.offset += self.skip @@ -65,9 +69,13 @@ self.offset %= self.size class MultiDimViewIterator(ConcreteArrayIterator): - def __init__(self, array, start, strides, backstrides, shape): + ''' The view iterator dtype can be different from the + array.dtype, this is what makes it a View + ''' + def __init__(self, array, dtype, start, strides, backstrides, shape): self.indexes = [0] * len(shape) self.array = array + self.dtype = dtype self.shape = shape self.offset = start self.shapelen = len(shape) @@ -131,12 +139,13 @@ self.offset = array.start self.dim = dim self.array = array + self.dtype = array.dtype def setitem(self, elem): - self.array.setitem(self.offset, elem) + self.dtype.setitem(self.array, self.offset, elem) def getitem(self): - return self.array.getitem(self.offset) + return self.dtype.getitem(self.array, self.offset) @jit.unroll_safe def next(self): @@ -220,6 +229,32 @@ new_shape, self) else: return None + + def get_real(self): + strides = self.get_strides() + backstrides = self.get_backstrides() + if self.dtype.is_complex_type(): + dtype = self.dtype.float_type + return SliceArray(self.start, strides, backstrides, + self.get_shape(), self, dtype=dtype) + return SliceArray(self.start, strides, backstrides, + self.get_shape(), self) + + def get_imag(self): + strides = self.get_strides() + backstrides = self.get_backstrides() + if self.dtype.is_complex_type(): + dtype = self.dtype.float_type + return SliceArray(self.start + dtype.get_size(), strides, + backstrides, self.get_shape(), self, dtype=dtype) + if self.dtype.is_flexible_type(): + # numpy returns self for self.imag + return SliceArray(self.start, strides, backstrides, + self.get_shape(), self) + impl = NonWritableArray(self.get_shape(), self.dtype, self.order, strides, + backstrides) + impl.fill(self.dtype.box(0)) + return impl # -------------------- applevel get/setitem ----------------------- @@ -377,7 +412,7 @@ def create_dot_iter(self, shape, skip): r = calculate_dot_strides(self.get_strides(), self.get_backstrides(), shape, skip) - return MultiDimViewIterator(self, self.start, r[0], r[1], shape) + return MultiDimViewIterator(self, self.dtype, self.start, r[0], r[1], shape) def swapaxes(self, axis1, axis2): shape = self.get_shape()[:] @@ -411,7 +446,7 @@ r = calculate_broadcast_strides(self.get_strides(), self.get_backstrides(), self.get_shape(), shape) - return MultiDimViewIterator(self, 0, r[0], r[1], shape) + return MultiDimViewIterator(self, self.dtype, 0, r[0], r[1], shape) def fill(self, box): self.dtype.fill(self.storage, box, 0, self.size) @@ -424,6 +459,12 @@ self.order) return SliceArray(0, strides, backstrides, new_shape, self) +class NonWritableArray(ConcreteArray): + def descr_setitem(self, space, w_index, w_value): + raise OperationError(space.w_RuntimeError, space.wrap( + "array is not writable")) + + class SliceArray(BaseConcreteArray): def __init__(self, start, strides, backstrides, shape, parent, dtype=None): self.strides = strides @@ -448,11 +489,12 @@ r = calculate_broadcast_strides(self.get_strides(), self.get_backstrides(), self.get_shape(), shape) - return MultiDimViewIterator(self.parent, + return MultiDimViewIterator(self.parent, self.dtype, self.start, r[0], r[1], shape) if len(self.get_shape()) == 1: - return OneDimViewIterator(self) - return MultiDimViewIterator(self.parent, self.start, + return OneDimViewIterator(self.parent, self.dtype, self.start, + self.get_strides(), self.get_shape()) + return MultiDimViewIterator(self.parent, self.dtype, self.start, self.get_strides(), self.get_backstrides(), self.get_shape()) diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -136,7 +136,7 @@ return self.kind == SIGNEDLTR def is_complex_type(self): - return (self.num == 14 or self.num == 15 or self.num == 16) + return False def is_bool_type(self): return self.kind == BOOLLTR @@ -155,6 +155,19 @@ def get_size(self): return self.itemtype.get_element_size() +class W_ComplexDtype(W_Dtype): + + def __init__(self, itemtype, num, kind, name, char, w_box_type, + alternate_constructors=[], aliases=[], + fields=None, fieldnames=None, native=True, float_type=None): + W_Dtype.__init__(self, itemtype, num, kind, name, char, w_box_type, + alternate_constructors=alternate_constructors, aliases=aliases, + fields=fields, fieldnames=fieldnames, native=native) + self.float_type = float_type + + def is_complex_type(self): + return True + def dtype_from_list(space, w_lst): lst_w = space.listview(w_lst) fields = {} @@ -413,15 +426,16 @@ alternate_constructors=[space.w_float], aliases=["float"], ) - self.w_complex64dtype = W_Dtype( + self.w_complex64dtype = W_ComplexDtype( types.Complex64(), num=14, kind=COMPLEXLTR, name="complex64", char="F", w_box_type = space.gettypefor(interp_boxes.W_Complex64Box), + float_type = self.w_float32dtype, ) - self.w_complex128dtype = W_Dtype( + self.w_complex128dtype = W_ComplexDtype( types.Complex128(), num=15, kind=COMPLEXLTR, @@ -430,6 +444,7 @@ w_box_type = space.gettypefor(interp_boxes.W_Complex128Box), alternate_constructors=[space.w_complex], aliases=["complex"], + float_type = self.w_float64dtype, ) if interp_boxes.long_double_size == 12: self.w_float96dtype = W_Dtype( @@ -443,7 +458,7 @@ ) self.w_longdouble = self.w_float96dtype - self.w_complex192dtype = W_Dtype( + self.w_complex192dtype = W_ComplexDtype( types.Complex192(), num=16, kind=COMPLEXLTR, @@ -452,6 +467,7 @@ w_box_type = space.gettypefor(interp_boxes.W_Complex192Box), alternate_constructors=[space.w_complex], aliases=["clongdouble", "clongfloat"], + float_type = self.w_float96dtype, ) self.w_clongdouble = self.w_complex192dtype @@ -467,7 +483,7 @@ ) self.w_longdouble = self.w_float128dtype - self.w_complex256dtype = W_Dtype( + self.w_complex256dtype = W_ComplexDtype( types.Complex256(), num=16, kind=COMPLEXLTR, @@ -476,6 +492,7 @@ w_box_type = space.gettypefor(interp_boxes.W_Complex256Box), alternate_constructors=[space.w_complex], aliases=["clongdouble", "clongfloat"], + float_type = self.w_float128dtype, ) self.w_clongdouble = self.w_complex256dtype else: diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -235,6 +235,29 @@ def descr_copy(self, space): return W_NDimArray(self.implementation.copy()) + def descr_get_real(self, space): + return W_NDimArray(self.implementation.get_real()) + + def descr_get_imag(self, space): + ret = self.implementation.get_imag() + if ret: + return W_NDimArray(ret) + raise OperationError(space.w_NotImplementedError, + space.wrap('imag not implemented for this dtype')) + + def descr_set_real(self, space, w_value): + # copy (broadcast) values into self + tmp = self.implementation.get_real() + tmp.setslice(space, convert_to_array(space, w_value)) + + def descr_set_imag(self, space, w_value): + # if possible, copy (broadcast) values into self + if not self.get_dtype().is_complex_type(): + raise OperationError(space.w_TypeError, + space.wrap('array does not have imaginary part to set')) + tmp = self.implementation.get_imag() + tmp.setslice(space, convert_to_array(space, w_value)) + def descr_reshape(self, space, args_w): """reshape(...) a.reshape(shape) @@ -387,8 +410,6 @@ descr_neg = _unaryop_impl("negative") descr_abs = _unaryop_impl("absolute") descr_invert = _unaryop_impl("invert") - descr_get_real = _unaryop_impl("real") - descr_get_imag = _unaryop_impl("imag") def descr_nonzero(self, space): if self.get_size() > 1: @@ -635,9 +656,10 @@ swapaxes = interp2app(W_NDimArray.descr_swapaxes), flat = GetSetProperty(W_NDimArray.descr_get_flatiter), item = interp2app(W_NDimArray.descr_item), - real = GetSetProperty(W_NDimArray.descr_get_real), - imag = GetSetProperty(W_NDimArray.descr_get_imag), - + real = GetSetProperty(W_NDimArray.descr_get_real, + W_NDimArray.descr_set_real), + imag = GetSetProperty(W_NDimArray.descr_get_imag, + W_NDimArray.descr_set_imag), __array_interface__ = GetSetProperty(W_NDimArray.descr_array_iface), ) diff --git a/pypy/module/micronumpy/test/test_iter.py b/pypy/module/micronumpy/test/test_iter.py --- a/pypy/module/micronumpy/test/test_iter.py +++ b/pypy/module/micronumpy/test/test_iter.py @@ -12,7 +12,7 @@ strides = [5, 1] backstrides = [x * (y - 1) for x,y in zip(strides, shape)] assert backstrides == [10, 4] - i = MultiDimViewIterator(MockArray, start, strides, backstrides, shape) + i = MultiDimViewIterator(MockArray, None, start, strides, backstrides, shape) i.next() i.next() i.next() @@ -30,7 +30,7 @@ strides = [1, 3] backstrides = [x * (y - 1) for x,y in zip(strides, shape)] assert backstrides == [2, 12] - i = MultiDimViewIterator(MockArray, start, strides, backstrides, shape) + i = MultiDimViewIterator(MockArray, None, start, strides, backstrides, shape) i.next() i.next() i.next() @@ -51,7 +51,7 @@ strides = [5, 1] backstrides = [x * (y - 1) for x,y in zip(strides, shape)] assert backstrides == [10, 4] - i = MultiDimViewIterator(MockArray, start, strides, backstrides, shape) + i = MultiDimViewIterator(MockArray, None, start, strides, backstrides, shape) i.next_skip_x(2) i.next_skip_x(2) i.next_skip_x(2) @@ -74,7 +74,7 @@ strides = [1, 3] backstrides = [x * (y - 1) for x,y in zip(strides, shape)] assert backstrides == [2, 12] - i = MultiDimViewIterator(MockArray, start, strides, backstrides, shape) + i = MultiDimViewIterator(MockArray, None, start, strides, backstrides, shape) i.next_skip_x(2) i.next_skip_x(2) i.next_skip_x(2) diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1169,8 +1169,8 @@ assert (d == [[1, 0, 0], [0, 1, 0], [0, 0, 1]]).all() def test_eye(self): - from _numpypy import eye, array - from _numpypy import int32, float64, dtype + from _numpypy import eye + from _numpypy import int32, dtype a = eye(0) assert len(a) == 0 assert a.dtype == dtype('float64') @@ -1345,6 +1345,42 @@ assert b[0] == 3 assert b[1] == 2 + def test_realimag_views(self): + from _numpypy import arange, array + a = arange(15) + b = a.real + b[5]=50 + assert a[5] == 50 + b = a.imag + assert b[7] == 0 + raises(RuntimeError, 'b[7] = -2') + raises(TypeError, 'a.imag = -2') + a = array(['abc','def'],dtype='S3') + b = a.real + assert a[0] == b[0] + assert a[1] == b[1] + b[1] = 'xyz' + assert a[1] == 'xyz' + assert a.imag[0] == 'abc' + raises(TypeError, 'a.imag = "qop"') + a=array([[1+1j, 2-3j, 4+5j],[-6+7j, 8-9j, -2-1j]]) + assert a.real[0,1] == 2 + a.real[0,1] = -20 + assert a[0,1].real == -20 + b = a.imag + assert b[1,2] == -1 + b[1,2] = 30 + assert a[1,2].imag == 30 + a.real = 13 + assert a[1,1].real == 13 + a=array([1+1j, 2-3j, 4+5j, -6+7j, 8-9j, -2-1j]) + a.real = 13 + assert a[3].real == 13 + a.imag = -5 + a.imag[3] = -10 + assert a[3].imag == -10 + assert a[2].imag == -5 + def test_tolist_scalar(self): from _numpypy import int32, bool_ x = int32(23) @@ -2273,7 +2309,7 @@ assert a[0]['x'] == 'a' def test_stringarray(self): - from _numpypy import array, flexible + from _numpypy import array a = array(['abc'],'S3') assert str(a.dtype) == '|S3' a = array(['abc']) @@ -2293,7 +2329,6 @@ def test_flexible_repr(self): # import overrides str(), repr() for array - from numpypy.core import arrayprint from _numpypy import array a = array(['abc'],'S3') s = repr(a) diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py --- a/pypy/module/pyexpat/interp_pyexpat.py +++ b/pypy/module/pyexpat/interp_pyexpat.py @@ -182,6 +182,8 @@ def clear(self): self.next_id = 0 + self._last_object_id = -1 + self._last_object = None self.storage = {} @staticmethod @@ -194,10 +196,18 @@ @staticmethod def get_object(id): - return global_storage.storage[id] + if id == global_storage._last_object_id: + return global_storage._last_object + result = global_storage.storage[id] + global_storage._last_object_id = id + global_storage._last_object = result + return result @staticmethod def free_nonmoving_id(id): + if id == global_storage._last_object_id: + global_storage._last_object = None + global_storage._last_object_id = -1 del global_storage.storage[id] global_storage = Storage() diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py --- a/pypy/module/pypyjit/interp_jit.py +++ b/pypy/module/pypyjit/interp_jit.py @@ -6,7 +6,7 @@ from rpython.tool.pairtype import extendabletype from rpython.rlib.rarithmetic import r_uint, intmask from rpython.rlib.jit import JitDriver, hint, we_are_jitted, dont_look_inside -from rpython.rlib import jit +from rpython.rlib import jit, objectmodel from rpython.rlib.jit import current_trace_length, unroll_parameters import pypy.interpreter.pyopcode # for side-effects from pypy.interpreter.error import OperationError, operationerrfmt @@ -97,6 +97,15 @@ is_being_profiled=self.is_being_profiled) return jumpto +callback_jit_driver = JitDriver(greens = ['name'], reds = 'auto') + +def callback_merge_point(name): + callback_jit_driver.jit_merge_point(name=name) + + at callback_jit_driver.inline(callback_merge_point) +def callback_hook(name): + pass + def _get_adapted_tick_counter(): # Normally, the tick counter is decremented by 100 for every # Python opcode. Here, to better support JIT compilation of diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py --- a/pypy/module/pypyjit/policy.py +++ b/pypy/module/pypyjit/policy.py @@ -106,7 +106,7 @@ 'posix', '_socket', '_sre', '_lsprof', '_weakref', '__pypy__', 'cStringIO', '_collections', 'struct', 'mmap', 'marshal', '_codecs', 'rctime', 'cppyy', - '_cffi_backend']: + '_cffi_backend', 'pyexpat']: if modname == 'pypyjit' and 'interp_resop' in rest: return False return True diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py --- a/pypy/module/signal/interp_signal.py +++ b/pypy/module/signal/interp_signal.py @@ -14,6 +14,7 @@ from rpython.rlib.rarithmetic import intmask from rpython.rlib.rsignal import * +WIN32 = sys.platform == 'win32' class SignalActionFlag(AbstractActionFlag): # This class uses the C-level pypysig_counter variable as the tick @@ -144,7 +145,10 @@ None -- if an unknown handler is in effect (XXX UNIMPLEMENTED) anything else -- the callable Python object used as a handler """ - check_signum_in_range(space, signum) + if WIN32: + check_signum_exists(space, signum) + else: + check_signum_in_range(space, signum) action = space.check_signal_action if signum in action.handlers_w: return action.handlers_w[signum] diff --git a/pypy/module/signal/test/test_signal.py b/pypy/module/signal/test/test_signal.py --- a/pypy/module/signal/test/test_signal.py +++ b/pypy/module/signal/test/test_signal.py @@ -55,7 +55,7 @@ skip("requires os.kill() and os.getpid()") signal = self.signal # the signal module to test if not hasattr(signal, 'SIGUSR1'): - py.test.skip("requires SIGUSR1 in signal") + skip("requires SIGUSR1 in signal") signum = signal.SIGUSR1 received = [] @@ -156,6 +156,7 @@ import sys if sys.platform == 'win32': raises(ValueError, signal, 42, lambda *args: None) + raises(ValueError, signal, 7, lambda *args: None) else: signal(42, lambda *args: None) signal(42, SIG_DFL) diff --git a/pypy/module/sys/app.py b/pypy/module/sys/app.py --- a/pypy/module/sys/app.py +++ b/pypy/module/sys/app.py @@ -66,11 +66,11 @@ return None copyright_str = """ -Copyright 2003-2012 PyPy development team. +Copyright 2003-2013 PyPy development team. All Rights Reserved. For further information, see -Portions Copyright (c) 2001-2012 Python Software Foundation. +Portions Copyright (c) 2001-2013 Python Software Foundation. All Rights Reserved. Portions Copyright (c) 2000 BeOpen.com. diff --git a/pypy/tool/option.py b/pypy/tool/option.py --- a/pypy/tool/option.py +++ b/pypy/tool/option.py @@ -6,7 +6,7 @@ import optparse extra_useage = """For detailed descriptions of all the options see -http://codespeak.net/pypy/dist/pypy/doc/config/commandline.html""" +http://doc.pypy.org/en/latest/config/commandline.html""" def get_standard_options(): config = get_pypy_config() diff --git a/pypy/tool/pypyjit_child.py b/pypy/tool/pypyjit_child.py --- a/pypy/tool/pypyjit_child.py +++ b/pypy/tool/pypyjit_child.py @@ -15,7 +15,7 @@ interp.heap.malloc_nonmovable = returns_null # XXX from rpython.jit.backend.llgraph.runner import LLtypeCPU - #LLtypeCPU.supports_floats = False # for now + #LLtypeCPU.supports_floats = False # for now apply_jit(interp, graph, LLtypeCPU) diff --git a/pypy/tool/release/make_release.py b/pypy/tool/release/make_release.py deleted file mode 100755 --- a/pypy/tool/release/make_release.py +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env python - -""" A tool to download correct pypy-c's from nightly build run and package them -into release packages. Note: you must run apropriate buildbots first and -make sure there are no failures. Use force-builds.py from the same directory. - -Usage: make_release.py - e.g.: make_release.py release-1.4.1 1.4.1 -""" - -import sys -import urllib2 -from xml.dom import minidom -import re -import py -from rpython.tool.udir import udir -from pypy.tool.release.package import package -import tarfile -import os -import shutil - -BASEURL = 'http://buildbot.pypy.org/nightly/' -PAUSE = False - -def browse_nightly(branch, - baseurl=BASEURL, - override_xml=None): - if override_xml is None: - url = baseurl + branch + '/' - xml = urllib2.urlopen(url).read() - else: - xml = override_xml - dom = minidom.parseString(xml) - refs = [node.getAttribute('href') for node in dom.getElementsByTagName('a') - if 'pypy' in node.getAttribute('href')] - # all refs are of form: pypy-c-{type}-{revnum}-{hghash}-{platform}.tar.bz2 - r = re.compile('pypy-c-([\w\d]+)-(\d+)-([0-9a-f]+)-([\w\d]+).tar.bz2$') - d = {} - for ref in refs: - kind, revnum, hghash, platform = r.match(ref).groups() - rev = int(revnum) - try: - lastrev, _ = d[(kind, platform)] - except KeyError: - lastrev = -1 - if rev > lastrev: - d[(kind, platform)] = rev, ref - return d - -def main(branch, release): - to_download = browse_nightly(branch) - tmpdir = udir.join('download') - tmpdir.ensure(dir=True) - alltars = [] - olddir = os.getcwd() - try: - os.chdir(str(tmpdir)) - print 'Using tmpdir', str(tmpdir) - for (kind, platform), (rev, name) in to_download.iteritems(): - if platform == 'win32': - print 'Ignoring %s, windows unsupported' % name - else: - print "Downloading %s at rev %d" % (name, rev) - url = BASEURL + branch + "/" + name - data = urllib2.urlopen(url).read() - tmpdir.join(name).write(data, mode="wb") - t = tarfile.open(str(tmpdir.join(name))) - dirname = t.getmembers()[0].name - t.extractall(path=str(tmpdir)) - if kind == 'jit': - kind = '' - else: - kind = '-' + kind - topdirname = 'pypy-%s-%s%s' % (release, platform, kind) - os.system('mv %s %s' % (str(tmpdir.join(dirname)), - str(tmpdir.join(topdirname)))) - if PAUSE: - print 'Pausing, press Enter...' - raw_input() - name = '%s.tar.bz2' % topdirname - print "Building %s" % name - t = tarfile.open(name, 'w:bz2') - t.add(topdirname) - alltars.append(name) - t.close() - shutil.rmtree(str(tmpdir.join(topdirname))) - for name in alltars: - print "Uploading %s" % name - os.system('scp %s codespeak.net:/www/pypy.org/htdocs/download' % name) - finally: - os.chdir(olddir) - -if __name__ == '__main__': - if len(sys.argv) != 3: - print __doc__ - sys.exit(1) - main(sys.argv[1], release=sys.argv[2]) - diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -121,7 +121,7 @@ try: os.chdir(str(builddir)) # - # 'strip' fun: see https://codespeak.net/issue/pypy-dev/issue587 + # 'strip' fun: see issue #587 for source, target in binaries: if sys.platform == 'win32': pass diff --git a/pypy/tool/release/test/test_make_release.py b/pypy/tool/release/test/test_make_release.py deleted file mode 100644 --- a/pypy/tool/release/test/test_make_release.py +++ /dev/null @@ -1,11 +0,0 @@ - -import py -from pypy.tool.release.make_release import browse_nightly - -XML = py.path.local(__file__).join('..', 'nightly.xml').read() - -def test_browse_nightly(): - res = browse_nightly('branch', override_xml=XML) - assert res[('jit', 'linux')] == (40170, 'pypy-c-jit-40170-c407c9dc5382-linux.tar.bz2') - assert len(res) == 6 - assert res[('nojit', 'linux64')] == (40170, 'pypy-c-nojit-40170-c407c9dc5382-linux64.tar.bz2') diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py --- a/rpython/flowspace/test/test_objspace.py +++ b/rpython/flowspace/test/test_objspace.py @@ -8,18 +8,17 @@ from rpython.flowspace.objspace import FlowObjSpace from rpython.flowspace.flowcontext import FlowingError, FlowSpaceFrame from rpython.conftest import option -from rpython.tool.stdlib_opcode import bytecode_spec, host_bytecode_spec +from rpython.tool.stdlib_opcode import host_bytecode_spec import os import operator is_operator = getattr(operator, 'is_', operator.eq) # it's not there 2.2 @contextmanager -def patching_opcodes(*opcodes): +def patching_opcodes(**opcodes): meth_names = host_bytecode_spec.method_names - opnums = [bytecode_spec.opmap[name] for name in opcodes] old_name = {} - for name, num in zip(opcodes, opnums): + for name, num in opcodes.items(): old_name[num] = meth_names[num] meth_names[num] = name yield @@ -898,7 +897,7 @@ """ Tests code generated by pypy-c compiled with CALL_METHOD bytecode """ - with patching_opcodes('CALL_METHOD', 'LOOKUP_METHOD'): + with patching_opcodes(CALL_METHOD=202, LOOKUP_METHOD=201): class X: def m(self): return 3 @@ -922,7 +921,7 @@ """ Tests code generated by pypy-c compiled with BUILD_LIST_FROM_ARG bytecode """ - with patching_opcodes('BUILD_LIST_FROM_ARG'): + with patching_opcodes(BUILD_LIST_FROM_ARG=203): def f(): return [i for i in "abc"] diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py --- a/rpython/jit/backend/arm/assembler.py +++ b/rpython/jit/backend/arm/assembler.py @@ -1,5 +1,6 @@ from __future__ import with_statement import os +from rpython.jit.backend.llsupport import jitframe from rpython.jit.backend.arm.helper.assembler import saved_registers from rpython.jit.backend.arm import conditions as c from rpython.jit.backend.arm import registers as r @@ -25,12 +26,12 @@ from rpython.rtyper.lltypesystem.lloperation import llop from rpython.jit.backend.arm.opassembler import ResOpAssembler from rpython.rlib.debug import (debug_print, debug_start, debug_stop, - have_debug_prints) + have_debug_prints, fatalerror) from rpython.rlib.jit import AsmInfo from rpython.rlib.objectmodel import compute_unique_id # XXX Move to llsupport -from rpython.jit.backend.x86.support import values_array, memcpy_fn +from rpython.jit.backend.x86.support import memcpy_fn DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', ('i', lltype.Signed), ('type', lltype.Char), # 'b'ridge, 'l'abel or @@ -44,14 +45,8 @@ debug = True - def __init__(self, cpu, failargs_limit=1000): + def __init__(self, cpu, translate_support_code=False): self.cpu = cpu - self.fail_boxes_int = values_array(lltype.Signed, failargs_limit) - self.fail_boxes_float = values_array(longlong.FLOATSTORAGE, - failargs_limit) - self.fail_boxes_ptr = values_array(llmemory.GCREF, failargs_limit) - self.fail_boxes_count = 0 - self.fail_force_index = 0 self.setup_failure_recovery() self.mc = None self.memcpy_addr = 0 @@ -68,6 +63,7 @@ self._debug = False self.loop_run_counters = [] self.debug_counter_descr = cpu.fielddescrof(DEBUG_COUNTER, 'i') + self.force_token_to_dead_frame = {} # XXX temporary hack def set_debug(self, v): r = self._debug @@ -112,9 +108,13 @@ gc_ll_descr.initialize() self._build_wb_slowpath(False) self._build_wb_slowpath(True) + self._build_failure_recovery(exc=True, withfloats=False) + self._build_failure_recovery(exc=False, withfloats=False) if self.cpu.supports_floats: self._build_wb_slowpath(False, withfloats=True) self._build_wb_slowpath(True, withfloats=True) + self._build_failure_recovery(exc=True, withfloats=True) + self._build_failure_recovery(exc=False, withfloats=True) self._build_propagate_exception_path() if gc_ll_descr.get_malloc_slowpath_addr is not None: self._build_malloc_slowpath() @@ -122,10 +122,7 @@ if gc_ll_descr.gcrootmap and gc_ll_descr.gcrootmap.is_shadow_stack: self._build_release_gil(gc_ll_descr.gcrootmap) self.memcpy_addr = self.cpu.cast_ptr_to_int(memcpy_fn) - self._exit_code_addr = self._gen_exit_path() - self._leave_jitted_hook_save_exc = \ - self._gen_leave_jitted_hook_code(True) - self._leave_jitted_hook = self._gen_leave_jitted_hook_code(False) + if not self._debug: # if self._debug is already set it means that someone called # set_debug by hand before initializing the assembler. Leave it @@ -220,36 +217,17 @@ self.releasegil_addr = rffi.cast(lltype.Signed, releasegil_func) self.reacqgil_addr = rffi.cast(lltype.Signed, reacqgil_func) - def _gen_leave_jitted_hook_code(self, save_exc): - mc = ARMv7Builder() - if self.cpu.supports_floats: - floats = r.caller_vfp_resp - else: - floats = [] - with saved_registers(mc, r.caller_resp + [r.lr], floats): - addr = self.cpu.get_on_leave_jitted_int(save_exception=save_exc) - mc.BL(addr) - assert self._exit_code_addr != 0 - mc.B(self._exit_code_addr) - return mc.materialize(self.cpu.asmmemmgr, [], - self.cpu.gc_ll_descr.gcrootmap) - def _build_propagate_exception_path(self): if self.cpu.propagate_exception_v < 0: return # not supported (for tests, or non-translated) # mc = ARMv7Builder() - # call on_leave_jitted_save_exc() - if self.cpu.supports_floats: - floats = r.caller_vfp_resp - else: - floats = [] - with saved_registers(mc, r.caller_resp + [r.lr], floats): - addr = self.cpu.get_on_leave_jitted_int(save_exception=True, - default_to_memoryerror=True) - mc.BL(addr) - mc.gen_load_int(r.ip.value, self.cpu.propagate_exception_v) - mc.MOV_rr(r.r0.value, r.ip.value) + # + # Call the helper, which will return a dead frame object with + # the correct exception set, or MemoryError by default + # XXX make sure we return the correct value here + addr = rffi.cast(lltype.Signed, self.cpu.get_propagate_exception()) + mc.BL(addr) self.gen_func_epilog(mc=mc) self.propagate_exception_path = mc.materialize(self.cpu.asmmemmgr, []) @@ -282,12 +260,12 @@ # restore registers and return # We check for c.EQ here, meaning all bits zero in this case mc.POP([reg.value for reg in r.argument_regs] + [r.pc.value], cond=c.EQ) - # call on_leave_jitted_save_exc() - addr = self.cpu.get_on_leave_jitted_int(save_exception=True) + # + # Call the helper, which will return a dead frame object with + # the correct exception set, or MemoryError by default + addr = rffi.cast(lltype.Signed, self.cpu.get_propagate_exception()) mc.BL(addr) # - mc.gen_load_int(r.r0.value, self.cpu.propagate_exception_v) - # # footer -- note the ADD, which skips the return address of this # function, and will instead return to the caller's caller. Note # also that we completely ignore the saved arguments, because we @@ -339,39 +317,82 @@ def setup_failure_recovery(self): - @rgc.no_collect + #@rgc.no_collect -- XXX still true, but hacked gc_set_extra_threshold def failure_recovery_func(mem_loc, frame_pointer, stack_pointer): """mem_loc is a structure in memory describing where the values for the failargs are stored. frame loc is the address of the frame pointer for the frame to be decoded frame """ - vfp_registers = rffi.cast(rffi.LONGLONGP, stack_pointer) - registers = rffi.ptradd(vfp_registers, len(r.all_vfp_regs)) + vfp_registers = rffi.cast(rffi.LONGP, stack_pointer) + registers = rffi.ptradd(vfp_registers, 2*len(r.all_vfp_regs)) registers = rffi.cast(rffi.LONGP, registers) - return self.decode_registers_and_descr(mem_loc, frame_pointer, + bytecode = rffi.cast(rffi.UCHARP, mem_loc) + return self.grab_frame_values(self.cpu, bytecode, frame_pointer, registers, vfp_registers) + self.failure_recovery_code = [0, 0, 0, 0] self.failure_recovery_func = failure_recovery_func - recovery_func_sign = lltype.Ptr(lltype.FuncType([lltype.Signed] * 3, - lltype.Signed)) + _FAILURE_RECOVERY_FUNC = lltype.Ptr(lltype.FuncType([rffi.LONGP] * 3, + llmemory.GCREF)) - @rgc.no_collect - def decode_registers_and_descr(self, mem_loc, frame_pointer, + @staticmethod + #@rgc.no_collect -- XXX still true, but hacked gc_set_extra_threshold + def grab_frame_values(cpu, bytecode, frame_pointer, registers, vfp_registers): - """Decode locations encoded in memory at mem_loc and write the values - to the failboxes. Values for spilled vars and registers are stored on - stack at frame_loc """ - assert frame_pointer & 1 == 0 - self.fail_force_index = frame_pointer - bytecode = rffi.cast(rffi.UCHARP, mem_loc) + # no malloc allowed here!! xxx apart from one, hacking a lot + force_index = rffi.cast(lltype.Signed, frame_pointer) num = 0 - value = 0 - fvalue = 0 + deadframe = lltype.nullptr(jitframe.DEADFRAME) + # step 1: lots of mess just to count the final value of 'num' + bytecode1 = bytecode + while 1: + code = rffi.cast(lltype.Signed, bytecode1[0]) + bytecode1 = rffi.ptradd(bytecode1, 1) + if code >= AssemblerARM.CODE_FROMSTACK: + while code > 0x7F: + code = rffi.cast(lltype.Signed, bytecode1[0]) + bytecode1 = rffi.ptradd(bytecode1, 1) + else: + kind = code & 3 + if kind == AssemblerARM.DESCR_SPECIAL: + if code == AssemblerARM.CODE_HOLE: + num += 1 + continue + if code == AssemblerARM.CODE_INPUTARG: + continue + if code == AssemblerARM.CODE_FORCED: + # resuming from a GUARD_NOT_FORCED + token = force_index + deadframe = ( + cpu.assembler.force_token_to_dead_frame.pop(token)) + deadframe = lltype.cast_opaque_ptr( + jitframe.DEADFRAMEPTR, deadframe) + continue + assert code == AssemblerARM.CODE_STOP + break + num += 1 + + # allocate the deadframe + if not deadframe: + # Remove the "reserve" at the end of the nursery. This means + # that it is guaranteed that the following malloc() works + # without requiring a collect(), but it needs to be re-added + # as soon as possible. + cpu.gc_clear_extra_threshold() + assert num <= cpu.get_failargs_limit() + try: + deadframe = lltype.malloc(jitframe.DEADFRAME, num) + except MemoryError: + fatalerror("memory usage error in grab_frame_values") + # fill it code_inputarg = False - while True: + num = 0 + value_hi = 0 + while 1: + # decode the next instruction from the bytecode code = rffi.cast(lltype.Signed, bytecode[0]) bytecode = rffi.ptradd(bytecode, 1) - if code >= self.CODE_FROMSTACK: + if code >= AssemblerARM.CODE_FROMSTACK: if code > 0x7F: shift = 7 code &= 0x7F @@ -384,54 +405,63 @@ break # load the value from the stack kind = code & 3 - code = int((code - self.CODE_FROMSTACK) >> 2) + code = (code - AssemblerARM.CODE_FROMSTACK) >> 2 if code_inputarg: code = ~code code_inputarg = False - if kind == self.DESCR_FLOAT: - # we use code + 1 to get the hi word of the double worded float - stackloc = frame_pointer - get_fp_offset(int(code) + 1) - assert stackloc & 3 == 0 - fvalue = rffi.cast(rffi.LONGLONGP, stackloc)[0] - else: - stackloc = frame_pointer - get_fp_offset(int(code)) - assert stackloc & 1 == 0 - value = rffi.cast(rffi.LONGP, stackloc)[0] + stackloc = force_index - get_fp_offset(int(code)) + value = rffi.cast(rffi.LONGP, stackloc)[0] + if kind == AssemblerARM.DESCR_FLOAT: + assert WORD == 4 + value_hi = value + value = rffi.cast(rffi.LONGP, stackloc - WORD)[0] else: - # 'code' identifies a register: load its value kind = code & 3 - if kind == self.DESCR_SPECIAL: - if code == self.CODE_HOLE: + if kind == AssemblerARM.DESCR_SPECIAL: + if code == AssemblerARM.CODE_HOLE: num += 1 continue - if code == self.CODE_INPUTARG: + if code == AssemblerARM.CODE_INPUTARG: code_inputarg = True continue - assert code == self.CODE_STOP + if code == AssemblerARM.CODE_FORCED: + continue + assert code == AssemblerARM.CODE_STOP break + # 'code' identifies a register: load its value code >>= 2 - if kind == self.DESCR_FLOAT: - fvalue = vfp_registers[code] + if kind == AssemblerARM.DESCR_FLOAT: + if WORD == 4: + value = vfp_registers[2*code] + value_hi = vfp_registers[2*code + 1] + else: + value = registers[code] else: value = registers[code] # store the loaded value into fail_boxes_ - if kind == self.DESCR_FLOAT: - tgt = self.fail_boxes_float.get_addr_for_num(num) - rffi.cast(rffi.LONGLONGP, tgt)[0] = fvalue + if kind == AssemblerARM.DESCR_INT: + deadframe.jf_values[num].int = value + elif kind == AssemblerARM.DESCR_REF: + deadframe.jf_values[num].ref = rffi.cast(llmemory.GCREF, value) + elif kind == AssemblerARM.DESCR_FLOAT: + assert WORD == 4 + assert not longlong.is_64_bit + floatvalue = rffi.cast(lltype.SignedLongLong, value_hi) + floatvalue <<= 32 + floatvalue |= rffi.cast(lltype.SignedLongLong, + rffi.cast(lltype.Unsigned, value)) + deadframe.jf_values[num].float = floatvalue else: - if kind == self.DESCR_INT: - tgt = self.fail_boxes_int.get_addr_for_num(num) - elif kind == self.DESCR_REF: - assert (value & 3) == 0, "misaligned pointer" - tgt = self.fail_boxes_ptr.get_addr_for_num(num) - else: - assert 0, "bogus kind" - rffi.cast(rffi.LONGP, tgt)[0] = value + assert 0, "bogus kind" num += 1 - self.fail_boxes_count = num + # + assert num == len(deadframe.jf_values) + if not we_are_translated(): + assert bytecode[4] == 0xCC fail_index = rffi.cast(rffi.INTP, bytecode)[0] - fail_index = rffi.cast(lltype.Signed, fail_index) - return fail_index + fail_descr = cpu.get_fail_descr_from_number(fail_index) + deadframe.jf_descr = fail_descr.hide(cpu) + return lltype.cast_opaque_ptr(llmemory.GCREF, deadframe) def decode_inputargs(self, code): descr_to_box_type = [REF, INT, FLOAT] @@ -514,12 +544,26 @@ self.mc.CMP_ri(r.r0.value, 0) self.mc.B(self.propagate_exception_path, c=c.EQ) - def _gen_exit_path(self): + def _build_failure_recovery(self, exc, withfloats=False): mc = ARMv7Builder() - decode_registers_addr = llhelper(self.recovery_func_sign, + failure_recovery = llhelper(self._FAILURE_RECOVERY_FUNC, self.failure_recovery_func) self._insert_checks(mc) - with saved_registers(mc, r.all_regs, r.all_vfp_regs): + if withfloats: + f = r.all_vfp_regs + else: + f = [] + with saved_registers(mc, r.all_regs, f): + if exc: + # We might have an exception pending. Load it into r4 + # (this is a register saved across calls) + mc.gen_load_int(r.r5.value, self.cpu.pos_exc_value()) + mc.LDR_ri(r.r4.value, self.cpu.pos_exc_value()) + # clear the exc flags + mc.gen_load_int(r.r6.value, 0) + mc.STR_ri(r.r6.value, r.r5.value) + mc.gen_load_int(r.r5.value, self.cpu.pos_exception()) + mc.STR_ri(r.r6.value, r.r5.value) # move mem block address, to r0 to pass as mc.MOV_rr(r.r0.value, r.lr.value) # pass the current frame pointer as second param @@ -527,23 +571,32 @@ # pass the current stack pointer as third param mc.MOV_rr(r.r2.value, r.sp.value) self._insert_checks(mc) - mc.BL(rffi.cast(lltype.Signed, decode_registers_addr)) + mc.BL(rffi.cast(lltype.Signed, failure_recovery)) + if exc: + # save ebx into 'jf_guard_exc' + from pypy.jit.backend.llsupport.descr import unpack_fielddescr + descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) + offset, size, _ = unpack_fielddescr(descrs.jf_guard_exc) + mc.STR_rr(r.r4.value, r.r0.value, offset, cond=c.AL) mc.MOV_rr(r.ip.value, r.r0.value) mc.MOV_rr(r.r0.value, r.ip.value) self.gen_func_epilog(mc=mc) - return mc.materialize(self.cpu.asmmemmgr, [], + rawstart = mc.materialize(self.cpu.asmmemmgr, [], self.cpu.gc_ll_descr.gcrootmap) + self.failure_recovery_code[exc + 2 * withfloats] = rawstart + self.mc = None DESCR_REF = 0x00 DESCR_INT = 0x01 DESCR_FLOAT = 0x02 DESCR_SPECIAL = 0x03 CODE_FROMSTACK = 64 - CODE_STOP = 0 | DESCR_SPECIAL - CODE_HOLE = 4 | DESCR_SPECIAL - CODE_INPUTARG = 8 | DESCR_SPECIAL + CODE_STOP = 0 | DESCR_SPECIAL + CODE_HOLE = 4 | DESCR_SPECIAL + CODE_INPUTARG = 8 | DESCR_SPECIAL + CODE_FORCED = 12 | DESCR_SPECIAL #XXX where should this be written? - def gen_descr_encoding(self, descr, failargs, locs): + def write_failure_recovery_description(self, descr, failargs, locs): assert self.mc is not None for i in range(len(failargs)): arg = failargs[i] @@ -575,26 +628,33 @@ self.mc.writechar(chr(n)) self.mc.writechar(chr(self.CODE_STOP)) - fdescr = self.cpu.get_fail_descr_number(descr) - self.mc.write32(fdescr) + + def generate_quick_failure(self, guardtok, fcond=c.AL): + assert isinstance(guardtok.save_exc, bool) + fail_index = self.cpu.get_fail_descr_number(guardtok.descr) + startpos = self.mc.currpos() + withfloats = False + for box in guardtok.failargs: + if box is not None and box.type == FLOAT: + withfloats = True + break + exc = guardtok.save_exc + target = self.failure_recovery_code[exc + 2 * withfloats] + assert target != 0 + self.mc.BL(target) + # write tight data that describes the failure recovery + if guardtok.is_guard_not_forced: + self.mc.writechar(chr(self.CODE_FORCED)) + self.write_failure_recovery_description(guardtok.descr, + guardtok.failargs, guardtok.faillocs[1:]) + self.mc.write32(fail_index) + # for testing the decoding, write a final byte 0xCC + if not we_are_translated(): + self.mc.writechar('\xCC') + faillocs = [loc for loc in guardtok.faillocs if loc is not None] + guardtok.descr._arm_debug_faillocs = faillocs self.align() - - # assert that the fail_boxes lists are big enough - assert len(failargs) <= self.fail_boxes_int.SIZE - - def _gen_path_to_exit_path(self, descr, args, arglocs, - save_exc, fcond=c.AL): - assert isinstance(save_exc, bool) - self.gen_exit_code(self.mc, save_exc, fcond) - self.gen_descr_encoding(descr, args, arglocs[1:]) - - def gen_exit_code(self, mc, save_exc, fcond=c.AL): - assert isinstance(save_exc, bool) - if save_exc: - path = self._leave_jitted_hook_save_exc - else: - path = self._leave_jitted_hook - mc.BL(path) + return startpos def align(self): while(self.mc.currpos() % FUNC_ALIGN != 0): @@ -841,15 +901,10 @@ def write_pending_failure_recoveries(self): for tok in self.pending_guards: - descr = tok.descr #generate the exit stub and the encoded representation - pos = self.mc.currpos() - tok.pos_recovery_stub = pos - - self._gen_path_to_exit_path(descr, tok.failargs, - tok.faillocs, save_exc=tok.save_exc) + tok.pos_recovery_stub = self.generate_quick_failure(tok) # store info on the descr - descr._arm_current_frame_depth = tok.faillocs[0].getint() + tok.descr._arm_current_frame_depth = tok.faillocs[0].getint() def process_pending_guards(self, block_start): clt = self.current_clt @@ -860,7 +915,7 @@ descr._arm_failure_recovery_block = failure_recovery_pos relative_offset = tok.pos_recovery_stub - tok.offset guard_pos = block_start + tok.offset - if not tok.is_invalidate: + if not tok.is_guard_not_invalidated: # patch the guard jumpt to the stub # overwrite the generate NOP with a B_offs to the pos of the # stub @@ -1255,11 +1310,6 @@ else: raise AssertionError('Trying to pop to an invalid location') - def leave_jitted_hook(self): - ptrs = self.fail_boxes_ptr.ar - llop.gc_assume_young_pointers(lltype.Void, - llmemory.cast_ptr_to_adr(ptrs)) - def malloc_cond(self, nursery_free_adr, nursery_top_adr, size): assert size & (WORD-1) == 0 # must be correctly aligned diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -16,6 +16,7 @@ gen_emit_unary_float_op, saved_registers, count_reg_args) +from rpython.jit.backend.arm.helper.regalloc import check_imm_arg from rpython.jit.backend.arm.codebuilder import ARMv7Builder, OverwritingBuilder from rpython.jit.backend.arm.jump import remap_frame_layout from rpython.jit.backend.arm.regalloc import TempInt, TempPtr @@ -27,18 +28,20 @@ from rpython.jit.metainterp.history import JitCellToken, TargetToken from rpython.jit.metainterp.resoperation import rop from rpython.rlib.objectmodel import we_are_translated -from rpython.rtyper.lltypesystem import rstr +from rpython.rlib import rgc +from rpython.rtyper.lltypesystem import rstr, rffi, lltype, llmemory NO_FORCE_INDEX = -1 class GuardToken(object): - def __init__(self, descr, failargs, faillocs, offset, - save_exc, fcond=c.AL, is_invalidate=False): + def __init__(self, descr, failargs, faillocs, offset, save_exc, fcond=c.AL, + is_guard_not_invalidated=False, is_guard_not_forced=False): assert isinstance(save_exc, bool) self.descr = descr self.offset = offset - self.is_invalidate = is_invalidate + self.is_guard_not_invalidated = is_guard_not_invalidated + self.is_guard_not_forced = is_guard_not_forced self.failargs = failargs self.faillocs = faillocs self.save_exc = save_exc @@ -190,7 +193,8 @@ return fcond def _emit_guard(self, op, arglocs, fcond, save_exc, - is_guard_not_invalidated=False): + is_guard_not_invalidated=False, + is_guard_not_forced=False): assert isinstance(save_exc, bool) assert isinstance(fcond, int) descr = op.getdescr() @@ -210,7 +214,8 @@ faillocs=arglocs, offset=pos, save_exc=save_exc, - is_invalidate=is_guard_not_invalidated, + is_guard_not_invalidated=is_guard_not_invalidated, + is_guard_not_forced=is_guard_not_forced, fcond=fcond)) return c.AL @@ -312,49 +317,10 @@ return fcond def emit_op_finish(self, op, arglocs, regalloc, fcond): - for i in range(len(arglocs) - 1): - loc = arglocs[i] - box = op.getarg(i) - if loc is None: - continue - if loc.is_reg(): - if box.type == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(i) - elif box.type == INT: - adr = self.fail_boxes_int.get_addr_for_num(i) - else: - assert 0 - self.mc.gen_load_int(r.ip.value, adr) - self.mc.STR_ri(loc.value, r.ip.value) - elif loc.is_vfp_reg(): - assert box.type == FLOAT - adr = self.fail_boxes_float.get_addr_for_num(i) - self.mc.gen_load_int(r.ip.value, adr) - self.mc.VSTR(loc.value, r.ip.value) - elif loc.is_stack() or loc.is_imm() or loc.is_imm_float(): - if box.type == FLOAT: - adr = self.fail_boxes_float.get_addr_for_num(i) - self.mov_loc_loc(loc, r.vfp_ip) - self.mc.gen_load_int(r.ip.value, adr) - self.mc.VSTR(r.vfp_ip.value, r.ip.value) - elif box.type == REF or box.type == INT: - if box.type == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(i) - elif box.type == INT: - adr = self.fail_boxes_int.get_addr_for_num(i) - else: - assert 0 - self.mov_loc_loc(loc, r.ip) - self.mc.gen_load_int(r.lr.value, adr) - self.mc.STR_ri(r.ip.value, r.lr.value) - else: - assert 0 - # note: no exception should currently be set in llop.get_exception_addr - # even if this finish may be an exit_frame_with_exception (in this case - # the exception instance is in arglocs[0]). - addr = self.cpu.get_on_leave_jitted_int(save_exception=False) - self.mc.BL(addr) - self.mc.gen_load_int(r.r0.value, arglocs[-1].value) + [argloc] = arglocs + if argloc is not r.r0: #XXX verify this + self.mov_loc_loc(argloc, r.r0, fcond) + # exit function self.gen_func_epilog() return fcond @@ -1152,8 +1118,26 @@ value = self.cpu.done_with_this_frame_float_v else: raise AssertionError(kind) - self.mc.gen_load_int(r.ip.value, value) - self.mc.CMP_rr(tmploc.value, r.ip.value) + from pypy.jit.backend.llsupport.descr import unpack_fielddescr + from pypy.jit.backend.llsupport.descr import unpack_interiorfielddescr + descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) + _offset, _size, _ = unpack_fielddescr(descrs.jf_descr) + fail_descr = self.cpu.get_fail_descr_from_number(value) + value = fail_descr.hide(self.cpu) + rgc._make_sure_does_not_move(value) + value = rffi.cast(lltype.Signed, value) + + if check_imm_arg(_offset): + self.mc.LDR_ri(r.ip.value, tmploc.value, imm=_offset) + else: + self.mc.gen_load_int(r.ip.value, _offset) + self.mc.LDR_rr(r.ip.value, tmploc.value, r.ip.value) + if check_imm_arg(value): + self.mc.CMP_ri(r.ip.value, imm=value) + else: + self.mc.gen_load_int(r.lr.value, value) + self.mc.CMP_rr(r.lr.value, r.ip.value) + #if values are equal we take the fast path # Slow path, calling helper @@ -1180,19 +1164,31 @@ if op.result is not None: # load the return value from fail_boxes_xxx[0] kind = op.result.type - if kind == INT: - adr = self.fail_boxes_int.get_addr_for_num(0) - elif kind == REF: - adr = self.fail_boxes_ptr.get_addr_for_num(0) - elif kind == FLOAT: - adr = self.fail_boxes_float.get_addr_for_num(0) + if kind == FLOAT: + t = unpack_interiorfielddescr(descrs.as_float)[0] + if not check_imm_arg(t): + self.mc.gen_load_int(r.ip.value, t, cond=fast_path_cond) + self.mc.ADD_rr(r.ip.value, r.r0.value, r.ip.value, + cond=fast_path_cond) + t = 0 + base = r.ip + else: + base = r.r0 + self.mc.VLDR(resloc.value, base.value, imm=t, + cond=fast_path_cond) else: - raise AssertionError(kind) - self.mc.gen_load_int(r.ip.value, adr, cond=fast_path_cond) - if op.result.type == FLOAT: - self.mc.VLDR(resloc.value, r.ip.value, cond=fast_path_cond) - else: - self.mc.LDR_ri(resloc.value, r.ip.value, cond=fast_path_cond) + assert resloc is r.r0 + if kind == INT: + t = unpack_interiorfielddescr(descrs.as_int)[0] + else: + t = unpack_interiorfielddescr(descrs.as_ref)[0] + if not check_imm_arg(t): + self.mc.gen_load_int(r.ip.value, t, cond=fast_path_cond) + self.mc.LDR_rr(resloc.value, resloc.value, r.ip.value, + cond=fast_path_cond) + else: + self.mc.LDR_ri(resloc.value, resloc.value, imm=t, + cond=fast_path_cond) # jump to merge point jmp_pos = self.mc.currpos() self.mc.BKPT() @@ -1266,7 +1262,8 @@ self.mc.LDR_ri(r.ip.value, r.fp.value) self.mc.CMP_ri(r.ip.value, 0) - self._emit_guard(guard_op, arglocs[1 + numargs:], c.GE, save_exc=True) + self._emit_guard(guard_op, arglocs[1 + numargs:], c.GE, + save_exc=True, is_guard_not_forced=True) return fcond def emit_guard_call_release_gil(self, op, guard_op, arglocs, regalloc, diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py --- a/rpython/jit/backend/arm/regalloc.py +++ b/rpython/jit/backend/arm/regalloc.py @@ -644,15 +644,9 @@ return args def prepare_op_finish(self, op, fcond): - args = [None] * (op.numargs() + 1) - for i in range(op.numargs()): - arg = op.getarg(i) - if arg: - args[i] = self.loc(arg) - self.possibly_free_var(arg) - n = self.cpu.get_fail_descr_number(op.getdescr()) - args[-1] = imm(n) - return args + loc = self.loc(op.getarg(0)) + self.possibly_free_var(op.getarg(0)) + return [loc] def prepare_op_guard_true(self, op, fcond): l0 = self.make_sure_var_in_reg(op.getarg(0)) diff --git a/rpython/jit/backend/arm/runner.py b/rpython/jit/backend/arm/runner.py --- a/rpython/jit/backend/arm/runner.py +++ b/rpython/jit/backend/arm/runner.py @@ -23,15 +23,21 @@ AbstractLLCPU.__init__(self, rtyper, stats, opts, translate_support_code, gcdescr) + from pypy.jit.backend.llsupport import jitframe + self.deadframe_size_max = llmemory.sizeof(jitframe.DEADFRAME, + self.get_failargs_limit()) + def set_debug(self, flag): return self.assembler.set_debug(flag) + def get_failargs_limit(self): + if self.opts is not None: + return self.opts.failargs_limit + else: + return 1000 + def setup(self): - if self.opts is not None: - failargs_limit = self.opts.failargs_limit - else: - failargs_limit = 1000 - self.assembler = AssemblerARM(self, failargs_limit=failargs_limit) + self.assembler = AssemblerARM(self, self.translate_support_code) def setup_once(self): self.assembler.setup_once() @@ -51,24 +57,6 @@ return self.assembler.assemble_bridge(faildescr, inputargs, operations, original_loop_token, log=log) - def get_latest_value_float(self, index): - return self.assembler.fail_boxes_float.getitem(index) - - def get_latest_value_int(self, index): - return self.assembler.fail_boxes_int.getitem(index) - - def get_latest_value_ref(self, index): - return self.assembler.fail_boxes_ptr.getitem(index) - - def get_latest_value_count(self): - return self.assembler.fail_boxes_count - - def get_latest_force_token(self): - return self.assembler.fail_force_index - - def get_on_leave_jitted_hook(self): - return self.assembler.leave_jitted_hook - def clear_latest_values(self, count): setitem = self.assembler.fail_boxes_ptr.setitem null = lltype.nullptr(llmemory.GCREF.TO) @@ -76,7 +64,7 @@ setitem(index, null) def make_execute_token(self, *ARGS): - FUNCPTR = lltype.Ptr(lltype.FuncType(ARGS, lltype.Signed)) + FUNCPTR = lltype.Ptr(lltype.FuncType(ARGS, llmemory.GCREF)) def execute_token(executable_token, *args): clt = executable_token.compiled_loop_token @@ -91,12 +79,13 @@ prev_interpreter = LLInterpreter.current_interpreter LLInterpreter.current_interpreter = self.debug_ll_interpreter try: - fail_index = func(*args) + deadframe = func(*args) finally: if not self.translate_support_code: From noreply at buildbot.pypy.org Sun Jan 13 08:02:50 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sun, 13 Jan 2013 08:02:50 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed resolve error Message-ID: <20130113070250.9F1F41C1322@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60014:57854b63747a Date: 2013-01-13 08:02 +0100 http://bitbucket.org/pypy/pypy/changeset/57854b63747a/ Log: Fixed resolve error diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py --- a/rpython/jit/backend/llgraph/runner.py +++ b/rpython/jit/backend/llgraph/runner.py @@ -1,12 +1,12 @@ import py, weakref -from pypy.jit.backend import model -from pypy.jit.backend.llgraph import support -from pypy.jit.metainterp.history import AbstractDescr -from pypy.jit.metainterp.history import Const, getkind -from pypy.jit.metainterp.history import INT, REF, FLOAT, VOID -from pypy.jit.metainterp.resoperation import rop -from pypy.jit.codewriter import longlong, heaptracker -from pypy.jit.codewriter.effectinfo import EffectInfo +from rpython.jit.backend import model +from rpython.jit.backend.llgraph import support +from rpython.jit.metainterp.history import AbstractDescr +from rpython.jit.metainterp.history import Const, getkind +from rpython.jit.metainterp.history import INT, REF, FLOAT, VOID +from rpython.jit.metainterp.resoperation import rop +from rpython.jit.codewriter import longlong, heaptracker +from rpython.jit.codewriter.effectinfo import EffectInfo from rpython.rtyper.llinterp import LLInterpreter, LLException from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rclass, rstr From noreply at buildbot.pypy.org Sun Jan 13 08:15:46 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sun, 13 Jan 2013 08:15:46 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Adjusted imports on new files Message-ID: <20130113071546.F11A31C029E@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60015:49bfec1167e6 Date: 2013-01-13 08:15 +0100 http://bitbucket.org/pypy/pypy/changeset/49bfec1167e6/ Log: Adjusted imports on new files diff --git a/rpython/jit/backend/llgraph/support.py b/rpython/jit/backend/llgraph/support.py --- a/rpython/jit/backend/llgraph/support.py +++ b/rpython/jit/backend/llgraph/support.py @@ -1,11 +1,11 @@ -from pypy.jit.codewriter import longlong -from pypy.jit.codewriter import heaptracker +from rpython.jit.codewriter import longlong +from rpython.jit.codewriter import heaptracker -from pypy.jit.metainterp.history import getkind +from rpython.jit.metainterp.history import getkind -from pypy.rlib.rarithmetic import r_longlong, r_ulonglong, r_uint -from pypy.rpython.lltypesystem import lltype, rffi, llmemory +from rpython.rlib.rarithmetic import r_longlong, r_ulonglong, r_uint +from rpython.rtyper.lltypesystem import lltype, rffi, llmemory IS_32_BIT = r_ulonglong is not r_uint diff --git a/rpython/jit/backend/llsupport/jitframe.py b/rpython/jit/backend/llsupport/jitframe.py --- a/rpython/jit/backend/llsupport/jitframe.py +++ b/rpython/jit/backend/llsupport/jitframe.py @@ -1,5 +1,5 @@ -from pypy.rpython.lltypesystem import lltype, llmemory -from pypy.jit.codewriter import longlong +from rpython.rtyper.lltypesystem import lltype, llmemory +from rpython.jit.codewriter import longlong # XXX not an actual union for now diff --git a/rpython/jit/metainterp/test/test_call.py b/rpython/jit/metainterp/test/test_call.py --- a/rpython/jit/metainterp/test/test_call.py +++ b/rpython/jit/metainterp/test/test_call.py @@ -1,6 +1,6 @@ -from pypy.jit.metainterp.test.support import LLJitMixin -from pypy.rlib import jit +from rpython.jit.metainterp.test.support import LLJitMixin +from rpython.rlib import jit class TestCall(LLJitMixin): def test_indirect_call(self): diff --git a/rpython/rlib/test/test_jit_libffi.py b/rpython/rlib/test/test_jit_libffi.py --- a/rpython/rlib/test/test_jit_libffi.py +++ b/rpython/rlib/test/test_jit_libffi.py @@ -1,10 +1,10 @@ import math import ctypes -from pypy.rpython.lltypesystem import lltype, rffi -from pypy.rlib import clibffi -from pypy.rlib.rarithmetic import intmask -from pypy.rlib.jit_libffi import CIF_DESCRIPTION -from pypy.rlib.jit_libffi import jit_ffi_prep_cif, jit_ffi_call +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rlib import clibffi +from rpython.rlib.rarithmetic import intmask +from rpython.rlib.jit_libffi import CIF_DESCRIPTION +from rpython.rlib.jit_libffi import jit_ffi_prep_cif, jit_ffi_call math_sin = intmask(ctypes.cast(ctypes.CDLL(None).sin, ctypes.c_void_p).value) From noreply at buildbot.pypy.org Sun Jan 13 08:34:22 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sun, 13 Jan 2013 08:34:22 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed more imports Message-ID: <20130113073422.89E9C1C054C@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60016:8c70875be135 Date: 2013-01-13 08:34 +0100 http://bitbucket.org/pypy/pypy/changeset/8c70875be135/ Log: Fixed more imports diff --git a/goal/targetpypystandalone.py b/goal/targetpypystandalone.py --- a/goal/targetpypystandalone.py +++ b/goal/targetpypystandalone.py @@ -31,7 +31,7 @@ if withjit: from pypy.module.pypyjit.interp_jit import callback_hook - from pypy.rlib import objectmodel + from rpython.rlib import objectmodel objectmodel.register_around_callback_hook(callback_hook) def entry_point(argv): diff --git a/pypy/doc/cli-backend.rst b/pypy/doc/cli-backend.rst --- a/pypy/doc/cli-backend.rst +++ b/pypy/doc/cli-backend.rst @@ -395,7 +395,7 @@ confidence not to break anything. The core of the testing framework is in the module -pypy.translator.cli.test.runtest; one of the most important function +rpython.translator.cli.test.runtest; one of the most important function of this module is compile_function(): it takes a Python function, compiles it to CLI and returns a Python object that runs the just created executable when called. @@ -407,7 +407,7 @@ on the standard output:: # Python source: foo.py - from pypy.translator.cli.test.runtest import compile_function + from rpython.translator.cli.test.runtest import compile_function def foo(x, y): return x+y, x*y diff --git a/pypy/doc/rffi.rst b/pypy/doc/rffi.rst --- a/pypy/doc/rffi.rst +++ b/pypy/doc/rffi.rst @@ -32,7 +32,7 @@ parameter:: from pypy.rpython.lltypesystem import rffi - from pypy.translator.tool.cbuild import ExternalCompilationInfo + from rpython.translator.tool.cbuild import ExternalCompilationInfo info = ExternalCompilationInfo(includes=[], libraries=[]) diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py --- a/rpython/jit/backend/arm/assembler.py +++ b/rpython/jit/backend/arm/assembler.py @@ -574,7 +574,7 @@ mc.BL(rffi.cast(lltype.Signed, failure_recovery)) if exc: # save ebx into 'jf_guard_exc' - from pypy.jit.backend.llsupport.descr import unpack_fielddescr + from rpython.jit.backend.llsupport.descr import unpack_fielddescr descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) offset, size, _ = unpack_fielddescr(descrs.jf_guard_exc) mc.STR_rr(r.r4.value, r.r0.value, offset, cond=c.AL) diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -1118,8 +1118,8 @@ value = self.cpu.done_with_this_frame_float_v else: raise AssertionError(kind) - from pypy.jit.backend.llsupport.descr import unpack_fielddescr - from pypy.jit.backend.llsupport.descr import unpack_interiorfielddescr + from rpython.jit.backend.llsupport.descr import unpack_fielddescr + from rpython.jit.backend.llsupport.descr import unpack_interiorfielddescr descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) _offset, _size, _ = unpack_fielddescr(descrs.jf_descr) fail_descr = self.cpu.get_fail_descr_from_number(value) diff --git a/rpython/jit/backend/arm/runner.py b/rpython/jit/backend/arm/runner.py --- a/rpython/jit/backend/arm/runner.py +++ b/rpython/jit/backend/arm/runner.py @@ -23,7 +23,7 @@ AbstractLLCPU.__init__(self, rtyper, stats, opts, translate_support_code, gcdescr) - from pypy.jit.backend.llsupport import jitframe + from rpython.jit.backend.llsupport import jitframe self.deadframe_size_max = llmemory.sizeof(jitframe.DEADFRAME, self.get_failargs_limit()) diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py --- a/rpython/jit/backend/llgraph/runner.py +++ b/rpython/jit/backend/llgraph/runner.py @@ -11,8 +11,8 @@ from rpython.rtyper.llinterp import LLInterpreter, LLException from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rclass, rstr -from pypy.rlib.rarithmetic import ovfcheck, r_uint, r_ulonglong -from pypy.rlib.rtimer import read_timestamp +from rpython.rlib.rarithmetic import ovfcheck, r_uint, r_ulonglong +from rpython.rlib.rtimer import read_timestamp class LLTrace(object): has_been_freed = False @@ -159,7 +159,7 @@ 'f': 0.0} class LLGraphCPU(model.AbstractCPU): - from pypy.jit.metainterp.typesystem import llhelper as ts + from rpython.jit.metainterp.typesystem import llhelper as ts supports_floats = True supports_longlong = r_uint is not r_ulonglong supports_singlefloats = True @@ -338,7 +338,7 @@ def _calldescr_dynamic_for_tests(self, atypes, rtype, abiname='FFI_DEFAULT_ABI'): # XXX WTF is that and why it breaks all abstractions? - from pypy.jit.backend.llsupport import ffisupport + from rpython.jit.backend.llsupport import ffisupport return ffisupport.calldescr_dynamic_for_tests(self, atypes, rtype, abiname) @@ -892,7 +892,7 @@ return x def execute_debug_merge_point(self, descr, *args): - from pypy.jit.metainterp.warmspot import get_stats + from rpython.jit.metainterp.warmspot import get_stats try: stats = get_stats() except AttributeError: @@ -925,7 +925,7 @@ def _setup(): def _make_impl_from_blackhole_interp(opname): - from pypy.jit.metainterp.blackhole import BlackholeInterpreter + from rpython.jit.metainterp.blackhole import BlackholeInterpreter name = 'bhimpl_' + opname.lower() try: func = BlackholeInterpreter.__dict__[name] diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -2152,7 +2152,7 @@ if exc: # save ebx into 'jf_guard_exc' - from pypy.jit.backend.llsupport.descr import unpack_fielddescr + from rpython.jit.backend.llsupport.descr import unpack_fielddescr descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) offset, size, _ = unpack_fielddescr(descrs.jf_guard_exc) mc.MOV_mr((eax.value, offset), ebx.value) @@ -2400,8 +2400,8 @@ else: raise AssertionError(kind) - from pypy.jit.backend.llsupport.descr import unpack_fielddescr - from pypy.jit.backend.llsupport.descr import unpack_interiorfielddescr + from rpython.jit.backend.llsupport.descr import unpack_fielddescr + from rpython.jit.backend.llsupport.descr import unpack_interiorfielddescr descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) _offset, _size, _ = unpack_fielddescr(descrs.jf_descr) fail_descr = self.cpu.get_fail_descr_from_number(value) diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py --- a/rpython/jit/backend/x86/runner.py +++ b/rpython/jit/backend/x86/runner.py @@ -45,7 +45,7 @@ self.profile_agent = profile_agent - from pypy.jit.backend.llsupport import jitframe + from rpython.jit.backend.llsupport import jitframe self.deadframe_size_max = llmemory.sizeof(jitframe.DEADFRAME, self.get_failargs_limit()) diff --git a/rpython/jit/codewriter/effectinfo.py b/rpython/jit/codewriter/effectinfo.py --- a/rpython/jit/codewriter/effectinfo.py +++ b/rpython/jit/codewriter/effectinfo.py @@ -169,7 +169,7 @@ ======= can_invalidate=False, call_release_gil_target=llmemory.NULL): - from pypy.translator.backendopt.writeanalyze import top_set + from rpython.translator.backendopt.writeanalyze import top_set >>>>>>> other if effects is top_set or extraeffect == EffectInfo.EF_RANDOM_EFFECTS: readonly_descrs_fields = None diff --git a/rpython/jit/codewriter/test/test_call.py b/rpython/jit/codewriter/test/test_call.py --- a/rpython/jit/codewriter/test/test_call.py +++ b/rpython/jit/codewriter/test/test_call.py @@ -176,7 +176,7 @@ <<<<<<< local from rpython.jit.backend.llgraph.runner import LLtypeCPU ======= - from pypy.jit.backend.llgraph.runner import LLGraphCPU + from rpython.jit.backend.llgraph.runner import LLGraphCPU >>>>>>> other T = rffi.CArrayPtr(rffi.TIME_T) @@ -199,7 +199,7 @@ assert call_descr.extrainfo.is_call_release_gil() is False def test_call_release_gil(): - from pypy.jit.backend.llgraph.runner import LLGraphCPU + from rpython.jit.backend.llgraph.runner import LLGraphCPU T = rffi.CArrayPtr(rffi.TIME_T) external = rffi.llexternal("time", [T], rffi.TIME_T, threadsafe=True) diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -658,7 +658,7 @@ return cpu.ts.cast_to_ref(ptr) @staticmethod def show(cpu, gcref): - from pypy.rpython.annlowlevel import cast_base_ptr_to_instance + from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance ptr = cpu.ts.cast_to_baseclass(gcref) return cast_base_ptr_to_instance(AllVirtuals, ptr) diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py --- a/rpython/jit/metainterp/history.py +++ b/rpython/jit/metainterp/history.py @@ -159,7 +159,7 @@ @staticmethod def show(cpu, descr_gcref): - from pypy.rpython.annlowlevel import cast_base_ptr_to_instance + from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance descr_ptr = cpu.ts.cast_to_baseclass(descr_gcref) return cast_base_ptr_to_instance(AbstractDescr, descr_ptr) diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py --- a/rpython/jit/metainterp/test/test_ajit.py +++ b/rpython/jit/metainterp/test/test_ajit.py @@ -3973,7 +3973,7 @@ assert self.interp_operations(f, [3]) == 6 def test_gc_add_memory_pressure(self): - from pypy.rlib import rgc + from rpython.rlib import rgc def f(): rgc.add_memory_pressure(1234) @@ -3982,7 +3982,7 @@ self.interp_operations(f, []) def test_external_call(self): - from pypy.rlib.objectmodel import invoke_around_extcall + from rpython.rlib.objectmodel import invoke_around_extcall T = rffi.CArrayPtr(rffi.TIME_T) external = rffi.llexternal("time", [T], rffi.TIME_T) diff --git a/rpython/jit/metainterp/test/test_fficall.py b/rpython/jit/metainterp/test/test_fficall.py --- a/rpython/jit/metainterp/test/test_fficall.py +++ b/rpython/jit/metainterp/test/test_fficall.py @@ -105,8 +105,8 @@ class TestFfiCall(FfiCallTests, LLJitMixin): def test_jit_fii_vref(self): - from pypy.rlib import clibffi - from pypy.rlib.jit_libffi import jit_ffi_prep_cif, jit_ffi_call + from rpython.rlib import clibffi + from rpython.rlib.jit_libffi import jit_ffi_prep_cif, jit_ffi_call math_sin = intmask(ctypes.cast(ctypes.CDLL(None).sin, ctypes.c_void_p).value) diff --git a/rpython/jit/metainterp/test/test_warmspot.py b/rpython/jit/metainterp/test/test_warmspot.py --- a/rpython/jit/metainterp/test/test_warmspot.py +++ b/rpython/jit/metainterp/test/test_warmspot.py @@ -537,9 +537,9 @@ def test_callback_jit_merge_point(self): - from pypy.rlib.objectmodel import register_around_callback_hook - from pypy.rpython.lltypesystem import lltype, rffi - from pypy.translator.tool.cbuild import ExternalCompilationInfo + from rpython.rlib.objectmodel import register_around_callback_hook + from rpython.rtyper.lltypesystem import lltype, rffi + from rpython.translator.tool.cbuild import ExternalCompilationInfo callback_jit_driver = JitDriver(greens = ['name'], reds = 'auto') diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py --- a/rpython/rlib/objectmodel.py +++ b/rpython/rlib/objectmodel.py @@ -599,8 +599,8 @@ """ Register a hook that's called before a callback from C calls RPython. Primary usage is for JIT to have 'started from' hook. """ - from pypy.rpython.lltypesystem import rffi - from pypy.rpython.annlowlevel import llhelper + from rpython.rtyper.lltypesystem import rffi + from rpython.rtyper.annlowlevel import llhelper rffi.aroundstate.callback_hook = hook llhelper(rffi.CallbackHookPtr, hook) From noreply at buildbot.pypy.org Sun Jan 13 11:03:50 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 13 Jan 2013 11:03:50 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fight with calls, not finished Message-ID: <20130113100350.5F5BF1C029E@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60017:d6934bbf2a8a Date: 2013-01-13 12:03 +0200 http://bitbucket.org/pypy/pypy/changeset/d6934bbf2a8a/ Log: fight with calls, not finished diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py --- a/pypy/jit/backend/test/runner_test.py +++ b/pypy/jit/backend/test/runner_test.py @@ -565,10 +565,11 @@ [funcbox, ConstInt(num), BoxInt(num)], 'int', descr=calldescr) assert res.value == 2 * num - if cpu.supports_floats: def func(f0, f1, f2, f3, f4, f5, f6, i0, f7, i1, f8, f9): + import pdb + pdb.set_trace() return f0 + f1 + f2 + f3 + f4 + f5 + f6 + float(i0 + i1) + f7 + f8 + f9 F = lltype.Float I = lltype.Signed diff --git a/pypy/jit/backend/x86/arch.py b/pypy/jit/backend/x86/arch.py --- a/pypy/jit/backend/x86/arch.py +++ b/pypy/jit/backend/x86/arch.py @@ -43,8 +43,9 @@ # XXX FORCE_INDEX might need to go to GC frame if WORD == 4: + # XXX rethink the fixed size # ebp + ebx + esi + edi + 4 extra words + force_index = 9 words - FRAME_FIXED_SIZE = 12 # 9 aligned to 16 bytes = 4 * WORD + FRAME_FIXED_SIZE = 6 FORCE_INDEX_OFS = 0 SAVED_REGISTERS = 1 # range(1, 5) MY_COPY_OF_REGS = 5 # range(5, 9) @@ -53,7 +54,7 @@ # reg, we don't save it else: # rbp + rbx + r12 + r13 + r14 + r15 + 11 extra words + force_index = 18 - FRAME_FIXED_SIZE = 18 # 18 aligned to 16 bytes = 2 * WORD + FRAME_FIXED_SIZE = 6 FORCE_INDEX_OFS = 0 SAVED_REGISTERS = 1 # range(1, 7) MY_COPY_OF_REGS = 7 # range(7, 18) diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -18,7 +18,7 @@ xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, r8, r9, r10, r11, r12, r13, r14, r15, X86_64_SCRATCH_REG, X86_64_XMM_SCRATCH_REG, RegLoc, StackLoc, ConstFloatLoc, ImmedLoc, AddressLoc, imm, - imm0, imm1, FloatImmedLoc, RawStackLoc) + imm0, imm1, FloatImmedLoc, RawStackLoc, RawEspLoc) from pypy.rlib.objectmodel import we_are_translated, specialize from pypy.jit.backend.x86 import rx86, codebuf from pypy.jit.metainterp.resoperation import rop, ResOperation @@ -749,9 +749,9 @@ def _call_header(self): # NB. the shape of the frame is hard-coded in get_basic_shape() too. # Also, make sure this is consistent with FRAME_FIXED_SIZE. - self.mc.PUSH_r(ebp.value) # XXX should be LEA? - self.mc.ADD_ri(esp.value, FRAME_FIXED_SIZE * WORD) + self.mc.SUB_ri(esp.value, FRAME_FIXED_SIZE * WORD) + self.mc.MOV_sr(0, ebp.value) if IS_X86_64: descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) _, ofs, _ = unpack_arraydescr(descrs.arraydescr) @@ -759,8 +759,8 @@ else: xxx - for loc in self.cpu.CALLEE_SAVE_REGISTERS: - self.mc.PUSH_r(loc.value) + for i, loc in enumerate(self.cpu.CALLEE_SAVE_REGISTERS): + self.mc.MOV_sr((i + 1) * WORD, loc.value) gcrootmap = self.cpu.gc_ll_descr.gcrootmap if gcrootmap and gcrootmap.is_shadow_stack: @@ -785,16 +785,16 @@ self._call_header() def _call_footer(self): - self.mc.SUB_ri(esp.value, FRAME_FIXED_SIZE * WORD) - gcrootmap = self.cpu.gc_ll_descr.gcrootmap if gcrootmap and gcrootmap.is_shadow_stack: self._call_footer_shadowstack(gcrootmap) for i in range(len(self.cpu.CALLEE_SAVE_REGISTERS)-1, -1, -1): - self.mc.POP_r(self.cpu.CALLEE_SAVE_REGISTERS[i].value) + self.mc.MOV_rs(self.cpu.CALLEE_SAVE_REGISTERS[i].value, + (i + 1) * WORD) - self.mc.POP_r(ebp.value) + self.mc.MOV_rs(ebp.value, 0) + self.mc.ADD_ri(esp.value, FRAME_FIXED_SIZE * WORD) self.mc.RET() def _call_header_shadowstack(self, gcrootmap): @@ -1056,7 +1056,7 @@ argtypes=None, callconv=FFI_DEFAULT_ABI): if IS_X86_64: return self._emit_call_64(force_index, x, arglocs, start, argtypes) - + XXX p = 0 n = len(arglocs) for i in range(start, n): @@ -1099,60 +1099,42 @@ dst_locs = [] xmm_src_locs = [] xmm_dst_locs = [] - pass_on_stack = [] singlefloats = None # In reverse order for use with pop() unused_gpr = [r9, r8, ecx, edx, esi, edi] unused_xmm = [xmm7, xmm6, xmm5, xmm4, xmm3, xmm2, xmm1, xmm0] + on_stack = 0 for i in range(start, len(arglocs)): loc = arglocs[i] - # XXX: Should be much simplier to tell whether a location is a - # float! It's so ugly because we have to "guard" the access to - # .type with isinstance, since not all AssemblerLocation classes - # are "typed" - if ((isinstance(loc, RegLoc) and loc.is_xmm) or - (isinstance(loc, StackLoc) and loc.type == FLOAT) or - (isinstance(loc, ConstFloatLoc))): + if loc.is_float(): + xmm_src_locs.append(loc) if len(unused_xmm) > 0: - xmm_src_locs.append(loc) xmm_dst_locs.append(unused_xmm.pop()) else: - pass_on_stack.append(loc) + xmm_dst_locs.append(RawEspLoc(on_stack * WORD, FLOAT)) + on_stack += 1 elif argtypes is not None and argtypes[i-start] == 'S': # Singlefloat argument if len(unused_xmm) > 0: - if singlefloats is None: singlefloats = [] + if singlefloats is None: + singlefloats = [] singlefloats.append((loc, unused_xmm.pop())) else: - pass_on_stack.append(loc) + singlefloats.append((loc, RawEspLoc(on_stack * WORD, INT))) + on_stack += 1 else: + src_locs.append(loc) if len(unused_gpr) > 0: - src_locs.append(loc) dst_locs.append(unused_gpr.pop()) else: - pass_on_stack.append(loc) + dst_locs.append(RawEspLoc(on_stack * WORD, INT)) + on_stack += 1 - # Emit instructions to pass the stack arguments - # XXX: Would be nice to let remap_frame_layout take care of this, but - # we'd need to create something like StackLoc, but relative to esp, - # and I don't know if it's worth it. - for i in range(len(pass_on_stack)): - loc = pass_on_stack[i] - if not isinstance(loc, RegLoc): - if isinstance(loc, StackLoc) and loc.type == FLOAT: - self.mc.MOVSD(X86_64_XMM_SCRATCH_REG, loc) - self.mc.MOVSD_sx(i*WORD, X86_64_XMM_SCRATCH_REG.value) - else: - self.mc.MOV(X86_64_SCRATCH_REG, loc) - self.mc.MOV_sr(i*WORD, X86_64_SCRATCH_REG.value) - else: - # It's a register - if loc.is_xmm: - self.mc.MOVSD_sx(i*WORD, loc.value) - else: - self.mc.MOV_sr(i*WORD, loc.value) + align = align_stack_words(on_stack) + if align: + self.mc.SUB_ri(esp.value, align * WORD) # Handle register arguments: first remap the xmm arguments remap_frame_layout(self, xmm_src_locs, xmm_dst_locs, @@ -1172,10 +1154,11 @@ dst_locs.append(r10) x = r10 remap_frame_layout(self, src_locs, dst_locs, X86_64_SCRATCH_REG) - + # align self.mc.CALL(x) + if align: + self.mc.ADD_ri(esp.value, align * WORD) self.mark_gc_roots(force_index) - self._regalloc.needed_extra_stack_locations(len(pass_on_stack)) def call(self, addr, args, res): force_index = self.write_new_force_index() diff --git a/pypy/jit/backend/x86/regloc.py b/pypy/jit/backend/x86/regloc.py --- a/pypy/jit/backend/x86/regloc.py +++ b/pypy/jit/backend/x86/regloc.py @@ -53,7 +53,7 @@ self.type = type def _getregkey(self): - return -(self.value + 1) + return -(self.value * 2 + 2) def get_width(self): if self.type == FLOAT: @@ -66,6 +66,36 @@ def assembler(self): return repr(self) + def is_float(self): + return self.type == FLOAT + +class RawEspLoc(AssemblerLocation): + """ Esp-based location + """ + _immutable_ = True + _location_code = 's' + + def __init__(self, value, type): + self.value = value + self.type = type + + def _getregkey(self): + return -(self.value * 2 + 1) + + def get_width(self): + if self.type == FLOAT: + return 8 + return WORD + + def __repr__(self): + return '%d(%%esp)' % (self.value,) + + def assembler(self): + return repr(self) + + def is_float(self): + return self.type == FLOAT + class StackLoc(RawStackLoc): def __init__(self, position, ebp_offset, type): # _getregkey() returns self.value; the value returned must not @@ -116,6 +146,9 @@ else: return eax + def is_float(self): + return self.is_xmm + class ImmediateAssemblerLocation(AssemblerLocation): _immutable_ = True @@ -123,10 +156,11 @@ _immutable_ = True _location_code = 'i' - def __init__(self, value): + def __init__(self, value, is_float=False): from pypy.rpython.lltypesystem import rffi, lltype # force as a real int self.value = rffi.cast(lltype.Signed, value) + self._is_float = is_float def getint(self): return self.value @@ -143,6 +177,9 @@ val -= 0x100 return ImmedLoc(val) + def is_float(self): + return self._is_float + class AddressLoc(AssemblerLocation): _immutable_ = True @@ -177,6 +214,10 @@ def get_width(self): return WORD + def is_float(self): + return False # not 100% true, but we don't use AddressLoc for locations + # really, so it's ok + def value_a(self): return self.loc_a @@ -224,6 +265,9 @@ def __repr__(self): return '' % (self.value,) + def is_float(self): + return True + if IS_X86_32: class FloatImmedLoc(ImmediateAssemblerLocation): # This stands for an immediate float. It cannot be directly used in @@ -255,11 +299,14 @@ floatvalue = longlong.getrealfloat(self.aslonglong) return '' % (floatvalue,) + def is_float(self): + return True + if IS_X86_64: def FloatImmedLoc(floatstorage): from pypy.rlib.longlong2float import float2longlong value = intmask(float2longlong(floatstorage)) - return ImmedLoc(value) + return ImmedLoc(value, True) REGLOCS = [RegLoc(i, is_xmm=False) for i in range(16)] From noreply at buildbot.pypy.org Sun Jan 13 11:06:59 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 13 Jan 2013 11:06:59 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: (arigo, fijal) fix Message-ID: <20130113100659.7E5DB1C029E@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60018:49df119bdf2e Date: 2013-01-13 12:06 +0200 http://bitbucket.org/pypy/pypy/changeset/49df119bdf2e/ Log: (arigo, fijal) fix diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py --- a/pypy/jit/backend/test/runner_test.py +++ b/pypy/jit/backend/test/runner_test.py @@ -568,8 +568,6 @@ if cpu.supports_floats: def func(f0, f1, f2, f3, f4, f5, f6, i0, f7, i1, f8, f9): - import pdb - pdb.set_trace() return f0 + f1 + f2 + f3 + f4 + f5 + f6 + float(i0 + i1) + f7 + f8 + f9 F = lltype.Float I = lltype.Signed diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -1132,7 +1132,7 @@ dst_locs.append(RawEspLoc(on_stack * WORD, INT)) on_stack += 1 - align = align_stack_words(on_stack) + align = align_stack_words(on_stack + 1) - 1 if align: self.mc.SUB_ri(esp.value, align * WORD) From noreply at buildbot.pypy.org Sun Jan 13 11:08:12 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 13 Jan 2013 11:08:12 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fix Message-ID: <20130113100812.45D991C029E@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60019:9dbce99922e5 Date: 2013-01-13 12:07 +0200 http://bitbucket.org/pypy/pypy/changeset/9dbce99922e5/ Log: fix diff --git a/pypy/jit/backend/x86/runner.py b/pypy/jit/backend/x86/runner.py --- a/pypy/jit/backend/x86/runner.py +++ b/pypy/jit/backend/x86/runner.py @@ -134,7 +134,7 @@ if IS_X86_32: num += WORD else: - xxx + self.set_ref_value(frame, num, arg) num += WORD descr_no = func(ll_frame) finally: From noreply at buildbot.pypy.org Sun Jan 13 11:11:52 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 13 Jan 2013 11:11:52 +0100 (CET) Subject: [pypy-commit] pypy default: Fix Message-ID: <20130113101152.992EE1C029E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r60020:cd501411ebdc Date: 2013-01-13 11:11 +0100 http://bitbucket.org/pypy/pypy/changeset/cd501411ebdc/ Log: Fix diff --git a/pypy/jit/backend/test/test_random.py b/pypy/jit/backend/test/test_random.py --- a/pypy/jit/backend/test/test_random.py +++ b/pypy/jit/backend/test/test_random.py @@ -618,6 +618,7 @@ if v not in used_later: endvars.append(v) r.shuffle(endvars) + endvars = endvars[:1] loop.operations.append(ResOperation(rop.FINISH, endvars, None, descr=BasicFailDescr())) if builder.should_fail_by: From noreply at buildbot.pypy.org Sun Jan 13 11:20:38 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 13 Jan 2013 11:20:38 +0100 (CET) Subject: [pypy-commit] pypy default: Fix Message-ID: <20130113102038.9CA691C054C@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r60021:cb0037c8f67c Date: 2013-01-13 11:20 +0100 http://bitbucket.org/pypy/pypy/changeset/cb0037c8f67c/ Log: Fix diff --git a/pypy/jit/backend/test/test_random.py b/pypy/jit/backend/test/test_random.py --- a/pypy/jit/backend/test/test_random.py +++ b/pypy/jit/backend/test/test_random.py @@ -26,6 +26,8 @@ class OperationBuilder(object): def __init__(self, cpu, loop, vars): self.cpu = cpu + if not hasattr(cpu, '_faildescr_keepalive'): + cpu._faildescr_keepalive = [] self.fakemetainterp = FakeMetaInterp() self.loop = loop self.intvars = [box for box in vars if isinstance(box, BoxInt)] @@ -206,6 +208,11 @@ if pytest.config.option.output: s.close() + def getfaildescr(self): + descr = BasicFailDescr() + self.cpu._faildescr_keepalive.append(descr) + return descr + class CannotProduceOperation(Exception): pass @@ -287,7 +294,7 @@ builder.intvars[:] = original_intvars else: op = ResOperation(rop.GUARD_NO_OVERFLOW, [], None) - op.setdescr(BasicFailDescr()) + op.setdescr(builder.getfaildescr()) op.setfailargs(fail_subset) builder.loop.operations.append(op) @@ -357,7 +364,7 @@ def produce_into(self, builder, r): op, passing = self.gen_guard(builder, r) builder.loop.operations.append(op) - op.setdescr(BasicFailDescr()) + op.setdescr(builder.getfaildescr()) op.setfailargs(builder.subset_of_intvars(r)) if not passing: builder.should_fail_by = op @@ -620,7 +627,7 @@ r.shuffle(endvars) endvars = endvars[:1] loop.operations.append(ResOperation(rop.FINISH, endvars, None, - descr=BasicFailDescr())) + descr=builder.getfaildescr())) if builder.should_fail_by: self.should_fail_by = builder.should_fail_by self.guard_op = builder.guard_op @@ -713,7 +720,7 @@ else: op = ResOperation(rop.GUARD_EXCEPTION, [guard_op._exc_box], BoxPtr()) - op.setdescr(BasicFailDescr()) + op.setdescr(self.builder.getfaildescr()) op.setfailargs([]) return op From noreply at buildbot.pypy.org Sun Jan 13 11:26:42 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 13 Jan 2013 11:26:42 +0100 (CET) Subject: [pypy-commit] pypy default: Try to fix the test, but it still fails Message-ID: <20130113102642.21EF81C054C@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r60022:f1ea1ef8cd72 Date: 2013-01-13 11:22 +0100 http://bitbucket.org/pypy/pypy/changeset/f1ea1ef8cd72/ Log: Try to fix the test, but it still fails diff --git a/pypy/jit/metainterp/test/test_fficall.py b/pypy/jit/metainterp/test/test_fficall.py --- a/pypy/jit/metainterp/test/test_fficall.py +++ b/pypy/jit/metainterp/test/test_fficall.py @@ -105,7 +105,7 @@ class TestFfiCall(FfiCallTests, LLJitMixin): - def test_jit_fii_vref(self): + def test_jit_ffi_vref(self): from pypy.rlib import clibffi from pypy.rlib.jit_libffi import jit_ffi_prep_cif, jit_ffi_call @@ -136,9 +136,10 @@ res = exb[3] lltype.free(exb, flavor='raw') # - lltype.free(atypes, flavor='raw') return res # res = self.interp_operations(f, []) lltype.free(cd, flavor='raw') assert res == math.sin(1.23) + + lltype.free(atypes, flavor='raw') From noreply at buildbot.pypy.org Sun Jan 13 12:23:09 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 13 Jan 2013 12:23:09 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: call_may_force Message-ID: <20130113112309.61D021C029E@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60023:4fe6e6433271 Date: 2013-01-13 13:22 +0200 http://bitbucket.org/pypy/pypy/changeset/4fe6e6433271/ Log: call_may_force diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py --- a/pypy/jit/backend/llsupport/gc.py +++ b/pypy/jit/backend/llsupport/gc.py @@ -117,6 +117,8 @@ descrs.jf_descr = cpu.fielddescrof(jitframe.JITFRAME, 'jf_descr') descrs.jf_guard_exc = cpu.fielddescrof(jitframe.JITFRAME, 'jf_guard_exc') + descrs.jf_force_index = cpu.fielddescrof(jitframe.JITFRAME, + 'jf_force_index') descrs.jf_frame_info = cpu.fielddescrof(jitframe.JITFRAME, 'jf_frame_info') descrs.jfi_frame_depth = cpu.fielddescrof(jitframe.JITFRAMEINFO, diff --git a/pypy/jit/backend/llsupport/jitframe.py b/pypy/jit/backend/llsupport/jitframe.py --- a/pypy/jit/backend/llsupport/jitframe.py +++ b/pypy/jit/backend/llsupport/jitframe.py @@ -24,6 +24,8 @@ # This field is also set immediately before doing CALL_MAY_FORCE # or CALL_ASSEMBLER. ('jf_descr', llmemory.GCREF), + # index of guard_not_force + ('jf_force_index', lltype.Signed), # For the front-end: a GCREF for the savedata ('jf_savedata', llmemory.GCREF), # For GUARD_(NO)_EXCEPTION and GUARD_NOT_FORCED: the exception we @@ -32,7 +34,8 @@ # exception is not stored there, but is simply kept as a variable there) ('jf_guard_exc', llmemory.GCREF), # the actual frame - ('jf_frame', lltype.Array(lltype.Signed, hints={'nolength': True})) + ('jf_frame', lltype.Array(lltype.Signed)) + # it should be: , hints={'nolength': True})), but ll2ctypes is complaining ) JITFRAMEPTR = lltype.Ptr(JITFRAME) diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py --- a/pypy/jit/backend/llsupport/llmodel.py +++ b/pypy/jit/backend/llsupport/llmodel.py @@ -169,13 +169,11 @@ return deadframe.jf_guard_exc def set_savedata_ref(self, deadframe, data): - XXX - deadframe = lltype.cast_opaque_ptr(jitframe.DEADFRAMEPTR, deadframe) + deadframe = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, deadframe) deadframe.jf_savedata = data def get_savedata_ref(self, deadframe): - XXXX - deadframe = lltype.cast_opaque_ptr(jitframe.DEADFRAMEPTR, deadframe) + deadframe = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, deadframe) return deadframe.jf_savedata def free_loop_and_bridges(self, compiled_loop_token): diff --git a/pypy/jit/backend/x86/arch.py b/pypy/jit/backend/x86/arch.py --- a/pypy/jit/backend/x86/arch.py +++ b/pypy/jit/backend/x86/arch.py @@ -40,13 +40,10 @@ # one word for the force_index, and some extra space used only # during a malloc that needs to go via its slow path. -# XXX FORCE_INDEX might need to go to GC frame - if WORD == 4: # XXX rethink the fixed size # ebp + ebx + esi + edi + 4 extra words + force_index = 9 words FRAME_FIXED_SIZE = 6 - FORCE_INDEX_OFS = 0 SAVED_REGISTERS = 1 # range(1, 5) MY_COPY_OF_REGS = 5 # range(5, 9) XXX @@ -55,7 +52,6 @@ else: # rbp + rbx + r12 + r13 + r14 + r15 + 11 extra words + force_index = 18 FRAME_FIXED_SIZE = 6 - FORCE_INDEX_OFS = 0 SAVED_REGISTERS = 1 # range(1, 7) MY_COPY_OF_REGS = 7 # range(7, 18) JITFRAME_FIXED_SIZE = 29 # 13 GPR + 16 XMM diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -12,7 +12,7 @@ from pypy.jit.backend.model import CompiledLoopToken from pypy.jit.backend.x86.regalloc import (RegAlloc, get_ebp_ofs, _get_scale, gpr_reg_mgr_cls, xmm_reg_mgr_cls, _valid_addressing_size) -from pypy.jit.backend.x86.arch import (FRAME_FIXED_SIZE, FORCE_INDEX_OFS, WORD, +from pypy.jit.backend.x86.arch import (FRAME_FIXED_SIZE, WORD, IS_X86_32, IS_X86_64) from pypy.jit.backend.x86.regloc import (eax, ecx, edx, ebx, esp, ebp, esi, edi, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, r8, r9, r10, r11, @@ -79,7 +79,6 @@ self.propagate_exception_path = 0 self.gcrootmap_retaddr_forced = 0 self.teardown() - self.force_token_to_dead_frame = {} # XXX temporary hack def set_debug(self, v): r = self._debug @@ -559,7 +558,7 @@ 'b', descr_number) descr = self.cpu.get_fail_descr_from_number(descr_number) - arglocs = self.rebuild_faillocs_from_descr(descr) + arglocs = self.rebuild_faillocs_from_descr(descr, inputargs) regalloc = RegAlloc(self, self.cpu.translate_support_code) startpos = self.mc.get_relative_pos() operations = regalloc.prepare_bridge(inputargs, arglocs, @@ -1154,7 +1153,6 @@ dst_locs.append(r10) x = r10 remap_frame_layout(self, src_locs, dst_locs, X86_64_SCRATCH_REG) - # align self.mc.CALL(x) if align: self.mc.ADD_ri(esp.value, align * WORD) @@ -1172,6 +1170,7 @@ # instruction that doesn't already have a force_index. gcrootmap = self.cpu.gc_ll_descr.gcrootmap if gcrootmap and gcrootmap.is_shadow_stack: + xxx clt = self.current_clt force_index = clt.reserve_and_record_some_faildescr_index() self.mc.MOV_bi(FORCE_INDEX_OFS, force_index) @@ -1794,9 +1793,6 @@ mc.PUSH(imm(fail_index)) mc.JMP_r(X86_64_SCRATCH_REG.value) # write tight data that describes the failure recovery - if guardtok.is_guard_not_forced: - XXX - mc.writechar(chr(self.CODE_FORCED)) positions = [0] * len(guardtok.fail_locs) for i, loc in enumerate(guardtok.fail_locs): if loc is None: @@ -1805,29 +1801,34 @@ positions[i] = loc.value else: assert isinstance(loc, RegLoc) - v = (gpr_reg_mgr_cls.all_reg_indexes[loc.value] + - loc.is_xmm * len(gpr_reg_mgr_cls.all_regs)) + if loc.is_xmm: + v = len(gpr_reg_mgr_cls.all_regs) + loc.value + else: + v = gpr_reg_mgr_cls.all_reg_indexes[loc.value] positions[i] = v * WORD guardtok.faildescr.rd_locs = positions # write fail_index too # for testing the decoding, write a final byte 0xCC return startpos - def rebuild_faillocs_from_descr(self, descr): + def rebuild_faillocs_from_descr(self, descr, inputargs): locs = [] GPR_REGS = len(gpr_reg_mgr_cls.all_regs) XMM_REGS = len(xmm_reg_mgr_cls.all_regs) + input_i = 0 for pos in descr.rd_locs: if pos == -1: - pass + continue elif pos < GPR_REGS * WORD: locs.append(gpr_reg_mgr_cls.all_regs[pos // WORD]) elif pos < (GPR_REGS + XMM_REGS) * WORD: - locs.append(xmm_reg_mgr_cls.xrm.all_regs[pos // WORD]) + pos = pos // WORD - GPR_REGS + locs.append(xmm_reg_mgr_cls.all_regs[pos]) else: i = pos // WORD - 2 * self.cpu.NUM_REGS - tp = xxx + tp = inputargs[input_i].type locs.append(StackLoc(i, pos, tp)) + input_i += 1 return locs def setup_failure_recovery(self): @@ -1961,13 +1962,18 @@ arglocs, result_loc): faildescr = guard_op.getdescr() fail_index = self.cpu.get_fail_descr_number(faildescr) - self.mc.MOV_bi(FORCE_INDEX_OFS, fail_index) + descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) + base_ofs = self.cpu.unpack_arraydescr(descrs.arraydescr) + ofs = self.cpu.unpack_fielddescr(descrs.jf_force_index) + self.mc.MOV_bi(ofs - base_ofs, fail_index) self._genop_call(op, arglocs, result_loc, fail_index) - self.mc.CMP_bi(FORCE_INDEX_OFS, 0) - self.implement_guard(guard_token, 'L') + ofs_fail = self.cpu.unpack_fielddescr(descrs.jf_descr) + self.mc.CMP_bi(ofs_fail - base_ofs, 0) + self.implement_guard(guard_token, 'NE') def genop_guard_call_release_gil(self, op, guard_op, guard_token, arglocs, result_loc): + xxx # first, close the stack in the sense of the asmgcc GC root tracker gcrootmap = self.cpu.gc_ll_descr.gcrootmap if gcrootmap: @@ -2087,6 +2093,7 @@ def genop_guard_call_assembler(self, op, guard_op, guard_token, arglocs, result_loc): + xxx faildescr = guard_op.getdescr() fail_index = self.cpu.get_fail_descr_number(faildescr) self.mc.MOV_bi(FORCE_INDEX_OFS, fail_index) diff --git a/pypy/jit/backend/x86/runner.py b/pypy/jit/backend/x86/runner.py --- a/pypy/jit/backend/x86/runner.py +++ b/pypy/jit/backend/x86/runner.py @@ -8,8 +8,7 @@ from pypy.jit.codewriter import longlong from pypy.jit.metainterp import history, compile from pypy.jit.backend.x86.assembler import Assembler386 -from pypy.jit.backend.x86.arch import (FORCE_INDEX_OFS, IS_X86_32, - JITFRAME_FIXED_SIZE) +from pypy.jit.backend.x86.arch import IS_X86_32, JITFRAME_FIXED_SIZE from pypy.jit.backend.x86.profagent import ProfileAgent from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU from pypy.jit.backend.llsupport import jitframe @@ -33,8 +32,6 @@ def __init__(self, rtyper, stats, opts=None, translate_support_code=False, gcdescr=None): - if gcdescr is not None: - gcdescr.force_index_ofs = FORCE_INDEX_OFS AbstractLLCPU.__init__(self, rtyper, stats, opts, translate_support_code, gcdescr) @@ -159,26 +156,13 @@ immortal=True) def force(self, addr_of_force_token): - TP = rffi.CArrayPtr(lltype.Signed) - addr_of_force_index = addr_of_force_token + FORCE_INDEX_OFS - fail_index = rffi.cast(TP, addr_of_force_index)[0] - assert fail_index >= 0, "already forced!" + descr = self.gc_ll_descr.getframedescrs(self).arraydescr + ofs = self.unpack_arraydescr(descr) + frame = rffi.cast(jitframe.JITFRAMEPTR, addr_of_force_token - ofs) + fail_index = frame.jf_force_index faildescr = self.get_fail_descr_from_number(fail_index) - rffi.cast(TP, addr_of_force_index)[0] = ~fail_index - frb = self.assembler._find_failure_recovery_bytecode(faildescr) - bytecode = rffi.cast(rffi.UCHARP, frb) - assert (rffi.cast(lltype.Signed, bytecode[0]) == - self.assembler.CODE_FORCED) - bytecode = rffi.ptradd(bytecode, 1) - deadframe = self.assembler.grab_frame_values( - self, - bytecode, - addr_of_force_token, - self.all_null_registers) - assert self.get_latest_descr(deadframe) is faildescr - self.assembler.force_token_to_dead_frame[addr_of_force_token] = ( - deadframe) - return deadframe + frame.jf_descr = cast_instance_to_gcref(faildescr) + return frame def redirect_call_assembler(self, oldlooptoken, newlooptoken): self.assembler.redirect_call_assembler(oldlooptoken, newlooptoken) From noreply at buildbot.pypy.org Sun Jan 13 12:24:17 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 13 Jan 2013 12:24:17 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: call_release_gil Message-ID: <20130113112417.AE2181C029E@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60024:781162d0409d Date: 2013-01-13 13:23 +0200 http://bitbucket.org/pypy/pypy/changeset/781162d0409d/ Log: call_release_gil diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -1973,7 +1973,6 @@ def genop_guard_call_release_gil(self, op, guard_op, guard_token, arglocs, result_loc): - xxx # first, close the stack in the sense of the asmgcc GC root tracker gcrootmap = self.cpu.gc_ll_descr.gcrootmap if gcrootmap: @@ -1981,14 +1980,18 @@ # do the call faildescr = guard_op.getdescr() fail_index = self.cpu.get_fail_descr_number(faildescr) - self.mc.MOV_bi(FORCE_INDEX_OFS, fail_index) + descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) + base_ofs = self.cpu.unpack_arraydescr(descrs.arraydescr) + ofs = self.cpu.unpack_fielddescr(descrs.jf_force_index) + self.mc.MOV_bi(ofs - base_ofs, fail_index) self._genop_call(op, arglocs, result_loc, fail_index) # then reopen the stack if gcrootmap: self.call_reacquire_gil(gcrootmap, result_loc) # finally, the guard_not_forced - self.mc.CMP_bi(FORCE_INDEX_OFS, 0) - self.implement_guard(guard_token, 'L') + ofs_fail = self.cpu.unpack_fielddescr(descrs.jf_descr) + self.mc.CMP_bi(ofs_fail - base_ofs, 0) + self.implement_guard(guard_token, 'NE') def call_release_gil(self, gcrootmap, save_registers): # First, we need to save away the registers listed in From noreply at buildbot.pypy.org Sun Jan 13 13:17:53 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 13 Jan 2013 13:17:53 +0100 (CET) Subject: [pypy-commit] pypy default: Don't use finally to call gc_thread_die: the exception has really nowhere to go. Message-ID: <20130113121753.629251C029E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r60025:509f13a420c6 Date: 2013-01-13 12:20 +0100 http://bitbucket.org/pypy/pypy/changeset/509f13a420c6/ Log: Don't use finally to call gc_thread_die: the exception has really nowhere to go. diff --git a/pypy/module/thread/os_thread.py b/pypy/module/thread/os_thread.py --- a/pypy/module/thread/os_thread.py +++ b/pypy/module/thread/os_thread.py @@ -2,6 +2,7 @@ Thread support based on OS-level threads. """ +import os from pypy.module.thread import ll_thread as thread from pypy.module.thread.error import wrap_thread_error from pypy.interpreter.error import OperationError, operationerrfmt @@ -91,14 +92,18 @@ # run! try: bootstrapper.run(space, w_callable, args) - finally: - bootstrapper.nbthreads -= 1 - # clean up space.threadlocals to remove the ExecutionContext - # entry corresponding to the current thread + # done + except Exception, e: + # oups! last-level attempt to recover. try: - space.threadlocals.leave_thread(space) - finally: - thread.gc_thread_die() + STDERR = 2 + os.write(STDERR, "Thread exited with ") + os.write(STDERR, str(e)) + os.write(STDERR, "\n") + except OSError: + pass + bootstrapper.nbthreads -= 1 + thread.gc_thread_die() bootstrap = staticmethod(bootstrap) def acquire(space, w_callable, args): @@ -129,6 +134,9 @@ where = 'thread %d started by ' % ident e.write_unraisable(space, where, w_callable) e.clear(space) + # clean up space.threadlocals to remove the ExecutionContext + # entry corresponding to the current thread + space.threadlocals.leave_thread(space) run = staticmethod(run) bootstrapper = Bootstrapper() From noreply at buildbot.pypy.org Sun Jan 13 13:17:54 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 13 Jan 2013 13:17:54 +0100 (CET) Subject: [pypy-commit] pypy default: Pushing and popping roots around gc_thread_run() and gc_thread_die() Message-ID: <20130113121754.9E0A11C029E@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r60026:e4428d6488d8 Date: 2013-01-13 13:21 +0100 http://bitbucket.org/pypy/pypy/changeset/e4428d6488d8/ Log: Pushing and popping roots around gc_thread_run() and gc_thread_die() doesn't make sense. Fixed by asserting we don't actually have to push any. Fix a test. PyPy translation was fixed by 509f13a420c6. diff --git a/pypy/module/thread/test/test_gil.py b/pypy/module/thread/test/test_gil.py --- a/pypy/module/thread/test/test_gil.py +++ b/pypy/module/thread/test/test_gil.py @@ -61,8 +61,9 @@ def bootstrap(): try: runme() - finally: - thread.gc_thread_die() + except Exception, e: + assert 0 + thread.gc_thread_die() def f(): state.data = [] state.datalen1 = 0 diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py --- a/pypy/rpython/memory/gctransform/framework.py +++ b/pypy/rpython/memory/gctransform/framework.py @@ -979,6 +979,7 @@ assert self.translator.config.translation.thread if hasattr(self.root_walker, 'thread_run_ptr'): livevars = self.push_roots(hop) + assert not livevars, "live GC var around %s!" % (hop.spaceop,) hop.genop("direct_call", [self.root_walker.thread_run_ptr]) self.pop_roots(hop, livevars) @@ -993,6 +994,7 @@ assert self.translator.config.translation.thread if hasattr(self.root_walker, 'thread_die_ptr'): livevars = self.push_roots(hop) + assert not livevars, "live GC var around %s!" % (hop.spaceop,) hop.genop("direct_call", [self.root_walker.thread_die_ptr]) self.pop_roots(hop, livevars) From noreply at buildbot.pypy.org Sun Jan 13 14:38:31 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 13 Jan 2013 14:38:31 +0100 (CET) Subject: [pypy-commit] pypy default: An assert that fails (unless I messed up with gdb) on jit+shadowstack translations. Message-ID: <20130113133831.2518A1C0884@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r60027:467151f84874 Date: 2013-01-13 14:18 +0100 http://bitbucket.org/pypy/pypy/changeset/467151f84874/ Log: An assert that fails (unless I messed up with gdb) on jit+shadowstack translations. diff --git a/pypy/rpython/memory/gctransform/shadowstack.py b/pypy/rpython/memory/gctransform/shadowstack.py --- a/pypy/rpython/memory/gctransform/shadowstack.py +++ b/pypy/rpython/memory/gctransform/shadowstack.py @@ -326,6 +326,8 @@ self.gcdata.root_stack_top = llmemory.NULL # to detect missing restore def forget_current_state(self): + ll_assert(self.gcdata.root_stack_base == self.gcdata.root_stack_top, + "forget_current_state: shadowstack not empty!") if self.unused_full_stack: llmemory.raw_free(self.unused_full_stack) self.unused_full_stack = self.gcdata.root_stack_base From noreply at buildbot.pypy.org Sun Jan 13 14:38:32 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 13 Jan 2013 14:38:32 +0100 (CET) Subject: [pypy-commit] pypy default: Add jitting to C callbacks only on demand, not systematically. The RPython callback must have Message-ID: <20130113133832.81ED61C0884@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r60028:ad1b2b6d8ced Date: 2013-01-13 14:41 +0100 http://bitbucket.org/pypy/pypy/changeset/ad1b2b6d8ced/ Log: Add jitting to C callbacks only on demand, not systematically. The RPython callback must have _callback_hook_="somestring". Might fix issue #1363, caused by the callback hook getting in the way of the shadowstack for the thread's bootstrapper. diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py --- a/pypy/module/_cffi_backend/ccallback.py +++ b/pypy/module/_cffi_backend/ccallback.py @@ -201,3 +201,4 @@ callback.write_error_return_value(ll_res) if ec is not None: cerrno.restore_errno_from(ec) +invoke_callback._callback_hook_ = "CFFI" diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py --- a/pypy/module/pyexpat/interp_pyexpat.py +++ b/pypy/module/pyexpat/interp_pyexpat.py @@ -312,6 +312,7 @@ """ % locals()) exec str(src) + callback._callback_hook_ = "XML_" + name c_name = 'XML_Set' + name callback_type = lltype.Ptr(lltype.FuncType( @@ -340,6 +341,7 @@ else: result = 1 return rffi.cast(rffi.INT, result) +UnknownEncodingHandlerData_callback._callback_hook_ = None # no jit needed callback_type = lltype.Ptr(lltype.FuncType( [rffi.VOIDP, rffi.CCHARP, XML_Encoding_Ptr], rffi.INT)) XML_SetUnknownEncodingHandler = expat_external( diff --git a/pypy/rlib/clibffi.py b/pypy/rlib/clibffi.py --- a/pypy/rlib/clibffi.py +++ b/pypy/rlib/clibffi.py @@ -431,6 +431,7 @@ """ userdata = rffi.cast(USERDATA_P, ll_userdata) userdata.callback(ll_args, ll_res, userdata) +ll_callback._callback_hook_ = "CLIBFFI" class StackCheckError(ValueError): message = None diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py --- a/pypy/rpython/lltypesystem/rffi.py +++ b/pypy/rpython/lltypesystem/rffi.py @@ -279,14 +279,16 @@ callable_name = getattr(callable, '__name__', '?') if callbackholder is not None: callbackholder.callbacks[callable] = True - callable_name_descr = str(callable).replace('"', '\\"') args = ', '.join(['a%d' % i for i in range(len(TP.TO.ARGS))]) + invoke_hook = getattr(callable, '_callback_hook_', None) + assert invoke_hook is None or isinstance(invoke_hook, str) + source = py.code.Source(r""" def inner_wrapper(%(args)s): - if aroundstate is not None: + if invoke_hook is not None and aroundstate is not None: callback_hook = aroundstate.callback_hook if callback_hook: - callback_hook(llstr("%(callable_name_descr)s")) + callback_hook(llstr(invoke_hook)) return callable(%(args)s) inner_wrapper._never_inline_ = True From noreply at buildbot.pypy.org Sun Jan 13 16:06:38 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 13 Jan 2013 16:06:38 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: rewrite call_assembler into allocating frame. not ideal so far sincei t's a call Message-ID: <20130113150638.220991C05DD@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60029:bb9e3d5e3d9d Date: 2013-01-13 17:06 +0200 http://bitbucket.org/pypy/pypy/changeset/bb9e3d5e3d9d/ Log: rewrite call_assembler into allocating frame. not ideal so far sincei t's a call diff --git a/pypy/jit/backend/llsupport/rewrite.py b/pypy/jit/backend/llsupport/rewrite.py --- a/pypy/jit/backend/llsupport/rewrite.py +++ b/pypy/jit/backend/llsupport/rewrite.py @@ -7,6 +7,7 @@ from pypy.jit.backend.llsupport.symbolic import WORD from pypy.jit.backend.llsupport.descr import SizeDescr, ArrayDescr from pypy.jit.backend.llsupport import jitframe +from pypy.rpython.lltypesystem import lltype, llmemory class GcRewriterAssembler(object): @@ -65,7 +66,11 @@ if op.getopnum() == rop.SETARRAYITEM_GC: self.handle_write_barrier_setarrayitem(op) continue - # ---------- + # ---------- call assembler ----------- + if op.getopnum() == rop.CALL_ASSEMBLER: + self.handle_call_assembler(op) + continue + # self.newops.append(op) return self.newops @@ -133,6 +138,27 @@ else: raise NotImplementedError(op.getopname()) + def handle_call_assembler(self, op): + descrs = self.gc_ll_descr.getframedescrs(self.cpu) + loop_token = op.getdescr() + assert isinstance(loop_token, history.JitCellToken) + lgt_box = history.BoxInt() + frame = history.BoxPtr() + jfi = loop_token.compiled_loop_token.frame_info + llref = lltype.cast_opaque_ptr(llmemory.GCREF, jfi) + op0 = ResOperation(rop.GETFIELD_GC, [history.ConstPtr(llref)], lgt_box, + descr=descrs.jfi_frame_depth) + self.newops.append(op0) + op1 = ResOperation(rop.NEW_ARRAY, [lgt_box], frame, + descr=descrs.arraydescr) + self.handle_new_array(descrs.arraydescr, op1) + op2 = ResOperation(rop.SETFIELD_GC, [frame, history.ConstPtr(llref)], + None, descr=descrs.jf_frame_info) + self.newops.append(op2) + lst = op.getarglist() + self.newops.append(ResOperation(rop.CALL_ASSEMBLER, [frame] + lst, + op.result, op.getdescr())) + # ---------- def emitting_an_operation_that_can_collect(self): diff --git a/pypy/jit/backend/llsupport/test/test_rewrite.py b/pypy/jit/backend/llsupport/test/test_rewrite.py --- a/pypy/jit/backend/llsupport/test/test_rewrite.py +++ b/pypy/jit/backend/llsupport/test/test_rewrite.py @@ -1,9 +1,11 @@ from pypy.jit.backend.llsupport.descr import * from pypy.jit.backend.llsupport.gc import * +from pypy.jit.backend.llsupport import jitframe from pypy.jit.metainterp.gc import get_description from pypy.jit.tool.oparser import parse from pypy.jit.metainterp.optimizeopt.util import equaloplists from pypy.jit.codewriter.heaptracker import register_known_gctype +from pypy.jit.metainterp.history import JitCellToken class Evaluator(object): @@ -13,6 +15,9 @@ return eval(key, self.scope) +class FakeLoopToken(object): + pass + class RewriteTests(object): def check_rewrite(self, frm_operations, to_operations, **namespace): S = lltype.GcStruct('S', ('x', lltype.Signed), @@ -60,6 +65,17 @@ unicodedescr = self.gc_ll_descr.unicode_descr strlendescr = strdescr.lendescr unicodelendescr = unicodedescr.lendescr + + casmdescr = JitCellToken() + clt = FakeLoopToken() + frame_info = lltype.malloc(jitframe.JITFRAMEINFO) + ll_frame_info = lltype.cast_opaque_ptr(llmemory.GCREF, frame_info) + clt.frame_info = frame_info + frame_info.jfi_frame_depth = 13 + framedescrs = self.gc_ll_descr.getframedescrs(self.cpu) + jfi_frame_depth = framedescrs.jfi_frame_depth + jf_frame_info = framedescrs.jf_frame_info + casmdescr.compiled_loop_token = clt # namespace.update(locals()) # @@ -76,10 +92,29 @@ []) equaloplists(operations, expected.operations) +class BaseFakeCPU(object): + def __init__(self): + self._cache = {} + + def arraydescrof(self, ARRAY): + try: + return self._cache[ARRAY] + except KeyError: + r = ArrayDescr(1, 2, FieldDescr('len', 0, 0, 0), 0) + self._cache[ARRAY] = r + return r + def fielddescrof(self, STRUCT, fname): + key = (STRUCT, fname) + try: + return self._cache[key] + except KeyError: + r = FieldDescr(fname, 1, 1, 1) + self._cache[key] = r + return r class TestBoehm(RewriteTests): def setup_method(self, meth): - class FakeCPU(object): + class FakeCPU(BaseFakeCPU): def sizeof(self, STRUCT): return SizeDescrWithVTable(102) self.cpu = FakeCPU() @@ -215,7 +250,7 @@ self.gc_ll_descr.write_barrier_descr.has_write_barrier_from_array = ( lambda cpu: True) # - class FakeCPU(object): + class FakeCPU(BaseFakeCPU): def sizeof(self, STRUCT): descr = SizeDescrWithVTable(104) descr.tid = 9315 @@ -677,3 +712,17 @@ setfield_raw(p0, p1, descr=tzdescr) jump() """) + + def test_rewrite_call_assembler(self): + self.check_rewrite(""" + [i0] + i1 = call_assembler(i0, descr=casmdescr) + """, """ + [i0] + i1 = getfield_gc(ConstPtr(ll_frame_info), descr=jfi_frame_depth) + p1 = call_malloc_gc(ConstClass(malloc_array_nonstandard), 1, 2, 0, 0, i1, descr=malloc_array_nonstandard_descr) + setfield_gc(p1, ConstPtr(ll_frame_info), descr=jf_frame_info) + i2 = call_assembler(p1, i0, descr=casmdescr) + """) + # XXX we want call_malloc_nursery actually, but let's not care + # for now, the array is a bit non-standard From noreply at buildbot.pypy.org Sun Jan 13 16:37:26 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 13 Jan 2013 16:37:26 +0100 (CET) Subject: [pypy-commit] pypy default: Kill callback_hook. Replace it with a simpler and more explicit solution, which is to decorate Message-ID: <20130113153726.AD2C71C0884@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r60030:6ce8443ec9eb Date: 2013-01-13 15:33 +0100 http://bitbucket.org/pypy/pypy/changeset/6ce8443ec9eb/ Log: Kill callback_hook. Replace it with a simpler and more explicit solution, which is to decorate explicitly the callbacks that you want the JIT to treat specially. Btw, another potential use of this would be to have the JIT see some function which is deep inside @dont_look_inside code, but which causes more app-level Python code to be invoked. This can be useful, to let the JIT see a bit more than just the app-level Python code (e.g. the wrapping of arguments). diff --git a/pypy/jit/metainterp/test/test_warmspot.py b/pypy/jit/metainterp/test/test_warmspot.py --- a/pypy/jit/metainterp/test/test_warmspot.py +++ b/pypy/jit/metainterp/test/test_warmspot.py @@ -536,44 +536,6 @@ self.check_trace_count(1) - def test_callback_jit_merge_point(self): - from pypy.rlib.objectmodel import register_around_callback_hook - from pypy.rpython.lltypesystem import lltype, rffi - from pypy.translator.tool.cbuild import ExternalCompilationInfo - - callback_jit_driver = JitDriver(greens = ['name'], reds = 'auto') - - def callback_merge_point(name): - callback_jit_driver.jit_merge_point(name=name) - - @callback_jit_driver.inline(callback_merge_point) - def callback_hook(name): - pass - - def callback(a, b): - if a > b: - return 1 - return -1 - - CB_TP = rffi.CCallback([lltype.Signed, lltype.Signed], lltype.Signed) - eci = ExternalCompilationInfo(includes=['stdlib.h']) - qsort = rffi.llexternal('qsort', - [rffi.VOIDP, lltype.Signed, lltype.Signed, - CB_TP], lltype.Void, compilation_info=eci) - ARR = rffi.CArray(lltype.Signed) - - def main(): - register_around_callback_hook(callback_hook) - raw = lltype.malloc(ARR, 10, flavor='raw') - for i in range(10): - raw[i] = 10 - i - qsort(raw, 10, rffi.sizeof(lltype.Signed), callback) - lltype.free(raw, flavor='raw') - - self.meta_interp(main, []) - self.check_trace_count(1) - - class TestLLWarmspot(WarmspotTests, LLJitMixin): CPUClass = runner.LLGraphCPU type_system = 'lltype' diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py --- a/pypy/module/_cffi_backend/ccallback.py +++ b/pypy/module/_cffi_backend/ccallback.py @@ -152,6 +152,7 @@ STDERR = 2 + at jit.jit_callback("CFFI") def invoke_callback(ffi_cif, ll_res, ll_args, ll_userdata): """ Callback specification. ffi_cif - something ffi specific, don't care @@ -201,4 +202,3 @@ callback.write_error_return_value(ll_res) if ec is not None: cerrno.restore_errno_from(ec) -invoke_callback._callback_hook_ = "CFFI" diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py --- a/pypy/module/pyexpat/interp_pyexpat.py +++ b/pypy/module/pyexpat/interp_pyexpat.py @@ -2,7 +2,7 @@ from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault from pypy.interpreter.error import OperationError -from pypy.rlib import rgc +from pypy.rlib import rgc, jit from pypy.rpython.lltypesystem import rffi, lltype from pypy.rpython.tool import rffi_platform from pypy.translator.tool.cbuild import ExternalCompilationInfo @@ -288,6 +288,7 @@ post_code = '' src = py.code.Source(""" + @jit.jit_callback('XML:%(name)s') def %(name)s_callback(%(first_arg)s, %(args)s): id = rffi.cast(lltype.Signed, %(ll_id)s) userdata = global_storage.get_object(id) @@ -312,7 +313,6 @@ """ % locals()) exec str(src) - callback._callback_hook_ = "XML_" + name c_name = 'XML_Set' + name callback_type = lltype.Ptr(lltype.FuncType( @@ -341,7 +341,6 @@ else: result = 1 return rffi.cast(rffi.INT, result) -UnknownEncodingHandlerData_callback._callback_hook_ = None # no jit needed callback_type = lltype.Ptr(lltype.FuncType( [rffi.VOIDP, rffi.CCHARP, XML_Encoding_Ptr], rffi.INT)) XML_SetUnknownEncodingHandler = expat_external( diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py --- a/pypy/module/pypyjit/interp_jit.py +++ b/pypy/module/pypyjit/interp_jit.py @@ -6,7 +6,7 @@ from pypy.tool.pairtype import extendabletype from pypy.rlib.rarithmetic import r_uint, intmask from pypy.rlib.jit import JitDriver, hint, we_are_jitted, dont_look_inside -from pypy.rlib import jit, objectmodel +from pypy.rlib import jit from pypy.rlib.jit import current_trace_length, unroll_parameters import pypy.interpreter.pyopcode # for side-effects from pypy.interpreter.error import OperationError, operationerrfmt @@ -97,15 +97,6 @@ is_being_profiled=self.is_being_profiled) return jumpto -callback_jit_driver = JitDriver(greens = ['name'], reds = 'auto') - -def callback_merge_point(name): - callback_jit_driver.jit_merge_point(name=name) - - at callback_jit_driver.inline(callback_merge_point) -def callback_hook(name): - pass - def _get_adapted_tick_counter(): # Normally, the tick counter is decremented by 100 for every # Python opcode. Here, to better support JIT compilation of diff --git a/pypy/rlib/clibffi.py b/pypy/rlib/clibffi.py --- a/pypy/rlib/clibffi.py +++ b/pypy/rlib/clibffi.py @@ -421,6 +421,7 @@ hints={'callback':True})) + at jit.jit_callback("CLIBFFI") def ll_callback(ffi_cif, ll_res, ll_args, ll_userdata): """ Callback specification. ffi_cif - something ffi specific, don't care @@ -431,7 +432,6 @@ """ userdata = rffi.cast(USERDATA_P, ll_userdata) userdata.callback(ll_args, ll_res, userdata) -ll_callback._callback_hook_ = "CLIBFFI" class StackCheckError(ValueError): message = None diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py --- a/pypy/rlib/jit.py +++ b/pypy/rlib/jit.py @@ -302,6 +302,32 @@ pass +def jit_callback(name): + """Use as a decorator for C callback functions, to insert a + jitdriver.jit_merge_point() at the start. Only for callbacks + that typically invoke more app-level Python code. + """ + def decorate(func): + from pypy.tool.sourcetools import compile2 + # + def get_printable_location(): + return name + jitdriver = JitDriver(get_printable_location=get_printable_location, + greens=[], reds='auto', name=name) + # + args = ','.join(['a%d' % i for i in range(func.func_code.co_argcount)]) + source = """def callback_with_jitdriver(%(args)s): + jitdriver.jit_merge_point() + return real_callback(%(args)s)""" % locals() + miniglobals = { + 'jitdriver': jitdriver, + 'real_callback': func, + } + exec compile2(source) in miniglobals + return miniglobals['callback_with_jitdriver'] + return decorate + + # ____________________________________________________________ # VRefs @@ -458,9 +484,8 @@ self.autoreds = True self.reds = [] self.numreds = None # see warmspot.autodetect_jit_markers_redvars - for hook in (get_jitcell_at, set_jitcell_at, get_printable_location, - confirm_enter_jit): - assert hook is None, "reds='auto' is not compatible with JitDriver hooks" + assert confirm_enter_jit is None, ( + "reds='auto' is not compatible with confirm_enter_jit") else: if reds is not None: self.reds = reds diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py --- a/pypy/rlib/objectmodel.py +++ b/pypy/rlib/objectmodel.py @@ -595,16 +595,6 @@ llhelper(rffi.AroundFnPtr, before) llhelper(rffi.AroundFnPtr, after) -def register_around_callback_hook(hook): - """ Register a hook that's called before a callback from C calls RPython. - Primary usage is for JIT to have 'started from' hook. - """ - from pypy.rpython.lltypesystem import rffi - from pypy.rpython.annlowlevel import llhelper - - rffi.aroundstate.callback_hook = hook - llhelper(rffi.CallbackHookPtr, hook) - def is_in_callback(): from pypy.rpython.lltypesystem import rffi return rffi.stackcounter.stacks_counter > 1 diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py --- a/pypy/rpython/lltypesystem/rffi.py +++ b/pypy/rpython/lltypesystem/rffi.py @@ -1,6 +1,6 @@ import py from pypy.annotation import model as annmodel -from pypy.rpython.lltypesystem import lltype, rstr +from pypy.rpython.lltypesystem import lltype from pypy.rpython.lltypesystem import ll2ctypes from pypy.rpython.lltypesystem.llmemory import cast_ptr_to_adr from pypy.rpython.lltypesystem.llmemory import itemoffsetof, raw_memcopy @@ -13,7 +13,7 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.rpython.tool.rfficache import platform, sizeof_c_type from pypy.translator.tool.cbuild import ExternalCompilationInfo -from pypy.rpython.annlowlevel import llhelper, llstr +from pypy.rpython.annlowlevel import llhelper from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.rstring import StringBuilder, UnicodeBuilder, assert_str0 from pypy.rlib import jit @@ -280,18 +280,7 @@ if callbackholder is not None: callbackholder.callbacks[callable] = True args = ', '.join(['a%d' % i for i in range(len(TP.TO.ARGS))]) - invoke_hook = getattr(callable, '_callback_hook_', None) - assert invoke_hook is None or isinstance(invoke_hook, str) - source = py.code.Source(r""" - def inner_wrapper(%(args)s): - if invoke_hook is not None and aroundstate is not None: - callback_hook = aroundstate.callback_hook - if callback_hook: - callback_hook(llstr(invoke_hook)) - return callable(%(args)s) - inner_wrapper._never_inline_ = True - def wrapper(%(args)s): # no *args - no GIL for mallocing the tuple llop.gc_stack_bottom(lltype.Void) # marker for trackgcroot.py if aroundstate is not None: @@ -301,7 +290,7 @@ # from now on we hold the GIL stackcounter.stacks_counter += 1 try: - result = inner_wrapper(%(args)s) + result = callable(%(args)s) except Exception, e: os.write(2, "Warning: uncaught exception in callback: %%s %%s\n" %% @@ -323,7 +312,6 @@ miniglobals = locals().copy() miniglobals['Exception'] = Exception miniglobals['os'] = os - miniglobals['llstr'] = llstr miniglobals['we_are_translated'] = we_are_translated miniglobals['stackcounter'] = stackcounter exec source.compile() in miniglobals @@ -331,11 +319,8 @@ _make_wrapper_for._annspecialcase_ = 'specialize:memo' AroundFnPtr = lltype.Ptr(lltype.FuncType([], lltype.Void)) -CallbackHookPtr = lltype.Ptr(lltype.FuncType([lltype.Ptr(rstr.STR)], lltype.Void)) class AroundState: - callback_hook = None - def _cleanup_(self): self.before = None # or a regular RPython function self.after = None # or a regular RPython function diff --git a/pypy/translator/goal/targetpypystandalone.py b/pypy/translator/goal/targetpypystandalone.py --- a/pypy/translator/goal/targetpypystandalone.py +++ b/pypy/translator/goal/targetpypystandalone.py @@ -28,11 +28,6 @@ w_call_startup_gateway = space.wrap(gateway.interp2app(call_startup)) withjit = space.config.objspace.usemodules.pypyjit - if withjit: - from pypy.module.pypyjit.interp_jit import callback_hook - from pypy.rlib import objectmodel - objectmodel.register_around_callback_hook(callback_hook) - def entry_point(argv): if withjit: from pypy.jit.backend.hlinfo import highleveljitinfo From noreply at buildbot.pypy.org Sun Jan 13 16:37:27 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 13 Jan 2013 16:37:27 +0100 (CET) Subject: [pypy-commit] pypy default: Reintroduce a test similar to the previous one, and fix broken code. Message-ID: <20130113153727.ED3141C0884@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r60031:73c6c33284b7 Date: 2013-01-13 15:41 +0100 http://bitbucket.org/pypy/pypy/changeset/73c6c33284b7/ Log: Reintroduce a test similar to the previous one, and fix broken code. diff --git a/pypy/jit/metainterp/test/test_warmspot.py b/pypy/jit/metainterp/test/test_warmspot.py --- a/pypy/jit/metainterp/test/test_warmspot.py +++ b/pypy/jit/metainterp/test/test_warmspot.py @@ -1,6 +1,6 @@ import py from pypy.jit.metainterp.warmspot import get_stats -from pypy.rlib.jit import JitDriver, set_param, unroll_safe +from pypy.rlib.jit import JitDriver, set_param, unroll_safe, jit_callback from pypy.jit.backend.llgraph import runner from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin @@ -536,6 +536,24 @@ self.check_trace_count(1) + def test_callback_jit_merge_point(self): + @jit_callback("testing") + def callback(a, b): + if a > b: + return 1 + return -1 + + def main(): + total = 0 + for i in range(10): + total += callback(i, 2) + return total + + res = self.meta_interp(main, []) + assert res == 7 - 3 + self.check_trace_count(2) + + class TestLLWarmspot(WarmspotTests, LLJitMixin): CPUClass = runner.LLGraphCPU type_system = 'lltype' diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py --- a/pypy/jit/metainterp/warmspot.py +++ b/pypy/jit/metainterp/warmspot.py @@ -360,13 +360,7 @@ jd._jit_merge_point_in = graph args = op.args[2:] s_binding = self.translator.annotator.binding - if op.args[1].value.autoreds: - # _portal_args_s is used only by _make_hook_graph, but for now we - # declare the various set_jitcell_at, get_printable_location, - # etc. as incompatible with autoreds - jd._portal_args_s = None - else: - jd._portal_args_s = [s_binding(v) for v in args] + jd._portal_args_s = [s_binding(v) for v in args] graph = copygraph(graph) [jmpp] = find_jit_merge_points([graph]) graph.startblock = support.split_before_jit_merge_point(*jmpp) @@ -614,9 +608,10 @@ if func is None: return None # - assert not jitdriver_sd.jitdriver.autoreds, ( - "reds='auto' is not compatible with JitDriver hooks such as " - "{get,set}_jitcell_at, get_printable_location, confirm_enter_jit, etc.") + if not onlygreens: + assert not jitdriver_sd.jitdriver.autoreds, ( + "reds='auto' is not compatible with JitDriver hooks such as " + "confirm_enter_jit") extra_args_s = [] if s_first_arg is not None: extra_args_s.append(s_first_arg) From noreply at buildbot.pypy.org Sun Jan 13 17:39:30 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sun, 13 Jan 2013 17:39:30 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed missing conflicts Message-ID: <20130113163930.C34E21C054C@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60032:a3294d331db9 Date: 2013-01-13 17:39 +0100 http://bitbucket.org/pypy/pypy/changeset/a3294d331db9/ Log: Fixed missing conflicts diff --git a/rpython/jit/codewriter/effectinfo.py b/rpython/jit/codewriter/effectinfo.py --- a/rpython/jit/codewriter/effectinfo.py +++ b/rpython/jit/codewriter/effectinfo.py @@ -163,14 +163,9 @@ def effectinfo_from_writeanalyze(effects, cpu, extraeffect=EffectInfo.EF_CAN_RAISE, oopspecindex=EffectInfo.OS_NONE, -<<<<<<< local - can_invalidate=False): - from rpython.translator.backendopt.writeanalyze import top_set -======= can_invalidate=False, call_release_gil_target=llmemory.NULL): from rpython.translator.backendopt.writeanalyze import top_set ->>>>>>> other if effects is top_set or extraeffect == EffectInfo.EF_RANDOM_EFFECTS: readonly_descrs_fields = None readonly_descrs_arrays = None diff --git a/rpython/jit/codewriter/test/test_call.py b/rpython/jit/codewriter/test/test_call.py --- a/rpython/jit/codewriter/test/test_call.py +++ b/rpython/jit/codewriter/test/test_call.py @@ -173,11 +173,7 @@ py.test.skip("XXX add a test for CallControl.getcalldescr() -> EF_xxx") def test_releases_gil_analyzer(): -<<<<<<< local - from rpython.jit.backend.llgraph.runner import LLtypeCPU -======= from rpython.jit.backend.llgraph.runner import LLGraphCPU ->>>>>>> other T = rffi.CArrayPtr(rffi.TIME_T) external = rffi.llexternal("time", [T], rffi.TIME_T, threadsafe=True) From noreply at buildbot.pypy.org Sun Jan 13 18:12:19 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 13 Jan 2013 18:12:19 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: argument passing to call_assembler Message-ID: <20130113171219.3AA131C029E@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60033:88ab83e424c5 Date: 2013-01-13 18:06 +0200 http://bitbucket.org/pypy/pypy/changeset/88ab83e424c5/ Log: argument passing to call_assembler diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py --- a/pypy/jit/backend/llsupport/gc.py +++ b/pypy/jit/backend/llsupport/gc.py @@ -125,6 +125,13 @@ 'jfi_frame_depth') return descrs + def getarraydescr_for_frame(self, type, index): + """ This function returns a tuple (descr, index) for storing + index inside a frame array. It's a little bit delicate, because on + say x86 32bit, we can't express all indexes of floats. + """ + raise NotImplementedError # cpu specific + class JitFrameDescrs: def _freeze_(self): return True diff --git a/pypy/jit/backend/llsupport/rewrite.py b/pypy/jit/backend/llsupport/rewrite.py --- a/pypy/jit/backend/llsupport/rewrite.py +++ b/pypy/jit/backend/llsupport/rewrite.py @@ -149,14 +149,20 @@ op0 = ResOperation(rop.GETFIELD_GC, [history.ConstPtr(llref)], lgt_box, descr=descrs.jfi_frame_depth) self.newops.append(op0) + # XXX for now it generates call_malloc_gc, instead of + # call_malloc_nursery, because the array is strange op1 = ResOperation(rop.NEW_ARRAY, [lgt_box], frame, descr=descrs.arraydescr) self.handle_new_array(descrs.arraydescr, op1) op2 = ResOperation(rop.SETFIELD_GC, [frame, history.ConstPtr(llref)], None, descr=descrs.jf_frame_info) self.newops.append(op2) - lst = op.getarglist() - self.newops.append(ResOperation(rop.CALL_ASSEMBLER, [frame] + lst, + for i, arg in enumerate(op.getarglist()): + descr = self.cpu.getarraydescr_for_frame(arg.type, i) + self.newops.append(ResOperation(rop.SETARRAYITEM_GC, + [frame, ConstInt(i), arg], + None, descr)) + self.newops.append(ResOperation(rop.CALL_ASSEMBLER, [frame], op.result, op.getdescr())) # ---------- diff --git a/pypy/jit/backend/llsupport/test/test_rewrite.py b/pypy/jit/backend/llsupport/test/test_rewrite.py --- a/pypy/jit/backend/llsupport/test/test_rewrite.py +++ b/pypy/jit/backend/llsupport/test/test_rewrite.py @@ -75,6 +75,8 @@ framedescrs = self.gc_ll_descr.getframedescrs(self.cpu) jfi_frame_depth = framedescrs.jfi_frame_depth jf_frame_info = framedescrs.jf_frame_info + signedframedescr = self.cpu.signedframedescr + floatframedescr = self.cpu.floatframedescr casmdescr.compiled_loop_token = clt # namespace.update(locals()) @@ -95,6 +97,13 @@ class BaseFakeCPU(object): def __init__(self): self._cache = {} + self.signedframedescr = ArrayDescr(3, 8, FieldDescr('len', 0, 0, 0), 0) + self.floatframedescr = ArrayDescr(5, 8, FieldDescr('len', 0, 0, 0), 0) + + def getarraydescr_for_frame(self, tp, index): + if tp == history.FLOAT: + return self.floatframedescr + return self.signedframedescr def arraydescrof(self, ARRAY): try: @@ -103,6 +112,7 @@ r = ArrayDescr(1, 2, FieldDescr('len', 0, 0, 0), 0) self._cache[ARRAY] = r return r + def fielddescrof(self, STRUCT, fname): key = (STRUCT, fname) try: @@ -715,14 +725,16 @@ def test_rewrite_call_assembler(self): self.check_rewrite(""" - [i0] - i1 = call_assembler(i0, descr=casmdescr) + [i0, f0] + i2 = call_assembler(i0, f0, descr=casmdescr) """, """ - [i0] + [i0, f0] i1 = getfield_gc(ConstPtr(ll_frame_info), descr=jfi_frame_depth) p1 = call_malloc_gc(ConstClass(malloc_array_nonstandard), 1, 2, 0, 0, i1, descr=malloc_array_nonstandard_descr) setfield_gc(p1, ConstPtr(ll_frame_info), descr=jf_frame_info) - i2 = call_assembler(p1, i0, descr=casmdescr) + setarrayitem_gc(p1, 0, i0, descr=signedframedescr) + setarrayitem_gc(p1, 1, f0, descr=floatframedescr) + i2 = call_assembler(p1, descr=casmdescr) """) # XXX we want call_malloc_nursery actually, but let's not care # for now, the array is a bit non-standard From noreply at buildbot.pypy.org Sun Jan 13 18:12:20 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 13 Jan 2013 18:12:20 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: after much fighting, pass call_assembler test, still not perfect though Message-ID: <20130113171220.930401C029E@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60034:03cc0f5a43b2 Date: 2013-01-13 19:12 +0200 http://bitbucket.org/pypy/pypy/changeset/03cc0f5a43b2/ Log: after much fighting, pass call_assembler test, still not perfect though diff --git a/pypy/jit/backend/llsupport/rewrite.py b/pypy/jit/backend/llsupport/rewrite.py --- a/pypy/jit/backend/llsupport/rewrite.py +++ b/pypy/jit/backend/llsupport/rewrite.py @@ -158,9 +158,9 @@ None, descr=descrs.jf_frame_info) self.newops.append(op2) for i, arg in enumerate(op.getarglist()): - descr = self.cpu.getarraydescr_for_frame(arg.type, i) + index, descr = self.cpu.getarraydescr_for_frame(arg.type, i) self.newops.append(ResOperation(rop.SETARRAYITEM_GC, - [frame, ConstInt(i), arg], + [frame, ConstInt(index), arg], None, descr)) self.newops.append(ResOperation(rop.CALL_ASSEMBLER, [frame], op.result, op.getdescr())) diff --git a/pypy/jit/backend/llsupport/test/test_rewrite.py b/pypy/jit/backend/llsupport/test/test_rewrite.py --- a/pypy/jit/backend/llsupport/test/test_rewrite.py +++ b/pypy/jit/backend/llsupport/test/test_rewrite.py @@ -102,8 +102,8 @@ def getarraydescr_for_frame(self, tp, index): if tp == history.FLOAT: - return self.floatframedescr - return self.signedframedescr + return index, self.floatframedescr + return index, self.signedframedescr def arraydescrof(self, ARRAY): try: diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py --- a/pypy/jit/backend/test/runner_test.py +++ b/pypy/jit/backend/test/runner_test.py @@ -14,12 +14,13 @@ from pypy.jit.tool.oparser import parse from pypy.rpython.lltypesystem import lltype, llmemory, rstr, rffi, rclass from pypy.rpython.ootypesystem import ootype -from pypy.rpython.annlowlevel import llhelper +from pypy.rpython.annlowlevel import llhelper, cast_instance_to_gcref from pypy.rpython.llinterp import LLException from pypy.jit.codewriter import heaptracker, longlong from pypy.rlib import longlong2float from pypy.rlib.rarithmetic import intmask, is_valid_int from pypy.jit.backend.detect_cpu import autodetect_main_model_and_size +from pypy.jit.backend.llsupport import jitframe def boxfloat(x): @@ -2756,12 +2757,17 @@ def test_assembler_call(self): called = [] - def assembler_helper(deadframe, virtualizable): + def assembler_helper(failindex, deadframe, virtualizable): + deadframe = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, + deadframe) + faildescr = self.cpu.get_fail_descr_from_number(failindex) + deadframe.jf_descr = cast_instance_to_gcref(faildescr) assert self.cpu.get_int_value(deadframe, 0) == 97 called.append(self.cpu.get_latest_descr(deadframe)) return 4 + 9 - FUNCPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF, llmemory.GCREF], + FUNCPTR = lltype.Ptr(lltype.FuncType([lltype.Signed, llmemory.GCREF, + llmemory.GCREF], lltype.Signed)) class FakeJitDriverSD: index_of_virtualizable = -1 diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -12,8 +12,8 @@ from pypy.jit.backend.model import CompiledLoopToken from pypy.jit.backend.x86.regalloc import (RegAlloc, get_ebp_ofs, _get_scale, gpr_reg_mgr_cls, xmm_reg_mgr_cls, _valid_addressing_size) -from pypy.jit.backend.x86.arch import (FRAME_FIXED_SIZE, WORD, - IS_X86_32, IS_X86_64) +from pypy.jit.backend.x86.arch import (FRAME_FIXED_SIZE, WORD, IS_X86_64, + JITFRAME_FIXED_SIZE, IS_X86_32) from pypy.jit.backend.x86.regloc import (eax, ecx, edx, ebx, esp, ebp, esi, edi, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, r8, r9, r10, r11, r12, r13, r14, r15, X86_64_SCRATCH_REG, X86_64_XMM_SCRATCH_REG, @@ -509,8 +509,7 @@ looppos = self.mc.get_relative_pos() looptoken._x86_loop_code = looppos frame_depth = self._assemble(regalloc, operations) - clt.frame_depth = frame_depth - clt.frame_info.jfi_frame_depth = frame_depth + clt.frame_info.jfi_frame_depth = frame_depth + JITFRAME_FIXED_SIZE # size_excluding_failure_stuff = self.mc.get_relative_pos() self.write_pending_failure_recoveries() @@ -583,8 +582,8 @@ self.patch_jump_for_descr(faildescr, rawstart) ops_offset = self.mc.ops_offset self.fixup_target_tokens(rawstart) - frame_depth = max(self.current_clt.frame_depth, frame_depth) - self.current_clt.frame_depth = frame_depth + frame_depth = max(self.current_clt.frame_depth, + frame_depth + JITFRAME_FIXED_SIZE) self.current_clt.frame_info.jfi_frame_depth = frame_depth self.teardown() # oprofile support @@ -741,7 +740,7 @@ frame_depth = regalloc.get_final_frame_depth() jump_target_descr = regalloc.jump_target_descr if jump_target_descr is not None: - target_frame_depth = jump_target_descr._x86_clt.frame_depth + target_frame_depth = jump_target_descr._x86_clt.frame_info.frame_depth frame_depth = max(frame_depth, target_frame_depth) return frame_depth @@ -825,6 +824,7 @@ self.mc.SUB_mi8((ebx.value, 0), 2*WORD) # SUB [ebx], 2*WORD def redirect_call_assembler(self, oldlooptoken, newlooptoken): + xxx # some minimal sanity checking old_nbargs = oldlooptoken.compiled_loop_token._debug_nbargs new_nbargs = newlooptoken.compiled_loop_token._debug_nbargs @@ -1958,19 +1958,28 @@ srcloc = eax self.load_from_mem(eax, srcloc, sizeloc, signloc) - def genop_guard_call_may_force(self, op, guard_op, guard_token, - arglocs, result_loc): + def _store_force_index(self, guard_op): faildescr = guard_op.getdescr() fail_index = self.cpu.get_fail_descr_number(faildescr) descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) base_ofs = self.cpu.unpack_arraydescr(descrs.arraydescr) ofs = self.cpu.unpack_fielddescr(descrs.jf_force_index) self.mc.MOV_bi(ofs - base_ofs, fail_index) - self._genop_call(op, arglocs, result_loc, fail_index) + return fail_index + + def _emit_guard_not_forced(self, guard_token): + descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) + base_ofs = self.cpu.unpack_arraydescr(descrs.arraydescr) ofs_fail = self.cpu.unpack_fielddescr(descrs.jf_descr) self.mc.CMP_bi(ofs_fail - base_ofs, 0) self.implement_guard(guard_token, 'NE') + def genop_guard_call_may_force(self, op, guard_op, guard_token, + arglocs, result_loc): + fail_index = self._store_force_index(guard_op) + self._genop_call(op, arglocs, result_loc, fail_index) + self._emit_guard_not_forced() + def genop_guard_call_release_gil(self, op, guard_op, guard_token, arglocs, result_loc): # first, close the stack in the sense of the asmgcc GC root tracker @@ -1978,20 +1987,13 @@ if gcrootmap: self.call_release_gil(gcrootmap, arglocs) # do the call - faildescr = guard_op.getdescr() - fail_index = self.cpu.get_fail_descr_number(faildescr) - descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) - base_ofs = self.cpu.unpack_arraydescr(descrs.arraydescr) - ofs = self.cpu.unpack_fielddescr(descrs.jf_force_index) - self.mc.MOV_bi(ofs - base_ofs, fail_index) + fail_index = self._store_force_index(guard_op) self._genop_call(op, arglocs, result_loc, fail_index) # then reopen the stack if gcrootmap: self.call_reacquire_gil(gcrootmap, result_loc) # finally, the guard_not_forced - ofs_fail = self.cpu.unpack_fielddescr(descrs.jf_descr) - self.mc.CMP_bi(ofs_fail - base_ofs, 0) - self.implement_guard(guard_token, 'NE') + self._emit_guard_not_forced(guard_token) def call_release_gil(self, gcrootmap, save_registers): # First, we need to save away the registers listed in @@ -2096,17 +2098,16 @@ def genop_guard_call_assembler(self, op, guard_op, guard_token, arglocs, result_loc): - xxx - faildescr = guard_op.getdescr() - fail_index = self.cpu.get_fail_descr_number(faildescr) - self.mc.MOV_bi(FORCE_INDEX_OFS, fail_index) + fail_index = self._store_force_index(guard_op) descr = op.getdescr() assert isinstance(descr, JitCellToken) - assert len(arglocs) - 2 == descr.compiled_loop_token._debug_nbargs + frame_loc = arglocs[0] # # Write a call to the target assembler + # we need to allocate the frame, keep in sync with runner's + # execute_token self._emit_call(fail_index, imm(descr._x86_function_addr), - arglocs, 2, tmp=eax) + [frame_loc], 0, tmp=eax) if op.result is None: assert result_loc is None value = self.cpu.done_with_this_frame_void_v @@ -2123,19 +2124,7 @@ else: raise AssertionError(kind) - from pypy.jit.backend.llsupport.descr import unpack_fielddescr - from pypy.jit.backend.llsupport.descr import unpack_interiorfielddescr - descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) - _offset, _size, _ = unpack_fielddescr(descrs.jf_descr) - fail_descr = self.cpu.get_fail_descr_from_number(value) - value = fail_descr.hide(self.cpu) - rgc._make_sure_does_not_move(value) - value = rffi.cast(lltype.Signed, value) - if rx86.fits_in_32bits(value): - self.mc.CMP_mi((eax.value, _offset), value) - else: - self.mc.MOV_ri(X86_64_SCRATCH_REG.value, value) - self.mc.CMP_mr((eax.value, _offset), X86_64_SCRATCH_REG.value) + self.mc.CMP_ri(eax.value, value) # patched later self.mc.J_il8(rx86.Conditions['E'], 0) # goto B if we get 'done_with_this_frame' je_location = self.mc.get_relative_pos() @@ -2144,8 +2133,8 @@ jd = descr.outermost_jitdriver_sd assert jd is not None asm_helper_adr = self.cpu.cast_adr_to_int(jd.assembler_helper_adr) - self._emit_call(fail_index, imm(asm_helper_adr), [eax, arglocs[1]], 0, - tmp=ecx) + self._emit_call(fail_index, imm(asm_helper_adr), + [eax, frame_loc, imm0], 0, tmp=ecx) if IS_X86_32 and isinstance(result_loc, StackLoc) and result_loc.type == FLOAT: self.mc.FSTPL_b(result_loc.value) #else: result_loc is already either eax or None, checked below @@ -2159,6 +2148,7 @@ # # Reset the vable token --- XXX really too much special logic here:-( if jd.index_of_virtualizable >= 0: + xxx from pypy.jit.backend.llsupport.descr import FieldDescr fielddescr = jd.vable_token_descr assert isinstance(fielddescr, FieldDescr) @@ -2169,26 +2159,26 @@ # if op.result is not None: # load the return value from the dead frame's value index 0 + assert isinstance(frame_loc, StackLoc) + self.mc.MOV_rb(eax.value, frame_loc.value) kind = op.result.type if kind == FLOAT: + xxx t = unpack_interiorfielddescr(descrs.as_float) self.mc.MOVSD_xm(xmm0.value, (eax.value, t[0])) if result_loc is not xmm0: self.mc.MOVSD(result_loc, xmm0) else: assert result_loc is eax - if kind == INT: - t = unpack_interiorfielddescr(descrs.as_int) - else: - t = unpack_interiorfielddescr(descrs.as_ref) - self.mc.MOV_rm(eax.value, (eax.value, t[0])) + _, descr = self.cpu.getarraydescr_for_frame(kind, 0) + ofs = self.cpu.unpack_arraydescr(descr) + self.mc.MOV_rm(eax.value, (eax.value, ofs)) # # Here we join Path A and Path B again offset = self.mc.get_relative_pos() - jmp_location assert 0 <= offset <= 127 self.mc.overwrite(jmp_location - 1, chr(offset)) - self.mc.CMP_bi(FORCE_INDEX_OFS, 0) - self.implement_guard(guard_token, 'L') + self._emit_guard_not_forced(guard_token) def genop_discard_cond_call_gc_wb(self, op, arglocs): # Write code equivalent to write_barrier() in the GC: it checks diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py --- a/pypy/jit/backend/x86/regalloc.py +++ b/pypy/jit/backend/x86/regalloc.py @@ -936,17 +936,13 @@ descr = op.getdescr() assert isinstance(descr, JitCellToken) jd = descr.outermost_jitdriver_sd - assert jd is not None - size = jd.portal_calldescr.get_result_size() + # we attached a frame as a first arg, move index of virtualizable by one vable_index = jd.index_of_virtualizable + self.rm._sync_var(op.getarg(0)) + frame_loc = self.fm.loc(op.getarg(0)) if vable_index >= 0: - self.rm._sync_var(op.getarg(vable_index)) - vable = self.fm.loc(op.getarg(vable_index)) - else: - vable = imm0 - self._call(op, [imm(size), vable] + - [self.loc(op.getarg(i)) for i in range(op.numargs())], - guard_not_forced_op=guard_op) + xxx + self._call(op, [frame_loc], guard_not_forced_op=guard_op) def consider_cond_call_gc_wb(self, op): assert op.result is None diff --git a/pypy/jit/backend/x86/runner.py b/pypy/jit/backend/x86/runner.py --- a/pypy/jit/backend/x86/runner.py +++ b/pypy/jit/backend/x86/runner.py @@ -112,9 +112,10 @@ addr = executable_token._x86_function_addr func = rffi.cast(FUNCPTR, addr) #llop.debug_print(lltype.Void, ">>>> Entering", addr) - frame = lltype.malloc(jitframe.JITFRAME, clt.frame_depth + - JITFRAME_FIXED_SIZE, zero=True) - frame.jf_frame_info = clt.frame_info + frame_info = clt.frame_info + frame = lltype.malloc(jitframe.JITFRAME, frame_info.jfi_frame_depth, + zero=True) + frame.jf_frame_info = frame_info ll_frame = lltype.cast_opaque_ptr(llmemory.GCREF, frame) prev_interpreter = None # help flow space if not self.translate_support_code: @@ -231,4 +232,11 @@ assert sys.maxint == (2**63 - 1) super(CPU_X86_64, self).__init__(*args, **kwargs) + def getarraydescr_for_frame(self, type, index): + if type != history.INT: + xxx + else: + descrs = self.gc_ll_descr.getframedescrs(self) + return JITFRAME_FIXED_SIZE + index, descrs.arraydescr + CPU = CPU386 diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py --- a/pypy/jit/metainterp/warmspot.py +++ b/pypy/jit/metainterp/warmspot.py @@ -658,7 +658,7 @@ else: assert False (_, jd._PTR_ASSEMBLER_HELPER_FUNCTYPE) = self.cpu.ts.get_FuncType( - [llmemory.GCREF, llmemory.GCREF], ASMRESTYPE) + [lltype.Signed, llmemory.GCREF, llmemory.GCREF], ASMRESTYPE) def rewrite_can_enter_jits(self): sublists = {} @@ -927,9 +927,12 @@ vinfo = jd.virtualizable_info - def assembler_call_helper(deadframe, virtualizableref): + def assembler_call_helper(failindex, deadframe, virtualizableref): + XXX # we got failindex, but the descr is not stored on the frame + # yet, it's our job fail_descr = self.cpu.get_latest_descr(deadframe) if vinfo is not None: + xxx virtualizable = lltype.cast_opaque_ptr( vinfo.VTYPEPTR, virtualizableref) vinfo.reset_vable_token(virtualizable) From noreply at buildbot.pypy.org Sun Jan 13 18:12:50 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 13 Jan 2013 18:12:50 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: revert this debugging help Message-ID: <20130113171250.D8D691C029E@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60035:9db4daaabba4 Date: 2013-01-13 19:12 +0200 http://bitbucket.org/pypy/pypy/changeset/9db4daaabba4/ Log: revert this debugging help diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -2101,13 +2101,13 @@ fail_index = self._store_force_index(guard_op) descr = op.getdescr() assert isinstance(descr, JitCellToken) - frame_loc = arglocs[0] + [frame_loc, argloc] = arglocs # # Write a call to the target assembler # we need to allocate the frame, keep in sync with runner's # execute_token self._emit_call(fail_index, imm(descr._x86_function_addr), - [frame_loc], 0, tmp=eax) + [argloc], 0, tmp=eax) if op.result is None: assert result_loc is None value = self.cpu.done_with_this_frame_void_v diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py --- a/pypy/jit/backend/x86/regalloc.py +++ b/pypy/jit/backend/x86/regalloc.py @@ -942,7 +942,8 @@ frame_loc = self.fm.loc(op.getarg(0)) if vable_index >= 0: xxx - self._call(op, [frame_loc], guard_not_forced_op=guard_op) + self._call(op, [frame_loc, self.loc(op.getarg(0))], + guard_not_forced_op=guard_op) def consider_cond_call_gc_wb(self, op): assert op.result is None From noreply at buildbot.pypy.org Sun Jan 13 18:27:06 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sun, 13 Jan 2013 18:27:06 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed typo Message-ID: <20130113172706.576FB1C071B@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60036:670201b7d4cf Date: 2013-01-13 18:26 +0100 http://bitbucket.org/pypy/pypy/changeset/670201b7d4cf/ Log: Fixed typo diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py --- a/rpython/jit/backend/llsupport/gc.py +++ b/rpython/jit/backend/llsupport/gc.py @@ -10,7 +10,7 @@ from rpython.jit.codewriter import heaptracker from rpython.jit.metainterp.history import ConstPtr, AbstractDescr from rpython.jit.metainterp.resoperation import ResOperation, rop -from rpython.jit.backend.llsupport import symbolic, , jitframe +from rpython.jit.backend.llsupport import symbolic, jitframe from rpython.jit.backend.llsupport.symbolic import WORD from rpython.jit.backend.llsupport.descr import SizeDescr, ArrayDescr from rpython.jit.backend.llsupport.descr import GcCache, get_field_descr From noreply at buildbot.pypy.org Sun Jan 13 20:23:08 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 13 Jan 2013 20:23:08 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: FLOAT support for call_assembler Message-ID: <20130113192308.5CDE71C071B@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60037:4eda81ccdfc7 Date: 2013-01-13 21:22 +0200 http://bitbucket.org/pypy/pypy/changeset/4eda81ccdfc7/ Log: FLOAT support for call_assembler diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py --- a/pypy/jit/backend/llsupport/llmodel.py +++ b/pypy/jit/backend/llsupport/llmodel.py @@ -259,6 +259,7 @@ abiname) def get_latest_descr(self, deadframe): + deadframe = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, deadframe) descr = deadframe.jf_descr return history.AbstractDescr.show(self, descr) diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py --- a/pypy/jit/backend/test/runner_test.py +++ b/pypy/jit/backend/test/runner_test.py @@ -2758,10 +2758,9 @@ def test_assembler_call(self): called = [] def assembler_helper(failindex, deadframe, virtualizable): - deadframe = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, - deadframe) faildescr = self.cpu.get_fail_descr_from_number(failindex) - deadframe.jf_descr = cast_instance_to_gcref(faildescr) + frame = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, deadframe) + frame.jf_descr = cast_instance_to_gcref(faildescr) assert self.cpu.get_int_value(deadframe, 0) == 97 called.append(self.cpu.get_latest_descr(deadframe)) return 4 + 9 @@ -2836,14 +2835,18 @@ if not self.cpu.supports_floats: py.test.skip("requires floats") called = [] - def assembler_helper(deadframe, virtualizable): + def assembler_helper(failindex, deadframe, virtualizable): + frame = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, deadframe) + faildescr = self.cpu.get_fail_descr_from_number(failindex) + frame.jf_descr = cast_instance_to_gcref(faildescr) x = self.cpu.get_float_value(deadframe, 0) assert longlong.getrealfloat(x) == 1.2 + 3.2 called.append(self.cpu.get_latest_descr(deadframe)) print '!' * 30 + 'assembler_helper' return 13.5 - FUNCPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF, llmemory.GCREF], + FUNCPTR = lltype.Ptr(lltype.FuncType([lltype.Signed, llmemory.GCREF, + llmemory.GCREF], lltype.Float)) class FakeJitDriverSD: index_of_virtualizable = -1 diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -2163,9 +2163,9 @@ self.mc.MOV_rb(eax.value, frame_loc.value) kind = op.result.type if kind == FLOAT: - xxx - t = unpack_interiorfielddescr(descrs.as_float) - self.mc.MOVSD_xm(xmm0.value, (eax.value, t[0])) + _, descr = self.cpu.getarraydescr_for_frame(kind, 0) + ofs = self.cpu.unpack_arraydescr(descr) + self.mc.MOVSD_xm(xmm0.value, (eax.value, ofs)) if result_loc is not xmm0: self.mc.MOVSD(result_loc, xmm0) else: diff --git a/pypy/jit/backend/x86/runner.py b/pypy/jit/backend/x86/runner.py --- a/pypy/jit/backend/x86/runner.py +++ b/pypy/jit/backend/x86/runner.py @@ -14,7 +14,9 @@ from pypy.jit.backend.llsupport import jitframe from pypy.jit.backend.x86 import regloc from pypy.jit.backend.llsupport.symbolic import WORD -from pypy.jit.backend.llsupport.descr import unpack_arraydescr +from pypy.jit.backend.llsupport.descr import ArrayDescr, FLAG_POINTER,\ + FLAG_FLOAT + import sys from pypy.tool.ansi_print import ansi_log @@ -231,12 +233,22 @@ def __init__(self, *args, **kwargs): assert sys.maxint == (2**63 - 1) super(CPU_X86_64, self).__init__(*args, **kwargs) + descrs = self.gc_ll_descr.getframedescrs(self) + ad = descrs.arraydescr + # the same as normal JITFRAME, however with an array of pointers + self.refarraydescr = ArrayDescr(ad.basesize, ad.itemsize, ad.lendescr, + FLAG_POINTER) + self.floatarraydescr = ArrayDescr(ad.basesize, ad.itemsize, ad.lendescr, + FLAG_FLOAT) def getarraydescr_for_frame(self, type, index): - if type != history.INT: - xxx + if type == history.FLOAT: + descr = self.floatarraydescr + elif type == history.REF: + descr = self.refarraydescr else: descrs = self.gc_ll_descr.getframedescrs(self) - return JITFRAME_FIXED_SIZE + index, descrs.arraydescr + descr = descrs.arraydescr + return JITFRAME_FIXED_SIZE + index, descr CPU = CPU386 From noreply at buildbot.pypy.org Sun Jan 13 20:32:01 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 13 Jan 2013 20:32:01 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fix fix fix Message-ID: <20130113193201.A97D71C0884@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60038:9a2f46a8e578 Date: 2013-01-13 21:31 +0200 http://bitbucket.org/pypy/pypy/changeset/9a2f46a8e578/ Log: fix fix fix diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py --- a/pypy/jit/backend/llsupport/llmodel.py +++ b/pypy/jit/backend/llsupport/llmodel.py @@ -1,7 +1,8 @@ from pypy.rpython.lltypesystem import lltype, llmemory, rffi, rclass, rstr from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.llinterp import LLInterpreter -from pypy.rpython.annlowlevel import llhelper, cast_instance_to_base_ptr +from pypy.rpython.annlowlevel import llhelper, cast_instance_to_base_ptr,\ + cast_instance_to_gcref from pypy.rlib.objectmodel import we_are_translated, specialize from pypy.jit.metainterp import history from pypy.jit.codewriter import heaptracker, longlong @@ -263,6 +264,11 @@ descr = deadframe.jf_descr return history.AbstractDescr.show(self, descr) + def store_fail_descr(self, deadframe, failindex): + faildescr = self.get_fail_descr_from_number(failindex) + frame = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, deadframe) + frame.jf_descr = cast_instance_to_gcref(faildescr) + def _decode_pos(self, deadframe, index): descr = self.get_latest_descr(deadframe) if descr.final_descr: diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py --- a/pypy/jit/backend/test/runner_test.py +++ b/pypy/jit/backend/test/runner_test.py @@ -2758,9 +2758,7 @@ def test_assembler_call(self): called = [] def assembler_helper(failindex, deadframe, virtualizable): - faildescr = self.cpu.get_fail_descr_from_number(failindex) - frame = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, deadframe) - frame.jf_descr = cast_instance_to_gcref(faildescr) + self.cpu.store_fail_descr(deadframe, failindex) assert self.cpu.get_int_value(deadframe, 0) == 97 called.append(self.cpu.get_latest_descr(deadframe)) return 4 + 9 @@ -2836,9 +2834,7 @@ py.test.skip("requires floats") called = [] def assembler_helper(failindex, deadframe, virtualizable): - frame = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, deadframe) - faildescr = self.cpu.get_fail_descr_from_number(failindex) - frame.jf_descr = cast_instance_to_gcref(faildescr) + self.cpu.store_fail_descr(deadframe, failindex) x = self.cpu.get_float_value(deadframe, 0) assert longlong.getrealfloat(x) == 1.2 + 3.2 called.append(self.cpu.get_latest_descr(deadframe)) @@ -2935,13 +2931,15 @@ if not self.cpu.supports_floats: py.test.skip("requires floats") called = [] - def assembler_helper(deadframe, virtualizable): + def assembler_helper(failindex, deadframe, virtualizable): + self.cpu.store_fail_descr(deadframe, failindex) x = self.cpu.get_float_value(deadframe, 0) assert longlong.getrealfloat(x) == 1.25 + 3.25 called.append(self.cpu.get_latest_descr(deadframe)) return 13.5 - FUNCPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF, llmemory.GCREF], + FUNCPTR = lltype.Ptr(lltype.FuncType([lltype.Signed, + llmemory.GCREF, llmemory.GCREF], lltype.Float)) class FakeJitDriverSD: index_of_virtualizable = -1 diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -582,7 +582,7 @@ self.patch_jump_for_descr(faildescr, rawstart) ops_offset = self.mc.ops_offset self.fixup_target_tokens(rawstart) - frame_depth = max(self.current_clt.frame_depth, + frame_depth = max(self.current_clt.frame_info.jfi_frame_depth, frame_depth + JITFRAME_FIXED_SIZE) self.current_clt.frame_info.jfi_frame_depth = frame_depth self.teardown() @@ -740,7 +740,7 @@ frame_depth = regalloc.get_final_frame_depth() jump_target_descr = regalloc.jump_target_descr if jump_target_descr is not None: - target_frame_depth = jump_target_descr._x86_clt.frame_info.frame_depth + target_frame_depth = jump_target_descr._x86_clt.frame_info.jfi_frame_depth frame_depth = max(frame_depth, target_frame_depth) return frame_depth @@ -824,7 +824,6 @@ self.mc.SUB_mi8((ebx.value, 0), 2*WORD) # SUB [ebx], 2*WORD def redirect_call_assembler(self, oldlooptoken, newlooptoken): - xxx # some minimal sanity checking old_nbargs = oldlooptoken.compiled_loop_token._debug_nbargs new_nbargs = newlooptoken.compiled_loop_token._debug_nbargs From noreply at buildbot.pypy.org Sun Jan 13 20:34:57 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 13 Jan 2013 20:34:57 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fix Message-ID: <20130113193457.16B411C0884@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60039:1059e8bf00eb Date: 2013-01-13 21:34 +0200 http://bitbucket.org/pypy/pypy/changeset/1059e8bf00eb/ Log: fix diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -490,6 +490,7 @@ clt = CompiledLoopToken(self.cpu, looptoken.number) clt.frame_info = lltype.malloc(jitframe.JITFRAMEINFO) clt.allgcrefs = [] + clt.frame_info.jfi_frame_depth = 0 # for now looptoken.compiled_loop_token = clt if not we_are_translated(): # Arguments should be unique From noreply at buildbot.pypy.org Sun Jan 13 20:56:30 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 13 Jan 2013 20:56:30 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fix enough to pass most of the tests Message-ID: <20130113195630.650E41C05DD@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60040:06712904340c Date: 2013-01-13 21:56 +0200 http://bitbucket.org/pypy/pypy/changeset/06712904340c/ Log: fix enough to pass most of the tests diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -1978,7 +1978,7 @@ arglocs, result_loc): fail_index = self._store_force_index(guard_op) self._genop_call(op, arglocs, result_loc, fail_index) - self._emit_guard_not_forced() + self._emit_guard_not_forced(guard_token) def genop_guard_call_release_gil(self, op, guard_op, guard_token, arglocs, result_loc): diff --git a/pypy/jit/backend/x86/test/test_runner.py b/pypy/jit/backend/x86/test/test_runner.py --- a/pypy/jit/backend/x86/test/test_runner.py +++ b/pypy/jit/backend/x86/test/test_runner.py @@ -4,7 +4,7 @@ from pypy.jit.metainterp.history import ResOperation, TargetToken, JitCellToken from pypy.jit.metainterp.history import (BoxInt, BoxPtr, ConstInt, ConstFloat, ConstPtr, Box, BoxFloat, - BasicFailDescr) + BasicFailDescr, BasicFinalDescr) from pypy.jit.backend.detect_cpu import getcpuclass from pypy.jit.backend.x86.arch import WORD from pypy.jit.backend.x86.rx86 import fits_in_32bits @@ -34,11 +34,7 @@ # ====> ../../test/runner_test.py add_loop_instructions = ['mov', 'add', 'test', 'je', 'jmp'] - if WORD == 4: - bridge_loop_instructions = ['lea', 'jmp'] - else: - # the 'mov' is part of the 'jmp' so far - bridge_loop_instructions = ['lea', 'mov', 'jmp'] + bridge_loop_instructions = ['mov', 'jmp'] def setup_method(self, meth): self.cpu = CPU(rtyper=None, stats=FakeStats()) @@ -285,13 +281,13 @@ ResOperation(guard, [f], None, descr=BasicFailDescr()), ResOperation(rop.FINISH, [ConstInt(0)], None, - descr=BasicFailDescr()), + descr=BasicFinalDescr()), ] ops[-2].setfailargs([i1]) looptoken = JitCellToken() self.cpu.compile_loop([b], ops, looptoken) deadframe = self.cpu.execute_token(looptoken, b.value) - result = self.cpu.get_latest_value_int(deadframe, 0) + result = self.cpu.get_int_value(deadframe, 0) if guard == rop.GUARD_FALSE: assert result == execute(self.cpu, None, op, None, b).value @@ -330,7 +326,7 @@ ResOperation(guard, [res], None, descr=BasicFailDescr()), ResOperation(rop.FINISH, [ConstInt(0)], None, - descr=BasicFailDescr()), + descr=BasicFinalDescr()), ] ops[-2].setfailargs([i1]) inputargs = [i for i in (a, b) if isinstance(i, Box)] @@ -338,7 +334,7 @@ self.cpu.compile_loop(inputargs, ops, looptoken) inputvalues = [box.value for box in inputargs] deadframe = self.cpu.execute_token(looptoken, *inputvalues) - result = self.cpu.get_latest_value_int(deadframe, 0) + result = self.cpu.get_int_value(deadframe, 0) expected = execute(self.cpu, None, op, None, a, b).value if guard == rop.GUARD_FALSE: assert result == expected @@ -406,7 +402,7 @@ deadframe = self.cpu.execute_token(looptoken, 2) fail = self.cpu.get_latest_descr(deadframe) assert fail.identifier == 2 - res = self.cpu.get_latest_value_int(deadframe, 0) + res = self.cpu.get_int_value(deadframe, 0) assert res == 20 def test_ops_offset(self): @@ -515,10 +511,10 @@ deadframe = self.cpu.execute_token(looptoken, 123450, 123408) fail = self.cpu.get_latest_descr(deadframe) assert fail.identifier == 0 - assert self.cpu.get_latest_value_int(deadframe, 0) == 42 - assert self.cpu.get_latest_value_int(deadframe, 1) == 42 - assert self.cpu.get_latest_value_int(deadframe, 2) == 42 - assert self.cpu.get_latest_value_int(deadframe, 3) == 42 + assert self.cpu.get_int_value(deadframe, 0) == 42 + assert self.cpu.get_int_value(deadframe, 1) == 42 + assert self.cpu.get_int_value(deadframe, 2) == 42 + assert self.cpu.get_int_value(deadframe, 3) == 42 class TestDebuggingAssembler(object): From noreply at buildbot.pypy.org Sun Jan 13 21:06:00 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 13 Jan 2013 21:06:00 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fix llgraph backend Message-ID: <20130113200600.8FFBC1C05DD@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60041:315200db40be Date: 2013-01-13 21:59 +0200 http://bitbucket.org/pypy/pypy/changeset/315200db40be/ Log: fix llgraph backend diff --git a/pypy/jit/backend/llgraph/runner.py b/pypy/jit/backend/llgraph/runner.py --- a/pypy/jit/backend/llgraph/runner.py +++ b/pypy/jit/backend/llgraph/runner.py @@ -563,6 +563,9 @@ def bh_read_timestamp(self): return read_timestamp() + def store_fail_descr(self, deadframe, descr): + pass # I *think* + class LLDeadFrame(object): _TYPE = llmemory.GCREF @@ -875,7 +878,7 @@ # assembler_helper_ptr = jd.assembler_helper_adr.ptr # fish try: - result = assembler_helper_ptr(pframe, vable) + result = assembler_helper_ptr(failindex, pframe, vable) except LLException, lle: assert self.last_exception is None, "exception left behind" self.last_exception = lle @@ -914,6 +917,7 @@ def execute_keepalive(self, descr, x): pass + def _getdescr(op): d = op.getdescr() if d is not None: From noreply at buildbot.pypy.org Sun Jan 13 21:06:01 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 13 Jan 2013 21:06:01 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: some frontend renaming Message-ID: <20130113200601.E88351C05DD@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60042:7cd874a1c1ce Date: 2013-01-13 22:05 +0200 http://bitbucket.org/pypy/pypy/changeset/7cd874a1c1ce/ Log: some frontend renaming diff --git a/pypy/jit/metainterp/compile.py b/pypy/jit/metainterp/compile.py --- a/pypy/jit/metainterp/compile.py +++ b/pypy/jit/metainterp/compile.py @@ -422,26 +422,26 @@ class DoneWithThisFrameDescrInt(_DoneWithThisFrameDescr): def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd): assert jitdriver_sd.result_type == history.INT - result = metainterp_sd.cpu.get_latest_value_int(deadframe, 0) + result = metainterp_sd.cpu.get_int_value(deadframe, 0) raise metainterp_sd.DoneWithThisFrameInt(result) class DoneWithThisFrameDescrRef(_DoneWithThisFrameDescr): def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd): assert jitdriver_sd.result_type == history.REF cpu = metainterp_sd.cpu - result = cpu.get_latest_value_ref(deadframe, 0) + result = cpu.get_ref_value(deadframe, 0) raise metainterp_sd.DoneWithThisFrameRef(cpu, result) class DoneWithThisFrameDescrFloat(_DoneWithThisFrameDescr): def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd): assert jitdriver_sd.result_type == history.FLOAT - result = metainterp_sd.cpu.get_latest_value_float(deadframe, 0) + result = metainterp_sd.cpu.get_float_value(deadframe, 0) raise metainterp_sd.DoneWithThisFrameFloat(result) class ExitFrameWithExceptionDescrRef(_DoneWithThisFrameDescr): def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd): cpu = metainterp_sd.cpu - value = cpu.get_latest_value_ref(deadframe, 0) + value = cpu.get_ref_value(deadframe, 0) raise metainterp_sd.ExitFrameWithExceptionRef(cpu, value) @@ -573,7 +573,7 @@ typetag = self._counter & self.CNT_TYPE_MASK counters = self._counters if typetag == self.CNT_INT: - intvalue = metainterp_sd.cpu.get_latest_value_int( + intvalue = metainterp_sd.cpu.get_int_value( deadframe, index) if counters is None: self._counters = counters = ResumeGuardCountersInt() @@ -581,7 +581,7 @@ assert isinstance(counters, ResumeGuardCountersInt) counter = counters.see_int(intvalue) elif typetag == self.CNT_REF: - refvalue = metainterp_sd.cpu.get_latest_value_ref( + refvalue = metainterp_sd.cpu.get_ref_value( deadframe, index) if counters is None: self._counters = counters = ResumeGuardCountersRef() @@ -589,7 +589,7 @@ assert isinstance(counters, ResumeGuardCountersRef) counter = counters.see_ref(refvalue) elif typetag == self.CNT_FLOAT: - floatvalue = metainterp_sd.cpu.get_latest_value_float( + floatvalue = metainterp_sd.cpu.get_float_value( deadframe, index) if counters is None: self._counters = counters = ResumeGuardCountersFloat() diff --git a/pypy/jit/metainterp/optimizeopt/test/test_util.py b/pypy/jit/metainterp/optimizeopt/test/test_util.py --- a/pypy/jit/metainterp/optimizeopt/test/test_util.py +++ b/pypy/jit/metainterp/optimizeopt/test/test_util.py @@ -383,7 +383,7 @@ boxkinds=boxkinds, invent_fail_descr=self.invent_fail_descr) - def invent_fail_descr(self, model, fail_args): + def invent_fail_descr(self, model, opnum, fail_args): if fail_args is None: return None descr = Storage() diff --git a/pypy/jit/metainterp/resume.py b/pypy/jit/metainterp/resume.py --- a/pypy/jit/metainterp/resume.py +++ b/pypy/jit/metainterp/resume.py @@ -974,11 +974,11 @@ num += len(self.liveboxes) assert num >= 0 if kind == INT: - box = BoxInt(self.cpu.get_latest_value_int(self.deadframe, num)) + box = BoxInt(self.cpu.get_int_value(self.deadframe, num)) elif kind == REF: - box = BoxPtr(self.cpu.get_latest_value_ref(self.deadframe, num)) + box = BoxPtr(self.cpu.get_ref_value(self.deadframe, num)) elif kind == FLOAT: - box = BoxFloat(self.cpu.get_latest_value_float(self.deadframe,num)) + box = BoxFloat(self.cpu.get_float_value(self.deadframe,num)) else: assert 0, "bad kind: %d" % ord(kind) self.liveboxes[num] = box @@ -1244,7 +1244,7 @@ assert tag == TAGBOX if num < 0: num += self.cpu.get_latest_value_count(self.deadframe) - return self.cpu.get_latest_value_int(self.deadframe, num) + return self.cpu.get_int_value(self.deadframe, num) def decode_ref(self, tagged): num, tag = untag(tagged) @@ -1258,7 +1258,7 @@ assert tag == TAGBOX if num < 0: num += self.cpu.get_latest_value_count(self.deadframe) - return self.cpu.get_latest_value_ref(self.deadframe, num) + return self.cpu.get_ref_value(self.deadframe, num) def decode_float(self, tagged): num, tag = untag(tagged) @@ -1268,7 +1268,7 @@ assert tag == TAGBOX if num < 0: num += self.cpu.get_latest_value_count(self.deadframe) - return self.cpu.get_latest_value_float(self.deadframe, num) + return self.cpu.get_float_value(self.deadframe, num) def write_an_int(self, index, int): self.blackholeinterp.setarg_i(index, int) diff --git a/pypy/jit/metainterp/test/support.py b/pypy/jit/metainterp/test/support.py --- a/pypy/jit/metainterp/test/support.py +++ b/pypy/jit/metainterp/test/support.py @@ -149,11 +149,11 @@ faildescr = cpu.get_latest_descr(deadframe) assert faildescr.__class__.__name__.startswith('DoneWithThisFrameDescr') if metainterp.jitdriver_sd.result_type == history.INT: - return cpu.get_latest_value_int(deadframe, 0) + return cpu.get_int_value(deadframe, 0) elif metainterp.jitdriver_sd.result_type == history.REF: - return cpu.get_latest_value_ref(deadframe, 0) + return cpu.get_ref_value(deadframe, 0) elif metainterp.jitdriver_sd.result_type == history.FLOAT: - return cpu.get_latest_value_float(deadframe, 0) + return cpu.get_float_value(deadframe, 0) else: return None From noreply at buildbot.pypy.org Sun Jan 13 21:33:06 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sun, 13 Jan 2013 21:33:06 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fix Message-ID: <20130113203306.621011C071B@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60043:46a50dcc27f1 Date: 2013-01-13 21:32 +0100 http://bitbucket.org/pypy/pypy/changeset/46a50dcc27f1/ Log: Fix diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -8,6 +8,7 @@ from rpython.rtyper.lltypesystem.lloperation import llop from rpython.rtyper.annlowlevel import llhelper from rpython.rlib.jit import AsmInfo +from rpython.rlib import longlong2float from rpython.jit.backend.model import CompiledLoopToken from rpython.jit.backend.x86.regalloc import (RegAlloc, get_ebp_ofs, _get_scale, gpr_reg_mgr_cls, xmm_reg_mgr_cls, _valid_addressing_size) diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py --- a/rpython/rlib/rthread.py +++ b/rpython/rlib/rthread.py @@ -196,6 +196,7 @@ # Thread integration. # These are six completely ad-hoc operations at the moment. + at jit.dont_look_inside def gc_thread_prepare(): """To call just before thread.start_new_thread(). This allocates a new shadow stack to be used by the future @@ -206,6 +207,7 @@ if we_are_translated(): llop.gc_thread_prepare(lltype.Void) + at jit.dont_look_inside def gc_thread_run(): """To call whenever the current thread (re-)acquired the GIL. """ @@ -213,12 +215,14 @@ llop.gc_thread_run(lltype.Void) gc_thread_run._always_inline_ = True + at jit.dont_look_inside def gc_thread_start(): """To call at the beginning of a new thread. """ if we_are_translated(): llop.gc_thread_start(lltype.Void) + at jit.dont_look_inside def gc_thread_die(): """To call just before the final GIL release done by a dying thread. After a thread_die(), no more gc operation should @@ -228,6 +232,7 @@ llop.gc_thread_die(lltype.Void) gc_thread_die._always_inline_ = True + at jit.dont_look_inside def gc_thread_before_fork(): """To call just before fork(). Prepares for forking, after which only the current thread will be alive. @@ -237,6 +242,7 @@ else: return llmemory.NULL + at jit.dont_look_inside def gc_thread_after_fork(result_of_fork, opaqueaddr): """To call just after fork(). """ From noreply at buildbot.pypy.org Mon Jan 14 01:10:51 2013 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Mon, 14 Jan 2013 01:10:51 +0100 (CET) Subject: [pypy-commit] pypy pytest: merge default Message-ID: <20130114001051.063C81C054C@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: pytest Changeset: r60044:7c8d34cf5c99 Date: 2013-01-14 01:01 +0100 http://bitbucket.org/pypy/pypy/changeset/7c8d34cf5c99/ Log: merge default diff too long, truncating to 2000 out of 107685 lines diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -2,3 +2,4 @@ b48df0bf4e75b81d98f19ce89d4a7dc3e1dab5e5 benchmarked d8ac7d23d3ec5f9a0fa1264972f74a010dbfd07f release-1.6 ff4af8f318821f7f5ca998613a60fca09aa137da release-1.7 +07e08e9c885ca67d89bcc304e45a32346daea2fa release-2.0-beta-1 diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -1,9 +1,10 @@ -License for files in the pypy/ directory -================================================== +License +======= Except when otherwise stated (look for LICENSE files in directories or information at the beginning of each file) all software and -documentation in the 'pypy' directories is licensed as follows: +documentation in the 'pypy', 'ctype_configure', 'dotviewer', 'demo', +and 'lib_pypy' directories is licensed as follows: The MIT License @@ -27,7 +28,7 @@ DEALINGS IN THE SOFTWARE. -PyPy Copyright holders 2003-2012 +PyPy Copyright holders 2003-2013 ----------------------------------- Except when otherwise stated (look for LICENSE files or information at diff --git a/README b/README deleted file mode 100644 --- a/README +++ /dev/null @@ -1,24 +0,0 @@ -===================================== -PyPy: Python in Python Implementation -===================================== - -Welcome to PyPy! - -PyPy is both an implementation of the Python programming language, and -an extensive compiler framework for dynamic language implementations. -You can build self-contained Python implementations which execute -independently from CPython. - -The home page is: - - http://pypy.org/ - -The getting-started document will help guide you: - - http://doc.pypy.org/en/latest/getting-started.html - -It will also point you to the rest of the documentation which is generated -from files in the pypy/doc directory within the source repositories. Enjoy -and send us feedback! - - the pypy-dev team diff --git a/README.rst b/README.rst new file mode 100644 --- /dev/null +++ b/README.rst @@ -0,0 +1,24 @@ +===================================== +PyPy: Python in Python Implementation +===================================== + +Welcome to PyPy! + +PyPy is both an implementation of the Python programming language, and +an extensive compiler framework for dynamic language implementations. +You can build self-contained Python implementations which execute +independently from CPython. + +The home page is: + + http://pypy.org/ + +The getting-started document will help guide you: + + http://doc.pypy.org/en/latest/getting-started.html + +It will also point you to the rest of the documentation which is generated +from files in the pypy/doc directory within the source repositories. Enjoy +and send us feedback! + + the pypy-dev team diff --git a/demo/autopath.py b/demo/autopath.py deleted file mode 100644 --- a/demo/autopath.py +++ /dev/null @@ -1,2 +0,0 @@ -import sys, os -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) diff --git a/demo/bpnn.py b/demo/bpnn.py deleted file mode 100755 --- a/demo/bpnn.py +++ /dev/null @@ -1,214 +0,0 @@ -#!/usr/bin/env python -""" - Translator Demo - - To analyse and type-annotate the functions and class defined in - this module, starting from the entry point function demo(), - use the following command line: - - ../pypy/translator/goal/translate.py bpnn.py - - Insert '--help' before 'bpnn.py' for a list of translation options, - or see the Overview of Command Line Options for translation at - http://codespeak.net/pypy/dist/pypy/doc/config/commandline.html -""" -# Back-Propagation Neural Networks -# -# Written in Python. See http://www.python.org/ -# -# Neil Schemenauer -# -# Modifications to the original (Armin Rigo): -# * import random from PyPy's lib, which is Python 2.2's plain -# Python implementation -# * print a doc about how to start the Translator - -import sys -import math -import time - -import autopath -from pypy.rlib import rrandom - -PRINT_IT = True - -random = rrandom.Random(1) - -# calculate a random number where: a <= rand < b -def rand(a, b): - return (b-a)*random.random() + a - -# Make a matrix (we could use NumPy to speed this up) -def makeMatrix(I, J, fill=0.0): - m = [] - for i in range(I): - m.append([fill]*J) - return m - -class NN: - - def __init__(self, ni, nh, no): - # number of input, hidden, and output nodes - self.ni = ni + 1 # +1 for bias node - self.nh = nh - self.no = no - - # activations for nodes - self.ai = [1.0]*self.ni - self.ah = [1.0]*self.nh - self.ao = [1.0]*self.no - - # create weights - self.wi = makeMatrix(self.ni, self.nh) - self.wo = makeMatrix(self.nh, self.no) - # set them to random vaules - for i in range(self.ni): - for j in range(self.nh): - self.wi[i][j] = rand(-2.0, 2.0) - for j in range(self.nh): - for k in range(self.no): - self.wo[j][k] = rand(-2.0, 2.0) - - # last change in weights for momentum - self.ci = makeMatrix(self.ni, self.nh) - self.co = makeMatrix(self.nh, self.no) - - def update(self, inputs): - if len(inputs) != self.ni-1: - raise ValueError, 'wrong number of inputs' - - # input activations - for i in range(self.ni-1): - #self.ai[i] = 1.0/(1.0+math.exp(-inputs[i])) - self.ai[i] = inputs[i] - - # hidden activations - for j in range(self.nh): - sum = 0.0 - for i in range(self.ni): - sum = sum + self.ai[i] * self.wi[i][j] - self.ah[j] = 1.0/(1.0+math.exp(-sum)) - - # output activations - for k in range(self.no): - sum = 0.0 - for j in range(self.nh): - sum = sum + self.ah[j] * self.wo[j][k] - self.ao[k] = 1.0/(1.0+math.exp(-sum)) - - return self.ao[:] - - - def backPropagate(self, targets, N, M): - if len(targets) != self.no: - raise ValueError, 'wrong number of target values' - - # calculate error terms for output - output_deltas = [0.0] * self.no - for k in range(self.no): - ao = self.ao[k] - output_deltas[k] = ao*(1-ao)*(targets[k]-ao) - - # calculate error terms for hidden - hidden_deltas = [0.0] * self.nh - for j in range(self.nh): - sum = 0.0 - for k in range(self.no): - sum = sum + output_deltas[k]*self.wo[j][k] - hidden_deltas[j] = self.ah[j]*(1-self.ah[j])*sum - - # update output weights - for j in range(self.nh): - for k in range(self.no): - change = output_deltas[k]*self.ah[j] - self.wo[j][k] = self.wo[j][k] + N*change + M*self.co[j][k] - self.co[j][k] = change - #print N*change, M*self.co[j][k] - - # update input weights - for i in range(self.ni): - for j in range(self.nh): - change = hidden_deltas[j]*self.ai[i] - self.wi[i][j] = self.wi[i][j] + N*change + M*self.ci[i][j] - self.ci[i][j] = change - - # calculate error - error = 0.0 - for k in range(len(targets)): - delta = targets[k]-self.ao[k] - error = error + 0.5*delta*delta - return error - - - def test(self, patterns): - for p in patterns: - if PRINT_IT: - print p[0], '->', self.update(p[0]) - - def weights(self): - if PRINT_IT: - print 'Input weights:' - for i in range(self.ni): - print self.wi[i] - print - print 'Output weights:' - for j in range(self.nh): - print self.wo[j] - - def train(self, patterns, iterations=2000, N=0.5, M=0.1): - # N: learning rate - # M: momentum factor - for i in xrange(iterations): - error = 0.0 - for p in patterns: - inputs = p[0] - targets = p[1] - self.update(inputs) - error = error + self.backPropagate(targets, N, M) - if PRINT_IT and i % 100 == 0: - print 'error', error - - -def demo(): - # Teach network XOR function - pat = [ - [[0,0], [0]], - [[0,1], [1]], - [[1,0], [1]], - [[1,1], [0]] - ] - - # create a network with two input, two hidden, and two output nodes - n = NN(2, 3, 1) - # train it with some patterns - n.train(pat, 2000) - # test it - n.test(pat) - - -# __________ Entry point for stand-alone builds __________ - -import time - -def entry_point(argv): - if len(argv) > 1: - N = int(argv[1]) - else: - N = 200 - T = time.time() - for i in range(N): - demo() - t1 = time.time() - T - print "%d iterations, %s milliseconds per iteration" % (N, 1000.0*t1/N) - return 0 - -# _____ Define and setup target ___ - -def target(*args): - return entry_point, None - -if __name__ == '__main__': - if len(sys.argv) == 1: - sys.argv.append('1') - entry_point(sys.argv) - print __doc__ diff --git a/demo/dis-goal.py b/demo/dis-goal.py deleted file mode 100644 --- a/demo/dis-goal.py +++ /dev/null @@ -1,7 +0,0 @@ -""" -An old-time classical example, and one of our first goals. -To run on top of PyPy. -""" - -import dis -dis.dis(dis.dis) diff --git a/demo/distribution/client.py b/demo/distribution/client.py deleted file mode 100644 --- a/demo/distribution/client.py +++ /dev/null @@ -1,35 +0,0 @@ -""" This a sample client, suitable for use with server.py from this -directory - -run by: -pypy-c client.py -""" - -HOST = '127.0.0.1' -PORT = 12222 - -from distributed.socklayer import connect -remote_handle = connect((HOST, PORT)) - -import code -code.interact(local=locals()) - -""" Things that can be done: 1. remote object access - -x = remote_handle.x -assert type(x) is remote_handle.X # typecheck -x.meth(lambda x: x + 10, 6) # remote call, with callback localy -x.meth(remote_handle.f, 3) # remote call, remote callback -remote_handle.sys._getframe(2).f_locals['x'] # remote frame access -# XXX should be 'is x' and shouldn't need (2) argument - -# XXX next one does not work, while it should. Too much mangling with remote -# traceback frames probably -try: - x.meth(1, 2) # non-callable argument, AssertionError -except: - import sys - e, c, tb = sys.exc_info() - import pdb - pdb.post_mortem(tb) -""" diff --git a/demo/distribution/fileclient.py b/demo/distribution/fileclient.py deleted file mode 100644 --- a/demo/distribution/fileclient.py +++ /dev/null @@ -1,25 +0,0 @@ - -""" This is sample client for a server based in fileserver.py, not counting -initialization, code.interact and __doc__ has just 2 lines! Usage: - -pypy-c fileclient.py - -The file_opener is a proxy for remote file object. Which means you can -perform same operations as locally, like file_opener('/etc/passwd').read() -or file_opener('/tmp/x', 'w').write('x') - -pypy-c needs to be compiled with --allworkingmodules in order to have socket -working. -""" - -HOST = '127.0.0.1' -PORT = 12221 - -from distributed.socklayer import connect -file_opener = connect((HOST, PORT)).open - -import code -code.interact(local=locals()) -# The file_opener is a proxy for remote file object. Which means you can -# perform same operations as locally, like file_opener('/etc/passwd').read() -# or file_opener('/tmp/x', 'w').write('x') diff --git a/demo/distribution/fileserver.py b/demo/distribution/fileserver.py deleted file mode 100644 --- a/demo/distribution/fileserver.py +++ /dev/null @@ -1,19 +0,0 @@ -""" This is a sample demo showcasing file server, done by the pypy -distribution library. - -Not counting __doc__ and initialization this is 2 line, -fully operational file server, -sample client which is in fileclient.py is included as well. - -run by: -pypy-c fileserver.py - -pypy-c needs to be compiled with --allworkingmodules in order to have socket -working. -""" - -HOST = '127.0.0.1' # defaults to localhost, not to export your files -PORT = 12221 - -from distributed.socklayer import socket_loop -socket_loop((HOST, PORT), {'open':open}) diff --git a/demo/distribution/server.py b/demo/distribution/server.py deleted file mode 100644 --- a/demo/distribution/server.py +++ /dev/null @@ -1,38 +0,0 @@ -""" This is a demo exposing all globals from the current process over -socket, to be accessible remotely. - -run by: -pypy-c server.py - -pypy-c needs to be compiled with --allworkingmodules in order to have socket -working. -""" - -# things to export -# function -def f(x): - return x + 3 - -# class -class X: - def __init__(self): - self.slot = 3 - - def meth(self, f, arg): - """ Method eating callable and calling it with an argument - """ - assert callable(f) - return f(arg) - -# object -x = X() - -# module -import sys - -# constants -HOST = '127.0.0.1' -PORT = 12222 - -from distributed.socklayer import socket_loop -socket_loop((HOST, PORT), globals()) diff --git a/demo/fibonacci.py b/demo/fibonacci.py deleted file mode 100644 --- a/demo/fibonacci.py +++ /dev/null @@ -1,43 +0,0 @@ -""" -Thunk (a.k.a. lazy objects) in PyPy. -To run on top of the thunk object space with the following command-line: - - py.py -o thunk fibonacci.py - -This is a typical Functional Programming Languages demo, computing the -Fibonacci sequence by using an infinite lazy linked list. -""" - -try: - from __pypy__ import thunk # only available in 'py.py -o thunk' -except ImportError: - print __doc__ - raise SystemExit(2) - -# ____________________________________________________________ - - -class ListNode: - def __init__(self, head, tail): - self.head = head # the first element of the list - self.tail = tail # the sublist of all remaining elements - - -def add_lists(list1, list2): - """Compute the linked-list equivalent of the Python expression - [a+b for (a,b) in zip(list1,list2)] - """ - return ListNode(list1.head + list2.head, - thunk(add_lists, list1.tail, list2.tail)) - - -# 1, 1, 2, 3, 5, 8, 13, 21, 34, ... -Fibonacci = ListNode(1, ListNode(1, None)) -Fibonacci.tail.tail = thunk(add_lists, Fibonacci, Fibonacci.tail) - - -if __name__ == '__main__': - node = Fibonacci - while True: - print node.head - node = node.tail diff --git a/demo/fibonacci2.py b/demo/fibonacci2.py deleted file mode 100644 --- a/demo/fibonacci2.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -Lazy functions in PyPy. -To run on top of the thunk object space with the following command-line: - - py.py -o thunk fibonacci2.py - -This is a typical Functional Programming Languages demo, computing the -Fibonacci sequence as nested 2-tuples. -""" - -import pprint - -try: - from __pypy__ import lazy -except ImportError: - print __doc__ - raise SystemExit(2) - - - at lazy -def fibo(a, b): - return (a, fibo(b, a + b)) - - -fibonacci = fibo(1, 1) - -pprint.pprint(fibonacci, depth=10) diff --git a/demo/foodbill.py b/demo/foodbill.py deleted file mode 100644 --- a/demo/foodbill.py +++ /dev/null @@ -1,59 +0,0 @@ -""" -Of historical interest: we computed the food bill of our first Gothenburg -sprint with PyPy :-) -""" - -slips=[(1, 'Kals MatMarkn', 6150, 'Chutney for Curry', 'dinner Saturday'), - (2, 'Kals MatMarkn', 32000, 'Spaghetti, Beer', 'dinner Monday'), - (2, 'Kals MatMarkn', -810, 'Deposit on Beer Bottles', 'various'), - (3, 'Fram', 7700, 'Rice and Curry Spice', 'dinner Saturday'), - (4, 'Kals MatMarkn', 25000, 'Alcohol-Free Beer, sundries', 'various'), - (4, 'Kals MatMarkn', -1570, "Michael's toothpaste", 'none'), - (4, 'Kals MatMarkn', -1690, "Laura's toothpaste", 'none'), - (4, 'Kals MatMarkn', -720, 'Deposit on Beer Bottles', 'various'), - (4, 'Kals MatMarkn', -60, 'Deposit on another Beer Bottle', 'various'), - (5, 'Kals MatMarkn', 26750, 'lunch bread meat cheese', 'lunch Monday'), - (6, 'Kals MatMarkn', 15950, 'various', 'dinner Tuesday and Thursday'), - (7, 'Kals MatMarkn', 3650, 'Drottningsylt, etc.', 'dinner Thursday'), - (8, 'Kals MatMarkn', 26150, 'Chicken and Mushroom Sauce', 'dinner Wed'), - (8, 'Kals MatMarkn', -2490, 'Jacob and Laura -- juice', 'dinner Wed'), - (8, 'Kals MatMarkn', -2990, "Chicken we didn't cook", 'dinner Wednesday'), - (9, 'Kals MatMarkn', 1380, 'fruit for Curry', 'dinner Saturday'), - (9, 'Kals MatMarkn', 1380, 'fruit for Curry', 'dinner Saturday'), - (10, 'Kals MatMarkn', 26900, 'Jansons Frestelse', 'dinner Sunday'), - (10, 'Kals MatMarkn', -540, 'Deposit on Beer Bottles', 'dinner Sunday'), - (11, 'Kals MatMarkn', 22650, 'lunch bread meat cheese', 'lunch Thursday'), - (11, 'Kals MatMarkn', -2190, 'Jacob and Laura -- juice', 'lunch Thursday'), - (11, 'Kals MatMarkn', -2790, 'Jacob and Laura -- cereal', 'lunch Thurs'), - (11, 'Kals MatMarkn', -760, 'Jacob and Laura -- milk', 'lunch Thursday'), - (12, 'Kals MatMarkn', 18850, 'lunch bread meat cheese', 'lunch Friday'), - (13, 'Kals MatMarkn', 18850, 'lunch bread meat cheese', 'guestimate Sun'), - (14, 'Kals MatMarkn', 18850, 'lunch bread meat cheese', 'guestimate Tues'), - (15, 'Kals MatMarkn', 20000, 'lunch bread meat cheese', 'guestimate Wed'), - (16, 'Kals MatMarkn', 42050, 'grillfest', 'dinner Friday'), - (16, 'Kals MatMarkn', -1350, 'Deposit on Beer Bottles', 'dinner Friday'), - (17, 'System Bolaget', 15500, 'Cederlunds Caloric', 'dinner Thursday'), - (17, 'System Bolaget', 22400, '4 x Farnese Sangiovese 56SEK', 'various'), - (17, 'System Bolaget', 22400, '4 x Farnese Sangiovese 56SEK', 'various'), - (17, 'System Bolaget', 13800, '2 x Jacobs Creek 69SEK', 'various'), - (18, 'J and Ls winecabinet', 10800, '2 x Parrotes 54SEK', 'various'), - (18, 'J and Ls winecabinet', 14700, '3 x Saint Paulin 49SEK', 'various'), - (18, 'J and Ls winecabinet', 10400, '2 x Farnese Sangioves 52SEK', - 'cheaper when we bought it'), - (18, 'J and Ls winecabinet', 17800, '2 x Le Poiane 89SEK', 'various'), - (18, 'J and Ls winecabinet', 9800, '2 x Something Else 49SEK', 'various'), - (19, 'Konsum', 26000, 'Saturday Bread and Fruit', 'Slip MISSING'), - (20, 'Konsum', 15245, 'Mooseburgers', 'found slip'), - (21, 'Kals MatMarkn', 20650, 'Grilling', 'Friday dinner'), - (22, 'J and Ls freezer', 21000, 'Meat for Curry, grilling', ''), - (22, 'J and Ls cupboard', 3000, 'Rice', ''), - (22, 'J and Ls cupboard', 4000, 'Charcoal', ''), - (23, 'Fram', 2975, 'Potatoes', '3.5 kg @ 8.50SEK'), - (23, 'Fram', 1421, 'Peas', 'Thursday dinner'), - (24, 'Kals MatMarkn', 20650, 'Grilling', 'Friday dinner'), - (24, 'Kals MatMarkn', -2990, 'TP', 'None'), - (24, 'Kals MatMarkn', -2320, 'T-Gul', 'None') - ] - -print [t[2] for t in slips] -print (reduce(lambda x, y: x+y, [t[2] for t in slips], 0))/900 diff --git a/demo/pickle_coroutine.py b/demo/pickle_coroutine.py deleted file mode 100644 --- a/demo/pickle_coroutine.py +++ /dev/null @@ -1,86 +0,0 @@ -""" -Stackless demo. - -This example only works on top of a pypy-c compiled with stackless features -and the signal module: - - translate.py --stackless targetpypystandalone --withmod-signal - -Usage: - - pypy-c pickle_coroutine.py --start demo.pickle - - Start the computation. You can interrupt it at any time by - pressing Ctrl-C; at this point, the state of the computing - coroutine is saved in demo.pickle. - - pypy-c pickle_coroutine.py --resume demo.pickle - - Reload the coroutine from demo.pickle and continue running it. - (It can be interrupted again with Ctrl-C.) - -This demo is documented in detail in pypy/doc/stackless.txt. -""" - -try: - import sys, pickle, signal - from stackless import coroutine -except ImportError: - print __doc__ - sys.exit(2) - - -def ackermann(x, y): - check() - if x == 0: - return y + 1 - if y == 0: - return ackermann(x - 1, 1) - return ackermann(x - 1, ackermann(x, y - 1)) - -# ____________________________________________________________ - -main = coroutine.getcurrent() -sys.setrecursionlimit(100000) - -interrupt_flag = False - -def interrupt_handler(*args): - global interrupt_flag - interrupt_flag = True - -def check(): - if interrupt_flag: - main.switch() - - -def execute(coro): - signal.signal(signal.SIGINT, interrupt_handler) - res = coro.switch() - if res is None and coro.is_alive: # interrupted! - print "interrupted! writing %s..." % (filename,) - f = open(filename, 'w') - pickle.dump(coro, f) - f.close() - print "done" - else: - print "result:", res - -try: - operation, filename = sys.argv[1:] -except ValueError: - print __doc__ - sys.exit(2) - -if operation == '--start': - coro = coroutine() - coro.bind(ackermann, 3, 7) - print "running from the start..." - execute(coro) -elif operation == '--resume': - print "reloading %s..." % (filename,) - f = open(filename) - coro = pickle.load(f) - f.close() - print "done, running now..." - execute(coro) diff --git a/demo/sharedref.py b/demo/sharedref.py deleted file mode 100644 --- a/demo/sharedref.py +++ /dev/null @@ -1,184 +0,0 @@ -""" - This is an example usage of the 'thunk' object space of PyPy. - It implements transparent distributed object manipulation. - - Start a server on a local port, say port 8888, with: - - $ py.py -o thunk sharedref.py 8888 - Waiting for connection on port 8888 - - Then start and connect a client from the same or another machine: - - $ py.py -o thunk sharedref.py ip_or_name:8888 - Connecting to ('...', 8888) - Ok - >>> l = [1,2,3] - >>> chan.send(l) # send the list to the server over the connexion - - On the server-side: - - Connected from ('...', 1046) - >>> l = chan.recv() # receive the list sent above - >>> l - [1, 2, 3] - >>> l.append(4) - - Back on the client-side: - - >>> l - [1, 2, 3, 4] - - The list behaves like a single distributed object, which both sides can - modify and access without needing further explicit synchronization. - There is no difference between who was the original sender or receiver of - the object, nor between which side was originally 'server' or 'client'. -""" - -import sys, marshal -from __pypy__ import thunk, become -from socket import * -from select import select - - -class Channel: - - def __init__(self, s, serverside): - # invariants: a shared object 'obj' is - # - either remote, and a thunk, and not a value in self.cache - # - or local (or at least on "our" side of this Channel), and - # then it has a corresponding key in self.cache - self.s = s - self.cache = {} - self.inputfifo = [] - self.count = int(not serverside) - -## def _check(self, obj): -## print '%s: cache=%r' % (self, self.cache.keys()), -## if is_thunk(obj): -## print 'THUNK' -## else: -## print obj - - def sendraw(self, obj): - data = marshal.dumps(obj) - hdr = str(len(data)) - hdr = '0'*(10-len(hdr)) + hdr - self.s.sendall(hdr + data) - - def _readbytes(self, count): - data = '' - while len(data) < count: - t = self.s.recv(count - len(data)) - if not t: - raise EOFError - data += t - return data - - def recvraw(self): - datasize = int(self._readbytes(10)) - data = self._readbytes(datasize) - return marshal.loads(data) - - def send(self, obj, n=None): - #print 'send', n,; self._check(obj) - if n is None: - n = self.count - self.count += 2 - data = (n, obj, None) - else: - data = (n, obj) - self.sendraw(data) - become(obj, thunk(self._resume, n)) - #print 'done', n,; self._check(obj) - - def recv(self): - obj = self.inputfifo.pop(0) - #print 'recv',; self._check(obj) - return obj - - def _resume(self, n): - #print 'resume', n,; sys.stdout.flush() - assert n not in self.cache - self.sendraw((n,)) - while n not in self.cache: - self.handle_once() - obj = self.cache[n] - #self._check(obj) - return obj - - def handle_once(self): - input = self.recvraw() - if len(input) > 1: - obj = input[1] - self.cache[input[0]] = obj - if len(input) > 2: - self.inputfifo.append(obj) - else: - n = input[0] - obj = self.cache[n] - self.send(obj, n) - del self.cache[n] - - -def mainloop(channels): - stdin = sys.stdin.fileno() - sockfd = [chan.s.fileno() for chan in channels] - while True: - sys.stdout.write('>>> ') - sys.stdout.flush() - while True: - iwtd, owtd, ewtd = select([stdin] + sockfd, [], [stdin]) - if stdin in iwtd or stdin in ewtd: break - for chan in channels: - if chan.s.fileno() in iwtd: - chan.handle_once() - code = raw_input() - if not code: break - try: - co = compile(code, '', 'single') - exec co in globals() - except Exception, e: - print e.__class__.__name__, str(e) - - -def server(port): - s = socket(AF_INET, SOCK_STREAM) - s.bind(('', port)) - s.listen(1) - print 'Waiting for connection on port', port - s, addr = s.accept() - print 'Connected from', addr - return Channel(s, True) - -def client(addr): - s = socket(AF_INET, SOCK_STREAM) - print 'Connecting to', addr - s.connect(addr) - print 'Ok' - return Channel(s, False) - - -if __name__ == '__main__': - try: - thunk, become # only available in 'py.py -o thunk' - except NameError: - print __doc__ - raise SystemExit(2) - - channels = [] - for a in sys.argv[1:]: - try: - port = int(a) - except ValueError: - host, port = a.split(':') - port = int(port) - chan = client((host, port)) - else: - chan = server(port) - channels.append(chan) - - try: - mainloop(channels) - finally: - for channel in channels: - channel.s.close() diff --git a/demo/tproxy/persistence.py b/demo/tproxy/persistence.py deleted file mode 100644 --- a/demo/tproxy/persistence.py +++ /dev/null @@ -1,62 +0,0 @@ -""" - -This small example implements a basic orthogonal persistence -mechanism on top of PyPy's transparent proxies. - -""" -from tputil import make_proxy - -list_changeops = set('__iadd__ __imul__ __delitem__ __setitem__ __setattr__' - '__delslice__ __setslice__ ' - 'append extend insert pop remove reverse sort'.split()) - -dict_changeops = set('__delitem__ __setitem__ __setattr__' - 'clear pop popitem setdefault update'.split()) - -def ischangeop(operation): - """ return True if this operation is a changing operation - on known builtins (dicts, lists). - """ - if isinstance(operation.obj, list): - changeops = list_changeops - elif isinstance(operation.obj, dict): - changeops = dict_changeops - else: - return False - return operation.opname in changeops - -def make_persistent_proxy(instance, storage): - def perform(operation): - res = operation.delegate() - if ischangeop(operation): - print "persisting after:", operation - storage.dump(instance) - if res is not operation.proxyobj and isinstance(res, (dict, list)): - res = make_proxy(perform, obj=res) - return res - return make_proxy(perform, obj=instance) - -def load(storage): - obj = storage.load() - return make_persistent_proxy(obj, storage) - -if __name__ == '__main__': - import py - storage = py.path.local("/tmp/dictpickle") - pdict = make_persistent_proxy({}, storage) - - # the code below is not aware of pdict being a proxy - assert type(pdict) is dict - pdict['hello'] = 'world' - pdict['somelist'] = [] - del pdict - - newdict = load(storage) - assert newdict == {'hello': 'world', 'somelist': []} - l = newdict['somelist'] - l.append(1) # this triggers persisting the whole dict - l.extend([2,3]) # this triggers persisting the whole dict - del newdict, l - - newdict = load(storage) - print newdict['somelist'] # will show [1,2,3] diff --git a/demo/tproxy/print_operations.py b/demo/tproxy/print_operations.py deleted file mode 100644 --- a/demo/tproxy/print_operations.py +++ /dev/null @@ -1,26 +0,0 @@ -""" - -This example transparently intercepts and shows operations on -builtin objects. Requires the "--objspace-std-withtproxy" option. - -""" - -from tputil import make_proxy - -def make_show_proxy(instance): - def controller(operation): - print "proxy sees:", operation - res = operation.delegate() - return res - tproxy = make_proxy(controller, obj=instance) - return tproxy - -if __name__ == '__main__': - mydict = make_show_proxy({}) - assert type(mydict) is dict # this looks exactly like a dict - mydict['hello'] = 'world' # will print __setitem__ - mydict[42] = 23 # will print __setitem__ - assert mydict.pop('hello') == 'world' # will print pop - assert mydict.popitem() == (42,23) # will print popitem - - diff --git a/lib-python/2.7/ctypes/test/test_internals.py b/lib-python/2.7/ctypes/test/test_internals.py --- a/lib-python/2.7/ctypes/test/test_internals.py +++ b/lib-python/2.7/ctypes/test/test_internals.py @@ -1,7 +1,10 @@ # This tests the internal _objects attribute import unittest from ctypes import * -from sys import getrefcount as grc +try: + from sys import getrefcount as grc +except ImportError: + grc = None # e.g. PyPy # XXX This test must be reviewed for correctness!!! @@ -22,6 +25,8 @@ self.assertEqual(id(a), id(b)) def test_ints(self): + if grc is None: + return unittest.skip("no sys.getrefcount()") i = 42000123 refcnt = grc(i) ci = c_int(i) @@ -29,6 +34,8 @@ self.assertEqual(ci._objects, None) def test_c_char_p(self): + if grc is None: + return unittest.skip("no sys.getrefcount()") s = "Hello, World" refcnt = grc(s) cs = c_char_p(s) diff --git a/lib-python/2.7/ctypes/test/test_memfunctions.py b/lib-python/2.7/ctypes/test/test_memfunctions.py --- a/lib-python/2.7/ctypes/test/test_memfunctions.py +++ b/lib-python/2.7/ctypes/test/test_memfunctions.py @@ -53,7 +53,8 @@ s = string_at("foo bar") # XXX The following may be wrong, depending on how Python # manages string instances - self.assertEqual(2, sys.getrefcount(s)) + if hasattr(sys, 'getrefcount'): + self.assertEqual(2, sys.getrefcount(s)) self.assertTrue(s, "foo bar") self.assertEqual(string_at("foo bar", 8), "foo bar\0") diff --git a/lib-python/2.7/ctypes/test/test_python_api.py b/lib-python/2.7/ctypes/test/test_python_api.py --- a/lib-python/2.7/ctypes/test/test_python_api.py +++ b/lib-python/2.7/ctypes/test/test_python_api.py @@ -9,7 +9,10 @@ ################################################################ -from sys import getrefcount as grc +try: + from sys import getrefcount as grc +except ImportError: + grc = None # e.g. PyPy if sys.version_info > (2, 4): c_py_ssize_t = c_size_t else: diff --git a/lib-python/2.7/ctypes/test/test_refcounts.py b/lib-python/2.7/ctypes/test/test_refcounts.py --- a/lib-python/2.7/ctypes/test/test_refcounts.py +++ b/lib-python/2.7/ctypes/test/test_refcounts.py @@ -11,7 +11,10 @@ class RefcountTestCase(unittest.TestCase): def test_1(self): - from sys import getrefcount as grc + try: + from sys import getrefcount as grc + except ImportError: + return unittest.skip("no sys.getrefcount()") f = dll._testfunc_callback_i_if f.restype = ctypes.c_int @@ -35,7 +38,10 @@ def test_refcount(self): - from sys import getrefcount as grc + try: + from sys import getrefcount as grc + except ImportError: + return unittest.skip("no sys.getrefcount()") def func(*args): pass # this is the standard refcount for func @@ -84,6 +90,10 @@ class AnotherLeak(unittest.TestCase): def test_callback(self): import sys + try: + from sys import getrefcount + except ImportError: + return unittest.skip("no sys.getrefcount()") proto = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int, ctypes.c_int) def func(a, b): diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py --- a/lib-python/2.7/distutils/command/build_ext.py +++ b/lib-python/2.7/distutils/command/build_ext.py @@ -699,6 +699,9 @@ shared extension. On most platforms, this is just 'ext.libraries'; on Windows and OS/2, we add the Python library (eg. python20.dll). """ + # For PyPy, we must not add any such Python library, on any platform + if "__pypy__" in sys.builtin_module_names: + return ext.libraries # The python library is always needed on Windows. if sys.platform == "win32": template = "python%d%d" diff --git a/lib-python/2.7/json/decoder.py b/lib-python/2.7/json/decoder.py --- a/lib-python/2.7/json/decoder.py +++ b/lib-python/2.7/json/decoder.py @@ -62,8 +62,7 @@ DEFAULT_ENCODING = "utf-8" -def py_scanstring(s, end, encoding=None, strict=True, - _b=BACKSLASH, _m=STRINGCHUNK.match): +def py_scanstring(s, end, encoding=None, strict=True): """Scan the string s for a JSON string. End is the index of the character in s after the quote that started the JSON string. Unescapes all valid JSON string escape sequences and raises ValueError @@ -78,7 +77,7 @@ _append = chunks.append begin = end - 1 while 1: - chunk = _m(s, end) + chunk = STRINGCHUNK.match(s, end) if chunk is None: raise ValueError( errmsg("Unterminated string starting at", s, begin)) @@ -109,7 +108,7 @@ # If not a unicode escape sequence, must be in the lookup table if esc != 'u': try: - char = _b[esc] + char = BACKSLASH[esc] except KeyError: msg = "Invalid \\escape: " + repr(esc) raise ValueError(errmsg(msg, s, end)) @@ -147,7 +146,7 @@ WHITESPACE_STR = ' \t\n\r' def JSONObject(s_and_end, encoding, strict, scan_once, object_hook, - object_pairs_hook, _w=WHITESPACE.match, _ws=WHITESPACE_STR): + object_pairs_hook): s, end = s_and_end pairs = [] pairs_append = pairs.append @@ -156,8 +155,8 @@ nextchar = s[end:end + 1] # Normally we expect nextchar == '"' if nextchar != '"': - if nextchar in _ws: - end = _w(s, end).end() + if nextchar in WHITESPACE_STR: + end = WHITESPACE.match(s, end).end() nextchar = s[end:end + 1] # Trivial empty object if nextchar == '}': @@ -177,17 +176,17 @@ # To skip some function call overhead we optimize the fast paths where # the JSON key separator is ": " or just ":". if s[end:end + 1] != ':': - end = _w(s, end).end() + end = WHITESPACE.match(s, end).end() if s[end:end + 1] != ':': raise ValueError(errmsg("Expecting : delimiter", s, end)) end += 1 try: - if s[end] in _ws: + if s[end] in WHITESPACE_STR: end += 1 - if s[end] in _ws: - end = _w(s, end + 1).end() + if s[end] in WHITESPACE_STR: + end = WHITESPACE.match(s, end + 1).end() except IndexError: pass @@ -199,8 +198,8 @@ try: nextchar = s[end] - if nextchar in _ws: - end = _w(s, end + 1).end() + if nextchar in WHITESPACE_STR: + end = WHITESPACE.match(s, end + 1).end() nextchar = s[end] except IndexError: nextchar = '' @@ -213,11 +212,11 @@ try: nextchar = s[end] - if nextchar in _ws: + if nextchar in WHITESPACE_STR: end += 1 nextchar = s[end] - if nextchar in _ws: - end = _w(s, end + 1).end() + if nextchar in WHITESPACE_STR: + end = WHITESPACE.match(s, end + 1).end() nextchar = s[end] except IndexError: nextchar = '' @@ -234,12 +233,12 @@ pairs = object_hook(pairs) return pairs, end -def JSONArray(s_and_end, scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR): +def JSONArray(s_and_end, scan_once): s, end = s_and_end values = [] nextchar = s[end:end + 1] - if nextchar in _ws: - end = _w(s, end + 1).end() + if nextchar in WHITESPACE_STR: + end = WHITESPACE.match(s, end + 1).end() nextchar = s[end:end + 1] # Look-ahead for trivial empty array if nextchar == ']': @@ -252,8 +251,8 @@ raise ValueError(errmsg("Expecting object", s, end)) _append(value) nextchar = s[end:end + 1] - if nextchar in _ws: - end = _w(s, end + 1).end() + if nextchar in WHITESPACE_STR: + end = WHITESPACE.match(s, end + 1).end() nextchar = s[end:end + 1] end += 1 if nextchar == ']': @@ -262,10 +261,10 @@ raise ValueError(errmsg("Expecting , delimiter", s, end)) try: - if s[end] in _ws: + if s[end] in WHITESPACE_STR: end += 1 - if s[end] in _ws: - end = _w(s, end + 1).end() + if s[end] in WHITESPACE_STR: + end = WHITESPACE.match(s, end + 1).end() except IndexError: pass @@ -358,13 +357,13 @@ self.parse_string = scanstring self.scan_once = scanner.make_scanner(self) - def decode(self, s, _w=WHITESPACE.match): + def decode(self, s): """Return the Python representation of ``s`` (a ``str`` or ``unicode`` instance containing a JSON document) """ - obj, end = self.raw_decode(s, idx=_w(s, 0).end()) - end = _w(s, end).end() + obj, end = self.raw_decode(s, idx=WHITESPACE.match(s, 0).end()) + end = WHITESPACE.match(s, end).end() if end != len(s): raise ValueError(errmsg("Extra data", s, end, len(s))) return obj diff --git a/lib-python/2.7/sre_parse.py b/lib-python/2.7/sre_parse.py --- a/lib-python/2.7/sre_parse.py +++ b/lib-python/2.7/sre_parse.py @@ -16,6 +16,12 @@ from sre_constants import * +try: + from __pypy__ import newdict +except ImportError: + def newdict(tp): + return {} + SPECIAL_CHARS = ".\\[{()*+?^$|" REPEAT_CHARS = "*+?{" @@ -68,7 +74,7 @@ self.flags = 0 self.open = [] self.groups = 1 - self.groupdict = {} + self.groupdict = newdict("module") def opengroup(self, name=None): gid = self.groups self.groups = gid + 1 diff --git a/lib-python/2.7/timeit.py b/lib-python/2.7/timeit.py --- a/lib-python/2.7/timeit.py +++ b/lib-python/2.7/timeit.py @@ -190,7 +190,8 @@ else: it = [None] * number gcold = gc.isenabled() - gc.disable() + if '__pypy__' not in sys.builtin_module_names: + gc.disable() # only do that on CPython try: timing = self.inner(it, self.timer) finally: diff --git a/lib-python/conftest.py b/lib-python/conftest.py --- a/lib-python/conftest.py +++ b/lib-python/conftest.py @@ -14,7 +14,7 @@ from pypy.interpreter.main import run_string, run_file # the following adds command line options as a side effect! -from pypy.conftest import gettestobjspace, option as pypy_option +from pypy.conftest import option as pypy_option from pypy.tool.pytest import appsupport from pypy.tool.pytest.confpath import pypydir, testdir, testresultdir diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py --- a/lib_pypy/_ctypes/structure.py +++ b/lib_pypy/_ctypes/structure.py @@ -69,7 +69,8 @@ resnames.append(name) names = resnames self._names = names - self.__dict__.update(fields) + for name, field in fields.items(): + setattr(self, name, field) class Field(object): def __init__(self, name, offset, size, ctype, num, is_bitfield): diff --git a/lib_pypy/_ctypes_test.py b/lib_pypy/_ctypes_test.py --- a/lib_pypy/_ctypes_test.py +++ b/lib_pypy/_ctypes_test.py @@ -2,10 +2,6 @@ import tempfile import gc -# Monkeypatch & hacks to let ctypes.tests import. -# This should be removed at some point. -sys.getrefcount = lambda x: len(gc.get_referrers(x)) - 1 - def compile_shared(): """Compile '_ctypes_test.c' into an extension module, and import it """ diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py --- a/lib_pypy/_sqlite3.py +++ b/lib_pypy/_sqlite3.py @@ -784,6 +784,9 @@ self.statement.reset() raise self.connection._get_exception(ret) + if self.statement.kind == DML: + self.statement.reset() + if self.statement.kind == DQL and ret == SQLITE_ROW: self.statement._build_row_cast_map() self.statement._readahead(self) @@ -791,9 +794,6 @@ self.statement.item = None self.statement.exhausted = True - if self.statement.kind == DML: - self.statement.reset() - self.rowcount = -1 if self.statement.kind == DML: self.rowcount = sqlite.sqlite3_changes(self.connection.db) diff --git a/lib_pypy/dbm.py b/lib_pypy/dbm.py --- a/lib_pypy/dbm.py +++ b/lib_pypy/dbm.py @@ -126,8 +126,11 @@ libpath = ctypes.util.find_library('db') if not libpath: # XXX this is hopeless... - libpath = ctypes.util.find_library('db-4.5') - if not libpath: + for c in '56789': + libpath = ctypes.util.find_library('db-4.%s' % c) + if libpath: + break + else: raise ImportError("Cannot find dbm library") lib = CDLL(libpath) # Linux _platform = 'bdb' diff --git a/lib_pypy/itertools.py b/lib_pypy/itertools.py deleted file mode 100644 --- a/lib_pypy/itertools.py +++ /dev/null @@ -1,670 +0,0 @@ -# Note that PyPy contains also a built-in module 'itertools' which will -# hide this one if compiled in. - -"""Functional tools for creating and using iterators. - -Infinite iterators: -count([n]) --> n, n+1, n+2, ... -cycle(p) --> p0, p1, ... plast, p0, p1, ... -repeat(elem [,n]) --> elem, elem, elem, ... endlessly or up to n times - -Iterators terminating on the shortest input sequence: -izip(p, q, ...) --> (p[0], q[0]), (p[1], q[1]), ... -ifilter(pred, seq) --> elements of seq where pred(elem) is True -ifilterfalse(pred, seq) --> elements of seq where pred(elem) is False -islice(seq, [start,] stop [, step]) --> elements from - seq[start:stop:step] -imap(fun, p, q, ...) --> fun(p0, q0), fun(p1, q1), ... -starmap(fun, seq) --> fun(*seq[0]), fun(*seq[1]), ... -tee(it, n=2) --> (it1, it2 , ... itn) splits one iterator into n -chain(p, q, ...) --> p0, p1, ... plast, q0, q1, ... -takewhile(pred, seq) --> seq[0], seq[1], until pred fails -dropwhile(pred, seq) --> seq[n], seq[n+1], starting when pred fails -groupby(iterable[, keyfunc]) --> sub-iterators grouped by value of keyfunc(v) -""" - -__all__ = ['chain', 'count', 'cycle', 'dropwhile', 'groupby', 'ifilter', - 'ifilterfalse', 'imap', 'islice', 'izip', 'repeat', 'starmap', - 'takewhile', 'tee', 'compress', 'product'] - -try: from __pypy__ import builtinify -except ImportError: builtinify = lambda f: f - - -class chain(object): - """Make an iterator that returns elements from the first iterable - until it is exhausted, then proceeds to the next iterable, until - all of the iterables are exhausted. Used for treating consecutive - sequences as a single sequence. - - Equivalent to : - - def chain(*iterables): - for it in iterables: - for element in it: - yield element - """ - def __init__(self, *iterables): - self._iterables_iter = iter(map(iter, iterables)) - # little trick for the first chain.next() call - self._cur_iterable_iter = iter([]) - - def __iter__(self): - return self - - def next(self): - while True: - try: - return self._cur_iterable_iter.next() - except StopIteration: - self._cur_iterable_iter = self._iterables_iter.next() - except AttributeError: - # CPython raises a TypeError when next() is not defined - raise TypeError('%s has no next() method' % \ - (self._cur_iterable_iter)) - - -class compress(object): - def __init__(self, data, selectors): - self.data = iter(data) - self.selectors = iter(selectors) - - def __iter__(self): - return self - - def next(self): - while True: - next_item = self.data.next() - next_selector = self.selectors.next() - if bool(next_selector): - return next_item - - -class count(object): - """Make an iterator that returns consecutive integers starting - with n. If not specified n defaults to zero. Does not currently - support python long integers. Often used as an argument to imap() - to generate consecutive data points. Also, used with izip() to - add sequence numbers. - - Equivalent to : - - def count(n=0): - if not isinstance(n, int): - raise TypeError("%s is not a regular integer" % n) - while True: - yield n - n += 1 - """ - def __init__(self, n=0): - if not isinstance(n, int): - raise TypeError('%s is not a regular integer' % n) - self.times = n-1 - - def __iter__(self): - return self - - def next(self): - self.times += 1 - return self.times - - def __repr__(self): - return 'count(%d)' % (self.times + 1) - - - -class cycle(object): - """Make an iterator returning elements from the iterable and - saving a copy of each. When the iterable is exhausted, return - elements from the saved copy. Repeats indefinitely. - - Equivalent to : - - def cycle(iterable): - saved = [] - for element in iterable: - yield element - saved.append(element) - while saved: - for element in saved: - yield element - """ - def __init__(self, iterable): - self._cur_iter = iter(iterable) - self._saved = [] - self._must_save = True - - def __iter__(self): - return self - - def next(self): - # XXX Could probably be improved - try: - next_elt = self._cur_iter.next() - if self._must_save: - self._saved.append(next_elt) - except StopIteration: - self._cur_iter = iter(self._saved) - next_elt = self._cur_iter.next() - self._must_save = False - except AttributeError: - # CPython raises a TypeError when next() is not defined - raise TypeError('%s has no next() method' % \ - (self._cur_iter)) - return next_elt - - -class dropwhile(object): - """Make an iterator that drops elements from the iterable as long - as the predicate is true; afterwards, returns every - element. Note, the iterator does not produce any output until the - predicate is true, so it may have a lengthy start-up time. - - Equivalent to : - - def dropwhile(predicate, iterable): - iterable = iter(iterable) - for x in iterable: - if not predicate(x): - yield x - break - for x in iterable: - yield x - """ - def __init__(self, predicate, iterable): - self._predicate = predicate - self._iter = iter(iterable) - self._dropped = False - - def __iter__(self): - return self - - def next(self): - try: - value = self._iter.next() - except AttributeError: - # CPython raises a TypeError when next() is not defined - raise TypeError('%s has no next() method' % \ - (self._iter)) - if self._dropped: - return value - while self._predicate(value): - value = self._iter.next() - self._dropped = True - return value - -class groupby(object): - """Make an iterator that returns consecutive keys and groups from the - iterable. The key is a function computing a key value for each - element. If not specified or is None, key defaults to an identity - function and returns the element unchanged. Generally, the - iterable needs to already be sorted on the same key function. - - The returned group is itself an iterator that shares the - underlying iterable with groupby(). Because the source is shared, - when the groupby object is advanced, the previous group is no - longer visible. So, if that data is needed later, it should be - stored as a list: - - groups = [] - uniquekeys = [] - for k, g in groupby(data, keyfunc): - groups.append(list(g)) # Store group iterator as a list - uniquekeys.append(k) - """ - def __init__(self, iterable, key=None): - if key is None: - key = lambda x: x - self.keyfunc = key - self.it = iter(iterable) - self.tgtkey = self.currkey = self.currvalue = xrange(0) - - def __iter__(self): - return self - - def next(self): - while self.currkey == self.tgtkey: - try: - self.currvalue = self.it.next() # Exit on StopIteration - except AttributeError: - # CPython raises a TypeError when next() is not defined - raise TypeError('%s has no next() method' % \ - (self.it)) - self.currkey = self.keyfunc(self.currvalue) - self.tgtkey = self.currkey - return (self.currkey, self._grouper(self.tgtkey)) - - def _grouper(self, tgtkey): - while self.currkey == tgtkey: - yield self.currvalue - self.currvalue = self.it.next() # Exit on StopIteration - self.currkey = self.keyfunc(self.currvalue) - - - -class _ifilter_base(object): - """base class for ifilter and ifilterflase""" - def __init__(self, predicate, iterable): - # Make sure iterable *IS* iterable - self._iter = iter(iterable) - if predicate is None: - self._predicate = bool - else: - self._predicate = predicate - - def __iter__(self): - return self - -class ifilter(_ifilter_base): - """Make an iterator that filters elements from iterable returning - only those for which the predicate is True. If predicate is - None, return the items that are true. - - Equivalent to : - - def ifilter: - if predicate is None: - predicate = bool - for x in iterable: - if predicate(x): - yield x - """ - def next(self): - try: - next_elt = self._iter.next() - except AttributeError: - # CPython raises a TypeError when next() is not defined - raise TypeError('%s has no next() method' % \ - (self._iter)) - while True: - if self._predicate(next_elt): - return next_elt - next_elt = self._iter.next() - -class ifilterfalse(_ifilter_base): - """Make an iterator that filters elements from iterable returning - only those for which the predicate is False. If predicate is - None, return the items that are false. - - Equivalent to : - - def ifilterfalse(predicate, iterable): - if predicate is None: - predicate = bool - for x in iterable: - if not predicate(x): - yield x - """ - def next(self): - try: - next_elt = self._iter.next() - except AttributeError: - # CPython raises a TypeError when next() is not defined - raise TypeError('%s has no next() method' % \ - (self._iter)) - while True: - if not self._predicate(next_elt): - return next_elt - next_elt = self._iter.next() - - - - -class imap(object): - """Make an iterator that computes the function using arguments - from each of the iterables. If function is set to None, then - imap() returns the arguments as a tuple. Like map() but stops - when the shortest iterable is exhausted instead of filling in - None for shorter iterables. The reason for the difference is that - infinite iterator arguments are typically an error for map() - (because the output is fully evaluated) but represent a common - and useful way of supplying arguments to imap(). - - Equivalent to : - - def imap(function, *iterables): - iterables = map(iter, iterables) - while True: - args = [i.next() for i in iterables] - if function is None: - yield tuple(args) - else: - yield function(*args) - - """ - def __init__(self, function, iterable, *other_iterables): - self._func = function - self._iters = map(iter, (iterable, ) + other_iterables) - - def __iter__(self): - return self - - def next(self): - try: - args = [it.next() for it in self._iters] - except AttributeError: - # CPython raises a TypeError when next() is not defined - raise TypeError('%s has no next() method' % \ - (it)) - if self._func is None: - return tuple(args) - else: - return self._func(*args) - - - -class islice(object): - """Make an iterator that returns selected elements from the - iterable. If start is non-zero, then elements from the iterable - are skipped until start is reached. Afterward, elements are - returned consecutively unless step is set higher than one which - results in items being skipped. If stop is None, then iteration - continues until the iterator is exhausted, if at all; otherwise, - it stops at the specified position. Unlike regular slicing, - islice() does not support negative values for start, stop, or - step. Can be used to extract related fields from data where the - internal structure has been flattened (for example, a multi-line - report may list a name field on every third line). - """ - def __init__(self, iterable, *args): - s = slice(*args) - self.start, self.stop, self.step = s.start or 0, s.stop, s.step - if not isinstance(self.start, (int, long)): - raise ValueError("Start argument must be an integer") - if self.stop is not None and not isinstance(self.stop, (int,long)): - raise ValueError("Stop argument must be an integer or None") - if self.step is None: - self.step = 1 - if self.start<0 or (self.stop is not None and self.stop<0 - ) or self.step<=0: - raise ValueError, "indices for islice() must be positive" - self.it = iter(iterable) - self.donext = None - self.cnt = 0 - - def __iter__(self): - return self - - def next(self): - if self.donext is None: - try: - self.donext = self.it.next - except AttributeError: - raise TypeError - nextindex = self.start - if self.stop is not None and nextindex >= self.stop: - raise StopIteration - while self.cnt <= nextindex: - nextitem = self.donext() - self.cnt += 1 - self.start += self.step - return nextitem - -class izip(object): - """Make an iterator that aggregates elements from each of the - iterables. Like zip() except that it returns an iterator instead - of a list. Used for lock-step iteration over several iterables at - a time. - - Equivalent to : - - def izip(*iterables): - iterables = map(iter, iterables) - while iterables: - result = [i.next() for i in iterables] - yield tuple(result) - """ - def __init__(self, *iterables): - self._iterators = map(iter, iterables) - self._result = [None] * len(self._iterators) - - def __iter__(self): - return self - - def next(self): - if not self._iterators: - raise StopIteration() - try: - return tuple([i.next() for i in self._iterators]) - except AttributeError: - # CPython raises a TypeError when next() is not defined - raise TypeError('%s has no next() method' % (i)) - - -class product(object): - - def __init__(self, *args, **kw): - if len(kw) > 1: - raise TypeError("product() takes at most 1 argument (%d given)" % - len(kw)) - self.repeat = kw.get('repeat', 1) - self.gears = [x for x in args] * self.repeat - self.num_gears = len(self.gears) - # initialization of indicies to loop over - self.indicies = [(0, len(self.gears[x])) - for x in range(0, self.num_gears)] - self.cont = True - - def roll_gears(self): - # Starting from the end of the gear indicies work to the front - # incrementing the gear until the limit is reached. When the limit - # is reached carry operation to the next gear - should_carry = True - for n in range(0, self.num_gears): - nth_gear = self.num_gears - n - 1 - if should_carry: - count, lim = self.indicies[nth_gear] - count += 1 - if count == lim and nth_gear == 0: - self.cont = False - if count == lim: - should_carry = True - count = 0 - else: - should_carry = False - self.indicies[nth_gear] = (count, lim) - else: - break - - def __iter__(self): - return self - - def next(self): - if not self.cont: - raise StopIteration - l = [] - for x in range(0, self.num_gears): - index, limit = self.indicies[x] - l.append(self.gears[x][index]) - self.roll_gears() - return tuple(l) - - -class repeat(object): - """Make an iterator that returns object over and over again. - Runs indefinitely unless the times argument is specified. Used - as argument to imap() for invariant parameters to the called - function. Also used with izip() to create an invariant part of a - tuple record. - - Equivalent to : - - def repeat(object, times=None): - if times is None: - while True: - yield object - else: - for i in xrange(times): - yield object - """ - def __init__(self, obj, times=None): - self._obj = obj - if times is not None: - xrange(times) # Raise a TypeError - if times < 0: - times = 0 - self._times = times - - def __iter__(self): - return self - - def next(self): - # next() *need* to decrement self._times when consumed - if self._times is not None: - if self._times <= 0: - raise StopIteration() - self._times -= 1 - return self._obj - - def __repr__(self): - if self._times is not None: - return 'repeat(%r, %r)' % (self._obj, self._times) - else: - return 'repeat(%r)' % (self._obj,) - - def __len__(self): - if self._times == -1 or self._times is None: - raise TypeError("len() of uniszed object") - return self._times - - -class starmap(object): - """Make an iterator that computes the function using arguments - tuples obtained from the iterable. Used instead of imap() when - argument parameters are already grouped in tuples from a single - iterable (the data has been ``pre-zipped''). The difference - between imap() and starmap() parallels the distinction between - function(a,b) and function(*c). - - Equivalent to : - - def starmap(function, iterable): - iterable = iter(iterable) - while True: - yield function(*iterable.next()) - """ - def __init__(self, function, iterable): - self._func = function - self._iter = iter(iterable) - - def __iter__(self): - return self - - def next(self): - # CPython raises a TypeError when the iterator doesn't return a tuple - try: - t = self._iter.next() - except AttributeError: - # CPython raises a TypeError when next() is not defined - raise TypeError('%s has no next() method' % self._iter) - if not isinstance(t, tuple): - raise TypeError("iterator must return a tuple") - return self._func(*t) - - - -class takewhile(object): - """Make an iterator that returns elements from the iterable as - long as the predicate is true. - - Equivalent to : - - def takewhile(predicate, iterable): - for x in iterable: - if predicate(x): - yield x - else: - break - """ - def __init__(self, predicate, iterable): - self._predicate = predicate - self._iter = iter(iterable) - - def __iter__(self): - return self - - def next(self): - try: - value = self._iter.next() - except AttributeError: - # CPython raises a TypeError when next() is not defined - raise TypeError('%s has no next() method' % \ - (self._iter)) - if not self._predicate(value): - raise StopIteration() - return value - - -class TeeData(object): - """Holds cached values for TeeObjects""" - def __init__(self, iterator): - self.data = [] - self._iter = iterator - - def __getitem__(self, i): - # iterates until 'i' if not done yet - while i>= len(self.data): - try: - self.data.append( self._iter.next() ) - except AttributeError: - # CPython raises a TypeError when next() is not defined - raise TypeError('%s has no next() method' % self._iter) - return self.data[i] - - -class TeeObject(object): - """Iterables / Iterators as returned by the tee() function""" - def __init__(self, iterable=None, tee_data=None): - if tee_data: - self.tee_data = tee_data - self.pos = 0 From noreply at buildbot.pypy.org Mon Jan 14 01:10:52 2013 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Mon, 14 Jan 2013 01:10:52 +0100 (CET) Subject: [pypy-commit] pypy pytest: clean up expect collectors Message-ID: <20130114001052.384391C054C@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: pytest Changeset: r60045:5310b273f150 Date: 2013-01-14 01:10 +0100 http://bitbucket.org/pypy/pypy/changeset/5310b273f150/ Log: clean up expect collectors diff --git a/pypy/tool/pytest/expecttest.py b/pypy/tool/pytest/expecttest.py --- a/pypy/tool/pytest/expecttest.py +++ b/pypy/tool/pytest/expecttest.py @@ -7,12 +7,14 @@ import py -import os, sys +import pytest +import os +import sys from pypy.tool.udir import udir from pypy.tool.autopath import pypydir -class ExpectTestMethod(py.test.collect.Function): +class ExpectTestMethod(pytest.collect.Function): @staticmethod def safe_name(target): s = "_".join(target) @@ -22,6 +24,8 @@ s = s.replace(os.sep, "_") return s + #XXX: we really want to get the tmpdir fixture and + # make a expect.py there def safe_filename(self): name = self.safe_name(self.listnames()) num = 0 @@ -42,12 +46,12 @@ def runtest(self): target = self.obj - import pexpect source = py.code.Source(target)[1:].deindent() filename = self.safe_filename() - source.lines = ['import sys', - 'sys.path.insert(0, %s)' % repr(os.path.dirname(pypydir)) - ] + source.lines + source.lines = [ + 'import sys', + 'sys.path.insert(0, %s)' % repr(os.path.dirname(pypydir)) + ] + source.lines source.lines.append('print "%s ok!"' % filename) f = udir.join(filename) f.write(source) @@ -57,18 +61,13 @@ child.expect(re.escape(filename + " ok!")) -class ExpectClassInstance(py.test.collect.Instance): +class ExpectClassInstance(pytest.collect.Instance): Function = ExpectTestMethod -class ExpectClassCollector(py.test.collect.Class): +class ExpectClassCollector(pytest.collect.Class): Instance = ExpectClassInstance def setup(self): super(ExpectClassCollector, self).setup() - try: - import pexpect - except ImportError: - py.test.skip("pexpect not found") - - + pytest.importorskip('pexpect') From noreply at buildbot.pypy.org Mon Jan 14 09:49:00 2013 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 14 Jan 2013 09:49:00 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: Trying to implement the idea of decreasing the limit more subtly: set it Message-ID: <20130114084900.C059F1C01EE@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r60046:b0df504dfd7b Date: 2013-01-14 09:48 +0100 http://bitbucket.org/pypy/pypy/changeset/b0df504dfd7b/ Log: Trying to implement the idea of decreasing the limit more subtly: set it to 94% of the read size at the point the abort occurred. (This is what I meant, and not f3d388a6c22d from stm-logging.) Measures needed. diff --git a/pypy/translator/stm/src_stm/et.c b/pypy/translator/stm/src_stm/et.c --- a/pypy/translator/stm/src_stm/et.c +++ b/pypy/translator/stm/src_stm/et.c @@ -367,6 +367,7 @@ static void AbortTransaction(int num) { struct tx_descriptor *d = thread_descriptor; + long limit; assert(d->active); assert(!is_inevitable(d)); assert(num < ABORT_REASONS); @@ -374,6 +375,15 @@ CancelLocks(d); + /* upon abort, set the reads size limit to 94% of how much was read + so far. This should ensure that, assuming the retry does the same + thing, it will commit just before it reaches the conflicting point. */ + limit = d->list_of_read_objects.size; + if (limit > 0) { + limit -= (limit >> 4); + d->reads_size_limit_nonatomic = limit; + } + gcptrlist_clear(&d->list_of_read_objects); gcptrlist_clear(&d->gcroots); g2l_clear(&d->global_to_local); diff --git a/pypy/translator/stm/src_stm/rpyintf.c b/pypy/translator/stm/src_stm/rpyintf.c --- a/pypy/translator/stm/src_stm/rpyintf.c +++ b/pypy/translator/stm/src_stm/rpyintf.c @@ -154,12 +154,19 @@ do { v_counter = counter + 1; - /* initialize 'reads_size_limit_nonatomic' from the configured - length limit, scaled down by a factor of 2 for each time we - retry an aborted transaction. Note that as soon as such a - shortened transaction succeeds, the next one will again have - full length, for now. */ - d->reads_size_limit_nonatomic = stm_regular_length_limit >> counter; + /* If counter==0, initialize 'reads_size_limit_nonatomic' from the + configured length limit. If counter>0, we did an abort, which + has configured 'reads_size_limit_nonatomic' to a smaller value. + When such a shortened transaction succeeds, the next one will + see its length limit doubled, up to the maximum. */ + if (counter == 0) { + long limit = d->reads_size_limit_nonatomic; + if (limit != 0 && limit < (stm_regular_length_limit >> 1)) + limit = (limit << 1) | 1; + else + limit = stm_regular_length_limit; + d->reads_size_limit_nonatomic = limit; + } if (!d->atomic) BeginTransaction(&_jmpbuf); From noreply at buildbot.pypy.org Mon Jan 14 10:16:55 2013 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 14 Jan 2013 10:16:55 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: done. Message-ID: <20130114091655.4600F1C0CD4@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r60047:5e7775d581bf Date: 2013-01-14 10:00 +0100 http://bitbucket.org/pypy/pypy/changeset/5e7775d581bf/ Log: done. diff --git a/TODO b/TODO --- a/TODO +++ b/TODO @@ -40,7 +40,3 @@ JIT: finish (missing: the call in execute_token(), reorganize pypy source, ?) ------------------------------------------------------------ - -investigate abusing jitdriver to do rstm.perform_transaction() - ------------------------------------------------------------- From noreply at buildbot.pypy.org Mon Jan 14 14:24:13 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 14 Jan 2013 14:24:13 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: simplification and fixes. runner test seems to pass Message-ID: <20130114132413.77D871C0DCB@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60048:7ca517276be9 Date: 2013-01-14 15:23 +0200 http://bitbucket.org/pypy/pypy/changeset/7ca517276be9/ Log: simplification and fixes. runner test seems to pass diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py --- a/pypy/jit/backend/llsupport/llmodel.py +++ b/pypy/jit/backend/llsupport/llmodel.py @@ -78,31 +78,9 @@ return (rffi.cast(lltype.Signed, _exception_emulator) + rffi.sizeof(lltype.Signed)) - def propagate_exception(): - exc = _exception_emulator[1] - _exception_emulator[0] = 0 - _exception_emulator[1] = 0 - assert self.propagate_exception_v >= 0 - faildescr = self.get_fail_descr_from_number( - self.propagate_exception_v) - faildescr = faildescr.hide(self) - if not exc: - deadframe = self.deadframe_memoryerror - if not deadframe.jf_descr: - deadframe.jf_descr = faildescr - else: - assert deadframe.jf_descr == faildescr - else: - XXX - deadframe = lltype.malloc(jitframe.DEADFRAME, 0) - deadframe.jf_guard_exc = rffi.cast(llmemory.GCREF, exc) - deadframe.jf_descr = faildescr - return lltype.cast_opaque_ptr(llmemory.GCREF, deadframe) - self.pos_exception = pos_exception self.pos_exc_value = pos_exc_value self.insert_stack_check = lambda: (0, 0, 0) - self._propagate_exception = propagate_exception def _setup_exception_handling_translated(self): @@ -124,46 +102,9 @@ slowpathaddr = rffi.cast(lltype.Signed, f) return endaddr, lengthaddr, slowpathaddr - def propagate_exception(): - addr = llop.get_exception_addr(llmemory.Address) - addr.address[0] = llmemory.NULL - addr = llop.get_exc_value_addr(llmemory.Address) - exc = rffi.cast(llmemory.GCREF, addr.address[0]) - addr.address[0] = llmemory.NULL - assert self.propagate_exception_v >= 0 - faildescr = self.get_fail_descr_from_number( - self.propagate_exception_v) - faildescr = faildescr.hide(self) - XXX - deadframe = lltype.nullptr(jitframe.DEADFRAME) - if exc: - try: - deadframe = lltype.malloc(jitframe.DEADFRAME, 0) - deadframe.jf_guard_exc = rffi.cast(llmemory.GCREF, exc) - deadframe.jf_descr = faildescr - except MemoryError: - deadframe = lltype.nullptr(jitframe.DEADFRAME) - if not deadframe: - deadframe = self.deadframe_memoryerror - if not deadframe.jf_descr: - exc = MemoryError() - exc = cast_instance_to_base_ptr(exc) - exc = lltype.cast_opaque_ptr(llmemory.GCREF, exc) - deadframe.jf_guard_exc = exc - deadframe.jf_descr = faildescr - else: - assert deadframe.jf_descr == faildescr - return lltype.cast_opaque_ptr(llmemory.GCREF, deadframe) - self.pos_exception = pos_exception self.pos_exc_value = pos_exc_value self.insert_stack_check = insert_stack_check - self._propagate_exception = propagate_exception - - PROPAGATE_EXCEPTION = lltype.Ptr(lltype.FuncType([], llmemory.GCREF)) - - def get_propagate_exception(self): - return llhelper(self.PROPAGATE_EXCEPTION, self._propagate_exception) def grab_exc_value(self, deadframe): deadframe = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, deadframe) diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py --- a/pypy/jit/backend/test/runner_test.py +++ b/pypy/jit/backend/test/runner_test.py @@ -3364,7 +3364,7 @@ fail = self.cpu.get_latest_descr(deadframe) assert fail.identifier == excdescr.identifier exc = self.cpu.grab_exc_value(deadframe) - assert exc == "memoryerror!" + assert not exc def test_math_sqrt(self): if not self.cpu.supports_floats: diff --git a/pypy/jit/backend/x86/arch.py b/pypy/jit/backend/x86/arch.py --- a/pypy/jit/backend/x86/arch.py +++ b/pypy/jit/backend/x86/arch.py @@ -43,6 +43,7 @@ if WORD == 4: # XXX rethink the fixed size # ebp + ebx + esi + edi + 4 extra words + force_index = 9 words + XX FRAME_FIXED_SIZE = 6 SAVED_REGISTERS = 1 # range(1, 5) MY_COPY_OF_REGS = 5 # range(5, 9) @@ -50,10 +51,9 @@ JITFRAME_FIXED_SIZE = 29 # 13 GPR + 16 XMM # reg, we don't save it else: - # rbp + rbx + r12 + r13 + r14 + r15 + 11 extra words + force_index = 18 - FRAME_FIXED_SIZE = 6 - SAVED_REGISTERS = 1 # range(1, 7) - MY_COPY_OF_REGS = 7 # range(7, 18) + # rbp + rbx + r12 + r13 + r14 + r15 + 12 extra words + return address = 18 + FRAME_FIXED_SIZE = 19 + PASS_ON_MY_FRAME = 12 JITFRAME_FIXED_SIZE = 29 # 13 GPR + 16 XMM # reg, we don't save it diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -13,7 +13,8 @@ from pypy.jit.backend.x86.regalloc import (RegAlloc, get_ebp_ofs, _get_scale, gpr_reg_mgr_cls, xmm_reg_mgr_cls, _valid_addressing_size) from pypy.jit.backend.x86.arch import (FRAME_FIXED_SIZE, WORD, IS_X86_64, - JITFRAME_FIXED_SIZE, IS_X86_32) + JITFRAME_FIXED_SIZE, IS_X86_32, + PASS_ON_MY_FRAME) from pypy.jit.backend.x86.regloc import (eax, ecx, edx, ebx, esp, ebp, esi, edi, xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, r8, r9, r10, r11, r12, r13, r14, r15, X86_64_SCRATCH_REG, X86_64_XMM_SCRATCH_REG, @@ -194,6 +195,7 @@ mc.MOV_br(ofs, reg.value) # if shadow_stack: + xxx # ---- shadowstack ---- mc.SUB_ri(esp.value, 16 - WORD) # stack alignment of 16 bytes if IS_X86_32: @@ -253,10 +255,12 @@ # self.mc = codebuf.MachineCodeBlockWrapper() # - # Call the helper, which will return a dead frame object with - # the correct exception set, or MemoryError by default - addr = rffi.cast(lltype.Signed, self.cpu.get_propagate_exception()) - self.mc.CALL(imm(addr)) + # read and reset the current exception + + self._store_and_reset_exception(eax) + ofs = self.cpu.get_ofs_of_frame_field('jf_guard_exc') + self.mc.MOV_br(ofs, eax.value) + self.mc.MOV_ri(eax.value, self.cpu.propagate_exception_v) # self._call_footer() rawstart = self.mc.materialize(self.cpu.asmmemmgr, []) @@ -267,6 +271,7 @@ _, _, slowpathaddr = self.cpu.insert_stack_check() if slowpathaddr == 0 or self.cpu.propagate_exception_v < 0: return # no stack check (for tests, or non-translated) + xxx # # make a "function" that is called immediately at the start of # an assembler function. In particular, the stack looks like: @@ -750,16 +755,15 @@ # Also, make sure this is consistent with FRAME_FIXED_SIZE. # XXX should be LEA? self.mc.SUB_ri(esp.value, FRAME_FIXED_SIZE * WORD) - self.mc.MOV_sr(0, ebp.value) + self.mc.MOV_sr(PASS_ON_MY_FRAME * WORD, ebp.value) if IS_X86_64: - descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) - _, ofs, _ = unpack_arraydescr(descrs.arraydescr) + ofs = self.cpu.get_baseofs_of_frame_field() self.mc.LEA_rm(ebp.value, (edi.value, ofs)) else: xxx for i, loc in enumerate(self.cpu.CALLEE_SAVE_REGISTERS): - self.mc.MOV_sr((i + 1) * WORD, loc.value) + self.mc.MOV_sr((PASS_ON_MY_FRAME + i + 1) * WORD, loc.value) gcrootmap = self.cpu.gc_ll_descr.gcrootmap if gcrootmap and gcrootmap.is_shadow_stack: @@ -769,6 +773,7 @@ if self.stack_check_slowpath == 0: pass # no stack check (e.g. not translated) else: + xxx endaddr, lengthaddr, _ = self.cpu.insert_stack_check() self.mc.MOV(eax, heap(endaddr)) # MOV eax, [start] self.mc.SUB(eax, esp) # SUB eax, current @@ -790,9 +795,9 @@ for i in range(len(self.cpu.CALLEE_SAVE_REGISTERS)-1, -1, -1): self.mc.MOV_rs(self.cpu.CALLEE_SAVE_REGISTERS[i].value, - (i + 1) * WORD) + (i + 1 + PASS_ON_MY_FRAME) * WORD) - self.mc.MOV_rs(ebp.value, 0) + self.mc.MOV_rs(ebp.value, PASS_ON_MY_FRAME * WORD) self.mc.ADD_ri(esp.value, FRAME_FIXED_SIZE * WORD) self.mc.RET() @@ -1105,6 +1110,20 @@ unused_xmm = [xmm7, xmm6, xmm5, xmm4, xmm3, xmm2, xmm1, xmm0] on_stack = 0 + # count the stack depth + floats = 0 + for i in range(start, len(arglocs)): + arg = arglocs[i] + if arg.is_float() or argtypes and argtypes[i - start] == 'S': + floats += 1 + all_args = len(arglocs) - start + stack_depth = (max(all_args - floats - len(unused_gpr), 0) + + max(floats - len(unused_xmm), 0)) + align = 0 + if stack_depth > PASS_ON_MY_FRAME: + stack_depth = align_stack_words(stack_depth) + align = (stack_depth - PASS_ON_MY_FRAME) + self.mc.SUB_ri(esp.value, align * WORD) for i in range(start, len(arglocs)): loc = arglocs[i] if loc.is_float(): @@ -1131,10 +1150,6 @@ dst_locs.append(RawEspLoc(on_stack * WORD, INT)) on_stack += 1 - align = align_stack_words(on_stack + 1) - 1 - if align: - self.mc.SUB_ri(esp.value, align * WORD) - # Handle register arguments: first remap the xmm arguments remap_frame_layout(self, xmm_src_locs, xmm_dst_locs, X86_64_XMM_SCRATCH_REG) @@ -1663,6 +1678,9 @@ self.mc.MOV(loc1, heap(self.cpu.pos_exception())) self.mc.CMP(loc1, loc) self.implement_guard(guard_token, 'NE') + self._store_and_reset_exception(resloc) + + def _store_and_reset_exception(self, resloc=None): if resloc is not None: self.mc.MOV(resloc, heap(self.cpu.pos_exc_value())) self.mc.MOV(heap(self.cpu.pos_exception()), imm0) @@ -1856,11 +1874,8 @@ if exc: # save ebx into 'jf_guard_exc' - from pypy.jit.backend.llsupport.descr import unpack_fielddescr - descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) - offset, size, _ = unpack_fielddescr(descrs.jf_guard_exc) - _, base_offset, _ = unpack_arraydescr(descrs.arraydescr) - mc.MOV_br(offset - base_offset, ebx.value) + offset = self.cpu.get_ofs_of_frame_field('jf_guard_exc') + mc.MOV_br(offset, ebx.value) # now we return from the complete frame, which starts from # _call_header_with_stack_check(). The LEA in _call_footer below @@ -1961,17 +1976,13 @@ def _store_force_index(self, guard_op): faildescr = guard_op.getdescr() fail_index = self.cpu.get_fail_descr_number(faildescr) - descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) - base_ofs = self.cpu.unpack_arraydescr(descrs.arraydescr) - ofs = self.cpu.unpack_fielddescr(descrs.jf_force_index) - self.mc.MOV_bi(ofs - base_ofs, fail_index) + ofs = self.cpu.get_ofs_of_frame_field('jf_force_index') + self.mc.MOV_bi(ofs, fail_index) return fail_index def _emit_guard_not_forced(self, guard_token): - descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) - base_ofs = self.cpu.unpack_arraydescr(descrs.arraydescr) - ofs_fail = self.cpu.unpack_fielddescr(descrs.jf_descr) - self.mc.CMP_bi(ofs_fail - base_ofs, 0) + ofs = self.cpu.get_ofs_of_frame_field('jf_descr') + self.mc.CMP_bi(ofs, 0) self.implement_guard(guard_token, 'NE') def genop_guard_call_may_force(self, op, guard_op, guard_token, diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py --- a/pypy/jit/backend/x86/regalloc.py +++ b/pypy/jit/backend/x86/regalloc.py @@ -24,7 +24,7 @@ from pypy.jit.backend.llsupport.regalloc import FrameManager, RegisterManager,\ TempBox, compute_vars_longevity, is_comparison_or_ovf_op from pypy.jit.backend.x86.arch import WORD, JITFRAME_FIXED_SIZE -from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64, MY_COPY_OF_REGS +from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64 from pypy.jit.backend.x86 import rx86 from pypy.rlib.rarithmetic import r_longlong @@ -41,12 +41,12 @@ esi: 2, edi: 3, } - REGLOC_TO_COPY_AREA_OFS = { - ecx: MY_COPY_OF_REGS + 0 * WORD, - ebx: MY_COPY_OF_REGS + 1 * WORD, - esi: MY_COPY_OF_REGS + 2 * WORD, - edi: MY_COPY_OF_REGS + 3 * WORD, - } + #REGLOC_TO_COPY_AREA_OFS = { + # ecx: MY_COPY_OF_REGS + 0 * WORD, + # ebx: MY_COPY_OF_REGS + 1 * WORD, + # esi: MY_COPY_OF_REGS + 2 * WORD, + # edi: MY_COPY_OF_REGS + 3 * WORD, + #} def call_result_location(self, v): return eax @@ -75,19 +75,19 @@ r14: 4, r15: 5, } - REGLOC_TO_COPY_AREA_OFS = { - ecx: MY_COPY_OF_REGS + 0 * WORD, - ebx: MY_COPY_OF_REGS + 1 * WORD, - esi: MY_COPY_OF_REGS + 2 * WORD, - edi: MY_COPY_OF_REGS + 3 * WORD, - r8: MY_COPY_OF_REGS + 4 * WORD, - r9: MY_COPY_OF_REGS + 5 * WORD, - r10: MY_COPY_OF_REGS + 6 * WORD, - r12: MY_COPY_OF_REGS + 7 * WORD, - r13: MY_COPY_OF_REGS + 8 * WORD, - r14: MY_COPY_OF_REGS + 9 * WORD, - r15: MY_COPY_OF_REGS + 10 * WORD, - } + #REGLOC_TO_COPY_AREA_OFS = { + # ecx: MY_COPY_OF_REGS + 0 * WORD, + # ebx: MY_COPY_OF_REGS + 1 * WORD, + # esi: MY_COPY_OF_REGS + 2 * WORD, + # edi: MY_COPY_OF_REGS + 3 * WORD, + # r8: MY_COPY_OF_REGS + 4 * WORD, + # r9: MY_COPY_OF_REGS + 5 * WORD, + # r10: MY_COPY_OF_REGS + 6 * WORD, + # r12: MY_COPY_OF_REGS + 7 * WORD, + # r13: MY_COPY_OF_REGS + 8 * WORD, + # r14: MY_COPY_OF_REGS + 9 * WORD, + #3 r15: MY_COPY_OF_REGS + 10 * WORD, + #} class X86XMMRegisterManager(RegisterManager): diff --git a/pypy/jit/backend/x86/runner.py b/pypy/jit/backend/x86/runner.py --- a/pypy/jit/backend/x86/runner.py +++ b/pypy/jit/backend/x86/runner.py @@ -208,6 +208,18 @@ ofs = self.unpack_arraydescr(descr) self.write_float_at_mem(newframe, ofs + index, value) + @specialize.arg(1) + def get_ofs_of_frame_field(self, name): + descrs = self.gc_ll_descr.getframedescrs(self) + base_ofs = self.unpack_arraydescr(descrs.arraydescr) + ofs = self.unpack_fielddescr(getattr(descrs, name)) + return ofs - base_ofs + + def get_baseofs_of_frame_field(self): + descrs = self.gc_ll_descr.getframedescrs(self) + base_ofs = self.unpack_arraydescr(descrs.arraydescr) + return base_ofs + class CPU386(AbstractX86CPU): backend_name = 'x86' WORD = 4 From noreply at buildbot.pypy.org Mon Jan 14 14:30:35 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 14 Jan 2013 14:30:35 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: frontend fixes Message-ID: <20130114133035.0FEAE1C0DCB@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60049:940e01414f34 Date: 2013-01-14 15:30 +0200 http://bitbucket.org/pypy/pypy/changeset/940e01414f34/ Log: frontend fixes diff --git a/pypy/jit/metainterp/compile.py b/pypy/jit/metainterp/compile.py --- a/pypy/jit/metainterp/compile.py +++ b/pypy/jit/metainterp/compile.py @@ -857,10 +857,14 @@ # ____________________________________________________________ +memory_error = MemoryError() + class PropagateExceptionDescr(AbstractFailDescr): def handle_fail(self, deadframe, metainterp_sd, jitdriver_sd): cpu = metainterp_sd.cpu exception = cpu.grab_exc_value(deadframe) + if not exception: + exception = memory_error assert exception, "PropagateExceptionDescr: no exception??" raise metainterp_sd.ExitFrameWithExceptionRef(cpu, exception) diff --git a/pypy/jit/metainterp/resume.py b/pypy/jit/metainterp/resume.py --- a/pypy/jit/metainterp/resume.py +++ b/pypy/jit/metainterp/resume.py @@ -694,6 +694,7 @@ def _init(self, cpu, storage): self.cpu = cpu self.cur_numb = storage.rd_numb + self.count = storage.rd_count self.consts = storage.rd_consts def _prepare(self, storage): @@ -1243,7 +1244,7 @@ else: assert tag == TAGBOX if num < 0: - num += self.cpu.get_latest_value_count(self.deadframe) + num += self.count return self.cpu.get_int_value(self.deadframe, num) def decode_ref(self, tagged): @@ -1257,7 +1258,7 @@ else: assert tag == TAGBOX if num < 0: - num += self.cpu.get_latest_value_count(self.deadframe) + num += self.count return self.cpu.get_ref_value(self.deadframe, num) def decode_float(self, tagged): @@ -1267,7 +1268,7 @@ else: assert tag == TAGBOX if num < 0: - num += self.cpu.get_latest_value_count(self.deadframe) + num += self.count return self.cpu.get_float_value(self.deadframe, num) def write_an_int(self, index, int): From noreply at buildbot.pypy.org Mon Jan 14 15:04:21 2013 From: noreply at buildbot.pypy.org (stepahn) Date: Mon, 14 Jan 2013 15:04:21 +0100 (CET) Subject: [pypy-commit] lang-js default: make W_Root settled Message-ID: <20130114140421.4642F1C0E00@cobra.cs.uni-duesseldorf.de> Author: Stephan Branch: Changeset: r338:b62ede861110 Date: 2013-01-09 10:34 +0100 http://bitbucket.org/pypy/lang-js/changeset/b62ede861110/ Log: make W_Root settled diff --git a/js/builtins/array.py b/js/builtins/array.py --- a/js/builtins/array.py +++ b/js/builtins/array.py @@ -64,6 +64,8 @@ array = this.ToObject() func = array.get(u'join') if func.is_callable(): + from js.jsobj import W_BasicFunction + assert isinstance(func, W_BasicFunction) return func.Call(this=this).to_string() else: return this.to_string() @@ -219,6 +221,8 @@ from js.execution import JsTypeError raise JsTypeError(u'') + from js.jsobj import W_BasicFunction + assert isinstance(comparefn, W_BasicFunction) res = comparefn.Call(args=[x, y], this=newundefined()) return res.ToInteger() diff --git a/js/jsobj.py b/js/jsobj.py --- a/js/jsobj.py +++ b/js/jsobj.py @@ -33,6 +33,7 @@ class W_Root(object): + _settled_ = True _immutable_fields_ = ['_type_'] _type_ = '' From noreply at buildbot.pypy.org Mon Jan 14 15:04:22 2013 From: noreply at buildbot.pypy.org (stepahn) Date: Mon, 14 Jan 2013 15:04:22 +0100 (CET) Subject: [pypy-commit] lang-js default: deleted unused opcode code Message-ID: <20130114140422.722501C11E8@cobra.cs.uni-duesseldorf.de> Author: Stephan Branch: Changeset: r339:91cc8d6ca943 Date: 2013-01-09 10:49 +0100 http://bitbucket.org/pypy/lang-js/changeset/91cc8d6ca943/ Log: deleted unused opcode code diff --git a/js/opcodes.py b/js/opcodes.py --- a/js/opcodes.py +++ b/js/opcodes.py @@ -66,12 +66,6 @@ _stack_change = 0 -class Undefined(Opcode): - def eval(self, ctx): - from js.object_space import newundefined - ctx.stack_append(newundefined()) - - class LOAD_INTCONSTANT(Opcode): _immutable_fields_ = ['w_intvalue'] @@ -614,65 +608,6 @@ return 'JUMP_IF_TRUE_NOPOP %d' % (self.where) -class DECLARE_FUNCTION(Opcode): - _stack_change = 0 - - def __init__(self, funcobj): - self.funcobj = funcobj - - def eval(self, ctx): - pass - #from js.jsobj import DONT_ENUM, READ_ONLY - ## 13.2 Creating Function Objects - ## TODO move this to W__Function.__init__ ? - - #func = W__Function(ctx, self.funcobj) - - #func.Put('length', W_IntNumber(len(self.funcobj.params())), flags = DONT_ENUM | READ_ONLY) - - #proto = W__Object() - #proto.Put('constructor', func, flags = DONT_ENUM) - - #func.Put('prototype', proto, flags = DONT_ENUM) - - #ctx.stack_append(funcobj) - - #if self.funcobj.name is not None: - #ctx.set_value(self.funcobj.name, func) - - def __str__(self): - funcobj = self.funcobj - if funcobj.name is None: - name = "" - else: - name = funcobj.name + " " - return 'DECLARE_FUNCTION %s%r' % (name, funcobj.params) - - ##def __repr__(self): - ## funcobj = self.funcobj - ## if funcobj.name is None: - ## name = "" - ## else: - ## name = funcobj.name + " " - ## codestr = '\n'.join([' %r' % (op,) for op in funcobj.opcodes]) - ## ##return 'DECLARE_FUNCTION %s%r [\n%s\n]' % (name, funcobj.params, codestr) - ## return 'DECLARE_FUNCTION %s%r' % (name, funcobj.params) - - -class DECLARE_VAR(Opcode): - _stack_change = 0 - - def __init__(self, name): - self.name = name - - def eval(self, ctx): - pass - #ctx.declare_variable(self.name) - - #def __repr__(self): - #return 'DECLARE_VAR "%s"' % (self.name,) - - class RETURN(Opcode): _stack_change = 0 @@ -970,34 +905,6 @@ ctx.stack_append(_w(res)) -class LOAD_LOCAL(Opcode): - _immutable_fields_ = ['local'] - - def __init__(self, local): - self.local = local - - def eval(self, ctx): - ctx.stack_append(ctx.get_local_value(self.local)) - - #def __repr__(self): - #return 'LOAD_LOCAL %d' % (self.local,) - - -class STORE_LOCAL(Opcode): - _stack_change = 0 - _immutable_fields_ = ['local'] - - def __init__(self, local): - self.local = local - - def eval(self, ctx): - value = ctx.stack_top() - ctx.assign_local(self.local, value) - - #def __repr__(self): - #return 'STORE_LOCAL %d' % (self.local,) - - class INSTANCEOF(Opcode): def eval(self, ctx): rval = ctx.stack_pop() diff --git a/js/operations.py b/js/operations.py --- a/js/operations.py +++ b/js/operations.py @@ -359,7 +359,6 @@ index = self.index - #bytecode.emit('DECLARE_FUNCTION', jsfunc) bytecode.emit('LOAD_FUNCTION', jsfunc) if index is not None: bytecode.emit('STORE', index, name) From noreply at buildbot.pypy.org Mon Jan 14 15:04:23 2013 From: noreply at buildbot.pypy.org (stepahn) Date: Mon, 14 Jan 2013 15:04:23 +0100 (CET) Subject: [pypy-commit] lang-js default: marked immutable fields Message-ID: <20130114140423.93D2C1C0E00@cobra.cs.uni-duesseldorf.de> Author: Stephan Branch: Changeset: r340:22e52dea7899 Date: 2013-01-09 11:00 +0100 http://bitbucket.org/pypy/lang-js/changeset/22e52dea7899/ Log: marked immutable fields diff --git a/js/opcodes.py b/js/opcodes.py --- a/js/opcodes.py +++ b/js/opcodes.py @@ -8,6 +8,8 @@ class Opcode(object): + _settled_ = True + _immutable_fields_ = ['_stack_change'] _stack_change = 1 def __init__(self): @@ -81,6 +83,8 @@ class LOAD_BOOLCONSTANT(Opcode): + _immutable_fields_ = ['w_boolval'] + def __init__(self, value): from js.object_space import newbool self.w_boolval = newbool(value) @@ -95,6 +99,8 @@ class LOAD_FLOATCONSTANT(Opcode): + _immutable_fields_ = ['w_floatvalue'] + def __init__(self, value): from js.object_space import newfloat self.w_floatvalue = newfloat(float(value)) @@ -107,7 +113,7 @@ class LOAD_STRINGCONSTANT(Opcode): - _immutable_fields_ = ['strval'] + _immutable_fields_ = ['w_strval'] def __init__(self, value): from js.object_space import newstring @@ -160,6 +166,8 @@ class LOAD_ARRAY(Opcode): + _immutable_fields_ = ['counter'] + def __init__(self, counter): self.counter = counter @@ -180,7 +188,8 @@ class LOAD_LIST(Opcode): - #_immutable_fields_ = ['counter'] + _immutable_fields_ = ['counter'] + def __init__(self, counter): self.counter = counter @@ -197,6 +206,8 @@ class LOAD_FUNCTION(Opcode): + #_immutable_fields_ = ['funcobj'] + def __init__(self, funcobj): self.funcobj = funcobj @@ -288,6 +299,8 @@ class TYPEOF_VARIABLE(Opcode): + _immutable_fields_ = ['index', 'name'] + def __init__(self, index, name): self.index = index self.name = name @@ -517,6 +530,7 @@ class LABEL(Opcode): _stack_change = 0 + _immutable_fields_ = ['num'] def __init__(self, num): self.num = num @@ -676,6 +690,8 @@ class TRYCATCHBLOCK(Opcode): + _immutable_fields_ = ['tryexec', 'catchexec', 'catchparam', 'finallyexec'] + def __init__(self, tryfunc, catchparam, catchfunc, finallyfunc): self.tryexec = tryfunc self.catchexec = catchfunc @@ -843,6 +859,8 @@ class WITH(Opcode): + _immutable_fields_ = ['body'] + def __init__(self, body): self.body = body @@ -865,6 +883,8 @@ class DELETE(Opcode): + _immutable_fields_ = ['name', 'index'] + def __init__(self, name, index): self.name = name self.index = index From noreply at buildbot.pypy.org Mon Jan 14 15:04:24 2013 From: noreply at buildbot.pypy.org (stepahn) Date: Mon, 14 Jan 2013 15:04:24 +0100 (CET) Subject: [pypy-commit] lang-js default: cleaned up commoncall and commonnew Message-ID: <20130114140424.AC27A1C0E00@cobra.cs.uni-duesseldorf.de> Author: Stephan Branch: Changeset: r341:301dfd045fd3 Date: 2013-01-09 11:09 +0100 http://bitbucket.org/pypy/lang-js/changeset/301dfd045fd3/ Log: cleaned up commoncall and commonnew diff --git a/js/opcodes.py b/js/opcodes.py --- a/js/opcodes.py +++ b/js/opcodes.py @@ -638,16 +638,17 @@ ctx.stack_pop() -def common_call(ctx, r1, args, this, name): - from js.jsobj import W_BasicFunction, W_List +def common_call(ctx, funcobj, args, this, identifyer): + if not funcobj.is_callable(): + err = u"%s is not a callable (%s)" % (funcobj.to_string(), identifyer.to_string()) + raise JsTypeError(err) + + from js.jsobj import W_List, W_BasicFunction assert isinstance(args, W_List) - # TODO - if not (isinstance(r1, W_BasicFunction)): - #err = (u"%s is not a callable (%s)"%(r1.to_string(), name.to_string())) - err = u"is not a callable (%s)" % (r1.to_string()) - raise JsTypeError(err) + assert isinstance(funcobj, W_BasicFunction) + argv = args.to_list() - res = r1.Call(args=argv, this=this, calling_context=ctx) + res = funcobj.Call(args=argv, this=this, calling_context=ctx) return res @@ -758,10 +759,12 @@ def commonnew(ctx, obj, args): + if not obj.is_callable(): + msg = u'%s is not a constructor' % (obj.to_string()) + raise JsTypeError(msg) + from js.jsobj import W_BasicFunction - - if not isinstance(obj, W_BasicFunction): - raise JsTypeError(u'not a constructor') + assert isinstance(obj, W_BasicFunction) res = obj.Construct(args=args) return res From noreply at buildbot.pypy.org Mon Jan 14 17:34:14 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 14 Jan 2013 17:34:14 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: make execute_token return a frame Message-ID: <20130114163414.E34B21C0246@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60050:71321730f1c8 Date: 2013-01-14 16:54 +0200 http://bitbucket.org/pypy/pypy/changeset/71321730f1c8/ Log: make execute_token return a frame diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py --- a/pypy/jit/backend/llsupport/gc.py +++ b/pypy/jit/backend/llsupport/gc.py @@ -100,6 +100,8 @@ p = v.value rgc._make_sure_does_not_move(p) gcrefs_output_list.append(p) + if op.is_guard() or op.getopnum() == rop.FINISH: + rgc._make_sure_does_not_move(op.getdescr()) def rewrite_assembler(self, cpu, operations, gcrefs_output_list): rewriter = GcRewriterAssembler(self, cpu) diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py --- a/pypy/jit/backend/test/runner_test.py +++ b/pypy/jit/backend/test/runner_test.py @@ -430,6 +430,7 @@ def test_int_operations(self): from pypy.jit.metainterp.test.test_executor import get_int_tests for opnum, boxargs, retvalue in get_int_tests(): + print "X" res = self.execute_operation(opnum, boxargs, 'int') assert res.value == retvalue diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -4,10 +4,9 @@ from pypy.jit.metainterp.history import Const, Box, BoxInt, ConstInt from pypy.jit.metainterp.history import AbstractFailDescr, INT, REF, FLOAT from pypy.jit.metainterp.history import JitCellToken -from pypy.jit.backend.llsupport.descr import unpack_arraydescr from pypy.rpython.lltypesystem import lltype, rffi, rstr, llmemory from pypy.rpython.lltypesystem.lloperation import llop -from pypy.rpython.annlowlevel import llhelper +from pypy.rpython.annlowlevel import llhelper, cast_instance_to_gcref from pypy.rlib.jit import AsmInfo from pypy.jit.backend.model import CompiledLoopToken from pypy.jit.backend.x86.regalloc import (RegAlloc, get_ebp_ofs, _get_scale, @@ -569,7 +568,6 @@ operations = regalloc.prepare_bridge(inputargs, arglocs, operations, self.current_clt.allgcrefs) - frame_depth = self._assemble(regalloc, operations) codeendpos = self.mc.get_relative_pos() self.write_pending_failure_recoveries() @@ -1793,7 +1791,6 @@ """Generate the initial code for handling a failure. We try to keep it as compact as possible. """ - fail_index = self.cpu.get_fail_descr_number(guardtok.faildescr) mc = self.mc startpos = mc.get_relative_pos() withfloats = False @@ -1803,12 +1800,14 @@ break exc = guardtok.exc target = self.failure_recovery_code[exc + 2 * withfloats] + fail_descr = cast_instance_to_gcref(guardtok.faildescr) + fail_descr = rffi.cast(lltype.Signed, fail_descr) if WORD == 4: - mc.PUSH(imm(fail_index)) + mc.PUSH(imm(fail_descr)) mc.JMP(imm(target)) else: mc.MOV_ri64(X86_64_SCRATCH_REG.value, target) - mc.PUSH(imm(fail_index)) + mc.PUSH(imm(fail_descr)) mc.JMP_r(X86_64_SCRATCH_REG.value) # write tight data that describes the failure recovery positions = [0] * len(guardtok.fail_locs) @@ -1881,7 +1880,11 @@ # _call_header_with_stack_check(). The LEA in _call_footer below # throws away most of the frame, including all the PUSHes that we # did just above. + ofs = self.cpu.get_ofs_of_frame_field('jf_descr') + base_ofs = self.cpu.get_baseofs_of_frame_field() mc.POP(eax) + mc.MOV_br(ofs, eax.value) + mc.LEA_rb(eax.value, -base_ofs) self._call_footer() rawstart = mc.materialize(self.cpu.asmmemmgr, []) @@ -1890,16 +1893,18 @@ def genop_finish(self, op, arglocs, result_loc): if len(arglocs) == 2: - [return_val, argloc] = arglocs + [return_val, fail_descr_loc] = arglocs if op.getarg(0).type == FLOAT and not IS_X86_64: size = WORD * 2 else: size = WORD self.save_into_mem(raw_stack(0), return_val, imm(size)) else: - [argloc] = arglocs - if argloc is not eax: - self.mov(argloc, eax) + [fail_descr_loc] = arglocs + ofs = self.cpu.get_ofs_of_frame_field('jf_descr') + base_ofs = self.cpu.get_baseofs_of_frame_field() + self.mov(fail_descr_loc, RawStackLoc(ofs)) + self.mc.LEA_rb(eax.value, -base_ofs) # exit function self._call_footer() diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py --- a/pypy/jit/backend/x86/regalloc.py +++ b/pypy/jit/backend/x86/regalloc.py @@ -9,6 +9,7 @@ TargetToken, JitCellToken) from pypy.jit.backend.x86.regloc import * from pypy.rpython.lltypesystem import lltype, rffi, rstr +from pypy.rpython.annlowlevel import cast_instance_to_gcref from pypy.rlib.objectmodel import we_are_translated from pypy.rlib import rgc from pypy.jit.backend.llsupport import symbolic @@ -201,7 +202,8 @@ return operations def prepare_bridge(self, inputargs, arglocs, operations, allgcrefs): - operations = self._prepare(inputargs, operations, allgcrefs) + operations = self._prepare(inputargs, operations, + allgcrefs) self._update_bindings(arglocs, inputargs) self.min_bytes_before_label = 0 return operations @@ -498,12 +500,12 @@ # the frame is in ebp, but we have to point where in the frame is # the potential argument to FINISH descr = op.getdescr() - fail_no = self.assembler.cpu.get_fail_descr_number(descr) + fail_descr = cast_instance_to_gcref(descr) if op.numargs() == 1: loc = self.make_sure_var_in_reg(op.getarg(0)) - locs = [loc, imm(fail_no)] + locs = [loc, imm(fail_descr)] else: - locs = [imm(fail_no)] + locs = [imm(fail_descr)] self.Perform(op, locs, None) if op.numargs() == 1: self.possibly_free_var(op.getarg(0)) diff --git a/pypy/jit/backend/x86/regloc.py b/pypy/jit/backend/x86/regloc.py --- a/pypy/jit/backend/x86/regloc.py +++ b/pypy/jit/backend/x86/regloc.py @@ -5,7 +5,7 @@ from pypy.tool.sourcetools import func_with_new_name from pypy.rlib.objectmodel import specialize, instantiate from pypy.rlib.rarithmetic import intmask -from pypy.jit.metainterp.history import FLOAT +from pypy.jit.metainterp.history import FLOAT, INT from pypy.jit.codewriter import longlong # @@ -48,7 +48,7 @@ _immutable_ = True _location_code = 'b' - def __init__(self, value, type): + def __init__(self, value, type=INT): self.value = value self.type = type diff --git a/pypy/jit/backend/x86/runner.py b/pypy/jit/backend/x86/runner.py --- a/pypy/jit/backend/x86/runner.py +++ b/pypy/jit/backend/x86/runner.py @@ -105,7 +105,7 @@ def make_execute_token(self, *ARGS): FUNCPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF], - lltype.Signed)) + llmemory.GCREF)) # def execute_token(executable_token, *args): clt = executable_token.compiled_loop_token @@ -136,14 +136,12 @@ else: self.set_ref_value(frame, num, arg) num += WORD - descr_no = func(ll_frame) + ll_frame = func(ll_frame) finally: if not self.translate_support_code: LLInterpreter.current_interpreter = prev_interpreter #llop.debug_print(lltype.Void, "<<<< Back") - descr = self.get_fail_descr_from_number(descr_no) - frame.jf_descr = cast_instance_to_gcref(descr) - return frame + return ll_frame return execute_token def cast_ptr_to_int(x): From noreply at buildbot.pypy.org Mon Jan 14 17:34:16 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 14 Jan 2013 17:34:16 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: kill the descr indexes Message-ID: <20130114163416.54F2C1C11E8@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60051:279c4bd34be3 Date: 2013-01-14 18:33 +0200 http://bitbucket.org/pypy/pypy/changeset/279c4bd34be3/ Log: kill the descr indexes diff --git a/pypy/jit/backend/llgraph/runner.py b/pypy/jit/backend/llgraph/runner.py --- a/pypy/jit/backend/llgraph/runner.py +++ b/pypy/jit/backend/llgraph/runner.py @@ -862,23 +862,22 @@ fielddescr = jd.vable_token_descr self.cpu.bh_setfield_gc(vable, 0, fielddescr) faildescr = self.cpu.get_latest_descr(pframe) - failindex = self.cpu.get_fail_descr_number(faildescr) - if failindex == self.cpu.done_with_this_frame_int_v: + if faildescr == self.cpu.done_with_this_frame_descr_int: reset_vable(jd, vable) return self.cpu.get_int_value(pframe, 0) - if failindex == self.cpu.done_with_this_frame_ref_v: + if faildescr == self.cpu.done_with_this_frame_descr_ref: reset_vable(jd, vable) return self.cpu.get_ref_value(pframe, 0) - if failindex == self.cpu.done_with_this_frame_float_v: + if faildescr == self.cpu.done_with_this_frame_descr_float: reset_vable(jd, vable) return self.cpu.get_float_value(pframe, 0) - if failindex == self.cpu.done_with_this_frame_void_v: + if faildescr == self.cpu.done_with_this_frame_descr_void: reset_vable(jd, vable) return None # assembler_helper_ptr = jd.assembler_helper_adr.ptr # fish try: - result = assembler_helper_ptr(failindex, pframe, vable) + result = assembler_helper_ptr(pframe, vable) except LLException, lle: assert self.last_exception is None, "exception left behind" self.last_exception = lle diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py --- a/pypy/jit/backend/llsupport/gc.py +++ b/pypy/jit/backend/llsupport/gc.py @@ -102,6 +102,7 @@ gcrefs_output_list.append(p) if op.is_guard() or op.getopnum() == rop.FINISH: rgc._make_sure_does_not_move(op.getdescr()) + gcrefs_output_list.append(op.getdescr()) def rewrite_assembler(self, cpu, operations, gcrefs_output_list): rewriter = GcRewriterAssembler(self, cpu) @@ -119,8 +120,8 @@ descrs.jf_descr = cpu.fielddescrof(jitframe.JITFRAME, 'jf_descr') descrs.jf_guard_exc = cpu.fielddescrof(jitframe.JITFRAME, 'jf_guard_exc') - descrs.jf_force_index = cpu.fielddescrof(jitframe.JITFRAME, - 'jf_force_index') + descrs.jf_force_descr = cpu.fielddescrof(jitframe.JITFRAME, + 'jf_force_descr') descrs.jf_frame_info = cpu.fielddescrof(jitframe.JITFRAME, 'jf_frame_info') descrs.jfi_frame_depth = cpu.fielddescrof(jitframe.JITFRAMEINFO, diff --git a/pypy/jit/backend/llsupport/jitframe.py b/pypy/jit/backend/llsupport/jitframe.py --- a/pypy/jit/backend/llsupport/jitframe.py +++ b/pypy/jit/backend/llsupport/jitframe.py @@ -24,8 +24,8 @@ # This field is also set immediately before doing CALL_MAY_FORCE # or CALL_ASSEMBLER. ('jf_descr', llmemory.GCREF), - # index of guard_not_force - ('jf_force_index', lltype.Signed), + # guard_not_forced descr + ('jf_force_descr', llmemory.GCREF), # For the front-end: a GCREF for the savedata ('jf_savedata', llmemory.GCREF), # For GUARD_(NO)_EXCEPTION and GUARD_NOT_FORCED: the exception we diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py --- a/pypy/jit/backend/llsupport/llmodel.py +++ b/pypy/jit/backend/llsupport/llmodel.py @@ -205,11 +205,6 @@ descr = deadframe.jf_descr return history.AbstractDescr.show(self, descr) - def store_fail_descr(self, deadframe, failindex): - faildescr = self.get_fail_descr_from_number(failindex) - frame = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, deadframe) - frame.jf_descr = cast_instance_to_gcref(faildescr) - def _decode_pos(self, deadframe, index): descr = self.get_latest_descr(deadframe) if descr.final_descr: diff --git a/pypy/jit/backend/model.py b/pypy/jit/backend/model.py --- a/pypy/jit/backend/model.py +++ b/pypy/jit/backend/model.py @@ -1,6 +1,5 @@ from pypy.rlib.debug import debug_start, debug_print, debug_stop -from pypy.rlib.objectmodel import we_are_translated -from pypy.jit.metainterp import history +from pypy.jit.metainterp import compile from pypy.rpython.lltypesystem import lltype @@ -12,52 +11,20 @@ # Boxes and Consts are BoxFloats and ConstFloats. supports_singlefloats = False - done_with_this_frame_void_v = -1 - done_with_this_frame_int_v = -1 - done_with_this_frame_ref_v = -1 - done_with_this_frame_float_v = -1 - propagate_exception_v = -1 total_compiled_loops = 0 total_compiled_bridges = 0 total_freed_loops = 0 total_freed_bridges = 0 + propagate_exception_descr = None + # for heaptracker # _all_size_descrs_with_vtable = None _vtable_to_descr_dict = None def __init__(self): - self.fail_descr_list = [] - self.fail_descr_free_list = [] - - def reserve_some_free_fail_descr_number(self): - lst = self.fail_descr_list - if len(self.fail_descr_free_list) > 0: - n = self.fail_descr_free_list.pop() - assert lst[n] is None - else: - n = len(lst) - lst.append(None) - return n - - def get_fail_descr_number(self, descr): - assert isinstance(descr, history.AbstractFailDescr) - if not we_are_translated(): - if not hasattr(descr, '_cpu'): - assert descr.index == -1, "descr.index is already >= 0??" - descr._cpu = self - assert descr._cpu is self,"another CPU has already seen the descr!" - # - n = descr.index - if n < 0: - n = self.reserve_some_free_fail_descr_number() - self.fail_descr_list[n] = descr - descr.index = n - return n - - def get_fail_descr_from_number(self, n): - return self.fail_descr_list[n] + self.__dict__.update(compile.make_done_loop_tokens()) def setup_once(self): """Called once by the front-end when the program starts.""" @@ -196,19 +163,7 @@ guarantees that at the point of the call to free_code_group(), none of the corresponding assembler is currently running. """ - # The base class provides a limited implementation: freeing the - # resume descrs. This is already quite helpful, because the - # resume descrs are the largest consumers of memory (about 3x - # more than the assembler, in the case of the x86 backend). - lst = self.fail_descr_list - # We expect 'compiled_loop_token' to be itself garbage-collected soon, - # but better safe than sorry: be ready to handle several calls to - # free_loop_and_bridges() for the same compiled_loop_token. - faildescr_indices = compiled_loop_token.faildescr_indices - compiled_loop_token.faildescr_indices = [] - for n in faildescr_indices: - lst[n] = None - self.fail_descr_free_list.extend(faildescr_indices) + pass def sizeof(self, S): raise NotImplementedError diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py --- a/pypy/jit/backend/test/runner_test.py +++ b/pypy/jit/backend/test/runner_test.py @@ -2758,13 +2758,12 @@ def test_assembler_call(self): called = [] - def assembler_helper(failindex, deadframe, virtualizable): - self.cpu.store_fail_descr(deadframe, failindex) + def assembler_helper(deadframe, virtualizable): assert self.cpu.get_int_value(deadframe, 0) == 97 called.append(self.cpu.get_latest_descr(deadframe)) return 4 + 9 - FUNCPTR = lltype.Ptr(lltype.FuncType([lltype.Signed, llmemory.GCREF, + FUNCPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF, llmemory.GCREF], lltype.Signed)) class FakeJitDriverSD: @@ -2773,9 +2772,6 @@ assembler_helper_adr = llmemory.cast_ptr_to_adr( _assembler_helper_ptr) - # ensure the fail_descr_number is not zero - for _ in range(10): - self.cpu.reserve_some_free_fail_descr_number() ops = ''' [i0, i1, i2, i3, i4, i5, i6, i7, i8, i9] i10 = int_add(i0, i1) @@ -2818,8 +2814,8 @@ # test the fast path, which should not call assembler_helper() del called[:] - self.cpu.done_with_this_frame_int_v = self.cpu.get_fail_descr_number( - finish_descr) + prev_descr = self.cpu.done_with_this_frame_descr_int + self.cpu.done_with_this_frame_descr_int = finish_descr try: othertoken = JitCellToken() self.cpu.compile_loop(loop.inputargs, loop.operations, othertoken) @@ -2828,21 +2824,20 @@ assert self.cpu.get_int_value(deadframe, 0) == 97 assert not called finally: - del self.cpu.done_with_this_frame_int_v + self.cpu.done_with_this_frame_int_v = prev_descr def test_assembler_call_float(self): if not self.cpu.supports_floats: py.test.skip("requires floats") called = [] - def assembler_helper(failindex, deadframe, virtualizable): - self.cpu.store_fail_descr(deadframe, failindex) + def assembler_helper(deadframe, virtualizable): x = self.cpu.get_float_value(deadframe, 0) assert longlong.getrealfloat(x) == 1.2 + 3.2 called.append(self.cpu.get_latest_descr(deadframe)) print '!' * 30 + 'assembler_helper' return 13.5 - FUNCPTR = lltype.Ptr(lltype.FuncType([lltype.Signed, llmemory.GCREF, + FUNCPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF, llmemory.GCREF], lltype.Float)) class FakeJitDriverSD: @@ -2856,8 +2851,6 @@ FakeJitDriverSD.portal_calldescr = self.cpu.calldescrof( lltype.Ptr(lltype.FuncType(ARGS, RES)), ARGS, RES, EffectInfo.MOST_GENERAL) - for _ in range(10): - self.cpu.reserve_some_free_fail_descr_number() ops = ''' [f0, f1] f2 = float_add(f0, f1) @@ -2890,8 +2883,8 @@ # test the fast path, which should not call assembler_helper() del called[:] - self.cpu.done_with_this_frame_float_v = self.cpu.get_fail_descr_number( - finish_descr) + prev_descr = self.cpu.done_with_this_frame_descr_float + self.cpu.done_with_this_frame_descr_float = finish_descr try: othertoken = JitCellToken() self.cpu.compile_loop(loop.inputargs, loop.operations, othertoken) @@ -2902,7 +2895,7 @@ assert longlong.getrealfloat(x) == 1.2 + 4.2 assert not called finally: - del self.cpu.done_with_this_frame_float_v + self.cpu.done_with_this_frame_descr_float = prev_descr def test_raw_malloced_getarrayitem(self): ARRAY = rffi.CArray(lltype.Signed) @@ -2932,15 +2925,13 @@ if not self.cpu.supports_floats: py.test.skip("requires floats") called = [] - def assembler_helper(failindex, deadframe, virtualizable): - self.cpu.store_fail_descr(deadframe, failindex) + def assembler_helper(deadframe, virtualizable): x = self.cpu.get_float_value(deadframe, 0) assert longlong.getrealfloat(x) == 1.25 + 3.25 called.append(self.cpu.get_latest_descr(deadframe)) return 13.5 - FUNCPTR = lltype.Ptr(lltype.FuncType([lltype.Signed, - llmemory.GCREF, llmemory.GCREF], + FUNCPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF, llmemory.GCREF], lltype.Float)) class FakeJitDriverSD: index_of_virtualizable = -1 @@ -2953,8 +2944,6 @@ FakeJitDriverSD.portal_calldescr = self.cpu.calldescrof( lltype.Ptr(lltype.FuncType(ARGS, RES)), ARGS, RES, EffectInfo.MOST_GENERAL) - for _ in range(10): - self.cpu.reserve_some_free_fail_descr_number() ops = ''' [f0, f1] f2 = float_add(f0, f1) @@ -3347,8 +3336,7 @@ def test_memoryerror(self): excdescr = BasicFailDescr(666) - self.cpu.propagate_exception_v = self.cpu.get_fail_descr_number( - excdescr) + self.cpu.propagate_exception_descr = excdescr self.cpu.setup_once() # xxx redo it, because we added # propagate_exception_v i0 = BoxInt() diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -249,7 +249,7 @@ self.malloc_slowpath2 = rawstart def _build_propagate_exception_path(self): - if self.cpu.propagate_exception_v < 0: + if not self.cpu.propagate_exception_descr: return # not supported (for tests, or non-translated) # self.mc = codebuf.MachineCodeBlockWrapper() @@ -259,7 +259,12 @@ self._store_and_reset_exception(eax) ofs = self.cpu.get_ofs_of_frame_field('jf_guard_exc') self.mc.MOV_br(ofs, eax.value) - self.mc.MOV_ri(eax.value, self.cpu.propagate_exception_v) + propagate_exception_descr = rffi.cast(lltype.Signed, + cast_instance_to_gcref(self.cpu.propagate_exception_descr)) + ofs = self.cpu.get_ofs_of_frame_field('jf_descr') + base_ofs = self.cpu.get_baseofs_of_frame_field() + self.mc.MOV_bi(ofs, propagate_exception_descr) + self.mc.LEA_rb(eax.value, -base_ofs) # self._call_footer() rawstart = self.mc.materialize(self.cpu.asmmemmgr, []) @@ -268,7 +273,7 @@ def _build_stack_check_slowpath(self): _, _, slowpathaddr = self.cpu.insert_stack_check() - if slowpathaddr == 0 or self.cpu.propagate_exception_v < 0: + if slowpathaddr == 0 or not self.cpu.propagate_exception_descr: return # no stack check (for tests, or non-translated) xxx # @@ -554,15 +559,13 @@ # Arguments should be unique assert len(set(inputargs)) == len(inputargs) - descr_number = self.cpu.get_fail_descr_number(faildescr) - self.setup(original_loop_token) + descr_number = compute_unique_id(faildescr) if log: operations = self._inject_debugging_code(faildescr, operations, 'b', descr_number) - descr = self.cpu.get_fail_descr_from_number(descr_number) - arglocs = self.rebuild_faillocs_from_descr(descr, inputargs) + arglocs = self.rebuild_faillocs_from_descr(faildescr, inputargs) regalloc = RegAlloc(self, self.cpu.translate_support_code) startpos = self.mc.get_relative_pos() operations = regalloc.prepare_bridge(inputargs, arglocs, @@ -575,7 +578,7 @@ # rawstart = self.materialize_loop(original_loop_token) debug_start("jit-backend-addr") - debug_print("bridge out of Guard %d has address %x to %x" % + debug_print("bridge out of Guard %x has address %x to %x" % (descr_number, rawstart, rawstart + codeendpos)) debug_stop("jit-backend-addr") self.patch_pending_failure_recoveries(rawstart) @@ -1980,10 +1983,9 @@ def _store_force_index(self, guard_op): faildescr = guard_op.getdescr() - fail_index = self.cpu.get_fail_descr_number(faildescr) - ofs = self.cpu.get_ofs_of_frame_field('jf_force_index') - self.mc.MOV_bi(ofs, fail_index) - return fail_index + ofs = self.cpu.get_ofs_of_frame_field('jf_force_descr') + self.mc.MOV_bi(ofs, rffi.cast(lltype.Signed, + cast_instance_to_gcref(faildescr))) def _emit_guard_not_forced(self, guard_token): ofs = self.cpu.get_ofs_of_frame_field('jf_descr') @@ -1992,8 +1994,8 @@ def genop_guard_call_may_force(self, op, guard_op, guard_token, arglocs, result_loc): - fail_index = self._store_force_index(guard_op) - self._genop_call(op, arglocs, result_loc, fail_index) + self._store_force_index(guard_op) + self._genop_call(op, arglocs, result_loc, -1) self._emit_guard_not_forced(guard_token) def genop_guard_call_release_gil(self, op, guard_op, guard_token, @@ -2126,21 +2128,24 @@ [argloc], 0, tmp=eax) if op.result is None: assert result_loc is None - value = self.cpu.done_with_this_frame_void_v + value = self.cpu.done_with_this_frame_descr_void else: kind = op.result.type if kind == INT: assert result_loc is eax - value = self.cpu.done_with_this_frame_int_v + value = self.cpu.done_with_this_frame_descr_int elif kind == REF: assert result_loc is eax - value = self.cpu.done_with_this_frame_ref_v + value = self.cpu.done_with_this_frame_descr_ref elif kind == FLOAT: - value = self.cpu.done_with_this_frame_float_v + value = self.cpu.done_with_this_frame_descr_float else: raise AssertionError(kind) - self.mc.CMP_ri(eax.value, value) + value = rffi.cast(lltype.Signed, cast_instance_to_gcref(value)) + base_ofs = self.cpu.get_baseofs_of_frame_field() + ofs = self.cpu.get_ofs_of_frame_field('jf_descr') + self.mc.CMP_mi((eax.value, base_ofs + ofs), value) # patched later self.mc.J_il8(rx86.Conditions['E'], 0) # goto B if we get 'done_with_this_frame' je_location = self.mc.get_relative_pos() @@ -2150,7 +2155,7 @@ assert jd is not None asm_helper_adr = self.cpu.cast_adr_to_int(jd.assembler_helper_adr) self._emit_call(fail_index, imm(asm_helper_adr), - [eax, frame_loc, imm0], 0, tmp=ecx) + [eax, imm0], 0, tmp=ecx) if IS_X86_32 and isinstance(result_loc, StackLoc) and result_loc.type == FLOAT: self.mc.FSTPL_b(result_loc.value) #else: result_loc is already either eax or None, checked below @@ -2175,8 +2180,6 @@ # if op.result is not None: # load the return value from the dead frame's value index 0 - assert isinstance(frame_loc, StackLoc) - self.mc.MOV_rb(eax.value, frame_loc.value) kind = op.result.type if kind == FLOAT: _, descr = self.cpu.getarraydescr_for_frame(kind, 0) diff --git a/pypy/jit/backend/x86/runner.py b/pypy/jit/backend/x86/runner.py --- a/pypy/jit/backend/x86/runner.py +++ b/pypy/jit/backend/x86/runner.py @@ -160,9 +160,7 @@ descr = self.gc_ll_descr.getframedescrs(self).arraydescr ofs = self.unpack_arraydescr(descr) frame = rffi.cast(jitframe.JITFRAMEPTR, addr_of_force_token - ofs) - fail_index = frame.jf_force_index - faildescr = self.get_fail_descr_from_number(fail_index) - frame.jf_descr = cast_instance_to_gcref(faildescr) + frame.jf_descr = frame.jf_force_descr return frame def redirect_call_assembler(self, oldlooptoken, newlooptoken): diff --git a/pypy/jit/metainterp/compile.py b/pypy/jit/metainterp/compile.py --- a/pypy/jit/metainterp/compile.py +++ b/pypy/jit/metainterp/compile.py @@ -459,7 +459,7 @@ exit_frame_with_exception_descr_ref = ExitFrameWithExceptionDescrRef() # pseudo loop tokens to make the life of optimize.py easier - return {'loop_tokens_done_with_this_frame_int': [ + d = {'loop_tokens_done_with_this_frame_int': [ TerminatingLoopToken(1, done_with_this_frame_descr_int) ], 'loop_tokens_done_with_this_frame_ref': [ @@ -474,7 +474,9 @@ 'loop_tokens_exit_frame_with_exception_ref': [ TerminatingLoopToken(1, exit_frame_with_exception_descr_ref) ], - } + } + d.update(locals()) + return d class ResumeDescr(AbstractFailDescr): pass diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py --- a/pypy/jit/metainterp/pyjitpl.py +++ b/pypy/jit/metainterp/pyjitpl.py @@ -1475,7 +1475,7 @@ jd.propagate_exc_descr = exc_descr # num = self.cpu.get_fail_descr_number(exc_descr) - self.cpu.propagate_exception_v = num + self.cpu.propagate_exc_descr = exc_descr # self.globaldata = MetaInterpGlobalData(self) From noreply at buildbot.pypy.org Mon Jan 14 18:11:12 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 14 Jan 2013 18:11:12 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: manage to write a failing test Message-ID: <20130114171112.8CC251C0DCB@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60052:28596864aa21 Date: 2013-01-14 19:10 +0200 http://bitbucket.org/pypy/pypy/changeset/28596864aa21/ Log: manage to write a failing test diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py --- a/pypy/jit/backend/test/runner_test.py +++ b/pypy/jit/backend/test/runner_test.py @@ -3725,3 +3725,64 @@ # make sure that force reads the registers from a zeroed piece of # memory assert values[0] == 0 + + def test_compile_bridge_while_running(self): + def func(): + bridge = parse(""" + [i1, i2] + i3 = int_add(i1, i2) + i4 = int_add(i1, i3) + i5 = int_add(i1, i4) + i6 = int_add(i4, i5) + i7 = int_add(i6, i5) + i8 = int_add(i5, 1) + i9 = int_add(i8, 1) + force_spill(i1) + force_spill(i2) + force_spill(i3) + force_spill(i4) + force_spill(i5) + force_spill(i6) + force_spill(i7) + force_spill(i8) + force_spill(i9) + call(ConstClass(func2_ptr), 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, descr=calldescr2) + guard_true(i1, descr=guarddescr) [i1, i2, i3, i4, i5, i6, i7, i8, i9] + finish(i1, descr=finaldescr) + """, namespace={'finaldescr': finaldescr, 'calldescr2': calldescr2, + 'guarddescr': guarddescr, 'func2_ptr': func2_ptr}) + self.cpu.compile_bridge(faildescr, bridge.inputargs, + bridge.operations, looptoken) + + cpu = self.cpu + finaldescr = BasicFinalDescr(13) + finaldescr2 = BasicFinalDescr(133) + guarddescr = BasicFailDescr(8) + + FUNC = self.FuncType([], lltype.Void) + FPTR = self.Ptr(FUNC) + func_ptr = llhelper(FPTR, func) + calldescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, + EffectInfo.MOST_GENERAL) + + def func2(a, b, c, d, e, f, g, h, i, j, k, l): + pass + + FUNC2 = self.FuncType([lltype.Signed] * 12, lltype.Void) + FPTR2 = self.Ptr(FUNC2) + func2_ptr = llhelper(FPTR2, func2) + calldescr2 = cpu.calldescrof(FUNC2, FUNC2.ARGS, FUNC2.RESULT, + EffectInfo.MOST_GENERAL) + + faildescr = BasicFailDescr(0) + + looptoken = JitCellToken() + loop = parse(""" + [i0, i1, i2] + call(ConstClass(func_ptr), descr=calldescr) + guard_true(i0, descr=faildescr) [i1, i2] + finish(i2, descr=finaldescr2) + """, namespace=locals()) + self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) + frame = self.cpu.execute_token(looptoken, 0, 0, 3) + assert self.cpu.get_latest_descr(frame) is guarddescr diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -571,6 +571,7 @@ operations = regalloc.prepare_bridge(inputargs, arglocs, operations, self.current_clt.allgcrefs) + self._check_frame_depth() frame_depth = self._assemble(regalloc, operations) codeendpos = self.mc.get_relative_pos() self.write_pending_failure_recoveries() @@ -650,6 +651,9 @@ mc.writeimm32(self.error_trampoline_64 - pos_after_jz) mc.copy_to_raw_memory(rawstart + pos_after_jz - 4) + def _check_frame_depth(self): + pass + def get_asmmemmgr_blocks(self, looptoken): clt = looptoken.compiled_loop_token if clt.asmmemmgr_blocks is None: From noreply at buildbot.pypy.org Mon Jan 14 18:30:01 2013 From: noreply at buildbot.pypy.org (cfbolz) Date: Mon, 14 Jan 2013 18:30:01 +0100 (CET) Subject: [pypy-commit] pypy type-specialized-instances: stop using strings as "the other type" Message-ID: <20130114173001.167C11C1197@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: type-specialized-instances Changeset: r60053:66f5d1b54b2c Date: 2013-01-14 13:29 +0100 http://bitbucket.org/pypy/pypy/changeset/66f5d1b54b2c/ Log: stop using strings as "the other type" diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py --- a/pypy/objspace/std/test/test_mapdict.py +++ b/pypy/objspace/std/test/test_mapdict.py @@ -313,9 +313,9 @@ assert space.eq_w(obj1.getdictvalue(space, "x"), space.wrap(1)) obj2 = cls.instantiate() - w_str = space.wrap("string") - obj2.setdictvalue(space, "x", w_str) - assert space.eq_w(obj2.getdictvalue(space, "x"), w_str) + w1 = W_Root() + obj2.setdictvalue(space, "x", w1) + assert obj2.getdictvalue(space, "x") is w1 assert obj1.map is not obj2.map assert isinstance(obj1.map, IntAttribute) @@ -327,7 +327,7 @@ assert obj1.map is obj3.map assert IntAttribute.unerase_item(obj1.storage[0]) == 1 - assert PlainAttribute.unerase_item(obj2.storage[0]) == w_str + assert PlainAttribute.unerase_item(obj2.storage[0]) == w1 def test_add_more_attributes(self): space = self.space @@ -341,11 +341,12 @@ obj2 = cls.instantiate() obj2.setdictvalue(space, "x", space.wrap(5)) # this is shared - obj2.setdictvalue(space, "y", space.wrap("str")) # this not + w1 = W_Root() + obj2.setdictvalue(space, "y", w1) # this not assert space.eq_w(obj2.getdictvalue(space, "x"), space.wrap(5)) - assert space.eq_w(obj2.getdictvalue(space, "y"), space.wrap("str")) + assert obj2.getdictvalue(space, "y") is w1 - def test_switch_attribute_types(self): + def test_switch_int_attribute_types(self): space = self.space cls = Class(sp=space) obj1 = cls.instantiate() @@ -354,11 +355,12 @@ assert isinstance(obj1.map, IntAttribute) assert space.eq_w(obj1.getdictvalue(space, "x"), space.wrap(1)) - obj1.setdictvalue(space, "y", space.wrap("str")) + w1 = W_Root() + obj1.setdictvalue(space, "y", w1) assert isinstance(obj1.map, PlainAttribute) - assert space.eq_w(obj1.getdictvalue(space, "y"), space.wrap("str")) + assert obj1.getdictvalue(space, "y") is w1 - def test_overwrite_attribute_with_another_type(self): + def test_overwrite_int_attribute_with_another_type(self): space = self.space cls = Class(sp=space) obj1 = cls.instantiate() @@ -367,11 +369,12 @@ assert isinstance(obj1.map, IntAttribute) assert space.eq_w(obj1.getdictvalue(space, "x"), space.wrap(1)) - obj1.setdictvalue(space, "x", space.wrap("a")) + w1 = W_Root() + obj1.setdictvalue(space, "x", w1) assert isinstance(obj1.map, PlainAttribute) - assert space.eq_w(obj1.getdictvalue(space, "x"), space.wrap("a")) + assert obj1.getdictvalue(space, "x") is w1 - def test_overwrite_attribute_with_another_type2(self): + def test_overwrite_int_attribute_with_another_type2(self): space = self.space cls = Class(sp=space) obj1 = cls.instantiate() @@ -385,9 +388,10 @@ assert space.eq_w(obj1.getdictvalue(space, "y"), space.wrap(2)) # overwrite 'x' with new type - obj1.setdictvalue(space, "x", space.wrap("a")) + w1 = W_Root() + obj1.setdictvalue(space, "x", w1) assert isinstance(obj1.map, PlainAttribute) - assert space.eq_w(obj1.getdictvalue(space, "x"), space.wrap("a")) + assert obj1.getdictvalue(space, "x") is w1 # check if 'y' is still reachable assert isinstance(obj1.map.back, IntAttribute) From noreply at buildbot.pypy.org Mon Jan 14 18:30:02 2013 From: noreply at buildbot.pypy.org (cfbolz) Date: Mon, 14 Jan 2013 18:30:02 +0100 (CET) Subject: [pypy-commit] pypy type-specialized-instances: type-specialize int attributes Message-ID: <20130114173002.586641C1197@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: type-specialized-instances Changeset: r60054:2007a3fae522 Date: 2013-01-14 18:26 +0100 http://bitbucket.org/pypy/pypy/changeset/2007a3fae522/ Log: type-specialize int attributes diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py --- a/pypy/objspace/std/mapdict.py +++ b/pypy/objspace/std/mapdict.py @@ -7,6 +7,7 @@ from pypy.objspace.std.dictmultiobject import W_DictMultiObject, DictStrategy, ObjectDictStrategy from pypy.objspace.std.dictmultiobject import BaseKeyIterator, BaseValueIterator, BaseItemIterator from pypy.objspace.std.dictmultiobject import _never_equal_to_string +from pypy.objspace.std.stringobject import W_StringObject from pypy.objspace.std.objectobject import W_ObjectObject from pypy.objspace.std.typeobject import TypeCell @@ -141,6 +142,8 @@ # XXX not so nice that the classes have to be listed if attrclass_key == PlainAttribute.attrclass_key: attr = PlainAttribute((name, index), self) + elif attrclass_key == StrAttribute.attrclass_key: + attr = StrAttribute((name, index), self) else: assert attrclass_key == IntAttribute.attrclass_key attr = IntAttribute((name, index), self) @@ -372,6 +375,26 @@ erased = self.erase_item(self.space.int_w(w_value)) obj._mapdict_write_storage(self.position, erased) +class StrAttribute(AbstractStoredAttribute): + attrclass_key = 2 + + erase_item, unerase_item = rerased.new_erasing_pair("mapdict storage string item") + erase_item = staticmethod(erase_item) + unerase_item = staticmethod(unerase_item) + + def read_attr(self, obj): + erased = obj._mapdict_read_storage(self.position) + value = self.unerase_item(erased) + return self.space.wrap(value) + + def write_attr(self, obj, w_value): + if type(w_value) is not W_StringObject: + self._replace(obj, self.selector, w_value) + return + erased = self.erase_item(self.space.str_w(w_value)) + obj._mapdict_write_storage(self.position, erased) + + def is_taggable_int(space, w_value): from pypy.objspace.std.intobject import W_IntObject if type(w_value) is W_IntObject: @@ -386,7 +409,8 @@ attrclass = PlainAttribute if is_taggable_int(space, w_value): attrclass = IntAttribute - + elif type(w_value) is W_StringObject: + attrclass = StrAttribute return attrclass def _become(w_obj, new_obj): diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py --- a/pypy/objspace/std/test/test_mapdict.py +++ b/pypy/objspace/std/test/test_mapdict.py @@ -407,6 +407,66 @@ assert isinstance(obj1.map, PlainAttribute) assert space.eq_w(obj1.getdictvalue(space, "x"), space.wrap(sys.maxint)) + def test_str_attributes(self): + space = self.space + cls = Class(sp=space) + obj1 = cls.instantiate() + obj1.setdictvalue(space, "x", space.wrap("a")) + assert space.eq_w(obj1.getdictvalue(space, "x"), space.wrap("a")) + + obj2 = cls.instantiate() + w1 = W_Root() + obj2.setdictvalue(space, "x", w1) + assert obj2.getdictvalue(space, "x") is w1 + + assert obj1.map is not obj2.map + assert isinstance(obj1.map, StrAttribute) + + obj3 = cls.instantiate() + obj3.setdictvalue(space, "x", space.wrap("a")) + assert space.eq_w(obj3.getdictvalue(space, "x"), space.wrap("a")) + + assert obj1.map is obj3.map + + assert StrAttribute.unerase_item(obj1.storage[0]) == "a" + assert PlainAttribute.unerase_item(obj2.storage[0]) == w1 + + def test_overwrite_str_attribute_with_another_type(self): + space = self.space + cls = Class(sp=space) + obj1 = cls.instantiate() + + obj1.setdictvalue(space, "x", space.wrap("a")) + assert isinstance(obj1.map, StrAttribute) + assert space.eq_w(obj1.getdictvalue(space, "x"), space.wrap("a")) + + w1 = W_Root() + obj1.setdictvalue(space, "x", w1) + assert isinstance(obj1.map, PlainAttribute) + assert obj1.getdictvalue(space, "x") is w1 + + def test_overwrite_str_attribute_with_another_type2(self): + space = self.space + cls = Class(sp=space) + obj1 = cls.instantiate() + + obj1.setdictvalue(space, "x", space.wrap("a")) + assert isinstance(obj1.map, StrAttribute) + assert space.eq_w(obj1.getdictvalue(space, "x"), space.wrap("a")) + + obj1.setdictvalue(space, "y", space.wrap("b")) + assert isinstance(obj1.map.back, StrAttribute) + assert space.eq_w(obj1.getdictvalue(space, "y"), space.wrap("b")) + + # overwrite 'x' with new type + w1 = W_Root() + obj1.setdictvalue(space, "x", w1) + assert isinstance(obj1.map, PlainAttribute) + assert obj1.getdictvalue(space, "x") is w1 + + # check if 'y' is still reachable + assert isinstance(obj1.map.back, StrAttribute) + assert space.eq_w(obj1.getdictvalue(space, "y"), space.wrap("b")) # ___________________________________________________________ # dict tests From noreply at buildbot.pypy.org Mon Jan 14 18:30:03 2013 From: noreply at buildbot.pypy.org (cfbolz) Date: Mon, 14 Jan 2013 18:30:03 +0100 (CET) Subject: [pypy-commit] pypy type-specialized-instances: specialize string fields Message-ID: <20130114173003.8073D1C1197@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: type-specialized-instances Changeset: r60055:8503866ec6b1 Date: 2013-01-14 18:26 +0100 http://bitbucket.org/pypy/pypy/changeset/8503866ec6b1/ Log: specialize string fields diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py --- a/pypy/objspace/std/mapdict.py +++ b/pypy/objspace/std/mapdict.py @@ -7,6 +7,7 @@ from pypy.objspace.std.dictmultiobject import W_DictMultiObject, DictStrategy, ObjectDictStrategy from pypy.objspace.std.dictmultiobject import BaseKeyIterator, BaseValueIterator, BaseItemIterator from pypy.objspace.std.dictmultiobject import _never_equal_to_string +from pypy.objspace.std.stringobject import W_StringObject from pypy.objspace.std.objectobject import W_ObjectObject from pypy.objspace.std.typeobject import TypeCell @@ -141,6 +142,8 @@ # XXX not so nice that the classes have to be listed if attrclass_key == PlainAttribute.attrclass_key: attr = PlainAttribute((name, index), self) + elif attrclass_key == StrAttribute.attrclass_key: + attr = StrAttribute((name, index), self) else: assert attrclass_key == IntAttribute.attrclass_key attr = IntAttribute((name, index), self) @@ -372,6 +375,26 @@ erased = self.erase_item(self.space.int_w(w_value)) obj._mapdict_write_storage(self.position, erased) +class StrAttribute(AbstractStoredAttribute): + attrclass_key = 2 + + erase_item, unerase_item = rerased.new_erasing_pair("mapdict storage string item") + erase_item = staticmethod(erase_item) + unerase_item = staticmethod(unerase_item) + + def read_attr(self, obj): + erased = obj._mapdict_read_storage(self.position) + value = self.unerase_item(erased) + return self.space.wrap(value) + + def write_attr(self, obj, w_value): + if type(w_value) is not W_StringObject: + self._replace(obj, self.selector, w_value) + return + erased = self.erase_item(self.space.str_w(w_value)) + obj._mapdict_write_storage(self.position, erased) + + def is_taggable_int(space, w_value): from pypy.objspace.std.intobject import W_IntObject if type(w_value) is W_IntObject: @@ -386,7 +409,8 @@ attrclass = PlainAttribute if is_taggable_int(space, w_value): attrclass = IntAttribute - + elif type(w_value) is W_StringObject: + attrclass = StrAttribute return attrclass def _become(w_obj, new_obj): diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py --- a/pypy/objspace/std/test/test_mapdict.py +++ b/pypy/objspace/std/test/test_mapdict.py @@ -407,6 +407,66 @@ assert isinstance(obj1.map, PlainAttribute) assert space.eq_w(obj1.getdictvalue(space, "x"), space.wrap(sys.maxint)) + def test_str_attributes(self): + space = self.space + cls = Class(sp=space) + obj1 = cls.instantiate() + obj1.setdictvalue(space, "x", space.wrap("a")) + assert space.eq_w(obj1.getdictvalue(space, "x"), space.wrap("a")) + + obj2 = cls.instantiate() + w1 = W_Root() + obj2.setdictvalue(space, "x", w1) + assert obj2.getdictvalue(space, "x") is w1 + + assert obj1.map is not obj2.map + assert isinstance(obj1.map, StrAttribute) + + obj3 = cls.instantiate() + obj3.setdictvalue(space, "x", space.wrap("a")) + assert space.eq_w(obj3.getdictvalue(space, "x"), space.wrap("a")) + + assert obj1.map is obj3.map + + assert StrAttribute.unerase_item(obj1.storage[0]) == "a" + assert PlainAttribute.unerase_item(obj2.storage[0]) == w1 + + def test_overwrite_str_attribute_with_another_type(self): + space = self.space + cls = Class(sp=space) + obj1 = cls.instantiate() + + obj1.setdictvalue(space, "x", space.wrap("a")) + assert isinstance(obj1.map, StrAttribute) + assert space.eq_w(obj1.getdictvalue(space, "x"), space.wrap("a")) + + w1 = W_Root() + obj1.setdictvalue(space, "x", w1) + assert isinstance(obj1.map, PlainAttribute) + assert obj1.getdictvalue(space, "x") is w1 + + def test_overwrite_str_attribute_with_another_type2(self): + space = self.space + cls = Class(sp=space) + obj1 = cls.instantiate() + + obj1.setdictvalue(space, "x", space.wrap("a")) + assert isinstance(obj1.map, StrAttribute) + assert space.eq_w(obj1.getdictvalue(space, "x"), space.wrap("a")) + + obj1.setdictvalue(space, "y", space.wrap("b")) + assert isinstance(obj1.map.back, StrAttribute) + assert space.eq_w(obj1.getdictvalue(space, "y"), space.wrap("b")) + + # overwrite 'x' with new type + w1 = W_Root() + obj1.setdictvalue(space, "x", w1) + assert isinstance(obj1.map, PlainAttribute) + assert obj1.getdictvalue(space, "x") is w1 + + # check if 'y' is still reachable + assert isinstance(obj1.map.back, StrAttribute) + assert space.eq_w(obj1.getdictvalue(space, "y"), space.wrap("b")) # ___________________________________________________________ # dict tests From noreply at buildbot.pypy.org Mon Jan 14 18:30:04 2013 From: noreply at buildbot.pypy.org (cfbolz) Date: Mon, 14 Jan 2013 18:30:04 +0100 (CET) Subject: [pypy-commit] pypy type-specialized-instances: merge Message-ID: <20130114173004.B8EEA1C1197@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: type-specialized-instances Changeset: r60056:7ec148a203e3 Date: 2013-01-14 18:28 +0100 http://bitbucket.org/pypy/pypy/changeset/7ec148a203e3/ Log: merge From noreply at buildbot.pypy.org Mon Jan 14 18:33:21 2013 From: noreply at buildbot.pypy.org (cfbolz) Date: Mon, 14 Jan 2013 18:33:21 +0100 (CET) Subject: [pypy-commit] pypy type-specialized-instances: merge default Message-ID: <20130114173321.5E0101C1197@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: type-specialized-instances Changeset: r60057:4f8edbe5b38d Date: 2013-01-14 18:33 +0100 http://bitbucket.org/pypy/pypy/changeset/4f8edbe5b38d/ Log: merge default diff --git a/pypy/doc/test/test_whatsnew.py b/pypy/doc/test/test_whatsnew.py --- a/pypy/doc/test/test_whatsnew.py +++ b/pypy/doc/test/test_whatsnew.py @@ -29,7 +29,7 @@ merge() and \ branch(default)) and \ not branch(default)' % (startrev, endrev) - cmd = r"hg log -R '%s' -r '%s' --template '{branches}\n'" % (path, revset) + cmd = r'hg log -R "%s" -r "%s" --template "{branches}\n"' % (path, revset) out = getoutput(cmd) branches = set(map(str.strip, out.splitlines())) return branches diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -15,6 +15,9 @@ .. branch: numpypy-longdouble Long double support for numpypy +.. branch: numpypy-real-as-view +Convert real, imag from ufuncs to views. This involves the beginning of +view() functionality .. branch: signatures Improved RPython typing diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -178,6 +178,11 @@ else: oparg = 0 + # note: the structure of the code here is such that it makes + # (after translation) a big "if/elif" chain, which is then + # turned into a switch(). It starts here: even if the first + # one is not an "if" but a "while" the effect is the same. + while opcode == self.opcodedesc.EXTENDED_ARG.index: opcode = ord(co_code[next_instr]) if opcode < self.HAVE_ARGUMENT: @@ -226,6 +231,8 @@ 'END_FINALLY', 'JUMP_ABSOLUTE'): continue # opcodes implemented above + # the following "if" is part of the big switch described + # above. if opcode == opdesc.index: # dispatch to the opcode method meth = getattr(self, opdesc.methodname) diff --git a/pypy/jit/backend/test/calling_convention_test.py b/pypy/jit/backend/test/calling_convention_test.py --- a/pypy/jit/backend/test/calling_convention_test.py +++ b/pypy/jit/backend/test/calling_convention_test.py @@ -105,11 +105,13 @@ ops = '[%s]\n' % arguments ops += '%s\n' % spill_ops ops += 'f99 = call(ConstClass(func_ptr), %s, descr=calldescr)\n' % arguments - ops += 'finish(f99, %s)\n' % arguments + ops += 'i99 = same_as(0)\n' + ops += 'guard_true(i99) [f99, %s]\n' % arguments + ops += 'finish()\n' loop = parse(ops, namespace=locals()) looptoken = JitCellToken() - done_number = self.cpu.get_fail_descr_number(loop.operations[-1].getdescr()) + done_number = self.cpu.get_fail_descr_number(loop.operations[-2].getdescr()) self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) argvals, expected_result = self._prepare_args(args, floats, ints) diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py --- a/pypy/jit/backend/test/runner_test.py +++ b/pypy/jit/backend/test/runner_test.py @@ -1192,7 +1192,8 @@ values.append(longlong.getfloatstorage(r.random())) # looptoken = JitCellToken() - faildescr = BasicFailDescr(42) + guarddescr = BasicFailDescr(42) + faildescr = BasicFailDescr(43) operations = [] retboxes = [] retvalues = [] @@ -1221,9 +1222,13 @@ retboxes.insert(kk, newbox) retvalues.insert(kk, y) # - operations.append( - ResOperation(rop.FINISH, retboxes, None, descr=faildescr) - ) + zero = BoxInt() + operations.extend([ + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=guarddescr), + ResOperation(rop.FINISH, [], None, descr=faildescr) + ]) + operations[-2].setfailargs(retboxes) print inputargs for op in operations: print op @@ -1344,12 +1349,15 @@ targettoken = TargetToken() faildescr1 = BasicFailDescr(1) faildescr2 = BasicFailDescr(2) + faildescr3 = BasicFailDescr(3) operations = [ ResOperation(rop.LABEL, fboxes, None, descr=targettoken), ResOperation(rop.FLOAT_LE, [fboxes[0], constfloat(9.2)], i2), ResOperation(rop.GUARD_TRUE, [i2], None, descr=faildescr1), - ResOperation(rop.FINISH, fboxes, None, descr=faildescr2), + ResOperation(rop.GUARD_FALSE, [i2], None, descr=faildescr2), + ResOperation(rop.FINISH, [], None, descr=faildescr3), ] + operations[-3].setfailargs(fboxes) operations[-2].setfailargs(fboxes) looptoken = JitCellToken() self.cpu.compile_loop(fboxes, operations, looptoken) @@ -1382,6 +1390,7 @@ py.test.skip("requires floats") fboxes = [BoxFloat() for i in range(3)] faildescr1 = BasicFailDescr(100) + faildescr2 = BasicFailDescr(102) loopops = """ [i0,f1, f2] f3 = float_add(f1, f2) @@ -1406,9 +1415,13 @@ assert longlong.getrealfloat(f2) == 0.75 assert longlong.getrealfloat(f3) == 133.0 + zero = BoxInt() bridgeops = [ - ResOperation(rop.FINISH, fboxes, None, descr=faildescr1), + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=faildescr1), + ResOperation(rop.FINISH, [], None, descr=faildescr2), ] + bridgeops[-2].setfailargs(fboxes[:]) self.cpu.compile_bridge(loop.operations[-2].getdescr(), fboxes, bridgeops, looptoken) args = [1, @@ -2002,7 +2015,7 @@ i1 = same_as(1) call(ConstClass(fptr), i0, descr=calldescr) p0 = guard_exception(ConstClass(xtp)) [i1] - finish(0, p0) + finish(p0) ''' FPTR = lltype.Ptr(lltype.FuncType([lltype.Signed], lltype.Void)) fptr = llhelper(FPTR, func) @@ -2023,8 +2036,7 @@ looptoken = JitCellToken() self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) deadframe = self.cpu.execute_token(looptoken, 1) - assert self.cpu.get_latest_value_int(deadframe, 0) == 0 - assert self.cpu.get_latest_value_ref(deadframe, 1) == xptr + assert self.cpu.get_latest_value_ref(deadframe, 0) == xptr excvalue = self.cpu.grab_exc_value(deadframe) assert not excvalue deadframe = self.cpu.execute_token(looptoken, 0) diff --git a/pypy/jit/backend/test/test_random.py b/pypy/jit/backend/test/test_random.py --- a/pypy/jit/backend/test/test_random.py +++ b/pypy/jit/backend/test/test_random.py @@ -26,6 +26,8 @@ class OperationBuilder(object): def __init__(self, cpu, loop, vars): self.cpu = cpu + if not hasattr(cpu, '_faildescr_keepalive'): + cpu._faildescr_keepalive = [] self.fakemetainterp = FakeMetaInterp() self.loop = loop self.intvars = [box for box in vars if isinstance(box, BoxInt)] @@ -206,6 +208,11 @@ if pytest.config.option.output: s.close() + def getfaildescr(self): + descr = BasicFailDescr() + self.cpu._faildescr_keepalive.append(descr) + return descr + class CannotProduceOperation(Exception): pass @@ -287,7 +294,7 @@ builder.intvars[:] = original_intvars else: op = ResOperation(rop.GUARD_NO_OVERFLOW, [], None) - op.setdescr(BasicFailDescr()) + op.setdescr(builder.getfaildescr()) op.setfailargs(fail_subset) builder.loop.operations.append(op) @@ -360,7 +367,7 @@ def produce_into(self, builder, r): op, passing = self.gen_guard(builder, r) builder.loop.operations.append(op) - op.setdescr(BasicFailDescr()) + op.setdescr(builder.getfaildescr()) op.setfailargs(builder.subset_of_intvars(r)) if not passing: builder.should_fail_by = op @@ -624,8 +631,9 @@ if v not in used_later: endvars.append(v) r.shuffle(endvars) + endvars = endvars[:1] loop.operations.append(ResOperation(rop.FINISH, endvars, None, - descr=BasicFailDescr())) + descr=builder.getfaildescr())) if builder.should_fail_by: self.should_fail_by = builder.should_fail_by self.guard_op = builder.guard_op @@ -718,7 +726,7 @@ else: op = ResOperation(rop.GUARD_EXCEPTION, [guard_op._exc_box], BoxPtr()) - op.setdescr(BasicFailDescr()) + op.setdescr(self.builder.getfaildescr()) op.setfailargs([]) return op diff --git a/pypy/jit/backend/x86/test/test_recompilation.py b/pypy/jit/backend/x86/test/test_recompilation.py --- a/pypy/jit/backend/x86/test/test_recompilation.py +++ b/pypy/jit/backend/x86/test/test_recompilation.py @@ -45,7 +45,8 @@ force_spill(i5) i8 = int_add(i7, 1) i9 = int_add(i8, 1) - finish(i3, i4, i5, i6, i7, i8, i9, descr=fdescr2) + guard_false(i3, descr=fdescr2) [i3, i4, i5, i6, i7, i8, i9] + finish() ''' bridge = self.attach_bridge(ops, loop, -2) descr = loop.operations[3].getdescr() diff --git a/pypy/jit/backend/x86/test/test_regalloc.py b/pypy/jit/backend/x86/test/test_regalloc.py --- a/pypy/jit/backend/x86/test/test_regalloc.py +++ b/pypy/jit/backend/x86/test/test_regalloc.py @@ -327,7 +327,7 @@ assert self.getint(0) == 0 bridge_ops = ''' [i0, i1] - finish(1, 2) + finish(2) ''' self.attach_bridge(bridge_ops, loop, 0) self.run(loop, 0, 1) @@ -428,7 +428,7 @@ ops = ''' [i0, i1, i2, i3, i4, i5, i6, i7] guard_value(i6, i1) [i0, i2, i3, i4, i5, i6] - finish(i0, i2, i3, i4, i5, i6) + finish(i0) ''' self.interpret(ops, [0, 0, 0, 0, 0, 0, 0, 0]) assert self.getint(0) == 0 @@ -438,14 +438,15 @@ [i0, i1, i2, i3, i4, i5, i6, i7, i8] i9 = same_as(0) guard_true(i0) [i9, i0, i1, i2, i3, i4, i5, i6, i7, i8] - finish(1, i0, i1, i2, i3, i4, i5, i6, i7, i8) + finish(1) ''' loop = self.interpret(ops, [0, 1, 2, 3, 4, 5, 6, 7, 8]) assert self.getint(0) == 0 bridge_ops = ''' [i9, i0, i1, i2, i3, i4, i5, i6, i7, i8] call(ConstClass(raising_fptr), 0, descr=raising_calldescr) - finish(i0, i1, i2, i3, i4, i5, i6, i7, i8) + guard_true(i9) [i0, i1, i2, i3, i4, i5, i6, i7, i8] + finish() ''' self.attach_bridge(bridge_ops, loop, 1) self.run(loop, 0, 1, 2, 3, 4, 5, 6, 7, 8) @@ -474,7 +475,8 @@ i1 = same_as(1) i2 = int_lt(i0, 100) guard_true(i3) [i1, i2] - finish(0, i2) + i4 = int_neg(i2) + finish(0) ''' self.interpret(ops, [0, 1]) assert self.getint(0) == 0 @@ -503,7 +505,8 @@ i15 = int_is_true(i5) i16 = int_is_true(i6) i17 = int_is_true(i7) - finish(i10, i11, i12, i13, i14, i15, i16, i17) + guard_true(i0) [i10, i11, i12, i13, i14, i15, i16, i17] + finish() ''' self.interpret(ops, [0, 42, 12, 0, 13, 0, 0, 3333]) assert self.getints(8) == [0, 1, 1, 0, 1, 0, 0, 1] @@ -517,7 +520,8 @@ i13 = int_eq(i5, i6) i14 = int_gt(i6, i2) i15 = int_ne(i2, i6) - finish(i10, i11, i12, i13, i14, i15) + guard_true(i0) [i10, i11, i12, i13, i14, i15] + finish() ''' self.interpret(ops, [0, 1, 2, 3, 4, 5, 6]) assert self.getints(6) == [1, 1, 0, 0, 1, 1] @@ -574,7 +578,9 @@ ops = ''' [f0, f1] f2 = float_add(f0, f1) - finish(f2, f0, f1) + i0 = same_as(0) + guard_true(i0) [f2, f0, f1] + finish() ''' self.interpret(ops, [3.0, 1.5]) assert self.getfloats(3) == [4.5, 3.0, 1.5] @@ -584,7 +590,9 @@ [f0, f1, f2, f3, f4, f5, f6, f7, f8] f9 = float_add(f0, f1) f10 = float_add(f8, 3.5) - finish(f9, f10, f2, f3, f4, f5, f6, f7, f8) + i0 = same_as(0) + guard_true(i0) [f9, f10, f2, f3, f4, f5, f6, f7, f8] + finish() ''' self.interpret(ops, [0.1, .2, .3, .4, .5, .6, .7, .8, .9]) assert self.getfloats(9) == [.1+.2, .9+3.5, .3, .4, .5, .6, .7, .8, .9] @@ -613,7 +621,8 @@ i7 = float_ne(f7, 0.0) i8 = float_ne(f8, 0.0) i9 = float_ne(f9, 0.0) - finish(i0, i1, i2, i3, i4, i5, i6, i7, i8, i9) + guard_true(i0) [i0, i1, i2, i3, i4, i5, i6, i7, i8, i9] + finish() ''' loop = self.interpret(ops, [0.0, .1, .2, .3, .4, .5, .6, .7, .8, .9]) assert self.getints(9) == [0, 1, 1, 1, 1, 1, 1, 1, 1] diff --git a/pypy/jit/backend/x86/test/test_regalloc2.py b/pypy/jit/backend/x86/test/test_regalloc2.py --- a/pypy/jit/backend/x86/test/test_regalloc2.py +++ b/pypy/jit/backend/x86/test/test_regalloc2.py @@ -11,13 +11,17 @@ v2 = BoxInt() v3 = BoxInt() v4 = BoxInt() + zero = BoxInt() inputargs = [v1] operations = [ ResOperation(rop.INT_ADD, [v1, v1], v2), ResOperation(rop.INT_INVERT, [v2], v3), ResOperation(rop.UINT_RSHIFT, [v1, ConstInt(3)], v4), - ResOperation(rop.FINISH, [v4, v3], None, descr=BasicFailDescr()), + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=BasicFailDescr()), + ResOperation(rop.FINISH, [], None, descr=BasicFailDescr()) ] + operations[-2].setfailargs([v4, v3]) cpu = CPU(None, None) cpu.setup_once() looptoken = JitCellToken() @@ -31,6 +35,7 @@ v2 = BoxInt() v3 = BoxInt() v4 = BoxInt() + zero = BoxInt() tmp5 = BoxInt() inputargs = [v1] operations = [ @@ -38,8 +43,11 @@ ResOperation(rop.INT_MUL, [v2, v1], v3), ResOperation(rop.INT_IS_TRUE, [v2], tmp5), ResOperation(rop.INT_IS_ZERO, [tmp5], v4), - ResOperation(rop.FINISH, [v4, v3, tmp5], None, descr=BasicFailDescr()), + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=BasicFailDescr()), + ResOperation(rop.FINISH, [], None, descr=BasicFailDescr()) ] + operations[-2].setfailargs([v4, v3, tmp5]) cpu = CPU(None, None) cpu.setup_once() looptoken = JitCellToken() @@ -90,6 +98,7 @@ v38 = BoxInt() v39 = BoxInt() v40 = BoxInt() + zero = BoxInt() tmp41 = BoxInt() tmp42 = BoxInt() tmp43 = BoxInt() @@ -134,8 +143,12 @@ ResOperation(rop.UINT_GT, [v33, ConstInt(-11)], v38), ResOperation(rop.INT_NEG, [v7], v39), ResOperation(rop.INT_GT, [v24, v32], v40), - ResOperation(rop.FINISH, [v40, v36, v37, v31, v16, v34, v35, v23, v22, v29, v14, v39, v30, v38], None, descr=BasicFailDescr()), + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=BasicFailDescr()), + ResOperation(rop.FINISH, [], None, descr=BasicFailDescr()) ] + operations[-2].setfailargs([v40, v36, v37, v31, v16, v34, v35, v23, + v22, v29, v14, v39, v30, v38]) cpu = CPU(None, None) cpu.setup_once() looptoken = JitCellToken() @@ -198,6 +211,7 @@ v38 = BoxInt() v39 = BoxInt() v40 = BoxInt() + zero = BoxInt() tmp41 = BoxInt() tmp42 = BoxInt() tmp43 = BoxInt() @@ -240,8 +254,13 @@ ResOperation(rop.INT_GT, [v4, v11], v38), ResOperation(rop.INT_LT, [v27, v22], v39), ResOperation(rop.INT_NEG, [v27], v40), - ResOperation(rop.FINISH, [v40, v10, v36, v26, v13, v30, v21, v33, v18, v25, v31, v32, v28, v29, v35, v38, v20, v39, v34, v23, v37], None, descr=BasicFailDescr()), + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=BasicFailDescr()), + ResOperation(rop.FINISH, [], None, descr=BasicFailDescr()) ] + operations[-2].setfailargs([v40, v10, v36, v26, v13, v30, v21, v33, + v18, v25, v31, v32, v28, v29, v35, v38, + v20, v39, v34, v23, v37]) cpu = CPU(None, None) cpu.setup_once() looptoken = JitCellToken() diff --git a/pypy/jit/backend/x86/test/test_runner.py b/pypy/jit/backend/x86/test/test_runner.py --- a/pypy/jit/backend/x86/test/test_runner.py +++ b/pypy/jit/backend/x86/test/test_runner.py @@ -499,9 +499,12 @@ i6, descr=calldescr), ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr), - ResOperation(rop.FINISH, [i3, i4, i5, i6], None, - descr=BasicFailDescr(0)) + ResOperation(rop.GUARD_FALSE, [i3], None, + descr=BasicFailDescr(0)), + ResOperation(rop.FINISH, [], None, + descr=BasicFailDescr(1)) ] + ops[-2].setfailargs([i3, i4, i5, i6]) ops[1].setfailargs([]) ops[3].setfailargs([]) ops[5].setfailargs([]) diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py --- a/pypy/jit/codewriter/support.py +++ b/pypy/jit/codewriter/support.py @@ -10,7 +10,7 @@ from pypy.rpython.extregistry import ExtRegistryEntry from pypy.translator.simplify import get_funcobj from pypy.translator.unsimplify import split_block -from pypy.objspace.flow.model import Constant +from pypy.objspace.flow.model import Variable, Constant from pypy.translator.translator import TranslationContext from pypy.annotation.policy import AnnotatorPolicy from pypy.annotation import model as annmodel @@ -59,6 +59,45 @@ rtyper = annotate(func, values) return rtyper.annotator.translator.graphs[0] +def autodetect_jit_markers_redvars(graph): + # the idea is to find all the jit_merge_point and + # add all the variables across the links to the reds. + for block, op in graph.iterblockops(): + if op.opname == 'jit_marker': + jitdriver = op.args[1].value + if not jitdriver.autoreds: + continue + # if we want to support also can_enter_jit, we should find a + # way to detect a consistent set of red vars to pass *both* to + # jit_merge_point and can_enter_jit. The current simple + # solution doesn't work because can_enter_jit might be in + # another block, so the set of alive_v will be different. + methname = op.args[0].value + assert methname == 'jit_merge_point', ( + "reds='auto' is supported only for jit drivers which " + "calls only jit_merge_point. Found a call to %s" % methname) + # + # compute the set of live variables across the jit_marker + alive_v = set() + for link in block.exits: + alive_v.update(link.args) + alive_v.difference_update(link.getextravars()) + for op1 in block.operations[::-1]: + if op1 is op: + break # stop when the meet the jit_marker + alive_v.discard(op1.result) + alive_v.update(op1.args) + greens_v = op.args[2:] + reds_v = alive_v - set(greens_v) + reds_v = [v for v in reds_v if isinstance(v, Variable) and + v.concretetype is not lltype.Void] + reds_v = sort_vars(reds_v) + op.args.extend(reds_v) + if jitdriver.numreds is None: + jitdriver.numreds = len(reds_v) + else: + assert jitdriver.numreds == len(reds_v), 'inconsistent number of reds_v' + def split_before_jit_merge_point(graph, portalblock, portalopindex): """Split the block just before the 'jit_merge_point', making sure the input args are in the canonical order. diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py --- a/pypy/jit/metainterp/resoperation.py +++ b/pypy/jit/metainterp/resoperation.py @@ -353,6 +353,9 @@ def initarglist(self, args): self._args = args + if not we_are_translated() and \ + self.__class__.__name__.startswith('FINISH'): # XXX remove me + assert len(args) <= 1 # FINISH operations take 0 or 1 arg now def getarglist(self): return self._args diff --git a/pypy/jit/metainterp/test/test_fficall.py b/pypy/jit/metainterp/test/test_fficall.py --- a/pypy/jit/metainterp/test/test_fficall.py +++ b/pypy/jit/metainterp/test/test_fficall.py @@ -105,7 +105,7 @@ class TestFfiCall(FfiCallTests, LLJitMixin): - def test_jit_fii_vref(self): + def test_jit_ffi_vref(self): from pypy.rlib import clibffi from pypy.rlib.jit_libffi import jit_ffi_prep_cif, jit_ffi_call @@ -136,9 +136,10 @@ res = exb[3] lltype.free(exb, flavor='raw') # - lltype.free(atypes, flavor='raw') return res # res = self.interp_operations(f, []) lltype.free(cd, flavor='raw') assert res == math.sin(1.23) + + lltype.free(atypes, flavor='raw') diff --git a/pypy/jit/metainterp/test/test_warmspot.py b/pypy/jit/metainterp/test/test_warmspot.py --- a/pypy/jit/metainterp/test/test_warmspot.py +++ b/pypy/jit/metainterp/test/test_warmspot.py @@ -1,6 +1,6 @@ import py from pypy.jit.metainterp.warmspot import get_stats -from pypy.rlib.jit import JitDriver, set_param, unroll_safe +from pypy.rlib.jit import JitDriver, set_param, unroll_safe, jit_callback from pypy.jit.backend.llgraph import runner from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin @@ -383,6 +383,23 @@ assert res == expected self.check_resops(int_sub=2, int_mul=0, int_add=2) + def test_loop_automatic_reds_not_too_many_redvars(self): + myjitdriver = JitDriver(greens = ['m'], reds = 'auto') + def one(): + return 1 + def f(n, m): + res = 0 + while n > 0: + n -= one() + myjitdriver.jit_merge_point(m=m) + res += m*2 + return res + expected = f(21, 5) + res = self.meta_interp(f, [21, 5]) + assert res == expected + oplabel = get_stats().loops[0].operations[0] + assert len(oplabel.getarglist()) == 2 # 'n', 'res' in some order + def test_inline_jit_merge_point(self): # test that the machinery to inline jit_merge_points in callers # works. The final user does not need to mess manually with the @@ -520,41 +537,21 @@ def test_callback_jit_merge_point(self): - from pypy.rlib.objectmodel import register_around_callback_hook - from pypy.rpython.lltypesystem import lltype, rffi - from pypy.translator.tool.cbuild import ExternalCompilationInfo - - callback_jit_driver = JitDriver(greens = ['name'], reds = 'auto') - - def callback_merge_point(name): - callback_jit_driver.jit_merge_point(name=name) - - @callback_jit_driver.inline(callback_merge_point) - def callback_hook(name): - pass - + @jit_callback("testing") def callback(a, b): if a > b: return 1 return -1 - CB_TP = rffi.CCallback([lltype.Signed, lltype.Signed], lltype.Signed) - eci = ExternalCompilationInfo(includes=['stdlib.h']) - qsort = rffi.llexternal('qsort', - [rffi.VOIDP, lltype.Signed, lltype.Signed, - CB_TP], lltype.Void, compilation_info=eci) - ARR = rffi.CArray(lltype.Signed) + def main(): + total = 0 + for i in range(10): + total += callback(i, 2) + return total - def main(): - register_around_callback_hook(callback_hook) - raw = lltype.malloc(ARR, 10, flavor='raw') - for i in range(10): - raw[i] = 10 - i - qsort(raw, 10, rffi.sizeof(lltype.Signed), callback) - lltype.free(raw, flavor='raw') - - self.meta_interp(main, []) - self.check_trace_count(1) + res = self.meta_interp(main, []) + assert res == 7 - 3 + self.check_trace_count(2) class TestLLWarmspot(WarmspotTests, LLJitMixin): diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py --- a/pypy/jit/metainterp/warmspot.py +++ b/pypy/jit/metainterp/warmspot.py @@ -347,62 +347,20 @@ self.jitdrivers_sd = [] graphs = self.translator.graphs for graph, block, pos in find_jit_merge_points(graphs): - self.autodetect_jit_markers_redvars(graph) + support.autodetect_jit_markers_redvars(graph) self.split_graph_and_record_jitdriver(graph, block, pos) # assert (len(set([jd.jitdriver for jd in self.jitdrivers_sd])) == len(self.jitdrivers_sd)), \ "there are multiple jit_merge_points with the same jitdriver" - def autodetect_jit_markers_redvars(self, graph): - # the idea is to find all the jit_merge_point and can_enter_jit and - # add all the variables across the links to the reds. - for block, op in graph.iterblockops(): - if op.opname == 'jit_marker': - jitdriver = op.args[1].value - if not jitdriver.autoreds: - continue - # if we want to support also can_enter_jit, we should find a - # way to detect a consistent set of red vars to pass *both* to - # jit_merge_point and can_enter_jit. The current simple - # solution doesn't work because can_enter_jit might be in - # another block, so the set of alive_v will be different. - methname = op.args[0].value - assert methname == 'jit_merge_point', ( - "reds='auto' is supported only for jit drivers which " - "calls only jit_merge_point. Found a call to %s" % methname) - # - # compute the set of live variables before the jit_marker - alive_v = set(block.inputargs) - for op1 in block.operations: - if op1 is op: - break # stop when the meet the jit_marker - if op1.result.concretetype != lltype.Void: - alive_v.add(op1.result) - greens_v = op.args[2:] - reds_v = alive_v - set(greens_v) - reds_v = [v for v in reds_v if v.concretetype is not lltype.Void] - reds_v = support.sort_vars(reds_v) - op.args.extend(reds_v) - if jitdriver.numreds is None: - jitdriver.numreds = len(reds_v) - else: - assert jitdriver.numreds == len(reds_v), 'inconsistent number of reds_v' - - def split_graph_and_record_jitdriver(self, graph, block, pos): op = block.operations[pos] jd = JitDriverStaticData() jd._jit_merge_point_in = graph args = op.args[2:] s_binding = self.translator.annotator.binding - if op.args[1].value.autoreds: - # _portal_args_s is used only by _make_hook_graph, but for now we - # declare the various set_jitcell_at, get_printable_location, - # etc. as incompatible with autoreds - jd._portal_args_s = None - else: - jd._portal_args_s = [s_binding(v) for v in args] + jd._portal_args_s = [s_binding(v) for v in args] graph = copygraph(graph) [jmpp] = find_jit_merge_points([graph]) graph.startblock = support.split_before_jit_merge_point(*jmpp) @@ -650,9 +608,10 @@ if func is None: return None # - assert not jitdriver_sd.jitdriver.autoreds, ( - "reds='auto' is not compatible with JitDriver hooks such as " - "{get,set}_jitcell_at, get_printable_location, confirm_enter_jit, etc.") + if not onlygreens: + assert not jitdriver_sd.jitdriver.autoreds, ( + "reds='auto' is not compatible with JitDriver hooks such as " + "confirm_enter_jit") extra_args_s = [] if s_first_arg is not None: extra_args_s.append(s_first_arg) @@ -717,6 +676,9 @@ jitdriver = op.args[1].value assert jitdriver in sublists, \ "can_enter_jit with no matching jit_merge_point" + assert not jitdriver.autoreds, ( + "can_enter_jit not supported with a jitdriver that " + "has reds='auto'") jd, sublist = sublists[jitdriver] origportalgraph = jd._jit_merge_point_in if graph is not origportalgraph: diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py --- a/pypy/module/_cffi_backend/ccallback.py +++ b/pypy/module/_cffi_backend/ccallback.py @@ -152,6 +152,7 @@ STDERR = 2 + at jit.jit_callback("CFFI") def invoke_callback(ffi_cif, ll_res, ll_args, ll_userdata): """ Callback specification. ffi_cif - something ffi specific, don't care diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py --- a/pypy/module/micronumpy/arrayimpl/concrete.py +++ b/pypy/module/micronumpy/arrayimpl/concrete.py @@ -13,6 +13,7 @@ from pypy.rlib.debug import make_sure_not_resized class ConcreteArrayIterator(base.BaseArrayIterator): + _immutable_fields_ = ['dtype', 'skip', 'size'] def __init__(self, array): self.array = array self.offset = 0 @@ -21,10 +22,10 @@ self.size = array.size def setitem(self, elem): - self.array.setitem(self.offset, elem) + self.dtype.setitem(self.array, self.offset, elem) def getitem(self): - return self.array.getitem(self.offset) + return self.dtype.getitem(self.array, self.offset) def getitem_bool(self): return self.dtype.getitem_bool(self.array, self.offset) @@ -42,13 +43,16 @@ self.offset %= self.size class OneDimViewIterator(ConcreteArrayIterator): - def __init__(self, array): + ''' The view iterator dtype can be different from the + array.dtype, this is what makes it a View + ''' + def __init__(self, array, dtype, start, strides, shape): self.array = array - self.offset = array.start - self.skip = array.get_strides()[0] - self.dtype = array.dtype + self.dtype = dtype + self.offset = start + self.skip = strides[0] self.index = 0 - self.size = array.get_shape()[0] + self.size = shape[0] def next(self): self.offset += self.skip @@ -65,9 +69,13 @@ self.offset %= self.size class MultiDimViewIterator(ConcreteArrayIterator): - def __init__(self, array, start, strides, backstrides, shape): + ''' The view iterator dtype can be different from the + array.dtype, this is what makes it a View + ''' + def __init__(self, array, dtype, start, strides, backstrides, shape): self.indexes = [0] * len(shape) self.array = array + self.dtype = dtype self.shape = shape self.offset = start self.shapelen = len(shape) @@ -131,12 +139,13 @@ self.offset = array.start self.dim = dim self.array = array + self.dtype = array.dtype def setitem(self, elem): - self.array.setitem(self.offset, elem) + self.dtype.setitem(self.array, self.offset, elem) def getitem(self): - return self.array.getitem(self.offset) + return self.dtype.getitem(self.array, self.offset) @jit.unroll_safe def next(self): @@ -220,6 +229,32 @@ new_shape, self) else: return None + + def get_real(self): + strides = self.get_strides() + backstrides = self.get_backstrides() + if self.dtype.is_complex_type(): + dtype = self.dtype.float_type + return SliceArray(self.start, strides, backstrides, + self.get_shape(), self, dtype=dtype) + return SliceArray(self.start, strides, backstrides, + self.get_shape(), self) + + def get_imag(self): + strides = self.get_strides() + backstrides = self.get_backstrides() + if self.dtype.is_complex_type(): + dtype = self.dtype.float_type + return SliceArray(self.start + dtype.get_size(), strides, + backstrides, self.get_shape(), self, dtype=dtype) + if self.dtype.is_flexible_type(): + # numpy returns self for self.imag + return SliceArray(self.start, strides, backstrides, + self.get_shape(), self) + impl = NonWritableArray(self.get_shape(), self.dtype, self.order, strides, + backstrides) + impl.fill(self.dtype.box(0)) + return impl # -------------------- applevel get/setitem ----------------------- @@ -377,7 +412,7 @@ def create_dot_iter(self, shape, skip): r = calculate_dot_strides(self.get_strides(), self.get_backstrides(), shape, skip) - return MultiDimViewIterator(self, self.start, r[0], r[1], shape) + return MultiDimViewIterator(self, self.dtype, self.start, r[0], r[1], shape) def swapaxes(self, axis1, axis2): shape = self.get_shape()[:] @@ -411,7 +446,7 @@ r = calculate_broadcast_strides(self.get_strides(), self.get_backstrides(), self.get_shape(), shape) - return MultiDimViewIterator(self, 0, r[0], r[1], shape) + return MultiDimViewIterator(self, self.dtype, 0, r[0], r[1], shape) def fill(self, box): self.dtype.fill(self.storage, box, 0, self.size) @@ -424,6 +459,12 @@ self.order) return SliceArray(0, strides, backstrides, new_shape, self) +class NonWritableArray(ConcreteArray): + def descr_setitem(self, space, w_index, w_value): + raise OperationError(space.w_RuntimeError, space.wrap( + "array is not writable")) + + class SliceArray(BaseConcreteArray): def __init__(self, start, strides, backstrides, shape, parent, dtype=None): self.strides = strides @@ -448,11 +489,12 @@ r = calculate_broadcast_strides(self.get_strides(), self.get_backstrides(), self.get_shape(), shape) - return MultiDimViewIterator(self.parent, + return MultiDimViewIterator(self.parent, self.dtype, self.start, r[0], r[1], shape) if len(self.get_shape()) == 1: - return OneDimViewIterator(self) - return MultiDimViewIterator(self.parent, self.start, + return OneDimViewIterator(self.parent, self.dtype, self.start, + self.get_strides(), self.get_shape()) + return MultiDimViewIterator(self.parent, self.dtype, self.start, self.get_strides(), self.get_backstrides(), self.get_shape()) diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -136,7 +136,7 @@ return self.kind == SIGNEDLTR def is_complex_type(self): - return (self.num == 14 or self.num == 15 or self.num == 16) + return False def is_bool_type(self): return self.kind == BOOLLTR @@ -155,6 +155,19 @@ def get_size(self): return self.itemtype.get_element_size() +class W_ComplexDtype(W_Dtype): + + def __init__(self, itemtype, num, kind, name, char, w_box_type, + alternate_constructors=[], aliases=[], + fields=None, fieldnames=None, native=True, float_type=None): + W_Dtype.__init__(self, itemtype, num, kind, name, char, w_box_type, + alternate_constructors=alternate_constructors, aliases=aliases, + fields=fields, fieldnames=fieldnames, native=native) + self.float_type = float_type + + def is_complex_type(self): + return True + def dtype_from_list(space, w_lst): lst_w = space.listview(w_lst) fields = {} @@ -413,15 +426,16 @@ alternate_constructors=[space.w_float], aliases=["float"], ) - self.w_complex64dtype = W_Dtype( + self.w_complex64dtype = W_ComplexDtype( types.Complex64(), num=14, kind=COMPLEXLTR, name="complex64", char="F", w_box_type = space.gettypefor(interp_boxes.W_Complex64Box), + float_type = self.w_float32dtype, ) - self.w_complex128dtype = W_Dtype( + self.w_complex128dtype = W_ComplexDtype( types.Complex128(), num=15, kind=COMPLEXLTR, @@ -430,6 +444,7 @@ w_box_type = space.gettypefor(interp_boxes.W_Complex128Box), alternate_constructors=[space.w_complex], aliases=["complex"], + float_type = self.w_float64dtype, ) if interp_boxes.long_double_size == 12: self.w_float96dtype = W_Dtype( @@ -443,7 +458,7 @@ ) self.w_longdouble = self.w_float96dtype - self.w_complex192dtype = W_Dtype( + self.w_complex192dtype = W_ComplexDtype( types.Complex192(), num=16, kind=COMPLEXLTR, @@ -452,6 +467,7 @@ w_box_type = space.gettypefor(interp_boxes.W_Complex192Box), alternate_constructors=[space.w_complex], aliases=["clongdouble", "clongfloat"], + float_type = self.w_float96dtype, ) self.w_clongdouble = self.w_complex192dtype @@ -467,7 +483,7 @@ ) self.w_longdouble = self.w_float128dtype - self.w_complex256dtype = W_Dtype( + self.w_complex256dtype = W_ComplexDtype( types.Complex256(), num=16, kind=COMPLEXLTR, @@ -476,6 +492,7 @@ w_box_type = space.gettypefor(interp_boxes.W_Complex256Box), alternate_constructors=[space.w_complex], aliases=["clongdouble", "clongfloat"], + float_type = self.w_float128dtype, ) self.w_clongdouble = self.w_complex256dtype else: diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -235,6 +235,29 @@ def descr_copy(self, space): return W_NDimArray(self.implementation.copy()) + def descr_get_real(self, space): + return W_NDimArray(self.implementation.get_real()) + + def descr_get_imag(self, space): + ret = self.implementation.get_imag() + if ret: + return W_NDimArray(ret) + raise OperationError(space.w_NotImplementedError, + space.wrap('imag not implemented for this dtype')) + + def descr_set_real(self, space, w_value): + # copy (broadcast) values into self + tmp = self.implementation.get_real() + tmp.setslice(space, convert_to_array(space, w_value)) + + def descr_set_imag(self, space, w_value): + # if possible, copy (broadcast) values into self + if not self.get_dtype().is_complex_type(): + raise OperationError(space.w_TypeError, + space.wrap('array does not have imaginary part to set')) + tmp = self.implementation.get_imag() + tmp.setslice(space, convert_to_array(space, w_value)) + def descr_reshape(self, space, args_w): """reshape(...) a.reshape(shape) @@ -387,8 +410,6 @@ descr_neg = _unaryop_impl("negative") descr_abs = _unaryop_impl("absolute") descr_invert = _unaryop_impl("invert") - descr_get_real = _unaryop_impl("real") - descr_get_imag = _unaryop_impl("imag") def descr_nonzero(self, space): if self.get_size() > 1: @@ -635,9 +656,10 @@ swapaxes = interp2app(W_NDimArray.descr_swapaxes), flat = GetSetProperty(W_NDimArray.descr_get_flatiter), item = interp2app(W_NDimArray.descr_item), - real = GetSetProperty(W_NDimArray.descr_get_real), - imag = GetSetProperty(W_NDimArray.descr_get_imag), - + real = GetSetProperty(W_NDimArray.descr_get_real, + W_NDimArray.descr_set_real), + imag = GetSetProperty(W_NDimArray.descr_get_imag, + W_NDimArray.descr_set_imag), __array_interface__ = GetSetProperty(W_NDimArray.descr_array_iface), ) diff --git a/pypy/module/micronumpy/test/test_iter.py b/pypy/module/micronumpy/test/test_iter.py --- a/pypy/module/micronumpy/test/test_iter.py +++ b/pypy/module/micronumpy/test/test_iter.py @@ -12,7 +12,7 @@ strides = [5, 1] backstrides = [x * (y - 1) for x,y in zip(strides, shape)] assert backstrides == [10, 4] - i = MultiDimViewIterator(MockArray, start, strides, backstrides, shape) + i = MultiDimViewIterator(MockArray, None, start, strides, backstrides, shape) i.next() i.next() i.next() @@ -30,7 +30,7 @@ strides = [1, 3] backstrides = [x * (y - 1) for x,y in zip(strides, shape)] assert backstrides == [2, 12] - i = MultiDimViewIterator(MockArray, start, strides, backstrides, shape) + i = MultiDimViewIterator(MockArray, None, start, strides, backstrides, shape) i.next() i.next() i.next() @@ -51,7 +51,7 @@ strides = [5, 1] backstrides = [x * (y - 1) for x,y in zip(strides, shape)] assert backstrides == [10, 4] - i = MultiDimViewIterator(MockArray, start, strides, backstrides, shape) + i = MultiDimViewIterator(MockArray, None, start, strides, backstrides, shape) i.next_skip_x(2) i.next_skip_x(2) i.next_skip_x(2) @@ -74,7 +74,7 @@ strides = [1, 3] backstrides = [x * (y - 1) for x,y in zip(strides, shape)] assert backstrides == [2, 12] - i = MultiDimViewIterator(MockArray, start, strides, backstrides, shape) + i = MultiDimViewIterator(MockArray, None, start, strides, backstrides, shape) i.next_skip_x(2) i.next_skip_x(2) i.next_skip_x(2) diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1169,8 +1169,8 @@ assert (d == [[1, 0, 0], [0, 1, 0], [0, 0, 1]]).all() def test_eye(self): - from _numpypy import eye, array - from _numpypy import int32, float64, dtype + from _numpypy import eye + from _numpypy import int32, dtype a = eye(0) assert len(a) == 0 assert a.dtype == dtype('float64') @@ -1345,6 +1345,42 @@ assert b[0] == 3 assert b[1] == 2 + def test_realimag_views(self): + from _numpypy import arange, array + a = arange(15) + b = a.real + b[5]=50 + assert a[5] == 50 + b = a.imag + assert b[7] == 0 + raises(RuntimeError, 'b[7] = -2') + raises(TypeError, 'a.imag = -2') + a = array(['abc','def'],dtype='S3') + b = a.real + assert a[0] == b[0] + assert a[1] == b[1] + b[1] = 'xyz' + assert a[1] == 'xyz' + assert a.imag[0] == 'abc' + raises(TypeError, 'a.imag = "qop"') + a=array([[1+1j, 2-3j, 4+5j],[-6+7j, 8-9j, -2-1j]]) + assert a.real[0,1] == 2 + a.real[0,1] = -20 + assert a[0,1].real == -20 + b = a.imag + assert b[1,2] == -1 + b[1,2] = 30 + assert a[1,2].imag == 30 + a.real = 13 + assert a[1,1].real == 13 + a=array([1+1j, 2-3j, 4+5j, -6+7j, 8-9j, -2-1j]) + a.real = 13 + assert a[3].real == 13 + a.imag = -5 + a.imag[3] = -10 + assert a[3].imag == -10 + assert a[2].imag == -5 + def test_tolist_scalar(self): from _numpypy import int32, bool_ x = int32(23) @@ -2273,7 +2309,7 @@ assert a[0]['x'] == 'a' def test_stringarray(self): - from _numpypy import array, flexible + from _numpypy import array a = array(['abc'],'S3') assert str(a.dtype) == '|S3' a = array(['abc']) @@ -2293,7 +2329,6 @@ def test_flexible_repr(self): # import overrides str(), repr() for array - from numpypy.core import arrayprint from _numpypy import array a = array(['abc'],'S3') s = repr(a) diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py --- a/pypy/module/pyexpat/interp_pyexpat.py +++ b/pypy/module/pyexpat/interp_pyexpat.py @@ -2,7 +2,7 @@ from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault from pypy.interpreter.error import OperationError -from pypy.rlib import rgc +from pypy.rlib import rgc, jit from pypy.rpython.lltypesystem import rffi, lltype from pypy.rpython.tool import rffi_platform from pypy.translator.tool.cbuild import ExternalCompilationInfo @@ -288,6 +288,7 @@ post_code = '' src = py.code.Source(""" + @jit.jit_callback('XML:%(name)s') def %(name)s_callback(%(first_arg)s, %(args)s): id = rffi.cast(lltype.Signed, %(ll_id)s) userdata = global_storage.get_object(id) diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py --- a/pypy/module/pypyjit/interp_jit.py +++ b/pypy/module/pypyjit/interp_jit.py @@ -6,7 +6,7 @@ from pypy.tool.pairtype import extendabletype from pypy.rlib.rarithmetic import r_uint, intmask from pypy.rlib.jit import JitDriver, hint, we_are_jitted, dont_look_inside -from pypy.rlib import jit, objectmodel +from pypy.rlib import jit from pypy.rlib.jit import current_trace_length, unroll_parameters import pypy.interpreter.pyopcode # for side-effects from pypy.interpreter.error import OperationError, operationerrfmt @@ -97,15 +97,6 @@ is_being_profiled=self.is_being_profiled) return jumpto -callback_jit_driver = JitDriver(greens = ['name'], reds = 'auto') - -def callback_merge_point(name): - callback_jit_driver.jit_merge_point(name=name) - - at callback_jit_driver.inline(callback_merge_point) -def callback_hook(name): - pass - def _get_adapted_tick_counter(): # Normally, the tick counter is decremented by 100 for every # Python opcode. Here, to better support JIT compilation of diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py --- a/pypy/module/signal/interp_signal.py +++ b/pypy/module/signal/interp_signal.py @@ -40,6 +40,7 @@ includes = ['stdlib.h', 'src/signals.h'] if sys.platform != 'win32': includes.append('sys/time.h') +WIN32 = sys.platform == 'win32' cdir = py.path.local(autopath.pypydir).join('translator', 'c') @@ -236,7 +237,10 @@ None -- if an unknown handler is in effect (XXX UNIMPLEMENTED) anything else -- the callable Python object used as a handler """ - check_signum_in_range(space, signum) + if WIN32: + check_signum_exists(space, signum) + else: + check_signum_in_range(space, signum) action = space.check_signal_action if signum in action.handlers_w: return action.handlers_w[signum] diff --git a/pypy/module/signal/test/test_signal.py b/pypy/module/signal/test/test_signal.py --- a/pypy/module/signal/test/test_signal.py +++ b/pypy/module/signal/test/test_signal.py @@ -55,7 +55,7 @@ skip("requires os.kill() and os.getpid()") signal = self.signal # the signal module to test if not hasattr(signal, 'SIGUSR1'): - py.test.skip("requires SIGUSR1 in signal") + skip("requires SIGUSR1 in signal") signum = signal.SIGUSR1 received = [] @@ -156,6 +156,7 @@ import sys if sys.platform == 'win32': raises(ValueError, signal, 42, lambda *args: None) + raises(ValueError, signal, 7, lambda *args: None) else: signal(42, lambda *args: None) signal(42, SIG_DFL) diff --git a/pypy/module/thread/os_thread.py b/pypy/module/thread/os_thread.py --- a/pypy/module/thread/os_thread.py +++ b/pypy/module/thread/os_thread.py @@ -2,6 +2,7 @@ Thread support based on OS-level threads. """ +import os from pypy.module.thread import ll_thread as thread from pypy.module.thread.error import wrap_thread_error from pypy.interpreter.error import OperationError, operationerrfmt @@ -91,14 +92,18 @@ # run! try: bootstrapper.run(space, w_callable, args) - finally: - bootstrapper.nbthreads -= 1 - # clean up space.threadlocals to remove the ExecutionContext - # entry corresponding to the current thread + # done + except Exception, e: + # oups! last-level attempt to recover. try: - space.threadlocals.leave_thread(space) - finally: - thread.gc_thread_die() + STDERR = 2 + os.write(STDERR, "Thread exited with ") + os.write(STDERR, str(e)) + os.write(STDERR, "\n") + except OSError: + pass + bootstrapper.nbthreads -= 1 + thread.gc_thread_die() bootstrap = staticmethod(bootstrap) def acquire(space, w_callable, args): @@ -129,6 +134,9 @@ where = 'thread %d started by ' % ident e.write_unraisable(space, where, w_callable) e.clear(space) + # clean up space.threadlocals to remove the ExecutionContext + # entry corresponding to the current thread + space.threadlocals.leave_thread(space) run = staticmethod(run) bootstrapper = Bootstrapper() diff --git a/pypy/module/thread/test/test_gil.py b/pypy/module/thread/test/test_gil.py --- a/pypy/module/thread/test/test_gil.py +++ b/pypy/module/thread/test/test_gil.py @@ -61,8 +61,9 @@ def bootstrap(): try: runme() - finally: - thread.gc_thread_die() + except Exception, e: + assert 0 + thread.gc_thread_die() def f(): state.data = [] state.datalen1 = 0 diff --git a/pypy/objspace/flow/test/test_objspace.py b/pypy/objspace/flow/test/test_objspace.py --- a/pypy/objspace/flow/test/test_objspace.py +++ b/pypy/objspace/flow/test/test_objspace.py @@ -8,18 +8,17 @@ from pypy.objspace.flow.objspace import FlowObjSpace from pypy.objspace.flow.flowcontext import FlowingError, FlowSpaceFrame from pypy import conftest -from pypy.tool.stdlib_opcode import bytecode_spec, host_bytecode_spec +from pypy.tool.stdlib_opcode import host_bytecode_spec import os import operator is_operator = getattr(operator, 'is_', operator.eq) # it's not there 2.2 @contextmanager -def patching_opcodes(*opcodes): +def patching_opcodes(**opcodes): meth_names = host_bytecode_spec.method_names - opnums = [bytecode_spec.opmap[name] for name in opcodes] old_name = {} - for name, num in zip(opcodes, opnums): + for name, num in opcodes.items(): old_name[num] = meth_names[num] meth_names[num] = name yield @@ -898,7 +897,7 @@ """ Tests code generated by pypy-c compiled with CALL_METHOD bytecode """ - with patching_opcodes('CALL_METHOD', 'LOOKUP_METHOD'): + with patching_opcodes(CALL_METHOD=202, LOOKUP_METHOD=201): class X: def m(self): return 3 @@ -922,7 +921,7 @@ """ Tests code generated by pypy-c compiled with BUILD_LIST_FROM_ARG bytecode """ - with patching_opcodes('BUILD_LIST_FROM_ARG'): + with patching_opcodes(BUILD_LIST_FROM_ARG=203): def f(): return [i for i in "abc"] diff --git a/pypy/rlib/clibffi.py b/pypy/rlib/clibffi.py --- a/pypy/rlib/clibffi.py +++ b/pypy/rlib/clibffi.py @@ -421,6 +421,7 @@ hints={'callback':True})) + at jit.jit_callback("CLIBFFI") def ll_callback(ffi_cif, ll_res, ll_args, ll_userdata): """ Callback specification. ffi_cif - something ffi specific, don't care diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py --- a/pypy/rlib/jit.py +++ b/pypy/rlib/jit.py @@ -302,6 +302,32 @@ pass +def jit_callback(name): + """Use as a decorator for C callback functions, to insert a + jitdriver.jit_merge_point() at the start. Only for callbacks + that typically invoke more app-level Python code. + """ + def decorate(func): + from pypy.tool.sourcetools import compile2 + # + def get_printable_location(): + return name + jitdriver = JitDriver(get_printable_location=get_printable_location, + greens=[], reds='auto', name=name) + # + args = ','.join(['a%d' % i for i in range(func.func_code.co_argcount)]) + source = """def callback_with_jitdriver(%(args)s): + jitdriver.jit_merge_point() + return real_callback(%(args)s)""" % locals() + miniglobals = { + 'jitdriver': jitdriver, + 'real_callback': func, + } + exec compile2(source) in miniglobals + return miniglobals['callback_with_jitdriver'] + return decorate + + # ____________________________________________________________ # VRefs @@ -458,9 +484,8 @@ self.autoreds = True self.reds = [] self.numreds = None # see warmspot.autodetect_jit_markers_redvars - for hook in (get_jitcell_at, set_jitcell_at, get_printable_location, - confirm_enter_jit): - assert hook is None, "reds='auto' is not compatible with JitDriver hooks" + assert confirm_enter_jit is None, ( + "reds='auto' is not compatible with confirm_enter_jit") else: if reds is not None: self.reds = reds diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py --- a/pypy/rlib/objectmodel.py +++ b/pypy/rlib/objectmodel.py @@ -595,16 +595,6 @@ llhelper(rffi.AroundFnPtr, before) llhelper(rffi.AroundFnPtr, after) -def register_around_callback_hook(hook): - """ Register a hook that's called before a callback from C calls RPython. - Primary usage is for JIT to have 'started from' hook. - """ - from pypy.rpython.lltypesystem import rffi - from pypy.rpython.annlowlevel import llhelper - - rffi.aroundstate.callback_hook = hook - llhelper(rffi.CallbackHookPtr, hook) - def is_in_callback(): from pypy.rpython.lltypesystem import rffi return rffi.stackcounter.stacks_counter > 1 diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py --- a/pypy/rpython/lltypesystem/rffi.py +++ b/pypy/rpython/lltypesystem/rffi.py @@ -1,19 +1,19 @@ import py from pypy.annotation import model as annmodel -from pypy.rpython.lltypesystem import lltype, rstr +from pypy.rpython.lltypesystem import lltype from pypy.rpython.lltypesystem import ll2ctypes -from pypy.rpython.lltypesystem.llmemory import cast_adr_to_ptr, cast_ptr_to_adr +from pypy.rpython.lltypesystem.llmemory import cast_ptr_to_adr from pypy.rpython.lltypesystem.llmemory import itemoffsetof, raw_memcopy from pypy.annotation.model import lltype_to_annotation from pypy.tool.sourcetools import func_with_new_name -from pypy.rlib.objectmodel import Symbolic, CDefinedIntSymbolic -from pypy.rlib.objectmodel import keepalive_until_here +from pypy.rlib.objectmodel import Symbolic +from pypy.rlib.objectmodel import keepalive_until_here, enforceargs from pypy.rlib import rarithmetic, rgc from pypy.rpython.extregistry import ExtRegistryEntry from pypy.rlib.unroll import unrolling_iterable from pypy.rpython.tool.rfficache import platform, sizeof_c_type from pypy.translator.tool.cbuild import ExternalCompilationInfo -from pypy.rpython.annlowlevel import llhelper, llstr +from pypy.rpython.annlowlevel import llhelper from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.rstring import StringBuilder, UnicodeBuilder, assert_str0 from pypy.rlib import jit @@ -279,17 +279,8 @@ callable_name = getattr(callable, '__name__', '?') if callbackholder is not None: callbackholder.callbacks[callable] = True - callable_name_descr = str(callable).replace('"', '\\"') args = ', '.join(['a%d' % i for i in range(len(TP.TO.ARGS))]) source = py.code.Source(r""" - def inner_wrapper(%(args)s): - if aroundstate is not None: - callback_hook = aroundstate.callback_hook - if callback_hook: - callback_hook(llstr("%(callable_name_descr)s")) - return callable(%(args)s) - inner_wrapper._never_inline_ = True - def wrapper(%(args)s): # no *args - no GIL for mallocing the tuple llop.gc_stack_bottom(lltype.Void) # marker for trackgcroot.py if aroundstate is not None: @@ -299,7 +290,7 @@ # from now on we hold the GIL stackcounter.stacks_counter += 1 try: - result = inner_wrapper(%(args)s) + result = callable(%(args)s) except Exception, e: os.write(2, "Warning: uncaught exception in callback: %%s %%s\n" %% @@ -321,7 +312,6 @@ miniglobals = locals().copy() miniglobals['Exception'] = Exception miniglobals['os'] = os - miniglobals['llstr'] = llstr miniglobals['we_are_translated'] = we_are_translated miniglobals['stackcounter'] = stackcounter exec source.compile() in miniglobals @@ -329,11 +319,8 @@ _make_wrapper_for._annspecialcase_ = 'specialize:memo' AroundFnPtr = lltype.Ptr(lltype.FuncType([], lltype.Void)) -CallbackHookPtr = lltype.Ptr(lltype.FuncType([lltype.Ptr(rstr.STR)], lltype.Void)) class AroundState: - callback_hook = None - def _cleanup_(self): self.before = None # or a regular RPython function self.after = None # or a regular RPython function @@ -795,6 +782,7 @@ # (char*, str, int, int) -> None @jit.dont_look_inside + @enforceargs(None, None, int, int) def str_from_buffer(raw_buf, gc_buf, allocated_size, needed_size): """ Converts from a pair returned by alloc_buffer to a high-level string. @@ -833,6 +821,7 @@ lltype.free(raw_buf, flavor='raw') # char* -> str, with an upper bound on the length in case there is no \x00 + @enforceargs(None, int) def charp2strn(cp, maxlen): b = builder_class(maxlen) i = 0 diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py --- a/pypy/rpython/memory/gctransform/framework.py +++ b/pypy/rpython/memory/gctransform/framework.py @@ -979,6 +979,7 @@ assert self.translator.config.translation.thread if hasattr(self.root_walker, 'thread_run_ptr'): livevars = self.push_roots(hop) + assert not livevars, "live GC var around %s!" % (hop.spaceop,) hop.genop("direct_call", [self.root_walker.thread_run_ptr]) self.pop_roots(hop, livevars) @@ -993,6 +994,7 @@ assert self.translator.config.translation.thread if hasattr(self.root_walker, 'thread_die_ptr'): livevars = self.push_roots(hop) + assert not livevars, "live GC var around %s!" % (hop.spaceop,) hop.genop("direct_call", [self.root_walker.thread_die_ptr]) self.pop_roots(hop, livevars) diff --git a/pypy/rpython/memory/gctransform/shadowstack.py b/pypy/rpython/memory/gctransform/shadowstack.py --- a/pypy/rpython/memory/gctransform/shadowstack.py +++ b/pypy/rpython/memory/gctransform/shadowstack.py @@ -326,6 +326,8 @@ self.gcdata.root_stack_top = llmemory.NULL # to detect missing restore def forget_current_state(self): + ll_assert(self.gcdata.root_stack_base == self.gcdata.root_stack_top, + "forget_current_state: shadowstack not empty!") if self.unused_full_stack: llmemory.raw_free(self.unused_full_stack) self.unused_full_stack = self.gcdata.root_stack_base diff --git a/pypy/translator/driver.py b/pypy/translator/driver.py --- a/pypy/translator/driver.py +++ b/pypy/translator/driver.py @@ -362,7 +362,7 @@ @taskdef([RTYPE], "JIT compiler generation") def task_pyjitpl_lltype(self): """ Generate bytecodes for JIT and flow the JIT helper functions - ootype version + lltype version """ get_policy = self.extra['jitpolicy'] self.jitpolicy = get_policy(self) diff --git a/pypy/translator/goal/targetpypystandalone.py b/pypy/translator/goal/targetpypystandalone.py --- a/pypy/translator/goal/targetpypystandalone.py +++ b/pypy/translator/goal/targetpypystandalone.py @@ -28,11 +28,6 @@ w_call_startup_gateway = space.wrap(gateway.interp2app(call_startup)) withjit = space.config.objspace.usemodules.pypyjit - if withjit: - from pypy.module.pypyjit.interp_jit import callback_hook - from pypy.rlib import objectmodel - objectmodel.register_around_callback_hook(callback_hook) - def entry_point(argv): if withjit: from pypy.jit.backend.hlinfo import highleveljitinfo From noreply at buildbot.pypy.org Mon Jan 14 19:17:30 2013 From: noreply at buildbot.pypy.org (hakanardo) Date: Mon, 14 Jan 2013 19:17:30 +0100 (CET) Subject: [pypy-commit] pypy jit-usable_retrace_3: random loop tests in progress: basic arithmetic Message-ID: <20130114181730.1F6C71C0DCB@cobra.cs.uni-duesseldorf.de> Author: Hakan Ardo Branch: jit-usable_retrace_3 Changeset: r60058:04dcba7544c1 Date: 2013-01-13 17:24 +0100 http://bitbucket.org/pypy/pypy/changeset/04dcba7544c1/ Log: random loop tests in progress: basic arithmetic diff --git a/pypy/jit/metainterp/test/test_random_loops.py b/pypy/jit/metainterp/test/test_random_loops.py new file mode 100644 --- /dev/null +++ b/pypy/jit/metainterp/test/test_random_loops.py @@ -0,0 +1,86 @@ +from pypy.jit.metainterp.test.support import LLJitMixin +from pypy.rlib.jit import JitDriver + +class IntBox(object): + def __init__(self, val): + self.val = val + + def value(self): + return self.val + + def add(self, other): + return IntBox(self.value() + other.value()) + + def sub(self, other): + return IntBox(self.value() - other.value()) + +class RandomLoopBase(object): + def check(self, bytecode, args=(0,0,0,0,0), **kwargs): + myjitdriver = JitDriver(greens = ['pc'], reds = ['a', 'b', 'c', 'd', 'e', 'value', 'prev']) + def interpreter(_a, _b, _c, _d, _e): + pc = 0 + value = prev = IntBox(0) + a = IntBox(_a) + b = IntBox(_b) + c = IntBox(_c) + d = IntBox(_d) + e = IntBox(_e) + while pc < len(bytecode): + myjitdriver.jit_merge_point(pc=pc, a=a, b=b, c=c, d=d, e=e, value=value, prev=prev) + op = bytecode[pc] + current = value + + if '0' <= op <= '9': + value = IntBox(ord(op) - ord('0')) + elif op == 'a': + value = a + elif op == 'b': + value = b + elif op == 'c': + value = c + elif op == 'd': + value = d + elif op == 'e': + value = e + elif op == 'A': + a = value + elif op == 'B': + b = value + elif op == 'C': + c = value + elif op == 'D': + d = value + elif op == 'E': + e = value + elif op == '+': + value = prev.add(value) + elif op == '-': + value = prev.sub(value) + else: + assert False + + prev = current + pc += 1 + return a.value(), b.value(), c.value(), d.value(), e.value() + + obj = self.meta_interp(interpreter, args)._obj + res = {'a': obj.item0, 'b': obj.item1, 'c': obj.item2, 'd': obj.item3, 'e': obj.item4} + obj = interpreter(*args) + expected = {'a': obj[0], 'b': obj[1], 'c': obj[2], 'd': obj[3], 'e': obj[4]} + assert res == expected + + for var, val in kwargs.items(): + assert res[var] == val + return res + + + +class BaseTests(RandomLoopBase): + def test_basic(self): + self.check('1A2B3C4D5E', a=1, b=2, c=3, d=4, e=5) + self.check('1', [6,7,8,9,0], a=6, b=7, c=8, d=9, e=0) + self.check('1a+A2b+B3c+C4d+D5e+E', [6,7,8,9,0], a=7, b=9, c=11, d=13, e=5) + self.check('ea+Eeb+Eec+Eed+E', [6,7,8,9,0], a=6, b=7, c=8, d=9, e=30) + +class TestLLtype(BaseTests, LLJitMixin): + pass From noreply at buildbot.pypy.org Mon Jan 14 19:17:31 2013 From: noreply at buildbot.pypy.org (hakanardo) Date: Mon, 14 Jan 2013 19:17:31 +0100 (CET) Subject: [pypy-commit] pypy jit-usable_retrace_3: while loops Message-ID: <20130114181731.511701C0DCB@cobra.cs.uni-duesseldorf.de> Author: Hakan Ardo Branch: jit-usable_retrace_3 Changeset: r60059:d86fb2edebc7 Date: 2013-01-13 18:19 +0100 http://bitbucket.org/pypy/pypy/changeset/d86fb2edebc7/ Log: while loops diff --git a/pypy/jit/metainterp/test/test_random_loops.py b/pypy/jit/metainterp/test/test_random_loops.py --- a/pypy/jit/metainterp/test/test_random_loops.py +++ b/pypy/jit/metainterp/test/test_random_loops.py @@ -14,9 +14,16 @@ def sub(self, other): return IntBox(self.value() - other.value()) +class UnknonwOpCode(Exception): + pass + class RandomLoopBase(object): def check(self, bytecode, args=(0,0,0,0,0), **kwargs): - myjitdriver = JitDriver(greens = ['pc'], reds = ['a', 'b', 'c', 'd', 'e', 'value', 'prev']) + offsets = self.offsets(bytecode) + def get_printable_location(pc): + return bytecode[pc] + myjitdriver = JitDriver(greens = ['pc'], reds = ['a', 'b', 'c', 'd', 'e', 'value', 'prev'], + get_printable_location=get_printable_location) def interpreter(_a, _b, _c, _d, _e): pc = 0 value = prev = IntBox(0) @@ -56,8 +63,14 @@ value = prev.add(value) elif op == '-': value = prev.sub(value) + elif op == '{': + pass + elif op == '}': + if value.value(): + pc -= offsets[pc] + myjitdriver.can_enter_jit(pc=pc, a=a, b=b, c=c, d=d, e=e, value=value, prev=prev) else: - assert False + raise UnknonwOpCode prev = current pc += 1 @@ -73,6 +86,18 @@ assert res[var] == val return res + def offsets(self, bytecode): + offsets = [0] * len(bytecode) + stack = [] + for pc, op in enumerate(bytecode): + if op in '{[(': + stack.append((pc, op)) + elif op in ')]}': + start_pc, start_op = stack.pop() + assert start_op + op in ('()', '[]', '{}') + offsets[start_pc] = offsets[pc] = pc - start_pc + return offsets + class BaseTests(RandomLoopBase): @@ -82,5 +107,8 @@ self.check('1a+A2b+B3c+C4d+D5e+E', [6,7,8,9,0], a=7, b=9, c=11, d=13, e=5) self.check('ea+Eeb+Eec+Eed+E', [6,7,8,9,0], a=6, b=7, c=8, d=9, e=30) + def test_loop(self): + self.check('0A9B{ab+Ab1-Bb}', a=45) + class TestLLtype(BaseTests, LLJitMixin): pass From noreply at buildbot.pypy.org Mon Jan 14 19:17:32 2013 From: noreply at buildbot.pypy.org (hakanardo) Date: Mon, 14 Jan 2013 19:17:32 +0100 (CET) Subject: [pypy-commit] pypy jit-usable_retrace_3: conditionals Message-ID: <20130114181732.73AFD1C0DCB@cobra.cs.uni-duesseldorf.de> Author: Hakan Ardo Branch: jit-usable_retrace_3 Changeset: r60060:c7d96f0b3ac3 Date: 2013-01-13 18:33 +0100 http://bitbucket.org/pypy/pypy/changeset/c7d96f0b3ac3/ Log: conditionals diff --git a/pypy/jit/metainterp/test/test_random_loops.py b/pypy/jit/metainterp/test/test_random_loops.py --- a/pypy/jit/metainterp/test/test_random_loops.py +++ b/pypy/jit/metainterp/test/test_random_loops.py @@ -14,6 +14,16 @@ def sub(self, other): return IntBox(self.value() - other.value()) + def gt(self, other): + return IntBox(self.value() > other.value()) + + def lt(self, other): + return IntBox(self.value() < other.value()) + + def eq(self, other): + return IntBox(self.value() == other.value()) + + class UnknonwOpCode(Exception): pass @@ -63,12 +73,25 @@ value = prev.add(value) elif op == '-': value = prev.sub(value) + elif op == '>': + value = prev.gt(value) + elif op == '<': + value = prev.lt(value) + elif op == '=': + value = prev.eq(value) elif op == '{': pass elif op == '}': if value.value(): pc -= offsets[pc] myjitdriver.can_enter_jit(pc=pc, a=a, b=b, c=c, d=d, e=e, value=value, prev=prev) + elif op == '(': + if not value.value(): + value = IntBox(1) + pc += offsets[pc] + myjitdriver.can_enter_jit(pc=pc, a=a, b=b, c=c, d=d, e=e, value=value, prev=prev) + elif op == ')': + value = IntBox(0) else: raise UnknonwOpCode @@ -110,5 +133,8 @@ def test_loop(self): self.check('0A9B{ab+Ab1-Bb}', a=45) + def test_conditional(self): + self.check('0A0C9B{b4<(a1+A)(c1+C)b1-Bb}', c=6, a=3) + class TestLLtype(BaseTests, LLJitMixin): pass From noreply at buildbot.pypy.org Mon Jan 14 19:17:33 2013 From: noreply at buildbot.pypy.org (hakanardo) Date: Mon, 14 Jan 2013 19:17:33 +0100 (CET) Subject: [pypy-commit] pypy jit-usable_retrace_3: break statement Message-ID: <20130114181733.D6C5A1C0DCB@cobra.cs.uni-duesseldorf.de> Author: Hakan Ardo Branch: jit-usable_retrace_3 Changeset: r60061:960797b5b0fc Date: 2013-01-13 18:53 +0100 http://bitbucket.org/pypy/pypy/changeset/960797b5b0fc/ Log: break statement diff --git a/pypy/jit/metainterp/test/test_random_loops.py b/pypy/jit/metainterp/test/test_random_loops.py --- a/pypy/jit/metainterp/test/test_random_loops.py +++ b/pypy/jit/metainterp/test/test_random_loops.py @@ -32,7 +32,7 @@ offsets = self.offsets(bytecode) def get_printable_location(pc): return bytecode[pc] - myjitdriver = JitDriver(greens = ['pc'], reds = ['a', 'b', 'c', 'd', 'e', 'value', 'prev'], + myjitdriver = JitDriver(greens = ['pc'], reds = ['a', 'b', 'c', 'd', 'e', 'value', 'prev', 'loop_stack'], get_printable_location=get_printable_location) def interpreter(_a, _b, _c, _d, _e): pc = 0 @@ -42,8 +42,10 @@ c = IntBox(_c) d = IntBox(_d) e = IntBox(_e) + loop_stack = [] while pc < len(bytecode): - myjitdriver.jit_merge_point(pc=pc, a=a, b=b, c=c, d=d, e=e, value=value, prev=prev) + myjitdriver.jit_merge_point(pc=pc, a=a, b=b, c=c, d=d, e=e, value=value, prev=prev, + loop_stack=loop_stack) op = bytecode[pc] current = value @@ -80,18 +82,27 @@ elif op == '=': value = prev.eq(value) elif op == '{': - pass + loop_stack.append(pc) elif op == '}': if value.value(): pc -= offsets[pc] - myjitdriver.can_enter_jit(pc=pc, a=a, b=b, c=c, d=d, e=e, value=value, prev=prev) + myjitdriver.can_enter_jit(pc=pc, a=a, b=b, c=c, d=d, e=e, value=value, prev=prev, + loop_stack=loop_stack) + else: + loop_stack.pop() + elif op == 'x': + pc = loop_stack.pop() + pc += offsets[pc] elif op == '(': if not value.value(): value = IntBox(1) pc += offsets[pc] - myjitdriver.can_enter_jit(pc=pc, a=a, b=b, c=c, d=d, e=e, value=value, prev=prev) + myjitdriver.can_enter_jit(pc=pc, a=a, b=b, c=c, d=d, e=e, value=value, prev=prev, + loop_stack=loop_stack) elif op == ')': value = IntBox(0) + elif op in ' \n': + pass else: raise UnknonwOpCode @@ -99,14 +110,15 @@ pc += 1 return a.value(), b.value(), c.value(), d.value(), e.value() + obj = interpreter(*args) + expected = {'a': obj[0], 'b': obj[1], 'c': obj[2], 'd': obj[3], 'e': obj[4]} + for var, val in kwargs.items(): + assert expected[var] == val + obj = self.meta_interp(interpreter, args)._obj res = {'a': obj.item0, 'b': obj.item1, 'c': obj.item2, 'd': obj.item3, 'e': obj.item4} - obj = interpreter(*args) - expected = {'a': obj[0], 'b': obj[1], 'c': obj[2], 'd': obj[3], 'e': obj[4]} assert res == expected - for var, val in kwargs.items(): - assert res[var] == val return res def offsets(self, bytecode): @@ -136,5 +148,21 @@ def test_conditional(self): self.check('0A0C9B{b4<(a1+A)(c1+C)b1-Bb}', c=6, a=3) + def test_break(self): + self.check('0A9B{ab+Ab1-Bb0=(x)1}', a=45) + + def test_nested(self): + self.check('''0A + 9B{ + 9C{ + ab+A + ac+A + c1-C + c0= (x) + 1} + b1-B + b0= (x) + 1}''', a=810) + class TestLLtype(BaseTests, LLJitMixin): pass From noreply at buildbot.pypy.org Mon Jan 14 19:17:35 2013 From: noreply at buildbot.pypy.org (hakanardo) Date: Mon, 14 Jan 2013 19:17:35 +0100 (CET) Subject: [pypy-commit] pypy jit-usable_retrace_3: dont can_enter_jit on forward jumps Message-ID: <20130114181735.259501C0DCB@cobra.cs.uni-duesseldorf.de> Author: Hakan Ardo Branch: jit-usable_retrace_3 Changeset: r60062:ad1ed0047199 Date: 2013-01-13 20:58 +0100 http://bitbucket.org/pypy/pypy/changeset/ad1ed0047199/ Log: dont can_enter_jit on forward jumps diff --git a/pypy/jit/metainterp/test/test_random_loops.py b/pypy/jit/metainterp/test/test_random_loops.py --- a/pypy/jit/metainterp/test/test_random_loops.py +++ b/pypy/jit/metainterp/test/test_random_loops.py @@ -85,9 +85,11 @@ loop_stack.append(pc) elif op == '}': if value.value(): - pc -= offsets[pc] + pc -= offsets[pc] - 1 + prev = current myjitdriver.can_enter_jit(pc=pc, a=a, b=b, c=c, d=d, e=e, value=value, prev=prev, loop_stack=loop_stack) + continue else: loop_stack.pop() elif op == 'x': @@ -97,8 +99,6 @@ if not value.value(): value = IntBox(1) pc += offsets[pc] - myjitdriver.can_enter_jit(pc=pc, a=a, b=b, c=c, d=d, e=e, value=value, prev=prev, - loop_stack=loop_stack) elif op == ')': value = IntBox(0) elif op in ' \n': From noreply at buildbot.pypy.org Mon Jan 14 19:17:36 2013 From: noreply at buildbot.pypy.org (hakanardo) Date: Mon, 14 Jan 2013 19:17:36 +0100 (CET) Subject: [pypy-commit] pypy jit-usable_retrace_3: kill whitespaces in bytecode Message-ID: <20130114181736.57CBF1C0DCB@cobra.cs.uni-duesseldorf.de> Author: Hakan Ardo Branch: jit-usable_retrace_3 Changeset: r60063:1537f18caa0c Date: 2013-01-13 21:02 +0100 http://bitbucket.org/pypy/pypy/changeset/1537f18caa0c/ Log: kill whitespaces in bytecode diff --git a/pypy/jit/metainterp/test/test_random_loops.py b/pypy/jit/metainterp/test/test_random_loops.py --- a/pypy/jit/metainterp/test/test_random_loops.py +++ b/pypy/jit/metainterp/test/test_random_loops.py @@ -1,5 +1,6 @@ from pypy.jit.metainterp.test.support import LLJitMixin from pypy.rlib.jit import JitDriver +import re class IntBox(object): def __init__(self, val): @@ -29,6 +30,7 @@ class RandomLoopBase(object): def check(self, bytecode, args=(0,0,0,0,0), **kwargs): + bytecode = re.subn('\s', '', bytecode)[0] offsets = self.offsets(bytecode) def get_printable_location(pc): return bytecode[pc] @@ -101,8 +103,6 @@ pc += offsets[pc] elif op == ')': value = IntBox(0) - elif op in ' \n': - pass else: raise UnknonwOpCode From noreply at buildbot.pypy.org Mon Jan 14 19:17:37 2013 From: noreply at buildbot.pypy.org (hakanardo) Date: Mon, 14 Jan 2013 19:17:37 +0100 (CET) Subject: [pypy-commit] pypy jit-usable_retrace_3: limit number of iterations Message-ID: <20130114181737.80E6F1C0DCB@cobra.cs.uni-duesseldorf.de> Author: Hakan Ardo Branch: jit-usable_retrace_3 Changeset: r60064:06be0d3d5f8a Date: 2013-01-13 22:03 +0100 http://bitbucket.org/pypy/pypy/changeset/06be0d3d5f8a/ Log: limit number of iterations diff --git a/pypy/jit/metainterp/test/test_random_loops.py b/pypy/jit/metainterp/test/test_random_loops.py --- a/pypy/jit/metainterp/test/test_random_loops.py +++ b/pypy/jit/metainterp/test/test_random_loops.py @@ -29,14 +29,19 @@ pass class RandomLoopBase(object): - def check(self, bytecode, args=(0,0,0,0,0), **kwargs): + def check(self, bytecode, args=(0,0,0,0,0), max_back_jumps=-1, **kwargs): bytecode = re.subn('\s', '', bytecode)[0] offsets = self.offsets(bytecode) + assert len(args) == 5 + for n in kwargs.keys(): + assert n in 'abcde' + args = tuple(args) + (max_back_jumps,) def get_printable_location(pc): return bytecode[pc] - myjitdriver = JitDriver(greens = ['pc'], reds = ['a', 'b', 'c', 'd', 'e', 'value', 'prev', 'loop_stack'], + myjitdriver = JitDriver(greens = ['pc'], reds = ['max_back_jumps', 'a', 'b', 'c', 'd', 'e', + 'value', 'prev', 'loop_stack'], get_printable_location=get_printable_location) - def interpreter(_a, _b, _c, _d, _e): + def interpreter(_a, _b, _c, _d, _e, max_back_jumps): pc = 0 value = prev = IntBox(0) a = IntBox(_a) @@ -47,7 +52,7 @@ loop_stack = [] while pc < len(bytecode): myjitdriver.jit_merge_point(pc=pc, a=a, b=b, c=c, d=d, e=e, value=value, prev=prev, - loop_stack=loop_stack) + loop_stack=loop_stack, max_back_jumps=max_back_jumps) op = bytecode[pc] current = value @@ -89,8 +94,12 @@ if value.value(): pc -= offsets[pc] - 1 prev = current + if max_back_jumps > 0: + max_back_jumps -= 1 + if not max_back_jumps: + break myjitdriver.can_enter_jit(pc=pc, a=a, b=b, c=c, d=d, e=e, value=value, prev=prev, - loop_stack=loop_stack) + loop_stack=loop_stack, max_back_jumps=max_back_jumps) continue else: loop_stack.pop() @@ -134,7 +143,6 @@ return offsets - class BaseTests(RandomLoopBase): def test_basic(self): self.check('1A2B3C4D5E', a=1, b=2, c=3, d=4, e=5) @@ -164,5 +172,8 @@ b0= (x) 1}''', a=810) + def test_jump_limit(self): + self.check('0A{a1+A1}', max_back_jumps=10, a=10) + class TestLLtype(BaseTests, LLJitMixin): pass From noreply at buildbot.pypy.org Mon Jan 14 19:17:38 2013 From: noreply at buildbot.pypy.org (hakanardo) Date: Mon, 14 Jan 2013 19:17:38 +0100 (CET) Subject: [pypy-commit] pypy jit-usable_retrace_3: generate random bytecodes Message-ID: <20130114181738.9EDAD1C0DCB@cobra.cs.uni-duesseldorf.de> Author: Hakan Ardo Branch: jit-usable_retrace_3 Changeset: r60065:9f0bb0453773 Date: 2013-01-14 18:47 +0100 http://bitbucket.org/pypy/pypy/changeset/9f0bb0453773/ Log: generate random bytecodes diff --git a/pypy/jit/metainterp/test/test_random_loops.py b/pypy/jit/metainterp/test/test_random_loops.py --- a/pypy/jit/metainterp/test/test_random_loops.py +++ b/pypy/jit/metainterp/test/test_random_loops.py @@ -1,5 +1,6 @@ from pypy.jit.metainterp.test.support import LLJitMixin from pypy.rlib.jit import JitDriver +from random import choice, randrange import re class IntBox(object): @@ -30,6 +31,7 @@ class RandomLoopBase(object): def check(self, bytecode, args=(0,0,0,0,0), max_back_jumps=-1, **kwargs): + print 'check:', bytecode, args, max_back_jumps bytecode = re.subn('\s', '', bytecode)[0] offsets = self.offsets(bytecode) assert len(args) == 5 @@ -142,6 +144,53 @@ offsets[start_pc] = offsets[pc] = pc - start_pc return offsets + def variable(self): + return choice('abcde') + + def constant(self): + return choice('0123456789') + + def value(self): + return choice([self.variable, self.constant])() + + def binop(self): + return self.value() + self.value() + choice('+-') + self.variable().upper() + + def break_loop(self): + return 'x' + + def compare(self): + return self.value() + self.value() + choice('<>=') + + def do_while(self): + self.levels -= 1 + code = '{' + self.block() + self.compare() + '}' + self.levels += 1 + return code + + def if_block(self): + self.levels -= 1 + code = self.compare() + '(' + self.block() + ')' + self.levels += 1 + return code + + def if_else_block(self): + self.levels -= 1 + code = self.compare() + '(' + self.block() + ')(' + self.block() + ')' + self.levels += 1 + return code + + def block(self): + stmts = [self.break_loop] + [self.binop] * 5 + if self.levels: + stmts += [self.do_while, self.if_block, self.if_else_block] + return ''.join(choice(stmts)() for i in xrange(randrange(self.max_stmts_per_block))) + + def random_loop(self, max_stmts_per_block=10, max_levels=5): + self.max_stmts_per_block = max_stmts_per_block + self.levels = max_levels + return '{{' + self.block() + '1}1}' + class BaseTests(RandomLoopBase): def test_basic(self): @@ -175,5 +224,10 @@ def test_jump_limit(self): self.check('0A{a1+A1}', max_back_jumps=10, a=10) + def test_random_bytecode(self): + for i in xrange(1000): + yield self.check, self.random_loop(), [randrange(100) for i in xrange(5)], 1000 + + class TestLLtype(BaseTests, LLJitMixin): pass From noreply at buildbot.pypy.org Mon Jan 14 19:17:39 2013 From: noreply at buildbot.pypy.org (hakanardo) Date: Mon, 14 Jan 2013 19:17:39 +0100 (CET) Subject: [pypy-commit] pypy jit-usable_retrace_3: first failing case found Message-ID: <20130114181739.C663D1C0DCB@cobra.cs.uni-duesseldorf.de> Author: Hakan Ardo Branch: jit-usable_retrace_3 Changeset: r60066:5b47a7e05437 Date: 2013-01-14 18:51 +0100 http://bitbucket.org/pypy/pypy/changeset/5b47a7e05437/ Log: first failing case found diff --git a/pypy/jit/metainterp/test/test_random_loops.py b/pypy/jit/metainterp/test/test_random_loops.py --- a/pypy/jit/metainterp/test/test_random_loops.py +++ b/pypy/jit/metainterp/test/test_random_loops.py @@ -228,6 +228,8 @@ for i in xrange(1000): yield self.check, self.random_loop(), [randrange(100) for i in xrange(5)], 1000 + def test_failure1(self): + self.check('{{c3+A{cc+Dda<({2b=}c7=(cd-Aae-Exe8+Ax)aa+Ab5-D2a-Dba=(0d+Be7-D6e+Bd3-A)0e+D)(a0=(x{79>})(0e>(x)9c+Ce3-Ccb>(5d=(08-Axx)da+B4d<(04-B7d+Eba+Axx09+B15-E))()33-Bc9<()({xba+Ec1+C0a=}4c+C79+Dxda<(0c+A)(5e+D9b+C2d-Bcc+D8b-A99-B3c-De4-Cc9+C)58-Bcc-Ae1-Be9-D)ae+A)xcc=(3c=(d5>(23+Bad+Bx8c-De5-Dac-Cd3-Cea+Ax)(xx)33-A{dc-D0b-Ab8-Cb1+Bd1+A28=}65+Aba<(x5c+Axba-C57+A3b-Deb+C)(cd+Bec+B30+Bbb+D45-De4-C)b8-Eae<(aa+Ce4-E1a+Dxa1+E)(d5-Ea2-Ex62+Bxx6a-D)be+B)c6+C3e+A)(5e+C4e-Ec9+Bx26-Ca8=()x)a3<(x10-Cd8-Aba-Ex{5c=})55+C)xd2-Ax98>}1}1}', [49, 30, 28, 93, 22], 1000) class TestLLtype(BaseTests, LLJitMixin): pass From noreply at buildbot.pypy.org Mon Jan 14 19:17:41 2013 From: noreply at buildbot.pypy.org (hakanardo) Date: Mon, 14 Jan 2013 19:17:41 +0100 (CET) Subject: [pypy-commit] pypy jit-usable_retrace_3: reorder Message-ID: <20130114181741.050701C0DCB@cobra.cs.uni-duesseldorf.de> Author: Hakan Ardo Branch: jit-usable_retrace_3 Changeset: r60067:e3289d43a141 Date: 2013-01-14 18:53 +0100 http://bitbucket.org/pypy/pypy/changeset/e3289d43a141/ Log: reorder diff --git a/pypy/jit/metainterp/test/test_random_loops.py b/pypy/jit/metainterp/test/test_random_loops.py --- a/pypy/jit/metainterp/test/test_random_loops.py +++ b/pypy/jit/metainterp/test/test_random_loops.py @@ -224,12 +224,12 @@ def test_jump_limit(self): self.check('0A{a1+A1}', max_back_jumps=10, a=10) + def test_failure1(self): + self.check('{{c3+A{cc+Dda<({2b=}c7=(cd-Aae-Exe8+Ax)aa+Ab5-D2a-Dba=(0d+Be7-D6e+Bd3-A)0e+D)(a0=(x{79>})(0e>(x)9c+Ce3-Ccb>(5d=(08-Axx)da+B4d<(04-B7d+Eba+Axx09+B15-E))()33-Bc9<()({xba+Ec1+C0a=}4c+C79+Dxda<(0c+A)(5e+D9b+C2d-Bcc+D8b-A99-B3c-De4-Cc9+C)58-Bcc-Ae1-Be9-D)ae+A)xcc=(3c=(d5>(23+Bad+Bx8c-De5-Dac-Cd3-Cea+Ax)(xx)33-A{dc-D0b-Ab8-Cb1+Bd1+A28=}65+Aba<(x5c+Axba-C57+A3b-Deb+C)(cd+Bec+B30+Bbb+D45-De4-C)b8-Eae<(aa+Ce4-E1a+Dxa1+E)(d5-Ea2-Ex62+Bxx6a-D)be+B)c6+C3e+A)(5e+C4e-Ec9+Bx26-Ca8=()x)a3<(x10-Cd8-Aba-Ex{5c=})55+C)xd2-Ax98>}1}1}', [49, 30, 28, 93, 22], 1000) + def test_random_bytecode(self): for i in xrange(1000): yield self.check, self.random_loop(), [randrange(100) for i in xrange(5)], 1000 - def test_failure1(self): - self.check('{{c3+A{cc+Dda<({2b=}c7=(cd-Aae-Exe8+Ax)aa+Ab5-D2a-Dba=(0d+Be7-D6e+Bd3-A)0e+D)(a0=(x{79>})(0e>(x)9c+Ce3-Ccb>(5d=(08-Axx)da+B4d<(04-B7d+Eba+Axx09+B15-E))()33-Bc9<()({xba+Ec1+C0a=}4c+C79+Dxda<(0c+A)(5e+D9b+C2d-Bcc+D8b-A99-B3c-De4-Cc9+C)58-Bcc-Ae1-Be9-D)ae+A)xcc=(3c=(d5>(23+Bad+Bx8c-De5-Dac-Cd3-Cea+Ax)(xx)33-A{dc-D0b-Ab8-Cb1+Bd1+A28=}65+Aba<(x5c+Axba-C57+A3b-Deb+C)(cd+Bec+B30+Bbb+D45-De4-C)b8-Eae<(aa+Ce4-E1a+Dxa1+E)(d5-Ea2-Ex62+Bxx6a-D)be+B)c6+C3e+A)(5e+C4e-Ec9+Bx26-Ca8=()x)a3<(x10-Cd8-Aba-Ex{5c=})55+C)xd2-Ax98>}1}1}', [49, 30, 28, 93, 22], 1000) - class TestLLtype(BaseTests, LLJitMixin): pass From noreply at buildbot.pypy.org Mon Jan 14 19:17:42 2013 From: noreply at buildbot.pypy.org (hakanardo) Date: Mon, 14 Jan 2013 19:17:42 +0100 (CET) Subject: [pypy-commit] pypy jit-usable_retrace_3: simplify a bit by making the backward jump unconditional and produce normal while loops Message-ID: <20130114181742.2CD371C0DCB@cobra.cs.uni-duesseldorf.de> Author: Hakan Ardo Branch: jit-usable_retrace_3 Changeset: r60068:b07a0bce0b61 Date: 2013-01-14 19:05 +0100 http://bitbucket.org/pypy/pypy/changeset/b07a0bce0b61/ Log: simplify a bit by making the backward jump unconditional and produce normal while loops diff --git a/pypy/jit/metainterp/test/test_random_loops.py b/pypy/jit/metainterp/test/test_random_loops.py --- a/pypy/jit/metainterp/test/test_random_loops.py +++ b/pypy/jit/metainterp/test/test_random_loops.py @@ -90,21 +90,20 @@ value = prev.lt(value) elif op == '=': value = prev.eq(value) + elif op == 'n': + value = IntBox(not value.value()) elif op == '{': loop_stack.append(pc) elif op == '}': - if value.value(): - pc -= offsets[pc] - 1 - prev = current - if max_back_jumps > 0: - max_back_jumps -= 1 - if not max_back_jumps: - break - myjitdriver.can_enter_jit(pc=pc, a=a, b=b, c=c, d=d, e=e, value=value, prev=prev, - loop_stack=loop_stack, max_back_jumps=max_back_jumps) - continue - else: - loop_stack.pop() + pc -= offsets[pc] - 1 + prev = current + if max_back_jumps > 0: + max_back_jumps -= 1 + if not max_back_jumps: + break + myjitdriver.can_enter_jit(pc=pc, a=a, b=b, c=c, d=d, e=e, value=value, prev=prev, + loop_stack=loop_stack, max_back_jumps=max_back_jumps) + continue elif op == 'x': pc = loop_stack.pop() pc += offsets[pc] @@ -162,9 +161,9 @@ def compare(self): return self.value() + self.value() + choice('<>=') - def do_while(self): + def while_loop(self): self.levels -= 1 - code = '{' + self.block() + self.compare() + '}' + code = '{' + self.compare() + 'n(x)' + self.block() + '}' self.levels += 1 return code @@ -183,7 +182,7 @@ def block(self): stmts = [self.break_loop] + [self.binop] * 5 if self.levels: - stmts += [self.do_while, self.if_block, self.if_else_block] + stmts += [self.while_loop, self.if_block, self.if_else_block] return ''.join(choice(stmts)() for i in xrange(randrange(self.max_stmts_per_block))) def random_loop(self, max_stmts_per_block=10, max_levels=5): @@ -200,13 +199,13 @@ self.check('ea+Eeb+Eec+Eed+E', [6,7,8,9,0], a=6, b=7, c=8, d=9, e=30) def test_loop(self): - self.check('0A9B{ab+Ab1-Bb}', a=45) + self.check('0A9B{ab+Ab1-Bbn(x)}', a=45) def test_conditional(self): - self.check('0A0C9B{b4<(a1+A)(c1+C)b1-Bb}', c=6, a=3) + self.check('0A0C9B{b4<(a1+A)(c1+C)b1-Bbn(x)}', c=6, a=3) def test_break(self): - self.check('0A9B{ab+Ab1-Bb0=(x)1}', a=45) + self.check('0A9B{ab+Ab1-Bb0=(x)}', a=45) def test_nested(self): self.check('''0A @@ -216,13 +215,13 @@ ac+A c1-C c0= (x) - 1} + } b1-B b0= (x) - 1}''', a=810) + }''', a=810) def test_jump_limit(self): - self.check('0A{a1+A1}', max_back_jumps=10, a=10) + self.check('0A{a1+A}', max_back_jumps=10, a=10) def test_failure1(self): self.check('{{c3+A{cc+Dda<({2b=}c7=(cd-Aae-Exe8+Ax)aa+Ab5-D2a-Dba=(0d+Be7-D6e+Bd3-A)0e+D)(a0=(x{79>})(0e>(x)9c+Ce3-Ccb>(5d=(08-Axx)da+B4d<(04-B7d+Eba+Axx09+B15-E))()33-Bc9<()({xba+Ec1+C0a=}4c+C79+Dxda<(0c+A)(5e+D9b+C2d-Bcc+D8b-A99-B3c-De4-Cc9+C)58-Bcc-Ae1-Be9-D)ae+A)xcc=(3c=(d5>(23+Bad+Bx8c-De5-Dac-Cd3-Cea+Ax)(xx)33-A{dc-D0b-Ab8-Cb1+Bd1+A28=}65+Aba<(x5c+Axba-C57+A3b-Deb+C)(cd+Bec+B30+Bbb+D45-De4-C)b8-Eae<(aa+Ce4-E1a+Dxa1+E)(d5-Ea2-Ex62+Bxx6a-D)be+B)c6+C3e+A)(5e+C4e-Ec9+Bx26-Ca8=()x)a3<(x10-Cd8-Aba-Ex{5c=})55+C)xd2-Ax98>}1}1}', [49, 30, 28, 93, 22], 1000) From noreply at buildbot.pypy.org Mon Jan 14 19:39:22 2013 From: noreply at buildbot.pypy.org (hakanardo) Date: Mon, 14 Jan 2013 19:39:22 +0100 (CET) Subject: [pypy-commit] pypy jit-usable_retrace_3: handle overflows corectly Message-ID: <20130114183922.45F371C0246@cobra.cs.uni-duesseldorf.de> Author: Hakan Ardo Branch: jit-usable_retrace_3 Changeset: r60069:fd8a0b5b9ea0 Date: 2013-01-14 19:36 +0100 http://bitbucket.org/pypy/pypy/changeset/fd8a0b5b9ea0/ Log: handle overflows corectly diff --git a/pypy/jit/metainterp/test/test_random_loops.py b/pypy/jit/metainterp/test/test_random_loops.py --- a/pypy/jit/metainterp/test/test_random_loops.py +++ b/pypy/jit/metainterp/test/test_random_loops.py @@ -1,6 +1,7 @@ from pypy.jit.metainterp.test.support import LLJitMixin from pypy.rlib.jit import JitDriver from random import choice, randrange +from pypy.rlib.rarithmetic import intmask import re class IntBox(object): @@ -11,10 +12,10 @@ return self.val def add(self, other): - return IntBox(self.value() + other.value()) + return IntBox(intmask(self.value() + other.value())) def sub(self, other): - return IntBox(self.value() - other.value()) + return IntBox(intmask(self.value() - other.value())) def gt(self, other): return IntBox(self.value() > other.value()) @@ -223,8 +224,9 @@ def test_jump_limit(self): self.check('0A{a1+A}', max_back_jumps=10, a=10) - def test_failure1(self): - self.check('{{c3+A{cc+Dda<({2b=}c7=(cd-Aae-Exe8+Ax)aa+Ab5-D2a-Dba=(0d+Be7-D6e+Bd3-A)0e+D)(a0=(x{79>})(0e>(x)9c+Ce3-Ccb>(5d=(08-Axx)da+B4d<(04-B7d+Eba+Axx09+B15-E))()33-Bc9<()({xba+Ec1+C0a=}4c+C79+Dxda<(0c+A)(5e+D9b+C2d-Bcc+D8b-A99-B3c-De4-Cc9+C)58-Bcc-Ae1-Be9-D)ae+A)xcc=(3c=(d5>(23+Bad+Bx8c-De5-Dac-Cd3-Cea+Ax)(xx)33-A{dc-D0b-Ab8-Cb1+Bd1+A28=}65+Aba<(x5c+Axba-C57+A3b-Deb+C)(cd+Bec+B30+Bbb+D45-De4-C)b8-Eae<(aa+Ce4-E1a+Dxa1+E)(d5-Ea2-Ex62+Bxx6a-D)be+B)c6+C3e+A)(5e+C4e-Ec9+Bx26-Ca8=()x)a3<(x10-Cd8-Aba-Ex{5c=})55+C)xd2-Ax98>}1}1}', [49, 30, 28, 93, 22], 1000) + def test_overflow(self): + self.check('9A{aa+A}', max_back_jumps=100) + self.check('09-A{aa+A}', max_back_jumps=100) def test_random_bytecode(self): for i in xrange(1000): From noreply at buildbot.pypy.org Mon Jan 14 20:39:08 2013 From: noreply at buildbot.pypy.org (hakanardo) Date: Mon, 14 Jan 2013 20:39:08 +0100 (CET) Subject: [pypy-commit] pypy jit-usable_retrace_3: failing case found Message-ID: <20130114193908.9AE4C1C1198@cobra.cs.uni-duesseldorf.de> Author: Hakan Ardo Branch: jit-usable_retrace_3 Changeset: r60070:58dc9cb7d261 Date: 2013-01-14 20:23 +0100 http://bitbucket.org/pypy/pypy/changeset/58dc9cb7d261/ Log: failing case found diff --git a/pypy/jit/metainterp/test/test_random_loops.py b/pypy/jit/metainterp/test/test_random_loops.py --- a/pypy/jit/metainterp/test/test_random_loops.py +++ b/pypy/jit/metainterp/test/test_random_loops.py @@ -228,6 +228,9 @@ self.check('9A{aa+A}', max_back_jumps=100) self.check('09-A{aa+A}', max_back_jumps=100) + def test_failure1(self): + self.check('{{7d<(37>(c6-A{a6>n(x)5b<()(bb+Bea-A){7d>n(x)1e+A7d-B3d+E35+Eac-B2b<(04+Cxe8+D9d+C)(){c6n(x)a5=(2c+E)(xxe9-A)ec>(b7-A40+Ce0-Cx06-A)(xc0-Ad2-A1e+B0d+Bx09+B0b-D)}be-Ab7=(3a+Ab7-Dde<(x))(ec+Cd4+D5d<(x10+A)(cb-Cb9-C))59<(2b-C00-Ab0<(bc-Bde+Bxa2-E4d+A))(e7+Eb3<(c4-Bxx)59-C7c+Aa8>(96-Exce+D3e-Ex)db+A76-Cdb-Bdc=(2b-A))}d2+D)(27-C26+B39+A9d+Bcc+Dab+Cdd=(ae+Eee-Ex{5d>n(x)}d0<(2a+C41+Dca+C6a>(47-Ae9-Eb7+Cx60+D63-C)bb+D9e+Ec5=(d2-Cxaa-D84-Ced+B20+A4c+B)65+A59-E)ed<(cd+B{a8=n(x)ed+Bxxd8+A}ba-D{66=n(x)a4+D30+Acd+C1c+A6c+Ac7-Dxc0+B}x)4b-Dx)e4-D{aa=n(x)44-Eac>(3e+Aae+A12<(a4-A)99<(ce-C4a+A7b+Dxdd+E)(c5+A10-D63-Eda-E22-E85+Dea+E)x)ed+Dbb=(e5-A9e-C)(x{can(x)}e5+Ba5+B}c9=()c0=(7b-Ae1+Daa+Aed+C)(91-C))3d=(a2>(1e-Cba=(2c-A)21+C6b>(aa-Bxd5-Bc7-E)ab<(8c-Ba8-Cxea-D4e+Ae8+Dc5+Ade+B)x33+B5a-C17+D)2a+B{ebn(x)x6a+Cb4+A27-Eab-C35-E25+B1d+C}cc+Aae-D}cc+C61-C68-D)(xe4>(e4+Db4-Cxdb=(xc7-Bcd-Bde-A00+A39+E9c+Dc9+B24-B){2b>n(x)x4d+E8d+Dbb+Ce7-Bxea+B54+A}80-A43<(89-B)(1c+Cx7b+Ae2+E02-C84+Exd0+A))(3d+Eb3-Ecd-A{8c=n(x)37-Bb0+Axxee+C11-C2c-De6-D46+D}cb+E)84+B05-Cbd+D)ad>(b6-C7d+Ae4-B4a+E)34-Cx9a=({c4>n(x)dd+B72<(a9+A)(b0-Db8-Dac-Bba-Excb+Ce2+Bx)ba-Bda+Cce-Cx}78+E97=(aa+Dx67+Dbc+B{de>n(x)b8-C14-Cx})c4+Cac+Ex80+C))xca=({55(de+Ax)da-B{d6>n(x)8d+A36-Bx8d-Bx49+Bbc-D88+Bx}x26-Da7-E)({9c>n(x)}bc<(0a-Cx61+Cce+A54-Ce7+A21-C9d+D)(6d-E)x{7e=n(x)6d-Ce6-Cx}5c>(bd+Ed5+Cc0-Dc2+C93-B)x69>(d9-Ac8+B8c-Dx94-E76+Ax)(ba+D)b6>(bb+Ebb-A5e+Bc3-D6c-E))54>(95+B46-D{15(xb6+Ec6-Aac-Bxc2-Ax99-C))b0-E58>(x0e+Bxa4+D5e>(7e-Dde-B1c+Bba-B8a-D)(35-Cce-C4c-C90-C1e+B9a+Be7+C1d-E)45>(xb9+A91+Ecc-Aac-B91+Cd1-De3+Ddd-B)c0+A0a>(45+Cbd+Bdc+Db4-D7c+C29+A7a+B60+A))(5d>(xx6e-Dxea+C)(c2-Cad+D36-Cx)61+E20-C9e-Cc2+Cce+Ad0+B{4b=n(x)})x{a1=n(x)e8<(x35-Dd3-Bd6-Bea+Eac-Acc+B79+Eaa-C)}x)xc7>(6b-C{57>n(x)e2>(1a+A97+Dxcc-E7b+A5d-B82+C2b-B)0a+Be1-Cbc+Ca2-E01+B{41>n(x)e1+Dc6+B47+Beb+Ac2+Bxab+D}}d3-D0e+E0d+Bae=(c7+Ce3-Bc3<(d2+E68+Bbc-Cdc+B1b-Ea2-Ac8-E)45-B43+Aac-E)xbe+C)dc=({ee>n(x)a4-Axd2+B1b+B}aa+Exxxb4>()(cb-C{4d(a0-Dab+A8c+Ddd-Edc+A)(xce-D)de=(d6+E83+Cxxc6-Axx)(0b+D78+Cx07-Eda+Ax1d-Eba+A)e6+E)de-E1c+Aae=(20-B25+Dba>(xxx71-A2b+Dc7+E){e9=n(x)7b+Ecd+Bda-Eec-A4b+De7-Ae3-D}ae+C7c+Aaa+Bx)86+B)}{cc=n(x)6b-B{cd(cc+D7e-Bc0+Bxde-A2c-B5d-C)(ba+D2b-Axxx07+Ca4-C)eb-C23-D7e<(xb8+Edd-De1+Add+C35-Dxx)x)(87-Ccc+Ee2=(9a-E78-C)()cb-A)39+D0b-Aa4>(ca+A)c6=({9c(bb-E22-D)x5e-C11<()9b=(6d-Bcd+A9e-E2a-E38+B4e-C)xaa-B{82=n(x)ae-B6a-Ab7+Ca9-A}08+C)d7+Cxcb-A}6c-B9c+D9c-E})3d+Cd4>(xab<()()3e+Ebd-Ba0-B{6c(63+Eb5+Bxec<(a1+E4e+Axdc-C)ea+Eaa=(x37-Edd-Cb4+B)(43-Ae0-Bx7e-C94+A)e8-Bd1-Da8-D)())(ce=(6c-B10+C7a+A3d+Ax19+Aca+B)(ed+A)1d>()d1+Ex43+A9d+Eb5+A)}8d+Ae9-C)(3d-D6c+Ee9<(29-C4c=({e9(4c+B)8a-Dc0-E82+E50+D}a1+D98=(be-B)3a-Ae9+De2-Add-E)35-D1}1}', [13, 40, 35, 44, 76], 1000) + def test_random_bytecode(self): for i in xrange(1000): yield self.check, self.random_loop(), [randrange(100) for i in xrange(5)], 1000 From noreply at buildbot.pypy.org Tue Jan 15 01:23:16 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Tue, 15 Jan 2013 01:23:16 +0100 (CET) Subject: [pypy-commit] pypy py3k: pow(negative, fraction) now promotes to complex Message-ID: <20130115002316.B279C1C1197@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60071:92dd4a5d9a8f Date: 2013-01-14 16:21 -0800 http://bitbucket.org/pypy/pypy/changeset/92dd4a5d9a8f/ Log: pow(negative, fraction) now promotes to complex diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py --- a/pypy/objspace/std/floatobject.py +++ b/pypy/objspace/std/floatobject.py @@ -493,9 +493,11 @@ if isnan(y): return W_FloatObject(NAN) if math.floor(y) != y: - raise OperationError(space.w_ValueError, - space.wrap("negative number cannot be " - "raised to a fractional power")) + # Negative numbers raised to fractional powers become + # complex + return space.pow(space.newcomplex(x, 0.0), + space.newcomplex(y, 0.0), + thirdArg) # y is an exact integer, albeit perhaps a very large one. # Replace x by its absolute value and remember to negate the # pow result if y is odd. diff --git a/pypy/objspace/std/test/test_floatobject.py b/pypy/objspace/std/test/test_floatobject.py --- a/pypy/objspace/std/test/test_floatobject.py +++ b/pypy/objspace/std/test/test_floatobject.py @@ -242,6 +242,11 @@ assert str(pw(float('-inf'), 1.0)) == '-inf' assert str(pw(float('-inf'), 2.0)) == 'inf' + def test_builtin_pow(self): + result = pow(-1, 0.5) + def assertAlmostEqual(x, y): assert round(abs(y - x), 7) == 0 + assertAlmostEqual(result, 1j) + def test_pow_neg_base(self): import math def pw(x, y): From noreply at buildbot.pypy.org Tue Jan 15 01:23:17 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Tue, 15 Jan 2013 01:23:17 +0100 (CET) Subject: [pypy-commit] pypy py3k: stricter boolean check, fixes test_builtin's obscure test_baddecorator Message-ID: <20130115002317.EE19C1C1197@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60072:ad0e6caa6591 Date: 2013-01-14 16:22 -0800 http://bitbucket.org/pypy/pypy/changeset/ad0e6caa6591/ Log: stricter boolean check, fixes test_builtin's obscure test_baddecorator diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py --- a/pypy/objspace/std/listobject.py +++ b/pypy/objspace/std/listobject.py @@ -1539,7 +1539,7 @@ def list_sort__List_ANY_ANY(space, w_list, w_keyfunc, w_reverse): has_key = not space.is_w(w_keyfunc, space.w_None) - has_reverse = space.is_true(w_reverse) + has_reverse = bool(space.int_w(w_reverse)) # create and setup a TimSort instance if 0: diff --git a/pypy/objspace/std/test/test_listobject.py b/pypy/objspace/std/test/test_listobject.py --- a/pypy/objspace/std/test/test_listobject.py +++ b/pypy/objspace/std/test/test_listobject.py @@ -588,6 +588,7 @@ l = [1] l.sort(reverse = True) assert l == [1] + raises(TypeError, sorted, [], None, lambda x, y: 0) def test_sort_cmp_key_reverse(self): def lower(x): return x.lower() From noreply at buildbot.pypy.org Tue Jan 15 01:23:19 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Tue, 15 Jan 2013 01:23:19 +0100 (CET) Subject: [pypy-commit] pypy py3k: improve starts/endswith error messages Message-ID: <20130115002319.3539A1C1197@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60073:f4f25ac26c4a Date: 2013-01-14 16:22 -0800 http://bitbucket.org/pypy/pypy/changeset/f4f25ac26c4a/ Log: improve starts/endswith error messages diff --git a/pypy/objspace/std/stringobject.py b/pypy/objspace/std/stringobject.py --- a/pypy/objspace/std/stringobject.py +++ b/pypy/objspace/std/stringobject.py @@ -705,11 +705,21 @@ u_self, u_start, u_end = _convert_idx_params(space, w_self, w_start, w_end) return wrapint(space, u_self.count(w_arg._value, u_start, u_end)) +def _suffix_to_str(space, w_suffix, funcname): + try: + return space.bufferstr_w(w_suffix) + except OperationError as e: + if e.match(space, space.w_TypeError): + msg = ("%s first arg must be bytes or a tuple of bytes, " + "not %s") + typename = space.type(w_suffix).getname(space) + raise operationerrfmt(space.w_TypeError, msg, funcname, typename) + def str_endswith__String_ANY_ANY_ANY(space, w_self, w_suffix, w_start, w_end): (u_self, start, end) = _convert_idx_params(space, w_self, w_start, w_end, True) - return space.newbool(stringendswith(u_self, space.bufferstr_w(w_suffix), - start, end)) + return space.newbool(stringendswith( + u_self, _suffix_to_str(space, w_suffix, 'endswith'), start, end)) def str_endswith__String_String_ANY_ANY(space, w_self, w_suffix, w_start, w_end): (u_self, start, end) = _convert_idx_params(space, w_self, w_start, @@ -728,8 +738,8 @@ def str_startswith__String_ANY_ANY_ANY(space, w_self, w_prefix, w_start, w_end): (u_self, start, end) = _convert_idx_params(space, w_self, w_start, w_end, True) - return space.newbool(stringstartswith(u_self, space.bufferstr_w(w_prefix), - start, end)) + return space.newbool(stringstartswith( + u_self, _suffix_to_str(space, w_prefix, 'startswith'), start, end)) def str_startswith__String_String_ANY_ANY(space, w_self, w_prefix, w_start, w_end): (u_self, start, end) = _convert_idx_params(space, w_self, w_start, diff --git a/pypy/objspace/std/test/test_stringobject.py b/pypy/objspace/std/test/test_stringobject.py --- a/pypy/objspace/std/test/test_stringobject.py +++ b/pypy/objspace/std/test/test_stringobject.py @@ -321,6 +321,12 @@ assert b'hello'.startswith((bytearray(b'he'), bytearray(b'hel'))) assert b'hello'.startswith((b'he', None, 123)) assert b'y'.startswith(b'xx') is False + try: + b'hello'.startswith([b'o']) + except TypeError as e: + assert 'bytes' in str(e) + else: + assert False, 'Expected TypeError' def test_startswith_more(self): assert b'ab'.startswith(b'a', 0) is True @@ -351,6 +357,12 @@ assert b''.endswith(b'a') is False assert b'x'.endswith(b'xx') is False assert b'y'.endswith(b'xx') is False + try: + b'hello'.endswith([b'o']) + except TypeError as e: + assert 'bytes' in str(e) + else: + assert False, 'Expected TypeError' def test_endswith_more(self): assert b'abc'.endswith(b'ab', 0, 2) is True From noreply at buildbot.pypy.org Tue Jan 15 02:49:35 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Tue, 15 Jan 2013 02:49:35 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: hg merge default Message-ID: <20130115014935.885BB1C0246@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60074:df8704aa6585 Date: 2013-01-15 02:49 +0100 http://bitbucket.org/pypy/pypy/changeset/df8704aa6585/ Log: hg merge default diff --git a/goal/targetpypystandalone.py b/goal/targetpypystandalone.py --- a/goal/targetpypystandalone.py +++ b/goal/targetpypystandalone.py @@ -29,11 +29,6 @@ w_call_startup_gateway = space.wrap(gateway.interp2app(call_startup)) withjit = space.config.objspace.usemodules.pypyjit - if withjit: - from pypy.module.pypyjit.interp_jit import callback_hook - from rpython.rlib import objectmodel - objectmodel.register_around_callback_hook(callback_hook) - def entry_point(argv): if withjit: from rpython.jit.backend.hlinfo import highleveljitinfo diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py --- a/pypy/module/_cffi_backend/ccallback.py +++ b/pypy/module/_cffi_backend/ccallback.py @@ -152,6 +152,7 @@ STDERR = 2 + at jit.jit_callback("CFFI") def invoke_callback(ffi_cif, ll_res, ll_args, ll_userdata): """ Callback specification. ffi_cif - something ffi specific, don't care diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py --- a/pypy/module/pyexpat/interp_pyexpat.py +++ b/pypy/module/pyexpat/interp_pyexpat.py @@ -2,7 +2,7 @@ from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault from pypy.interpreter.error import OperationError -from rpython.rlib import rgc +from rpython.rlib import rgc, jit from rpython.rtyper.lltypesystem import rffi, lltype from rpython.rtyper.tool import rffi_platform from rpython.translator.tool.cbuild import ExternalCompilationInfo @@ -288,6 +288,7 @@ post_code = '' src = py.code.Source(""" + @jit.jit_callback('XML:%(name)s') def %(name)s_callback(%(first_arg)s, %(args)s): id = rffi.cast(lltype.Signed, %(ll_id)s) userdata = global_storage.get_object(id) diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py --- a/pypy/module/pypyjit/interp_jit.py +++ b/pypy/module/pypyjit/interp_jit.py @@ -6,7 +6,7 @@ from rpython.tool.pairtype import extendabletype from rpython.rlib.rarithmetic import r_uint, intmask from rpython.rlib.jit import JitDriver, hint, we_are_jitted, dont_look_inside -from rpython.rlib import jit, objectmodel +from rpython.rlib import jit from rpython.rlib.jit import current_trace_length, unroll_parameters import pypy.interpreter.pyopcode # for side-effects from pypy.interpreter.error import OperationError, operationerrfmt @@ -97,15 +97,6 @@ is_being_profiled=self.is_being_profiled) return jumpto -callback_jit_driver = JitDriver(greens = ['name'], reds = 'auto') - -def callback_merge_point(name): - callback_jit_driver.jit_merge_point(name=name) - - at callback_jit_driver.inline(callback_merge_point) -def callback_hook(name): - pass - def _get_adapted_tick_counter(): # Normally, the tick counter is decremented by 100 for every # Python opcode. Here, to better support JIT compilation of diff --git a/pypy/module/thread/os_thread.py b/pypy/module/thread/os_thread.py --- a/pypy/module/thread/os_thread.py +++ b/pypy/module/thread/os_thread.py @@ -2,7 +2,8 @@ Thread support based on OS-level threads. """ -from rpython.rlib import rthread as thread +import os +from pypy.module.thread import ll_thread as thread from pypy.module.thread.error import wrap_thread_error from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.gateway import unwrap_spec, Arguments @@ -91,14 +92,18 @@ # run! try: bootstrapper.run(space, w_callable, args) - finally: - bootstrapper.nbthreads -= 1 - # clean up space.threadlocals to remove the ExecutionContext - # entry corresponding to the current thread + # done + except Exception, e: + # oups! last-level attempt to recover. try: - space.threadlocals.leave_thread(space) - finally: - thread.gc_thread_die() + STDERR = 2 + os.write(STDERR, "Thread exited with ") + os.write(STDERR, str(e)) + os.write(STDERR, "\n") + except OSError: + pass + bootstrapper.nbthreads -= 1 + thread.gc_thread_die() bootstrap = staticmethod(bootstrap) def acquire(space, w_callable, args): @@ -129,6 +134,9 @@ where = 'thread %d started by ' % ident e.write_unraisable(space, where, w_callable) e.clear(space) + # clean up space.threadlocals to remove the ExecutionContext + # entry corresponding to the current thread + space.threadlocals.leave_thread(space) run = staticmethod(run) bootstrapper = Bootstrapper() diff --git a/pypy/module/thread/test/test_gil.py b/pypy/module/thread/test/test_gil.py --- a/pypy/module/thread/test/test_gil.py +++ b/pypy/module/thread/test/test_gil.py @@ -61,8 +61,9 @@ def bootstrap(): try: runme() - finally: - thread.gc_thread_die() + except Exception, e: + assert 0 + thread.gc_thread_die() def f(): state.data = [] state.datalen1 = 0 diff --git a/rpython/jit/backend/test/test_random.py b/rpython/jit/backend/test/test_random.py --- a/rpython/jit/backend/test/test_random.py +++ b/rpython/jit/backend/test/test_random.py @@ -26,6 +26,8 @@ class OperationBuilder(object): def __init__(self, cpu, loop, vars): self.cpu = cpu + if not hasattr(cpu, '_faildescr_keepalive'): + cpu._faildescr_keepalive = [] self.fakemetainterp = FakeMetaInterp() self.loop = loop self.intvars = [box for box in vars if isinstance(box, BoxInt)] @@ -206,6 +208,11 @@ if pytest.config.option.output: s.close() + def getfaildescr(self): + descr = BasicFailDescr() + self.cpu._faildescr_keepalive.append(descr) + return descr + class CannotProduceOperation(Exception): pass @@ -287,7 +294,7 @@ builder.intvars[:] = original_intvars else: op = ResOperation(rop.GUARD_NO_OVERFLOW, [], None) - op.setdescr(BasicFailDescr()) + op.setdescr(builder.getfaildescr()) op.setfailargs(fail_subset) builder.loop.operations.append(op) @@ -357,7 +364,7 @@ def produce_into(self, builder, r): op, passing = self.gen_guard(builder, r) builder.loop.operations.append(op) - op.setdescr(BasicFailDescr()) + op.setdescr(builder.getfaildescr()) op.setfailargs(builder.subset_of_intvars(r)) if not passing: builder.should_fail_by = op @@ -618,8 +625,9 @@ if v not in used_later: endvars.append(v) r.shuffle(endvars) + endvars = endvars[:1] loop.operations.append(ResOperation(rop.FINISH, endvars, None, - descr=BasicFailDescr())) + descr=builder.getfaildescr())) if builder.should_fail_by: self.should_fail_by = builder.should_fail_by self.guard_op = builder.guard_op @@ -712,7 +720,7 @@ else: op = ResOperation(rop.GUARD_EXCEPTION, [guard_op._exc_box], BoxPtr()) - op.setdescr(BasicFailDescr()) + op.setdescr(self.builder.getfaildescr()) op.setfailargs([]) return op diff --git a/rpython/jit/metainterp/test/test_fficall.py b/rpython/jit/metainterp/test/test_fficall.py --- a/rpython/jit/metainterp/test/test_fficall.py +++ b/rpython/jit/metainterp/test/test_fficall.py @@ -104,7 +104,7 @@ class TestFfiCall(FfiCallTests, LLJitMixin): - def test_jit_fii_vref(self): + def test_jit_ffi_vref(self): from rpython.rlib import clibffi from rpython.rlib.jit_libffi import jit_ffi_prep_cif, jit_ffi_call @@ -135,9 +135,10 @@ res = exb[3] lltype.free(exb, flavor='raw') # - lltype.free(atypes, flavor='raw') return res # res = self.interp_operations(f, []) lltype.free(cd, flavor='raw') assert res == math.sin(1.23) + + lltype.free(atypes, flavor='raw') diff --git a/rpython/jit/metainterp/test/test_warmspot.py b/rpython/jit/metainterp/test/test_warmspot.py --- a/rpython/jit/metainterp/test/test_warmspot.py +++ b/rpython/jit/metainterp/test/test_warmspot.py @@ -1,6 +1,6 @@ import py from rpython.jit.metainterp.warmspot import get_stats -from rpython.rlib.jit import JitDriver, set_param, unroll_safe +from rpython.rlib.jit import JitDriver, set_param, unroll_safe, jit_callback from rpython.jit.backend.llgraph import runner from rpython.jit.metainterp.test.support import LLJitMixin, OOJitMixin @@ -537,6 +537,7 @@ def test_callback_jit_merge_point(self): +<<<<<<< local from rpython.rlib.objectmodel import register_around_callback_hook from rpython.rtyper.lltypesystem import lltype, rffi from rpython.translator.tool.cbuild import ExternalCompilationInfo @@ -550,28 +551,23 @@ def callback_hook(name): pass +======= + @jit_callback("testing") +>>>>>>> other def callback(a, b): if a > b: return 1 return -1 - CB_TP = rffi.CCallback([lltype.Signed, lltype.Signed], lltype.Signed) - eci = ExternalCompilationInfo(includes=['stdlib.h']) - qsort = rffi.llexternal('qsort', - [rffi.VOIDP, lltype.Signed, lltype.Signed, - CB_TP], lltype.Void, compilation_info=eci) - ARR = rffi.CArray(lltype.Signed) + def main(): + total = 0 + for i in range(10): + total += callback(i, 2) + return total - def main(): - register_around_callback_hook(callback_hook) - raw = lltype.malloc(ARR, 10, flavor='raw') - for i in range(10): - raw[i] = 10 - i - qsort(raw, 10, rffi.sizeof(lltype.Signed), callback) - lltype.free(raw, flavor='raw') - - self.meta_interp(main, []) - self.check_trace_count(1) + res = self.meta_interp(main, []) + assert res == 7 - 3 + self.check_trace_count(2) class TestLLWarmspot(WarmspotTests, LLJitMixin): diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py --- a/rpython/jit/metainterp/warmspot.py +++ b/rpython/jit/metainterp/warmspot.py @@ -360,13 +360,7 @@ jd._jit_merge_point_in = graph args = op.args[2:] s_binding = self.translator.annotator.binding - if op.args[1].value.autoreds: - # _portal_args_s is used only by _make_hook_graph, but for now we - # declare the various set_jitcell_at, get_printable_location, - # etc. as incompatible with autoreds - jd._portal_args_s = None - else: - jd._portal_args_s = [s_binding(v) for v in args] + jd._portal_args_s = [s_binding(v) for v in args] graph = copygraph(graph) [jmpp] = find_jit_merge_points([graph]) graph.startblock = support.split_before_jit_merge_point(*jmpp) @@ -614,9 +608,10 @@ if func is None: return None # - assert not jitdriver_sd.jitdriver.autoreds, ( - "reds='auto' is not compatible with JitDriver hooks such as " - "{get,set}_jitcell_at, get_printable_location, confirm_enter_jit, etc.") + if not onlygreens: + assert not jitdriver_sd.jitdriver.autoreds, ( + "reds='auto' is not compatible with JitDriver hooks such as " + "confirm_enter_jit") extra_args_s = [] if s_first_arg is not None: extra_args_s.append(s_first_arg) diff --git a/rpython/rlib/clibffi.py b/rpython/rlib/clibffi.py --- a/rpython/rlib/clibffi.py +++ b/rpython/rlib/clibffi.py @@ -421,6 +421,7 @@ hints={'callback':True})) + at jit.jit_callback("CLIBFFI") def ll_callback(ffi_cif, ll_res, ll_args, ll_userdata): """ Callback specification. ffi_cif - something ffi specific, don't care diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -302,6 +302,32 @@ pass +def jit_callback(name): + """Use as a decorator for C callback functions, to insert a + jitdriver.jit_merge_point() at the start. Only for callbacks + that typically invoke more app-level Python code. + """ + def decorate(func): + from pypy.tool.sourcetools import compile2 + # + def get_printable_location(): + return name + jitdriver = JitDriver(get_printable_location=get_printable_location, + greens=[], reds='auto', name=name) + # + args = ','.join(['a%d' % i for i in range(func.func_code.co_argcount)]) + source = """def callback_with_jitdriver(%(args)s): + jitdriver.jit_merge_point() + return real_callback(%(args)s)""" % locals() + miniglobals = { + 'jitdriver': jitdriver, + 'real_callback': func, + } + exec compile2(source) in miniglobals + return miniglobals['callback_with_jitdriver'] + return decorate + + # ____________________________________________________________ # VRefs @@ -458,9 +484,8 @@ self.autoreds = True self.reds = [] self.numreds = None # see warmspot.autodetect_jit_markers_redvars - for hook in (get_jitcell_at, set_jitcell_at, get_printable_location, - confirm_enter_jit): - assert hook is None, "reds='auto' is not compatible with JitDriver hooks" + assert confirm_enter_jit is None, ( + "reds='auto' is not compatible with confirm_enter_jit") else: if reds is not None: self.reds = reds diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py --- a/rpython/rlib/objectmodel.py +++ b/rpython/rlib/objectmodel.py @@ -595,16 +595,6 @@ llhelper(rffi.AroundFnPtr, before) llhelper(rffi.AroundFnPtr, after) -def register_around_callback_hook(hook): - """ Register a hook that's called before a callback from C calls RPython. - Primary usage is for JIT to have 'started from' hook. - """ - from rpython.rtyper.lltypesystem import rffi - from rpython.rtyper.annlowlevel import llhelper - - rffi.aroundstate.callback_hook = hook - llhelper(rffi.CallbackHookPtr, hook) - def is_in_callback(): from rpython.rtyper.lltypesystem import rffi return rffi.stackcounter.stacks_counter > 1 diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py --- a/rpython/rtyper/lltypesystem/rffi.py +++ b/rpython/rtyper/lltypesystem/rffi.py @@ -13,7 +13,7 @@ from rpython.rlib.unroll import unrolling_iterable from rpython.rtyper.tool.rfficache import platform, sizeof_c_type from rpython.translator.tool.cbuild import ExternalCompilationInfo -from rpython.rtyper.annlowlevel import llhelper, llstr +from rpython.rtyper.annlowlevel import llhelper from rpython.rlib.objectmodel import we_are_translated from rpython.rlib.rstring import StringBuilder, UnicodeBuilder, assert_str0 from rpython.rlib import jit @@ -279,17 +279,8 @@ callable_name = getattr(callable, '__name__', '?') if callbackholder is not None: callbackholder.callbacks[callable] = True - callable_name_descr = str(callable).replace('"', '\\"') args = ', '.join(['a%d' % i for i in range(len(TP.TO.ARGS))]) source = py.code.Source(r""" - def inner_wrapper(%(args)s): - if aroundstate is not None: - callback_hook = aroundstate.callback_hook - if callback_hook: - callback_hook(llstr("%(callable_name_descr)s")) - return callable(%(args)s) - inner_wrapper._never_inline_ = True - def wrapper(%(args)s): # no *args - no GIL for mallocing the tuple llop.gc_stack_bottom(lltype.Void) # marker for trackgcroot.py if aroundstate is not None: @@ -299,7 +290,7 @@ # from now on we hold the GIL stackcounter.stacks_counter += 1 try: - result = inner_wrapper(%(args)s) + result = callable(%(args)s) except Exception, e: os.write(2, "Warning: uncaught exception in callback: %%s %%s\n" %% @@ -321,7 +312,6 @@ miniglobals = locals().copy() miniglobals['Exception'] = Exception miniglobals['os'] = os - miniglobals['llstr'] = llstr miniglobals['we_are_translated'] = we_are_translated miniglobals['stackcounter'] = stackcounter exec source.compile() in miniglobals @@ -329,11 +319,8 @@ _make_wrapper_for._annspecialcase_ = 'specialize:memo' AroundFnPtr = lltype.Ptr(lltype.FuncType([], lltype.Void)) -CallbackHookPtr = lltype.Ptr(lltype.FuncType([lltype.Ptr(rstr.STR)], lltype.Void)) class AroundState: - callback_hook = None - def _cleanup_(self): self.before = None # or a regular RPython function self.after = None # or a regular RPython function diff --git a/rpython/rtyper/memory/gctransform/framework.py b/rpython/rtyper/memory/gctransform/framework.py --- a/rpython/rtyper/memory/gctransform/framework.py +++ b/rpython/rtyper/memory/gctransform/framework.py @@ -979,6 +979,7 @@ assert self.translator.config.translation.thread if hasattr(self.root_walker, 'thread_run_ptr'): livevars = self.push_roots(hop) + assert not livevars, "live GC var around %s!" % (hop.spaceop,) hop.genop("direct_call", [self.root_walker.thread_run_ptr]) self.pop_roots(hop, livevars) @@ -993,6 +994,7 @@ assert self.translator.config.translation.thread if hasattr(self.root_walker, 'thread_die_ptr'): livevars = self.push_roots(hop) + assert not livevars, "live GC var around %s!" % (hop.spaceop,) hop.genop("direct_call", [self.root_walker.thread_die_ptr]) self.pop_roots(hop, livevars) diff --git a/rpython/rtyper/memory/gctransform/shadowstack.py b/rpython/rtyper/memory/gctransform/shadowstack.py --- a/rpython/rtyper/memory/gctransform/shadowstack.py +++ b/rpython/rtyper/memory/gctransform/shadowstack.py @@ -326,6 +326,8 @@ self.gcdata.root_stack_top = llmemory.NULL # to detect missing restore def forget_current_state(self): + ll_assert(self.gcdata.root_stack_base == self.gcdata.root_stack_top, + "forget_current_state: shadowstack not empty!") if self.unused_full_stack: llmemory.raw_free(self.unused_full_stack) self.unused_full_stack = self.gcdata.root_stack_base From noreply at buildbot.pypy.org Tue Jan 15 02:51:49 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Tue, 15 Jan 2013 02:51:49 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: missed conflict Message-ID: <20130115015149.6FE631C0246@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60075:9960bb86e2a6 Date: 2013-01-15 02:51 +0100 http://bitbucket.org/pypy/pypy/changeset/9960bb86e2a6/ Log: missed conflict diff --git a/rpython/jit/metainterp/test/test_warmspot.py b/rpython/jit/metainterp/test/test_warmspot.py --- a/rpython/jit/metainterp/test/test_warmspot.py +++ b/rpython/jit/metainterp/test/test_warmspot.py @@ -537,23 +537,7 @@ def test_callback_jit_merge_point(self): -<<<<<<< local - from rpython.rlib.objectmodel import register_around_callback_hook - from rpython.rtyper.lltypesystem import lltype, rffi - from rpython.translator.tool.cbuild import ExternalCompilationInfo - - callback_jit_driver = JitDriver(greens = ['name'], reds = 'auto') - - def callback_merge_point(name): - callback_jit_driver.jit_merge_point(name=name) - - @callback_jit_driver.inline(callback_merge_point) - def callback_hook(name): - pass - -======= @jit_callback("testing") ->>>>>>> other def callback(a, b): if a > b: return 1 From noreply at buildbot.pypy.org Tue Jan 15 09:31:42 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Tue, 15 Jan 2013 09:31:42 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed missing rename Message-ID: <20130115083142.B0C2E1C0131@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60076:2d390094069c Date: 2013-01-15 09:31 +0100 http://bitbucket.org/pypy/pypy/changeset/2d390094069c/ Log: Fixed missing rename diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -308,7 +308,7 @@ that typically invoke more app-level Python code. """ def decorate(func): - from pypy.tool.sourcetools import compile2 + from rpython.tool.sourcetools import compile2 # def get_printable_location(): return name From noreply at buildbot.pypy.org Tue Jan 15 10:27:35 2013 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 15 Jan 2013 10:27:35 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: missing pytest.ini Message-ID: <20130115092735.0BAFE1C02C4@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: split-rpython Changeset: r60077:cddefdada58d Date: 2013-01-15 11:26 +0200 http://bitbucket.org/pypy/pypy/changeset/cddefdada58d/ Log: missing pytest.ini diff --git a/rpython/pytest.ini b/rpython/pytest.ini new file mode 100644 --- /dev/null +++ b/rpython/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +addopts = --assert=reinterp -rf From noreply at buildbot.pypy.org Tue Jan 15 10:44:18 2013 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 15 Jan 2013 10:44:18 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: fix annotation tests Message-ID: <20130115094418.DE9CE1C02C4@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: split-rpython Changeset: r60078:5d4d9f975d2f Date: 2013-01-15 11:28 +0200 http://bitbucket.org/pypy/pypy/changeset/5d4d9f975d2f/ Log: fix annotation tests diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py --- a/rpython/annotator/test/test_annrpython.py +++ b/rpython/annotator/test/test_annrpython.py @@ -3655,7 +3655,7 @@ cls = C return cls().foo a = self.RPythonAnnotator() - raises(Exception, a.build_types, f, [int]) + py.test.raises(Exception, a.build_types, f, [int]) def test_range_variable_step(self): def g(n): From noreply at buildbot.pypy.org Tue Jan 15 10:44:23 2013 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 15 Jan 2013 10:44:23 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: fix another test Message-ID: <20130115094423.6B6621C02C4@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: split-rpython Changeset: r60079:4450811557b3 Date: 2013-01-15 11:30 +0200 http://bitbucket.org/pypy/pypy/changeset/4450811557b3/ Log: fix another test diff --git a/rpython/rtyper/lltypesystem/module/test/test_ll_math.py b/rpython/rtyper/lltypesystem/module/test/test_ll_math.py --- a/rpython/rtyper/lltypesystem/module/test/test_ll_math.py +++ b/rpython/rtyper/lltypesystem/module/test/test_ll_math.py @@ -296,7 +296,7 @@ assert expected == OverflowError, "%s: got an OverflowError" % ( repr,) else: - if not (expected)(got): + if not get_tester(expected)(got): raise AssertionError("%r: got %r, expected %r" % (repr, got, expected)) # From noreply at buildbot.pypy.org Tue Jan 15 11:49:05 2013 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Tue, 15 Jan 2013 11:49:05 +0100 (CET) Subject: [pypy-commit] pyrepl py3k-readline: take in changes from pypy Message-ID: <20130115104905.B5E1D1C0131@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: py3k-readline Changeset: r207:3400dcbc7022 Date: 2013-01-15 11:48 +0100 http://bitbucket.org/pypy/pyrepl/changeset/3400dcbc7022/ Log: take in changes from pypy diff --git a/pyrepl/readline.py b/pyrepl/readline.py --- a/pyrepl/readline.py +++ b/pyrepl/readline.py @@ -185,7 +185,7 @@ def __init__(self): self.f_in = os.dup(0) - self.f_ut = os.dup(1) + self.f_out = os.dup(1) def get_reader(self): if self.reader is None: @@ -241,7 +241,7 @@ try: return unicode(line, ENCODING) except UnicodeDecodeError: # bah, silently fall back... - return unicode(line, 'utf-8') + return unicode(line, 'utf-8', 'replace') def get_history_length(self): return self.saved_history_length diff --git a/pyrepl/unix_console.py b/pyrepl/unix_console.py --- a/pyrepl/unix_console.py +++ b/pyrepl/unix_console.py @@ -477,7 +477,7 @@ if iscode: self.__tputs(text) else: - os.write(self.output_fd, text.encode(self.encoding)) + os.write(self.output_fd, text.encode(self.encoding, 'replace')) del self.__buffer[:] def __tputs(self, fmt, prog=delayprog): From noreply at buildbot.pypy.org Tue Jan 15 11:55:07 2013 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Tue, 15 Jan 2013 11:55:07 +0100 (CET) Subject: [pypy-commit] pyrepl default: shorten some repeated license statements Message-ID: <20130115105507.646261C0131@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: Changeset: r208:20f83b930447 Date: 2013-01-05 01:03 +0100 http://bitbucket.org/pypy/pyrepl/changeset/20f83b930447/ Log: shorten some repeated license statements diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -1,7 +1,3 @@ - Copyright 2000-2002 Michael Hudson mwh at python.net - - All Rights Reserved - Permission to use, copy, modify, and distribute this software and its documentation for any purpose is hereby granted without fee, provided that the above copyright notice appear in all copies and that both diff --git a/testing/test_functional.py b/testing/test_functional.py --- a/testing/test_functional.py +++ b/testing/test_functional.py @@ -1,23 +1,6 @@ # Copyright 2000-2007 Michael Hudson-Doyle # Maciek Fijalkowski -# -# All Rights Reserved -# -# -# Permission to use, copy, modify, and distribute this software and -# its documentation for any purpose is hereby granted without fee, -# provided that the above copyright notice appear in all copies and -# that both that copyright notice and this permission notice appear in -# supporting documentation. -# -# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO -# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, -# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER -# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF -# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - +# License: MIT # some functional tests, to see if this is really working import pytest From noreply at buildbot.pypy.org Tue Jan 15 11:55:08 2013 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Tue, 15 Jan 2013 11:55:08 +0100 (CET) Subject: [pypy-commit] pyrepl default: kill encopyright Message-ID: <20130115105508.8256F1C0131@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: Changeset: r209:1a187de426b7 Date: 2013-01-05 01:40 +0100 http://bitbucket.org/pypy/pyrepl/changeset/1a187de426b7/ Log: kill encopyright diff --git a/encopyright.py b/encopyright.py deleted file mode 100644 --- a/encopyright.py +++ /dev/null @@ -1,157 +0,0 @@ -#!/usr/local/bin/python - -# Copyright 2000-2003 Michael Hudson mwh at python.net -# -# All Rights Reserved -# -# -# Permission to use, copy, modify, and distribute this software and -# its documentation for any purpose is hereby granted without fee, -# provided that the above copyright notice appear in all copies and -# that both that copyright notice and this permission notice appear in -# supporting documentation. -# -# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO -# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, -# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER -# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF -# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import os, time, sys -import py - -header_template = """\ -# Copyright 2000-%(lastyear)s Michael Hudson-Doyle %(others)s -# -# All Rights Reserved -# -# -# Permission to use, copy, modify, and distribute this software and -# its documentation for any purpose is hereby granted without fee, -# provided that the above copyright notice appear in all copies and -# that both that copyright notice and this permission notice appear in -# supporting documentation. -# -# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO -# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, -# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER -# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF -# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.\ -""" - -author_template = "\n#%s%%s"%(' '*(header_template.index("Michael")+1),) - - - -author_map = { - u'mwh': None, - u'micahel': None, - u'Michael Hudson ': None, - u'arigo': u"Armin Rigo", - u'antocuni': u'Antonio Cuni', - u'anto': u'Antonio Cuni', - u'bob': u'Bob Ippolito', - u'fijal': u'Maciek Fijalkowski', - u'agaynor': u'Alex Gaynor', - u'hpk': u'Holger Krekel', - u'Ronny': u'Ronny Pfannschmidt', - u'amauryfa': u"Amaury Forgeot d'Arc", - } - - -def author_revs(path): - proc = py.std.subprocess.Popen([ - 'hg','log', str(path), - '--template', '{author|user} {date}\n', - '-r', 'not keyword("encopyright")', - ], stdout=py.std.subprocess.PIPE) - output, _ = proc.communicate() - lines = output.splitlines() - for line in lines: - try: - name, date = line.split(None, 1) - except ValueError: - pass - else: - if '-' in date: - date = date.split('-')[0] - yield name, float(date) - - -def process(path): - ilines = path.readlines() - revs = sorted(author_revs(path), key=lambda x:x[1]) - modified_year = time.gmtime(revs[-1][1])[0] - if not modified_year: - print 'E: no sensible modified_year found for', path - modified_year = time.gmtime(time.time())[0] - extra_authors = [] - authors = set(rev[0] for rev in revs) - for a in authors: - if a not in author_map: - print 'E: need real name for', a - ea = author_map.get(a) - if ea: - extra_authors.append(ea) - extra_authors.sort() - header = header_template % { - 'lastyear': modified_year, - 'others': ''.join([author_template%ea for ea in extra_authors]) - } - header_lines = header.splitlines() - prelines = [] - old_copyright = [] - - if not ilines: - print "W ignoring empty file", path - return - - i = 0 - - if ilines[0][:2] == '#!': - prelines.append(ilines[0]) - i += 1 - - while i < len(ilines) and not ilines[i].strip(): - prelines.append(ilines[i]) - i += 1 - - while i < len(ilines) and ilines[i][:1] == '#': - old_copyright.append(ilines[i]) - i += 1 - - if abs(len(old_copyright) - len(header_lines)) < 2 + len(extra_authors): - for x, y in zip(old_copyright, header_lines): - if x[:-1] != y: - print "C change needed in", path - ofile = path.open("w") - for l in prelines: - ofile.write(l) - ofile.write(header + "\n") - for l in ilines[i:]: - ofile.write(l) - ofile.close() - break - else: - print "M no change needed in", path - else: - print "A no (c) in", file - with path.open("w") as ofile: - for l in prelines: - ofile.write(l) - ofile.write(header + "\n\n") - for l in ilines[len(prelines):]: - ofile.write(l) - - -for thing in sys.argv[1:]: - path = py.path.local(thing) - if path.check(dir=1): - for item in path.visit('*.py'): - process(item) - elif path.check(file=1, ext='py'): - process(path) From noreply at buildbot.pypy.org Tue Jan 15 11:55:09 2013 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Tue, 15 Jan 2013 11:55:09 +0100 (CET) Subject: [pypy-commit] pyrepl codecheck-clean: make the tests pass codechecks Message-ID: <20130115105509.9622D1C0131@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: codecheck-clean Changeset: r210:31da5c045587 Date: 2013-01-14 21:59 +0100 http://bitbucket.org/pypy/pyrepl/changeset/31da5c045587/ Log: make the tests pass codechecks diff --git a/testing/test_unix_reader.py b/testing/test_unix_reader.py --- a/testing/test_unix_reader.py +++ b/testing/test_unix_reader.py @@ -1,9 +1,6 @@ from __future__ import unicode_literals from pyrepl.unix_eventqueue import EncodedQueue -from pyrepl import curses - - def test_simple(): q = EncodedQueue({}, 'utf-8') diff --git a/testing/test_wishes.py b/testing/test_wishes.py --- a/testing/test_wishes.py +++ b/testing/test_wishes.py @@ -24,8 +24,8 @@ def test_quoted_insert_repeat(): - read_spec([(('digit-arg', '3'), ['']), - (('quoted-insert', None), ['']), - (('self-insert', '\033'), ['^[^[^[']), - (('accept', None), None)]) - + read_spec([ + (('digit-arg', '3'), ['']), + (('quoted-insert', None), ['']), + (('self-insert', '\033'), ['^[^[^[']), + (('accept', None), None)]) From noreply at buildbot.pypy.org Tue Jan 15 11:55:10 2013 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Tue, 15 Jan 2013 11:55:10 +0100 (CET) Subject: [pypy-commit] pyrepl codecheck-clean: add .cache to hgingore Message-ID: <20130115105510.AC9581C0131@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: codecheck-clean Changeset: r211:8e2c0d0a50c8 Date: 2013-01-15 11:09 +0100 http://bitbucket.org/pypy/pyrepl/changeset/8e2c0d0a50c8/ Log: add .cache to hgingore diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -1,4 +1,5 @@ dist/ build/ +.cache/ \.tox/ .*\.egg-info From noreply at buildbot.pypy.org Tue Jan 15 11:55:11 2013 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Tue, 15 Jan 2013 11:55:11 +0100 (CET) Subject: [pypy-commit] pyrepl codecheck-clean: clean setup.py Message-ID: <20130115105511.CF7D71C0131@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: codecheck-clean Changeset: r212:cde0a6ed277a Date: 2013-01-15 11:09 +0100 http://bitbucket.org/pypy/pyrepl/changeset/cde0a6ed277a/ Log: clean setup.py diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -from setuptools import setup, Extension +from setuptools import setup long_desc = """\ pyrepl is a Python library, inspired by readline, for building flexible @@ -31,17 +31,17 @@ setup( - name = "pyrepl", - version = "0.8.4", - author = "Michael Hudson-Doyle", - author_email = "micahel at gmail.com", + name="pyrepl", + version="0.8.4", + author="Michael Hudson-Doyle", + author_email="micahel at gmail.com", maintainer="Ronny Pfannschmidt", maintainer_email="ronny.pfannschmidt at gmx.de", - url = "http://bitbucket.org/pypy/pyrepl/", - license = "MIT X11 style", - description = "A library for building flexible command line interfaces", - platforms = ["unix", "linux"], - packages = ["pyrepl" ], - scripts = ["pythoni", "pythoni1"], - long_description = long_desc, - ) + url="http://bitbucket.org/pypy/pyrepl/", + license="MIT X11 style", + description="A library for building flexible command line interfaces", + platforms=["unix", "linux"], + packages=["pyrepl"], + scripts=["pythoni", "pythoni1"], + long_description=long_desc, +) From noreply at buildbot.pypy.org Tue Jan 15 11:55:12 2013 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Tue, 15 Jan 2013 11:55:12 +0100 (CET) Subject: [pypy-commit] pyrepl codecheck-clean: code clean and refactor cmdrepl Message-ID: <20130115105512.E5D3C1C0131@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: codecheck-clean Changeset: r213:fa54870f1d7a Date: 2013-01-15 11:11 +0100 http://bitbucket.org/pypy/pyrepl/changeset/fa54870f1d7a/ Log: code clean and refactor cmdrepl diff --git a/pyrepl/cmdrepl.py b/pyrepl/cmdrepl.py --- a/pyrepl/cmdrepl.py +++ b/pyrepl/cmdrepl.py @@ -35,26 +35,28 @@ from __future__ import print_function -from pyrepl import completing_reader as cr, reader, completer +from pyrepl import completer from pyrepl.completing_reader import CompletingReader as CR import cmd + class CmdReader(CR): def collect_keymap(self): return super(CmdReader, self).collect_keymap() + ( ("\\M-\\n", "invalid-key"), ("\\n", "accept")) - - CR_init = CR.__init__ + def __init__(self, completions): - self.CR_init(self) + super(CmdReader, self).__init__() self.completions = completions def get_completions(self, stem): if len(stem) != self.pos: return [] - return sorted(set(s for s in self.completions - if s.startswith(stem))) + return sorted(set(s + for s in self.completions + if s.startswith(stem))) + def replize(klass, history_across_invocations=1): @@ -71,26 +73,25 @@ for s in completer.get_class_members(klass) if s.startswith("do_")] - if not issubclass(klass, cmd.Cmd): - raise Exception + assert issubclass(klass, cmd.Cmd) # if klass.cmdloop.im_class is not cmd.Cmd: # print "this may not work" - class CmdRepl(klass): - k_init = klass.__init__ + class MultiHist(object): + __history = [] - if history_across_invocations: - _CmdRepl__history = [] - def __init__(self, *args, **kw): - self.k_init(*args, **kw) - self.__reader = CmdReader(completions) - self.__reader.history = CmdRepl._CmdRepl__history - self.__reader.historyi = len(CmdRepl._CmdRepl__history) - else: - def __init__(self, *args, **kw): - self.k_init(*args, **kw) - self.__reader = CmdReader(completions) - + def __init__(self, *args, **kw): + super(MultiHist, self).__init__(*args, **kw) + self.__reader = CmdReader(completions) + self.__reader.history = self.__history + self.__reader.historyi = len(self.__history) + + class SimpleHist(object): + def __init__(self, *args, **kw): + super(SimpleHist, self).__init__(*args, **kw) + self.__reader = CmdReader(completions) + + class CmdLoopMixin(object): def cmdloop(self, intro=None): self.preloop() if intro is not None: @@ -113,6 +114,8 @@ stop = self.postcmd(stop, line) self.postloop() - CmdRepl.__name__ = "replize(%s.%s)"%(klass.__module__, klass.__name__) + hist = MultiHist if history_across_invocations else SimpleHist + + class CmdRepl(hist, CmdLoopMixin, klass): + __name__ = "replize(%s.%s)" % (klass.__module__, klass.__name__) return CmdRepl - From noreply at buildbot.pypy.org Tue Jan 15 11:55:14 2013 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Tue, 15 Jan 2013 11:55:14 +0100 (CET) Subject: [pypy-commit] pyrepl codecheck-clean: clean some more files Message-ID: <20130115105514.0C9181C0131@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: codecheck-clean Changeset: r214:e07229306d89 Date: 2013-01-15 11:26 +0100 http://bitbucket.org/pypy/pyrepl/changeset/e07229306d89/ Log: clean some more files diff --git a/pyrepl/_minimal_curses.py b/pyrepl/_minimal_curses.py --- a/pyrepl/_minimal_curses.py +++ b/pyrepl/_minimal_curses.py @@ -9,7 +9,8 @@ hide this one if compiled in. """ -import ctypes, ctypes.util +import ctypes.util + class error(Exception): pass @@ -42,8 +43,12 @@ # ____________________________________________________________ -try: from __pypy__ import builtinify -except ImportError: builtinify = lambda f: f +try: + from __pypy__ import builtinify + builtinify # silence broken pyflakes +except ImportError: + builtinify = lambda f: f + @builtinify def setupterm(termstr, fd): @@ -52,6 +57,7 @@ if result == ERR: raise error("setupterm() failed (err=%d)" % err.value) + @builtinify def tigetstr(cap): if not isinstance(cap, bytes): @@ -61,6 +67,7 @@ return None return ctypes.cast(result, ctypes.c_char_p).value + @builtinify def tparm(str, i1=0, i2=0, i3=0, i4=0, i5=0, i6=0, i7=0, i8=0, i9=0): result = clib.tparm(str, i1, i2, i3, i4, i5, i6, i7, i8, i9) diff --git a/pyrepl/completer.py b/pyrepl/completer.py --- a/pyrepl/completer.py +++ b/pyrepl/completer.py @@ -19,10 +19,12 @@ try: import __builtin__ as builtins + builtins # silence broken pyflakes except ImportError: import builtins -class Completer: + +class Completer(object): def __init__(self, ns): self.ns = ns @@ -79,11 +81,10 @@ matches.append("%s.%s" % (expr, word)) return matches + def get_class_members(klass): ret = dir(klass) if hasattr(klass, '__bases__'): for base in klass.__bases__: ret = ret + get_class_members(base) return ret - - diff --git a/pyrepl/completing_reader.py b/pyrepl/completing_reader.py --- a/pyrepl/completing_reader.py +++ b/pyrepl/completing_reader.py @@ -18,11 +18,12 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +import re from pyrepl import commands, reader from pyrepl.reader import Reader -def prefix(wordlist, j = 0): +def prefix(wordlist, j=0): d = {} i = j try: @@ -36,14 +37,18 @@ except IndexError: return wordlist[0][j:i] -import re + +STRIPCOLOR_REGEX = re.compile(r"\x1B\[([0-9]{1,3}(;[0-9]{1,2})?)?[m|K]") + + def stripcolor(s): - return stripcolor.regexp.sub('', s) -stripcolor.regexp = re.compile(r"\x1B\[([0-9]{1,3}(;[0-9]{1,2})?)?[m|K]") + return STRIPCOLOR_REGEX.regexp.sub('', s) + def real_len(s): return len(stripcolor(s)) + def left_align(s, maxlen): stripped = stripcolor(s) if len(stripped) > maxlen: @@ -52,6 +57,7 @@ padding = maxlen - len(stripped) return s + ' '*padding + def build_menu(cons, wordlist, start, use_brackets, sort_in_column): if use_brackets: item = "[ %s ]" @@ -66,14 +72,14 @@ if sort_in_column: # sort_in_column=False (default) sort_in_column=True # A B C A D G - # D E F B E + # D E F B E # G C F # # "fill" the table with empty words, so we always have the same amout # of rows for each column missing = cols*rows - len(wordlist) wordlist = wordlist + ['']*missing - indexes = [(i%cols)*rows + i//cols for i in range(len(wordlist))] + indexes = [(i % cols) * rows + i // cols for i in range(len(wordlist))] wordlist = [wordlist[i] for i in indexes] menu = [] i = start @@ -84,14 +90,14 @@ i += 1 if i >= len(wordlist): break - menu.append( ''.join(row) ) + menu.append(''.join(row)) if i >= len(wordlist): i = 0 break if r + 5 > cons.height: - menu.append(" %d more... "%(len(wordlist) - i)) + menu.append(" %d more... " % (len(wordlist) - i)) break - return menu, i + return menu, i # this gets somewhat user interface-y, and as a result the logic gets # very convoluted. @@ -118,7 +124,7 @@ # only if the ``assume_immutable_completions`` is True. # # now it gets complicated. -# +# # for the first press of a completion key: # if there's a common prefix, stick it in. @@ -140,22 +146,22 @@ # for subsequent bangs, rotate the menu around (if there are sufficient # choices). + class complete(commands.Command): def do(self): r = self.reader + last_is_completer = r.last_command_is(self.__class__) + immutable_completions = r.assume_immutable_completions + completions_unchangable = last_is_completer and immutable_completions stem = r.get_stem() - if r.assume_immutable_completions and \ - r.last_command_is(self.__class__): - completions = r.cmpltn_menu_choices - else: - r.cmpltn_menu_choices = completions = \ - r.get_completions(stem) - if len(completions) == 0: + if not completions_unchangable: + r.cmpltn_menu_choices = r.get_completions(stem) + + completions = r.cmpltn_menu_choices + if not completions: r.error("no matches") elif len(completions) == 1: - if r.assume_immutable_completions and \ - len(completions[0]) == len(stem) and \ - r.last_command_is(self.__class__): + if completions_unchangable and len(completions[0]) == len(stem): r.msg = "[ sole completion ]" r.dirty = 1 r.insert(completions[0][len(stem):]) @@ -163,7 +169,7 @@ p = prefix(completions, len(stem)) if p: r.insert(p) - if r.last_command_is(self.__class__): + if last_is_completer: if not r.cmpltn_menu_vis: r.cmpltn_menu_vis = 1 r.cmpltn_menu, r.cmpltn_menu_end = build_menu( @@ -177,6 +183,7 @@ r.msg = "[ not unique ]" r.dirty = 1 + class self_insert(commands.self_insert): def do(self): commands.self_insert.do(self) @@ -195,6 +202,7 @@ else: r.cmpltn_reset() + class CompletingReader(Reader): """Adds completion support @@ -204,26 +212,25 @@ """ # see the comment for the complete command assume_immutable_completions = True - use_brackets = True # display completions inside [] + use_brackets = True # display completions inside [] sort_in_column = False - + def collect_keymap(self): return super(CompletingReader, self).collect_keymap() + ( (r'\t', 'complete'),) - + def __init__(self, console): super(CompletingReader, self).__init__(console) self.cmpltn_menu = ["[ menu 1 ]", "[ menu 2 ]"] self.cmpltn_menu_vis = 0 self.cmpltn_menu_end = 0 - for c in [complete, self_insert]: + for c in (complete, self_insert): self.commands[c.__name__] = c - self.commands[c.__name__.replace('_', '-')] = c + self.commands[c.__name__.replace('_', '-')] = c def after_command(self, cmd): super(CompletingReader, self).after_command(cmd) - if not isinstance(cmd, self.commands['complete']) \ - and not isinstance(cmd, self.commands['self_insert']): + if not isinstance(cmd, (complete, self_insert)): self.cmpltn_reset() def calc_screen(self): @@ -243,7 +250,7 @@ self.cmpltn_menu = [] self.cmpltn_menu_vis = 0 self.cmpltn_menu_end = 0 - self.cmpltn_menu_choices = [] + self.cmpltn_menu_choices = [] def get_stem(self): st = self.syntax_table @@ -257,11 +264,14 @@ def get_completions(self, stem): return [] + def test(): class TestReader(CompletingReader): def get_completions(self, stem): - return [s for l in map(lambda x:x.split(),self.history) - for s in l if s and s.startswith(stem)] + return [s for l in self.history + for s in l.split() + if s and s.startswith(stem)] + reader = TestReader() reader.ps1 = "c**> " reader.ps2 = "c/*> " @@ -270,5 +280,6 @@ while reader.readline(): pass -if __name__=='__main__': + +if __name__ == '__main__': test() diff --git a/pyrepl/console.py b/pyrepl/console.py --- a/pyrepl/console.py +++ b/pyrepl/console.py @@ -17,6 +17,7 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + class Event(object): """An Event. `evt' is 'key' or somesuch.""" __slots__ = 'evt', 'data', 'raw' @@ -27,7 +28,8 @@ self.raw = raw def __repr__(self): - return 'Event(%r, %r)'%(self.evt, self.data) + return 'Event(%r, %r)' % (self.evt, self.data) + class Console(object): """Attributes: @@ -36,7 +38,7 @@ height, width, """ - + def refresh(self, screen, xy): pass From noreply at buildbot.pypy.org Tue Jan 15 11:55:15 2013 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Tue, 15 Jan 2013 11:55:15 +0100 (CET) Subject: [pypy-commit] pyrepl codecheck-clean: clean up unix event queue Message-ID: <20130115105515.1B32D1C0131@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: codecheck-clean Changeset: r215:034dcaf7d07c Date: 2013-01-15 11:37 +0100 http://bitbucket.org/pypy/pyrepl/changeset/034dcaf7d07c/ Log: clean up unix event queue diff --git a/pyrepl/unix_eventqueue.py b/pyrepl/unix_eventqueue.py --- a/pyrepl/unix_eventqueue.py +++ b/pyrepl/unix_eventqueue.py @@ -21,6 +21,8 @@ # Bah, this would be easier to test if curses/terminfo didn't have so # much non-introspectable global state. +from collections import deque + from pyrepl import keymap from pyrepl.console import Event from pyrepl import curses @@ -34,23 +36,23 @@ _keynames = { - "delete" : "kdch1", - "down" : "kcud1", - "end" : "kend", - "enter" : "kent", - "f1" : "kf1", "f2" : "kf2", "f3" : "kf3", "f4" : "kf4", - "f5" : "kf5", "f6" : "kf6", "f7" : "kf7", "f8" : "kf8", - "f9" : "kf9", "f10" : "kf10", "f11" : "kf11", "f12" : "kf12", - "f13" : "kf13", "f14" : "kf14", "f15" : "kf15", "f16" : "kf16", - "f17" : "kf17", "f18" : "kf18", "f19" : "kf19", "f20" : "kf20", - "home" : "khome", - "insert" : "kich1", - "left" : "kcub1", - "page down" : "knp", - "page up" : "kpp", - "right" : "kcuf1", - "up" : "kcuu1", - } + "delete": "kdch1", + "down": "kcud1", + "end": "kend", + "enter": "kent", + "home": "khome", + "insert": "kich1", + "left": "kcub1", + "page down": "knp", + "page up": "kpp", + "right": "kcuf1", + "up": "kcuu1", +} + + +#function keys x in 1-20 -> fX: kfX +_keynames.update(('f%d' % i, 'kf%d' % i) for i in range(1, 21)) + def general_keycodes(): keycodes = {} @@ -62,7 +64,6 @@ return keycodes - def EventQueue(fd, encoding): keycodes = general_keycodes() if os.isatty(fd): @@ -72,16 +73,17 @@ trace('keymap {k!r}', k=k) return EncodedQueue(k, encoding) + class EncodedQueue(object): def __init__(self, keymap, encoding): self.k = self.ck = keymap - self.events = [] + self.events = deque() self.buf = bytearray() - self.encoding=encoding + self.encoding = encoding def get(self): if self.events: - return self.events.pop(0) + return self.events.popleft() else: return None From noreply at buildbot.pypy.org Tue Jan 15 11:55:16 2013 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Tue, 15 Jan 2013 11:55:16 +0100 (CET) Subject: [pypy-commit] pyrepl commands-as-functions: turn most commands into functions Message-ID: <20130115105516.256E51C0131@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: commands-as-functions Changeset: r216:c1f83654d126 Date: 2013-01-15 11:38 +0100 http://bitbucket.org/pypy/pyrepl/changeset/c1f83654d126/ Log: turn most commands into functions diff --git a/pyrepl/commands.py b/pyrepl/commands.py --- a/pyrepl/commands.py +++ b/pyrepl/commands.py @@ -19,7 +19,8 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -import sys, os +import os +import signal # Catgories of actions: # killing @@ -30,6 +31,7 @@ # finishing # [completion] + class Command(object): finish = 0 kills_digit_arg = 1 @@ -42,137 +44,168 @@ def do(self): pass + @classmethod + def usereader(cls, func): + is_method = 'self' in func.__code__.co_varnames + + def wrap(f): + if is_method: + return f + else: + return staticmethod(f) + + class ReaderCommand(cls): + do_reader = wrap(func) + + def do(self): + self.do_reader(self.reader) + + ReaderCommand.__name__ = func.__name__ + return ReaderCommand + + +def kill_range(reader, start, end): + if start == end: + return + r = reader + b = reader.buffer + text = b[start:end] + del b[start:end] + if is_kill(r.last_command): + if start < r.pos: + r.kill_ring[-1] = text + r.kill_ring[-1] + else: + r.kill_ring[-1] = r.kill_ring[-1] + text + else: + r.kill_ring.append(text) + r.pos = start + r.dirty = 1 + + class KillCommand(Command): - def kill_range(self, start, end): - if start == end: - return - r = self.reader - b = r.buffer - text = b[start:end] - del b[start:end] - if is_kill(r.last_command): - if start < r.pos: - r.kill_ring[-1] = text + r.kill_ring[-1] - else: - r.kill_ring[-1] = r.kill_ring[-1] + text - else: - r.kill_ring.append(text) - r.pos = start - r.dirty = 1 + pass + class YankCommand(Command): pass + class MotionCommand(Command): pass + class EditCommand(Command): pass + class FinishCommand(Command): finish = 1 - pass + def is_kill(command): return command and issubclass(command, KillCommand) + def is_yank(command): return command and issubclass(command, YankCommand) # etc + class digit_arg(Command): kills_digit_arg = 0 + def do(self): r = self.reader c = self.event[-1] if c == "-": - if r.arg is not None: + if r.arg is None: + r.arg = -1 + else: r.arg = -r.arg - else: - r.arg = -1 else: d = int(c) if r.arg is None: r.arg = d else: if r.arg < 0: - r.arg = 10*r.arg - d + r.arg = 10 * r.arg - d else: - r.arg = 10*r.arg + d + r.arg = 10 * r.arg + d r.dirty = 1 -class clear_screen(Command): - def do(self): - r = self.reader - r.console.clear() - r.dirty = 1 -class refresh(Command): - def do(self): - self.reader.dirty = 1 + at Command.usereader +def clear_screen(r): + r.console.clear() + r.dirty = 1 -class repaint(Command): - def do(self): - self.reader.dirty = 1 - self.reader.console.repaint_prep() -class kill_line(KillCommand): - def do(self): - r = self.reader - b = r.buffer - eol = r.eol() - for c in b[r.pos:eol]: - if not c.isspace(): - self.kill_range(r.pos, eol) - return - else: - self.kill_range(r.pos, eol+1) + at Command.usereader +def refresh(reader): + reader.dirty = 1 -class unix_line_discard(KillCommand): - def do(self): - r = self.reader - self.kill_range(r.bol(), r.pos) + + at Command.usereader +def repaint(self): + self.reader.dirty = 1 + self.reader.console.repaint_prep() + + + at KillCommand.usereader +def kill_line(r): + b = r.buffer + eol = r.eol() + for c in b[r.pos:eol]: + if not c.isspace(): + kill_range(r, r.pos, eol) + return + else: + kill_range(r, r.pos, eol + 1) + + + at KillCommand.usereader +def unix_line_discard(r): + kill_range(r, r.bol(), r.pos) # XXX unix_word_rubout and backward_kill_word should actually # do different things... -class unix_word_rubout(KillCommand): - def do(self): - r = self.reader - for i in range(r.get_arg()): - self.kill_range(r.bow(), r.pos) -class kill_word(KillCommand): - def do(self): - r = self.reader - for i in range(r.get_arg()): - self.kill_range(r.pos, r.eow()) + at KillCommand.usereader +def unix_word_rubout(r): + for i in range(r.get_arg()): + kill_range(r, r.bow(), r.pos) -class backward_kill_word(KillCommand): - def do(self): - r = self.reader - for i in range(r.get_arg()): - self.kill_range(r.bow(), r.pos) -class yank(YankCommand): - def do(self): - r = self.reader - if not r.kill_ring: - r.error("nothing to yank") - return + at KillCommand.usereader +def kill_word(r): + for i in range(r.get_arg()): + kill_range(r, r.pos, r.eow()) + + + at KillCommand.usereader +def backward_kill_word(r): + for i in range(r.get_arg()): + kill_range(r, r.bow(), r.pos) + + + at YankCommand.usereader +def yank(r): + if r.kill_ring: r.insert(r.kill_ring[-1]) + else: + r.error("nothing to yank") -class yank_pop(YankCommand): - def do(self): - r = self.reader + + at YankCommand.usereader +def yank_pop(r): + if not r.kill_ring: + r.error("nothing to yank") + elif not is_yank(r.last_command): + r.error("previous command was not a yank") + else: b = r.buffer - if not r.kill_ring: - r.error("nothing to yank") - return - if not is_yank(r.last_command): - r.error("previous command was not a yank") - return repl = len(r.kill_ring[-1]) r.kill_ring.insert(0, r.kill_ring.pop()) t = r.kill_ring[-1] @@ -180,208 +213,209 @@ r.pos = r.pos - repl + len(t) r.dirty = 1 -class interrupt(FinishCommand): - def do(self): - import signal - self.reader.console.finish() - os.kill(os.getpid(), signal.SIGINT) -class suspend(Command): - def do(self): - import signal - r = self.reader - p = r.pos + at FinishCommand.usereader +def interrupt(r): + r.console.finish() + os.kill(os.getpid(), signal.SIGINT) + + + at Command.usereader +def suspend(r): + p = r.pos + r.console.finish() + os.kill(os.getpid(), signal.SIGSTOP) + ## this should probably be done + ## in a handler for SIGCONT? + r.console.prepare() + r.pos = p + r.posxy = 0, 0 + r.dirty = 1 + r.console.screen = [] + + + at MotionCommand.usereader +def up(r): + for i in range(r.get_arg()): + bol1 = r.bol() + if bol1 == 0: + if r.historyi > 0: + r.select_item(r.historyi - 1) + return + r.pos = 0 + r.error("start of buffer") + return + bol2 = r.bol(bol1 - 1) + line_pos = r.pos - bol1 + if line_pos > bol1 - bol2 - 1: + r.sticky_y = line_pos + r.pos = bol1 - 1 + else: + r.pos = bol2 + line_pos + + + at MotionCommand.usereader +def down(r): + b = r.buffer + for i in range(r.get_arg()): + bol1 = r.bol() + eol1 = r.eol() + if eol1 == len(b): + if r.historyi < len(r.history): + r.select_item(r.historyi + 1) + r.pos = r.eol(0) + return + r.pos = len(b) + r.error("end of buffer") + return + eol2 = r.eol(eol1 + 1) + if r.pos - bol1 > eol2 - eol1 - 1: + r.pos = eol2 + else: + r.pos = eol1 + (r.pos - bol1) + 1 + + + at MotionCommand.usereader +def left(r): + new_pos = r.pos - r.get_arg() + r.pos = max(0, new_pos) + if new_pos < 0: + r.error("start of buffer") + + + at MotionCommand.usereader +def right(r): + new_pos = r.pos + r.get_arg() + buffsize = len(r.buffer) + r.pos = min(new_pos, buffsize) + if new_pos > buffsize: + r.error("end of buffer") + + + at MotionCommand.usereader +def beginning_of_line(r): + r.pos = r.bol() + + + at MotionCommand.usereader +def end_of_line(r): + r.pos = r.eol() + + + at MotionCommand.usereader +def home(r): + r.pos = 0 + + + at MotionCommand.usereader +def end(r): + r.pos = len(r.buffer) + + + at MotionCommand.usereader +def forward_word(r): + for i in range(r.get_arg()): + r.pos = r.eow() + + + at MotionCommand.usereader +def backward_word(r): + for i in range(r.get_arg()): + r.pos = r.bow() + + + at EditCommand.usereader +def self_insert(self, r): + r.insert(self.event * r.get_arg()) + + + at EditCommand.usereader +def insert_nl(r): + r.insert("\n" * r.get_arg()) + + + at EditCommand.usereader +def transpose_characters(r): + b = r.buffer + s = r.pos - 1 + if s < 0: + r.error("cannot transpose at start of buffer") + else: + if s == len(b): + s -= 1 + t = min(s + r.get_arg(), len(b) - 1) + c = b[s] + del b[s] + b.insert(t, c) + r.pos = t + r.dirty = 1 + + + at EditCommand.usereader +def backspace(r): + b = r.buffer + for i in range(r.get_arg()): + if r.pos > 0: + r.pos -= 1 + del b[r.pos] + r.dirty = 1 + else: + r.error("can't backspace at start") + + + at EditCommand.usereader +def delete(self, r): + b = r.buffer + if r.pos == 0 and not b and self.event[-1] == "\004": + r.update_screen() r.console.finish() - os.kill(os.getpid(), signal.SIGSTOP) - ## this should probably be done - ## in a handler for SIGCONT? - r.console.prepare() - r.pos = p - r.posxy = 0, 0 - r.dirty = 1 - r.console.screen = [] + raise EOFError + for i in range(r.get_arg()): + if r.pos != len(b): + del b[r.pos] + r.dirty = 1 + else: + r.error("end of buffer") -class up(MotionCommand): - def do(self): - r = self.reader - for i in range(r.get_arg()): - bol1 = r.bol() - if bol1 == 0: - if r.historyi > 0: - r.select_item(r.historyi - 1) - return - r.pos = 0 - r.error("start of buffer") - return - bol2 = r.bol(bol1-1) - line_pos = r.pos - bol1 - if line_pos > bol1 - bol2 - 1: - r.sticky_y = line_pos - r.pos = bol1 - 1 - else: - r.pos = bol2 + line_pos - -class down(MotionCommand): - def do(self): - r = self.reader - b = r.buffer - for i in range(r.get_arg()): - bol1 = r.bol() - eol1 = r.eol() - if eol1 == len(b): - if r.historyi < len(r.history): - r.select_item(r.historyi + 1) - r.pos = r.eol(0) - return - r.pos = len(b) - r.error("end of buffer") - return - eol2 = r.eol(eol1+1) - if r.pos - bol1 > eol2 - eol1 - 1: - r.pos = eol2 - else: - r.pos = eol1 + (r.pos - bol1) + 1 - -class left(MotionCommand): - def do(self): - r = self.reader - for i in range(r.get_arg()): - p = r.pos - 1 - if p >= 0: - r.pos = p - else: - self.reader.error("start of buffer") - -class right(MotionCommand): - def do(self): - r = self.reader - b = r.buffer - for i in range(r.get_arg()): - p = r.pos + 1 - if p <= len(b): - r.pos = p - else: - self.reader.error("end of buffer") - -class beginning_of_line(MotionCommand): - def do(self): - self.reader.pos = self.reader.bol() - -class end_of_line(MotionCommand): - def do(self): - r = self.reader - self.reader.pos = self.reader.eol() - -class home(MotionCommand): - def do(self): - self.reader.pos = 0 - -class end(MotionCommand): - def do(self): - self.reader.pos = len(self.reader.buffer) - -class forward_word(MotionCommand): - def do(self): - r = self.reader - for i in range(r.get_arg()): - r.pos = r.eow() - -class backward_word(MotionCommand): - def do(self): - r = self.reader - for i in range(r.get_arg()): - r.pos = r.bow() - -class self_insert(EditCommand): - def do(self): - r = self.reader - r.insert(self.event * r.get_arg()) - -class insert_nl(EditCommand): - def do(self): - r = self.reader - r.insert("\n" * r.get_arg()) - -class transpose_characters(EditCommand): - def do(self): - r = self.reader - b = r.buffer - s = r.pos - 1 - if s < 0: - r.error("cannot transpose at start of buffer") - else: - if s == len(b): - s -= 1 - t = min(s + r.get_arg(), len(b) - 1) - c = b[s] - del b[s] - b.insert(t, c) - r.pos = t - r.dirty = 1 - -class backspace(EditCommand): - def do(self): - r = self.reader - b = r.buffer - for i in range(r.get_arg()): - if r.pos > 0: - r.pos -= 1 - del b[r.pos] - r.dirty = 1 - else: - self.reader.error("can't backspace at start") - -class delete(EditCommand): - def do(self): - r = self.reader - b = r.buffer - if ( r.pos == 0 and len(b) == 0 # this is something of a hack - and self.event[-1] == "\004"): - r.update_screen() - r.console.finish() - raise EOFError - for i in range(r.get_arg()): - if r.pos != len(b): - del b[r.pos] - r.dirty = 1 - else: - self.reader.error("end of buffer") class accept(FinishCommand): - def do(self): - pass + pass -class help(Command): - def do(self): - self.reader.msg = self.reader.help_text - self.reader.dirty = 1 -class invalid_key(Command): - def do(self): - pending = self.reader.console.getpending() - s = ''.join(self.event) + pending.data - self.reader.error("`%r' not bound"%s) + at Command.usereader +def help(reader): + reader.msg = reader.help_text + reader.dirty = 1 -class invalid_command(Command): - def do(self): - s = self.event_name - self.reader.error("command `%s' not known"%s) -class qIHelp(Command): - def do(self): - r = self.reader - r.insert((self.event + r.console.getpending().data) * r.get_arg()) - r.pop_input_trans() + at Command.usereader +def invalid_key(self, reader): + pending = reader.console.getpending() + s = ''.join(self.event) + pending.data + reader.error("`%r' not bound" % s) -from pyrepl import input + + at Command.usereader +def invalid_command(self, reader): + reader.error("command `%s' not known" % self.event_name) + + + at Command.usereader +def qIHelp(self, r): + r.insert((self.event + r.console.getpending().data) * r.get_arg()) + r.pop_input_trans() + class QITrans(object): def push(self, evt): self.evt = evt + def get(self): return ('qIHelp', self.evt.raw) + class quoted_insert(Command): kills_digit_arg = 0 + def do(self): self.reader.push_input_trans(QITrans()) From noreply at buildbot.pypy.org Tue Jan 15 11:55:17 2013 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Tue, 15 Jan 2013 11:55:17 +0100 (CET) Subject: [pypy-commit] pyrepl py3k-readline: temporary allow to use our own curses binding Message-ID: <20130115105517.325801C0131@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: py3k-readline Changeset: r217:ac5047e2ea0f Date: 2013-01-15 11:52 +0100 http://bitbucket.org/pypy/pyrepl/changeset/ac5047e2ea0f/ Log: temporary allow to use our own curses binding diff --git a/pyrepl/curses.py b/pyrepl/curses.py --- a/pyrepl/curses.py +++ b/pyrepl/curses.py @@ -32,5 +32,4 @@ tparm = _curses.tparm error = _curses.error except ImportError: - raise from ._minimal_curses import setupterm, tigetstr, tparm, error From noreply at buildbot.pypy.org Tue Jan 15 11:55:18 2013 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Tue, 15 Jan 2013 11:55:18 +0100 (CET) Subject: [pypy-commit] pyrepl py3k-readline: merge default Message-ID: <20130115105518.52E231C0131@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: py3k-readline Changeset: r218:21b1a3e5cc66 Date: 2013-01-15 11:53 +0100 http://bitbucket.org/pypy/pyrepl/changeset/21b1a3e5cc66/ Log: merge default diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -1,3 +1,4 @@ 9e6ce97035736092e9eb7815816b36bee5e92cb3 pyrepl080 5f47f9f65ff7127668bdeda102e675c23224d321 pyrepl081 61de8bdd14264ab8af5b336be854cb9bfa720542 pyrepl082 +62f2256014af7b74b97c00827f1a7789e00dd814 v0.8.4 diff --git a/CHANGES b/CHANGES --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,11 @@ This file contains more verbose descriptions of the changes between versions than those in README. +New in Version 0.8.x: + + * propper command usage + fix one wrong call (thanks Trundle) + + New in 0.7.3: + ^T at the start of a line should not crash. diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -1,7 +1,3 @@ - Copyright 2000-2002 Michael Hudson mwh at python.net - - All Rights Reserved - Permission to use, copy, modify, and distribute this software and its documentation for any purpose is hereby granted without fee, provided that the above copyright notice appear in all copies and that both diff --git a/README b/README --- a/README +++ b/README @@ -1,6 +1,4 @@ -This is pyrepl 0.8.3, a readline-a-like in Python. - -http://pyrepl.codespeak.net/ +This is pyrepl 0.8.4, a readline-a-like in Python. It requires python 2.7 (or newer) and features: @@ -36,13 +34,6 @@ which will also install the above "pythoni" script. -Please direct comments to micahel at gmail.com. - -(If you've released open source software you'll know how frustrating -it is to see a couple of hundred downloads and only get a handful of -emails, so don't think I'll be irritated by the banality of your -comments!) - Summary of 0.8.4: diff --git a/encopyright.py b/encopyright.py deleted file mode 100644 --- a/encopyright.py +++ /dev/null @@ -1,157 +0,0 @@ -#!/usr/local/bin/python - -# Copyright 2000-2003 Michael Hudson mwh at python.net -# -# All Rights Reserved -# -# -# Permission to use, copy, modify, and distribute this software and -# its documentation for any purpose is hereby granted without fee, -# provided that the above copyright notice appear in all copies and -# that both that copyright notice and this permission notice appear in -# supporting documentation. -# -# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO -# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, -# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER -# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF -# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -import os, time, sys -import py - -header_template = """\ -# Copyright 2000-%(lastyear)s Michael Hudson-Doyle %(others)s -# -# All Rights Reserved -# -# -# Permission to use, copy, modify, and distribute this software and -# its documentation for any purpose is hereby granted without fee, -# provided that the above copyright notice appear in all copies and -# that both that copyright notice and this permission notice appear in -# supporting documentation. -# -# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO -# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, -# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER -# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF -# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.\ -""" - -author_template = "\n#%s%%s"%(' '*(header_template.index("Michael")+1),) - - - -author_map = { - u'mwh': None, - u'micahel': None, - u'Michael Hudson ': None, - u'arigo': u"Armin Rigo", - u'antocuni': u'Antonio Cuni', - u'anto': u'Antonio Cuni', - u'bob': u'Bob Ippolito', - u'fijal': u'Maciek Fijalkowski', - u'agaynor': u'Alex Gaynor', - u'hpk': u'Holger Krekel', - u'Ronny': u'Ronny Pfannschmidt', - u'amauryfa': u"Amaury Forgeot d'Arc", - } - - -def author_revs(path): - proc = py.std.subprocess.Popen([ - 'hg','log', str(path), - '--template', '{author|user} {date}\n', - '-r', 'not keyword("encopyright")', - ], stdout=py.std.subprocess.PIPE) - output, _ = proc.communicate() - lines = output.splitlines() - for line in lines: - try: - name, date = line.split(None, 1) - except ValueError: - pass - else: - if '-' in date: - date = date.split('-')[0] - yield name, float(date) - - -def process(path): - ilines = path.readlines() - revs = sorted(author_revs(path), key=lambda x:x[1]) - modified_year = time.gmtime(revs[-1][1])[0] - if not modified_year: - print 'E: no sensible modified_year found for', path - modified_year = time.gmtime(time.time())[0] - extra_authors = [] - authors = set(rev[0] for rev in revs) - for a in authors: - if a not in author_map: - print 'E: need real name for', a - ea = author_map.get(a) - if ea: - extra_authors.append(ea) - extra_authors.sort() - header = header_template % { - 'lastyear': modified_year, - 'others': ''.join([author_template%ea for ea in extra_authors]) - } - header_lines = header.splitlines() - prelines = [] - old_copyright = [] - - if not ilines: - print "W ignoring empty file", path - return - - i = 0 - - if ilines[0][:2] == '#!': - prelines.append(ilines[0]) - i += 1 - - while i < len(ilines) and not ilines[i].strip(): - prelines.append(ilines[i]) - i += 1 - - while i < len(ilines) and ilines[i][:1] == '#': - old_copyright.append(ilines[i]) - i += 1 - - if abs(len(old_copyright) - len(header_lines)) < 2 + len(extra_authors): - for x, y in zip(old_copyright, header_lines): - if x[:-1] != y: - print "C change needed in", path - ofile = path.open("w") - for l in prelines: - ofile.write(l) - ofile.write(header + "\n") - for l in ilines[i:]: - ofile.write(l) - ofile.close() - break - else: - print "M no change needed in", path - else: - print "A no (c) in", file - with path.open("w") as ofile: - for l in prelines: - ofile.write(l) - ofile.write(header + "\n\n") - for l in ilines[len(prelines):]: - ofile.write(l) - - -for thing in sys.argv[1:]: - path = py.path.local(thing) - if path.check(dir=1): - for item in path.visit('*.py'): - process(item) - elif path.check(file=1, ext='py'): - process(path) diff --git a/pyrepl/cmdrepl.py b/pyrepl/cmdrepl.py --- a/pyrepl/cmdrepl.py +++ b/pyrepl/cmdrepl.py @@ -53,8 +53,8 @@ def get_completions(self, stem): if len(stem) != self.pos: return [] - return cr.uniqify([s for s in self.completions - if s.startswith(stem)]) + return sorted(set(s for s in self.completions + if s.startswith(stem))) def replize(klass, history_across_invocations=1): diff --git a/pyrepl/completing_reader.py b/pyrepl/completing_reader.py --- a/pyrepl/completing_reader.py +++ b/pyrepl/completing_reader.py @@ -21,13 +21,6 @@ from pyrepl import commands, reader from pyrepl.reader import Reader -def uniqify(l): - d = {} - for i in l: - d[i] = 1 - r = d.keys() - r.sort() - return r def prefix(wordlist, j = 0): d = {} diff --git a/pyrepl/module_lister.py b/pyrepl/module_lister.py --- a/pyrepl/module_lister.py +++ b/pyrepl/module_lister.py @@ -17,7 +17,6 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -from pyrepl.completing_reader import uniqify import os, sys # for the completion support. @@ -38,9 +37,7 @@ l.append( prefix + fname ) _packages[prefix + fname] = _make_module_list_dir( file, suffs, prefix + fname + '.' ) - l = uniqify(l) - l.sort() - return l + return sorted(set(l)) def _make_module_list(): import imp diff --git a/pyrepl/python_reader.py b/pyrepl/python_reader.py --- a/pyrepl/python_reader.py +++ b/pyrepl/python_reader.py @@ -155,7 +155,7 @@ return [x[len(mod) + 1:] for x in l if x.startswith(mod + '.' + name)] try: - l = completing_reader.uniqify(self.completer.complete(stem)) + l = sorted(set(self.completer.complete(stem))) return l except (NameError, AttributeError): return [] diff --git a/pyrepl/readline.py b/pyrepl/readline.py --- a/pyrepl/readline.py +++ b/pyrepl/readline.py @@ -200,7 +200,7 @@ except _error: return _old_raw_input(prompt) reader.ps1 = prompt - return reader.readline(reader, startup_hook=self.startup_hook) + return reader.readline(startup_hook=self.startup_hook) def multiline_input(self, more_lines, ps1, ps2, returns_unicode=False): """Read an input on possibly multiple lines, asking for more diff --git a/pyrepl/unix_console.py b/pyrepl/unix_console.py --- a/pyrepl/unix_console.py +++ b/pyrepl/unix_console.py @@ -358,6 +358,7 @@ # per-readline preparations: self.__svtermstate = tcgetattr(self.input_fd) raw = self.__svtermstate.copy() + raw.iflag |= termios.ICRNL raw.iflag &=~ (termios.BRKINT | termios.INPCK | termios.ISTRIP | termios.IXON) raw.oflag &=~ (termios.OPOST) diff --git a/pyrepl_utilsmodule.c b/pyrepl_utilsmodule.c deleted file mode 100644 --- a/pyrepl_utilsmodule.c +++ /dev/null @@ -1,175 +0,0 @@ - -/* Copyright 2000-2001 Michael Hudson mwh at python.net - * - * All Rights Reserved - * - * - * Permission to use, copy, modify, and distribute this software and - * its documentation for any purpose is hereby granted without fee, - * provided that the above copyright notice appear in all copies and - * that both that copyright notice and this permission notice appear in - * supporting documentation. - * - * THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO - * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY - * AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, - * INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER - * RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF - * CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN - * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -#include - -/* nothing here that can't be done in Python, but it helps to be able - to do it quicker */ - -static char* _unctrl_map[255]; - -static PyObject* -pyrepl_utils_init_unctrl_map(PyObject* self, PyObject* args) -{ - PyObject* dict; - PyObject* pyc; - PyObject* pys; - int c = 0; - char cc; - - if (!PyArg_ParseTuple(args, "O", &dict)) { - return NULL; - } - - if (!PyDict_Check(dict)) { - PyErr_SetString(PyExc_TypeError, - "init_unctrl_map: must be dict"); - } - - for (c = 0; c < 256; c++) { - cc = c; - pyc = PyString_FromStringAndSize(&cc,1); - if (!pyc) return NULL; - pys = PyDict_GetItem(dict, pyc); - if (!pys) { - PyErr_Format(PyExc_KeyError, - "%c",cc); - _unctrl_map[0] = NULL; - return NULL; - } - if (!PyString_Check(pys)) { - PyErr_SetString(PyExc_TypeError, - "init_unctrl_map: found non-string"); - } - Py_INCREF(pys); /* this ain't going away */ - _unctrl_map[c] = PyString_AS_STRING(pys); - Py_DECREF(pyc); - } - - Py_INCREF(Py_None); - return Py_None; -} - -static char pyrepl_utils_init_unctrl_map_doc[] = -" init_unctrl_map(unctrl_map:dict) -> None\n\ -\n\ -Call this before calling disp_str."; - -static PyObject* -pyrepl_utils_disp_str(PyObject* self, PyObject* args) -{ - char *s; - char *r; - char **temp; - int slen = 0, rlen = 0; - int i, j, k, n; - PyObject *list, *ret; - - if (!PyArg_ParseTuple(args, "s#", &s, &slen)) { - return NULL; - } - - if (!_unctrl_map[0]) { - PyErr_SetString(PyExc_RuntimeError, - "bad boy!"); - return NULL; - } - - temp = malloc(sizeof(char*)*slen); - if (!temp) { - PyErr_NoMemory(); - return NULL; - } - - for (i = 0; i < slen; i++) { - temp[i] = _unctrl_map[(unsigned char)s[i]]; - rlen += strlen(temp[i]); - } - - r = malloc(rlen + 1); - if (!r) { - free(temp); - PyErr_NoMemory(); - return NULL; - } - - list = PyList_New(rlen); - if (!list) { - free(r); - free(temp); - return NULL; - } - - for (i = 0, j = 0; i < slen; i++) { - n = strlen(temp[i]); - memcpy(&r[j], temp[i], n); - PyList_SET_ITEM(list, j, PyInt_FromLong(1)); - k = j + 1; - j += n; - while (k < j) { - PyList_SET_ITEM(list, k, PyInt_FromLong(0)); - k++; - } - } - - free(temp); - r[rlen] = '\000'; - - ret = Py_BuildValue("(sN)", r, list); - - free(r); - - return ret; -} - -static char pyrepl_utils_disp_str_doc[] = -" disp_str(buffer:string) -> (string, [int])\n\ -\n\ -Return the string that should be the printed represenation of\n\ -|buffer| and a list detailing where the characters of |buffer|\n\ -get used up. E.g:\n\ -\n\ ->>> disp_str('\\003')\n\ -('^C', [1, 0])\n\ -\n\ -the list always contains 0s or 1s at present; it could conceivably\n\ -go higher as and when unicode support happens.\n\ -\n\ -You MUST call init_unctrl_map before using this version."; - - -PyMethodDef pyrepl_utils_methods[] = { - { "init_unctrl_map", pyrepl_utils_init_unctrl_map, - METH_VARARGS, pyrepl_utils_init_unctrl_map_doc }, - { "disp_str", pyrepl_utils_disp_str, - METH_VARARGS, pyrepl_utils_disp_str_doc }, - { NULL, NULL } -}; - -static char pyrepl_utils_doc[] = -"Utilities to help speed up pyrepl."; - -void init_pyrepl_utils(void) -{ - Py_InitModule3("_pyrepl_utils", - pyrepl_utils_methods, - pyrepl_utils_doc); -} diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -42,7 +42,6 @@ description = "A library for building flexible command line interfaces", platforms = ["unix", "linux"], packages = ["pyrepl" ], - #ext_modules = [Extension("_pyrepl_utils", ["pyrepl_utilsmodule.c"])], scripts = ["pythoni", "pythoni1"], long_description = long_desc, ) diff --git a/testing/test_bugs.py b/testing/test_bugs.py --- a/testing/test_bugs.py +++ b/testing/test_bugs.py @@ -17,15 +17,26 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -from .infrastructure import EA, read_spec +from pyrepl.historical_reader import HistoricalReader +from .infrastructure import EA, TestReader, read_spec # this test case should contain as-verbatim-as-possible versions of # (applicable) bug reports import pytest +class HistoricalTestReader(HistoricalReader, TestReader): + pass + @pytest.mark.xfail(reason='event missing', run=False) def test_transpose_at_start(): read_spec([( 'transpose', [EA, '']), ( 'accept', [''])]) +def test_cmd_instantiation_crash(): + spec = [ + ('reverse-history-isearch', ["(r-search `') "]), + (('key', 'left'), ['']), + ('accept', ['']) + ] + read_spec(spec, HistoricalTestReader) diff --git a/testing/test_functional.py b/testing/test_functional.py --- a/testing/test_functional.py +++ b/testing/test_functional.py @@ -1,23 +1,6 @@ # Copyright 2000-2007 Michael Hudson-Doyle # Maciek Fijalkowski -# -# All Rights Reserved -# -# -# Permission to use, copy, modify, and distribute this software and -# its documentation for any purpose is hereby granted without fee, -# provided that the above copyright notice appear in all copies and -# that both that copyright notice and this permission notice appear in -# supporting documentation. -# -# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO -# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, -# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER -# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF -# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - +# License: MIT # some functional tests, to see if this is really working import pytest diff --git a/testing/test_readline.py b/testing/test_readline.py new file mode 100644 --- /dev/null +++ b/testing/test_readline.py @@ -0,0 +1,13 @@ +from pyrepl.readline import _ReadlineWrapper +import os, pty + +def test_raw_input(): + readline_wrapper = _ReadlineWrapper() + master, slave = pty.openpty() + readline_wrapper.f_in = slave + os.write(master, 'input\n') + result = readline_wrapper.raw_input('prompt:') + assert result == 'input' + # A bytes string on python2, a unicode string on python3. + assert isinstance(result, str) + diff --git a/testing/test_unix_reader.py b/testing/test_unix_reader.py --- a/testing/test_unix_reader.py +++ b/testing/test_unix_reader.py @@ -14,4 +14,6 @@ event = q.get() assert q.get() is None + assert event.data == a + assert event.raw == b From noreply at buildbot.pypy.org Tue Jan 15 12:02:02 2013 From: noreply at buildbot.pypy.org (cfbolz) Date: Tue, 15 Jan 2013 12:02:02 +0100 (CET) Subject: [pypy-commit] pypy type-specialized-instances: also add a unicode specialization Message-ID: <20130115110202.5FB0C1C0131@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: type-specialized-instances Changeset: r60080:e66173bbd88b Date: 2013-01-15 12:01 +0100 http://bitbucket.org/pypy/pypy/changeset/e66173bbd88b/ Log: also add a unicode specialization diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py --- a/pypy/objspace/std/mapdict.py +++ b/pypy/objspace/std/mapdict.py @@ -8,6 +8,7 @@ from pypy.objspace.std.dictmultiobject import BaseKeyIterator, BaseValueIterator, BaseItemIterator from pypy.objspace.std.dictmultiobject import _never_equal_to_string from pypy.objspace.std.stringobject import W_StringObject +from pypy.objspace.std.unicodeobject import W_UnicodeObject from pypy.objspace.std.objectobject import W_ObjectObject from pypy.objspace.std.typeobject import TypeCell @@ -140,10 +141,14 @@ attr = cache.get(key, None) if attr is None: # XXX not so nice that the classes have to be listed + # it's necessary because instantiate inside an elidable function + # confuses the JIT if attrclass_key == PlainAttribute.attrclass_key: attr = PlainAttribute((name, index), self) elif attrclass_key == StrAttribute.attrclass_key: attr = StrAttribute((name, index), self) + elif attrclass_key == UnicodeAttribute.attrclass_key: + attr = UnicodeAttribute((name, index), self) else: assert attrclass_key == IntAttribute.attrclass_key attr = IntAttribute((name, index), self) @@ -394,6 +399,26 @@ erased = self.erase_item(self.space.str_w(w_value)) obj._mapdict_write_storage(self.position, erased) +class UnicodeAttribute(AbstractStoredAttribute): + attrclass_key = 3 + + erase_item, unerase_item = rerased.new_erasing_pair("mapdict storage unicode item") + erase_item = staticmethod(erase_item) + unerase_item = staticmethod(unerase_item) + + def read_attr(self, obj): + erased = obj._mapdict_read_storage(self.position) + value = self.unerase_item(erased) + return self.space.wrap(value) + + def write_attr(self, obj, w_value): + if type(w_value) is not W_UnicodeObject: + self._replace(obj, self.selector, w_value) + return + erased = self.erase_item(self.space.unicode_w(w_value)) + obj._mapdict_write_storage(self.position, erased) + + def is_taggable_int(space, w_value): from pypy.objspace.std.intobject import W_IntObject @@ -411,6 +436,8 @@ attrclass = IntAttribute elif type(w_value) is W_StringObject: attrclass = StrAttribute + elif type(w_value) is W_UnicodeObject: + attrclass = UnicodeAttribute return attrclass def _become(w_obj, new_obj): diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py --- a/pypy/objspace/std/test/test_mapdict.py +++ b/pypy/objspace/std/test/test_mapdict.py @@ -467,6 +467,67 @@ # check if 'y' is still reachable assert isinstance(obj1.map.back, StrAttribute) assert space.eq_w(obj1.getdictvalue(space, "y"), space.wrap("b")) + + def test_unicode_attributes(self): + space = self.space + cls = Class(sp=space) + obj1 = cls.instantiate() + obj1.setdictvalue(space, "x", space.wrap(u"a")) + assert space.eq_w(obj1.getdictvalue(space, "x"), space.wrap(u"a")) + + obj2 = cls.instantiate() + w1 = W_Root() + obj2.setdictvalue(space, "x", w1) + assert obj2.getdictvalue(space, "x") is w1 + + assert obj1.map is not obj2.map + assert isinstance(obj1.map, UnicodeAttribute) + + obj3 = cls.instantiate() + obj3.setdictvalue(space, "x", space.wrap(u"a")) + assert space.eq_w(obj3.getdictvalue(space, "x"), space.wrap(u"a")) + + assert obj1.map is obj3.map + + assert UnicodeAttribute.unerase_item(obj1.storage[0]) == "a" + assert PlainAttribute.unerase_item(obj2.storage[0]) == w1 + + def test_overwrite_unicode_attribute_with_another_type(self): + space = self.space + cls = Class(sp=space) + obj1 = cls.instantiate() + + obj1.setdictvalue(space, "x", space.wrap(u"a")) + assert isinstance(obj1.map, UnicodeAttribute) + assert space.eq_w(obj1.getdictvalue(space, "x"), space.wrap(u"a")) + + w1 = W_Root() + obj1.setdictvalue(space, "x", w1) + assert isinstance(obj1.map, PlainAttribute) + assert obj1.getdictvalue(space, "x") is w1 + + def test_overwrite_unicode_attribute_with_another_type2(self): + space = self.space + cls = Class(sp=space) + obj1 = cls.instantiate() + + obj1.setdictvalue(space, "x", space.wrap(u"a")) + assert isinstance(obj1.map, UnicodeAttribute) + assert space.eq_w(obj1.getdictvalue(space, "x"), space.wrap(u"a")) + + obj1.setdictvalue(space, "y", space.wrap(u"b")) + assert isinstance(obj1.map.back, UnicodeAttribute) + assert space.eq_w(obj1.getdictvalue(space, "y"), space.wrap(u"b")) + + # overwrite 'x' with new type + w1 = W_Root() + obj1.setdictvalue(space, "x", w1) + assert isinstance(obj1.map, PlainAttribute) + assert obj1.getdictvalue(space, "x") is w1 + + # check if 'y' is still reachable + assert isinstance(obj1.map.back, UnicodeAttribute) + assert space.eq_w(obj1.getdictvalue(space, "y"), space.wrap(u"b")) # ___________________________________________________________ # dict tests From noreply at buildbot.pypy.org Tue Jan 15 12:24:19 2013 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 15 Jan 2013 12:24:19 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: work on stack check failure, failing to call it so far Message-ID: <20130115112419.2174C1C07B5@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60081:2a740e45803a Date: 2013-01-15 13:23 +0200 http://bitbucket.org/pypy/pypy/changeset/2a740e45803a/ Log: work on stack check failure, failing to call it so far diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py --- a/pypy/jit/backend/llsupport/llmodel.py +++ b/pypy/jit/backend/llsupport/llmodel.py @@ -45,11 +45,24 @@ else: self._setup_exception_handling_untranslated() self.asmmemmgr = AsmMemoryManager() + self._setup_frame_realloc() self.setup() def setup(self): pass + def _setup_frame_realloc(self): + FUNC_TP = lltype.Ptr(lltype.FuncType([llmemory.GCREF], + llmemory.GCREF)) + + def realloc_frame(frame): + frame = lltype.cast_opaque_ptr(jitframe.JITFRAME, frame) + import pdb + pdb.set_trace() + return frame + + f = llhelper(FUNC_TP, realloc_frame) + self.realloc_frame = heaptracker.adr2int(llmemory.cast_ptr_to_adr(f)) def _setup_exception_handling_untranslated(self): # for running un-translated only, all exceptions occurring in the diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py --- a/pypy/jit/backend/test/runner_test.py +++ b/pypy/jit/backend/test/runner_test.py @@ -3766,7 +3766,8 @@ EffectInfo.MOST_GENERAL) def func2(a, b, c, d, e, f, g, h, i, j, k, l): - pass + import pdb + pdb.set_trace() FUNC2 = self.FuncType([lltype.Signed] * 12, lltype.Void) FPTR2 = self.Ptr(FUNC2) diff --git a/pypy/jit/backend/x86/arch.py b/pypy/jit/backend/x86/arch.py --- a/pypy/jit/backend/x86/arch.py +++ b/pypy/jit/backend/x86/arch.py @@ -51,7 +51,7 @@ JITFRAME_FIXED_SIZE = 29 # 13 GPR + 16 XMM # reg, we don't save it else: - # rbp + rbx + r12 + r13 + r14 + r15 + 12 extra words + return address = 18 + # rbp + rbx + r12 + r13 + r14 + r15 + 12 extra words + return address = 19 FRAME_FIXED_SIZE = 19 PASS_ON_MY_FRAME = 12 JITFRAME_FIXED_SIZE = 29 # 13 GPR + 16 XMM diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -94,6 +94,7 @@ self._build_failure_recovery(True) self._build_wb_slowpath(False) self._build_wb_slowpath(True) + self._build_stack_check_failure() if self.cpu.supports_floats: self._build_failure_recovery(False, withfloats=True) self._build_failure_recovery(True, withfloats=True) @@ -167,6 +168,19 @@ self.float_const_neg_addr = float_constants self.float_const_abs_addr = float_constants + 16 + def _build_stack_check_failure(self): + mc = codebuf.MachineCodeBlockWrapper() + base_ofs = self.cpu.get_baseofs_of_frame_field() + self._push_all_regs_to_frame(mc, self.cpu.supports_floats) + assert not IS_X86_32 + # push first arg + mc.LEA_rb(edi.value, -base_ofs) + mc.CALL(imm(self.cpu.realloc_frame)) + mc.LEA_rm(ebp.value, (eax.value, base_ofs)) + self._pop_all_regs_from_frame(mc, self.cpu.supports_floats) + mc.RET() + self._stack_check_failure = mc.materialize(self.cpu.asmmemmgr, []) + def _build_malloc_slowpath(self): # With asmgcc, we need two helpers, so that we can write two CALL # instructions in assembler, with a mark_gc_roots in between. @@ -571,7 +585,7 @@ operations = regalloc.prepare_bridge(inputargs, arglocs, operations, self.current_clt.allgcrefs) - self._check_frame_depth() + stack_check_patch_ofs = self._check_frame_depth() frame_depth = self._assemble(regalloc, operations) codeendpos = self.mc.get_relative_pos() self.write_pending_failure_recoveries() @@ -589,9 +603,10 @@ # patch the jump from original guard self.patch_jump_for_descr(faildescr, rawstart) ops_offset = self.mc.ops_offset - self.fixup_target_tokens(rawstart) frame_depth = max(self.current_clt.frame_info.jfi_frame_depth, frame_depth + JITFRAME_FIXED_SIZE) + self._patch_stackadjust(stack_check_patch_ofs + rawstart, frame_depth) + self.fixup_target_tokens(rawstart) self.current_clt.frame_info.jfi_frame_depth = frame_depth self.teardown() # oprofile support @@ -652,8 +667,21 @@ mc.copy_to_raw_memory(rawstart + pos_after_jz - 4) def _check_frame_depth(self): - pass + descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) + ofs = self.cpu.unpack_fielddescr(descrs.arraydescr.lendescr) + base_ofs = self.cpu.get_baseofs_of_frame_field() + self.mc.CMP_bi(ofs - base_ofs, 0xffffff) + stack_check_cmp_ofs = self.mc.get_relative_pos() - 4 + assert not IS_X86_32 + self.mc.J_il8(rx86.Conditions['G'], 9) + self.mc.CALL(imm(self._stack_check_failure)) + return stack_check_cmp_ofs + def _patch_stackadjust(self, adr, allocated_depth): + mc = codebuf.MachineCodeBlockWrapper() + mc.writeimm32(allocated_depth) + mc.copy_to_raw_memory(adr) + def get_asmmemmgr_blocks(self, looptoken): clt = looptoken.compiled_loop_token if clt.asmmemmgr_blocks is None: @@ -1858,13 +1886,31 @@ def setup_failure_recovery(self): self.failure_recovery_code = [0, 0, 0, 0] + def _push_all_regs_to_frame(self, mc, withfloats): + # Push all general purpose registers + for i, gpr in enumerate(gpr_reg_mgr_cls.all_regs): + mc.MOV_br(i * WORD, gpr.value) + if withfloats: + # Push all XMM regs + ofs = len(gpr_reg_mgr_cls.all_regs) + for i in range(self.cpu.NUM_REGS): + mc.MOVSD_bx((ofs + i) * WORD, i) + + def _pop_all_regs_from_frame(self, mc, withfloats): + # Pop all general purpose registers + for i, gpr in enumerate(gpr_reg_mgr_cls.all_regs): + mc.MOV_rb(gpr.value, i * WORD) + if withfloats: + # Pop all XMM regs + ofs = len(gpr_reg_mgr_cls.all_regs) + for i in range(self.cpu.NUM_REGS): + mc.MOVSD_xb(i, (ofs + i) * WORD) + def _build_failure_recovery(self, exc, withfloats=False): mc = codebuf.MachineCodeBlockWrapper() self.mc = mc - # Push all general purpose registers - for i, gpr in enumerate(gpr_reg_mgr_cls.all_regs): - mc.MOV_br(i * WORD, gpr.value) + self._push_all_regs_to_frame(mc, withfloats) if exc: # We might have an exception pending. Load it into ebx @@ -1872,13 +1918,6 @@ mc.MOV(ebx, heap(self.cpu.pos_exc_value())) mc.MOV(heap(self.cpu.pos_exception()), imm0) mc.MOV(heap(self.cpu.pos_exc_value()), imm0) - - if withfloats: - ofs = len(gpr_reg_mgr_cls.all_regs) - for i in range(self.cpu.NUM_REGS): - mc.MOVSD_bx((ofs + i) * WORD, i) - - if exc: # save ebx into 'jf_guard_exc' offset = self.cpu.get_ofs_of_frame_field('jf_guard_exc') mc.MOV_br(offset, ebx.value) @@ -2007,12 +2046,14 @@ # first, close the stack in the sense of the asmgcc GC root tracker gcrootmap = self.cpu.gc_ll_descr.gcrootmap if gcrootmap: + xxx self.call_release_gil(gcrootmap, arglocs) # do the call fail_index = self._store_force_index(guard_op) self._genop_call(op, arglocs, result_loc, fail_index) # then reopen the stack if gcrootmap: + xxx self.call_reacquire_gil(gcrootmap, result_loc) # finally, the guard_not_forced self._emit_guard_not_forced(guard_token) diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py --- a/pypy/jit/backend/x86/regalloc.py +++ b/pypy/jit/backend/x86/regalloc.py @@ -212,14 +212,10 @@ self.min_bytes_before_label = max(self.min_bytes_before_label, at_least_position) - def needed_extra_stack_locations(self, n): - # call *after* you needed extra stack locations: (%esp), (%esp+4)... - min_frame_depth = self.fm.get_frame_depth() + n + def get_final_frame_depth(self): + min_frame_depth = self.fm.get_frame_depth() if min_frame_depth > self.min_frame_depth: self.min_frame_depth = min_frame_depth - - def get_final_frame_depth(self): - self.needed_extra_stack_locations(0) # update min_frame_depth return self.min_frame_depth def _set_initial_bindings(self, inputargs): diff --git a/pypy/jit/backend/x86/test/test_runner.py b/pypy/jit/backend/x86/test/test_runner.py --- a/pypy/jit/backend/x86/test/test_runner.py +++ b/pypy/jit/backend/x86/test/test_runner.py @@ -34,7 +34,7 @@ # ====> ../../test/runner_test.py add_loop_instructions = ['mov', 'add', 'test', 'je', 'jmp'] - bridge_loop_instructions = ['mov', 'jmp'] + bridge_loop_instructions = ['cmp', 'jg', 'mov', 'call', 'mov', 'jmp'] def setup_method(self, meth): self.cpu = CPU(rtyper=None, stats=FakeStats()) From noreply at buildbot.pypy.org Tue Jan 15 13:10:14 2013 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 15 Jan 2013 13:10:14 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fixes Message-ID: <20130115121014.A2CFE1C02C4@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60082:bc9a7d2f84ff Date: 2013-01-15 14:09 +0200 http://bitbucket.org/pypy/pypy/changeset/bc9a7d2f84ff/ Log: fixes diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py --- a/pypy/jit/backend/llsupport/llmodel.py +++ b/pypy/jit/backend/llsupport/llmodel.py @@ -56,10 +56,22 @@ llmemory.GCREF)) def realloc_frame(frame): - frame = lltype.cast_opaque_ptr(jitframe.JITFRAME, frame) - import pdb - pdb.set_trace() - return frame + frame = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, frame) + frame_info = frame.jf_frame_info + new_frame = lltype.malloc(jitframe.JITFRAME, + frame_info.jfi_frame_depth) + new_frame.jf_frame_info = frame_info + # we need to do this, because we're not sure what things + # are GC pointers and which ones are not + llop.gc_writebarrier_before_copy(lltype.Bool, frame, new_frame, + 0, 0, len(frame.jf_frame)) + i = 0 + while i < len(frame.jf_frame): + new_frame.jf_frame[i] = frame.jf_frame[i] + i += 1 + new_frame.jf_savedata = frame.jf_savedata + # all other fields are empty + return lltype.cast_opaque_ptr(llmemory.GCREF, new_frame) f = llhelper(FUNC_TP, realloc_frame) self.realloc_frame = heaptracker.adr2int(llmemory.cast_ptr_to_adr(f)) diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py --- a/pypy/jit/backend/test/runner_test.py +++ b/pypy/jit/backend/test/runner_test.py @@ -3766,8 +3766,7 @@ EffectInfo.MOST_GENERAL) def func2(a, b, c, d, e, f, g, h, i, j, k, l): - import pdb - pdb.set_trace() + pass FUNC2 = self.FuncType([lltype.Signed] * 12, lltype.Void) FPTR2 = self.Ptr(FUNC2) @@ -3786,4 +3785,11 @@ """, namespace=locals()) self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) frame = self.cpu.execute_token(looptoken, 0, 0, 3) - assert self.cpu.get_latest_descr(frame) is guarddescr + #assert self.cpu.get_latest_descr(frame) is guarddescr + from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU + + if not isinstance(self.cpu, AbstractLLCPU): + py.test.skip("pointless test on non-asm") + + frame = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, frame) + assert len(frame.jf_frame) == frame.jf_frame_info.jfi_frame_depth diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -175,7 +175,9 @@ assert not IS_X86_32 # push first arg mc.LEA_rb(edi.value, -base_ofs) + mc.SUB_ri(esp.value, WORD) # we need that cause we're inside a call mc.CALL(imm(self.cpu.realloc_frame)) + mc.ADD_ri(esp.value, WORD) mc.LEA_rm(ebp.value, (eax.value, base_ofs)) self._pop_all_regs_from_frame(mc, self.cpu.supports_floats) mc.RET() diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py --- a/pypy/rpython/lltypesystem/opimpl.py +++ b/pypy/rpython/lltypesystem/opimpl.py @@ -524,9 +524,12 @@ source_start, dest_start, length): A = lltype.typeOf(source) assert A == lltype.typeOf(dest) - assert isinstance(A.TO, lltype.GcArray) - assert isinstance(A.TO.OF, lltype.Ptr) - assert A.TO.OF.TO._gckind == 'gc' + if isinstance(A.TO, lltype.GcArray): + assert isinstance(A.TO.OF, lltype.Ptr) + assert A.TO.OF.TO._gckind == 'gc' + else: + assert isinstance(A.TO, lltype.GcStruct) + assert hasattr(A.TO, '_arrayfld') assert type(source_start) is int assert type(dest_start) is int assert type(length) is int From noreply at buildbot.pypy.org Tue Jan 15 13:10:15 2013 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 15 Jan 2013 13:10:15 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: reenable the assert Message-ID: <20130115121015.E12681C02C4@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60083:6a5398f7e828 Date: 2013-01-15 14:09 +0200 http://bitbucket.org/pypy/pypy/changeset/6a5398f7e828/ Log: reenable the assert diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py --- a/pypy/jit/backend/test/runner_test.py +++ b/pypy/jit/backend/test/runner_test.py @@ -3785,7 +3785,7 @@ """, namespace=locals()) self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) frame = self.cpu.execute_token(looptoken, 0, 0, 3) - #assert self.cpu.get_latest_descr(frame) is guarddescr + assert self.cpu.get_latest_descr(frame) is guarddescr from pypy.jit.backend.llsupport.llmodel import AbstractLLCPU if not isinstance(self.cpu, AbstractLLCPU): From noreply at buildbot.pypy.org Tue Jan 15 13:27:13 2013 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 15 Jan 2013 13:27:13 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fix subtle bug Message-ID: <20130115122713.25D1B1C068C@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60084:904929d995dc Date: 2013-01-15 14:26 +0200 http://bitbucket.org/pypy/pypy/changeset/904929d995dc/ Log: fix subtle bug diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -1879,7 +1879,7 @@ pos = pos // WORD - GPR_REGS locs.append(xmm_reg_mgr_cls.all_regs[pos]) else: - i = pos // WORD - 2 * self.cpu.NUM_REGS + i = pos // WORD - GPR_REGS - XMM_REGS tp = inputargs[input_i].type locs.append(StackLoc(i, pos, tp)) input_i += 1 From noreply at buildbot.pypy.org Tue Jan 15 18:22:34 2013 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 15 Jan 2013 18:22:34 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: some test_random fixes Message-ID: <20130115172234.972861C0131@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60085:cf18e5dc7c50 Date: 2013-01-15 14:42 +0200 http://bitbucket.org/pypy/pypy/changeset/cf18e5dc7c50/ Log: some test_random fixes diff --git a/pypy/jit/backend/test/test_random.py b/pypy/jit/backend/test/test_random.py --- a/pypy/jit/backend/test/test_random.py +++ b/pypy/jit/backend/test/test_random.py @@ -2,7 +2,7 @@ import pytest from pypy.rlib.rarithmetic import intmask, LONG_BIT from pypy.rpython.lltypesystem import llmemory -from pypy.jit.metainterp.history import BasicFailDescr, TreeLoop +from pypy.jit.metainterp.history import BasicFailDescr, TreeLoop, BasicFinalDescr from pypy.jit.metainterp.history import BoxInt, ConstInt, JitCellToken from pypy.jit.metainterp.history import BoxPtr, ConstPtr, TargetToken from pypy.jit.metainterp.history import BoxFloat, ConstFloat, Const @@ -198,9 +198,9 @@ for i, v in enumerate(fail_args): if isinstance(v, (BoxFloat, ConstFloat)): print >>s, (' assert longlong.getrealfloat(' - 'cpu.get_latest_value_float(%d)) == %r' % (i, v.value)) + 'cpu.get_float_value(%d)) == %r' % (i, v.value)) else: - print >>s, (' assert cpu.get_latest_value_int(%d) == %d' + print >>s, (' assert cpu.get_int_value(%d) == %d' % (i, v.value)) self.names = names if pytest.config.option.output: @@ -619,7 +619,7 @@ endvars.append(v) r.shuffle(endvars) loop.operations.append(ResOperation(rop.FINISH, endvars, None, - descr=BasicFailDescr())) + descr=BasicFinalDescr())) if builder.should_fail_by: self.should_fail_by = builder.should_fail_by self.guard_op = builder.guard_op @@ -686,9 +686,9 @@ self.should_fail_by.getdescr())) for i, v in enumerate(self.get_fail_args()): if isinstance(v, (BoxFloat, ConstFloat)): - value = cpu.get_latest_value_float(deadframe, i) + value = cpu.get_float_value(deadframe, i) else: - value = cpu.get_latest_value_int(deadframe, i) + value = cpu.get_int_value(deadframe, i) do_assert(value == self.expected[v], "Got %r, expected %r for value #%d" % (value, self.expected[v], From noreply at buildbot.pypy.org Tue Jan 15 18:22:36 2013 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 15 Jan 2013 18:22:36 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: Merge default and fix Message-ID: <20130115172236.5EFE11C0275@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60086:86e82d36c7c9 Date: 2013-01-15 19:22 +0200 http://bitbucket.org/pypy/pypy/changeset/86e82d36c7c9/ Log: Merge default and fix diff --git a/pypy/jit/backend/test/test_random.py b/pypy/jit/backend/test/test_random.py --- a/pypy/jit/backend/test/test_random.py +++ b/pypy/jit/backend/test/test_random.py @@ -26,6 +26,8 @@ class OperationBuilder(object): def __init__(self, cpu, loop, vars): self.cpu = cpu + if not hasattr(cpu, '_faildescr_keepalive'): + cpu._faildescr_keepalive = [] self.fakemetainterp = FakeMetaInterp() self.loop = loop self.intvars = [box for box in vars if isinstance(box, BoxInt)] @@ -206,6 +208,14 @@ if pytest.config.option.output: s.close() + def getfaildescr(self, is_finish=False): + if is_finish: + descr = BasicFinalDescr() + else: + descr = BasicFailDescr() + self.cpu._faildescr_keepalive.append(descr) + return descr + class CannotProduceOperation(Exception): pass @@ -287,7 +297,7 @@ builder.intvars[:] = original_intvars else: op = ResOperation(rop.GUARD_NO_OVERFLOW, [], None) - op.setdescr(BasicFailDescr()) + op.setdescr(builder.getfaildescr()) op.setfailargs(fail_subset) builder.loop.operations.append(op) @@ -357,7 +367,7 @@ def produce_into(self, builder, r): op, passing = self.gen_guard(builder, r) builder.loop.operations.append(op) - op.setdescr(BasicFailDescr()) + op.setdescr(builder.getfaildescr()) op.setfailargs(builder.subset_of_intvars(r)) if not passing: builder.should_fail_by = op @@ -618,8 +628,9 @@ if v not in used_later: endvars.append(v) r.shuffle(endvars) + endvars = endvars[:1] loop.operations.append(ResOperation(rop.FINISH, endvars, None, - descr=BasicFinalDescr())) + descr=builder.getfaildescr(is_finish=True))) if builder.should_fail_by: self.should_fail_by = builder.should_fail_by self.guard_op = builder.guard_op @@ -712,7 +723,7 @@ else: op = ResOperation(rop.GUARD_EXCEPTION, [guard_op._exc_box], BoxPtr()) - op.setdescr(BasicFailDescr()) + op.setdescr(self.builder.getfaildescr()) op.setfailargs([]) return op diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -675,8 +675,13 @@ self.mc.CMP_bi(ofs - base_ofs, 0xffffff) stack_check_cmp_ofs = self.mc.get_relative_pos() - 4 assert not IS_X86_32 - self.mc.J_il8(rx86.Conditions['G'], 9) + self.mc.J_il8(rx86.Conditions['G'], 0) + jg_location = self.mc.get_relative_pos() self.mc.CALL(imm(self._stack_check_failure)) + # patch the JG above + offset = self.mc.get_relative_pos() - jg_location + assert 0 < offset <= 127 + self.mc.overwrite(jg_location-1, chr(offset)) return stack_check_cmp_ofs def _patch_stackadjust(self, adr, allocated_depth): @@ -1860,9 +1865,9 @@ else: v = gpr_reg_mgr_cls.all_reg_indexes[loc.value] positions[i] = v * WORD + # write down the positions of locs guardtok.faildescr.rd_locs = positions - # write fail_index too - # for testing the decoding, write a final byte 0xCC + # write down the GC pattern return startpos def rebuild_faillocs_from_descr(self, descr, inputargs): diff --git a/pypy/jit/metainterp/test/test_fficall.py b/pypy/jit/metainterp/test/test_fficall.py --- a/pypy/jit/metainterp/test/test_fficall.py +++ b/pypy/jit/metainterp/test/test_fficall.py @@ -106,7 +106,7 @@ class TestFfiCall(FfiCallTests, LLJitMixin): - def test_jit_fii_vref(self): + def test_jit_ffi_vref(self): py.test.skip("unsupported so far") from pypy.rlib import clibffi from pypy.rlib.jit_libffi import jit_ffi_prep_cif, jit_ffi_call @@ -138,9 +138,10 @@ res = exb[3] lltype.free(exb, flavor='raw') # - lltype.free(atypes, flavor='raw') return res # res = self.interp_operations(f, []) lltype.free(cd, flavor='raw') assert res == math.sin(1.23) + + lltype.free(atypes, flavor='raw') diff --git a/pypy/jit/metainterp/test/test_warmspot.py b/pypy/jit/metainterp/test/test_warmspot.py --- a/pypy/jit/metainterp/test/test_warmspot.py +++ b/pypy/jit/metainterp/test/test_warmspot.py @@ -1,6 +1,6 @@ import py from pypy.jit.metainterp.warmspot import get_stats -from pypy.rlib.jit import JitDriver, set_param, unroll_safe +from pypy.rlib.jit import JitDriver, set_param, unroll_safe, jit_callback from pypy.jit.backend.llgraph import runner from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin @@ -537,41 +537,21 @@ def test_callback_jit_merge_point(self): - from pypy.rlib.objectmodel import register_around_callback_hook - from pypy.rpython.lltypesystem import lltype, rffi - from pypy.translator.tool.cbuild import ExternalCompilationInfo - - callback_jit_driver = JitDriver(greens = ['name'], reds = 'auto') - - def callback_merge_point(name): - callback_jit_driver.jit_merge_point(name=name) - - @callback_jit_driver.inline(callback_merge_point) - def callback_hook(name): - pass - + @jit_callback("testing") def callback(a, b): if a > b: return 1 return -1 - CB_TP = rffi.CCallback([lltype.Signed, lltype.Signed], lltype.Signed) - eci = ExternalCompilationInfo(includes=['stdlib.h']) - qsort = rffi.llexternal('qsort', - [rffi.VOIDP, lltype.Signed, lltype.Signed, - CB_TP], lltype.Void, compilation_info=eci) - ARR = rffi.CArray(lltype.Signed) + def main(): + total = 0 + for i in range(10): + total += callback(i, 2) + return total - def main(): - register_around_callback_hook(callback_hook) - raw = lltype.malloc(ARR, 10, flavor='raw') - for i in range(10): - raw[i] = 10 - i - qsort(raw, 10, rffi.sizeof(lltype.Signed), callback) - lltype.free(raw, flavor='raw') - - self.meta_interp(main, []) - self.check_trace_count(1) + res = self.meta_interp(main, []) + assert res == 7 - 3 + self.check_trace_count(2) class TestLLWarmspot(WarmspotTests, LLJitMixin): diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py --- a/pypy/jit/metainterp/warmspot.py +++ b/pypy/jit/metainterp/warmspot.py @@ -360,13 +360,7 @@ jd._jit_merge_point_in = graph args = op.args[2:] s_binding = self.translator.annotator.binding - if op.args[1].value.autoreds: - # _portal_args_s is used only by _make_hook_graph, but for now we - # declare the various set_jitcell_at, get_printable_location, - # etc. as incompatible with autoreds - jd._portal_args_s = None - else: - jd._portal_args_s = [s_binding(v) for v in args] + jd._portal_args_s = [s_binding(v) for v in args] graph = copygraph(graph) [jmpp] = find_jit_merge_points([graph]) graph.startblock = support.split_before_jit_merge_point(*jmpp) @@ -614,9 +608,10 @@ if func is None: return None # - assert not jitdriver_sd.jitdriver.autoreds, ( - "reds='auto' is not compatible with JitDriver hooks such as " - "{get,set}_jitcell_at, get_printable_location, confirm_enter_jit, etc.") + if not onlygreens: + assert not jitdriver_sd.jitdriver.autoreds, ( + "reds='auto' is not compatible with JitDriver hooks such as " + "confirm_enter_jit") extra_args_s = [] if s_first_arg is not None: extra_args_s.append(s_first_arg) diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py --- a/pypy/module/_cffi_backend/ccallback.py +++ b/pypy/module/_cffi_backend/ccallback.py @@ -152,6 +152,7 @@ STDERR = 2 + at jit.jit_callback("CFFI") def invoke_callback(ffi_cif, ll_res, ll_args, ll_userdata): """ Callback specification. ffi_cif - something ffi specific, don't care diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py --- a/pypy/module/pyexpat/interp_pyexpat.py +++ b/pypy/module/pyexpat/interp_pyexpat.py @@ -2,7 +2,7 @@ from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault from pypy.interpreter.error import OperationError -from pypy.rlib import rgc +from pypy.rlib import rgc, jit from pypy.rpython.lltypesystem import rffi, lltype from pypy.rpython.tool import rffi_platform from pypy.translator.tool.cbuild import ExternalCompilationInfo @@ -288,6 +288,7 @@ post_code = '' src = py.code.Source(""" + @jit.jit_callback('XML:%(name)s') def %(name)s_callback(%(first_arg)s, %(args)s): id = rffi.cast(lltype.Signed, %(ll_id)s) userdata = global_storage.get_object(id) diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py --- a/pypy/module/pypyjit/interp_jit.py +++ b/pypy/module/pypyjit/interp_jit.py @@ -6,7 +6,7 @@ from pypy.tool.pairtype import extendabletype from pypy.rlib.rarithmetic import r_uint, intmask from pypy.rlib.jit import JitDriver, hint, we_are_jitted, dont_look_inside -from pypy.rlib import jit, objectmodel +from pypy.rlib import jit from pypy.rlib.jit import current_trace_length, unroll_parameters import pypy.interpreter.pyopcode # for side-effects from pypy.interpreter.error import OperationError, operationerrfmt @@ -97,15 +97,6 @@ is_being_profiled=self.is_being_profiled) return jumpto -callback_jit_driver = JitDriver(greens = ['name'], reds = 'auto') - -def callback_merge_point(name): - callback_jit_driver.jit_merge_point(name=name) - - at callback_jit_driver.inline(callback_merge_point) -def callback_hook(name): - pass - def _get_adapted_tick_counter(): # Normally, the tick counter is decremented by 100 for every # Python opcode. Here, to better support JIT compilation of diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py --- a/pypy/module/signal/interp_signal.py +++ b/pypy/module/signal/interp_signal.py @@ -40,6 +40,7 @@ includes = ['stdlib.h', 'src/signals.h'] if sys.platform != 'win32': includes.append('sys/time.h') +WIN32 = sys.platform == 'win32' cdir = py.path.local(autopath.pypydir).join('translator', 'c') @@ -236,7 +237,10 @@ None -- if an unknown handler is in effect (XXX UNIMPLEMENTED) anything else -- the callable Python object used as a handler """ - check_signum_in_range(space, signum) + if WIN32: + check_signum_exists(space, signum) + else: + check_signum_in_range(space, signum) action = space.check_signal_action if signum in action.handlers_w: return action.handlers_w[signum] diff --git a/pypy/module/signal/test/test_signal.py b/pypy/module/signal/test/test_signal.py --- a/pypy/module/signal/test/test_signal.py +++ b/pypy/module/signal/test/test_signal.py @@ -55,7 +55,7 @@ skip("requires os.kill() and os.getpid()") signal = self.signal # the signal module to test if not hasattr(signal, 'SIGUSR1'): - py.test.skip("requires SIGUSR1 in signal") + skip("requires SIGUSR1 in signal") signum = signal.SIGUSR1 received = [] @@ -156,6 +156,7 @@ import sys if sys.platform == 'win32': raises(ValueError, signal, 42, lambda *args: None) + raises(ValueError, signal, 7, lambda *args: None) else: signal(42, lambda *args: None) signal(42, SIG_DFL) diff --git a/pypy/module/thread/os_thread.py b/pypy/module/thread/os_thread.py --- a/pypy/module/thread/os_thread.py +++ b/pypy/module/thread/os_thread.py @@ -2,6 +2,7 @@ Thread support based on OS-level threads. """ +import os from pypy.module.thread import ll_thread as thread from pypy.module.thread.error import wrap_thread_error from pypy.interpreter.error import OperationError, operationerrfmt @@ -91,14 +92,18 @@ # run! try: bootstrapper.run(space, w_callable, args) - finally: - bootstrapper.nbthreads -= 1 - # clean up space.threadlocals to remove the ExecutionContext - # entry corresponding to the current thread + # done + except Exception, e: + # oups! last-level attempt to recover. try: - space.threadlocals.leave_thread(space) - finally: - thread.gc_thread_die() + STDERR = 2 + os.write(STDERR, "Thread exited with ") + os.write(STDERR, str(e)) + os.write(STDERR, "\n") + except OSError: + pass + bootstrapper.nbthreads -= 1 + thread.gc_thread_die() bootstrap = staticmethod(bootstrap) def acquire(space, w_callable, args): @@ -129,6 +134,9 @@ where = 'thread %d started by ' % ident e.write_unraisable(space, where, w_callable) e.clear(space) + # clean up space.threadlocals to remove the ExecutionContext + # entry corresponding to the current thread + space.threadlocals.leave_thread(space) run = staticmethod(run) bootstrapper = Bootstrapper() diff --git a/pypy/module/thread/test/test_gil.py b/pypy/module/thread/test/test_gil.py --- a/pypy/module/thread/test/test_gil.py +++ b/pypy/module/thread/test/test_gil.py @@ -61,8 +61,9 @@ def bootstrap(): try: runme() - finally: - thread.gc_thread_die() + except Exception, e: + assert 0 + thread.gc_thread_die() def f(): state.data = [] state.datalen1 = 0 diff --git a/pypy/rlib/clibffi.py b/pypy/rlib/clibffi.py --- a/pypy/rlib/clibffi.py +++ b/pypy/rlib/clibffi.py @@ -421,6 +421,7 @@ hints={'callback':True})) + at jit.jit_callback("CLIBFFI") def ll_callback(ffi_cif, ll_res, ll_args, ll_userdata): """ Callback specification. ffi_cif - something ffi specific, don't care diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py --- a/pypy/rlib/jit.py +++ b/pypy/rlib/jit.py @@ -302,6 +302,32 @@ pass +def jit_callback(name): + """Use as a decorator for C callback functions, to insert a + jitdriver.jit_merge_point() at the start. Only for callbacks + that typically invoke more app-level Python code. + """ + def decorate(func): + from pypy.tool.sourcetools import compile2 + # + def get_printable_location(): + return name + jitdriver = JitDriver(get_printable_location=get_printable_location, + greens=[], reds='auto', name=name) + # + args = ','.join(['a%d' % i for i in range(func.func_code.co_argcount)]) + source = """def callback_with_jitdriver(%(args)s): + jitdriver.jit_merge_point() + return real_callback(%(args)s)""" % locals() + miniglobals = { + 'jitdriver': jitdriver, + 'real_callback': func, + } + exec compile2(source) in miniglobals + return miniglobals['callback_with_jitdriver'] + return decorate + + # ____________________________________________________________ # VRefs @@ -458,9 +484,8 @@ self.autoreds = True self.reds = [] self.numreds = None # see warmspot.autodetect_jit_markers_redvars - for hook in (get_jitcell_at, set_jitcell_at, get_printable_location, - confirm_enter_jit): - assert hook is None, "reds='auto' is not compatible with JitDriver hooks" + assert confirm_enter_jit is None, ( + "reds='auto' is not compatible with confirm_enter_jit") else: if reds is not None: self.reds = reds diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py --- a/pypy/rlib/objectmodel.py +++ b/pypy/rlib/objectmodel.py @@ -595,16 +595,6 @@ llhelper(rffi.AroundFnPtr, before) llhelper(rffi.AroundFnPtr, after) -def register_around_callback_hook(hook): - """ Register a hook that's called before a callback from C calls RPython. - Primary usage is for JIT to have 'started from' hook. - """ - from pypy.rpython.lltypesystem import rffi - from pypy.rpython.annlowlevel import llhelper - - rffi.aroundstate.callback_hook = hook - llhelper(rffi.CallbackHookPtr, hook) - def is_in_callback(): from pypy.rpython.lltypesystem import rffi return rffi.stackcounter.stacks_counter > 1 diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py --- a/pypy/rpython/lltypesystem/rffi.py +++ b/pypy/rpython/lltypesystem/rffi.py @@ -1,6 +1,6 @@ import py from pypy.annotation import model as annmodel -from pypy.rpython.lltypesystem import lltype, rstr +from pypy.rpython.lltypesystem import lltype from pypy.rpython.lltypesystem import ll2ctypes from pypy.rpython.lltypesystem.llmemory import cast_ptr_to_adr from pypy.rpython.lltypesystem.llmemory import itemoffsetof, raw_memcopy @@ -13,7 +13,7 @@ from pypy.rlib.unroll import unrolling_iterable from pypy.rpython.tool.rfficache import platform, sizeof_c_type from pypy.translator.tool.cbuild import ExternalCompilationInfo -from pypy.rpython.annlowlevel import llhelper, llstr +from pypy.rpython.annlowlevel import llhelper from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.rstring import StringBuilder, UnicodeBuilder, assert_str0 from pypy.rlib import jit @@ -279,17 +279,8 @@ callable_name = getattr(callable, '__name__', '?') if callbackholder is not None: callbackholder.callbacks[callable] = True - callable_name_descr = str(callable).replace('"', '\\"') args = ', '.join(['a%d' % i for i in range(len(TP.TO.ARGS))]) source = py.code.Source(r""" - def inner_wrapper(%(args)s): - if aroundstate is not None: - callback_hook = aroundstate.callback_hook - if callback_hook: - callback_hook(llstr("%(callable_name_descr)s")) - return callable(%(args)s) - inner_wrapper._never_inline_ = True - def wrapper(%(args)s): # no *args - no GIL for mallocing the tuple llop.gc_stack_bottom(lltype.Void) # marker for trackgcroot.py if aroundstate is not None: @@ -299,7 +290,7 @@ # from now on we hold the GIL stackcounter.stacks_counter += 1 try: - result = inner_wrapper(%(args)s) + result = callable(%(args)s) except Exception, e: os.write(2, "Warning: uncaught exception in callback: %%s %%s\n" %% @@ -321,7 +312,6 @@ miniglobals = locals().copy() miniglobals['Exception'] = Exception miniglobals['os'] = os - miniglobals['llstr'] = llstr miniglobals['we_are_translated'] = we_are_translated miniglobals['stackcounter'] = stackcounter exec source.compile() in miniglobals @@ -329,11 +319,8 @@ _make_wrapper_for._annspecialcase_ = 'specialize:memo' AroundFnPtr = lltype.Ptr(lltype.FuncType([], lltype.Void)) -CallbackHookPtr = lltype.Ptr(lltype.FuncType([lltype.Ptr(rstr.STR)], lltype.Void)) class AroundState: - callback_hook = None - def _cleanup_(self): self.before = None # or a regular RPython function self.after = None # or a regular RPython function diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py --- a/pypy/rpython/memory/gctransform/framework.py +++ b/pypy/rpython/memory/gctransform/framework.py @@ -968,6 +968,7 @@ assert self.translator.config.translation.thread if hasattr(self.root_walker, 'thread_run_ptr'): livevars = self.push_roots(hop) + assert not livevars, "live GC var around %s!" % (hop.spaceop,) hop.genop("direct_call", [self.root_walker.thread_run_ptr]) self.pop_roots(hop, livevars) @@ -982,6 +983,7 @@ assert self.translator.config.translation.thread if hasattr(self.root_walker, 'thread_die_ptr'): livevars = self.push_roots(hop) + assert not livevars, "live GC var around %s!" % (hop.spaceop,) hop.genop("direct_call", [self.root_walker.thread_die_ptr]) self.pop_roots(hop, livevars) diff --git a/pypy/rpython/memory/gctransform/shadowstack.py b/pypy/rpython/memory/gctransform/shadowstack.py --- a/pypy/rpython/memory/gctransform/shadowstack.py +++ b/pypy/rpython/memory/gctransform/shadowstack.py @@ -326,6 +326,8 @@ self.gcdata.root_stack_top = llmemory.NULL # to detect missing restore def forget_current_state(self): + ll_assert(self.gcdata.root_stack_base == self.gcdata.root_stack_top, + "forget_current_state: shadowstack not empty!") if self.unused_full_stack: llmemory.raw_free(self.unused_full_stack) self.unused_full_stack = self.gcdata.root_stack_base diff --git a/pypy/translator/goal/targetpypystandalone.py b/pypy/translator/goal/targetpypystandalone.py --- a/pypy/translator/goal/targetpypystandalone.py +++ b/pypy/translator/goal/targetpypystandalone.py @@ -28,11 +28,6 @@ w_call_startup_gateway = space.wrap(gateway.interp2app(call_startup)) withjit = space.config.objspace.usemodules.pypyjit - if withjit: - from pypy.module.pypyjit.interp_jit import callback_hook - from pypy.rlib import objectmodel - objectmodel.register_around_callback_hook(callback_hook) - def entry_point(argv): if withjit: from pypy.jit.backend.hlinfo import highleveljitinfo From noreply at buildbot.pypy.org Tue Jan 15 18:48:07 2013 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Tue, 15 Jan 2013 18:48:07 +0100 (CET) Subject: [pypy-commit] pypy pytest: use pylib reraise for py3k compat Message-ID: <20130115174807.458E41C0131@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: pytest Changeset: r60087:812281a24135 Date: 2013-01-14 01:19 +0100 http://bitbucket.org/pypy/pypy/changeset/812281a24135/ Log: use pylib reraise for py3k compat diff --git a/pypy/tool/pytest/inttest.py b/pypy/tool/pytest/inttest.py --- a/pypy/tool/pytest/inttest.py +++ b/pypy/tool/pytest/inttest.py @@ -14,7 +14,7 @@ try: if e.w_type.name == 'KeyboardInterrupt': tb = sys.exc_info()[2] - raise KeyboardInterrupt, KeyboardInterrupt(), tb + py.builtin._reraise(KeyboardInterrupt, KeyboardInterrupt(), tb) except AttributeError: pass From noreply at buildbot.pypy.org Tue Jan 15 18:48:08 2013 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Tue, 15 Jan 2013 18:48:08 +0100 (CET) Subject: [pypy-commit] pypy pytest: add pytest-cache dir to hgignore Message-ID: <20130115174808.6706E1C0131@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: pytest Changeset: r60088:be293d809c4f Date: 2013-01-14 01:19 +0100 http://bitbucket.org/pypy/pypy/changeset/be293d809c4f/ Log: add pytest-cache dir to hgignore diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -7,6 +7,8 @@ .project .pydevproject +\.cache + syntax: regexp ^testresult$ ^site-packages$ From noreply at buildbot.pypy.org Tue Jan 15 18:48:09 2013 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Tue, 15 Jan 2013 18:48:09 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: move pytest.ini to the toplevel Message-ID: <20130115174809.A6DF51C0131@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: split-rpython Changeset: r60089:4ae735de7803 Date: 2013-01-15 18:32 +0100 http://bitbucket.org/pypy/pypy/changeset/4ae735de7803/ Log: move pytest.ini to the toplevel diff --git a/pypy/pytest.ini b/pytest.ini rename from pypy/pytest.ini rename to pytest.ini From noreply at buildbot.pypy.org Tue Jan 15 18:48:11 2013 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Tue, 15 Jan 2013 18:48:11 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: merge Message-ID: <20130115174811.48B481C0131@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: split-rpython Changeset: r60090:e375529de2e8 Date: 2013-01-15 18:47 +0100 http://bitbucket.org/pypy/pypy/changeset/e375529de2e8/ Log: merge diff --git a/goal/targetpypystandalone.py b/goal/targetpypystandalone.py --- a/goal/targetpypystandalone.py +++ b/goal/targetpypystandalone.py @@ -29,11 +29,6 @@ w_call_startup_gateway = space.wrap(gateway.interp2app(call_startup)) withjit = space.config.objspace.usemodules.pypyjit - if withjit: - from pypy.module.pypyjit.interp_jit import callback_hook - from rpython.rlib import objectmodel - objectmodel.register_around_callback_hook(callback_hook) - def entry_point(argv): if withjit: from rpython.jit.backend.hlinfo import highleveljitinfo diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py --- a/pypy/module/_cffi_backend/ccallback.py +++ b/pypy/module/_cffi_backend/ccallback.py @@ -152,6 +152,7 @@ STDERR = 2 + at jit.jit_callback("CFFI") def invoke_callback(ffi_cif, ll_res, ll_args, ll_userdata): """ Callback specification. ffi_cif - something ffi specific, don't care diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py --- a/pypy/module/pyexpat/interp_pyexpat.py +++ b/pypy/module/pyexpat/interp_pyexpat.py @@ -2,7 +2,7 @@ from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault from pypy.interpreter.error import OperationError -from rpython.rlib import rgc +from rpython.rlib import rgc, jit from rpython.rtyper.lltypesystem import rffi, lltype from rpython.rtyper.tool import rffi_platform from rpython.translator.tool.cbuild import ExternalCompilationInfo @@ -288,6 +288,7 @@ post_code = '' src = py.code.Source(""" + @jit.jit_callback('XML:%(name)s') def %(name)s_callback(%(first_arg)s, %(args)s): id = rffi.cast(lltype.Signed, %(ll_id)s) userdata = global_storage.get_object(id) diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py --- a/pypy/module/pypyjit/interp_jit.py +++ b/pypy/module/pypyjit/interp_jit.py @@ -6,7 +6,7 @@ from rpython.tool.pairtype import extendabletype from rpython.rlib.rarithmetic import r_uint, intmask from rpython.rlib.jit import JitDriver, hint, we_are_jitted, dont_look_inside -from rpython.rlib import jit, objectmodel +from rpython.rlib import jit from rpython.rlib.jit import current_trace_length, unroll_parameters import pypy.interpreter.pyopcode # for side-effects from pypy.interpreter.error import OperationError, operationerrfmt @@ -97,15 +97,6 @@ is_being_profiled=self.is_being_profiled) return jumpto -callback_jit_driver = JitDriver(greens = ['name'], reds = 'auto') - -def callback_merge_point(name): - callback_jit_driver.jit_merge_point(name=name) - - at callback_jit_driver.inline(callback_merge_point) -def callback_hook(name): - pass - def _get_adapted_tick_counter(): # Normally, the tick counter is decremented by 100 for every # Python opcode. Here, to better support JIT compilation of diff --git a/pypy/module/thread/os_thread.py b/pypy/module/thread/os_thread.py --- a/pypy/module/thread/os_thread.py +++ b/pypy/module/thread/os_thread.py @@ -2,7 +2,8 @@ Thread support based on OS-level threads. """ -from rpython.rlib import rthread as thread +import os +from pypy.module.thread import ll_thread as thread from pypy.module.thread.error import wrap_thread_error from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.gateway import unwrap_spec, Arguments @@ -91,14 +92,18 @@ # run! try: bootstrapper.run(space, w_callable, args) - finally: - bootstrapper.nbthreads -= 1 - # clean up space.threadlocals to remove the ExecutionContext - # entry corresponding to the current thread + # done + except Exception, e: + # oups! last-level attempt to recover. try: - space.threadlocals.leave_thread(space) - finally: - thread.gc_thread_die() + STDERR = 2 + os.write(STDERR, "Thread exited with ") + os.write(STDERR, str(e)) + os.write(STDERR, "\n") + except OSError: + pass + bootstrapper.nbthreads -= 1 + thread.gc_thread_die() bootstrap = staticmethod(bootstrap) def acquire(space, w_callable, args): @@ -129,6 +134,9 @@ where = 'thread %d started by ' % ident e.write_unraisable(space, where, w_callable) e.clear(space) + # clean up space.threadlocals to remove the ExecutionContext + # entry corresponding to the current thread + space.threadlocals.leave_thread(space) run = staticmethod(run) bootstrapper = Bootstrapper() diff --git a/pypy/module/thread/test/test_gil.py b/pypy/module/thread/test/test_gil.py --- a/pypy/module/thread/test/test_gil.py +++ b/pypy/module/thread/test/test_gil.py @@ -61,8 +61,9 @@ def bootstrap(): try: runme() - finally: - thread.gc_thread_die() + except Exception, e: + assert 0 + thread.gc_thread_die() def f(): state.data = [] state.datalen1 = 0 diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py --- a/rpython/annotator/test/test_annrpython.py +++ b/rpython/annotator/test/test_annrpython.py @@ -3655,7 +3655,7 @@ cls = C return cls().foo a = self.RPythonAnnotator() - raises(Exception, a.build_types, f, [int]) + py.test.raises(Exception, a.build_types, f, [int]) def test_range_variable_step(self): def g(n): diff --git a/rpython/jit/backend/test/test_random.py b/rpython/jit/backend/test/test_random.py --- a/rpython/jit/backend/test/test_random.py +++ b/rpython/jit/backend/test/test_random.py @@ -26,6 +26,8 @@ class OperationBuilder(object): def __init__(self, cpu, loop, vars): self.cpu = cpu + if not hasattr(cpu, '_faildescr_keepalive'): + cpu._faildescr_keepalive = [] self.fakemetainterp = FakeMetaInterp() self.loop = loop self.intvars = [box for box in vars if isinstance(box, BoxInt)] @@ -206,6 +208,11 @@ if pytest.config.option.output: s.close() + def getfaildescr(self): + descr = BasicFailDescr() + self.cpu._faildescr_keepalive.append(descr) + return descr + class CannotProduceOperation(Exception): pass @@ -287,7 +294,7 @@ builder.intvars[:] = original_intvars else: op = ResOperation(rop.GUARD_NO_OVERFLOW, [], None) - op.setdescr(BasicFailDescr()) + op.setdescr(builder.getfaildescr()) op.setfailargs(fail_subset) builder.loop.operations.append(op) @@ -357,7 +364,7 @@ def produce_into(self, builder, r): op, passing = self.gen_guard(builder, r) builder.loop.operations.append(op) - op.setdescr(BasicFailDescr()) + op.setdescr(builder.getfaildescr()) op.setfailargs(builder.subset_of_intvars(r)) if not passing: builder.should_fail_by = op @@ -618,8 +625,9 @@ if v not in used_later: endvars.append(v) r.shuffle(endvars) + endvars = endvars[:1] loop.operations.append(ResOperation(rop.FINISH, endvars, None, - descr=BasicFailDescr())) + descr=builder.getfaildescr())) if builder.should_fail_by: self.should_fail_by = builder.should_fail_by self.guard_op = builder.guard_op @@ -712,7 +720,7 @@ else: op = ResOperation(rop.GUARD_EXCEPTION, [guard_op._exc_box], BoxPtr()) - op.setdescr(BasicFailDescr()) + op.setdescr(self.builder.getfaildescr()) op.setfailargs([]) return op diff --git a/rpython/jit/metainterp/test/test_fficall.py b/rpython/jit/metainterp/test/test_fficall.py --- a/rpython/jit/metainterp/test/test_fficall.py +++ b/rpython/jit/metainterp/test/test_fficall.py @@ -104,7 +104,7 @@ class TestFfiCall(FfiCallTests, LLJitMixin): - def test_jit_fii_vref(self): + def test_jit_ffi_vref(self): from rpython.rlib import clibffi from rpython.rlib.jit_libffi import jit_ffi_prep_cif, jit_ffi_call @@ -135,9 +135,10 @@ res = exb[3] lltype.free(exb, flavor='raw') # - lltype.free(atypes, flavor='raw') return res # res = self.interp_operations(f, []) lltype.free(cd, flavor='raw') assert res == math.sin(1.23) + + lltype.free(atypes, flavor='raw') diff --git a/rpython/jit/metainterp/test/test_warmspot.py b/rpython/jit/metainterp/test/test_warmspot.py --- a/rpython/jit/metainterp/test/test_warmspot.py +++ b/rpython/jit/metainterp/test/test_warmspot.py @@ -1,6 +1,6 @@ import py from rpython.jit.metainterp.warmspot import get_stats -from rpython.rlib.jit import JitDriver, set_param, unroll_safe +from rpython.rlib.jit import JitDriver, set_param, unroll_safe, jit_callback from rpython.jit.backend.llgraph import runner from rpython.jit.metainterp.test.support import LLJitMixin, OOJitMixin @@ -537,41 +537,21 @@ def test_callback_jit_merge_point(self): - from rpython.rlib.objectmodel import register_around_callback_hook - from rpython.rtyper.lltypesystem import lltype, rffi - from rpython.translator.tool.cbuild import ExternalCompilationInfo - - callback_jit_driver = JitDriver(greens = ['name'], reds = 'auto') - - def callback_merge_point(name): - callback_jit_driver.jit_merge_point(name=name) - - @callback_jit_driver.inline(callback_merge_point) - def callback_hook(name): - pass - + @jit_callback("testing") def callback(a, b): if a > b: return 1 return -1 - CB_TP = rffi.CCallback([lltype.Signed, lltype.Signed], lltype.Signed) - eci = ExternalCompilationInfo(includes=['stdlib.h']) - qsort = rffi.llexternal('qsort', - [rffi.VOIDP, lltype.Signed, lltype.Signed, - CB_TP], lltype.Void, compilation_info=eci) - ARR = rffi.CArray(lltype.Signed) + def main(): + total = 0 + for i in range(10): + total += callback(i, 2) + return total - def main(): - register_around_callback_hook(callback_hook) - raw = lltype.malloc(ARR, 10, flavor='raw') - for i in range(10): - raw[i] = 10 - i - qsort(raw, 10, rffi.sizeof(lltype.Signed), callback) - lltype.free(raw, flavor='raw') - - self.meta_interp(main, []) - self.check_trace_count(1) + res = self.meta_interp(main, []) + assert res == 7 - 3 + self.check_trace_count(2) class TestLLWarmspot(WarmspotTests, LLJitMixin): diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py --- a/rpython/jit/metainterp/warmspot.py +++ b/rpython/jit/metainterp/warmspot.py @@ -360,13 +360,7 @@ jd._jit_merge_point_in = graph args = op.args[2:] s_binding = self.translator.annotator.binding - if op.args[1].value.autoreds: - # _portal_args_s is used only by _make_hook_graph, but for now we - # declare the various set_jitcell_at, get_printable_location, - # etc. as incompatible with autoreds - jd._portal_args_s = None - else: - jd._portal_args_s = [s_binding(v) for v in args] + jd._portal_args_s = [s_binding(v) for v in args] graph = copygraph(graph) [jmpp] = find_jit_merge_points([graph]) graph.startblock = support.split_before_jit_merge_point(*jmpp) @@ -614,9 +608,10 @@ if func is None: return None # - assert not jitdriver_sd.jitdriver.autoreds, ( - "reds='auto' is not compatible with JitDriver hooks such as " - "{get,set}_jitcell_at, get_printable_location, confirm_enter_jit, etc.") + if not onlygreens: + assert not jitdriver_sd.jitdriver.autoreds, ( + "reds='auto' is not compatible with JitDriver hooks such as " + "confirm_enter_jit") extra_args_s = [] if s_first_arg is not None: extra_args_s.append(s_first_arg) diff --git a/rpython/pytest.ini b/rpython/pytest.ini new file mode 100644 --- /dev/null +++ b/rpython/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +addopts = --assert=reinterp -rf diff --git a/rpython/rlib/clibffi.py b/rpython/rlib/clibffi.py --- a/rpython/rlib/clibffi.py +++ b/rpython/rlib/clibffi.py @@ -421,6 +421,7 @@ hints={'callback':True})) + at jit.jit_callback("CLIBFFI") def ll_callback(ffi_cif, ll_res, ll_args, ll_userdata): """ Callback specification. ffi_cif - something ffi specific, don't care diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -302,6 +302,32 @@ pass +def jit_callback(name): + """Use as a decorator for C callback functions, to insert a + jitdriver.jit_merge_point() at the start. Only for callbacks + that typically invoke more app-level Python code. + """ + def decorate(func): + from rpython.tool.sourcetools import compile2 + # + def get_printable_location(): + return name + jitdriver = JitDriver(get_printable_location=get_printable_location, + greens=[], reds='auto', name=name) + # + args = ','.join(['a%d' % i for i in range(func.func_code.co_argcount)]) + source = """def callback_with_jitdriver(%(args)s): + jitdriver.jit_merge_point() + return real_callback(%(args)s)""" % locals() + miniglobals = { + 'jitdriver': jitdriver, + 'real_callback': func, + } + exec compile2(source) in miniglobals + return miniglobals['callback_with_jitdriver'] + return decorate + + # ____________________________________________________________ # VRefs @@ -458,9 +484,8 @@ self.autoreds = True self.reds = [] self.numreds = None # see warmspot.autodetect_jit_markers_redvars - for hook in (get_jitcell_at, set_jitcell_at, get_printable_location, - confirm_enter_jit): - assert hook is None, "reds='auto' is not compatible with JitDriver hooks" + assert confirm_enter_jit is None, ( + "reds='auto' is not compatible with confirm_enter_jit") else: if reds is not None: self.reds = reds diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py --- a/rpython/rlib/objectmodel.py +++ b/rpython/rlib/objectmodel.py @@ -595,16 +595,6 @@ llhelper(rffi.AroundFnPtr, before) llhelper(rffi.AroundFnPtr, after) -def register_around_callback_hook(hook): - """ Register a hook that's called before a callback from C calls RPython. - Primary usage is for JIT to have 'started from' hook. - """ - from rpython.rtyper.lltypesystem import rffi - from rpython.rtyper.annlowlevel import llhelper - - rffi.aroundstate.callback_hook = hook - llhelper(rffi.CallbackHookPtr, hook) - def is_in_callback(): from rpython.rtyper.lltypesystem import rffi return rffi.stackcounter.stacks_counter > 1 diff --git a/rpython/rtyper/lltypesystem/module/test/test_ll_math.py b/rpython/rtyper/lltypesystem/module/test/test_ll_math.py --- a/rpython/rtyper/lltypesystem/module/test/test_ll_math.py +++ b/rpython/rtyper/lltypesystem/module/test/test_ll_math.py @@ -296,7 +296,7 @@ assert expected == OverflowError, "%s: got an OverflowError" % ( repr,) else: - if not (expected)(got): + if not get_tester(expected)(got): raise AssertionError("%r: got %r, expected %r" % (repr, got, expected)) # diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py --- a/rpython/rtyper/lltypesystem/rffi.py +++ b/rpython/rtyper/lltypesystem/rffi.py @@ -13,7 +13,7 @@ from rpython.rlib.unroll import unrolling_iterable from rpython.rtyper.tool.rfficache import platform, sizeof_c_type from rpython.translator.tool.cbuild import ExternalCompilationInfo -from rpython.rtyper.annlowlevel import llhelper, llstr +from rpython.rtyper.annlowlevel import llhelper from rpython.rlib.objectmodel import we_are_translated from rpython.rlib.rstring import StringBuilder, UnicodeBuilder, assert_str0 from rpython.rlib import jit @@ -279,17 +279,8 @@ callable_name = getattr(callable, '__name__', '?') if callbackholder is not None: callbackholder.callbacks[callable] = True - callable_name_descr = str(callable).replace('"', '\\"') args = ', '.join(['a%d' % i for i in range(len(TP.TO.ARGS))]) source = py.code.Source(r""" - def inner_wrapper(%(args)s): - if aroundstate is not None: - callback_hook = aroundstate.callback_hook - if callback_hook: - callback_hook(llstr("%(callable_name_descr)s")) - return callable(%(args)s) - inner_wrapper._never_inline_ = True - def wrapper(%(args)s): # no *args - no GIL for mallocing the tuple llop.gc_stack_bottom(lltype.Void) # marker for trackgcroot.py if aroundstate is not None: @@ -299,7 +290,7 @@ # from now on we hold the GIL stackcounter.stacks_counter += 1 try: - result = inner_wrapper(%(args)s) + result = callable(%(args)s) except Exception, e: os.write(2, "Warning: uncaught exception in callback: %%s %%s\n" %% @@ -321,7 +312,6 @@ miniglobals = locals().copy() miniglobals['Exception'] = Exception miniglobals['os'] = os - miniglobals['llstr'] = llstr miniglobals['we_are_translated'] = we_are_translated miniglobals['stackcounter'] = stackcounter exec source.compile() in miniglobals @@ -329,11 +319,8 @@ _make_wrapper_for._annspecialcase_ = 'specialize:memo' AroundFnPtr = lltype.Ptr(lltype.FuncType([], lltype.Void)) -CallbackHookPtr = lltype.Ptr(lltype.FuncType([lltype.Ptr(rstr.STR)], lltype.Void)) class AroundState: - callback_hook = None - def _cleanup_(self): self.before = None # or a regular RPython function self.after = None # or a regular RPython function diff --git a/rpython/rtyper/memory/gctransform/framework.py b/rpython/rtyper/memory/gctransform/framework.py --- a/rpython/rtyper/memory/gctransform/framework.py +++ b/rpython/rtyper/memory/gctransform/framework.py @@ -979,6 +979,7 @@ assert self.translator.config.translation.thread if hasattr(self.root_walker, 'thread_run_ptr'): livevars = self.push_roots(hop) + assert not livevars, "live GC var around %s!" % (hop.spaceop,) hop.genop("direct_call", [self.root_walker.thread_run_ptr]) self.pop_roots(hop, livevars) @@ -993,6 +994,7 @@ assert self.translator.config.translation.thread if hasattr(self.root_walker, 'thread_die_ptr'): livevars = self.push_roots(hop) + assert not livevars, "live GC var around %s!" % (hop.spaceop,) hop.genop("direct_call", [self.root_walker.thread_die_ptr]) self.pop_roots(hop, livevars) diff --git a/rpython/rtyper/memory/gctransform/shadowstack.py b/rpython/rtyper/memory/gctransform/shadowstack.py --- a/rpython/rtyper/memory/gctransform/shadowstack.py +++ b/rpython/rtyper/memory/gctransform/shadowstack.py @@ -326,6 +326,8 @@ self.gcdata.root_stack_top = llmemory.NULL # to detect missing restore def forget_current_state(self): + ll_assert(self.gcdata.root_stack_base == self.gcdata.root_stack_top, + "forget_current_state: shadowstack not empty!") if self.unused_full_stack: llmemory.raw_free(self.unused_full_stack) self.unused_full_stack = self.gcdata.root_stack_base From noreply at buildbot.pypy.org Tue Jan 15 19:13:03 2013 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Tue, 15 Jan 2013 19:13:03 +0100 (CET) Subject: [pypy-commit] buildbot default: add a extra rpython test step for the rpython split Message-ID: <20130115181303.299321C1353@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: Changeset: r733:940bbdfd9a85 Date: 2013-01-15 19:12 +0100 http://bitbucket.org/pypy/buildbot/changeset/940bbdfd9a85/ Log: add a extra rpython test step for the rpython split diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -367,7 +367,7 @@ timeout=kwargs.get('timeout', 4000) self.addStep(PytestCmd( - description="pytest", + description="pytest pypy", command=["python", "testrunner/runner.py", "--logfile=testrun.log", "--config=pypy/testrunner_cfg.py", @@ -379,6 +379,19 @@ env={"PYTHONPATH": ['.'], "PYPYCHERRYPICK": cherrypick})) + self.addStep(PytestCmd( + description="pytest rpython", + command=["python", "testrunner/runner.py", + "--logfile=testrun.log", + "--config=pypy/testrunner_cfg.py", + "--config=~/machine_cfg.py", + "--root=rpython", "--timeout=%s" % (timeout,) + ] + ["--config=%s" % cfg for cfg in extra_cfgs], + logfiles={'pytestLog': 'testrun.log'}, + timeout=timeout, + env={"PYTHONPATH": ['.'], + "PYPYCHERRYPICK": cherrypick})) + class Translated(factory.BuildFactory): From noreply at buildbot.pypy.org Tue Jan 15 19:20:55 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Tue, 15 Jan 2013 19:20:55 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: import py.test on test_rwin32 Message-ID: <20130115182055.F07161C1353@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60091:a8308f6a3af7 Date: 2013-01-15 18:07 +0100 http://bitbucket.org/pypy/pypy/changeset/a8308f6a3af7/ Log: import py.test on test_rwin32 diff --git a/rpython/rlib/test/test_rwin32.py b/rpython/rlib/test/test_rwin32.py --- a/rpython/rlib/test/test_rwin32.py +++ b/rpython/rlib/test/test_rwin32.py @@ -1,6 +1,6 @@ -import os +import os, py if os.name != 'nt': - skip('tests for win32 only') + py.test.skip('tests for win32 only') from rpython.rlib import rwin32 from rpython.tool.udir import udir @@ -11,12 +11,12 @@ fd = fid.fileno() rwin32.get_osfhandle(fd) fid.close() - raises(OSError, rwin32.get_osfhandle, fd) + py.test.raises(OSError, rwin32.get_osfhandle, fd) rwin32.get_osfhandle(0) def test_get_osfhandle_raising(): #try to test what kind of exception get_osfhandle raises w/out fd validation - skip('Crashes python') + py.test.skip('Crashes python') fid = open(str(udir.join('validate_test.txt')), 'w') fd = fid.fileno() fid.close() @@ -32,7 +32,7 @@ assert pid != 0 handle = rwin32.OpenProcess(rwin32.PROCESS_QUERY_INFORMATION, False, pid) rwin32.CloseHandle(handle) - raises(WindowsError, rwin32.OpenProcess, rwin32.PROCESS_TERMINATE, False, 0) + py.test.raises(WindowsError, rwin32.OpenProcess, rwin32.PROCESS_TERMINATE, False, 0) def test_terminate_process(): import subprocess, signal, sys From noreply at buildbot.pypy.org Tue Jan 15 19:20:57 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Tue, 15 Jan 2013 19:20:57 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed missing rename Message-ID: <20130115182057.609E61C1353@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60092:c0bac6429aa6 Date: 2013-01-15 18:11 +0100 http://bitbucket.org/pypy/pypy/changeset/c0bac6429aa6/ Log: Fixed missing rename diff --git a/pypy/module/thread/os_thread.py b/pypy/module/thread/os_thread.py --- a/pypy/module/thread/os_thread.py +++ b/pypy/module/thread/os_thread.py @@ -3,7 +3,7 @@ """ import os -from pypy.module.thread import ll_thread as thread +from rpython.rlib import rthread as thread from pypy.module.thread.error import wrap_thread_error from pypy.interpreter.error import OperationError, operationerrfmt from pypy.interpreter.gateway import unwrap_spec, Arguments From noreply at buildbot.pypy.org Tue Jan 15 19:20:58 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Tue, 15 Jan 2013 19:20:58 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: import py on test_re Message-ID: <20130115182058.853B61C1353@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60093:357b17707835 Date: 2013-01-15 18:16 +0100 http://bitbucket.org/pypy/pypy/changeset/357b17707835/ Log: import py on test_re diff --git a/rpython/rlib/rsre/test/test_re.py b/rpython/rlib/rsre/test/test_re.py --- a/rpython/rlib/rsre/test/test_re.py +++ b/rpython/rlib/rsre/test/test_re.py @@ -1,4 +1,4 @@ -import sys, os +import sys, os, py from rpython.rlib.rsre.test.test_match import get_code from rpython.rlib.rsre import rsre_re as re @@ -112,18 +112,18 @@ assert re.sub('x', r'\400', 'x') == '\0' assert re.sub('x', r'\777', 'x') == '\377' - raises(re.error, re.sub, 'x', r'\1', 'x') - raises(re.error, re.sub, 'x', r'\8', 'x') - raises(re.error, re.sub, 'x', r'\9', 'x') - raises(re.error, re.sub, 'x', r'\11', 'x') - raises(re.error, re.sub, 'x', r'\18', 'x') - raises(re.error, re.sub, 'x', r'\1a', 'x') - raises(re.error, re.sub, 'x', r'\90', 'x') - raises(re.error, re.sub, 'x', r'\99', 'x') - raises(re.error, re.sub, 'x', r'\118', 'x') # r'\11' + '8' - raises(re.error, re.sub, 'x', r'\11a', 'x') - raises(re.error, re.sub, 'x', r'\181', 'x') # r'\18' + '1' - raises(re.error, re.sub, 'x', r'\800', 'x') # r'\80' + '0' + py.test.raises(re.error, re.sub, 'x', r'\1', 'x') + py.test.raises(re.error, re.sub, 'x', r'\8', 'x') + py.test.raises(re.error, re.sub, 'x', r'\9', 'x') + py.test.raises(re.error, re.sub, 'x', r'\11', 'x') + py.test.raises(re.error, re.sub, 'x', r'\18', 'x') + py.test.raises(re.error, re.sub, 'x', r'\1a', 'x') + py.test.raises(re.error, re.sub, 'x', r'\90', 'x') + py.test.raises(re.error, re.sub, 'x', r'\99', 'x') + py.test.raises(re.error, re.sub, 'x', r'\118', 'x') # r'\11' + '8' + py.test.raises(re.error, re.sub, 'x', r'\11a', 'x') + py.test.raises(re.error, re.sub, 'x', r'\181', 'x') # r'\18' + '1' + py.test.raises(re.error, re.sub, 'x', r'\800', 'x') # r'\80' + '0' # in python2.3 (etc), these loop endlessly in sre_parser.py assert re.sub('(((((((((((x)))))))))))', r'\11', 'x') == 'x' @@ -146,15 +146,15 @@ assert re.sub('x+', '-', 'abxd') == 'ab-d' def test_symbolic_refs(self): - raises(re.error, re.sub, '(?Px)', '\gx)', '\g<', 'xx') - raises(re.error, re.sub, '(?Px)', '\g', 'xx') - raises(re.error, re.sub, '(?Px)', '\g', 'xx') - raises(re.error, re.sub, '(?Px)', '\g<1a1>', 'xx') - raises(IndexError, re.sub, '(?Px)', '\g', 'xx') - raises(re.error, re.sub, '(?Px)|(?Py)', '\g', 'xx') - raises(re.error, re.sub, '(?Px)|(?Py)', '\\2', 'xx') - raises(re.error, re.sub, '(?Px)', '\g<-1>', 'xx') + py.test.raises(re.error, re.sub, '(?Px)', '\gx)', '\g<', 'xx') + py.test.raises(re.error, re.sub, '(?Px)', '\g', 'xx') + py.test.raises(re.error, re.sub, '(?Px)', '\g', 'xx') + py.test.raises(re.error, re.sub, '(?Px)', '\g<1a1>', 'xx') + py.test.raises(IndexError, re.sub, '(?Px)', '\g', 'xx') + py.test.raises(re.error, re.sub, '(?Px)|(?Py)', '\g', 'xx') + py.test.raises(re.error, re.sub, '(?Px)|(?Py)', '\\2', 'xx') + py.test.raises(re.error, re.sub, '(?Px)', '\g<-1>', 'xx') def test_re_subn(self): assert re.subn("(?i)b+", "x", "bbbb BBBB") == ('x x', 2) @@ -466,7 +466,7 @@ assert re.match(r"\x%02x" % i, chr(i)) != None assert re.match(r"\x%02x0" % i, chr(i)+"0") != None assert re.match(r"\x%02xz" % i, chr(i)+"z") != None - raises(re.error, re.match, "\911", "") + py.test.raises(re.error, re.match, "\911", "") def test_sre_character_class_literals(self): for i in [0, 8, 16, 32, 64, 127, 128, 255]: @@ -476,7 +476,7 @@ assert re.match(r"[\x%02x]" % i, chr(i)) != None assert re.match(r"[\x%02x0]" % i, chr(i)) != None assert re.match(r"[\x%02xz]" % i, chr(i)) != None - raises(re.error, re.match, "[\911]", "") + py.test.raises(re.error, re.match, "[\911]", "") def test_bug_113254(self): assert re.match(r'(a)|(b)', 'b').start(1) == -1 @@ -494,7 +494,7 @@ def test_bug_545855(self): # bug 545855 -- This pattern failed to cause a compile error as it # should, instead provoking a TypeError. - raises(re.error, re.compile, 'foo[a-') + py.test.raises(re.error, re.compile, 'foo[a-') def test_bug_418626(self): # bugs 418626 at al. -- Testing Greg Chapman's addition of op code @@ -609,7 +609,7 @@ def test_bug_581080(self): iter = re.finditer(r"\s", "a b") assert iter.next().span() == (1,2) - raises(StopIteration, iter.next) + py.test.raises(StopIteration, iter.next) if 0: # XXX scanner = re.compile(r"\s").scanner("a b") @@ -620,7 +620,7 @@ iter = re.finditer(r".*", "asdf") assert iter.next().span() == (0, 4) assert iter.next().span() == (4, 4) - raises(StopIteration, iter.next) + py.test.raises(StopIteration, iter.next) def test_empty_array(self): # SF buf 1647541 From noreply at buildbot.pypy.org Tue Jan 15 19:20:59 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Tue, 15 Jan 2013 19:20:59 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed tests Message-ID: <20130115182059.A75E51C1353@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60094:ed119bf74a79 Date: 2013-01-15 19:18 +0100 http://bitbucket.org/pypy/pypy/changeset/ed119bf74a79/ Log: Fixed tests diff --git a/rpython/rlib/test/test_jit.py b/rpython/rlib/test/test_jit.py --- a/rpython/rlib/test/test_jit.py +++ b/rpython/rlib/test/test_jit.py @@ -222,7 +222,7 @@ myjitdriver = JitDriver(greens=['r1', 'i1', 'f1'], reds=[]) class A(object): pass - e = raises(AssertionError, + e = py.test.raises(AssertionError, myjitdriver.jit_merge_point, i1=42, r1=A(), f1=3.5) def test_argument_order_more_precision_later(self): @@ -230,7 +230,7 @@ class A(object): pass myjitdriver.jit_merge_point(i1=42, r1=None, r2=None, f1=3.5) - e = raises(AssertionError, + e = py.test.raises(AssertionError, myjitdriver.jit_merge_point, i1=42, r1=A(), r2=None, f1=3.5) assert "got ['2:REF', '1:INT', '?', '3:FLOAT']" in repr(e.value) @@ -239,7 +239,7 @@ class A(object): pass myjitdriver.jit_merge_point(i1=42, r1=None, r2=A(), f1=3.5) - e = raises(AssertionError, + e = py.test.raises(AssertionError, myjitdriver.jit_merge_point, i1=42, r1=A(), r2=None, f1=3.5) assert "got ['2:REF', '1:INT', '2:REF', '3:FLOAT']" in repr(e.value) diff --git a/rpython/rlib/test/test_libffi.py b/rpython/rlib/test/test_libffi.py --- a/rpython/rlib/test/test_libffi.py +++ b/rpython/rlib/test/test_libffi.py @@ -178,7 +178,7 @@ def test_windll(self): if os.name != 'nt': - skip('Run only on windows') + py.test.skip('Run only on windows') from rpython.rlib.libffi import WinDLL dll = WinDLL('Kernel32.dll') sleep = dll.getpointer('Sleep',[types.uint], types.void) diff --git a/rpython/rlib/test/test_rarithmetic.py b/rpython/rlib/test/test_rarithmetic.py --- a/rpython/rlib/test/test_rarithmetic.py +++ b/rpython/rlib/test/test_rarithmetic.py @@ -216,7 +216,7 @@ n = -sys.maxint-1 y = sys.maxint-1 # sanity - raises(AssertionError, ovfcheck, r_uint(0)) + py.test.raises(AssertionError, ovfcheck, r_uint(0)) # not overflowing try: diff --git a/rpython/rlib/test/test_rbigint.py b/rpython/rlib/test/test_rbigint.py --- a/rpython/rlib/test/test_rbigint.py +++ b/rpython/rlib/test/test_rbigint.py @@ -249,7 +249,7 @@ assert d == float(x) x = x ** 100 f1 = rbigint.fromlong(x) - assert raises(OverflowError, f1.tofloat) + assert py.test.raises(OverflowError, f1.tofloat) f2 = rbigint.fromlong(2097152 << SHIFT) d = f2.tofloat() assert d == float(2097152 << SHIFT) @@ -282,8 +282,8 @@ # XXX don't use such consts. marshal doesn't handle them right. x = 12345.6789e200 x *= x - assert raises(OverflowError, rbigint.fromfloat, x) - assert raises(ValueError, rbigint.fromfloat, NAN) + assert py.test.raises(OverflowError, rbigint.fromfloat, x) + assert py.test.raises(ValueError, rbigint.fromfloat, NAN) # f1 = rbigint.fromfloat(9007199254740991.0) assert f1.tolong() == 9007199254740991 @@ -781,13 +781,13 @@ s = "\xFF\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12\x13\x14\x15\xFF" bigint = rbigint.frombytes(s, byteorder="big", signed=False) assert s == bigint.tobytes(16, byteorder="big", signed=False) - raises(InvalidEndiannessError, bigint.frombytes, '\xFF', 'foo', + py.test.raises(InvalidEndiannessError, bigint.frombytes, '\xFF', 'foo', signed=True) def test_tobytes(self): assert rbigint.fromint(0).tobytes(1, 'big', signed=True) == '\x00' assert rbigint.fromint(1).tobytes(2, 'big', signed=True) == '\x00\x01' - raises(OverflowError, rbigint.fromint(255).tobytes, 1, 'big', signed=True) + py.test.raises(OverflowError, rbigint.fromint(255).tobytes, 1, 'big', signed=True) assert rbigint.fromint(-129).tobytes(2, 'big', signed=True) == '\xff\x7f' assert rbigint.fromint(-129).tobytes(2, 'little', signed=True) == '\x7f\xff' assert rbigint.fromint(65535).tobytes(3, 'big', signed=True) == '\x00\xff\xff' @@ -795,7 +795,7 @@ assert rbigint.fromint(65535).tobytes(2, 'big', signed=False) == '\xff\xff' assert rbigint.fromint(-8388608).tobytes(3, 'little', signed=True) == '\x00\x00\x80' i = rbigint.fromint(-8388608) - raises(InvalidEndiannessError, i.tobytes, 3, 'foo', signed=True) - raises(InvalidSignednessError, i.tobytes, 3, 'little', signed=False) - raises(OverflowError, i.tobytes, 2, 'little', signed=True) + py.test.raises(InvalidEndiannessError, i.tobytes, 3, 'foo', signed=True) + py.test.raises(InvalidSignednessError, i.tobytes, 3, 'little', signed=False) + py.test.raises(OverflowError, i.tobytes, 2, 'little', signed=True) From noreply at buildbot.pypy.org Tue Jan 15 19:21:00 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Tue, 15 Jan 2013 19:21:00 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: merge Message-ID: <20130115182100.C98331C1353@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60095:56313bbca2ea Date: 2013-01-15 19:20 +0100 http://bitbucket.org/pypy/pypy/changeset/56313bbca2ea/ Log: merge diff --git a/pypy/pytest.ini b/pytest.ini rename from pypy/pytest.ini rename to pytest.ini diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py --- a/rpython/annotator/test/test_annrpython.py +++ b/rpython/annotator/test/test_annrpython.py @@ -3655,7 +3655,7 @@ cls = C return cls().foo a = self.RPythonAnnotator() - raises(Exception, a.build_types, f, [int]) + py.test.raises(Exception, a.build_types, f, [int]) def test_range_variable_step(self): def g(n): diff --git a/rpython/pytest.ini b/rpython/pytest.ini new file mode 100644 --- /dev/null +++ b/rpython/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +addopts = --assert=reinterp -rf diff --git a/rpython/rtyper/lltypesystem/module/test/test_ll_math.py b/rpython/rtyper/lltypesystem/module/test/test_ll_math.py --- a/rpython/rtyper/lltypesystem/module/test/test_ll_math.py +++ b/rpython/rtyper/lltypesystem/module/test/test_ll_math.py @@ -296,7 +296,7 @@ assert expected == OverflowError, "%s: got an OverflowError" % ( repr,) else: - if not (expected)(got): + if not get_tester(expected)(got): raise AssertionError("%r: got %r, expected %r" % (repr, got, expected)) # From noreply at buildbot.pypy.org Tue Jan 15 19:31:15 2013 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 15 Jan 2013 19:31:15 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: rip off some unused tests Message-ID: <20130115183115.CFD181C1353@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60096:47864e6b39c1 Date: 2013-01-15 20:30 +0200 http://bitbucket.org/pypy/pypy/changeset/47864e6b39c1/ Log: rip off some unused tests diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -1853,6 +1853,7 @@ mc.JMP_r(X86_64_SCRATCH_REG.value) # write tight data that describes the failure recovery positions = [0] * len(guardtok.fail_locs) + gcpattern = 0 for i, loc in enumerate(guardtok.fail_locs): if loc is None: positions[i] = -1 @@ -1864,10 +1865,13 @@ v = len(gpr_reg_mgr_cls.all_regs) + loc.value else: v = gpr_reg_mgr_cls.all_reg_indexes[loc.value] + if guardtok.failargs[i].type == REF: + gcpattern |= v positions[i] = v * WORD # write down the positions of locs guardtok.faildescr.rd_locs = positions # write down the GC pattern + guardtok.faildescr.rd_gcpattern = gcpattern return startpos def rebuild_faillocs_from_descr(self, descr, inputargs): diff --git a/pypy/jit/backend/x86/test/test_assembler.py b/pypy/jit/backend/x86/test/test_assembler.py --- a/pypy/jit/backend/x86/test/test_assembler.py +++ b/pypy/jit/backend/x86/test/test_assembler.py @@ -45,217 +45,6 @@ def hide(self, cpu): return rffi.cast(llmemory.GCREF, 123) -def test_write_failure_recovery_description(): - assembler = Assembler386(FakeCPU()) - mc = FakeMC() - failargs = [BoxInt(), BoxPtr(), BoxFloat()] * 3 - failargs.insert(6, None) - failargs.insert(7, None) - locs = [X86FrameManager.frame_pos(0, INT), - X86FrameManager.frame_pos(1, REF), - X86FrameManager.frame_pos(10, FLOAT), - X86FrameManager.frame_pos(100, INT), - X86FrameManager.frame_pos(101, REF), - X86FrameManager.frame_pos(110, FLOAT), - None, - None, - ebx, - esi, - xmm2] - assert len(failargs) == len(locs) - assembler.write_failure_recovery_description(mc, failargs, locs) - base = 8 + 8*IS_X86_64 - nums = [Assembler386.DESCR_INT + 4*(base+0), - Assembler386.DESCR_REF + 4*(base+1), - Assembler386.DESCR_FLOAT + 4*(base+10), - Assembler386.DESCR_INT + 4*(base+100), - Assembler386.DESCR_REF + 4*(base+101), - Assembler386.DESCR_FLOAT + 4*(base+110), - Assembler386.CODE_HOLE, - Assembler386.CODE_HOLE, - Assembler386.DESCR_INT + 4*ebx.value, - Assembler386.DESCR_REF + 4*esi.value, - Assembler386.DESCR_FLOAT + 4*xmm2.value] - double_byte_nums = [] - for num in nums[3:6]: - double_byte_nums.append((num & 0x7F) | 0x80) - double_byte_nums.append(num >> 7) - assert mc.content == (nums[:3] + double_byte_nums + nums[6:] + - [assembler.CODE_STOP]) - - # also test rebuild_faillocs_from_descr(), which should not - # reproduce the holes at all - bytecode = lltype.malloc(rffi.UCHARP.TO, len(mc.content), flavor='raw', - immortal=True) - for i in range(len(mc.content)): - assert 0 <= mc.content[i] <= 255 - bytecode[i] = rffi.cast(rffi.UCHAR, mc.content[i]) - bytecode_addr = rffi.cast(lltype.Signed, bytecode) - newlocs = assembler.rebuild_faillocs_from_descr(bytecode_addr) - assert ([loc.assembler() for loc in newlocs] == - [loc.assembler() for loc in locs if loc is not None]) - -# ____________________________________________________________ - -def test_failure_recovery_func_no_floats(): - do_failure_recovery_func(withfloats=False) - -def test_failure_recovery_func_with_floats(): - do_failure_recovery_func(withfloats=True) - -def do_failure_recovery_func(withfloats): - import random - S = lltype.GcStruct('S') - - def get_random_int(): - return random.randrange(-10000, 10000) - - def get_random_ptr(): - return lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S)) - - def get_random_float(): - # Returns , , - # NB: on 64-bit, will be the entire float and - # will be random garbage from malloc! - assert withfloats - value = random.random() - 0.5 - # make sure it fits into 64 bits - tmp = lltype.malloc(rffi.LONGP.TO, 2, flavor='raw', - track_allocation=False) - rffi.cast(rffi.DOUBLEP, tmp)[0] = value - return rffi.cast(rffi.DOUBLEP, tmp)[0], tmp[0], tmp[1] - - if IS_X86_32: - main_registers = X86RegisterManager.all_regs - xmm_registers = X86XMMRegisterManager.all_regs - elif IS_X86_64: - main_registers = X86_64_RegisterManager.all_regs - xmm_registers = X86_64_XMMRegisterManager.all_regs - - # memory locations: 26 integers, 26 pointers, 26 floats - # main registers: half of them as signed and the other half as ptrs - # xmm registers: all floats, from xmm0 to xmm(7|15) - # holes: 8 - locations = [] - baseloc = 4 - for i in range(26+26+26): - if baseloc < 128: - baseloc += random.randrange(2, 20) - else: - baseloc += random.randrange(2, 1000) - locations.append(baseloc) - random.shuffle(locations) - content = ([('int', locations.pop()) for _ in range(26)] + - [('ptr', locations.pop()) for _ in range(26)] + - [(['int', 'ptr'][random.randrange(0, 2)], reg) - for reg in main_registers]) - if withfloats: - content += ([('float', locations.pop()) for _ in range(26)] + - [('float', reg) for reg in xmm_registers]) - for i in range(8): - content.append(('hole', None)) - random.shuffle(content) - - # prepare the expected target arrays, the descr_bytecode, - # the 'registers' and the 'stack' arrays according to 'content' - xmmregisters = lltype.malloc(rffi.LONGP.TO, 16+ACTUAL_CPU.NUM_REGS+1, - flavor='raw', immortal=True) - registers = rffi.ptradd(xmmregisters, 16) - stacklen = baseloc + 30 - stack = lltype.malloc(rffi.LONGP.TO, stacklen, flavor='raw', - immortal=True) - expected_ints = [None] * len(content) - expected_ptrs = [None] * len(content) - expected_floats = [None] * len(content) - - def write_in_stack(loc, value): - assert loc >= 0 - ofs = get_ebp_ofs(loc) - assert ofs < 0 - assert (ofs % WORD) == 0 - stack[stacklen + ofs//WORD] = value - - descr_bytecode = [] - for i, (kind, loc) in enumerate(content): - if kind == 'hole': - num = Assembler386.CODE_HOLE - else: - if kind == 'float': - value, lo, hi = get_random_float() - expected_floats[i] = longlong.getfloatstorage(value) - kind = Assembler386.DESCR_FLOAT - if isinstance(loc, RegLoc): - if WORD == 4: - xmmregisters[2*loc.value] = lo - xmmregisters[2*loc.value+1] = hi - elif WORD == 8: - xmmregisters[loc.value] = lo - else: - if WORD == 4: - write_in_stack(loc, hi) - write_in_stack(loc+1, lo) - elif WORD == 8: - write_in_stack(loc, lo) - else: - if kind == 'int': - value = get_random_int() - expected_ints[i] = value - kind = Assembler386.DESCR_INT - elif kind == 'ptr': - value = get_random_ptr() - expected_ptrs[i] = value - kind = Assembler386.DESCR_REF - value = rffi.cast(rffi.LONG, value) - else: - assert 0, kind - if isinstance(loc, RegLoc): - registers[loc.value] = value - else: - write_in_stack(loc, value) - - if isinstance(loc, RegLoc): - num = kind + 4*loc.value - else: - num = kind + Assembler386.CODE_FROMSTACK + (4*loc) - while num >= 0x80: - descr_bytecode.append((num & 0x7F) | 0x80) - num >>= 7 - descr_bytecode.append(num) - - descr_bytecode.append(Assembler386.CODE_STOP) - descr_bytecode.append(0xC3) # fail_index = 0x1C3 - descr_bytecode.append(0x01) - descr_bytecode.append(0x00) - descr_bytecode.append(0x00) - descr_bytecode.append(0xCC) # end marker - descr_bytes = lltype.malloc(rffi.UCHARP.TO, len(descr_bytecode), - flavor='raw', immortal=True) - for i in range(len(descr_bytecode)): - assert 0 <= descr_bytecode[i] <= 255 - descr_bytes[i] = rffi.cast(rffi.UCHAR, descr_bytecode[i]) - registers[ACTUAL_CPU.NUM_REGS] = rffi.cast(rffi.LONG, descr_bytes) - registers[ebp.value] = rffi.cast(rffi.LONG, stack) + WORD*stacklen - - XXX # rewrite - - # run! - assembler = Assembler386(FakeCPU()) - deadframe = assembler.failure_recovery_func(registers) - deadframe = lltype.cast_opaque_ptr(jitframe.DEADFRAMEPTR, deadframe) - assert deadframe.jf_descr == rffi.cast(llmemory.GCREF, 123) - - # check the fail_boxes - for i in range(len(content)): - if expected_ints[i] is not None: - assert deadframe.jf_values[i].int == expected_ints[i] - if expected_ptrs[i] is not None: - assert deadframe.jf_values[i].ref == expected_ptrs[i] - # note: we expect *exact* results below. If you have only - # an approximate result, it might mean that only the first 32 - # bits of the float were correctly saved and restored. - if expected_floats[i] is not None: - assert deadframe.jf_values[i].float == expected_floats[i] - # ____________________________________________________________ class TestRegallocPushPop(object): From noreply at buildbot.pypy.org Tue Jan 15 19:40:46 2013 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 15 Jan 2013 19:40:46 +0100 (CET) Subject: [pypy-commit] extradoc extradoc: abstract for SEA 2013 Message-ID: <20130115184046.9ACEB1C0131@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: extradoc Changeset: r4941:47317bf880e4 Date: 2013-01-15 20:19 +0200 http://bitbucket.org/pypy/extradoc/changeset/47317bf880e4/ Log: abstract for SEA 2013 diff --git a/talk/sea2013/abstract.rst b/talk/sea2013/abstract.rst new file mode 100644 --- /dev/null +++ b/talk/sea2013/abstract.rst @@ -0,0 +1,22 @@ +High performance Python +======================= + +I would like to present an intermediate level talk on High Performance Python. +Python has seen quite a lot of adoption by the scientific community, however, +the main part of it has been as a glue language. Python is used a lot for +data processing, driving simulations and presenting, however the main +part of scientific processing is usually done with Fortran, C or C++. +Recent advancements in numeric tools available for Python make it feasible +to develop a prototype in Python and often even leave your entire model +in Python. I will give a brief overview on available tools, profilers and +literature how you can achieve that. This talk will briefly cover CPython, +PyPy, NumPy, Cython, Numba, Numexpr and other available performance-related +packages. + +I am a core developer of the PyPy project as well as an implementor of +the numpy package for PyPy. I have extensive knowledge in profiling +python software including numeric packages. Furthermore I've implemented +various tools that simplify profiling. + +The preferred talk length is one hour. + From noreply at buildbot.pypy.org Tue Jan 15 19:41:59 2013 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 15 Jan 2013 19:41:59 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fix some tests, skip others Message-ID: <20130115184159.B63601C0131@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60097:656ef70c79ae Date: 2013-01-15 20:41 +0200 http://bitbucket.org/pypy/pypy/changeset/656ef70c79ae/ Log: fix some tests, skip others diff --git a/pypy/jit/backend/x86/test/test_regalloc.py b/pypy/jit/backend/x86/test/test_regalloc.py --- a/pypy/jit/backend/x86/test/test_regalloc.py +++ b/pypy/jit/backend/x86/test/test_regalloc.py @@ -178,22 +178,22 @@ return regalloc def getint(self, index): - return self.cpu.get_latest_value_int(self.deadframe, index) + return self.cpu.get_int_value(self.deadframe, index) def getfloat(self, index): - return self.cpu.get_latest_value_float(self.deadframe, index) + return self.cpu.get_float_value(self.deadframe, index) def getints(self, end): - return [self.cpu.get_latest_value_int(self.deadframe, index) for + return [self.cpu.get_int_value(self.deadframe, index) for index in range(0, end)] def getfloats(self, end): return [longlong.getrealfloat( - self.cpu.get_latest_value_float(self.deadframe, index)) + self.cpu.get_float_value(self.deadframe, index)) for index in range(0, end)] def getptr(self, index, T): - gcref = self.cpu.get_latest_value_ref(self.deadframe, index) + gcref = self.cpu.get_ref_value(self.deadframe, index) return lltype.cast_opaque_ptr(T, gcref) def attach_bridge(self, ops, loop, guard_op_index, **kwds): @@ -459,12 +459,9 @@ jump(i4, i1, i2, i3) """ regalloc = self.prepare_loop(ops) - if IS_X86_64: - assert len(regalloc.rm.reg_bindings) == 4 - assert len(regalloc.fm.bindings) == 0 - else: - assert len(regalloc.rm.reg_bindings) == 0 - assert len(regalloc.fm.bindings) == 4 + # we pass stuff on the frame + assert len(regalloc.rm.reg_bindings) == 0 + assert len(regalloc.fm.bindings) == 4 class TestRegallocCompOps(BaseTestRegalloc): @@ -628,6 +625,9 @@ assert self.getints(9) == [0, 1, 1, 1, 1, 1, 1, 1, 1] class TestRegAllocCallAndStackDepth(BaseTestRegalloc): + def setup_class(cls): + py.test.skip("skip for now, not sure what do we do") + def expected_frame_depth(self, num_call_args, num_pushed_input_args=0): # Assumes the arguments are all non-float if IS_X86_32: diff --git a/pypy/jit/backend/x86/test/test_regalloc2.py b/pypy/jit/backend/x86/test/test_regalloc2.py --- a/pypy/jit/backend/x86/test/test_regalloc2.py +++ b/pypy/jit/backend/x86/test/test_regalloc2.py @@ -1,6 +1,6 @@ import py from pypy.jit.metainterp.history import ResOperation, BoxInt, ConstInt,\ - BoxPtr, ConstPtr, BasicFailDescr, JitCellToken + BoxPtr, ConstPtr, BasicFailDescr, JitCellToken, BasicFinalDescr from pypy.jit.metainterp.resoperation import rop from pypy.jit.backend.detect_cpu import getcpuclass from pypy.jit.backend.x86.arch import WORD @@ -19,7 +19,7 @@ ResOperation(rop.UINT_RSHIFT, [v1, ConstInt(3)], v4), ResOperation(rop.SAME_AS, [ConstInt(0)], zero), ResOperation(rop.GUARD_TRUE, [zero], None, descr=BasicFailDescr()), - ResOperation(rop.FINISH, [], None, descr=BasicFailDescr()) + ResOperation(rop.FINISH, [], None, descr=BasicFinalDescr()) ] operations[-2].setfailargs([v4, v3]) cpu = CPU(None, None) @@ -27,8 +27,8 @@ looptoken = JitCellToken() cpu.compile_loop(inputargs, operations, looptoken) deadframe = cpu.execute_token(looptoken, 9) - assert cpu.get_latest_value_int(deadframe, 0) == (9 >> 3) - assert cpu.get_latest_value_int(deadframe, 1) == (~18) + assert cpu.get_int_value(deadframe, 0) == (9 >> 3) + assert cpu.get_int_value(deadframe, 1) == (~18) def test_bug_int_is_true_1(): v1 = BoxInt() @@ -45,7 +45,7 @@ ResOperation(rop.INT_IS_ZERO, [tmp5], v4), ResOperation(rop.SAME_AS, [ConstInt(0)], zero), ResOperation(rop.GUARD_TRUE, [zero], None, descr=BasicFailDescr()), - ResOperation(rop.FINISH, [], None, descr=BasicFailDescr()) + ResOperation(rop.FINISH, [], None, descr=BasicFinalDescr()) ] operations[-2].setfailargs([v4, v3, tmp5]) cpu = CPU(None, None) @@ -53,9 +53,9 @@ looptoken = JitCellToken() cpu.compile_loop(inputargs, operations, looptoken) deadframe = cpu.execute_token(looptoken, -10) - assert cpu.get_latest_value_int(deadframe, 0) == 0 - assert cpu.get_latest_value_int(deadframe, 1) == -1000 - assert cpu.get_latest_value_int(deadframe, 2) == 1 + assert cpu.get_int_value(deadframe, 0) == 0 + assert cpu.get_int_value(deadframe, 1) == -1000 + assert cpu.get_int_value(deadframe, 2) == 1 def test_bug_0(): v1 = BoxInt() @@ -145,7 +145,7 @@ ResOperation(rop.INT_GT, [v24, v32], v40), ResOperation(rop.SAME_AS, [ConstInt(0)], zero), ResOperation(rop.GUARD_TRUE, [zero], None, descr=BasicFailDescr()), - ResOperation(rop.FINISH, [], None, descr=BasicFailDescr()) + ResOperation(rop.FINISH, [], None, descr=BasicFinalDescr()) ] operations[-2].setfailargs([v40, v36, v37, v31, v16, v34, v35, v23, v22, v29, v14, v39, v30, v38]) @@ -155,20 +155,20 @@ cpu.compile_loop(inputargs, operations, looptoken) deadframe = cpu.execute_token(looptoken, -13, 10, 10, 8, -8, -16, -18, 46, -12, 26) - assert cpu.get_latest_value_int(deadframe, 0) == 0 - assert cpu.get_latest_value_int(deadframe, 1) == 0 - assert cpu.get_latest_value_int(deadframe, 2) == 0 - assert cpu.get_latest_value_int(deadframe, 3) == 0 - assert cpu.get_latest_value_int(deadframe, 4) == 1 - assert cpu.get_latest_value_int(deadframe, 5) == -7 - assert cpu.get_latest_value_int(deadframe, 6) == 1 - assert cpu.get_latest_value_int(deadframe, 7) == 0 - assert cpu.get_latest_value_int(deadframe, 8) == -2 - assert cpu.get_latest_value_int(deadframe, 9) == 18 - assert cpu.get_latest_value_int(deadframe, 10) == 1 - assert cpu.get_latest_value_int(deadframe, 11) == 18 - assert cpu.get_latest_value_int(deadframe, 12) == -1 - assert cpu.get_latest_value_int(deadframe, 13) == 0 + assert cpu.get_int_value(deadframe, 0) == 0 + assert cpu.get_int_value(deadframe, 1) == 0 + assert cpu.get_int_value(deadframe, 2) == 0 + assert cpu.get_int_value(deadframe, 3) == 0 + assert cpu.get_int_value(deadframe, 4) == 1 + assert cpu.get_int_value(deadframe, 5) == -7 + assert cpu.get_int_value(deadframe, 6) == 1 + assert cpu.get_int_value(deadframe, 7) == 0 + assert cpu.get_int_value(deadframe, 8) == -2 + assert cpu.get_int_value(deadframe, 9) == 18 + assert cpu.get_int_value(deadframe, 10) == 1 + assert cpu.get_int_value(deadframe, 11) == 18 + assert cpu.get_int_value(deadframe, 12) == -1 + assert cpu.get_int_value(deadframe, 13) == 0 def test_bug_1(): v1 = BoxInt() @@ -256,7 +256,7 @@ ResOperation(rop.INT_NEG, [v27], v40), ResOperation(rop.SAME_AS, [ConstInt(0)], zero), ResOperation(rop.GUARD_TRUE, [zero], None, descr=BasicFailDescr()), - ResOperation(rop.FINISH, [], None, descr=BasicFailDescr()) + ResOperation(rop.FINISH, [], None, descr=BasicFinalDescr()) ] operations[-2].setfailargs([v40, v10, v36, v26, v13, v30, v21, v33, v18, v25, v31, v32, v28, v29, v35, v38, @@ -267,27 +267,27 @@ cpu.compile_loop(inputargs, operations, looptoken) deadframe = cpu.execute_token(looptoken, 17, -20, -6, 6, 1, 13, 13, 9, 49, 8) - assert cpu.get_latest_value_int(deadframe, 0) == 0 - assert cpu.get_latest_value_int(deadframe, 1) == 8 - assert cpu.get_latest_value_int(deadframe, 2) == 1 - assert cpu.get_latest_value_int(deadframe, 3) == 131072 - assert cpu.get_latest_value_int(deadframe, 4) == 20 - assert cpu.get_latest_value_int(deadframe, 5) == -1 - assert cpu.get_latest_value_int(deadframe, 6) == 0 - assert cpu.get_latest_value_int(deadframe, 7) == -19 - assert cpu.get_latest_value_int(deadframe, 8) == 6 - assert cpu.get_latest_value_int(deadframe, 9) == 26 - assert cpu.get_latest_value_int(deadframe, 10) == 12 - assert cpu.get_latest_value_int(deadframe, 11) == 0 - assert cpu.get_latest_value_int(deadframe, 12) == 0 - assert cpu.get_latest_value_int(deadframe, 13) == 2 - assert cpu.get_latest_value_int(deadframe, 14) == 2 - assert cpu.get_latest_value_int(deadframe, 15) == 1 - assert cpu.get_latest_value_int(deadframe, 16) == -57344 - assert cpu.get_latest_value_int(deadframe, 17) == 1 - assert cpu.get_latest_value_int(deadframe, 18) == -1 + assert cpu.get_int_value(deadframe, 0) == 0 + assert cpu.get_int_value(deadframe, 1) == 8 + assert cpu.get_int_value(deadframe, 2) == 1 + assert cpu.get_int_value(deadframe, 3) == 131072 + assert cpu.get_int_value(deadframe, 4) == 20 + assert cpu.get_int_value(deadframe, 5) == -1 + assert cpu.get_int_value(deadframe, 6) == 0 + assert cpu.get_int_value(deadframe, 7) == -19 + assert cpu.get_int_value(deadframe, 8) == 6 + assert cpu.get_int_value(deadframe, 9) == 26 + assert cpu.get_int_value(deadframe, 10) == 12 + assert cpu.get_int_value(deadframe, 11) == 0 + assert cpu.get_int_value(deadframe, 12) == 0 + assert cpu.get_int_value(deadframe, 13) == 2 + assert cpu.get_int_value(deadframe, 14) == 2 + assert cpu.get_int_value(deadframe, 15) == 1 + assert cpu.get_int_value(deadframe, 16) == -57344 + assert cpu.get_int_value(deadframe, 17) == 1 + assert cpu.get_int_value(deadframe, 18) == -1 if WORD == 4: - assert cpu.get_latest_value_int(deadframe, 19) == -2147483648 + assert cpu.get_int_value(deadframe, 19) == -2147483648 elif WORD == 8: - assert cpu.get_latest_value_int(deadframe, 19) == 19327352832 - assert cpu.get_latest_value_int(deadframe, 20) == -49 + assert cpu.get_int_value(deadframe, 19) == 19327352832 + assert cpu.get_int_value(deadframe, 20) == -49 From noreply at buildbot.pypy.org Tue Jan 15 20:26:38 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Tue, 15 Jan 2013 20:26:38 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed a bunch of tests Message-ID: <20130115192638.797391C1353@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60098:cfb2f3a5e88f Date: 2013-01-15 20:26 +0100 http://bitbucket.org/pypy/pypy/changeset/cfb2f3a5e88f/ Log: Fixed a bunch of tests diff --git a/rpython/rlib/test/test_rdtoa.py b/rpython/rlib/test/test_rdtoa.py --- a/rpython/rlib/test/test_rdtoa.py +++ b/rpython/rlib/test/test_rdtoa.py @@ -1,3 +1,4 @@ +import py from rpython.rlib.rdtoa import strtod, dtoa from rpython.rlib import rfloat @@ -6,11 +7,11 @@ assert strtod("1.1") == 1.1 assert strtod("3.47") == 3.47 assert strtod(".125") == .125 - raises(ValueError, strtod, "123A") - raises(ValueError, strtod, "") - raises(ValueError, strtod, " ") - raises(ValueError, strtod, "\0") - raises(ValueError, strtod, "3\09") + py.test.raises(ValueError, strtod, "123A") + py.test.raises(ValueError, strtod, "") + py.test.raises(ValueError, strtod, " ") + py.test.raises(ValueError, strtod, "\0") + py.test.raises(ValueError, strtod, "3\09") def test_dtoa(): assert dtoa(3.47) == "3.47" diff --git a/rpython/rlib/test/test_rmmap.py b/rpython/rlib/test/test_rmmap.py --- a/rpython/rlib/test/test_rmmap.py +++ b/rpython/rlib/test/test_rmmap.py @@ -1,5 +1,5 @@ from rpython.tool.udir import udir -import os, sys +import os, sys, py from rpython.rtyper.test.test_llinterp import interpret from rpython.rlib.rarithmetic import intmask from rpython.rlib import rmmap as mmap @@ -255,24 +255,24 @@ def test_write_readonly(self): if os.name == "nt": - skip("Needs PROT_READ") + py.test.skip("Needs PROT_READ") f = open(self.tmpname + "l", "w+") f.write("foobar") f.flush() m = mmap.mmap(f.fileno(), 6, prot=mmap.PROT_READ) - raises(RTypeError, m.write, "foo") + py.test.raises(RTypeError, m.write, "foo") m.close() f.close() def test_write_without_protwrite(self): if os.name == "nt": - skip("Needs PROT_WRITE") + py.test.skip("Needs PROT_WRITE") f = open(self.tmpname + "l2", "w+") f.write("foobar") f.flush() m = mmap.mmap(f.fileno(), 6, prot=~mmap.PROT_WRITE) - raises(RTypeError, m.write_byte, 'a') - raises(RTypeError, m.write, "foo") + py.test.raises(RTypeError, m.write_byte, 'a') + py.test.raises(RTypeError, m.write, "foo") m.close() f.close() @@ -325,7 +325,7 @@ def test_resize(self): if ("darwin" in sys.platform) or ("freebsd" in sys.platform): - skip("resize does not work under OSX or FreeBSD") + py.test.skip("resize does not work under OSX or FreeBSD") import os @@ -430,7 +430,7 @@ def test_windows_crasher_1(self): if sys.platform != "win32": - skip("Windows-only test") + py.test.skip("Windows-only test") def func(): m = mmap.mmap(-1, 1000, tagname="foo") # same tagname, but larger size @@ -444,7 +444,7 @@ def test_windows_crasher_2(self): if sys.platform != "win32": - skip("Windows-only test") + py.test.skip("Windows-only test") f = open(self.tmpname + "t", "w+") f.write("foobar") @@ -453,8 +453,8 @@ f = open(self.tmpname + "t", "r+b") m = mmap.mmap(f.fileno(), 0) f.close() - raises(WindowsError, m.resize, 0) - raises(RValueError, m.getitem, 0) + py.test.raises(WindowsError, m.resize, 0) + py.test.raises(RValueError, m.getitem, 0) m.close() def test_alloc_free(): diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py --- a/rpython/rlib/test/test_rposix.py +++ b/rpython/rlib/test/test_rposix.py @@ -134,7 +134,7 @@ def test_is_valid_fd(self): if os.name != 'nt': - skip('relevant for windows only') + py.test.skip('relevant for windows only') assert rposix.is_valid_fd(0) == 1 fid = open(str(udir.join('validate_test.txt')), 'w') fd = fid.fileno() diff --git a/rpython/rlib/test/test_rsignal.py b/rpython/rlib/test/test_rsignal.py --- a/rpython/rlib/test/test_rsignal.py +++ b/rpython/rlib/test/test_rsignal.py @@ -5,7 +5,7 @@ def setup_module(mod): if not hasattr(os, 'kill') or not hasattr(os, 'getpid'): py.test.skip("requires os.kill() and os.getpid()") - if not hasattr(signals, 'SIGUSR1'): + if not hasattr(rsignal, 'SIGUSR1'): py.test.skip("requires SIGUSR1 in signal") diff --git a/rpython/rlib/test/test_rstring.py b/rpython/rlib/test/test_rstring.py --- a/rpython/rlib/test/test_rstring.py +++ b/rpython/rlib/test/test_rstring.py @@ -1,4 +1,4 @@ -import sys +import sys, py from rpython.rlib.rstring import StringBuilder, UnicodeBuilder, split, rsplit @@ -11,7 +11,7 @@ assert split('a|b|c|d', '|', 2) == ['a', 'b', 'c|d'] assert split('a//b//c//d', '//') == ['a', 'b', 'c', 'd'] assert split('endcase test', 'test') == ['endcase ', ''] - raises(ValueError, split, 'abc', '') + py.test.raises(ValueError, split, 'abc', '') def test_rsplit(): assert rsplit("a", "a", 1) == ['', ''] @@ -21,7 +21,7 @@ assert rsplit('a|b|c|d', '|', 2) == ['a|b', 'c', 'd'] assert rsplit('a//b//c//d', '//') == ['a', 'b', 'c', 'd'] assert rsplit('endcase test', 'test') == ['endcase ', ''] - raises(ValueError, rsplit, "abc", '') + py.test.raises(ValueError, rsplit, "abc", '') def test_string_builder(): s = StringBuilder() diff --git a/rpython/rlib/test/test_runicode.py b/rpython/rlib/test/test_runicode.py --- a/rpython/rlib/test/test_runicode.py +++ b/rpython/rlib/test/test_runicode.py @@ -251,7 +251,7 @@ def test_utf8_surrogate(self): # surrogates used to be allowed by python 2.x - raises(UnicodeDecodeError, self.checkdecode, u"\ud800", "utf-8") + py.test.raises(UnicodeDecodeError, self.checkdecode, u"\ud800", "utf-8") def test_invalid_start_byte(self): """ @@ -263,7 +263,7 @@ """ FFFD = u'\ufffd' for byte in '\x80\xA0\x9F\xBF\xC0\xC1\xF5\xFF': - raises(UnicodeDecodeError, self.decoder, byte, 1, None, final=True) + py.test.raises(UnicodeDecodeError, self.decoder, byte, 1, None, final=True) self.checkdecodeerror(byte, 'utf-8', 0, 1, addstuff=False, msg='invalid start byte') assert self.decoder(byte, 1, 'replace', final=True) == (FFFD, 1) @@ -298,7 +298,7 @@ FFFD = u'\ufffd' for seq in sequences: seq = self.to_bytestring(seq) - raises(UnicodeDecodeError, self.decoder, seq, len(seq), + py.test.raises(UnicodeDecodeError, self.decoder, seq, len(seq), None, final=True) self.checkdecodeerror(seq, 'utf-8', 0, len(seq), addstuff=False, msg='unexpected end of data') @@ -332,7 +332,7 @@ ] for seq, res in sequences: seq = self.to_bytestring(seq) - raises(UnicodeDecodeError, self.decoder, seq, len(seq), + py.test.raises(UnicodeDecodeError, self.decoder, seq, len(seq), None, final=True) self.checkdecodeerror(seq, 'utf-8', 0, 1, addstuff=False, msg='invalid continuation byte') @@ -404,7 +404,7 @@ ] for seq, res in sequences: seq = self.to_bytestring(seq) - raises(UnicodeDecodeError, self.decoder, seq, len(seq), + py.test.raises(UnicodeDecodeError, self.decoder, seq, len(seq), None, final=True) self.checkdecodeerror(seq, 'utf-8', 0, len(seq)-1, addstuff=False, msg='invalid continuation byte') @@ -496,7 +496,7 @@ ] for seq, res in sequences: seq = self.to_bytestring(seq) - raises(UnicodeDecodeError, self.decoder, seq, len(seq), + py.test.raises(UnicodeDecodeError, self.decoder, seq, len(seq), None, final=True) self.checkdecodeerror(seq, 'utf-8', 0, len(seq)-1, addstuff=False, msg='invalid continuation byte') @@ -626,7 +626,7 @@ for n, (seq, res) in enumerate(sequences): decoder = self.getdecoder('utf-8') - raises(UnicodeDecodeError, decoder, seq, len(seq), None, final=True) + py.test.raises(UnicodeDecodeError, decoder, seq, len(seq), None, final=True) assert decoder(seq, len(seq), 'replace', final=True ) == (res, len(seq)) assert decoder(seq + 'b', len(seq) + 1, 'replace', final=True @@ -694,7 +694,7 @@ self.checkencode(u, "utf-8") else: # This is not done in wide unicode builds - raises(UnicodeEncodeError, self.checkencode, u, "utf-8") + py.test.raises(UnicodeEncodeError, self.checkencode, u, "utf-8") def test_ascii_error(self): self.checkencodeerror(u"abc\xFF\xFF\xFFcde", "ascii", 3, 6) @@ -714,7 +714,7 @@ def test_encode_decimal(self): encoder = self.getencoder('decimal') assert encoder(u' 12, 34 ', 8, None) == ' 12, 34 ' - raises(UnicodeEncodeError, encoder, u' 12, \u1234 ', 7, None) + py.test.raises(UnicodeEncodeError, encoder, u' 12, \u1234 ', 7, None) assert encoder(u'u\u1234', 2, 'replace') == 'u?' class TestTranslation(object): diff --git a/rpython/rlib/test/test_rzlib.py b/rpython/rlib/test/test_rzlib.py --- a/rpython/rlib/test/test_rzlib.py +++ b/rpython/rlib/test/test_rzlib.py @@ -70,8 +70,8 @@ deflateInit() should raise ValueError when an out of bounds level is passed to it. """ - raises(ValueError, rzlib.deflateInit, -2) - raises(ValueError, rzlib.deflateInit, 10) + py.test.raises(ValueError, rzlib.deflateInit, -2) + py.test.raises(ValueError, rzlib.deflateInit, 10) def test_deflate_init_end(): diff --git a/rpython/rlib/unicodedata/test/test_unicodedata.py b/rpython/rlib/unicodedata/test/test_unicodedata.py --- a/rpython/rlib/unicodedata/test/test_unicodedata.py +++ b/rpython/rlib/unicodedata/test/test_unicodedata.py @@ -1,3 +1,4 @@ +import py from rpython.rlib.unicodedata import unicodedb_3_2_0, unicodedb_5_2_0 class AppTestUnicodeData: @@ -39,8 +40,8 @@ assert unicodedata.name(unichr(code)) == name assert unicodedata.lookup(name) == unichr(code) # Test outside the range - raises(ValueError, unicodedata.name, unichr(0xAC00 - 1)) - raises(ValueError, unicodedata.name, unichr(0xD7A3 + 1)) + py.test.raises(ValueError, unicodedata.name, unichr(0xAC00 - 1)) + py.test.raises(ValueError, unicodedata.name, unichr(0xD7A3 + 1)) def test_cjk(self): import sys @@ -71,7 +72,7 @@ unicodedata.name(char) except ValueError, e: assert e.message == 'no such name' - raises(KeyError, unicodedata.lookup, charname) + py.test.raises(KeyError, unicodedata.lookup, charname) def test_bug_1704793(self): # from CPython import unicodedata @@ -79,7 +80,7 @@ def test_normalize(self): import unicodedata - raises(TypeError, unicodedata.normalize, 'x') + py.test.raises(TypeError, unicodedata.normalize, 'x') def test_normalize_wide(self): import sys, unicodedata @@ -128,7 +129,7 @@ def test_random_missing_chars(self): for chr in self.nocharlist: - raises(KeyError, unicodedb_5_2_0.name, ord(chr)) + py.test.raises(KeyError, unicodedb_5_2_0.name, ord(chr)) def test_compare_functions(self): import unicodedata # CPython implementation @@ -173,7 +174,7 @@ def test_differences(self): assert unicodedb_5_2_0.name(9187) == 'BENZENE RING WITH CIRCLE' assert unicodedb_5_2_0.lookup('BENZENE RING WITH CIRCLE') == 9187 - raises(KeyError, unicodedb_3_2_0.lookup, 'BENZENE RING WITH CIRCLE') - raises(KeyError, unicodedb_3_2_0.name, 9187) + py.test.raises(KeyError, unicodedb_3_2_0.lookup, 'BENZENE RING WITH CIRCLE') + py.test.raises(KeyError, unicodedb_3_2_0.name, 9187) diff --git a/rpython/rtyper/lltypesystem/test/test_ll2ctypes.py b/rpython/rtyper/lltypesystem/test/test_ll2ctypes.py --- a/rpython/rtyper/lltypesystem/test/test_ll2ctypes.py +++ b/rpython/rtyper/lltypesystem/test/test_ll2ctypes.py @@ -514,8 +514,8 @@ # and it can be called with the expected number of arguments res = cmul(41, 42) assert res == 41 * 42 - raises(TypeError, cmul, 41) - raises(TypeError, cmul, 41, 42, 43) + py.test.raises(TypeError, cmul, 41) + py.test.raises(TypeError, cmul, 41, 42, 43) def test_qsort(self): CMPFUNC = lltype.FuncType([rffi.VOIDP, rffi.VOIDP], rffi.INT) diff --git a/rpython/rtyper/module/test/test_ll_os.py b/rpython/rtyper/module/test/test_ll_os.py --- a/rpython/rtyper/module/test/test_ll_os.py +++ b/rpython/rtyper/module/test/test_ll_os.py @@ -102,7 +102,7 @@ def test_mkdir(): filename = str(udir.join('test_mkdir.dir')) getllimpl(os.mkdir)(filename, 0) - exc = raises(OSError, getllimpl(os.mkdir), filename, 0) + exc = py.test.raises(OSError, getllimpl(os.mkdir), filename, 0) assert exc.value.errno == errno.EEXIST if sys.platform == 'win32': assert exc.type is WindowsError @@ -202,7 +202,7 @@ assert fid.read() == "Hello world" fd = os.open(fname, os.O_WRONLY|os.O_CREAT, 0777) os.close(fd) - raises(OSError, f, fd, 'Hello world') + py.test.raises(OSError, f, fd, 'Hello world') def test_os_close(): fname = str(udir.join('os_test.txt')) @@ -211,7 +211,7 @@ os.write(fd, 'Hello world') f = getllimpl(os.close) f(fd) - raises(OSError, f, fd) + py.test.raises(OSError, f, fd) def test_os_lseek(): fname = str(udir.join('os_test.txt')) @@ -222,7 +222,7 @@ f(fd,0,0) assert os.read(fd, 11) == 'Hello world' os.close(fd) - raises(OSError, f, fd, 0, 0) + py.test.raises(OSError, f, fd, 0, 0) def test_os_fsync(): fname = str(udir.join('os_test.txt')) @@ -235,13 +235,13 @@ fid = open(fname) assert fid.read() == 'Hello world' fid.close() - raises(OSError, f, fd) + py.test.raises(OSError, f, fd) def test_os_fdatasync(): try: f = getllimpl(os.fdatasync) except: - skip('No fdatasync in os') + py.test.skip('No fdatasync in os') fname = str(udir.join('os_test.txt')) fd = os.open(fname, os.O_WRONLY|os.O_CREAT, 0777) assert fd >= 0 @@ -250,12 +250,12 @@ fid = open(fname) assert fid.read() == 'Hello world' os.close(fd) - raises(OSError, f, fd) + py.test.raises(OSError, f, fd) def test_os_kill(): if not hasattr(os,'kill') or sys.platform == 'win32': - skip('No kill in os') + py.test.skip('No kill in os') f = getllimpl(os.kill) import subprocess import signal @@ -272,7 +272,7 @@ try: f = getllimpl(os.isatty) except: - skip('No isatty in os') + py.test.skip('No isatty in os') assert f(-1) == False diff --git a/rpython/rtyper/module/test/test_posix.py b/rpython/rtyper/module/test/test_posix.py --- a/rpython/rtyper/module/test/test_posix.py +++ b/rpython/rtyper/module/test/test_posix.py @@ -141,7 +141,7 @@ text = 'This is a test' os.write(fi,text) res = self.interpret(f,[fi]) - raises( OSError, os.fstat, fi) + py.test.raises( OSError, os.fstat, fi) if hasattr(os, 'ftruncate'): def test_ftruncate(self): diff --git a/rpython/rtyper/test/test_extfunc.py b/rpython/rtyper/test/test_extfunc.py --- a/rpython/rtyper/test/test_extfunc.py +++ b/rpython/rtyper/test/test_extfunc.py @@ -1,3 +1,4 @@ +import py from rpython.rtyper.extfunc import ExtFuncEntry, register_external,\ is_external, lazy_register @@ -177,7 +178,7 @@ a.translator.config.translation.check_str_without_nul=True def g(s): return os_open(s) - raises(Exception, a.build_types, g, [str]) + py.test.raises(Exception, a.build_types, g, [str]) a.build_types(g, [str0]) # Does not raise def test_list_of_str0(self): @@ -195,7 +196,7 @@ a.translator.config.translation.check_str_without_nul=True def g(l): return os_execve(l) - raises(Exception, a.build_types, g, [[str]]) + py.test.raises(Exception, a.build_types, g, [[str]]) a.build_types(g, [[str0]]) # Does not raise diff --git a/rpython/rtyper/test/test_rbuiltin.py b/rpython/rtyper/test/test_rbuiltin.py --- a/rpython/rtyper/test/test_rbuiltin.py +++ b/rpython/rtyper/test/test_rbuiltin.py @@ -206,7 +206,7 @@ assert hello == "hello world" fd = os.open(tmpdir, os.O_WRONLY|os.O_CREAT, 777) os.close(fd) - raises(OSError, os.write, fd, "hello world") + py.test.raises(OSError, os.write, fd, "hello world") def test_os_write_single_char(self): tmpdir = str(udir.udir.join("os_write_test_char")) diff --git a/rpython/rtyper/test/test_rfloat.py b/rpython/rtyper/test/test_rfloat.py --- a/rpython/rtyper/test/test_rfloat.py +++ b/rpython/rtyper/test/test_rfloat.py @@ -1,4 +1,4 @@ -import sys +import sys, py from rpython.translator.translator import TranslationContext from rpython.annotator import unaryop, binaryop from rpython.rtyper.test import snippet @@ -234,7 +234,7 @@ assert break_up_float('-') == ('-', '', '', '') assert break_up_float('e1') == ('', '', '', '1') - raises(ValueError, break_up_float, 'e') + py.test.raises(ValueError, break_up_float, 'e') def test_formatd(self): from rpython.rlib.rfloat import formatd @@ -397,13 +397,13 @@ class TestOOtype(BaseTestRfloat, OORtypeMixin): def test_formatd(self): - skip('formatd is broken on ootype') + py.test.skip('formatd is broken on ootype') def test_formatd_repr(self): - skip('formatd is broken on ootype') + py.test.skip('formatd is broken on ootype') def test_formatd_huge(self): - skip('formatd is broken on ootype') + py.test.skip('formatd is broken on ootype') def test_parts_to_float(self): - skip('parts_to_float is broken on ootype') + py.test.skip('parts_to_float is broken on ootype') diff --git a/rpython/rtyper/test/test_rlist.py b/rpython/rtyper/test/test_rlist.py --- a/rpython/rtyper/test/test_rlist.py +++ b/rpython/rtyper/test/test_rlist.py @@ -1545,7 +1545,7 @@ raise KeyError # expected case return prev(l, index) rlist.ll_getitem_foldable_nonneg = seen_ok - e = raises(LLException, self.interpret, dummyfn, []) + e = py.test.raises(LLException, self.interpret, dummyfn, []) assert 'KeyError' in str(e.value) finally: rlist.ll_getitem_foldable_nonneg = prev @@ -1571,7 +1571,7 @@ raise KeyError # expected case return prev(l, index) rlist.ll_getitem_foldable_nonneg = seen_ok - e = raises(LLException, self.interpret, dummyfn, []) + e = py.test.raises(LLException, self.interpret, dummyfn, []) assert 'KeyError' in str(e.value) finally: rlist.ll_getitem_foldable_nonneg = prev diff --git a/rpython/rtyper/test/test_rstr.py b/rpython/rtyper/test/test_rstr.py --- a/rpython/rtyper/test/test_rstr.py +++ b/rpython/rtyper/test/test_rstr.py @@ -357,11 +357,11 @@ def f(): s = const('abc') s.find(s, 0, -10) - raises(TyperError, self.interpret, f, ()) + py.test.raises(TyperError, self.interpret, f, ()) def f(): s = const('abc') s.find(s, -10) - raises(TyperError, self.interpret, f, ()) + py.test.raises(TyperError, self.interpret, f, ()) def test_find_empty_string(self): const = self.const @@ -752,11 +752,11 @@ def fn(): s = const('abbccc') s = s.replace(const('a'), const('baz')) - raises(TyperError, self.interpret, fn, ()) + py.test.raises(TyperError, self.interpret, fn, ()) def fn(): s = const('abbccc') s = s.replace(const('abb'), const('c')) - raises(TyperError, self.interpret, fn, ()) + py.test.raises(TyperError, self.interpret, fn, ()) def test_int(self): const = self.const @@ -888,11 +888,11 @@ def f(): s = const('abc') s.count(s, 0, -10) - raises(TyperError, self.interpret, f, ()) + py.test.raises(TyperError, self.interpret, f, ()) def f(): s = const('abc') s.count(s, -10) - raises(TyperError, self.interpret, f, ()) + py.test.raises(TyperError, self.interpret, f, ()) def test_getitem_exc(self): const = self.const diff --git a/rpython/translator/c/test/test_math.py b/rpython/translator/c/test/test_math.py --- a/rpython/translator/c/test/test_math.py +++ b/rpython/translator/c/test/test_math.py @@ -1,6 +1,6 @@ import py, math from rpython.rtyper.lltypesystem.module.test.test_ll_math import (MathTests, - getTester) + get_tester) from rpython.translator.c.test.test_standalone import StandaloneTests from rpython.rlib import rfloat From noreply at buildbot.pypy.org Tue Jan 15 20:49:20 2013 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 15 Jan 2013 20:49:20 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: start fighting with the GC by ripping out some unused code Message-ID: <20130115194920.A523F1C1353@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60099:2614825692d3 Date: 2013-01-15 21:49 +0200 http://bitbucket.org/pypy/pypy/changeset/2614825692d3/ Log: start fighting with the GC by ripping out some unused code diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py --- a/pypy/jit/backend/llsupport/gc.py +++ b/pypy/jit/backend/llsupport/gc.py @@ -117,13 +117,9 @@ def getframedescrs(self, cpu): descrs = JitFrameDescrs() descrs.arraydescr = cpu.arraydescrof(jitframe.JITFRAME) - descrs.jf_descr = cpu.fielddescrof(jitframe.JITFRAME, 'jf_descr') - descrs.jf_guard_exc = cpu.fielddescrof(jitframe.JITFRAME, - 'jf_guard_exc') - descrs.jf_force_descr = cpu.fielddescrof(jitframe.JITFRAME, - 'jf_force_descr') - descrs.jf_frame_info = cpu.fielddescrof(jitframe.JITFRAME, - 'jf_frame_info') + for name in ['jf_descr', 'jf_guard_exc', 'jf_force_descr', + 'jf_frame_info', 'jf_gcpattern']: + setattr(descrs, name, cpu.fielddescrof(jitframe.JITFRAME, name)) descrs.jfi_frame_depth = cpu.fielddescrof(jitframe.JITFRAMEINFO, 'jfi_frame_depth') return descrs diff --git a/pypy/jit/backend/llsupport/jitframe.py b/pypy/jit/backend/llsupport/jitframe.py --- a/pypy/jit/backend/llsupport/jitframe.py +++ b/pypy/jit/backend/llsupport/jitframe.py @@ -26,6 +26,9 @@ ('jf_descr', llmemory.GCREF), # guard_not_forced descr ('jf_force_descr', llmemory.GCREF), + # a bitmask of where are GCREFS in the top of the frame (saved registers) + # used for calls and failures + ('jf_gcpattern', lltype.Signed), # For the front-end: a GCREF for the savedata ('jf_savedata', llmemory.GCREF), # For GUARD_(NO)_EXCEPTION and GUARD_NOT_FORCED: the exception we diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -184,6 +184,7 @@ self._stack_check_failure = mc.materialize(self.cpu.asmmemmgr, []) def _build_malloc_slowpath(self): + xxx # With asmgcc, we need two helpers, so that we can write two CALL # instructions in assembler, with a mark_gc_roots in between. # With shadowstack, this is not needed, so we produce a single helper. @@ -1096,10 +1097,10 @@ self.implement_guard(guard_token, checkfalsecond) return genop_cmp_guard_float - def _emit_call(self, force_index, x, arglocs, start=0, tmp=eax, + def _emit_call(self, x, arglocs, start=0, tmp=eax, argtypes=None, callconv=FFI_DEFAULT_ABI): if IS_X86_64: - return self._emit_call_64(force_index, x, arglocs, start, argtypes) + return self._emit_call_64(x, arglocs, start, argtypes) XXX p = 0 n = len(arglocs) @@ -1124,7 +1125,6 @@ p += loc.get_width() # x is a location self.mc.CALL(x) - self.mark_gc_roots(force_index) # if callconv != FFI_DEFAULT_ABI: self._fix_stdcall(callconv, p) @@ -1138,7 +1138,7 @@ # the called function just added 'p' to ESP, by subtracting it again. self.mc.SUB_ri(esp.value, p) - def _emit_call_64(self, force_index, x, arglocs, start, argtypes): + def _emit_call_64(self, x, arglocs, start, argtypes): src_locs = [] dst_locs = [] xmm_src_locs = [] @@ -1211,30 +1211,11 @@ self.mc.CALL(x) if align: self.mc.ADD_ri(esp.value, align * WORD) - self.mark_gc_roots(force_index) def call(self, addr, args, res): - force_index = self.write_new_force_index() - self._emit_call(force_index, imm(addr), args) + self._emit_call(imm(addr), args) assert res is eax - def write_new_force_index(self): - # for shadowstack only: get a new, unused force_index number and - # write it to FORCE_INDEX_OFS. Used to record the call shape - # (i.e. where the GC pointers are in the stack) around a CALL - # instruction that doesn't already have a force_index. - gcrootmap = self.cpu.gc_ll_descr.gcrootmap - if gcrootmap and gcrootmap.is_shadow_stack: - xxx - clt = self.current_clt - force_index = clt.reserve_and_record_some_faildescr_index() - self.mc.MOV_bi(FORCE_INDEX_OFS, force_index) - return force_index - else: - # the return value is ignored, apart from the fact that it - # is not negative. - return 0 - genop_int_neg = _unaryop("NEG") genop_int_invert = _unaryop("NOT") genop_int_add = _binaryop_or_lea("ADD", True) @@ -1844,14 +1825,6 @@ target = self.failure_recovery_code[exc + 2 * withfloats] fail_descr = cast_instance_to_gcref(guardtok.faildescr) fail_descr = rffi.cast(lltype.Signed, fail_descr) - if WORD == 4: - mc.PUSH(imm(fail_descr)) - mc.JMP(imm(target)) - else: - mc.MOV_ri64(X86_64_SCRATCH_REG.value, target) - mc.PUSH(imm(fail_descr)) - mc.JMP_r(X86_64_SCRATCH_REG.value) - # write tight data that describes the failure recovery positions = [0] * len(guardtok.fail_locs) gcpattern = 0 for i, loc in enumerate(guardtok.fail_locs): @@ -1870,8 +1843,15 @@ positions[i] = v * WORD # write down the positions of locs guardtok.faildescr.rd_locs = positions - # write down the GC pattern - guardtok.faildescr.rd_gcpattern = gcpattern + if WORD == 4: + mc.PUSH(imm(fail_descr)) + mc.PUSH(imm(gcpattern)) + mc.JMP(imm(target)) + else: + mc.MOV_ri64(X86_64_SCRATCH_REG.value, target) + mc.PUSH(imm(fail_descr)) + mc.PUSH(imm(gcpattern)) + mc.JMP_r(X86_64_SCRATCH_REG.value) return startpos def rebuild_faillocs_from_descr(self, descr, inputargs): @@ -1938,9 +1918,13 @@ # throws away most of the frame, including all the PUSHes that we # did just above. ofs = self.cpu.get_ofs_of_frame_field('jf_descr') + ofs2 = self.cpu.get_ofs_of_frame_field('jf_gcpattern') base_ofs = self.cpu.get_baseofs_of_frame_field() mc.POP(eax) + mc.MOV_br(ofs2, eax.value) + mc.POP(eax) mc.MOV_br(ofs, eax.value) + # store the gc pattern mc.LEA_rb(eax.value, -base_ofs) self._call_footer() @@ -2376,23 +2360,6 @@ not_implemented("not implemented operation (guard): %s" % op.getopname()) - def mark_gc_roots(self, force_index, use_copy_area=False): - if force_index < 0: - return # not needed - gcrootmap = self.cpu.gc_ll_descr.gcrootmap - if gcrootmap: - mark = self._regalloc.get_mark_gc_roots(gcrootmap, use_copy_area) - if gcrootmap.is_shadow_stack: - gcrootmap.write_callshape(mark, force_index) - else: - if self.gcrootmap_retaddr_forced == 0: - self.mc.insert_gcroot_marker(mark) # common case - else: - assert self.gcrootmap_retaddr_forced != -1, ( - "two mark_gc_roots() in a CALL_RELEASE_GIL") - gcrootmap.put(self.gcrootmap_retaddr_forced, mark) - self.gcrootmap_retaddr_forced = -1 - def closing_jump(self, target_token): # The backend's logic assumes that the target code is in a piece of # assembler that was also called with the same number of arguments, @@ -2434,7 +2401,6 @@ # there are two helpers to call only with asmgcc slowpath_addr1 = self.malloc_slowpath1 self.mc.CALL(imm(slowpath_addr1)) - self.mark_gc_roots(self.write_new_force_index(), use_copy_area=True) slowpath_addr2 = self.malloc_slowpath2 self.mc.CALL(imm(slowpath_addr2)) diff --git a/pypy/jit/backend/x86/test/test_gc_integration.py b/pypy/jit/backend/x86/test/test_gc_integration.py --- a/pypy/jit/backend/x86/test/test_gc_integration.py +++ b/pypy/jit/backend/x86/test/test_gc_integration.py @@ -42,41 +42,6 @@ class MockGcDescr(GcLLDescr_boehm): gcrootmap = MockGcRootMap() -class TestRegallocDirectGcIntegration(object): - - def test_mark_gc_roots(self): - cpu = CPU(None, None) - cpu.setup_once() - regalloc = RegAlloc(MockAssembler(cpu, MockGcDescr(None, None, None))) - regalloc.assembler.datablockwrapper = 'fakedatablockwrapper' - boxes = [BoxPtr() for i in range(len(X86RegisterManager.all_regs))] - longevity = {} - for box in boxes: - longevity[box] = (0, 1) - regalloc.fm = X86FrameManager() - regalloc.rm = X86RegisterManager(longevity, regalloc.fm, - assembler=regalloc.assembler) - regalloc.xrm = X86XMMRegisterManager(longevity, regalloc.fm, - assembler=regalloc.assembler) - cpu = regalloc.assembler.cpu - for box in boxes: - regalloc.rm.try_allocate_reg(box) - TP = lltype.FuncType([], lltype.Signed) - calldescr = cpu.calldescrof(TP, TP.ARGS, TP.RESULT, - EffectInfo.MOST_GENERAL) - regalloc.rm._check_invariants() - box = boxes[0] - regalloc.position = 0 - regalloc.consider_call(ResOperation(rop.CALL, [box], BoxInt(), - calldescr)) - assert len(regalloc.assembler.movs) == 3 - # - mark = regalloc.get_mark_gc_roots(cpu.gc_ll_descr.gcrootmap) - assert mark[0] == 'compressed' - base = -WORD * FRAME_FIXED_SIZE - expected = ['ebx', 'esi', 'edi', base, base-WORD, base-WORD*2] - assert dict.fromkeys(mark[1:]) == dict.fromkeys(expected) - class TestRegallocGcIntegration(BaseTestRegalloc): cpu = CPU(None, None) From noreply at buildbot.pypy.org Tue Jan 15 21:01:38 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Tue, 15 Jan 2013 21:01:38 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed more tests Message-ID: <20130115200138.7B9331C1353@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60100:d76c4740cba4 Date: 2013-01-15 21:01 +0100 http://bitbucket.org/pypy/pypy/changeset/d76c4740cba4/ Log: Fixed more tests diff --git a/rpython/jit/backend/x86/test/test_zmath.py b/rpython/jit/backend/x86/test/test_zmath.py --- a/rpython/jit/backend/x86/test/test_zmath.py +++ b/rpython/jit/backend/x86/test/test_zmath.py @@ -8,7 +8,7 @@ from rpython.rlib.unroll import unrolling_iterable from rpython.rlib.debug import debug_print from rpython.rtyper.lltypesystem.module.test.test_ll_math import (MathTests, - getTester) + get_tester) def get_test_case((fnname, args, expected)): try: diff --git a/rpython/jit/codewriter/test/test_jtransform.py b/rpython/jit/codewriter/test/test_jtransform.py --- a/rpython/jit/codewriter/test/test_jtransform.py +++ b/rpython/jit/codewriter/test/test_jtransform.py @@ -1249,7 +1249,7 @@ op = SpaceOperation('getfield', [v_x, Constant('x', lltype.Void)], varoftype(lltype.Signed)) tr = Transformer(None, None) - raises(NotImplementedError, tr.rewrite_operation, op) + py.test.raises(NotImplementedError, tr.rewrite_operation, op) def test_cast_opaque_ptr(): S = lltype.GcStruct("S", ("x", lltype.Signed)) diff --git a/rpython/rtyper/lltypesystem/test/test_ll2ctypes.py b/rpython/rtyper/lltypesystem/test/test_ll2ctypes.py --- a/rpython/rtyper/lltypesystem/test/test_ll2ctypes.py +++ b/rpython/rtyper/lltypesystem/test/test_ll2ctypes.py @@ -1017,6 +1017,7 @@ assert ftest == [-5, -5, -5] def test_c_callback_with_void_arg_3(self): + from rpython.rtyper.lltypesystem.rstr import LLHelpers def f(i): x = 'X' * i return x[-2] @@ -1028,8 +1029,8 @@ graph = a.translator.graphs[0] op = graph.startblock.operations[-1] assert op.opname == 'direct_call' - assert op.args[0].value._obj._callable == rpython.rtyper.lltypesystem.rstr.LLHelpers.ll_stritem.im_func - assert op.args[1].value == rpython.rtyper.lltypesystem.rstr.LLHelpers + assert op.args[0].value._obj._callable == LLHelpers.ll_stritem.im_func + assert op.args[1].value == LLHelpers assert op.args[3].value == -2 def test_recursive_struct_more(self): From noreply at buildbot.pypy.org Tue Jan 15 21:36:18 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Tue, 15 Jan 2013 21:36:18 +0100 (CET) Subject: [pypy-commit] pypy py3k: fix sorted's default now that sort is stricter Message-ID: <20130115203618.E86651C1359@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60101:4cb5a3adc59e Date: 2013-01-15 12:27 -0800 http://bitbucket.org/pypy/pypy/changeset/4cb5a3adc59e/ Log: fix sorted's default now that sort is stricter diff --git a/pypy/module/__builtin__/app_functional.py b/pypy/module/__builtin__/app_functional.py --- a/pypy/module/__builtin__/app_functional.py +++ b/pypy/module/__builtin__/app_functional.py @@ -5,7 +5,7 @@ # ____________________________________________________________ -def sorted(lst, key=None, reverse=None): +def sorted(lst, key=None, reverse=False): "sorted(iterable, key=None, reverse=False) --> new sorted list" sorted_lst = list(lst) sorted_lst.sort(key=key, reverse=reverse) From noreply at buildbot.pypy.org Tue Jan 15 21:36:20 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Tue, 15 Jan 2013 21:36:20 +0100 (CET) Subject: [pypy-commit] pypy py3k: workaround our 3.4 length_hint behavior Message-ID: <20130115203620.4DF581C1367@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60102:c7d2ff404236 Date: 2013-01-15 12:28 -0800 http://bitbucket.org/pypy/pypy/changeset/c7d2ff404236/ Log: workaround our 3.4 length_hint behavior diff --git a/lib-python/3.2/test/test_itertools.py b/lib-python/3.2/test/test_itertools.py --- a/lib-python/3.2/test/test_itertools.py +++ b/lib-python/3.2/test/test_itertools.py @@ -1398,7 +1398,11 @@ def test_repeat(self): from test.test_iterlen import len self.assertEqual(len(repeat(None, 50)), 50) - self.assertRaises(TypeError, len, repeat(None)) + if support.check_impl_detail(pypy=True): + # 3.4 behavior + self.assertEqual(len(repeat(None)), NotImplemented) + else: + self.assertRaises(TypeError, len, repeat(None)) class RegressionTests(unittest.TestCase): From noreply at buildbot.pypy.org Tue Jan 15 21:36:21 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Tue, 15 Jan 2013 21:36:21 +0100 (CET) Subject: [pypy-commit] pypy py3k: itertools.accumulate Message-ID: <20130115203621.881A31C1368@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60103:b6a97a30f8f4 Date: 2013-01-15 12:30 -0800 http://bitbucket.org/pypy/pypy/changeset/b6a97a30f8f4/ Log: itertools.accumulate diff --git a/pypy/module/itertools/__init__.py b/pypy/module/itertools/__init__.py --- a/pypy/module/itertools/__init__.py +++ b/pypy/module/itertools/__init__.py @@ -23,6 +23,7 @@ """ interpleveldefs = { + 'accumulate' : 'interp_itertools.W_Accumulate', 'chain' : 'interp_itertools.W_Chain', 'combinations' : 'interp_itertools.W_Combinations', 'combinations_with_replacement' : 'interp_itertools.W_CombinationsWithReplacement', diff --git a/pypy/module/itertools/interp_itertools.py b/pypy/module/itertools/interp_itertools.py --- a/pypy/module/itertools/interp_itertools.py +++ b/pypy/module/itertools/interp_itertools.py @@ -1185,3 +1185,38 @@ permutations(range(3), 2) --> (0,1), (0,2), (1,0), (1,2), (2,0), (2,1)""", ) + + +class W_Accumulate(Wrappable): + + def __init__(self, space, w_iterable): + self.space = space + self.w_iterable = w_iterable + self.w_total = None + + def iter_w(self): + return self.space.wrap(self) + + def next_w(self): + w_value = self.space.next(self.w_iterable) + if self.w_total is None: + self.w_total = w_value + return w_value + + self.w_total = self.space.add(self.w_total, w_value) + return self.w_total + +def W_Accumulate__new__(space, w_subtype, w_iterable): + r = space.allocate_instance(W_Accumulate, w_subtype) + r.__init__(space, space.iter(w_iterable)) + return space.wrap(r) + +W_Accumulate.typedef = TypeDef("accumulate", + __module__ = 'itertools', + __new__ = interp2app(W_Accumulate__new__), + __iter__ = interp2app(W_Accumulate.iter_w), + __next__ = interp2app(W_Accumulate.next_w), + __doc__ = """\ +"accumulate(iterable) --> accumulate object + +Return series of accumulated sums.""") diff --git a/pypy/module/itertools/test/test_itertools.py b/pypy/module/itertools/test/test_itertools.py --- a/pypy/module/itertools/test/test_itertools.py +++ b/pypy/module/itertools/test/test_itertools.py @@ -918,3 +918,18 @@ assert list(itertools.islice(c2, 3)) == expected c3 = pickle.loads(pickle.dumps(c)) assert list(itertools.islice(c3, 3)) == expected + +class AppTestItertools32: + spaceconfig = dict(usemodules=['itertools']) + + def setup_class(cls): + if cls.space.is_true(cls.space.appexec([], """(): + import sys; return sys.version_info < (3, 2) + """)): + py.test.skip("Requires Python 3.2") + + def test_accumulate(self): + from itertools import accumulate + expected = [0, 1, 3, 6, 10, 15, 21, 28, 36, 45] + assert list(accumulate(range(10))) == expected + assert list(accumulate(iterable=range(10))) == expected From noreply at buildbot.pypy.org Tue Jan 15 21:48:13 2013 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 15 Jan 2013 21:48:13 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: cleanup a little Message-ID: <20130115204813.4AB3F1C0131@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60104:5a80de633257 Date: 2013-01-15 22:47 +0200 http://bitbucket.org/pypy/pypy/changeset/5a80de633257/ Log: cleanup a little diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -1959,10 +1959,6 @@ self.pending_guard_tokens.append(guard_token) def genop_call(self, op, arglocs, resloc): - force_index = self.write_new_force_index() - self._genop_call(op, arglocs, resloc, force_index) - - def _genop_call(self, op, arglocs, resloc, force_index): from pypy.jit.backend.llsupport.descr import CallDescr sizeloc = arglocs[0] @@ -1979,7 +1975,7 @@ descr = op.getdescr() assert isinstance(descr, CallDescr) - self._emit_call(force_index, x, arglocs, 3, tmp=tmp, + self._emit_call(x, arglocs, 3, tmp=tmp, argtypes=descr.get_arg_types(), callconv=descr.get_call_conv()) @@ -2033,7 +2029,7 @@ def genop_guard_call_may_force(self, op, guard_op, guard_token, arglocs, result_loc): self._store_force_index(guard_op) - self._genop_call(op, arglocs, result_loc, -1) + self.genop_call(op, arglocs, result_loc) self._emit_guard_not_forced(guard_token) def genop_guard_call_release_gil(self, op, guard_op, guard_token, @@ -2044,8 +2040,8 @@ xxx self.call_release_gil(gcrootmap, arglocs) # do the call - fail_index = self._store_force_index(guard_op) - self._genop_call(op, arglocs, result_loc, fail_index) + self._store_force_index(guard_op) + self.genop_call(op, arglocs, result_loc) # then reopen the stack if gcrootmap: xxx @@ -2115,7 +2111,7 @@ self.mc.LEA_rb(reg.value, css) args = [reg] # - self._emit_call(-1, imm(self.releasegil_addr), args) + self._emit_call(imm(self.releasegil_addr), args) # Finally, restore the registers saved above. if IS_X86_32: p = WORD @@ -2148,7 +2144,7 @@ reg = edi self.mc.LEA_rb(reg.value, css) args = [reg] - self._emit_call(-1, imm(self.reacqgil_addr), args) + self._emit_call(imm(self.reacqgil_addr), args) # restore the result from the stack if isinstance(save_loc, RegLoc) and not save_loc.is_xmm: self.mc.MOV_rs(save_loc.value, WORD) @@ -2156,7 +2152,7 @@ def genop_guard_call_assembler(self, op, guard_op, guard_token, arglocs, result_loc): - fail_index = self._store_force_index(guard_op) + self._store_force_index(guard_op) descr = op.getdescr() assert isinstance(descr, JitCellToken) [frame_loc, argloc] = arglocs @@ -2164,7 +2160,7 @@ # Write a call to the target assembler # we need to allocate the frame, keep in sync with runner's # execute_token - self._emit_call(fail_index, imm(descr._x86_function_addr), + self._emit_call(imm(descr._x86_function_addr), [argloc], 0, tmp=eax) if op.result is None: assert result_loc is None @@ -2194,7 +2190,7 @@ jd = descr.outermost_jitdriver_sd assert jd is not None asm_helper_adr = self.cpu.cast_adr_to_int(jd.assembler_helper_adr) - self._emit_call(fail_index, imm(asm_helper_adr), + self._emit_call(imm(asm_helper_adr), [eax, imm0], 0, tmp=ecx) if IS_X86_32 and isinstance(result_loc, StackLoc) and result_loc.type == FLOAT: self.mc.FSTPL_b(result_loc.value) diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py --- a/pypy/jit/backend/x86/regalloc.py +++ b/pypy/jit/backend/x86/regalloc.py @@ -1230,7 +1230,7 @@ # call memcpy() self.rm.before_call() self.xrm.before_call() - self.assembler._emit_call(-1, imm(self.assembler.memcpy_addr), + self.assembler._emit_call(imm(self.assembler.memcpy_addr), [dstaddr_loc, srcaddr_loc, length_loc]) self.rm.possibly_free_var(length_box) self.rm.possibly_free_var(dstaddr_box) From noreply at buildbot.pypy.org Tue Jan 15 23:55:21 2013 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 15 Jan 2013 23:55:21 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: die in fire!!!! Message-ID: <20130115225521.D3FB81C0131@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60105:0722c7563762 Date: 2013-01-16 00:54 +0200 http://bitbucket.org/pypy/pypy/changeset/0722c7563762/ Log: die in fire!!!! diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py --- a/pypy/jit/backend/llsupport/gc.py +++ b/pypy/jit/backend/llsupport/gc.py @@ -228,196 +228,6 @@ # All code below is for the hybrid or minimark GC -class GcRootMap_asmgcc(object): - """Handles locating the stack roots in the assembler. - This is the class supporting --gcrootfinder=asmgcc. - """ - is_shadow_stack = False - is_64_bit = (WORD == 8) - - LOC_REG = 0 - LOC_ESP_PLUS = 1 - LOC_EBP_PLUS = 2 - LOC_EBP_MINUS = 3 - - GCMAP_ARRAY = rffi.CArray(lltype.Signed) - CALLSHAPE_ARRAY_PTR = rffi.CArrayPtr(rffi.UCHAR) - - def __init__(self, gcdescr=None): - # '_gcmap' is an array of length '_gcmap_maxlength' of addresses. - # '_gcmap_curlength' tells how full the array really is. - # The addresses are actually grouped in pairs: - # (addr-after-the-CALL-in-assembler, addr-of-the-call-shape). - # '_gcmap_deadentries' counts pairs marked dead (2nd item is NULL). - # '_gcmap_sorted' is True only if we know the array is sorted. - self._gcmap = lltype.nullptr(self.GCMAP_ARRAY) - self._gcmap_curlength = 0 - self._gcmap_maxlength = 0 - self._gcmap_deadentries = 0 - self._gcmap_sorted = True - - def add_jit2gc_hooks(self, jit2gc): - jit2gc.update({ - 'gcmapstart': lambda: self.gcmapstart(), - 'gcmapend': lambda: self.gcmapend(), - 'gcmarksorted': lambda: self.gcmarksorted(), - }) - - def initialize(self): - # hack hack hack. Remove these lines and see MissingRTypeAttribute - # when the rtyper tries to annotate these methods only when GC-ing... - self.gcmapstart() - self.gcmapend() - self.gcmarksorted() - - def gcmapstart(self): - return rffi.cast(llmemory.Address, self._gcmap) - - def gcmapend(self): - addr = self.gcmapstart() - if self._gcmap_curlength: - addr += rffi.sizeof(lltype.Signed) * self._gcmap_curlength - if not we_are_translated() and type(addr) is long: - from pypy.rpython.lltypesystem import ll2ctypes - addr = ll2ctypes._lladdress(addr) # XXX workaround - return addr - - def gcmarksorted(self): - # Called by the GC when it is about to sort [gcmapstart():gcmapend()]. - # Returns the previous sortedness flag -- i.e. returns True if it - # is already sorted, False if sorting is needed. - sorted = self._gcmap_sorted - self._gcmap_sorted = True - return sorted - - def put(self, retaddr, callshapeaddr): - """'retaddr' is the address just after the CALL. - 'callshapeaddr' is the address of the raw 'shape' marker. - Both addresses are actually integers here.""" - index = self._gcmap_curlength - if index + 2 > self._gcmap_maxlength: - index = self._enlarge_gcmap() - self._gcmap[index] = retaddr - self._gcmap[index+1] = callshapeaddr - self._gcmap_curlength = index + 2 - self._gcmap_sorted = False - - @rgc.no_collect - def _enlarge_gcmap(self): - oldgcmap = self._gcmap - if self._gcmap_deadentries * 3 * 2 > self._gcmap_maxlength: - # More than 1/3rd of the entries are dead. Don't actually - # enlarge the gcmap table, but just clean up the dead entries. - newgcmap = oldgcmap - else: - # Normal path: enlarge the array. - newlength = 250 + (self._gcmap_maxlength // 3) * 4 - newgcmap = lltype.malloc(self.GCMAP_ARRAY, newlength, flavor='raw', - track_allocation=False) - self._gcmap_maxlength = newlength - # - j = 0 - i = 0 - end = self._gcmap_curlength - while i < end: - if oldgcmap[i + 1]: - newgcmap[j] = oldgcmap[i] - newgcmap[j + 1] = oldgcmap[i + 1] - j += 2 - i += 2 - self._gcmap_curlength = j - self._gcmap_deadentries = 0 - if oldgcmap != newgcmap: - self._gcmap = newgcmap - if oldgcmap: - lltype.free(oldgcmap, flavor='raw', track_allocation=False) - return j - - @rgc.no_collect - def freeing_block(self, start, stop): - # if [start:stop] is a raw block of assembler, then look up the - # corresponding gcroot markers, and mark them as freed now in - # self._gcmap by setting the 2nd address of every entry to NULL. - gcmapstart = self.gcmapstart() - gcmapend = self.gcmapend() - if gcmapstart == gcmapend: - return - if not self.gcmarksorted(): - asmgcroot.sort_gcmap(gcmapstart, gcmapend) - # A note about gcmarksorted(): the deletion we do here keeps the - # array sorted. This avoids needing too many sort_gcmap()s. - # Indeed, freeing_block() is typically called many times in a row, - # so it will call sort_gcmap() at most the first time. - startaddr = rffi.cast(llmemory.Address, start) - stopaddr = rffi.cast(llmemory.Address, stop) - item = asmgcroot.binary_search(gcmapstart, gcmapend, startaddr) - # 'item' points to one of the entries. Because the whole array - # is sorted, we know that it points either to the first entry we - # want to kill, or to the previous entry. - if item.address[0] < startaddr: - item += asmgcroot.arrayitemsize # go forward one entry - assert item == gcmapend or item.address[0] >= startaddr - while item != gcmapend and item.address[0] < stopaddr: - item.address[1] = llmemory.NULL - self._gcmap_deadentries += 1 - item += asmgcroot.arrayitemsize - - def get_basic_shape(self): - # XXX: Should this code even really know about stack frame layout of - # the JIT? - if self.is_64_bit: - return [chr(self.LOC_EBP_PLUS | 4), # return addr: at 8(%rbp) - chr(self.LOC_EBP_MINUS | 4), # saved %rbx: at -8(%rbp) - chr(self.LOC_EBP_MINUS | 8), # saved %r12: at -16(%rbp) - chr(self.LOC_EBP_MINUS | 12), # saved %r13: at -24(%rbp) - chr(self.LOC_EBP_MINUS | 16), # saved %r14: at -32(%rbp) - chr(self.LOC_EBP_MINUS | 20), # saved %r15: at -40(%rbp) - chr(self.LOC_EBP_PLUS | 0), # saved %rbp: at (%rbp) - chr(0)] - else: - return [chr(self.LOC_EBP_PLUS | 4), # return addr: at 4(%ebp) - chr(self.LOC_EBP_MINUS | 4), # saved %ebx: at -4(%ebp) - chr(self.LOC_EBP_MINUS | 8), # saved %esi: at -8(%ebp) - chr(self.LOC_EBP_MINUS | 12), # saved %edi: at -12(%ebp) - chr(self.LOC_EBP_PLUS | 0), # saved %ebp: at (%ebp) - chr(0)] - - def _encode_num(self, shape, number): - assert number >= 0 - flag = 0 - while number >= 0x80: - shape.append(chr((number & 0x7F) | flag)) - flag = 0x80 - number >>= 7 - shape.append(chr(number | flag)) - - def add_frame_offset(self, shape, offset): - if self.is_64_bit: - assert (offset & 7) == 0 - offset >>= 1 - else: - assert (offset & 3) == 0 - if offset >= 0: - num = self.LOC_EBP_PLUS | offset - else: - num = self.LOC_EBP_MINUS | (-offset) - self._encode_num(shape, num) - - def add_callee_save_reg(self, shape, reg_index): - assert reg_index > 0 - shape.append(chr(self.LOC_REG | (reg_index << 2))) - - def compress_callshape(self, shape, datablockwrapper): - # Similar to compress_callshape() in trackgcroot.py. - # Returns an address to raw memory (as an integer). - length = len(shape) - rawaddr = datablockwrapper.malloc_aligned(length, 1) - p = rffi.cast(self.CALLSHAPE_ARRAY_PTR, rawaddr) - for i in range(length): - p[length-1-i] = rffi.cast(rffi.UCHAR, shape[i]) - return rawaddr - - class GcRootMap_shadowstack(object): """Handles locating the stack roots in the assembler. This is the class supporting --gcrootfinder=shadowstack. From noreply at buildbot.pypy.org Tue Jan 15 23:56:31 2013 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 15 Jan 2013 23:56:31 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: this guy dies too Message-ID: <20130115225631.38E481C0131@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60106:e0dbf5f5acd5 Date: 2013-01-16 00:56 +0200 http://bitbucket.org/pypy/pypy/changeset/e0dbf5f5acd5/ Log: this guy dies too diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py --- a/pypy/jit/backend/llsupport/gc.py +++ b/pypy/jit/backend/llsupport/gc.py @@ -228,186 +228,6 @@ # All code below is for the hybrid or minimark GC -class GcRootMap_shadowstack(object): - """Handles locating the stack roots in the assembler. - This is the class supporting --gcrootfinder=shadowstack. - """ - is_shadow_stack = True - MARKER_FRAME = 8 # this marker now *follows* the frame addr - - # The "shadowstack" is a portable way in which the GC finds the - # roots that live in the stack. Normally it is just a list of - # pointers to GC objects. The pointers may be moved around by a GC - # collection. But with the JIT, an entry can also be MARKER_FRAME, - # in which case the previous entry points to an assembler stack frame. - # During a residual CALL from the assembler (which may indirectly - # call the GC), we use the force_index stored in the assembler - # stack frame to identify the call: we can go from the force_index - # to a list of where the GC pointers are in the frame (this is the - # purpose of the present class). - # - # Note that across CALL_MAY_FORCE or CALL_ASSEMBLER, we can also go - # from the force_index to a ResumeGuardForcedDescr instance, which - # is used if the virtualizable or the virtualrefs need to be forced - # (see pypy.jit.backend.model). The force_index number in the stack - # frame is initially set to a non-negative value x, but it is - # occasionally turned into (~x) in case of forcing. - - INTARRAYPTR = rffi.CArrayPtr(rffi.INT) - CALLSHAPES_ARRAY = rffi.CArray(INTARRAYPTR) - - def __init__(self, gcdescr): - self._callshapes = lltype.nullptr(self.CALLSHAPES_ARRAY) - self._callshapes_maxlength = 0 - self.force_index_ofs = gcdescr.force_index_ofs - - def add_jit2gc_hooks(self, jit2gc): - # - # --------------- - # This is used to enumerate the shadowstack in the presence - # of the JIT. It is also used by the stacklet support in - # rlib/_stacklet_shadowstack. That's why it is written as - # an iterator that can also be used with a custom_trace. - # - class RootIterator: - _alloc_flavor_ = "raw" - - def setcontext(iself, context): - iself.context = context - - def nextleft(iself, gc, range_lowest, prev): - # Return the next valid GC object's address, in right-to-left - # order from the shadowstack array. This usually means just - # returning "prev - sizeofaddr", until we reach "range_lowest", - # except that we are skipping NULLs. If "prev - sizeofaddr" - # contains a MARKER_FRAME instead, then we go into - # JIT-frame-lookup mode. - # - while True: - # - # If we are not iterating right now in a JIT frame - if iself.frame_addr == 0: - # - # Look for the next shadowstack address that - # contains a valid pointer - while prev != range_lowest: - prev -= llmemory.sizeof(llmemory.Address) - if prev.signed[0] == self.MARKER_FRAME: - break - if gc.points_to_valid_gc_object(prev): - return prev - else: - return llmemory.NULL # done - # - # It's a JIT frame. Save away 'prev' for later, and - # go into JIT-frame-exploring mode. - prev -= llmemory.sizeof(llmemory.Address) - frame_addr = prev.signed[0] - iself.saved_prev = prev - iself.frame_addr = frame_addr - addr = llmemory.cast_int_to_adr(frame_addr + - self.force_index_ofs) - addr = iself.translateptr(iself.context, addr) - force_index = addr.signed[0] - if force_index < 0: - force_index = ~force_index - # NB: the next line reads a still-alive _callshapes, - # because we ensure that just before we called this - # piece of assembler, we put on the (same) stack a - # pointer to a loop_token that keeps the force_index - # alive. - callshape = self._callshapes[force_index] - else: - # Continuing to explore this JIT frame - callshape = iself.callshape - # - # 'callshape' points to the next INT of the callshape. - # If it's zero we are done with the JIT frame. - while rffi.cast(lltype.Signed, callshape[0]) != 0: - # - # Non-zero: it's an offset inside the JIT frame. - # Read it and increment 'callshape'. - offset = rffi.cast(lltype.Signed, callshape[0]) - callshape = lltype.direct_ptradd(callshape, 1) - addr = llmemory.cast_int_to_adr(iself.frame_addr + - offset) - addr = iself.translateptr(iself.context, addr) - if gc.points_to_valid_gc_object(addr): - # - # The JIT frame contains a valid GC pointer at - # this address (as opposed to NULL). Save - # 'callshape' for the next call, and return the - # address. - iself.callshape = callshape - return addr - # - # Restore 'prev' and loop back to the start. - iself.frame_addr = 0 - prev = iself.saved_prev - - # --------------- - # - root_iterator = RootIterator() - root_iterator.frame_addr = 0 - root_iterator.context = llmemory.NULL - root_iterator.translateptr = lambda context, addr: addr - jit2gc.update({ - 'root_iterator': root_iterator, - }) - - def initialize(self): - pass - - def get_basic_shape(self): - return [] - - def add_frame_offset(self, shape, offset): - assert offset != 0 - shape.append(offset) - - def add_callee_save_reg(self, shape, register): - msg = "GC pointer in %s was not spilled" % register - os.write(2, '[llsupport/gc] %s\n' % msg) - raise AssertionError(msg) - - def compress_callshape(self, shape, datablockwrapper): - length = len(shape) - SZINT = rffi.sizeof(rffi.INT) - rawaddr = datablockwrapper.malloc_aligned((length + 1) * SZINT, SZINT) - p = rffi.cast(self.INTARRAYPTR, rawaddr) - for i in range(length): - p[i] = rffi.cast(rffi.INT, shape[i]) - p[length] = rffi.cast(rffi.INT, 0) - return p - - def write_callshape(self, p, force_index): - if force_index >= self._callshapes_maxlength: - self._enlarge_callshape_list(force_index + 1) - self._callshapes[force_index] = p - - def _enlarge_callshape_list(self, minsize): - newlength = 250 + (self._callshapes_maxlength // 3) * 4 - if newlength < minsize: - newlength = minsize - newarray = lltype.malloc(self.CALLSHAPES_ARRAY, newlength, - flavor='raw', track_allocation=False) - if self._callshapes: - i = self._callshapes_maxlength - 1 - while i >= 0: - newarray[i] = self._callshapes[i] - i -= 1 - lltype.free(self._callshapes, flavor='raw', track_allocation=False) - self._callshapes = newarray - self._callshapes_maxlength = newlength - - def freeing_block(self, start, stop): - pass # nothing needed here - - def get_root_stack_top_addr(self): - rst_addr = llop.gc_adr_of_root_stack_top(llmemory.Address) - return rffi.cast(lltype.Signed, rst_addr) - - class WriteBarrierDescr(AbstractDescr): def __init__(self, gc_ll_descr): self.llop1 = gc_ll_descr.llop1 From noreply at buildbot.pypy.org Wed Jan 16 00:45:45 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Wed, 16 Jan 2013 00:45:45 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Dont assume working path Message-ID: <20130115234545.618471C1353@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60107:ee184342fa3f Date: 2013-01-16 00:45 +0100 http://bitbucket.org/pypy/pypy/changeset/ee184342fa3f/ Log: Dont assume working path diff --git a/pypy/bin/pyinteractive.py b/pypy/bin/pyinteractive.py --- a/pypy/bin/pyinteractive.py +++ b/pypy/bin/pyinteractive.py @@ -9,6 +9,8 @@ import os, sys import time +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) + import pypy from pypy.tool import option from optparse import make_option diff --git a/rpython/translator/c/gcc/trackgcroot.py b/rpython/translator/c/gcc/trackgcroot.py --- a/rpython/translator/c/gcc/trackgcroot.py +++ b/rpython/translator/c/gcc/trackgcroot.py @@ -1,6 +1,8 @@ #! /usr/bin/env python import re, sys, os, random +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', '..', '..')) + from rpython.translator.c.gcc.instruction import Insn, Label, InsnCall, InsnRet from rpython.translator.c.gcc.instruction import InsnFunctionStart, InsnStop from rpython.translator.c.gcc.instruction import InsnSetLocal, InsnCopyLocal From noreply at buildbot.pypy.org Wed Jan 16 18:19:40 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 16 Jan 2013 18:19:40 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: a test and a fix Message-ID: <20130116171940.0DB071C125D@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60108:ffa091b8f472 Date: 2013-01-16 19:19 +0200 http://bitbucket.org/pypy/pypy/changeset/ffa091b8f472/ Log: a test and a fix diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -67,8 +67,7 @@ self.loop_run_counters = [] self.float_const_neg_addr = 0 self.float_const_abs_addr = 0 - self.malloc_slowpath1 = 0 - self.malloc_slowpath2 = 0 + self.malloc_slowpath = 0 self.wb_slowpath = [0, 0, 0, 0] self.memcpy_addr = 0 self.setup_failure_recovery() @@ -1839,7 +1838,7 @@ else: v = gpr_reg_mgr_cls.all_reg_indexes[loc.value] if guardtok.failargs[i].type == REF: - gcpattern |= v + gcpattern |= (1 << v) positions[i] = v * WORD # write down the positions of locs guardtok.faildescr.rd_locs = positions diff --git a/pypy/jit/backend/x86/test/test_gc_integration.py b/pypy/jit/backend/x86/test/test_gc_integration.py --- a/pypy/jit/backend/x86/test/test_gc_integration.py +++ b/pypy/jit/backend/x86/test/test_gc_integration.py @@ -2,26 +2,15 @@ """ Tests for register allocation for common constructs """ -import py -from pypy.jit.metainterp.history import BoxInt, ConstInt,\ - BoxPtr, ConstPtr, TreeLoop, TargetToken -from pypy.jit.metainterp.resoperation import rop, ResOperation -from pypy.jit.codewriter import heaptracker -from pypy.jit.codewriter.effectinfo import EffectInfo -from pypy.jit.backend.llsupport.descr import FieldDescr, FLAG_SIGNED +from pypy.jit.metainterp.history import TargetToken, AbstractDescr from pypy.jit.backend.llsupport.gc import GcLLDescription, GcLLDescr_boehm from pypy.jit.backend.detect_cpu import getcpuclass -from pypy.jit.backend.x86.regalloc import RegAlloc -from pypy.jit.backend.x86.arch import WORD, FRAME_FIXED_SIZE -from pypy.jit.tool.oparser import parse +from pypy.jit.backend.x86.arch import WORD +from pypy.jit.backend.llsupport import jitframe from pypy.rpython.lltypesystem import lltype, llmemory, rffi -from pypy.rpython.annlowlevel import llhelper -from pypy.rpython.lltypesystem import rclass, rstr -from pypy.jit.backend.x86.test.test_regalloc import MockAssembler from pypy.jit.backend.x86.test.test_regalloc import BaseTestRegalloc -from pypy.jit.backend.x86.regalloc import X86RegisterManager, X86FrameManager,\ - X86XMMRegisterManager +from pypy.jit.backend.x86.regalloc import gpr_reg_mgr_cls CPU = getcpuclass() @@ -42,6 +31,10 @@ class MockGcDescr(GcLLDescr_boehm): gcrootmap = MockGcRootMap() +class TestDirectGcIntegration(object): + def test_gcmap(self): + pass + class TestRegallocGcIntegration(BaseTestRegalloc): cpu = CPU(None, None) @@ -60,7 +53,7 @@ struct_ptr.field = child_ptr - descr0 = cpu.fielddescrof(S, 'int') + intdescr = cpu.fielddescrof(S, 'int') ptr0 = struct_ref targettoken = TargetToken() @@ -76,6 +69,23 @@ self.interpret(ops, [self.struct_ptr]) assert not self.getptr(0, lltype.Ptr(self.S)) + def test_guard(self): + ops = ''' + [i0, p0, i1, p1] + p3 = getfield_gc(p0, descr=fielddescr) + guard_true(i0) [p0, i1, p1, p3] + ''' + s1 = lltype.malloc(self.S) + s2 = lltype.malloc(self.S) + s1.field = s2 + self.interpret(ops, [0, s1, 1, s2]) + frame = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, self.deadframe) + # p0 and p3 should be in registers, p1 not so much + assert self.getptr(0, lltype.Ptr(self.S)) == s1 + # this is a fairly CPU specific check + all = len(gpr_reg_mgr_cls.all_regs) + assert frame.jf_gcpattern == (1 << (all - 1)) | (1 << (all - 2)) + def test_rewrite_constptr(self): ops = ''' [] @@ -91,30 +101,30 @@ label(i0, i1, i2, i3, i4, i5, i6, i7, i8, descr=targettoken) guard_value(i2, 1) [i2, i3, i4, i5, i6, i7, i0, i1, i8] guard_class(i4, 138998336) [i4, i5, i6, i7, i0, i1, i8] - i11 = getfield_gc(i4, descr=descr0) + i11 = getfield_gc(i4, descr=intdescr) guard_nonnull(i11) [i4, i5, i6, i7, i0, i1, i11, i8] - i13 = getfield_gc(i11, descr=descr0) + i13 = getfield_gc(i11, descr=intdescr) guard_isnull(i13) [i4, i5, i6, i7, i0, i1, i11, i8] - i15 = getfield_gc(i4, descr=descr0) + i15 = getfield_gc(i4, descr=intdescr) i17 = int_lt(i15, 0) guard_false(i17) [i4, i5, i6, i7, i0, i1, i11, i15, i8] - i18 = getfield_gc(i11, descr=descr0) + i18 = getfield_gc(i11, descr=intdescr) i19 = int_ge(i15, i18) guard_false(i19) [i4, i5, i6, i7, i0, i1, i11, i15, i8] i20 = int_lt(i15, 0) guard_false(i20) [i4, i5, i6, i7, i0, i1, i11, i15, i8] - i21 = getfield_gc(i11, descr=descr0) - i22 = getfield_gc(i11, descr=descr0) + i21 = getfield_gc(i11, descr=intdescr) + i22 = getfield_gc(i11, descr=intdescr) i23 = int_mul(i15, i22) i24 = int_add(i21, i23) - i25 = getfield_gc(i4, descr=descr0) + i25 = getfield_gc(i4, descr=intdescr) i27 = int_add(i25, 1) - setfield_gc(i4, i27, descr=descr0) - i29 = getfield_raw(144839744, descr=descr0) + setfield_gc(i4, i27, descr=intdescr) + i29 = getfield_raw(144839744, descr=intdescr) i31 = int_and(i29, -2141192192) i32 = int_is_true(i31) guard_false(i32) [i4, i6, i7, i0, i1, i24] - i33 = getfield_gc(i0, descr=descr0) + i33 = getfield_gc(i0, descr=intdescr) guard_value(i33, ConstPtr(ptr0)) [i4, i6, i7, i0, i1, i33, i24] jump(i0, i1, 1, 17, i4, ConstPtr(ptr0), i6, i7, i24, descr=targettoken) ''' From noreply at buildbot.pypy.org Wed Jan 16 20:39:53 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Wed, 16 Jan 2013 20:39:53 +0100 (CET) Subject: [pypy-commit] pypy default: emit BOMs w/ utf16/32 empty strings. a py3k incremental encoding test Message-ID: <20130116193953.E5B051C137A@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: Changeset: r60109:1804a92ff657 Date: 2013-01-16 11:35 -0800 http://bitbucket.org/pypy/pypy/changeset/1804a92ff657/ Log: emit BOMs w/ utf16/32 empty strings. a py3k incremental encoding test (test_codecs's test_decoder_state) actually relies on this diff --git a/pypy/rlib/runicode.py b/pypy/rlib/runicode.py --- a/pypy/rlib/runicode.py +++ b/pypy/rlib/runicode.py @@ -460,6 +460,10 @@ errorhandler=None, byteorder='little'): if size == 0: + if byteorder == 'native': + result = StringBuilder(2) + _STORECHAR(result, 0xFEFF, BYTEORDER) + return result.build() return "" result = StringBuilder(size * 2 + 2) @@ -621,6 +625,10 @@ errorhandler=None, byteorder='little'): if size == 0: + if byteorder == 'native': + result = StringBuilder(4) + _STORECHAR32(result, 0xFEFF, BYTEORDER) + return result.build() return "" result = StringBuilder(size * 4 + 4) diff --git a/pypy/rlib/test/test_runicode.py b/pypy/rlib/test/test_runicode.py --- a/pypy/rlib/test/test_runicode.py +++ b/pypy/rlib/test/test_runicode.py @@ -679,6 +679,11 @@ "utf-32 utf-32-be utf-32-le").split(): self.checkencode(uni, encoding) + def test_empty(self): + for encoding in ("utf-8 utf-16 utf-16-be utf-16-le " + "utf-32 utf-32-be utf-32-le").split(): + self.checkencode(u'', encoding) + def test_single_chars_utf8(self): # check every number of bytes per char for s in ["\xd7\x90", "\xd6\x96", "\xeb\x96\x95", "\xf0\x90\x91\x93"]: From noreply at buildbot.pypy.org Thu Jan 17 01:10:24 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Thu, 17 Jan 2013 01:10:24 +0100 (CET) Subject: [pypy-commit] pypy py3k: merge default Message-ID: <20130117001024.D71111C1379@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60110:1272eba02bdc Date: 2013-01-16 15:05 -0800 http://bitbucket.org/pypy/pypy/changeset/1272eba02bdc/ Log: merge default diff too long, truncating to 2000 out of 2068 lines diff --git a/pypy/doc/Makefile b/pypy/doc/Makefile --- a/pypy/doc/Makefile +++ b/pypy/doc/Makefile @@ -32,38 +32,38 @@ -rm -rf $(BUILDDIR)/* html: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." pickle: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ @@ -73,7 +73,7 @@ @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/PyPy.qhc" latex: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @@ -81,26 +81,26 @@ "run these through (pdf)latex." man: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man" changes: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." doctest: - python config/generate.py + # python config/generate.py #readthedocs will not run this Makefile $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -38,7 +38,7 @@ # General information about the project. project = u'PyPy' -copyright = u'2011, The PyPy Project' +copyright = u'2013, The PyPy Project' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/pypy/doc/config/translation.no__thread.txt b/pypy/doc/config/translation.no__thread.txt --- a/pypy/doc/config/translation.no__thread.txt +++ b/pypy/doc/config/translation.no__thread.txt @@ -1,3 +1,3 @@ Don't use gcc __thread attribute for fast thread local storage -implementation . Increases the chance that moving the resulting +implementation. Increases the chance that moving the resulting executable to another same processor Linux machine will work. diff --git a/pypy/doc/pypyconfig.py b/pypy/doc/pypyconfig.py --- a/pypy/doc/pypyconfig.py +++ b/pypy/doc/pypyconfig.py @@ -3,6 +3,10 @@ def setup(app): import sys, os sys.path.insert(0, os.path.abspath("../../")) + + #Autmatically calls make_cmdlline_overview + from pypy.doc.config import generate + from pypy.config import makerestdoc import py role = makerestdoc.register_config_role(py.path.local()) diff --git a/pypy/doc/test/test_whatsnew.py b/pypy/doc/test/test_whatsnew.py --- a/pypy/doc/test/test_whatsnew.py +++ b/pypy/doc/test/test_whatsnew.py @@ -29,7 +29,7 @@ merge() and \ branch(default)) and \ not branch(default)' % (startrev, endrev) - cmd = r"hg log -R '%s' -r '%s' --template '{branches}\n'" % (path, revset) + cmd = r'hg log -R "%s" -r "%s" --template "{branches}\n"' % (path, revset) out = getoutput(cmd) branches = set(map(str.strip, out.splitlines())) return branches diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -15,6 +15,9 @@ .. branch: numpypy-longdouble Long double support for numpypy +.. branch: numpypy-real-as-view +Convert real, imag from ufuncs to views. This involves the beginning of +view() functionality .. branch: signatures Improved RPython typing diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -182,6 +182,11 @@ else: oparg = 0 + # note: the structure of the code here is such that it makes + # (after translation) a big "if/elif" chain, which is then + # turned into a switch(). It starts here: even if the first + # one is not an "if" but a "while" the effect is the same. + while opcode == self.opcodedesc.EXTENDED_ARG.index: opcode = ord(co_code[next_instr]) if opcode < self.HAVE_ARGUMENT: @@ -230,6 +235,8 @@ 'END_FINALLY', 'JUMP_ABSOLUTE'): continue # opcodes implemented above + # the following "if" is part of the big switch described + # above. if opcode == opdesc.index: # dispatch to the opcode method meth = getattr(self, opdesc.methodname) diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py --- a/pypy/interpreter/typedef.py +++ b/pypy/interpreter/typedef.py @@ -105,6 +105,13 @@ # typical usage (case 1 is the most common kind of app-level subclasses; # case 2 is the memory-saving kind defined with __slots__). # +# +----------------------------------------------------------------+ +# | NOTE: if withmapdict is enabled, the following doesn't apply! | +# | Map dicts can flexibly allow any slots/__dict__/__weakref__ to | +# | show up only when needed. In particular there is no way with | +# | mapdict to prevent some objects from being weakrefable. | +# +----------------------------------------------------------------+ +# # dict slots del weakrefable # # 1. Y N N Y UserDictWeakref diff --git a/pypy/jit/backend/arm/test/test_gc_integration.py b/pypy/jit/backend/arm/test/test_gc_integration.py --- a/pypy/jit/backend/arm/test/test_gc_integration.py +++ b/pypy/jit/backend/arm/test/test_gc_integration.py @@ -2,25 +2,13 @@ """ Tests for register allocation for common constructs """ -import py -from pypy.jit.metainterp.history import BoxInt, \ - BoxPtr, TreeLoop, TargetToken -from pypy.jit.metainterp.resoperation import rop, ResOperation -from pypy.jit.codewriter import heaptracker -from pypy.jit.backend.llsupport.descr import GcCache -from pypy.jit.backend.llsupport.gc import GcLLDescription +from pypy.jit.metainterp.history import TargetToken +from pypy.jit.backend.llsupport.gc import GcLLDescription, GcLLDescr_boehm from pypy.jit.backend.detect_cpu import getcpuclass from pypy.jit.backend.arm.arch import WORD from pypy.rpython.lltypesystem import lltype, llmemory, rffi -from pypy.rpython.annlowlevel import llhelper -from pypy.rpython.lltypesystem import rclass -from pypy.jit.backend.llsupport.gc import GcLLDescr_framework -from pypy.jit.backend.arm.test.test_regalloc import MockAssembler from pypy.jit.backend.arm.test.test_regalloc import BaseTestRegalloc -from pypy.jit.backend.arm.regalloc import ARMFrameManager, VFPRegisterManager -from pypy.jit.codewriter.effectinfo import EffectInfo -from pypy.jit.backend.arm.regalloc import Regalloc CPU = getcpuclass() @@ -44,23 +32,14 @@ return ['compressed'] + shape[1:] -class MockGcDescr(GcCache): - get_malloc_slowpath_addr = None - write_barrier_descr = None - moving_gc = True +class MockGcDescr(GcLLDescr_boehm): gcrootmap = MockGcRootMap() - def initialize(self): - pass - - _record_constptrs = GcLLDescr_framework._record_constptrs.im_func - rewrite_assembler = GcLLDescr_framework.rewrite_assembler.im_func - class TestRegallocGcIntegration(BaseTestRegalloc): cpu = CPU(None, None) - cpu.gc_ll_descr = MockGcDescr(False) + cpu.gc_ll_descr = MockGcDescr(None, None, None) cpu.setup_once() S = lltype.GcForwardReference() diff --git a/pypy/jit/backend/test/calling_convention_test.py b/pypy/jit/backend/test/calling_convention_test.py --- a/pypy/jit/backend/test/calling_convention_test.py +++ b/pypy/jit/backend/test/calling_convention_test.py @@ -105,11 +105,13 @@ ops = '[%s]\n' % arguments ops += '%s\n' % spill_ops ops += 'f99 = call(ConstClass(func_ptr), %s, descr=calldescr)\n' % arguments - ops += 'finish(f99, %s)\n' % arguments + ops += 'i99 = same_as(0)\n' + ops += 'guard_true(i99) [f99, %s]\n' % arguments + ops += 'finish()\n' loop = parse(ops, namespace=locals()) looptoken = JitCellToken() - done_number = self.cpu.get_fail_descr_number(loop.operations[-1].getdescr()) + done_number = self.cpu.get_fail_descr_number(loop.operations[-2].getdescr()) self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) argvals, expected_result = self._prepare_args(args, floats, ints) diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py --- a/pypy/jit/backend/test/runner_test.py +++ b/pypy/jit/backend/test/runner_test.py @@ -1192,7 +1192,8 @@ values.append(longlong.getfloatstorage(r.random())) # looptoken = JitCellToken() - faildescr = BasicFailDescr(42) + guarddescr = BasicFailDescr(42) + faildescr = BasicFailDescr(43) operations = [] retboxes = [] retvalues = [] @@ -1221,9 +1222,13 @@ retboxes.insert(kk, newbox) retvalues.insert(kk, y) # - operations.append( - ResOperation(rop.FINISH, retboxes, None, descr=faildescr) - ) + zero = BoxInt() + operations.extend([ + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=guarddescr), + ResOperation(rop.FINISH, [], None, descr=faildescr) + ]) + operations[-2].setfailargs(retboxes) print inputargs for op in operations: print op @@ -1344,12 +1349,15 @@ targettoken = TargetToken() faildescr1 = BasicFailDescr(1) faildescr2 = BasicFailDescr(2) + faildescr3 = BasicFailDescr(3) operations = [ ResOperation(rop.LABEL, fboxes, None, descr=targettoken), ResOperation(rop.FLOAT_LE, [fboxes[0], constfloat(9.2)], i2), ResOperation(rop.GUARD_TRUE, [i2], None, descr=faildescr1), - ResOperation(rop.FINISH, fboxes, None, descr=faildescr2), + ResOperation(rop.GUARD_FALSE, [i2], None, descr=faildescr2), + ResOperation(rop.FINISH, [], None, descr=faildescr3), ] + operations[-3].setfailargs(fboxes) operations[-2].setfailargs(fboxes) looptoken = JitCellToken() self.cpu.compile_loop(fboxes, operations, looptoken) @@ -1382,6 +1390,7 @@ py.test.skip("requires floats") fboxes = [BoxFloat() for i in range(3)] faildescr1 = BasicFailDescr(100) + faildescr2 = BasicFailDescr(102) loopops = """ [i0,f1, f2] f3 = float_add(f1, f2) @@ -1406,9 +1415,13 @@ assert longlong.getrealfloat(f2) == 0.75 assert longlong.getrealfloat(f3) == 133.0 + zero = BoxInt() bridgeops = [ - ResOperation(rop.FINISH, fboxes, None, descr=faildescr1), + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=faildescr1), + ResOperation(rop.FINISH, [], None, descr=faildescr2), ] + bridgeops[-2].setfailargs(fboxes[:]) self.cpu.compile_bridge(loop.operations[-2].getdescr(), fboxes, bridgeops, looptoken) args = [1, @@ -2002,7 +2015,7 @@ i1 = same_as(1) call(ConstClass(fptr), i0, descr=calldescr) p0 = guard_exception(ConstClass(xtp)) [i1] - finish(0, p0) + finish(p0) ''' FPTR = lltype.Ptr(lltype.FuncType([lltype.Signed], lltype.Void)) fptr = llhelper(FPTR, func) @@ -2023,8 +2036,7 @@ looptoken = JitCellToken() self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) deadframe = self.cpu.execute_token(looptoken, 1) - assert self.cpu.get_latest_value_int(deadframe, 0) == 0 - assert self.cpu.get_latest_value_ref(deadframe, 1) == xptr + assert self.cpu.get_latest_value_ref(deadframe, 0) == xptr excvalue = self.cpu.grab_exc_value(deadframe) assert not excvalue deadframe = self.cpu.execute_token(looptoken, 0) diff --git a/pypy/jit/backend/test/test_random.py b/pypy/jit/backend/test/test_random.py --- a/pypy/jit/backend/test/test_random.py +++ b/pypy/jit/backend/test/test_random.py @@ -26,6 +26,8 @@ class OperationBuilder(object): def __init__(self, cpu, loop, vars): self.cpu = cpu + if not hasattr(cpu, '_faildescr_keepalive'): + cpu._faildescr_keepalive = [] self.fakemetainterp = FakeMetaInterp() self.loop = loop self.intvars = [box for box in vars if isinstance(box, BoxInt)] @@ -206,6 +208,11 @@ if pytest.config.option.output: s.close() + def getfaildescr(self): + descr = BasicFailDescr() + self.cpu._faildescr_keepalive.append(descr) + return descr + class CannotProduceOperation(Exception): pass @@ -287,7 +294,7 @@ builder.intvars[:] = original_intvars else: op = ResOperation(rop.GUARD_NO_OVERFLOW, [], None) - op.setdescr(BasicFailDescr()) + op.setdescr(builder.getfaildescr()) op.setfailargs(fail_subset) builder.loop.operations.append(op) @@ -357,7 +364,7 @@ def produce_into(self, builder, r): op, passing = self.gen_guard(builder, r) builder.loop.operations.append(op) - op.setdescr(BasicFailDescr()) + op.setdescr(builder.getfaildescr()) op.setfailargs(builder.subset_of_intvars(r)) if not passing: builder.should_fail_by = op @@ -618,8 +625,9 @@ if v not in used_later: endvars.append(v) r.shuffle(endvars) + endvars = endvars[:1] loop.operations.append(ResOperation(rop.FINISH, endvars, None, - descr=BasicFailDescr())) + descr=builder.getfaildescr())) if builder.should_fail_by: self.should_fail_by = builder.should_fail_by self.guard_op = builder.guard_op @@ -712,7 +720,7 @@ else: op = ResOperation(rop.GUARD_EXCEPTION, [guard_op._exc_box], BoxPtr()) - op.setdescr(BasicFailDescr()) + op.setdescr(self.builder.getfaildescr()) op.setfailargs([]) return op diff --git a/pypy/jit/backend/x86/test/test_recompilation.py b/pypy/jit/backend/x86/test/test_recompilation.py --- a/pypy/jit/backend/x86/test/test_recompilation.py +++ b/pypy/jit/backend/x86/test/test_recompilation.py @@ -45,7 +45,8 @@ force_spill(i5) i8 = int_add(i7, 1) i9 = int_add(i8, 1) - finish(i3, i4, i5, i6, i7, i8, i9, descr=fdescr2) + guard_false(i3, descr=fdescr2) [i3, i4, i5, i6, i7, i8, i9] + finish() ''' bridge = self.attach_bridge(ops, loop, -2) descr = loop.operations[3].getdescr() diff --git a/pypy/jit/backend/x86/test/test_regalloc.py b/pypy/jit/backend/x86/test/test_regalloc.py --- a/pypy/jit/backend/x86/test/test_regalloc.py +++ b/pypy/jit/backend/x86/test/test_regalloc.py @@ -327,7 +327,7 @@ assert self.getint(0) == 0 bridge_ops = ''' [i0, i1] - finish(1, 2) + finish(2) ''' self.attach_bridge(bridge_ops, loop, 0) self.run(loop, 0, 1) @@ -428,7 +428,7 @@ ops = ''' [i0, i1, i2, i3, i4, i5, i6, i7] guard_value(i6, i1) [i0, i2, i3, i4, i5, i6] - finish(i0, i2, i3, i4, i5, i6) + finish(i0) ''' self.interpret(ops, [0, 0, 0, 0, 0, 0, 0, 0]) assert self.getint(0) == 0 @@ -438,14 +438,15 @@ [i0, i1, i2, i3, i4, i5, i6, i7, i8] i9 = same_as(0) guard_true(i0) [i9, i0, i1, i2, i3, i4, i5, i6, i7, i8] - finish(1, i0, i1, i2, i3, i4, i5, i6, i7, i8) + finish(1) ''' loop = self.interpret(ops, [0, 1, 2, 3, 4, 5, 6, 7, 8]) assert self.getint(0) == 0 bridge_ops = ''' [i9, i0, i1, i2, i3, i4, i5, i6, i7, i8] call(ConstClass(raising_fptr), 0, descr=raising_calldescr) - finish(i0, i1, i2, i3, i4, i5, i6, i7, i8) + guard_true(i9) [i0, i1, i2, i3, i4, i5, i6, i7, i8] + finish() ''' self.attach_bridge(bridge_ops, loop, 1) self.run(loop, 0, 1, 2, 3, 4, 5, 6, 7, 8) @@ -474,7 +475,8 @@ i1 = same_as(1) i2 = int_lt(i0, 100) guard_true(i3) [i1, i2] - finish(0, i2) + i4 = int_neg(i2) + finish(0) ''' self.interpret(ops, [0, 1]) assert self.getint(0) == 0 @@ -503,7 +505,8 @@ i15 = int_is_true(i5) i16 = int_is_true(i6) i17 = int_is_true(i7) - finish(i10, i11, i12, i13, i14, i15, i16, i17) + guard_true(i0) [i10, i11, i12, i13, i14, i15, i16, i17] + finish() ''' self.interpret(ops, [0, 42, 12, 0, 13, 0, 0, 3333]) assert self.getints(8) == [0, 1, 1, 0, 1, 0, 0, 1] @@ -517,7 +520,8 @@ i13 = int_eq(i5, i6) i14 = int_gt(i6, i2) i15 = int_ne(i2, i6) - finish(i10, i11, i12, i13, i14, i15) + guard_true(i0) [i10, i11, i12, i13, i14, i15] + finish() ''' self.interpret(ops, [0, 1, 2, 3, 4, 5, 6]) assert self.getints(6) == [1, 1, 0, 0, 1, 1] @@ -574,7 +578,9 @@ ops = ''' [f0, f1] f2 = float_add(f0, f1) - finish(f2, f0, f1) + i0 = same_as(0) + guard_true(i0) [f2, f0, f1] + finish() ''' self.interpret(ops, [3.0, 1.5]) assert self.getfloats(3) == [4.5, 3.0, 1.5] @@ -584,7 +590,9 @@ [f0, f1, f2, f3, f4, f5, f6, f7, f8] f9 = float_add(f0, f1) f10 = float_add(f8, 3.5) - finish(f9, f10, f2, f3, f4, f5, f6, f7, f8) + i0 = same_as(0) + guard_true(i0) [f9, f10, f2, f3, f4, f5, f6, f7, f8] + finish() ''' self.interpret(ops, [0.1, .2, .3, .4, .5, .6, .7, .8, .9]) assert self.getfloats(9) == [.1+.2, .9+3.5, .3, .4, .5, .6, .7, .8, .9] @@ -613,7 +621,8 @@ i7 = float_ne(f7, 0.0) i8 = float_ne(f8, 0.0) i9 = float_ne(f9, 0.0) - finish(i0, i1, i2, i3, i4, i5, i6, i7, i8, i9) + guard_true(i0) [i0, i1, i2, i3, i4, i5, i6, i7, i8, i9] + finish() ''' loop = self.interpret(ops, [0.0, .1, .2, .3, .4, .5, .6, .7, .8, .9]) assert self.getints(9) == [0, 1, 1, 1, 1, 1, 1, 1, 1] diff --git a/pypy/jit/backend/x86/test/test_regalloc2.py b/pypy/jit/backend/x86/test/test_regalloc2.py --- a/pypy/jit/backend/x86/test/test_regalloc2.py +++ b/pypy/jit/backend/x86/test/test_regalloc2.py @@ -11,13 +11,17 @@ v2 = BoxInt() v3 = BoxInt() v4 = BoxInt() + zero = BoxInt() inputargs = [v1] operations = [ ResOperation(rop.INT_ADD, [v1, v1], v2), ResOperation(rop.INT_INVERT, [v2], v3), ResOperation(rop.UINT_RSHIFT, [v1, ConstInt(3)], v4), - ResOperation(rop.FINISH, [v4, v3], None, descr=BasicFailDescr()), + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=BasicFailDescr()), + ResOperation(rop.FINISH, [], None, descr=BasicFailDescr()) ] + operations[-2].setfailargs([v4, v3]) cpu = CPU(None, None) cpu.setup_once() looptoken = JitCellToken() @@ -31,6 +35,7 @@ v2 = BoxInt() v3 = BoxInt() v4 = BoxInt() + zero = BoxInt() tmp5 = BoxInt() inputargs = [v1] operations = [ @@ -38,8 +43,11 @@ ResOperation(rop.INT_MUL, [v2, v1], v3), ResOperation(rop.INT_IS_TRUE, [v2], tmp5), ResOperation(rop.INT_IS_ZERO, [tmp5], v4), - ResOperation(rop.FINISH, [v4, v3, tmp5], None, descr=BasicFailDescr()), + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=BasicFailDescr()), + ResOperation(rop.FINISH, [], None, descr=BasicFailDescr()) ] + operations[-2].setfailargs([v4, v3, tmp5]) cpu = CPU(None, None) cpu.setup_once() looptoken = JitCellToken() @@ -90,6 +98,7 @@ v38 = BoxInt() v39 = BoxInt() v40 = BoxInt() + zero = BoxInt() tmp41 = BoxInt() tmp42 = BoxInt() tmp43 = BoxInt() @@ -134,8 +143,12 @@ ResOperation(rop.UINT_GT, [v33, ConstInt(-11)], v38), ResOperation(rop.INT_NEG, [v7], v39), ResOperation(rop.INT_GT, [v24, v32], v40), - ResOperation(rop.FINISH, [v40, v36, v37, v31, v16, v34, v35, v23, v22, v29, v14, v39, v30, v38], None, descr=BasicFailDescr()), + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=BasicFailDescr()), + ResOperation(rop.FINISH, [], None, descr=BasicFailDescr()) ] + operations[-2].setfailargs([v40, v36, v37, v31, v16, v34, v35, v23, + v22, v29, v14, v39, v30, v38]) cpu = CPU(None, None) cpu.setup_once() looptoken = JitCellToken() @@ -198,6 +211,7 @@ v38 = BoxInt() v39 = BoxInt() v40 = BoxInt() + zero = BoxInt() tmp41 = BoxInt() tmp42 = BoxInt() tmp43 = BoxInt() @@ -240,8 +254,13 @@ ResOperation(rop.INT_GT, [v4, v11], v38), ResOperation(rop.INT_LT, [v27, v22], v39), ResOperation(rop.INT_NEG, [v27], v40), - ResOperation(rop.FINISH, [v40, v10, v36, v26, v13, v30, v21, v33, v18, v25, v31, v32, v28, v29, v35, v38, v20, v39, v34, v23, v37], None, descr=BasicFailDescr()), + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=BasicFailDescr()), + ResOperation(rop.FINISH, [], None, descr=BasicFailDescr()) ] + operations[-2].setfailargs([v40, v10, v36, v26, v13, v30, v21, v33, + v18, v25, v31, v32, v28, v29, v35, v38, + v20, v39, v34, v23, v37]) cpu = CPU(None, None) cpu.setup_once() looptoken = JitCellToken() diff --git a/pypy/jit/backend/x86/test/test_runner.py b/pypy/jit/backend/x86/test/test_runner.py --- a/pypy/jit/backend/x86/test/test_runner.py +++ b/pypy/jit/backend/x86/test/test_runner.py @@ -499,9 +499,12 @@ i6, descr=calldescr), ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr), - ResOperation(rop.FINISH, [i3, i4, i5, i6], None, - descr=BasicFailDescr(0)) + ResOperation(rop.GUARD_FALSE, [i3], None, + descr=BasicFailDescr(0)), + ResOperation(rop.FINISH, [], None, + descr=BasicFailDescr(1)) ] + ops[-2].setfailargs([i3, i4, i5, i6]) ops[1].setfailargs([]) ops[3].setfailargs([]) ops[5].setfailargs([]) diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py --- a/pypy/jit/codewriter/support.py +++ b/pypy/jit/codewriter/support.py @@ -10,7 +10,7 @@ from pypy.rpython.extregistry import ExtRegistryEntry from pypy.translator.simplify import get_funcobj from pypy.translator.unsimplify import split_block -from pypy.objspace.flow.model import Constant +from pypy.objspace.flow.model import Variable, Constant from pypy.translator.translator import TranslationContext from pypy.annotation.policy import AnnotatorPolicy from pypy.annotation import model as annmodel @@ -59,6 +59,45 @@ rtyper = annotate(func, values) return rtyper.annotator.translator.graphs[0] +def autodetect_jit_markers_redvars(graph): + # the idea is to find all the jit_merge_point and + # add all the variables across the links to the reds. + for block, op in graph.iterblockops(): + if op.opname == 'jit_marker': + jitdriver = op.args[1].value + if not jitdriver.autoreds: + continue + # if we want to support also can_enter_jit, we should find a + # way to detect a consistent set of red vars to pass *both* to + # jit_merge_point and can_enter_jit. The current simple + # solution doesn't work because can_enter_jit might be in + # another block, so the set of alive_v will be different. + methname = op.args[0].value + assert methname == 'jit_merge_point', ( + "reds='auto' is supported only for jit drivers which " + "calls only jit_merge_point. Found a call to %s" % methname) + # + # compute the set of live variables across the jit_marker + alive_v = set() + for link in block.exits: + alive_v.update(link.args) + alive_v.difference_update(link.getextravars()) + for op1 in block.operations[::-1]: + if op1 is op: + break # stop when the meet the jit_marker + alive_v.discard(op1.result) + alive_v.update(op1.args) + greens_v = op.args[2:] + reds_v = alive_v - set(greens_v) + reds_v = [v for v in reds_v if isinstance(v, Variable) and + v.concretetype is not lltype.Void] + reds_v = sort_vars(reds_v) + op.args.extend(reds_v) + if jitdriver.numreds is None: + jitdriver.numreds = len(reds_v) + else: + assert jitdriver.numreds == len(reds_v), 'inconsistent number of reds_v' + def split_before_jit_merge_point(graph, portalblock, portalopindex): """Split the block just before the 'jit_merge_point', making sure the input args are in the canonical order. diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py --- a/pypy/jit/metainterp/resoperation.py +++ b/pypy/jit/metainterp/resoperation.py @@ -353,6 +353,9 @@ def initarglist(self, args): self._args = args + if not we_are_translated() and \ + self.__class__.__name__.startswith('FINISH'): # XXX remove me + assert len(args) <= 1 # FINISH operations take 0 or 1 arg now def getarglist(self): return self._args diff --git a/pypy/jit/metainterp/test/test_fficall.py b/pypy/jit/metainterp/test/test_fficall.py --- a/pypy/jit/metainterp/test/test_fficall.py +++ b/pypy/jit/metainterp/test/test_fficall.py @@ -105,7 +105,7 @@ class TestFfiCall(FfiCallTests, LLJitMixin): - def test_jit_fii_vref(self): + def test_jit_ffi_vref(self): from pypy.rlib import clibffi from pypy.rlib.jit_libffi import jit_ffi_prep_cif, jit_ffi_call @@ -136,9 +136,10 @@ res = exb[3] lltype.free(exb, flavor='raw') # - lltype.free(atypes, flavor='raw') return res # res = self.interp_operations(f, []) lltype.free(cd, flavor='raw') assert res == math.sin(1.23) + + lltype.free(atypes, flavor='raw') diff --git a/pypy/jit/metainterp/test/test_warmspot.py b/pypy/jit/metainterp/test/test_warmspot.py --- a/pypy/jit/metainterp/test/test_warmspot.py +++ b/pypy/jit/metainterp/test/test_warmspot.py @@ -1,6 +1,6 @@ import py from pypy.jit.metainterp.warmspot import get_stats -from pypy.rlib.jit import JitDriver, set_param, unroll_safe +from pypy.rlib.jit import JitDriver, set_param, unroll_safe, jit_callback from pypy.jit.backend.llgraph import runner from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin @@ -383,6 +383,23 @@ assert res == expected self.check_resops(int_sub=2, int_mul=0, int_add=2) + def test_loop_automatic_reds_not_too_many_redvars(self): + myjitdriver = JitDriver(greens = ['m'], reds = 'auto') + def one(): + return 1 + def f(n, m): + res = 0 + while n > 0: + n -= one() + myjitdriver.jit_merge_point(m=m) + res += m*2 + return res + expected = f(21, 5) + res = self.meta_interp(f, [21, 5]) + assert res == expected + oplabel = get_stats().loops[0].operations[0] + assert len(oplabel.getarglist()) == 2 # 'n', 'res' in some order + def test_inline_jit_merge_point(self): # test that the machinery to inline jit_merge_points in callers # works. The final user does not need to mess manually with the @@ -520,41 +537,21 @@ def test_callback_jit_merge_point(self): - from pypy.rlib.objectmodel import register_around_callback_hook - from pypy.rpython.lltypesystem import lltype, rffi - from pypy.translator.tool.cbuild import ExternalCompilationInfo - - callback_jit_driver = JitDriver(greens = ['name'], reds = 'auto') - - def callback_merge_point(name): - callback_jit_driver.jit_merge_point(name=name) - - @callback_jit_driver.inline(callback_merge_point) - def callback_hook(name): - pass - + @jit_callback("testing") def callback(a, b): if a > b: return 1 return -1 - CB_TP = rffi.CCallback([lltype.Signed, lltype.Signed], lltype.Signed) - eci = ExternalCompilationInfo(includes=['stdlib.h']) - qsort = rffi.llexternal('qsort', - [rffi.VOIDP, lltype.Signed, lltype.Signed, - CB_TP], lltype.Void, compilation_info=eci) - ARR = rffi.CArray(lltype.Signed) + def main(): + total = 0 + for i in range(10): + total += callback(i, 2) + return total - def main(): - register_around_callback_hook(callback_hook) - raw = lltype.malloc(ARR, 10, flavor='raw') - for i in range(10): - raw[i] = 10 - i - qsort(raw, 10, rffi.sizeof(lltype.Signed), callback) - lltype.free(raw, flavor='raw') - - self.meta_interp(main, []) - self.check_trace_count(1) + res = self.meta_interp(main, []) + assert res == 7 - 3 + self.check_trace_count(2) class TestLLWarmspot(WarmspotTests, LLJitMixin): diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py --- a/pypy/jit/metainterp/warmspot.py +++ b/pypy/jit/metainterp/warmspot.py @@ -347,62 +347,20 @@ self.jitdrivers_sd = [] graphs = self.translator.graphs for graph, block, pos in find_jit_merge_points(graphs): - self.autodetect_jit_markers_redvars(graph) + support.autodetect_jit_markers_redvars(graph) self.split_graph_and_record_jitdriver(graph, block, pos) # assert (len(set([jd.jitdriver for jd in self.jitdrivers_sd])) == len(self.jitdrivers_sd)), \ "there are multiple jit_merge_points with the same jitdriver" - def autodetect_jit_markers_redvars(self, graph): - # the idea is to find all the jit_merge_point and can_enter_jit and - # add all the variables across the links to the reds. - for block, op in graph.iterblockops(): - if op.opname == 'jit_marker': - jitdriver = op.args[1].value - if not jitdriver.autoreds: - continue - # if we want to support also can_enter_jit, we should find a - # way to detect a consistent set of red vars to pass *both* to - # jit_merge_point and can_enter_jit. The current simple - # solution doesn't work because can_enter_jit might be in - # another block, so the set of alive_v will be different. - methname = op.args[0].value - assert methname == 'jit_merge_point', ( - "reds='auto' is supported only for jit drivers which " - "calls only jit_merge_point. Found a call to %s" % methname) - # - # compute the set of live variables before the jit_marker - alive_v = set(block.inputargs) - for op1 in block.operations: - if op1 is op: - break # stop when the meet the jit_marker - if op1.result.concretetype != lltype.Void: - alive_v.add(op1.result) - greens_v = op.args[2:] - reds_v = alive_v - set(greens_v) - reds_v = [v for v in reds_v if v.concretetype is not lltype.Void] - reds_v = support.sort_vars(reds_v) - op.args.extend(reds_v) - if jitdriver.numreds is None: - jitdriver.numreds = len(reds_v) - else: - assert jitdriver.numreds == len(reds_v), 'inconsistent number of reds_v' - - def split_graph_and_record_jitdriver(self, graph, block, pos): op = block.operations[pos] jd = JitDriverStaticData() jd._jit_merge_point_in = graph args = op.args[2:] s_binding = self.translator.annotator.binding - if op.args[1].value.autoreds: - # _portal_args_s is used only by _make_hook_graph, but for now we - # declare the various set_jitcell_at, get_printable_location, - # etc. as incompatible with autoreds - jd._portal_args_s = None - else: - jd._portal_args_s = [s_binding(v) for v in args] + jd._portal_args_s = [s_binding(v) for v in args] graph = copygraph(graph) [jmpp] = find_jit_merge_points([graph]) graph.startblock = support.split_before_jit_merge_point(*jmpp) @@ -650,9 +608,10 @@ if func is None: return None # - assert not jitdriver_sd.jitdriver.autoreds, ( - "reds='auto' is not compatible with JitDriver hooks such as " - "{get,set}_jitcell_at, get_printable_location, confirm_enter_jit, etc.") + if not onlygreens: + assert not jitdriver_sd.jitdriver.autoreds, ( + "reds='auto' is not compatible with JitDriver hooks such as " + "confirm_enter_jit") extra_args_s = [] if s_first_arg is not None: extra_args_s.append(s_first_arg) @@ -717,6 +676,9 @@ jitdriver = op.args[1].value assert jitdriver in sublists, \ "can_enter_jit with no matching jit_merge_point" + assert not jitdriver.autoreds, ( + "can_enter_jit not supported with a jitdriver that " + "has reds='auto'") jd, sublist = sublists[jitdriver] origportalgraph = jd._jit_merge_point_in if graph is not origportalgraph: diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py --- a/pypy/module/_cffi_backend/ccallback.py +++ b/pypy/module/_cffi_backend/ccallback.py @@ -152,6 +152,7 @@ STDERR = 2 + at jit.jit_callback("CFFI") def invoke_callback(ffi_cif, ll_res, ll_args, ll_userdata): """ Callback specification. ffi_cif - something ffi specific, don't care diff --git a/pypy/module/_weakref/test/test_weakref.py b/pypy/module/_weakref/test/test_weakref.py --- a/pypy/module/_weakref/test/test_weakref.py +++ b/pypy/module/_weakref/test/test_weakref.py @@ -114,19 +114,28 @@ class A(object): def __eq__(self, other): return True + def __ne__(self, other): + return False a1 = A() a2 = A() ref1 = _weakref.ref(a1) ref2 = _weakref.ref(a2) assert ref1 == ref2 + assert not (ref1 != ref2) + assert not (ref1 == []) + assert ref1 != [] del a1 gc.collect() assert not ref1 == ref2 assert ref1 != ref2 + assert not (ref1 == []) + assert ref1 != [] del a2 gc.collect() assert not ref1 == ref2 assert ref1 != ref2 + assert not (ref1 == []) + assert ref1 != [] def test_getweakrefs(self): import _weakref, gc @@ -300,6 +309,13 @@ if seen_callback: assert seen_callback == [True, True, True] + def test_type_weakrefable(self): + import _weakref, gc + w = _weakref.ref(list) + assert w() is list + gc.collect() + assert w() is list + class AppTestProxy(object): spaceconfig = dict(usemodules=('_weakref',)) @@ -434,6 +450,8 @@ class A(object): def __eq__(self, other): return True + def __ne__(self, other): + return False a = A() assert _weakref.ref(a) == a diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py --- a/pypy/module/micronumpy/arrayimpl/concrete.py +++ b/pypy/module/micronumpy/arrayimpl/concrete.py @@ -13,6 +13,7 @@ from pypy.rlib.debug import make_sure_not_resized class ConcreteArrayIterator(base.BaseArrayIterator): + _immutable_fields_ = ['dtype', 'skip', 'size'] def __init__(self, array): self.array = array self.offset = 0 @@ -21,10 +22,10 @@ self.size = array.size def setitem(self, elem): - self.array.setitem(self.offset, elem) + self.dtype.setitem(self.array, self.offset, elem) def getitem(self): - return self.array.getitem(self.offset) + return self.dtype.getitem(self.array, self.offset) def getitem_bool(self): return self.dtype.getitem_bool(self.array, self.offset) @@ -42,13 +43,16 @@ self.offset %= self.size class OneDimViewIterator(ConcreteArrayIterator): - def __init__(self, array): + ''' The view iterator dtype can be different from the + array.dtype, this is what makes it a View + ''' + def __init__(self, array, dtype, start, strides, shape): self.array = array - self.offset = array.start - self.skip = array.get_strides()[0] - self.dtype = array.dtype + self.dtype = dtype + self.offset = start + self.skip = strides[0] self.index = 0 - self.size = array.get_shape()[0] + self.size = shape[0] def next(self): self.offset += self.skip @@ -65,9 +69,13 @@ self.offset %= self.size class MultiDimViewIterator(ConcreteArrayIterator): - def __init__(self, array, start, strides, backstrides, shape): + ''' The view iterator dtype can be different from the + array.dtype, this is what makes it a View + ''' + def __init__(self, array, dtype, start, strides, backstrides, shape): self.indexes = [0] * len(shape) self.array = array + self.dtype = dtype self.shape = shape self.offset = start self.shapelen = len(shape) @@ -131,12 +139,13 @@ self.offset = array.start self.dim = dim self.array = array + self.dtype = array.dtype def setitem(self, elem): - self.array.setitem(self.offset, elem) + self.dtype.setitem(self.array, self.offset, elem) def getitem(self): - return self.array.getitem(self.offset) + return self.dtype.getitem(self.array, self.offset) @jit.unroll_safe def next(self): @@ -220,6 +229,32 @@ new_shape, self) else: return None + + def get_real(self): + strides = self.get_strides() + backstrides = self.get_backstrides() + if self.dtype.is_complex_type(): + dtype = self.dtype.float_type + return SliceArray(self.start, strides, backstrides, + self.get_shape(), self, dtype=dtype) + return SliceArray(self.start, strides, backstrides, + self.get_shape(), self) + + def get_imag(self): + strides = self.get_strides() + backstrides = self.get_backstrides() + if self.dtype.is_complex_type(): + dtype = self.dtype.float_type + return SliceArray(self.start + dtype.get_size(), strides, + backstrides, self.get_shape(), self, dtype=dtype) + if self.dtype.is_flexible_type(): + # numpy returns self for self.imag + return SliceArray(self.start, strides, backstrides, + self.get_shape(), self) + impl = NonWritableArray(self.get_shape(), self.dtype, self.order, strides, + backstrides) + impl.fill(self.dtype.box(0)) + return impl # -------------------- applevel get/setitem ----------------------- @@ -377,7 +412,7 @@ def create_dot_iter(self, shape, skip): r = calculate_dot_strides(self.get_strides(), self.get_backstrides(), shape, skip) - return MultiDimViewIterator(self, self.start, r[0], r[1], shape) + return MultiDimViewIterator(self, self.dtype, self.start, r[0], r[1], shape) def swapaxes(self, axis1, axis2): shape = self.get_shape()[:] @@ -411,7 +446,7 @@ r = calculate_broadcast_strides(self.get_strides(), self.get_backstrides(), self.get_shape(), shape) - return MultiDimViewIterator(self, 0, r[0], r[1], shape) + return MultiDimViewIterator(self, self.dtype, 0, r[0], r[1], shape) def fill(self, box): self.dtype.fill(self.storage, box, 0, self.size) @@ -424,6 +459,12 @@ self.order) return SliceArray(0, strides, backstrides, new_shape, self) +class NonWritableArray(ConcreteArray): + def descr_setitem(self, space, w_index, w_value): + raise OperationError(space.w_RuntimeError, space.wrap( + "array is not writable")) + + class SliceArray(BaseConcreteArray): def __init__(self, start, strides, backstrides, shape, parent, dtype=None): self.strides = strides @@ -448,11 +489,12 @@ r = calculate_broadcast_strides(self.get_strides(), self.get_backstrides(), self.get_shape(), shape) - return MultiDimViewIterator(self.parent, + return MultiDimViewIterator(self.parent, self.dtype, self.start, r[0], r[1], shape) if len(self.get_shape()) == 1: - return OneDimViewIterator(self) - return MultiDimViewIterator(self.parent, self.start, + return OneDimViewIterator(self.parent, self.dtype, self.start, + self.get_strides(), self.get_shape()) + return MultiDimViewIterator(self.parent, self.dtype, self.start, self.get_strides(), self.get_backstrides(), self.get_shape()) diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -136,7 +136,7 @@ return self.kind == SIGNEDLTR def is_complex_type(self): - return (self.num == 14 or self.num == 15 or self.num == 16) + return False def is_bool_type(self): return self.kind == BOOLLTR @@ -155,6 +155,19 @@ def get_size(self): return self.itemtype.get_element_size() +class W_ComplexDtype(W_Dtype): + + def __init__(self, itemtype, num, kind, name, char, w_box_type, + alternate_constructors=[], aliases=[], + fields=None, fieldnames=None, native=True, float_type=None): + W_Dtype.__init__(self, itemtype, num, kind, name, char, w_box_type, + alternate_constructors=alternate_constructors, aliases=aliases, + fields=fields, fieldnames=fieldnames, native=native) + self.float_type = float_type + + def is_complex_type(self): + return True + def dtype_from_list(space, w_lst): lst_w = space.listview(w_lst) fields = {} @@ -413,15 +426,16 @@ alternate_constructors=[space.w_float], aliases=["float"], ) - self.w_complex64dtype = W_Dtype( + self.w_complex64dtype = W_ComplexDtype( types.Complex64(), num=14, kind=COMPLEXLTR, name="complex64", char="F", w_box_type = space.gettypefor(interp_boxes.W_Complex64Box), + float_type = self.w_float32dtype, ) - self.w_complex128dtype = W_Dtype( + self.w_complex128dtype = W_ComplexDtype( types.Complex128(), num=15, kind=COMPLEXLTR, @@ -430,6 +444,7 @@ w_box_type = space.gettypefor(interp_boxes.W_Complex128Box), alternate_constructors=[space.w_complex], aliases=["complex"], + float_type = self.w_float64dtype, ) if interp_boxes.long_double_size == 12: self.w_float96dtype = W_Dtype( @@ -443,7 +458,7 @@ ) self.w_longdouble = self.w_float96dtype - self.w_complex192dtype = W_Dtype( + self.w_complex192dtype = W_ComplexDtype( types.Complex192(), num=16, kind=COMPLEXLTR, @@ -452,6 +467,7 @@ w_box_type = space.gettypefor(interp_boxes.W_Complex192Box), alternate_constructors=[space.w_complex], aliases=["clongdouble", "clongfloat"], + float_type = self.w_float96dtype, ) self.w_clongdouble = self.w_complex192dtype @@ -467,7 +483,7 @@ ) self.w_longdouble = self.w_float128dtype - self.w_complex256dtype = W_Dtype( + self.w_complex256dtype = W_ComplexDtype( types.Complex256(), num=16, kind=COMPLEXLTR, @@ -476,6 +492,7 @@ w_box_type = space.gettypefor(interp_boxes.W_Complex256Box), alternate_constructors=[space.w_complex], aliases=["clongdouble", "clongfloat"], + float_type = self.w_float128dtype, ) self.w_clongdouble = self.w_complex256dtype else: diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -235,6 +235,29 @@ def descr_copy(self, space): return W_NDimArray(self.implementation.copy()) + def descr_get_real(self, space): + return W_NDimArray(self.implementation.get_real()) + + def descr_get_imag(self, space): + ret = self.implementation.get_imag() + if ret: + return W_NDimArray(ret) + raise OperationError(space.w_NotImplementedError, + space.wrap('imag not implemented for this dtype')) + + def descr_set_real(self, space, w_value): + # copy (broadcast) values into self + tmp = self.implementation.get_real() + tmp.setslice(space, convert_to_array(space, w_value)) + + def descr_set_imag(self, space, w_value): + # if possible, copy (broadcast) values into self + if not self.get_dtype().is_complex_type(): + raise OperationError(space.w_TypeError, + space.wrap('array does not have imaginary part to set')) + tmp = self.implementation.get_imag() + tmp.setslice(space, convert_to_array(space, w_value)) + def descr_reshape(self, space, args_w): """reshape(...) a.reshape(shape) @@ -387,8 +410,6 @@ descr_neg = _unaryop_impl("negative") descr_abs = _unaryop_impl("absolute") descr_invert = _unaryop_impl("invert") - descr_get_real = _unaryop_impl("real") - descr_get_imag = _unaryop_impl("imag") def descr_nonzero(self, space): if self.get_size() > 1: @@ -635,9 +656,10 @@ swapaxes = interp2app(W_NDimArray.descr_swapaxes), flat = GetSetProperty(W_NDimArray.descr_get_flatiter), item = interp2app(W_NDimArray.descr_item), - real = GetSetProperty(W_NDimArray.descr_get_real), - imag = GetSetProperty(W_NDimArray.descr_get_imag), - + real = GetSetProperty(W_NDimArray.descr_get_real, + W_NDimArray.descr_set_real), + imag = GetSetProperty(W_NDimArray.descr_get_imag, + W_NDimArray.descr_set_imag), __array_interface__ = GetSetProperty(W_NDimArray.descr_array_iface), ) diff --git a/pypy/module/micronumpy/test/test_iter.py b/pypy/module/micronumpy/test/test_iter.py --- a/pypy/module/micronumpy/test/test_iter.py +++ b/pypy/module/micronumpy/test/test_iter.py @@ -12,7 +12,7 @@ strides = [5, 1] backstrides = [x * (y - 1) for x,y in zip(strides, shape)] assert backstrides == [10, 4] - i = MultiDimViewIterator(MockArray, start, strides, backstrides, shape) + i = MultiDimViewIterator(MockArray, None, start, strides, backstrides, shape) i.next() i.next() i.next() @@ -30,7 +30,7 @@ strides = [1, 3] backstrides = [x * (y - 1) for x,y in zip(strides, shape)] assert backstrides == [2, 12] - i = MultiDimViewIterator(MockArray, start, strides, backstrides, shape) + i = MultiDimViewIterator(MockArray, None, start, strides, backstrides, shape) i.next() i.next() i.next() @@ -51,7 +51,7 @@ strides = [5, 1] backstrides = [x * (y - 1) for x,y in zip(strides, shape)] assert backstrides == [10, 4] - i = MultiDimViewIterator(MockArray, start, strides, backstrides, shape) + i = MultiDimViewIterator(MockArray, None, start, strides, backstrides, shape) i.next_skip_x(2) i.next_skip_x(2) i.next_skip_x(2) @@ -74,7 +74,7 @@ strides = [1, 3] backstrides = [x * (y - 1) for x,y in zip(strides, shape)] assert backstrides == [2, 12] - i = MultiDimViewIterator(MockArray, start, strides, backstrides, shape) + i = MultiDimViewIterator(MockArray, None, start, strides, backstrides, shape) i.next_skip_x(2) i.next_skip_x(2) i.next_skip_x(2) diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1169,8 +1169,8 @@ assert (d == [[1, 0, 0], [0, 1, 0], [0, 0, 1]]).all() def test_eye(self): - from _numpypy import eye, array - from _numpypy import int32, float64, dtype + from _numpypy import eye + from _numpypy import int32, dtype a = eye(0) assert len(a) == 0 assert a.dtype == dtype('float64') @@ -1345,6 +1345,42 @@ assert b[0] == 3 assert b[1] == 2 + def test_realimag_views(self): + from _numpypy import arange, array + a = arange(15) + b = a.real + b[5]=50 + assert a[5] == 50 + b = a.imag + assert b[7] == 0 + raises(RuntimeError, 'b[7] = -2') + raises(TypeError, 'a.imag = -2') + a = array(['abc','def'],dtype='S3') + b = a.real + assert a[0] == b[0] + assert a[1] == b[1] + b[1] = 'xyz' + assert a[1] == 'xyz' + assert a.imag[0] == 'abc' + raises(TypeError, 'a.imag = "qop"') + a=array([[1+1j, 2-3j, 4+5j],[-6+7j, 8-9j, -2-1j]]) + assert a.real[0,1] == 2 + a.real[0,1] = -20 + assert a[0,1].real == -20 + b = a.imag + assert b[1,2] == -1 + b[1,2] = 30 + assert a[1,2].imag == 30 + a.real = 13 + assert a[1,1].real == 13 + a=array([1+1j, 2-3j, 4+5j, -6+7j, 8-9j, -2-1j]) + a.real = 13 + assert a[3].real == 13 + a.imag = -5 + a.imag[3] = -10 + assert a[3].imag == -10 + assert a[2].imag == -5 + def test_tolist_scalar(self): from _numpypy import int32, bool_ x = int32(23) @@ -2274,7 +2310,7 @@ assert a[0]['x'] == 'a' def test_stringarray(self): - from _numpypy import array, flexible + from _numpypy import array a = array(['abc'],'S3') assert str(a.dtype) == '|S3' a = array(['abc']) @@ -2294,7 +2330,6 @@ def test_flexible_repr(self): # import overrides str(), repr() for array - from numpypy.core import arrayprint from _numpypy import array a = array(['abc'],'S3') s = repr(a) diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py --- a/pypy/module/pyexpat/interp_pyexpat.py +++ b/pypy/module/pyexpat/interp_pyexpat.py @@ -2,7 +2,7 @@ from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault from pypy.interpreter.error import OperationError -from pypy.rlib import rgc +from pypy.rlib import rgc, jit from pypy.rpython.lltypesystem import rffi, lltype from pypy.rpython.tool import rffi_platform from pypy.translator.tool.cbuild import ExternalCompilationInfo @@ -288,6 +288,7 @@ post_code = '' src = py.code.Source(""" + @jit.jit_callback('XML:%(name)s') def %(name)s_callback(%(first_arg)s, %(args)s): id = rffi.cast(lltype.Signed, %(ll_id)s) userdata = global_storage.get_object(id) diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py --- a/pypy/module/pypyjit/interp_jit.py +++ b/pypy/module/pypyjit/interp_jit.py @@ -6,7 +6,7 @@ from pypy.tool.pairtype import extendabletype from pypy.rlib.rarithmetic import r_uint, intmask from pypy.rlib.jit import JitDriver, hint, we_are_jitted, dont_look_inside -from pypy.rlib import jit, objectmodel +from pypy.rlib import jit from pypy.rlib.jit import current_trace_length, unroll_parameters import pypy.interpreter.pyopcode # for side-effects from pypy.interpreter.error import OperationError, operationerrfmt @@ -97,15 +97,6 @@ is_being_profiled=self.is_being_profiled) return jumpto -callback_jit_driver = JitDriver(greens = ['name'], reds = 'auto') - -def callback_merge_point(name): - callback_jit_driver.jit_merge_point(name=name) - - at callback_jit_driver.inline(callback_merge_point) -def callback_hook(name): - pass - def _get_adapted_tick_counter(): # Normally, the tick counter is decremented by 100 for every # Python opcode. Here, to better support JIT compilation of diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py --- a/pypy/module/signal/interp_signal.py +++ b/pypy/module/signal/interp_signal.py @@ -40,6 +40,7 @@ includes = ['stdlib.h', 'src/signals.h'] if sys.platform != 'win32': includes.append('sys/time.h') +WIN32 = sys.platform == 'win32' cdir = py.path.local(autopath.pypydir).join('translator', 'c') @@ -236,7 +237,10 @@ None -- if an unknown handler is in effect (XXX UNIMPLEMENTED) anything else -- the callable Python object used as a handler """ - check_signum_in_range(space, signum) + if WIN32: + check_signum_exists(space, signum) + else: + check_signum_in_range(space, signum) action = space.check_signal_action if signum in action.handlers_w: return action.handlers_w[signum] diff --git a/pypy/module/signal/test/test_signal.py b/pypy/module/signal/test/test_signal.py --- a/pypy/module/signal/test/test_signal.py +++ b/pypy/module/signal/test/test_signal.py @@ -55,7 +55,7 @@ skip("requires os.kill() and os.getpid()") signal = self.signal # the signal module to test if not hasattr(signal, 'SIGUSR1'): - py.test.skip("requires SIGUSR1 in signal") + skip("requires SIGUSR1 in signal") signum = signal.SIGUSR1 received = [] @@ -156,6 +156,7 @@ import sys if sys.platform == 'win32': raises(ValueError, signal, 42, lambda *args: None) + raises(ValueError, signal, 7, lambda *args: None) else: signal(42, lambda *args: None) signal(42, SIG_DFL) diff --git a/pypy/module/thread/os_thread.py b/pypy/module/thread/os_thread.py --- a/pypy/module/thread/os_thread.py +++ b/pypy/module/thread/os_thread.py @@ -2,6 +2,7 @@ Thread support based on OS-level threads. """ +import os from pypy.module.thread import ll_thread as thread from pypy.module.thread.error import wrap_thread_error from pypy.interpreter.error import OperationError, operationerrfmt @@ -91,14 +92,18 @@ # run! try: bootstrapper.run(space, w_callable, args) - finally: - bootstrapper.nbthreads -= 1 - # clean up space.threadlocals to remove the ExecutionContext - # entry corresponding to the current thread + # done + except Exception, e: + # oups! last-level attempt to recover. try: - space.threadlocals.leave_thread(space) - finally: - thread.gc_thread_die() + STDERR = 2 + os.write(STDERR, "Thread exited with ") + os.write(STDERR, str(e)) + os.write(STDERR, "\n") + except OSError: + pass + bootstrapper.nbthreads -= 1 + thread.gc_thread_die() bootstrap = staticmethod(bootstrap) def acquire(space, w_callable, args): @@ -129,6 +134,9 @@ where = 'thread %d started by ' % ident e.write_unraisable(space, where, w_callable) e.clear(space) + # clean up space.threadlocals to remove the ExecutionContext + # entry corresponding to the current thread + space.threadlocals.leave_thread(space) run = staticmethod(run) bootstrapper = Bootstrapper() diff --git a/pypy/module/thread/test/test_gil.py b/pypy/module/thread/test/test_gil.py --- a/pypy/module/thread/test/test_gil.py +++ b/pypy/module/thread/test/test_gil.py @@ -61,8 +61,9 @@ def bootstrap(): try: runme() - finally: - thread.gc_thread_die() + except Exception, e: + assert 0 + thread.gc_thread_die() def f(): state.data = [] state.datalen1 = 0 diff --git a/pypy/objspace/flow/test/test_objspace.py b/pypy/objspace/flow/test/test_objspace.py --- a/pypy/objspace/flow/test/test_objspace.py +++ b/pypy/objspace/flow/test/test_objspace.py @@ -8,18 +8,17 @@ from pypy.objspace.flow.objspace import FlowObjSpace from pypy.objspace.flow.flowcontext import FlowingError, FlowSpaceFrame from pypy import conftest -from pypy.tool.stdlib_opcode import bytecode_spec, host_bytecode_spec +from pypy.tool.stdlib_opcode import host_bytecode_spec import os import operator is_operator = getattr(operator, 'is_', operator.eq) # it's not there 2.2 @contextmanager -def patching_opcodes(*opcodes): +def patching_opcodes(**opcodes): meth_names = host_bytecode_spec.method_names - opnums = [bytecode_spec.opmap[name] for name in opcodes] old_name = {} - for name, num in zip(opcodes, opnums): + for name, num in opcodes.items(): old_name[num] = meth_names[num] meth_names[num] = name yield @@ -905,7 +904,7 @@ """ Tests code generated by pypy-c compiled with CALL_METHOD bytecode """ - with patching_opcodes('CALL_METHOD', 'LOOKUP_METHOD'): + with patching_opcodes(CALL_METHOD=202, LOOKUP_METHOD=201): class X: def m(self): return 3 @@ -929,7 +928,7 @@ """ Tests code generated by pypy-c compiled with BUILD_LIST_FROM_ARG bytecode """ - with patching_opcodes('BUILD_LIST_FROM_ARG'): + with patching_opcodes(BUILD_LIST_FROM_ARG=203): def f(): return [i for i in "abc"] diff --git a/pypy/rlib/clibffi.py b/pypy/rlib/clibffi.py --- a/pypy/rlib/clibffi.py +++ b/pypy/rlib/clibffi.py @@ -421,6 +421,7 @@ hints={'callback':True})) + at jit.jit_callback("CLIBFFI") def ll_callback(ffi_cif, ll_res, ll_args, ll_userdata): """ Callback specification. ffi_cif - something ffi specific, don't care diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py --- a/pypy/rlib/jit.py +++ b/pypy/rlib/jit.py @@ -302,6 +302,32 @@ pass +def jit_callback(name): + """Use as a decorator for C callback functions, to insert a + jitdriver.jit_merge_point() at the start. Only for callbacks + that typically invoke more app-level Python code. + """ + def decorate(func): + from pypy.tool.sourcetools import compile2 + # + def get_printable_location(): + return name + jitdriver = JitDriver(get_printable_location=get_printable_location, + greens=[], reds='auto', name=name) + # + args = ','.join(['a%d' % i for i in range(func.func_code.co_argcount)]) + source = """def callback_with_jitdriver(%(args)s): + jitdriver.jit_merge_point() + return real_callback(%(args)s)""" % locals() + miniglobals = { + 'jitdriver': jitdriver, + 'real_callback': func, + } + exec compile2(source) in miniglobals + return miniglobals['callback_with_jitdriver'] + return decorate + + # ____________________________________________________________ # VRefs @@ -458,9 +484,8 @@ self.autoreds = True self.reds = [] self.numreds = None # see warmspot.autodetect_jit_markers_redvars - for hook in (get_jitcell_at, set_jitcell_at, get_printable_location, - confirm_enter_jit): - assert hook is None, "reds='auto' is not compatible with JitDriver hooks" + assert confirm_enter_jit is None, ( + "reds='auto' is not compatible with confirm_enter_jit") else: if reds is not None: self.reds = reds diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py --- a/pypy/rlib/objectmodel.py +++ b/pypy/rlib/objectmodel.py @@ -599,16 +599,6 @@ llhelper(rffi.AroundFnPtr, before) llhelper(rffi.AroundFnPtr, after) -def register_around_callback_hook(hook): - """ Register a hook that's called before a callback from C calls RPython. - Primary usage is for JIT to have 'started from' hook. - """ - from pypy.rpython.lltypesystem import rffi - from pypy.rpython.annlowlevel import llhelper - - rffi.aroundstate.callback_hook = hook - llhelper(rffi.CallbackHookPtr, hook) - def is_in_callback(): from pypy.rpython.lltypesystem import rffi return rffi.stackcounter.stacks_counter > 1 diff --git a/pypy/rlib/runicode.py b/pypy/rlib/runicode.py --- a/pypy/rlib/runicode.py +++ b/pypy/rlib/runicode.py @@ -460,6 +460,10 @@ errorhandler=None, byteorder='little'): if size == 0: + if byteorder == 'native': + result = StringBuilder(2) + _STORECHAR(result, 0xFEFF, BYTEORDER) + return result.build() return "" result = StringBuilder(size * 2 + 2) @@ -621,6 +625,10 @@ errorhandler=None, byteorder='little'): if size == 0: + if byteorder == 'native': + result = StringBuilder(4) + _STORECHAR32(result, 0xFEFF, BYTEORDER) + return result.build() return "" result = StringBuilder(size * 4 + 4) diff --git a/pypy/rlib/test/test_runicode.py b/pypy/rlib/test/test_runicode.py --- a/pypy/rlib/test/test_runicode.py +++ b/pypy/rlib/test/test_runicode.py @@ -679,6 +679,11 @@ "utf-32 utf-32-be utf-32-le").split(): self.checkencode(uni, encoding) + def test_empty(self): + for encoding in ("utf-8 utf-16 utf-16-be utf-16-le " + "utf-32 utf-32-be utf-32-le").split(): + self.checkencode(u'', encoding) + def test_single_chars_utf8(self): # check every number of bytes per char for s in ["\xd7\x90", "\xd6\x96", "\xeb\x96\x95", "\xf0\x90\x91\x93"]: diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py --- a/pypy/rpython/lltypesystem/rffi.py +++ b/pypy/rpython/lltypesystem/rffi.py @@ -1,19 +1,19 @@ import py from pypy.annotation import model as annmodel -from pypy.rpython.lltypesystem import lltype, rstr +from pypy.rpython.lltypesystem import lltype from pypy.rpython.lltypesystem import ll2ctypes -from pypy.rpython.lltypesystem.llmemory import cast_adr_to_ptr, cast_ptr_to_adr +from pypy.rpython.lltypesystem.llmemory import cast_ptr_to_adr from pypy.rpython.lltypesystem.llmemory import itemoffsetof, raw_memcopy from pypy.annotation.model import lltype_to_annotation from pypy.tool.sourcetools import func_with_new_name -from pypy.rlib.objectmodel import Symbolic, CDefinedIntSymbolic -from pypy.rlib.objectmodel import keepalive_until_here +from pypy.rlib.objectmodel import Symbolic +from pypy.rlib.objectmodel import keepalive_until_here, enforceargs from pypy.rlib import rarithmetic, rgc from pypy.rpython.extregistry import ExtRegistryEntry from pypy.rlib.unroll import unrolling_iterable from pypy.rpython.tool.rfficache import platform, sizeof_c_type from pypy.translator.tool.cbuild import ExternalCompilationInfo -from pypy.rpython.annlowlevel import llhelper, llstr +from pypy.rpython.annlowlevel import llhelper from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.rstring import StringBuilder, UnicodeBuilder, assert_str0 from pypy.rlib import jit @@ -279,17 +279,8 @@ callable_name = getattr(callable, '__name__', '?') if callbackholder is not None: callbackholder.callbacks[callable] = True - callable_name_descr = str(callable).replace('"', '\\"') args = ', '.join(['a%d' % i for i in range(len(TP.TO.ARGS))]) source = py.code.Source(r""" - def inner_wrapper(%(args)s): - if aroundstate is not None: - callback_hook = aroundstate.callback_hook - if callback_hook: - callback_hook(llstr("%(callable_name_descr)s")) - return callable(%(args)s) - inner_wrapper._never_inline_ = True - def wrapper(%(args)s): # no *args - no GIL for mallocing the tuple llop.gc_stack_bottom(lltype.Void) # marker for trackgcroot.py if aroundstate is not None: @@ -299,7 +290,7 @@ # from now on we hold the GIL stackcounter.stacks_counter += 1 try: - result = inner_wrapper(%(args)s) + result = callable(%(args)s) except Exception, e: os.write(2, "Warning: uncaught exception in callback: %%s %%s\n" %% @@ -321,7 +312,6 @@ miniglobals = locals().copy() miniglobals['Exception'] = Exception miniglobals['os'] = os - miniglobals['llstr'] = llstr miniglobals['we_are_translated'] = we_are_translated miniglobals['stackcounter'] = stackcounter exec source.compile() in miniglobals @@ -329,11 +319,8 @@ _make_wrapper_for._annspecialcase_ = 'specialize:memo' AroundFnPtr = lltype.Ptr(lltype.FuncType([], lltype.Void)) -CallbackHookPtr = lltype.Ptr(lltype.FuncType([lltype.Ptr(rstr.STR)], lltype.Void)) class AroundState: - callback_hook = None - def _cleanup_(self): self.before = None # or a regular RPython function self.after = None # or a regular RPython function @@ -795,6 +782,7 @@ # (char*, str, int, int) -> None @jit.dont_look_inside + @enforceargs(None, None, int, int) def str_from_buffer(raw_buf, gc_buf, allocated_size, needed_size): """ Converts from a pair returned by alloc_buffer to a high-level string. @@ -833,6 +821,7 @@ lltype.free(raw_buf, flavor='raw') # char* -> str, with an upper bound on the length in case there is no \x00 + @enforceargs(None, int) def charp2strn(cp, maxlen): b = builder_class(maxlen) i = 0 diff --git a/pypy/rpython/memory/gc/minimark.py b/pypy/rpython/memory/gc/minimark.py --- a/pypy/rpython/memory/gc/minimark.py +++ b/pypy/rpython/memory/gc/minimark.py @@ -2000,6 +2000,17 @@ (obj + offset).address[0] = llmemory.NULL continue # no need to remember this weakref any longer # + elif self.header(pointing_to).tid & GCFLAG_NO_HEAP_PTRS: + # see test_weakref_to_prebuilt: it's not useful to put + # weakrefs into 'old_objects_with_weakrefs' if they point + # to a prebuilt object (they are immortal). If moreover + # the 'pointing_to' prebuilt object still has the + # GCFLAG_NO_HEAP_PTRS flag, then it's even wrong, because + # 'pointing_to' will not get the GCFLAG_VISITED during + # the next major collection. Solve this by not registering + # the weakref into 'old_objects_with_weakrefs'. + continue + # self.old_objects_with_weakrefs.append(obj) def invalidate_old_weakrefs(self): @@ -2013,6 +2024,9 @@ continue # weakref itself dies offset = self.weakpointer_offset(self.get_type_id(obj)) pointing_to = (obj + offset).address[0] + ll_assert((self.header(pointing_to).tid & GCFLAG_NO_HEAP_PTRS) + == 0, "registered old weakref should not " + "point to a NO_HEAP_PTRS obj") if self.header(pointing_to).tid & GCFLAG_VISITED: new_with_weakref.append(obj) else: diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py --- a/pypy/rpython/memory/gctransform/framework.py +++ b/pypy/rpython/memory/gctransform/framework.py @@ -979,6 +979,7 @@ assert self.translator.config.translation.thread if hasattr(self.root_walker, 'thread_run_ptr'): livevars = self.push_roots(hop) + assert not livevars, "live GC var around %s!" % (hop.spaceop,) hop.genop("direct_call", [self.root_walker.thread_run_ptr]) self.pop_roots(hop, livevars) @@ -993,6 +994,7 @@ assert self.translator.config.translation.thread if hasattr(self.root_walker, 'thread_die_ptr'): livevars = self.push_roots(hop) + assert not livevars, "live GC var around %s!" % (hop.spaceop,) hop.genop("direct_call", [self.root_walker.thread_die_ptr]) self.pop_roots(hop, livevars) diff --git a/pypy/rpython/memory/gctransform/shadowstack.py b/pypy/rpython/memory/gctransform/shadowstack.py --- a/pypy/rpython/memory/gctransform/shadowstack.py +++ b/pypy/rpython/memory/gctransform/shadowstack.py @@ -326,6 +326,8 @@ self.gcdata.root_stack_top = llmemory.NULL # to detect missing restore def forget_current_state(self): + ll_assert(self.gcdata.root_stack_base == self.gcdata.root_stack_top, + "forget_current_state: shadowstack not empty!") if self.unused_full_stack: llmemory.raw_free(self.unused_full_stack) self.unused_full_stack = self.gcdata.root_stack_base diff --git a/pypy/rpython/memory/test/test_gc.py b/pypy/rpython/memory/test/test_gc.py --- a/pypy/rpython/memory/test/test_gc.py +++ b/pypy/rpython/memory/test/test_gc.py @@ -557,6 +557,19 @@ res = self.interpret(f, [20]) # for GenerationGC, enough for a minor collection assert res == True + def test_weakref_to_prebuilt(self): + import weakref + class A: + pass + a = A() + def f(x): + ref = weakref.ref(a) + assert ref() is a + llop.gc__collect(lltype.Void) + return ref() is a + res = self.interpret(f, [20]) # for GenerationGC, enough for a minor collection + assert res == True + def test_many_weakrefs(self): # test for the case where allocating the weakref itself triggers # a collection diff --git a/pypy/translator/driver.py b/pypy/translator/driver.py --- a/pypy/translator/driver.py +++ b/pypy/translator/driver.py @@ -362,7 +362,7 @@ @taskdef([RTYPE], "JIT compiler generation") def task_pyjitpl_lltype(self): """ Generate bytecodes for JIT and flow the JIT helper functions - ootype version + lltype version """ get_policy = self.extra['jitpolicy'] self.jitpolicy = get_policy(self) diff --git a/pypy/translator/goal/app_main.py b/pypy/translator/goal/app_main.py --- a/pypy/translator/goal/app_main.py +++ b/pypy/translator/goal/app_main.py @@ -599,7 +599,12 @@ python_startup, 'exec') exec_(co_python_startup, mainmodule.__dict__) + mainmodule.__file__ = python_startup run_toplevel(run_it) + try: + del mainmodule.__file__ + except (AttributeError, TypeError): + pass # Then we need a prompt. inspect = True else: diff --git a/pypy/translator/goal/targetpypystandalone.py b/pypy/translator/goal/targetpypystandalone.py --- a/pypy/translator/goal/targetpypystandalone.py +++ b/pypy/translator/goal/targetpypystandalone.py @@ -28,11 +28,6 @@ w_call_startup_gateway = space.wrap(gateway.interp2app(call_startup)) withjit = space.config.objspace.usemodules.pypyjit - if withjit: - from pypy.module.pypyjit.interp_jit import callback_hook - from pypy.rlib import objectmodel - objectmodel.register_around_callback_hook(callback_hook) - def entry_point(argv): if withjit: from pypy.jit.backend.hlinfo import highleveljitinfo diff --git a/pypy/translator/goal/test2/test_app_main.py b/pypy/translator/goal/test2/test_app_main.py --- a/pypy/translator/goal/test2/test_app_main.py +++ b/pypy/translator/goal/test2/test_app_main.py @@ -398,6 +398,30 @@ child.expect('Traceback') child.expect('NameError') + def test_pythonstartup_file1(self, monkeypatch): + monkeypatch.setenv('PYTHONPATH', None) + monkeypatch.setenv('PYTHONSTARTUP', demo_script) + child = self.spawn([]) + child.expect('File: [^\n]+\.py') + child.expect('goodbye') + child.expect('>>> ') + child.sendline('[myvalue]') + child.expect(re.escape('[42]')) + child.expect('>>> ') + child.sendline('__file__') + child.expect('Traceback') + child.expect('NameError') + + def test_pythonstartup_file2(self, monkeypatch): + monkeypatch.setenv('PYTHONPATH', None) + monkeypatch.setenv('PYTHONSTARTUP', crashing_demo_script) + child = self.spawn([]) + child.expect('Traceback') + child.expect('>>> ') + child.sendline('__file__') + child.expect('Traceback') + child.expect('NameError') + def test_ignore_python_startup(self, crashing_demo_script): old = os.environ.get('PYTHONSTARTUP', '') try: diff --git a/pypy/translator/platform/linux.py b/pypy/translator/platform/linux.py --- a/pypy/translator/platform/linux.py +++ b/pypy/translator/platform/linux.py @@ -36,7 +36,13 @@ # places where we need to look for libffi.a # XXX obscuuure! only look for libffi.a if run with translate.py if 'translate' in sys.modules: - return self.library_dirs_for_libffi() + ['/usr/lib'] + if sys.maxint > 2**32: + host = 'x86_64' + else: + host = 'x86' + return self.library_dirs_for_libffi() + [ + '/usr/lib', + '/usr/lib/%s-linux-gnu/' % host] else: return [] diff --git a/pypy/translator/platform/test/test_distutils.py b/pypy/translator/platform/test/test_distutils.py --- a/pypy/translator/platform/test/test_distutils.py +++ b/pypy/translator/platform/test/test_distutils.py @@ -8,3 +8,6 @@ def test_nice_errors(self): py.test.skip("Unsupported") + + def test_900_files(self): + py.test.skip('Makefiles not suppoerted') diff --git a/pypy/translator/platform/test/test_platform.py b/pypy/translator/platform/test/test_platform.py --- a/pypy/translator/platform/test/test_platform.py +++ b/pypy/translator/platform/test/test_platform.py @@ -59,6 +59,34 @@ res = self.platform.execute(executable) self.check_res(res) + def test_900_files(self): + txt = '#include \n' + for i in range(900): + txt += 'int func%03d();\n' % i From noreply at buildbot.pypy.org Thu Jan 17 01:10:26 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Thu, 17 Jan 2013 01:10:26 +0100 (CET) Subject: [pypy-commit] pypy py3k: adapt our distutils changes from 2.7.3 Message-ID: <20130117001026.326161C1379@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60111:4b8c42f02d1c Date: 2013-01-16 15:05 -0800 http://bitbucket.org/pypy/pypy/changeset/4b8c42f02d1c/ Log: adapt our distutils changes from 2.7.3 diff --git a/lib-python/3.2/distutils/command/bdist_wininst.py b/lib-python/3.2/distutils/command/bdist_wininst.py --- a/lib-python/3.2/distutils/command/bdist_wininst.py +++ b/lib-python/3.2/distutils/command/bdist_wininst.py @@ -287,7 +287,8 @@ bitmaplen, # number of bytes in bitmap ) file.write(header) - file.write(open(arcname, "rb").read()) + with open(arcname, "rb") as arcfile: + file.write(arcfile.read()) def get_installer_filename(self, fullname): # Factored out to allow overriding in subclasses diff --git a/lib-python/3.2/distutils/command/build_ext.py b/lib-python/3.2/distutils/command/build_ext.py --- a/lib-python/3.2/distutils/command/build_ext.py +++ b/lib-python/3.2/distutils/command/build_ext.py @@ -188,7 +188,7 @@ # the 'libs' directory is for binary installs - we assume that # must be the *native* platform. But we don't really support # cross-compiling via a binary install anyway, so we let it go. - self.library_dirs.append(os.path.join(sys.exec_prefix, 'libs')) + self.library_dirs.append(os.path.join(sys.exec_prefix, 'include')) if self.debug: self.build_temp = os.path.join(self.build_temp, "Debug") else: @@ -196,8 +196,13 @@ # Append the source distribution include and library directories, # this allows distutils on windows to work in the source tree - self.include_dirs.append(os.path.join(sys.exec_prefix, 'PC')) - if MSVC_VERSION == 9: + if 0: + # pypy has no PC directory + self.include_dirs.append(os.path.join(sys.exec_prefix, 'PC')) + if 1: + # pypy has no PCBuild directory + pass + elif MSVC_VERSION == 9: # Use the .lib files for the correct architecture if self.plat_name == 'win32': suffix = '' @@ -688,24 +693,17 @@ shared extension. On most platforms, this is just 'ext.libraries'; on Windows and OS/2, we add the Python library (eg. python20.dll). """ - # The python library is always needed on Windows. For MSVC, this - # is redundant, since the library is mentioned in a pragma in - # pyconfig.h that MSVC groks. The other Windows compilers all seem - # to need it mentioned explicitly, though, so that's what we do. - # Append '_d' to the python import library on debug builds. + # For PyPy, we must not add any such Python library, on any platform + if "__pypy__" in sys.builtin_module_names: + return ext.libraries + # The python library is always needed on Windows. if sys.platform == "win32": - from distutils.msvccompiler import MSVCCompiler - if not isinstance(self.compiler, MSVCCompiler): - template = "python%d%d" - if self.debug: - template = template + '_d' - pythonlib = (template % - (sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff)) - # don't extend ext.libraries, it may be shared with other - # extensions, it is a reference to the original list - return ext.libraries + [pythonlib] - else: - return ext.libraries + template = "python%d%d" + pythonlib = (template % + (sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff)) + # don't extend ext.libraries, it may be shared with other + # extensions, it is a reference to the original list + return ext.libraries + [pythonlib] elif sys.platform == "os2emx": # EMX/GCC requires the python library explicitly, and I # believe VACPP does as well (though not confirmed) - AIM Apr01 diff --git a/lib-python/3.2/distutils/command/install.py b/lib-python/3.2/distutils/command/install.py --- a/lib-python/3.2/distutils/command/install.py +++ b/lib-python/3.2/distutils/command/install.py @@ -65,6 +65,13 @@ 'scripts': '$base/Scripts', 'data' : '$base', }, + 'pypy': { + 'purelib': '$base/site-packages', + 'platlib': '$base/site-packages', + 'headers': '$base/include', + 'scripts': '$base/bin', + 'data' : '$base', + }, } # user site schemes @@ -482,6 +489,8 @@ def select_scheme(self, name): """Sets the install directories by applying the install schemes.""" # it's the caller's problem if they supply a bad name! + if hasattr(sys, 'pypy_version_info'): + name = 'pypy' scheme = INSTALL_SCHEMES[name] for key in SCHEME_KEYS: attrname = 'install_' + key diff --git a/lib-python/3.2/distutils/cygwinccompiler.py b/lib-python/3.2/distutils/cygwinccompiler.py --- a/lib-python/3.2/distutils/cygwinccompiler.py +++ b/lib-python/3.2/distutils/cygwinccompiler.py @@ -78,6 +78,9 @@ elif msc_ver == '1500': # VS2008 / MSVC 9.0 return ['msvcr90'] + elif msc_ver == '1600': + # VS2010 / MSVC 10.0 + return ['msvcr100'] else: raise ValueError("Unknown MS Compiler version %s " % msc_ver) diff --git a/lib-python/3.2/distutils/msvc9compiler.py b/lib-python/3.2/distutils/msvc9compiler.py --- a/lib-python/3.2/distutils/msvc9compiler.py +++ b/lib-python/3.2/distutils/msvc9compiler.py @@ -667,6 +667,7 @@ temp_manifest = os.path.join( build_temp, os.path.basename(output_filename) + ".manifest") + ld_args.append('/MANIFEST') ld_args.append('/MANIFESTFILE:' + temp_manifest) def manifest_get_embed_info(self, target_desc, ld_args): diff --git a/lib-python/3.2/distutils/spawn.py b/lib-python/3.2/distutils/spawn.py --- a/lib-python/3.2/distutils/spawn.py +++ b/lib-python/3.2/distutils/spawn.py @@ -56,7 +56,6 @@ def _spawn_nt(cmd, search_path=1, verbose=0, dry_run=0): executable = cmd[0] - cmd = _nt_quote_args(cmd) if search_path: # either we find one or it stays the same executable = find_executable(executable) or executable @@ -64,7 +63,8 @@ if not dry_run: # spawn for NT requires a full path to the .exe try: - rc = os.spawnv(os.P_WAIT, executable, cmd) + import subprocess + rc = subprocess.call(cmd) except OSError as exc: # this seems to happen when the command isn't found raise DistutilsExecError( diff --git a/lib-python/3.2/distutils/tests/test_install.py b/lib-python/3.2/distutils/tests/test_install.py --- a/lib-python/3.2/distutils/tests/test_install.py +++ b/lib-python/3.2/distutils/tests/test_install.py @@ -6,7 +6,7 @@ import unittest import site -from test.support import captured_stdout, run_unittest +from test.support import captured_stdout, check_impl_detail, run_unittest from distutils import sysconfig from distutils.command.install import install @@ -58,14 +58,15 @@ expected = os.path.normpath(expected) self.assertEqual(got, expected) - libdir = os.path.join(destination, "lib", "python") - check_path(cmd.install_lib, libdir) - check_path(cmd.install_platlib, libdir) - check_path(cmd.install_purelib, libdir) - check_path(cmd.install_headers, - os.path.join(destination, "include", "python", "foopkg")) - check_path(cmd.install_scripts, os.path.join(destination, "bin")) - check_path(cmd.install_data, destination) + if check_impl_detail(): + libdir = os.path.join(destination, "lib", "python") + check_path(cmd.install_lib, libdir) + check_path(cmd.install_platlib, libdir) + check_path(cmd.install_purelib, libdir) + check_path(cmd.install_headers, + os.path.join(destination, "include", "python", "foopkg")) + check_path(cmd.install_scripts, os.path.join(destination, "bin")) + check_path(cmd.install_data, destination) def test_user_site(self): # test install with --user diff --git a/lib-python/3.2/distutils/unixccompiler.py b/lib-python/3.2/distutils/unixccompiler.py --- a/lib-python/3.2/distutils/unixccompiler.py +++ b/lib-python/3.2/distutils/unixccompiler.py @@ -122,7 +122,22 @@ } if sys.platform[:6] == "darwin": + import platform + if platform.machine() == 'i386': + if platform.architecture()[0] == '32bit': + arch = 'i386' + else: + arch = 'x86_64' + else: + # just a guess + arch = platform.machine() executables['ranlib'] = ["ranlib"] + executables['linker_so'] += ['-undefined', 'dynamic_lookup'] + + for k, v in executables.iteritems(): + if v and v[0] == 'cc': + v += ['-arch', arch] + # Needed for the filename generation methods provided by the base # class, CCompiler. NB. whoever instantiates/uses a particular @@ -319,7 +334,7 @@ # On OSX users can specify an alternate SDK using # '-isysroot', calculate the SDK root if it is specified # (and use it further on) - cflags = sysconfig.get_config_var('CFLAGS') + cflags = sysconfig.get_config_var('CFLAGS') or '' m = re.search(r'-isysroot\s+(\S+)', cflags) if m is None: sysroot = '/' From noreply at buildbot.pypy.org Thu Jan 17 01:10:27 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Thu, 17 Jan 2013 01:10:27 +0100 (CET) Subject: [pypy-commit] pypy py3k: fix some tests under -A and add some new failures Message-ID: <20130117001027.5DD9F1C1379@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60112:c1245fc2be55 Date: 2013-01-16 16:04 -0800 http://bitbucket.org/pypy/pypy/changeset/c1245fc2be55/ Log: fix some tests under -A and add some new failures diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -371,9 +371,14 @@ for (i, line) in enumerate(reader): assert line == s[i] - def test_array(self): + def test_readbuffer_encode(self): + import _codecs + assert _codecs.readbuffer_encode("") == (b"", 0) + + def test_readbuffer_encode_array(self): import _codecs, array - _codecs.readbuffer_encode(array.array('b', b'spam')) == ('spam', 4) + assert (_codecs.readbuffer_encode(array.array('b', b'spam')) == + (b'spam', 4)) def test_utf8sig(self): import codecs @@ -381,6 +386,20 @@ s = "spam" assert d.decode(s.encode("utf-8-sig")) == s + def test_decoder_state(self): + import codecs + encoding = 'utf16' + u = 'abc123' + s = u.encode(encoding) + for i in range(len(u) + 1): + d = codecs.getincrementalencoder(encoding)() + part1 = d.encode(u[:i]) + state = d.getstate() + d = codecs.getincrementalencoder(encoding)() + d.setstate(state) + part2 = d.encode(u[i:], True) + assert s == part1 + part2 + def test_escape_decode_escaped_newline(self): import _codecs s = b'\\\n' @@ -628,17 +647,19 @@ assert codecs.getdecoder('utf-16')( b'\xff\xfe1\x002\x003\x00a\x00b\x00c\x00') == (x, 14) - def test_unicode_escape(self): + def test_unicode_escape(self): + import _codecs assert '\\'.encode('unicode-escape') == b'\\\\' assert b'\\\\'.decode('unicode-escape') == '\\' assert '\ud801'.encode('unicode-escape') == b'\\ud801' assert '\u0013'.encode('unicode-escape') == b'\\x13' + assert _codecs.unicode_escape_decode(r"\u1234") == ("\u1234", 6) def test_mbcs(self): import sys if sys.platform != 'win32': return - assert 'test'.encode('mbcs') == 'test' - assert 'caf\xe9'.encode('mbcs') == 'caf\xe9' - assert '\u040a'.encode('mbcs') == '?' # some cyrillic letter - assert 'cafx\e9'.decode('mbcs') == 'cafx\e9' + assert 'test'.encode('mbcs') == b'test' + assert 'caf\xe9'.encode('mbcs') == b'caf\xe9' + assert '\u040a'.encode('mbcs') == b'?' # some cyrillic letter + assert 'cafx\e9'.decode('mbcs') == b'cafx\e9' From noreply at buildbot.pypy.org Thu Jan 17 01:10:28 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Thu, 17 Jan 2013 01:10:28 +0100 (CET) Subject: [pypy-commit] pypy py3k: o add a bufferstr_or_u_w that acts like the 2.x bufferstr_w (accepts unicode and Message-ID: <20130117001028.91F611C1379@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60113:b67edd58b7c1 Date: 2013-01-16 16:08 -0800 http://bitbucket.org/pypy/pypy/changeset/b67edd58b7c1/ Log: o add a bufferstr_or_u_w that acts like the 2.x bufferstr_w (accepts unicode and returns it encoded to the default encoding) mostly for use by codecs o fix readbuffer_encode to return bytes diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -1298,6 +1298,17 @@ buffer = self.buffer_w(w_obj) return buffer.as_str() + def bufferstr_or_u_w(self, w_obj): + """Returns an interp-level str, directly if possible. + + Accepts unicode or any type supporting the buffer + interface. Unicode objects will be encoded to the default + encoding (UTF-8) + """ + if self.isinstance_w(w_obj, self.w_unicode): + return w_obj.identifier_w(self) + return self.bufferstr_w(w_obj) + def str_or_None_w(self, w_obj): if self.is_w(w_obj, self.w_None): return None diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py --- a/pypy/interpreter/gateway.py +++ b/pypy/interpreter/gateway.py @@ -125,6 +125,9 @@ def visit_bufferstr(self, el, app_sig): self.checked_space_method(el, app_sig) + def visit_bufferstr_or_u(self, el, app_sig): + self.checked_space_method(el, app_sig) + def visit_str_or_None(self, el, app_sig): self.checked_space_method(el, app_sig) @@ -243,6 +246,9 @@ def visit_bufferstr(self, typ): self.run_args.append("space.bufferstr_w(%s)" % (self.scopenext(),)) + def visit_bufferstr_or_u(self, typ): + self.run_args.append("space.bufferstr_or_u_w(%s)" % (self.scopenext(),)) + def visit_str_or_None(self, typ): self.run_args.append("space.str_or_None_w(%s)" % (self.scopenext(),)) @@ -380,6 +386,9 @@ def visit_bufferstr(self, typ): self.unwrap.append("space.bufferstr_w(%s)" % (self.nextarg(),)) + def visit_bufferstr_or_u(self, typ): + self.unwrap.append("space.bufferstr_or_u_w(%s)" % (self.nextarg(),)) + def visit_str_or_None(self, typ): self.unwrap.append("space.str_or_None_w(%s)" % (self.nextarg(),)) diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py --- a/pypy/module/_codecs/interp_codecs.py +++ b/pypy/module/_codecs/interp_codecs.py @@ -402,9 +402,9 @@ w_res = space.call_function(w_encoder, w_obj, space.wrap(errors)) return space.getitem(w_res, space.wrap(0)) - at unwrap_spec(s='bufferstr', errors='str_or_None') + at unwrap_spec(s='bufferstr_or_u', errors='str_or_None') def buffer_encode(space, s, errors='strict'): - return space.newtuple([space.wrap(s), space.wrap(len(s))]) + return space.newtuple([space.wrapbytes(s), space.wrap(len(s))]) @unwrap_spec(errors=str) def decode(space, w_obj, w_encoding=None, errors='strict'): @@ -749,7 +749,7 @@ return -1 return space.int_w(w_code) - at unwrap_spec(string='bufferstr', errors='str_or_None', + at unwrap_spec(string='bufferstr_or_u', errors='str_or_None', w_final=WrappedDefault(False)) def unicode_escape_decode(space, string, errors="strict", w_final=None): if errors is None: @@ -799,7 +799,7 @@ result = string_escape_encode(data, False) return space.newtuple([space.wrapbytes(result), space.wrap(len(data))]) - at unwrap_spec(data="bufferstr", errors='str_or_None') + at unwrap_spec(data='bufferstr_or_u', errors='str_or_None') def escape_decode(space, data, errors='strict'): from pypy.interpreter.pyparser.parsestring import PyString_DecodeEscape result = PyString_DecodeEscape(space, data, None) From noreply at buildbot.pypy.org Thu Jan 17 01:10:29 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Thu, 17 Jan 2013 01:10:29 +0100 (CET) Subject: [pypy-commit] pypy py3k: improve bytearray's starts/endswith error messages too Message-ID: <20130117001029.D9BD71C1379@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60114:d50054f9d114 Date: 2013-01-16 16:09 -0800 http://bitbucket.org/pypy/pypy/changeset/d50054f9d114/ Log: improve bytearray's starts/endswith error messages too diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py --- a/pypy/objspace/std/bytearrayobject.py +++ b/pypy/objspace/std/bytearrayobject.py @@ -261,8 +261,18 @@ return stringobject.str_rfind__String_String_ANY_ANY(space, w_str, w_char, w_start, w_stop) +def _suffix_to_str(space, w_suffix, funcname): + try: + return space.bufferstr_new_w(w_suffix) + except OperationError as e: + if e.match(space, space.w_TypeError): + msg = ("%s first arg must be bytes or a tuple of bytes, " + "not %s") + typename = space.type(w_suffix).getname(space) + raise operationerrfmt(space.w_TypeError, msg, funcname, typename) + def str_startswith__Bytearray_ANY_ANY_ANY(space, w_bytearray, w_prefix, w_start, w_stop): - w_prefix = space.wrapbytes(space.bufferstr_new_w(w_prefix)) + w_prefix = space.wrapbytes(_suffix_to_str(space, w_prefix, 'startswith')) w_str = _to_bytes(space, w_bytearray) return stringobject.str_startswith__String_String_ANY_ANY(space, w_str, w_prefix, w_start, w_stop) @@ -275,7 +285,7 @@ w_start, w_stop) def str_endswith__Bytearray_ANY_ANY_ANY(space, w_bytearray, w_suffix, w_start, w_stop): - w_suffix = space.wrapbytes(space.bufferstr_new_w(w_suffix)) + w_suffix = space.wrapbytes(_suffix_to_str(space, w_suffix, 'endswith')) w_str = _to_bytes(space, w_bytearray) return stringobject.str_endswith__String_String_ANY_ANY(space, w_str, w_suffix, w_start, w_stop) diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py --- a/pypy/objspace/std/test/test_bytearrayobject.py +++ b/pypy/objspace/std/test/test_bytearrayobject.py @@ -183,6 +183,18 @@ assert bytearray(b'hello').endswith(b'lo') assert bytearray(b'hello').endswith(bytearray(b'lo')) assert bytearray(b'hello').endswith((bytearray(b'lo'), b'he')) + try: + bytearray(b'hello').startswith([b'o']) + except TypeError as e: + assert 'bytes' in str(e) + else: + assert False, 'Expected TypeError' + try: + bytearray(b'hello').endswith([b'o']) + except TypeError as e: + assert 'bytes' in str(e) + else: + assert False, 'Expected TypeError' def test_stringlike_conversions(self): # methods that should return bytearray (and not str) From noreply at buildbot.pypy.org Thu Jan 17 03:03:00 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Thu, 17 Jan 2013 03:03:00 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: hg merge default Message-ID: <20130117020300.58CEB1C1382@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60115:daaeb86c9502 Date: 2013-01-17 00:52 +0100 http://bitbucket.org/pypy/pypy/changeset/daaeb86c9502/ Log: hg merge default diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py --- a/rpython/rlib/runicode.py +++ b/rpython/rlib/runicode.py @@ -460,6 +460,10 @@ errorhandler=None, byteorder='little'): if size == 0: + if byteorder == 'native': + result = StringBuilder(2) + _STORECHAR(result, 0xFEFF, BYTEORDER) + return result.build() return "" result = StringBuilder(size * 2 + 2) @@ -621,6 +625,10 @@ errorhandler=None, byteorder='little'): if size == 0: + if byteorder == 'native': + result = StringBuilder(4) + _STORECHAR32(result, 0xFEFF, BYTEORDER) + return result.build() return "" result = StringBuilder(size * 4 + 4) diff --git a/rpython/rlib/test/test_runicode.py b/rpython/rlib/test/test_runicode.py --- a/rpython/rlib/test/test_runicode.py +++ b/rpython/rlib/test/test_runicode.py @@ -679,6 +679,11 @@ "utf-32 utf-32-be utf-32-le").split(): self.checkencode(uni, encoding) + def test_empty(self): + for encoding in ("utf-8 utf-16 utf-16-be utf-16-le " + "utf-32 utf-32-be utf-32-le").split(): + self.checkencode(u'', encoding) + def test_single_chars_utf8(self): # check every number of bytes per char for s in ["\xd7\x90", "\xd6\x96", "\xeb\x96\x95", "\xf0\x90\x91\x93"]: From noreply at buildbot.pypy.org Thu Jan 17 03:03:01 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Thu, 17 Jan 2013 03:03:01 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: wtf did i really write rpython.interpreter? Message-ID: <20130117020301.C43781C1382@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60116:aef9d5edf02e Date: 2013-01-17 02:56 +0100 http://bitbucket.org/pypy/pypy/changeset/aef9d5edf02e/ Log: wtf did i really write rpython.interpreter? diff --git a/pypy/bin/pyinteractive.py b/pypy/bin/pyinteractive.py --- a/pypy/bin/pyinteractive.py +++ b/pypy/bin/pyinteractive.py @@ -14,7 +14,7 @@ import pypy from pypy.tool import option from optparse import make_option -from rpython.interpreter import main, interactive, error, gateway +from pypy.interpreter import main, interactive, error, gateway from rpython.config.config import OptionDescription, BoolOption, StrOption from rpython.config.config import Config, to_optparse from pypy.config import pypyoption From noreply at buildbot.pypy.org Thu Jan 17 04:08:56 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Thu, 17 Jan 2013 04:08:56 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Moved LeakFinder from pypy.tool.pytest.plugins to rpython.conftest Message-ID: <20130117030856.422E81C1379@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60117:05d23aa0cf1c Date: 2013-01-17 03:52 +0100 http://bitbucket.org/pypy/pypy/changeset/05d23aa0cf1c/ Log: Moved LeakFinder from pypy.tool.pytest.plugins to rpython.conftest diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -25,12 +25,6 @@ def pytest_report_header(): return "pytest-%s from %s" %(pytest.__version__, pytest.__file__) - -def pytest_addhooks(pluginmanager): - from pypy.tool.pytest.plugins import LeakFinder - pluginmanager.register(LeakFinder()) - - def pytest_configure(config): global option option = config.option diff --git a/pypy/tool/pytest/plugins.py b/pypy/tool/pytest/plugins.py deleted file mode 100644 --- a/pypy/tool/pytest/plugins.py +++ /dev/null @@ -1,36 +0,0 @@ -# pytest hooks, installed by pypy.conftest. - -import py -from rpython.tool import leakfinder - -class LeakFinder: - """Track memory allocations during test execution. - - So far, only used by the function lltype.malloc(flavor='raw'). - """ - def pytest_runtest_setup(self, __multicall__, item): - __multicall__.execute() - if not isinstance(item, py.test.collect.Function): - return - if not getattr(item.obj, 'dont_track_allocations', False): - leakfinder.start_tracking_allocations() - - def pytest_runtest_call(self, __multicall__, item): - __multicall__.execute() - if not isinstance(item, py.test.collect.Function): - return - item._success = True - - def pytest_runtest_teardown(self, __multicall__, item): - __multicall__.execute() - if not isinstance(item, py.test.collect.Function): - return - if (not getattr(item.obj, 'dont_track_allocations', False) - and leakfinder.TRACK_ALLOCATIONS): - item._pypytest_leaks = leakfinder.stop_tracking_allocations(False) - else: # stop_tracking_allocations() already called - item._pypytest_leaks = None - - # check for leaks, but only if the test passed so far - if getattr(item, '_success', False) and item._pypytest_leaks: - raise leakfinder.MallocMismatch(item._pypytest_leaks) diff --git a/rpython/conftest.py b/rpython/conftest.py --- a/rpython/conftest.py +++ b/rpython/conftest.py @@ -40,3 +40,38 @@ group.addoption('--viewloops', action="store_true", default=False, dest="viewloops", help="show only the compiled loops") + +def pytest_addhooks(pluginmanager): + pluginmanager.register(LeakFinder()) + +class LeakFinder: + """Track memory allocations during test execution. + + So far, only used by the function lltype.malloc(flavor='raw'). + """ + def pytest_runtest_setup(self, __multicall__, item): + __multicall__.execute() + if not isinstance(item, py.test.collect.Function): + return + if not getattr(item.obj, 'dont_track_allocations', False): + leakfinder.start_tracking_allocations() + + def pytest_runtest_call(self, __multicall__, item): + __multicall__.execute() + if not isinstance(item, py.test.collect.Function): + return + item._success = True + + def pytest_runtest_teardown(self, __multicall__, item): + __multicall__.execute() + if not isinstance(item, py.test.collect.Function): + return + if (not getattr(item.obj, 'dont_track_allocations', False) + and leakfinder.TRACK_ALLOCATIONS): + item._pypytest_leaks = leakfinder.stop_tracking_allocations(False) + else: # stop_tracking_allocations() already called + item._pypytest_leaks = None + + # check for leaks, but only if the test passed so far + if getattr(item, '_success', False) and item._pypytest_leaks: + raise leakfinder.MallocMismatch(item._pypytest_leaks) From noreply at buildbot.pypy.org Thu Jan 17 04:08:57 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Thu, 17 Jan 2013 04:08:57 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed test Message-ID: <20130117030857.9BB221C1379@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60118:b6b2fe14b3dc Date: 2013-01-17 04:08 +0100 http://bitbucket.org/pypy/pypy/changeset/b6b2fe14b3dc/ Log: Fixed test diff --git a/rpython/translator/c/test/test_genc.py b/rpython/translator/c/test/test_genc.py --- a/rpython/translator/c/test/test_genc.py +++ b/rpython/translator/c/test/test_genc.py @@ -560,7 +560,7 @@ t.context._graphof(foobar_fn).inhibit_tail_call = True t.source_c() lines = t.driver.cbuilder.c_source_filename.join('..', - 'pypy_translator_c_test_test_genc.c').readlines() + 'rpython_translator_c_test_test_genc.c').readlines() for i, line in enumerate(lines): if '= pypy_g_foobar_fn' in line: break From noreply at buildbot.pypy.org Thu Jan 17 04:31:51 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Thu, 17 Jan 2013 04:31:51 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed missing import Message-ID: <20130117033151.D25551C1382@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60119:ee0334430e03 Date: 2013-01-17 04:31 +0100 http://bitbucket.org/pypy/pypy/changeset/ee0334430e03/ Log: Fixed missing import diff --git a/rpython/conftest.py b/rpython/conftest.py --- a/rpython/conftest.py +++ b/rpython/conftest.py @@ -1,5 +1,6 @@ from os.path import * import py, pytest +from rpython.tool import leakfinder cdir = realpath(join(dirname(__file__), 'translator', 'c')) cache_dir = realpath(join(dirname(__file__), '_cache')) From noreply at buildbot.pypy.org Thu Jan 17 05:18:40 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Thu, 17 Jan 2013 05:18:40 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Added Leakfinder to pypy.conftest too Message-ID: <20130117041840.686651C137A@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60120:045d0377e690 Date: 2013-01-17 05:17 +0100 http://bitbucket.org/pypy/pypy/changeset/045d0377e690/ Log: Added Leakfinder to pypy.conftest too diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -25,6 +25,10 @@ def pytest_report_header(): return "pytest-%s from %s" %(pytest.__version__, pytest.__file__) +def pytest_addhooks(pluginmanager): + from rpython.conftest import LeakFinder + pluginmanager.register(LeakFinder()) + def pytest_configure(config): global option option = config.option From noreply at buildbot.pypy.org Thu Jan 17 05:59:40 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Thu, 17 Jan 2013 05:59:40 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed isolate_slave Message-ID: <20130117045940.870E31C137B@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60121:cd43dc8bcaae Date: 2013-01-17 05:59 +0100 http://bitbucket.org/pypy/pypy/changeset/cd43dc8bcaae/ Log: Fixed isolate_slave diff --git a/pypy/tool/isolate_slave.py b/pypy/tool/isolate_slave.py --- a/pypy/tool/isolate_slave.py +++ b/pypy/tool/isolate_slave.py @@ -1,4 +1,4 @@ -import sys, imp +import sys, os, imp sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..'))) from pypy.tool import slaveproc From noreply at buildbot.pypy.org Thu Jan 17 06:12:05 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Thu, 17 Jan 2013 06:12:05 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Removed unused code in test_rarithmetic which created errors Message-ID: <20130117051205.13C841C1379@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60122:7b430de28490 Date: 2013-01-17 06:11 +0100 http://bitbucket.org/pypy/pypy/changeset/7b430de28490/ Log: Removed unused code in test_rarithmetic which created errors diff --git a/rpython/rlib/test/test_rarithmetic.py b/rpython/rlib/test/test_rarithmetic.py --- a/rpython/rlib/test/test_rarithmetic.py +++ b/rpython/rlib/test/test_rarithmetic.py @@ -16,10 +16,6 @@ class Test_r_int: - - def setup_method(self,method): - space = self.space - def test__add__(self): self.binary_test(lambda x, y: x + y) def test__sub__(self): @@ -73,10 +69,6 @@ assert res == cmp class Test_r_uint: - - def setup_method(self,method): - space = self.space - def test__add__(self): self.binary_test(lambda x, y: x + y) def test__sub__(self): From noreply at buildbot.pypy.org Thu Jan 17 07:00:47 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Thu, 17 Jan 2013 07:00:47 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed new generated files names Message-ID: <20130117060047.9CFA01C12FA@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60123:6e43b9654ca9 Date: 2013-01-17 06:58 +0100 http://bitbucket.org/pypy/pypy/changeset/6e43b9654ca9/ Log: Fixed new generated files names diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py --- a/rpython/translator/c/test/test_standalone.py +++ b/rpython/translator/c/test/test_standalone.py @@ -57,9 +57,9 @@ # Verify that the generated C files have sane names: gen_c_files = [str(f) for f in cbuilder.extrafiles] - for expfile in ('pypy_rlib_rposix.c', - 'pypy_rpython_lltypesystem_rstr.c', - 'pypy_translator_c_test_test_standalone.c'): + for expfile in ('rpython_rlib_rposix.c', + 'rpython_rtyper_lltypesystem_rstr.c', + 'rpython_translator_c_test_test_standalone.c'): assert cbuilder.targetdir.join(expfile) in gen_c_files def test_print(self): From noreply at buildbot.pypy.org Thu Jan 17 07:44:13 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Thu, 17 Jan 2013 07:44:13 +0100 (CET) Subject: [pypy-commit] pypy default: Reorganize where the dont_look_inside is to push it to a lower level. Doesn't affect anything yet because the JIT never looks in here. Message-ID: <20130117064413.94A891C1167@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: Changeset: r60124:3ebd8e0fd171 Date: 2013-01-16 22:43 -0800 http://bitbucket.org/pypy/pypy/changeset/3ebd8e0fd171/ Log: Reorganize where the dont_look_inside is to push it to a lower level. Doesn't affect anything yet because the JIT never looks in here. diff --git a/pypy/jit/metainterp/compile.py b/pypy/jit/metainterp/compile.py --- a/pypy/jit/metainterp/compile.py +++ b/pypy/jit/metainterp/compile.py @@ -5,7 +5,7 @@ from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.debug import debug_start, debug_stop, debug_print from pypy.rlib import rstack -from pypy.rlib.jit import JitDebugInfo, Counters +from pypy.rlib.jit import JitDebugInfo, Counters, dont_look_inside from pypy.conftest import option from pypy.tool.sourcetools import func_with_new_name @@ -685,6 +685,7 @@ assert 0, "unreachable" @staticmethod + @dont_look_inside def force_now(cpu, token): # Called during a residual call from the assembler, if the code # actually needs to force one of the virtualrefs or the virtualizable. diff --git a/pypy/jit/metainterp/virtualref.py b/pypy/jit/metainterp/virtualref.py --- a/pypy/jit/metainterp/virtualref.py +++ b/pypy/jit/metainterp/virtualref.py @@ -169,4 +169,3 @@ # token == TOKEN_NONE and the vref was not forced: it's invalid raise InvalidVirtualRef return vref.forced - force_virtual._dont_inline_ = True From noreply at buildbot.pypy.org Thu Jan 17 07:44:14 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Thu, 17 Jan 2013 07:44:14 +0100 (CET) Subject: [pypy-commit] pypy default: Merged upstream. Message-ID: <20130117064414.D68EE1C1167@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: Changeset: r60125:b67cae978aff Date: 2013-01-16 22:44 -0800 http://bitbucket.org/pypy/pypy/changeset/b67cae978aff/ Log: Merged upstream. diff --git a/pypy/rlib/runicode.py b/pypy/rlib/runicode.py --- a/pypy/rlib/runicode.py +++ b/pypy/rlib/runicode.py @@ -460,6 +460,10 @@ errorhandler=None, byteorder='little'): if size == 0: + if byteorder == 'native': + result = StringBuilder(2) + _STORECHAR(result, 0xFEFF, BYTEORDER) + return result.build() return "" result = StringBuilder(size * 2 + 2) @@ -621,6 +625,10 @@ errorhandler=None, byteorder='little'): if size == 0: + if byteorder == 'native': + result = StringBuilder(4) + _STORECHAR32(result, 0xFEFF, BYTEORDER) + return result.build() return "" result = StringBuilder(size * 4 + 4) diff --git a/pypy/rlib/test/test_runicode.py b/pypy/rlib/test/test_runicode.py --- a/pypy/rlib/test/test_runicode.py +++ b/pypy/rlib/test/test_runicode.py @@ -679,6 +679,11 @@ "utf-32 utf-32-be utf-32-le").split(): self.checkencode(uni, encoding) + def test_empty(self): + for encoding in ("utf-8 utf-16 utf-16-be utf-16-le " + "utf-32 utf-32-be utf-32-le").split(): + self.checkencode(u'', encoding) + def test_single_chars_utf8(self): # check every number of bytes per char for s in ["\xd7\x90", "\xd6\x96", "\xeb\x96\x95", "\xf0\x90\x91\x93"]: From noreply at buildbot.pypy.org Thu Jan 17 08:29:26 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Thu, 17 Jan 2013 08:29:26 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: To fix. Message-ID: <20130117072926.C74411C12FC@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60126:c5f9caec1ea7 Date: 2013-01-17 08:29 +0100 http://bitbucket.org/pypy/pypy/changeset/c5f9caec1ea7/ Log: To fix. diff --git a/rpython/conftest.py b/rpython/conftest.py --- a/rpython/conftest.py +++ b/rpython/conftest.py @@ -2,6 +2,13 @@ import py, pytest from rpython.tool import leakfinder +# XXX +# Aquana, i just ensured that with the pypy conftest we get all skips, +# i'll investigate differences in collection tommorow, for now, can you just +# import the makemodule hook from pypy and add a comment that im responsible +# for fixing? +from pypy.conftest import pytest_pycollect_makemodule + cdir = realpath(join(dirname(__file__), 'translator', 'c')) cache_dir = realpath(join(dirname(__file__), '_cache')) option = None From noreply at buildbot.pypy.org Thu Jan 17 10:48:32 2013 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Thu, 17 Jan 2013 10:48:32 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: fix collection of the jvm backend tests Message-ID: <20130117094832.317311C00BE@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: split-rpython Changeset: r60127:1bcb31da666a Date: 2013-01-17 10:47 +0100 http://bitbucket.org/pypy/pypy/changeset/1bcb31da666a/ Log: fix collection of the jvm backend tests modern pytest ignores test classes with a own __init__ due to general missuse diff --git a/rpython/conftest.py b/rpython/conftest.py --- a/rpython/conftest.py +++ b/rpython/conftest.py @@ -2,12 +2,6 @@ import py, pytest from rpython.tool import leakfinder -# XXX -# Aquana, i just ensured that with the pypy conftest we get all skips, -# i'll investigate differences in collection tommorow, for now, can you just -# import the makemodule hook from pypy and add a comment that im responsible -# for fixing? -from pypy.conftest import pytest_pycollect_makemodule cdir = realpath(join(dirname(__file__), 'translator', 'c')) cache_dir = realpath(join(dirname(__file__), '_cache')) diff --git a/rpython/translator/jvm/test/runtest.py b/rpython/translator/jvm/test/runtest.py --- a/rpython/translator/jvm/test/runtest.py +++ b/rpython/translator/jvm/test/runtest.py @@ -98,10 +98,9 @@ FLOAT_PRECISION = 7 - def __init__(self): - self._func = None - self._ann = None - self._jvm_src = None + _func = None + _ann = None + _jvm_src = None def compile(self, fn, args, ann=None, backendopt=False): if ann is None: From noreply at buildbot.pypy.org Thu Jan 17 10:57:42 2013 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Thu, 17 Jan 2013 10:57:42 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Backed out changeset 1bcb31da666a, defer the fix to pytest Message-ID: <20130117095742.BB1681C00BE@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: split-rpython Changeset: r60128:59bc9f371b06 Date: 2013-01-17 10:57 +0100 http://bitbucket.org/pypy/pypy/changeset/59bc9f371b06/ Log: Backed out changeset 1bcb31da666a, defer the fix to pytest diff --git a/rpython/conftest.py b/rpython/conftest.py --- a/rpython/conftest.py +++ b/rpython/conftest.py @@ -2,6 +2,12 @@ import py, pytest from rpython.tool import leakfinder +# XXX +# Aquana, i just ensured that with the pypy conftest we get all skips, +# i'll investigate differences in collection tommorow, for now, can you just +# import the makemodule hook from pypy and add a comment that im responsible +# for fixing? +from pypy.conftest import pytest_pycollect_makemodule cdir = realpath(join(dirname(__file__), 'translator', 'c')) cache_dir = realpath(join(dirname(__file__), '_cache')) diff --git a/rpython/translator/jvm/test/runtest.py b/rpython/translator/jvm/test/runtest.py --- a/rpython/translator/jvm/test/runtest.py +++ b/rpython/translator/jvm/test/runtest.py @@ -98,9 +98,10 @@ FLOAT_PRECISION = 7 - _func = None - _ann = None - _jvm_src = None + def __init__(self): + self._func = None + self._ann = None + self._jvm_src = None def compile(self, fn, args, ann=None, backendopt=False): if ann is None: From noreply at buildbot.pypy.org Thu Jan 17 12:00:26 2013 From: noreply at buildbot.pypy.org (antocuni) Date: Thu, 17 Jan 2013 12:00:26 +0100 (CET) Subject: [pypy-commit] pypy default: add a way to construct a numpy array from an existing buffer. PyPy-only implementation detail Message-ID: <20130117110026.57D561C00BE@cobra.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: Changeset: r60129:c437c0261c7e Date: 2013-01-17 11:00 +0000 http://bitbucket.org/pypy/pypy/changeset/c437c0261c7e/ Log: add a way to construct a numpy array from an existing buffer. PyPy- only implementation detail diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py --- a/pypy/module/micronumpy/arrayimpl/concrete.py +++ b/pypy/module/micronumpy/arrayimpl/concrete.py @@ -9,7 +9,7 @@ from pypy.interpreter.error import OperationError, operationerrfmt from pypy.rpython.lltypesystem import rffi, lltype from pypy.rlib import jit -from pypy.rlib.rawstorage import free_raw_storage +from pypy.rlib.rawstorage import free_raw_storage, RAW_STORAGE from pypy.rlib.debug import make_sure_not_resized class ConcreteArrayIterator(base.BaseArrayIterator): @@ -427,18 +427,18 @@ def get_storage_as_int(self, space): return rffi.cast(lltype.Signed, self.storage) -class ConcreteArray(BaseConcreteArray): - def __init__(self, shape, dtype, order, strides, backstrides): +class ConcreteArrayNotOwning(BaseConcreteArray): + def __init__(self, shape, dtype, order, strides, backstrides, storage): make_sure_not_resized(shape) make_sure_not_resized(strides) make_sure_not_resized(backstrides) self.shape = shape self.size = support.product(shape) * dtype.get_size() - self.storage = dtype.itemtype.malloc(self.size) self.order = order self.dtype = dtype self.strides = strides self.backstrides = backstrides + self.storage = storage def create_iter(self, shape=None): if shape is None or shape == self.get_shape(): @@ -451,14 +451,24 @@ def fill(self, box): self.dtype.fill(self.storage, box, 0, self.size) - def __del__(self): - free_raw_storage(self.storage, track_allocation=False) - def set_shape(self, space, new_shape): strides, backstrides = support.calc_strides(new_shape, self.dtype, self.order) return SliceArray(0, strides, backstrides, new_shape, self) +class ConcreteArray(ConcreteArrayNotOwning): + def __init__(self, shape, dtype, order, strides, backstrides): + # we allocate the actual storage later because we need to compute + # self.size first + null_storage = lltype.nullptr(RAW_STORAGE) + ConcreteArrayNotOwning.__init__(self, shape, dtype, order, strides, backstrides, + null_storage) + self.storage = dtype.itemtype.malloc(self.size) + + def __del__(self): + free_raw_storage(self.storage, track_allocation=False) + + class NonWritableArray(ConcreteArray): def descr_setitem(self, space, w_index, w_value): raise OperationError(space.w_RuntimeError, space.wrap( diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py --- a/pypy/module/micronumpy/base.py +++ b/pypy/module/micronumpy/base.py @@ -28,6 +28,16 @@ return W_NDimArray(impl) @staticmethod + def from_shape_and_storage(shape, storage, dtype, order='C'): + from pypy.module.micronumpy.arrayimpl import concrete + assert shape + strides, backstrides = calc_strides(shape, dtype, order) + impl = concrete.ConcreteArrayNotOwning(shape, dtype, order, strides, + backstrides, storage) + return W_NDimArray(impl) + + + @staticmethod def new_slice(offset, strides, backstrides, shape, parent, dtype=None): from pypy.module.micronumpy.arrayimpl import concrete diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -397,7 +397,6 @@ space.w_False])) return w_d - # --------------------- operations ---------------------------- def _unaryop_impl(ufunc_name): @@ -565,6 +564,20 @@ return W_NDimArray.new_scalar(space, dtype) return W_NDimArray.from_shape(shape, dtype) + at unwrap_spec(addr=int) +def descr__from_shape_and_storage(space, w_cls, w_shape, addr, w_dtype): + """ + Create an array from an existing buffer, given its address as int. + PyPy-only implementation detail. + """ + from pypy.rpython.lltypesystem import rffi + from pypy.rlib.rawstorage import RAW_STORAGE_PTR + shape = _find_shape(space, w_shape) + storage = rffi.cast(RAW_STORAGE_PTR, addr) + dtype = space.interp_w(interp_dtype.W_Dtype, + space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)) + return W_NDimArray.from_shape_and_storage(shape, storage, dtype) + W_NDimArray.typedef = TypeDef( "ndarray", __new__ = interp2app(descr_new_array), @@ -661,6 +674,8 @@ imag = GetSetProperty(W_NDimArray.descr_get_imag, W_NDimArray.descr_set_imag), __array_interface__ = GetSetProperty(W_NDimArray.descr_array_iface), + _from_shape_and_storage = interp2app(descr__from_shape_and_storage, + as_classmethod=True) ) @unwrap_spec(ndmin=int, copy=bool, subok=bool) diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -189,7 +189,23 @@ assert shape == [2] assert space.str_w(elems[0]) == "a" assert space.str_w(elems[1]) == "b" - + + def test_from_shape_and_storage(self): + from pypy.rlib.rawstorage import alloc_raw_storage, raw_storage_setitem + from pypy.rpython.lltypesystem import rffi + from pypy.module.micronumpy.interp_dtype import get_dtype_cache + storage = alloc_raw_storage(4, track_allocation=False, zero=True) + for i in range(4): + raw_storage_setitem(storage, i, rffi.cast(rffi.UCHAR, i)) + # + dtypes = get_dtype_cache(self.space) + w_array = W_NDimArray.from_shape_and_storage([2, 2], storage, dtypes.w_int8dtype) + def get(i, j): + return w_array.getitem(self.space, [i, j]).value + assert get(0, 0) == 0 + assert get(0, 1) == 1 + assert get(1, 0) == 2 + assert get(1, 1) == 3 class AppTestNumArray(BaseNumpyAppTest): def w_CustomIndexObject(self, index): @@ -2340,7 +2356,7 @@ s = repr(a) assert s.replace('\n', '') == \ "array(['abc', 'defg', 'ab'], dtype='|S4')" - + class AppTestPyPy(BaseNumpyAppTest): def setup_class(cls): @@ -2360,3 +2376,12 @@ assert a[0][1] == 2 a = _numpypy.array(([[[1, 2], [3, 4], [5, 6]]])) assert (a[0, 1] == [3, 4]).all() + + def test_from_shape_and_storage(self): + from _numpypy import array, ndarray + x = array([1, 2, 3, 4]) + addr, _ = x.__array_interface__['data'] + y = ndarray._from_shape_and_storage([2, 2], addr, x.dtype) + assert y[0, 1] == 2 + y[0, 1] = 42 + assert x[1] == 42 From noreply at buildbot.pypy.org Thu Jan 17 14:29:27 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 17 Jan 2013 14:29:27 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: rework the frame manager to use freelists Message-ID: <20130117132927.DD3EF1C00BE@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60130:3fe697670da9 Date: 2013-01-17 15:29 +0200 http://bitbucket.org/pypy/pypy/changeset/3fe697670da9/ Log: rework the frame manager to use freelists diff --git a/pypy/jit/backend/llsupport/regalloc.py b/pypy/jit/backend/llsupport/regalloc.py --- a/pypy/jit/backend/llsupport/regalloc.py +++ b/pypy/jit/backend/llsupport/regalloc.py @@ -13,16 +13,117 @@ class NoVariableToSpill(Exception): pass +class Node(object): + def __init__(self, val, next): + self.val = val + self.next = next + +class LinkedList(object): + def __init__(self, fm, lst=None): + # assume the list is sorted + if lst is not None: + node = None + for item in range(len(lst) - 1, -1, -1): + node = Node(fm.get_loc_index(item), node) + self.master_node = node + else: + self.master_node = None + self.fm = fm + + def append(self, size, item): + key = self.fm.get_loc_index(item) + if size == 2: + self._append(key) + self._append(key + 1) + else: + assert size == 1 + self._append(key) + + def _append(self, key): + if self.master_node is None or self.master_node.val > key: + self.master_node = Node(key, self.master_node) + else: + node = self.master_node + prev_node = self.master_node + while node and node.val < key: + prev_node = node + node = node.next + prev_node.next = Node(key, node) + + def pop(self, size, tp): + if size == 2: + return self._pop_two(tp) + assert size == 1 + if not self.master_node: + return None + node = self.master_node + self.master_node = node.next + return self.fm.frame_pos(node.val, tp) + + def _candidate(self, node): + return node.val + 1 == node.next.val + + def _pop_two(self, tp): + node = self.master_node + if node is None or node.next is None: + return None + if self._candidate(node): + self.master_node = node.next.next + return self.fm.frame_pos(node.val, tp) + prev_node = node + node = node.next + while True: + if node.next is None: + return None + if self._candidate(node): + # pop two + prev_node.next = node.next.next + return self.fm.frame_pos(node.val, tp) + node = node.next + + def __len__(self): + """ For tests only + """ + node = self.master_node + c = 0 + while node: + node = node.next + c += 1 + return c + + def __repr__(self): + if not self.master_node: + return 'LinkedList()' + node = self.master_node + l = [] + while node: + l.append(str(node.val)) + node = node.next + return 'LinkedList(%s)' % '->'.join(l) + class FrameManager(object): """ Manage frame positions + + start_free_depth is the start where we can allocate in whatever order + we like. + + freelist_gcrefs and freelist_others are free lists of locations that + can be used for gcrefs and others. below stack_free_depth. Note + that if floats are occupying more than one spot, in order to allocate + the correct size, we need to use more than one from the freelist in + the consecutive order. """ - def __init__(self): + def __init__(self, start_free_depth=0, freelist_gcrefs=None, + freelist_others=None): self.bindings = {} - self.used = [] # list of bools - self.hint_frame_locations = {} + self.current_frame_depth = start_free_depth + # we disable hints for now + #self.hint_frame_locations = {} + self.freelist_gcrefs = LinkedList(self, freelist_gcrefs) + self.freelist_others = LinkedList(self, freelist_others) def get_frame_depth(self): - return len(self.used) + return self.current_frame_depth def get(self, box): return self.bindings.get(box, None) @@ -35,12 +136,12 @@ except KeyError: pass # check if we have a hint for this box - if box in self.hint_frame_locations: - # if we do, try to reuse the location for this box - loc = self.hint_frame_locations[box] - if self.try_to_reuse_location(box, loc): - return loc - # no valid hint. make up a new free location + #if box in self.hint_frame_locations: + # # if we do, try to reuse the location for this box + # loc = self.hint_frame_locations[box] + # if self.try_to_reuse_location(box, loc): + # return loc + ## no valid hint. make up a new free location return self.get_new_loc(box) def get_new_loc(self, box): @@ -49,22 +150,26 @@ # that 'size' is a power of two. The reason for doing so is to # avoid obscure issues in jump.py with stack locations that try # to move from position (6,7) to position (7,8). - while self.get_frame_depth() & (size - 1): - self.used.append(False) - # - index = self.get_frame_depth() - newloc = self.frame_pos(index, box.type) - for i in range(size): - self.used.append(True) - # - if not we_are_translated(): # extra testing - testindex = self.get_loc_index(newloc) - assert testindex == index - # + if box.type == REF: + newloc = self.freelist_gcrefs.pop(1, box.type) + else: + newloc = self.freelist_others.pop(size, box.type) + + if newloc is None: + # + index = self.get_frame_depth() + newloc = self.frame_pos(index, box.type) + self.current_frame_depth += size + # + if not we_are_translated(): # extra testing + testindex = self.get_loc_index(newloc) + assert testindex == index + # self.bindings[box] = newloc return newloc def set_binding(self, box, loc): + xxx self.bindings[box] = loc # index = self.get_loc_index(loc) @@ -77,29 +182,20 @@ self.used[index] = True index += 1 - def reserve_location_in_frame(self, size): - frame_depth = self.get_frame_depth() - for i in range(size): - self.used.append(True) - return frame_depth - def mark_as_free(self, box): try: loc = self.bindings[box] except KeyError: return # already gone del self.bindings[box] - # - size = self.frame_size(box.type) - baseindex = self.get_loc_index(loc) - if baseindex < 0: - return - for i in range(size): - index = baseindex + i - assert 0 <= index < len(self.used) - self.used[index] = False + if box.type == REF: + self.freelist_gcrefs.append(1, loc) + else: + size = self.frame_size(box) + self.freelist_others.append(size, loc) def try_to_reuse_location(self, box, loc): + xxx index = self.get_loc_index(loc) if index < 0: return False @@ -125,7 +221,11 @@ @staticmethod def get_loc_index(loc): raise NotImplementedError("Purely abstract") - + @staticmethod + def newloc(pos, size, tp): + """ Reverse of get_loc_index + """ + raise NotImplementedError("Purely abstract") class RegisterManager(object): """ Class that keeps track of register allocations diff --git a/pypy/jit/backend/llsupport/test/test_regalloc.py b/pypy/jit/backend/llsupport/test/test_regalloc.py --- a/pypy/jit/backend/llsupport/test/test_regalloc.py +++ b/pypy/jit/backend/llsupport/test/test_regalloc.py @@ -1,6 +1,7 @@ import py -from pypy.jit.metainterp.history import BoxInt, ConstInt, BoxFloat, INT, FLOAT -from pypy.jit.backend.llsupport.regalloc import FrameManager +from pypy.jit.metainterp.history import BoxInt, ConstInt, BoxFloat, INT, FLOAT,\ + BoxPtr +from pypy.jit.backend.llsupport.regalloc import FrameManager, LinkedList from pypy.jit.backend.llsupport.regalloc import RegisterManager as BaseRegMan from pypy.jit.tool.oparser import parse from pypy.jit.backend.detect_cpu import getcpuclass @@ -8,6 +9,9 @@ def newboxes(*values): return [BoxInt(v) for v in values] +def newrefboxes(count): + return [BoxPtr() for _ in range(count)] + def boxes_and_longevity(num): res = [] longevity = {} @@ -35,6 +39,21 @@ def __init__(self, pos, box_type): self.pos = pos self.box_type = box_type + def __repr__(self): + return 'FramePos<%d,%s>' % (self.pos, self.box_type) + def __eq__(self, other): + return self.pos == other.pos and self.box_type == other.box_type + def __ne__(self, other): + return not self == other + +class TFrameManagerEqual(FrameManager): + def frame_pos(self, i, box_type): + return FakeFramePos(i, box_type) + def frame_size(self, box_type): + return 1 + def get_loc_index(self, loc): + assert isinstance(loc, FakeFramePos) + return loc.pos class TFrameManager(FrameManager): def frame_pos(self, i, box_type): @@ -42,10 +61,8 @@ def frame_size(self, box_type): if box_type == FLOAT: return 2 - elif box_type == INT: + else: return 1 - else: - raise ValueError(box_type) def get_loc_index(self, loc): assert isinstance(loc, FakeFramePos) return loc.pos @@ -343,9 +360,7 @@ xrm.loc(f0) rm.loc(b0) assert fm.get_frame_depth() == 3 - - - + def test_spilling(self): b0, b1, b2, b3, b4, b5 = newboxes(0, 1, 2, 3, 4, 5) longevity = {b0: (0, 3), b1: (0, 3), b3: (0, 5), b2: (0, 2), b4: (1, 4), b5: (1, 3)} @@ -366,6 +381,7 @@ def test_hint_frame_locations_1(self): + py.test.skip("xxx") b0, = newboxes(0) fm = TFrameManager() loc123 = FakeFramePos(123, INT) @@ -376,6 +392,7 @@ assert fm.get_frame_depth() == 124 def test_hint_frame_locations_2(self): + py.test.skip("xxx") b0, b1, b2 = newboxes(0, 1, 2) longevity = {b0: (0, 1), b1: (0, 2), b2: (0, 2)} fm = TFrameManager() @@ -407,6 +424,115 @@ # rm._check_invariants() + def test_linkedlist(self): + class Loc(object): + def __init__(self, pos, size, tp): + self.pos = pos + self.size = size + self.tp = tp + + class FrameManager(object): + @staticmethod + def get_loc_index(item): + return item.pos + @staticmethod + def frame_pos(pos, tp): + if tp == 13: + size = 2 + else: + size = 1 + return Loc(pos, size, tp) + + fm = FrameManager() + l = LinkedList(fm) + l.append(1, Loc(1, 1, 0)) + l.append(1, Loc(4, 1, 0)) + l.append(1, Loc(2, 1, 0)) + l.append(1, Loc(0, 1, 0)) + assert l.master_node.val == 0 + assert l.master_node.next.val == 1 + assert l.master_node.next.next.val == 2 + assert l.master_node.next.next.next.val == 4 + assert l.master_node.next.next.next.next is None + item = l.pop(1, 0) + assert item.pos == 0 + item = l.pop(1, 0) + assert item.pos == 1 + item = l.pop(1, 0) + assert item.pos == 2 + item = l.pop(1, 0) + assert item.pos == 4 + assert l.pop(1, 0) is None + l.append(1, Loc(1, 1, 0)) + l.append(1, Loc(5, 1, 0)) + l.append(1, Loc(2, 1, 0)) + l.append(1, Loc(0, 1, 0)) + item = l.pop(2, 13) + assert item.tp == 13 + assert item.pos == 0 + assert item.size == 2 + assert l.pop(2, 0) is None # 2 and 4 + l.append(1, Loc(4, 1, 0)) + item = l.pop(2, 13) + assert item.pos == 4 + assert item.size == 2 + assert l.pop(1, 0).pos == 2 + assert l.pop(1, 0) is None + l.append(2, Loc(1, 2, 0)) + item = l.pop(2, 13) + assert item.pos == 1 + assert item.tp == 13 + assert item.size == 2 + + def test_frame_manager_basic_equal(self): + b0, b1 = newboxes(0, 1) + fm = TFrameManagerEqual() + loc0 = fm.loc(b0) + assert fm.get_loc_index(loc0) == 0 + # + assert fm.get(b1) is None + loc1 = fm.loc(b1) + assert fm.get_loc_index(loc1) == 1 + assert fm.get(b1) == loc1 + # + loc0b = fm.loc(b0) + assert loc0b == loc0 + # + fm.loc(BoxInt()) + assert fm.get_frame_depth() == 3 + # + f0 = BoxFloat() + locf0 = fm.loc(f0) + assert fm.get_loc_index(locf0) == 3 + assert fm.get_frame_depth() == 4 + # + f1 = BoxFloat() + locf1 = fm.loc(f1) + assert fm.get_loc_index(locf1) == 4 + assert fm.get_frame_depth() == 5 + fm.mark_as_free(b1) + assert fm.freelist_others + b2 = BoxInt() + fm.loc(b2) # should be in the same spot as b1 before + assert fm.get(b1) is None + assert fm.get(b2) == loc1 + fm.mark_as_free(b0) + p0 = BoxPtr() + ploc = fm.loc(p0) + assert fm.get_loc_index(ploc) == 5 + assert fm.get_frame_depth() == 6 + assert ploc != loc1 + p1 = BoxPtr() + p1loc = fm.loc(p1) + assert fm.get_loc_index(p1loc) == 6 + assert fm.get_frame_depth() == 7 + fm.mark_as_free(p0) + p2 = BoxPtr() + p2loc = fm.loc(p2) + assert p2loc == ploc + assert not fm.freelist_gcrefs + assert len(fm.freelist_others) == 1 + def test_frame_manager_basic(self): b0, b1 = newboxes(0, 1) fm = TFrameManager() @@ -426,60 +552,39 @@ # f0 = BoxFloat() locf0 = fm.loc(f0) - assert fm.get_loc_index(locf0) == 4 - assert fm.get_frame_depth() == 6 + assert fm.get_loc_index(locf0) == 3 + assert fm.get_frame_depth() == 5 # f1 = BoxFloat() locf1 = fm.loc(f1) - assert fm.get_loc_index(locf1) == 6 + assert fm.get_loc_index(locf1) == 5 + assert fm.get_frame_depth() == 7 + fm.mark_as_free(b1) + assert fm.freelist_others + b2 = BoxInt() + fm.loc(b2) # should be in the same spot as b1 before + assert fm.get(b1) is None + assert fm.get(b2) == loc1 + fm.mark_as_free(b0) + p0 = BoxPtr() + ploc = fm.loc(p0) + assert fm.get_loc_index(ploc) == 7 assert fm.get_frame_depth() == 8 - assert fm.used == [True, True, True, False, True, True, True, True] - # - fm.mark_as_free(b0) - assert fm.used == [False, True, True, False, True, True, True, True] - fm.mark_as_free(b0) - assert fm.used == [False, True, True, False, True, True, True, True] - fm.mark_as_free(f1) - assert fm.used == [False, True, True, False, True, True, False, False] - # - fm.reserve_location_in_frame(1) + assert ploc != loc1 + p1 = BoxPtr() + p1loc = fm.loc(p1) + assert fm.get_loc_index(p1loc) == 8 assert fm.get_frame_depth() == 9 - assert fm.used == [False, True, True, False, True, True, False, False, True] - # - assert b0 not in fm.bindings - fm.set_binding(b0, loc0) - assert b0 in fm.bindings - assert fm.used == [True, True, True, False, True, True, False, False, True] - # - b3 = BoxInt() - assert not fm.try_to_reuse_location(b3, loc0) - assert fm.used == [True, True, True, False, True, True, False, False, True] - # - fm.mark_as_free(b0) - assert fm.used == [False, True, True, False, True, True, False, False, True] - assert fm.try_to_reuse_location(b3, loc0) - assert fm.used == [True, True, True, False, True, True, False, False, True] - # - fm.mark_as_free(b0) # already free - assert fm.used == [True, True, True, False, True, True, False, False, True] - # - fm.mark_as_free(b3) - assert fm.used == [False, True, True, False, True, True, False, False, True] + fm.mark_as_free(p0) + p2 = BoxPtr() + p2loc = fm.loc(p2) + assert p2loc == ploc + assert not fm.freelist_gcrefs + assert len(fm.freelist_others) == 1 + fm.mark_as_free(b2) f3 = BoxFloat() - assert not fm.try_to_reuse_location(f3, fm.frame_pos(0, FLOAT)) - assert not fm.try_to_reuse_location(f3, fm.frame_pos(1, FLOAT)) - assert not fm.try_to_reuse_location(f3, fm.frame_pos(2, FLOAT)) - assert not fm.try_to_reuse_location(f3, fm.frame_pos(3, FLOAT)) - assert not fm.try_to_reuse_location(f3, fm.frame_pos(4, FLOAT)) - assert not fm.try_to_reuse_location(f3, fm.frame_pos(5, FLOAT)) - assert fm.used == [False, True, True, False, True, True, False, False, True] - assert fm.try_to_reuse_location(f3, fm.frame_pos(6, FLOAT)) - assert fm.used == [False, True, True, False, True, True, True, True, True] - # - fm.used = [False] - assert fm.try_to_reuse_location(BoxFloat(), fm.frame_pos(0, FLOAT)) - assert fm.used == [True, True] - # - fm.used = [True] - assert not fm.try_to_reuse_location(BoxFloat(), fm.frame_pos(0, FLOAT)) - assert fm.used == [True] + floc = fm.loc(f3) + assert fm.get_loc_index(floc) == 0 + + def test_frame_manager_2(self): + pass diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -2072,6 +2072,7 @@ # like %eax that would be destroyed by this call, *and* they are # used by arglocs for the *next* call, then trouble; for now we # will just push/pop them. + xxx from pypy.rpython.memory.gctransform import asmgcroot css = self._regalloc.close_stack_struct if css == 0: diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py --- a/pypy/jit/backend/x86/regalloc.py +++ b/pypy/jit/backend/x86/regalloc.py @@ -331,6 +331,7 @@ def _update_bindings(self, locs, inputargs): # XXX this should probably go to llsupport/regalloc.py + xxx used = {} i = 0 for loc in locs: @@ -1286,6 +1287,7 @@ # optimization only: fill in the 'hint_frame_locations' dictionary # of 'fm' based on the JUMP at the end of the loop, by looking # at where we would like the boxes to be after the jump. + return # XXX disabled for now op = operations[-1] if op.getopnum() != rop.JUMP: return From noreply at buildbot.pypy.org Thu Jan 17 14:29:51 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 17 Jan 2013 14:29:51 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: oops, empty test Message-ID: <20130117132951.003001C00BE@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60131:39345139bc3f Date: 2013-01-17 15:29 +0200 http://bitbucket.org/pypy/pypy/changeset/39345139bc3f/ Log: oops, empty test diff --git a/pypy/jit/backend/llsupport/test/test_regalloc.py b/pypy/jit/backend/llsupport/test/test_regalloc.py --- a/pypy/jit/backend/llsupport/test/test_regalloc.py +++ b/pypy/jit/backend/llsupport/test/test_regalloc.py @@ -585,6 +585,3 @@ f3 = BoxFloat() floc = fm.loc(f3) assert fm.get_loc_index(floc) == 0 - - def test_frame_manager_2(self): - pass From noreply at buildbot.pypy.org Thu Jan 17 14:54:09 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 17 Jan 2013 14:54:09 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: Add a TODO Message-ID: <20130117135409.B90911C1353@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r60132:810ce3027d32 Date: 2013-01-17 14:53 +0100 http://bitbucket.org/pypy/pypy/changeset/810ce3027d32/ Log: Add a TODO diff --git a/TODO b/TODO --- a/TODO +++ b/TODO @@ -40,3 +40,5 @@ JIT: finish (missing: the call in execute_token(), reorganize pypy source, ?) ------------------------------------------------------------ + +implement thread-locals in RPython (for the executioncontext) From noreply at buildbot.pypy.org Thu Jan 17 16:48:27 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 17 Jan 2013 16:48:27 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: gcmap Message-ID: <20130117154827.BE5B41C1353@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60133:0f2e3e9d6d9e Date: 2013-01-17 15:40 +0200 http://bitbucket.org/pypy/pypy/changeset/0f2e3e9d6d9e/ Log: gcmap diff --git a/pypy/jit/backend/llsupport/regalloc.py b/pypy/jit/backend/llsupport/regalloc.py --- a/pypy/jit/backend/llsupport/regalloc.py +++ b/pypy/jit/backend/llsupport/regalloc.py @@ -1,6 +1,6 @@ import os from pypy.jit.metainterp.history import Const, Box, REF -from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.objectmodel import we_are_translated, specialize from pypy.jit.metainterp.resoperation import rop class TempBox(Box): @@ -50,6 +50,13 @@ node = node.next prev_node.next = Node(key, node) + @specialize.arg(1) + def foreach(self, function, arg): + node = self.master_node + while node is not None: + function(arg, node.val) + node = node.next + def pop(self, size, tp): if size == 2: return self._pop_two(tp) @@ -81,9 +88,7 @@ return self.fm.frame_pos(node.val, tp) node = node.next - def __len__(self): - """ For tests only - """ + def len(self): node = self.master_node c = 0 while node: @@ -91,6 +96,11 @@ c += 1 return c + def __len__(self): + """ For tests only + """ + return self.len() + def __repr__(self): if not self.master_node: return 'LinkedList()' @@ -211,6 +221,20 @@ self.bindings[box] = loc return True + @staticmethod + def _gather_gcroots(lst, var): + lst.append(var) + + def get_gc_map(self): + """ returns a list of locations where GC pointers are + """ + assert not self.bindings + # XXX unsure, maybe what we want is to + # free everything instead + lst = [] + self.freelist_gcrefs.foreach(self._gather_gcroots, lst) + return lst + # abstract methods that need to be overwritten for specific assemblers @staticmethod def frame_pos(loc, type): diff --git a/pypy/jit/backend/llsupport/test/test_regalloc.py b/pypy/jit/backend/llsupport/test/test_regalloc.py --- a/pypy/jit/backend/llsupport/test/test_regalloc.py +++ b/pypy/jit/backend/llsupport/test/test_regalloc.py @@ -532,6 +532,9 @@ assert p2loc == ploc assert not fm.freelist_gcrefs assert len(fm.freelist_others) == 1 + for box in fm.bindings.keys(): + fm.mark_as_free(box) + assert fm.get_gc_map() == [5, 6] def test_frame_manager_basic(self): b0, b1 = newboxes(0, 1) @@ -585,3 +588,6 @@ f3 = BoxFloat() floc = fm.loc(f3) assert fm.get_loc_index(floc) == 0 + for box in fm.bindings.keys(): + fm.mark_as_free(box) + assert fm.get_gc_map() == [7, 8] From noreply at buildbot.pypy.org Thu Jan 17 16:48:29 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 17 Jan 2013 16:48:29 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fight fight fight. freelists on the frame manager, so we get more compact Message-ID: <20130117154829.196141C1353@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60134:68788e589b96 Date: 2013-01-17 17:48 +0200 http://bitbucket.org/pypy/pypy/changeset/68788e589b96/ Log: fight fight fight. freelists on the frame manager, so we get more compact frames (and we can respect GC-vs-non-GC) diff --git a/pypy/jit/backend/llsupport/jitframe.py b/pypy/jit/backend/llsupport/jitframe.py --- a/pypy/jit/backend/llsupport/jitframe.py +++ b/pypy/jit/backend/llsupport/jitframe.py @@ -4,18 +4,37 @@ # compiled loop token (in fact we could use this as a compiled loop token # XXX do this +GCMAP = lltype.GcArray(lltype.Signed) +NULLGCMAP = lltype.nullptr(GCMAP) +# XXX make it SHORT not Signed + JITFRAMEINFO = lltype.GcStruct( 'JITFRAMEINFO', # the depth of frame ('jfi_frame_depth', lltype.Signed), # gcindexlist is a list of indexes of GC ptrs # in the actual array jf_frame of JITFRAME - ('jfi_gcindexlist', lltype.Ptr(lltype.GcArray(lltype.Signed))), - ) + ('jfi_gcmap', lltype.Ptr(GCMAP)), +) + +NULLFRAMEINFO = lltype.nullptr(JITFRAMEINFO) # the JITFRAME that's stored on the heap. See backend//arch.py for # detailed explanation how it is on your architecture +def jitframe_allocate(frame_info): + frame = lltype.malloc(JITFRAME, frame_info.jfi_frame_depth, zero=True) + frame.jf_gcmap = frame_info.jfi_gcmap + frame.jf_frame_info = frame_info + return frame + +def jitframe_copy(frame): + frame_info = frame.jf_frame_info + new_frame = lltype.malloc(JITFRAME, frame_info.jfi_frame_depth, zero=True) + new_frame.jf_gcmap = frame_info.jfi_gcmap + new_frame.jf_frame_info = frame_info + return new_frame + JITFRAME = lltype.GcStruct( 'JITFRAME', ('jf_frame_info', lltype.Ptr(JITFRAMEINFO)), @@ -29,6 +48,8 @@ # a bitmask of where are GCREFS in the top of the frame (saved registers) # used for calls and failures ('jf_gcpattern', lltype.Signed), + # a copy of gcmap from frameinfo + ('jf_gcmap', lltype.Ptr(GCMAP)), # For the front-end: a GCREF for the savedata ('jf_savedata', llmemory.GCREF), # For GUARD_(NO)_EXCEPTION and GUARD_NOT_FORCED: the exception we @@ -37,8 +58,13 @@ # exception is not stored there, but is simply kept as a variable there) ('jf_guard_exc', llmemory.GCREF), # the actual frame - ('jf_frame', lltype.Array(lltype.Signed)) - # it should be: , hints={'nolength': True})), but ll2ctypes is complaining + ('jf_frame', lltype.Array(lltype.Signed)), + # note that we keep length field, because it's crucial to have the data + # about GCrefs here and not in frame info which might change + adtmeths = { + 'allocate': jitframe_allocate, + 'copy': jitframe_copy, + } ) JITFRAMEPTR = lltype.Ptr(JITFRAME) diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py --- a/pypy/jit/backend/llsupport/llmodel.py +++ b/pypy/jit/backend/llsupport/llmodel.py @@ -57,10 +57,8 @@ def realloc_frame(frame): frame = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, frame) - frame_info = frame.jf_frame_info - new_frame = lltype.malloc(jitframe.JITFRAME, - frame_info.jfi_frame_depth) - new_frame.jf_frame_info = frame_info + new_frame = frame.copy() + # XXX now we know, rewrite this # we need to do this, because we're not sure what things # are GC pointers and which ones are not llop.gc_writebarrier_before_copy(lltype.Bool, frame, new_frame, diff --git a/pypy/jit/backend/llsupport/regalloc.py b/pypy/jit/backend/llsupport/regalloc.py --- a/pypy/jit/backend/llsupport/regalloc.py +++ b/pypy/jit/backend/llsupport/regalloc.py @@ -23,8 +23,9 @@ # assume the list is sorted if lst is not None: node = None - for item in range(len(lst) - 1, -1, -1): - node = Node(fm.get_loc_index(item), node) + for i in range(len(lst) - 1, -1, -1): + item = lst[i] + node = Node(item, node) self.master_node = node else: self.master_node = None @@ -111,6 +112,37 @@ node = node.next return 'LinkedList(%s)' % '->'.join(l) +def frame_manager_from_gcmap(FmClass, gcmap, depth, frame_bindings): + if not gcmap: + return FmClass() + rev_bindings = [False] * depth + for arg, loc in frame_bindings.iteritems(): + size = FmClass.frame_size(arg.type) + if size == 2: + rev_bindings[FmClass.get_loc_index(loc) + 1] = True + assert size == 1 + rev_bindings[FmClass.get_loc_index(loc)] = True + if not gcmap: + return FmClass() + gcrefs = [] + others = [] + c = 0 + for i in range(len(gcmap)): + item = gcmap[i] + while c < item: + if not rev_bindings[c]: + others.append(c) + c += 1 + if not rev_bindings[item]: + gcrefs.append(item) + c += 1 + for i in range(c, depth): + others.append(i) + fm = FmClass(depth, gcrefs, others) + for arg, loc in frame_bindings.iteritems(): + fm.bindings[arg] = loc + return fm + class FrameManager(object): """ Manage frame positions @@ -605,6 +637,7 @@ """ Platform specific - Allocates a temporary register """ raise NotImplementedError("Abstract") + def compute_vars_longevity(inputargs, operations): # compute a dictionary that maps variables to index in # operations that is a "last-time-seen" diff --git a/pypy/jit/backend/llsupport/test/test_regalloc.py b/pypy/jit/backend/llsupport/test/test_regalloc.py --- a/pypy/jit/backend/llsupport/test/test_regalloc.py +++ b/pypy/jit/backend/llsupport/test/test_regalloc.py @@ -1,10 +1,9 @@ import py from pypy.jit.metainterp.history import BoxInt, ConstInt, BoxFloat, INT, FLOAT,\ BoxPtr -from pypy.jit.backend.llsupport.regalloc import FrameManager, LinkedList +from pypy.jit.backend.llsupport.regalloc import FrameManager, LinkedList,\ + frame_manager_from_gcmap from pypy.jit.backend.llsupport.regalloc import RegisterManager as BaseRegMan -from pypy.jit.tool.oparser import parse -from pypy.jit.backend.detect_cpu import getcpuclass def newboxes(*values): return [BoxInt(v) for v in values] @@ -591,3 +590,25 @@ for box in fm.bindings.keys(): fm.mark_as_free(box) assert fm.get_gc_map() == [7, 8] + + def test_fm_from_gcmap(self): + class Loc(object): + def __init__(self, l): + self.l = l + + class Fm(FrameManager): + @staticmethod + def get_loc_index(l): + return l.l + + b0 = BoxInt() + b1 = BoxInt() + l0 = Loc(5) + l1 = Loc(2) + bindings = {b0: l0, b1: l1} + fm = frame_manager_from_gcmap(Fm, [1, 5, 6, 8], 13, + bindings) + assert repr(fm.freelist_gcrefs) == "LinkedList(1->6->8)" + assert repr(fm.freelist_others) == "LinkedList(0->3->4->7->9->10->11->12)" + assert fm.current_frame_depth == 13 + assert fm.bindings == bindings diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py --- a/pypy/jit/backend/test/runner_test.py +++ b/pypy/jit/backend/test/runner_test.py @@ -430,7 +430,7 @@ def test_int_operations(self): from pypy.jit.metainterp.test.test_executor import get_int_tests for opnum, boxargs, retvalue in get_int_tests(): - print "X" + print opnum res = self.execute_operation(opnum, boxargs, 'int') assert res.value == retvalue diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -534,8 +534,9 @@ looptoken, clt.allgcrefs) looppos = self.mc.get_relative_pos() looptoken._x86_loop_code = looppos - frame_depth = self._assemble(regalloc, operations) + frame_depth = self._assemble(regalloc, inputargs, operations) clt.frame_info.jfi_frame_depth = frame_depth + JITFRAME_FIXED_SIZE + self._update_gcmap(clt.frame_info, regalloc) # size_excluding_failure_stuff = self.mc.get_relative_pos() self.write_pending_failure_recoveries() @@ -586,9 +587,10 @@ startpos = self.mc.get_relative_pos() operations = regalloc.prepare_bridge(inputargs, arglocs, operations, - self.current_clt.allgcrefs) + self.current_clt.allgcrefs, + self.current_clt.frame_info) stack_check_patch_ofs = self._check_frame_depth() - frame_depth = self._assemble(regalloc, operations) + frame_depth = self._assemble(regalloc, inputargs, operations) codeendpos = self.mc.get_relative_pos() self.write_pending_failure_recoveries() fullsize = self.mc.get_relative_pos() @@ -610,6 +612,7 @@ self._patch_stackadjust(stack_check_patch_ofs + rawstart, frame_depth) self.fixup_target_tokens(rawstart) self.current_clt.frame_info.jfi_frame_depth = frame_depth + self._update_gcmap(self.current_clt.frame_info, regalloc) self.teardown() # oprofile support if self.cpu.profile_agent is not None: @@ -688,6 +691,13 @@ mc = codebuf.MachineCodeBlockWrapper() mc.writeimm32(allocated_depth) mc.copy_to_raw_memory(adr) + + def _update_gcmap(self, frame_info, regalloc): + gcmap = regalloc.get_gc_map() + frame_info.jfi_gcmap = lltype.malloc(jitframe.GCMAP, + len(gcmap)) + for i in range(len(gcmap)): + frame_info.jfi_gcmap[i] = gcmap[i] def get_asmmemmgr_blocks(self, looptoken): clt = looptoken.compiled_loop_token @@ -777,16 +787,17 @@ operations = newoperations return operations - def _assemble(self, regalloc, operations): + def _assemble(self, regalloc, inputargs, operations): self._regalloc = regalloc regalloc.compute_hint_frame_locations(operations) - regalloc.walk_operations(operations) + regalloc.walk_operations(inputargs, operations) if we_are_translated() or self.cpu.dont_keepalive_stuff: self._regalloc = None # else keep it around for debugging frame_depth = regalloc.get_final_frame_depth() jump_target_descr = regalloc.jump_target_descr if jump_target_descr is not None: - target_frame_depth = jump_target_descr._x86_clt.frame_info.jfi_frame_depth + tgt_depth = jump_target_descr._x86_clt.frame_info.jfi_frame_depth + target_frame_depth = tgt_depth - JITFRAME_FIXED_SIZE frame_depth = max(frame_depth, target_frame_depth) return frame_depth diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py --- a/pypy/jit/backend/x86/regalloc.py +++ b/pypy/jit/backend/x86/regalloc.py @@ -17,13 +17,15 @@ from pypy.jit.codewriter import longlong from pypy.jit.codewriter.effectinfo import EffectInfo from pypy.jit.metainterp.resoperation import rop +from pypy.jit.backend.llsupport.jitframe import NULLGCMAP from pypy.jit.backend.llsupport.descr import ArrayDescr from pypy.jit.backend.llsupport.descr import CallDescr from pypy.jit.backend.llsupport.descr import unpack_arraydescr from pypy.jit.backend.llsupport.descr import unpack_fielddescr from pypy.jit.backend.llsupport.descr import unpack_interiorfielddescr from pypy.jit.backend.llsupport.regalloc import FrameManager, RegisterManager,\ - TempBox, compute_vars_longevity, is_comparison_or_ovf_op + TempBox, compute_vars_longevity, is_comparison_or_ovf_op,\ + frame_manager_from_gcmap from pypy.jit.backend.x86.arch import WORD, JITFRAME_FIXED_SIZE from pypy.jit.backend.x86.arch import IS_X86_32, IS_X86_64 from pypy.jit.backend.x86 import rx86 @@ -171,9 +173,11 @@ self.close_stack_struct = 0 self.final_jump_op = None - def _prepare(self, inputargs, operations, allgcrefs): - self.fm = X86FrameManager() - self.min_frame_depth = 0 + def _prepare(self, inputargs, operations, allgcrefs, gcmap=NULLGCMAP, + parent_frame_depth=0, frame_bindings=None): + self.fm = frame_manager_from_gcmap(X86FrameManager, gcmap, + parent_frame_depth, + frame_bindings) cpu = self.assembler.cpu operations = cpu.gc_ll_descr.rewrite_assembler(cpu, operations, allgcrefs) @@ -201,9 +205,16 @@ self.min_bytes_before_label = 13 return operations - def prepare_bridge(self, inputargs, arglocs, operations, allgcrefs): - operations = self._prepare(inputargs, operations, - allgcrefs) + def get_gc_map(self): + return self.fm.get_gc_map() + + def prepare_bridge(self, inputargs, arglocs, operations, allgcrefs, + frame_info): + frame_bindings = self._frame_bindings(arglocs, inputargs) + operations = self._prepare(inputargs, operations, allgcrefs, + frame_info.jfi_gcmap, + frame_info.jfi_frame_depth, + frame_bindings) self._update_bindings(arglocs, inputargs) self.min_bytes_before_label = 0 return operations @@ -213,70 +224,12 @@ at_least_position) def get_final_frame_depth(self): - min_frame_depth = self.fm.get_frame_depth() - if min_frame_depth > self.min_frame_depth: - self.min_frame_depth = min_frame_depth - return self.min_frame_depth + return self.fm.get_frame_depth() def _set_initial_bindings(self, inputargs): for box in inputargs: assert isinstance(box, Box) self.fm.get_new_loc(box) - #loc = self.fm.frame_pos(cur_frame_pos, box.type) - #self.fm.set_binding(box, loc) - - # if IS_X86_64: - # inputargs = self._set_initial_bindings_regs_64(inputargs) - # # ... - # # stack layout: arg2 - # # arg1 - # # arg0 - # # return address - # # saved ebp <-- ebp points here - # # ... - # XXX # adjust the address to count for the fact that we're passing - # # jitframe as a first arg - # cur_frame_pos = - 1 - FRAME_FIXED_SIZE - # assert get_ebp_ofs(cur_frame_pos-1) == 2*WORD - # assert get_ebp_ofs(cur_frame_pos-2) == 3*WORD - # # - # for box in inputargs: - # assert isinstance(box, Box) - # # - # if IS_X86_32 and box.type == FLOAT: - # cur_frame_pos -= 2 - # else: - # cur_frame_pos -= 1 - # loc = self.fm.frame_pos(cur_frame_pos, box.type) - # self.fm.set_binding(box, loc) - - # def _set_initial_bindings_regs_64(self, inputargs): - # # In reverse order for use with pop() - # unused_gpr = [r9, r8, ecx, edx, esi] # jitframe comes in edi. don't use - # # it for parameter parsing - # unused_xmm = [xmm7, xmm6, xmm5, xmm4, xmm3, xmm2, xmm1, xmm0] - # # - # pass_on_stack = [] - # # - # for box in inputargs: - # assert isinstance(box, Box) - # # - # if box.type == FLOAT: - # if len(unused_xmm) > 0: - # ask = unused_xmm.pop() - # got = self.xrm.try_allocate_reg(box, selected_reg=ask) - # assert ask == got - # else: - # pass_on_stack.append(box) - # else: - # if len(unused_gpr) > 0: - # ask = unused_gpr.pop() - # got = self.rm.try_allocate_reg(box, selected_reg=ask) - # assert ask == got - # else: - # pass_on_stack.append(box) - # # - # return pass_on_stack def possibly_free_var(self, var): if var.type == FLOAT: @@ -329,9 +282,20 @@ else: return self.xrm.make_sure_var_in_reg(var, forbidden_vars) + def _frame_bindings(self, locs, inputargs): + bindings = {} + i = 0 + for loc in locs: + if loc is None: + continue + arg = inputargs[i] + i += 1 + if not isinstance(loc, RegLoc): + bindings[arg] = loc + return bindings + def _update_bindings(self, locs, inputargs): # XXX this should probably go to llsupport/regalloc.py - xxx used = {} i = 0 for loc in locs: @@ -339,21 +303,16 @@ continue arg = inputargs[i] i += 1 - if arg.type == FLOAT: - if isinstance(loc, RegLoc): + if isinstance(loc, RegLoc): + if arg.type == FLOAT: self.xrm.reg_bindings[arg] = loc used[loc] = None else: - self.fm.set_binding(arg, loc) - else: - if isinstance(loc, RegLoc): if loc is ebp: self.rm.bindings_to_frame_reg[arg] = None else: self.rm.reg_bindings[arg] = loc used[loc] = None - else: - self.fm.set_binding(arg, loc) self.rm.free_regs = [] for reg in self.rm.all_regs: if reg not in used: @@ -362,8 +321,6 @@ for reg in self.xrm.all_regs: if reg not in used: self.xrm.free_regs.append(reg) - # note: we need to make a copy of inputargs because possibly_free_vars - # is also used on op args, which is a non-resizable list self.possibly_free_vars(list(inputargs)) self.rm._check_invariants() self.xrm._check_invariants() @@ -437,7 +394,7 @@ return False return True - def walk_operations(self, operations): + def walk_operations(self, inputargs, operations): i = 0 #self.operations = operations while i < len(operations): @@ -465,6 +422,8 @@ assert not self.xrm.reg_bindings self.flush_loop() self.assembler.mc.mark_op(None) # end of the loop + for arg in inputargs: + self.possibly_free_var(arg) def flush_loop(self): # rare case: if the loop is too short, or if we are just after @@ -921,6 +880,7 @@ if oopspecindex == EffectInfo.OS_MATH_SQRT: return self._consider_math_sqrt(op) self._consider_call(op) + self.possibly_free_vars(op.getarglist()) def consider_call_may_force(self, op, guard_op): assert guard_op is not None @@ -943,6 +903,7 @@ xxx self._call(op, [frame_loc, self.loc(op.getarg(0))], guard_not_forced_op=guard_op) + self.possibly_free_var(op.getarg(0)) def consider_cond_call_gc_wb(self, op): assert op.result is None diff --git a/pypy/jit/backend/x86/runner.py b/pypy/jit/backend/x86/runner.py --- a/pypy/jit/backend/x86/runner.py +++ b/pypy/jit/backend/x86/runner.py @@ -115,9 +115,7 @@ func = rffi.cast(FUNCPTR, addr) #llop.debug_print(lltype.Void, ">>>> Entering", addr) frame_info = clt.frame_info - frame = lltype.malloc(jitframe.JITFRAME, frame_info.jfi_frame_depth, - zero=True) - frame.jf_frame_info = frame_info + frame = jitframe.JITFRAME.allocate(frame_info) ll_frame = lltype.cast_opaque_ptr(llmemory.GCREF, frame) prev_interpreter = None # help flow space if not self.translate_support_code: From noreply at buildbot.pypy.org Thu Jan 17 17:55:32 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 17 Jan 2013 17:55:32 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: pretty crucial fix Message-ID: <20130117165532.663DC1C00BE@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60135:996aae6e0891 Date: 2013-01-17 18:52 +0200 http://bitbucket.org/pypy/pypy/changeset/996aae6e0891/ Log: pretty crucial fix diff --git a/pypy/jit/backend/llsupport/regalloc.py b/pypy/jit/backend/llsupport/regalloc.py --- a/pypy/jit/backend/llsupport/regalloc.py +++ b/pypy/jit/backend/llsupport/regalloc.py @@ -122,8 +122,6 @@ rev_bindings[FmClass.get_loc_index(loc) + 1] = True assert size == 1 rev_bindings[FmClass.get_loc_index(loc)] = True - if not gcmap: - return FmClass() gcrefs = [] others = [] c = 0 @@ -137,7 +135,8 @@ gcrefs.append(item) c += 1 for i in range(c, depth): - others.append(i) + if not rev_bindings[i]: + others.append(i) fm = FmClass(depth, gcrefs, others) for arg, loc in frame_bindings.iteritems(): fm.bindings[arg] = loc @@ -196,7 +195,6 @@ newloc = self.freelist_gcrefs.pop(1, box.type) else: newloc = self.freelist_others.pop(size, box.type) - if newloc is None: # index = self.get_frame_depth() @@ -207,23 +205,10 @@ testindex = self.get_loc_index(newloc) assert testindex == index # + self.bindings[box] = newloc return newloc - def set_binding(self, box, loc): - xxx - self.bindings[box] = loc - # - index = self.get_loc_index(loc) - if index < 0: - return - endindex = index + self.frame_size(box.type) - while len(self.used) < endindex: - self.used.append(False) - while index < endindex: - self.used[index] = True - index += 1 - def mark_as_free(self, box): try: loc = self.bindings[box] diff --git a/pypy/jit/backend/x86/arch.py b/pypy/jit/backend/x86/arch.py --- a/pypy/jit/backend/x86/arch.py +++ b/pypy/jit/backend/x86/arch.py @@ -48,14 +48,12 @@ SAVED_REGISTERS = 1 # range(1, 5) MY_COPY_OF_REGS = 5 # range(5, 9) XXX - JITFRAME_FIXED_SIZE = 29 # 13 GPR + 16 XMM - # reg, we don't save it + JITFRAME_FIXED_SIZE = 29 # 13 GPR + 15 XMM + one word for alignment else: # rbp + rbx + r12 + r13 + r14 + r15 + 12 extra words + return address = 19 FRAME_FIXED_SIZE = 19 PASS_ON_MY_FRAME = 12 - JITFRAME_FIXED_SIZE = 29 # 13 GPR + 16 XMM - # reg, we don't save it + JITFRAME_FIXED_SIZE = 29 # 13 GPR + 15 XMM + one word for alignment # "My copy of regs" has room for almost all registers, apart from eax and edx # which are used in the malloc itself. They are: diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -1878,7 +1878,8 @@ pos = pos // WORD - GPR_REGS locs.append(xmm_reg_mgr_cls.all_regs[pos]) else: - i = pos // WORD - GPR_REGS - XMM_REGS + i = pos // WORD - JITFRAME_FIXED_SIZE + assert i >= 0 tp = inputargs[input_i].type locs.append(StackLoc(i, pos, tp)) input_i += 1 diff --git a/pypy/jit/backend/x86/regloc.py b/pypy/jit/backend/x86/regloc.py --- a/pypy/jit/backend/x86/regloc.py +++ b/pypy/jit/backend/x86/regloc.py @@ -98,12 +98,15 @@ class StackLoc(RawStackLoc): def __init__(self, position, ebp_offset, type): + from pypy.jit.backend.x86.arch import JITFRAME_FIXED_SIZE, WORD + # _getregkey() returns self.value; the value returned must not # conflict with RegLoc._getregkey(). It doesn't a bit by chance, # so let it fail the following assert if it no longer does. assert ebp_offset >= 0 #assert not (0 <= ebp_offset < 8 + 8 * IS_X86_64) self.position = position + assert (position + JITFRAME_FIXED_SIZE) * WORD == ebp_offset self.value = ebp_offset # One of INT, REF, FLOAT self.type = type From noreply at buildbot.pypy.org Thu Jan 17 17:58:00 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 17 Jan 2013 17:58:00 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: free more stuff Message-ID: <20130117165800.970F51C00BE@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60136:44bbd6d3e56b Date: 2013-01-17 18:57 +0200 http://bitbucket.org/pypy/pypy/changeset/44bbd6d3e56b/ Log: free more stuff diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py --- a/pypy/jit/backend/x86/regalloc.py +++ b/pypy/jit/backend/x86/regalloc.py @@ -851,6 +851,7 @@ self._call(op, [imm(size), sign_loc] + [self.loc(op.getarg(i)) for i in range(op.numargs())], guard_not_forced_op=guard_not_forced_op) + self.possibly_free_vars(op.getarglist()) def consider_call(self, op): effectinfo = op.getdescr().get_extra_info() @@ -880,7 +881,6 @@ if oopspecindex == EffectInfo.OS_MATH_SQRT: return self._consider_math_sqrt(op) self._consider_call(op) - self.possibly_free_vars(op.getarglist()) def consider_call_may_force(self, op, guard_op): assert guard_op is not None From noreply at buildbot.pypy.org Thu Jan 17 18:38:27 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 17 Jan 2013 18:38:27 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: frontend fixes Message-ID: <20130117173827.A683E1C12FC@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60137:866412438b36 Date: 2013-01-17 19:38 +0200 http://bitbucket.org/pypy/pypy/changeset/866412438b36/ Log: frontend fixes diff --git a/pypy/jit/backend/llgraph/runner.py b/pypy/jit/backend/llgraph/runner.py --- a/pypy/jit/backend/llgraph/runner.py +++ b/pypy/jit/backend/llgraph/runner.py @@ -22,6 +22,7 @@ # We need to clone the list of operations because the # front-end will mutate them under our feet again. We also # need to make sure things get freed. + self.alldescrs = [] def mapping(box, _cache={}): if isinstance(box, Const) or box is None: return box @@ -35,6 +36,8 @@ self.operations = [] for op in operations: if op.getdescr() is not None: + # XXX is this hack needed any more? + self.alldescrs.append(op.getdescr()) newdescr = WeakrefDescr(op.getdescr()) else: newdescr = None diff --git a/pypy/jit/metainterp/compile.py b/pypy/jit/metainterp/compile.py --- a/pypy/jit/metainterp/compile.py +++ b/pypy/jit/metainterp/compile.py @@ -368,7 +368,6 @@ def send_bridge_to_backend(jitdriver_sd, metainterp_sd, faildescr, inputargs, operations, original_loop_token): - n = metainterp_sd.cpu.get_fail_descr_number(faildescr) if not we_are_translated(): show_procedures(metainterp_sd) seen = dict.fromkeys(inputargs) @@ -377,7 +376,7 @@ hooks = metainterp_sd.warmrunnerdesc.hooks debug_info = JitDebugInfo(jitdriver_sd, metainterp_sd.logger_ops, original_loop_token, operations, 'bridge', - fail_descr_no=n) + fail_descr=faildescr) hooks.before_compile_bridge(debug_info) else: hooks = None @@ -403,7 +402,8 @@ ops_offset = asminfo.ops_offset else: ops_offset = None - metainterp_sd.logger_ops.log_bridge(inputargs, operations, n, ops_offset) + metainterp_sd.logger_ops.log_bridge(inputargs, operations, faildescr, + ops_offset) # #if metainterp_sd.warmrunnerdesc is not None: # for tests # metainterp_sd.warmrunnerdesc.memory_manager.keep_loop_alive( diff --git a/pypy/jit/metainterp/logger.py b/pypy/jit/metainterp/logger.py --- a/pypy/jit/metainterp/logger.py +++ b/pypy/jit/metainterp/logger.py @@ -1,7 +1,7 @@ import os from pypy.rlib.debug import have_debug_prints from pypy.rlib.debug import debug_start, debug_stop, debug_print -from pypy.rlib.objectmodel import we_are_translated +from pypy.rlib.objectmodel import we_are_translated, compute_unique_id from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.history import Const, ConstInt, Box, \ @@ -30,7 +30,9 @@ debug_stop("jit-log-opt-loop") return logops - def log_bridge(self, inputargs, operations, number=-1, ops_offset=None): + def log_bridge(self, inputargs, operations, descr=None, ops_offset=None): + # XXX great idea to pass number = -1 or -2 to mean stuff + return if number == -1: debug_start("jit-log-noopt-bridge") logops = self._log_operations(inputargs, operations, ops_offset) @@ -131,8 +133,8 @@ if op.getdescr() is not None: descr = op.getdescr() if is_guard and self.guard_number: - index = self.metainterp_sd.cpu.get_fail_descr_number(descr) - r = "" % index + hash = compute_unique_id(descr) + r = "" % hash else: r = self.repr_of_descr(descr) if args: diff --git a/pypy/jit/metainterp/pyjitpl.py b/pypy/jit/metainterp/pyjitpl.py --- a/pypy/jit/metainterp/pyjitpl.py +++ b/pypy/jit/metainterp/pyjitpl.py @@ -1470,11 +1470,8 @@ history.VOID: 'void'}[jd.result_type] tokens = getattr(self, 'loop_tokens_done_with_this_frame_%s' % name) jd.portal_finishtoken = tokens[0].finishdescr - num = self.cpu.get_fail_descr_number(tokens[0].finishdescr) - setattr(self.cpu, 'done_with_this_frame_%s_v' % name, num) jd.propagate_exc_descr = exc_descr # - num = self.cpu.get_fail_descr_number(exc_descr) self.cpu.propagate_exc_descr = exc_descr # self.globaldata = MetaInterpGlobalData(self) diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py --- a/pypy/jit/metainterp/warmspot.py +++ b/pypy/jit/metainterp/warmspot.py @@ -653,7 +653,7 @@ else: assert False (_, jd._PTR_ASSEMBLER_HELPER_FUNCTYPE) = self.cpu.ts.get_FuncType( - [lltype.Signed, llmemory.GCREF, llmemory.GCREF], ASMRESTYPE) + [llmemory.GCREF, llmemory.GCREF], ASMRESTYPE) def rewrite_can_enter_jits(self): sublists = {} @@ -922,9 +922,7 @@ vinfo = jd.virtualizable_info - def assembler_call_helper(failindex, deadframe, virtualizableref): - XXX # we got failindex, but the descr is not stored on the frame - # yet, it's our job + def assembler_call_helper(deadframe, virtualizableref): fail_descr = self.cpu.get_latest_descr(deadframe) if vinfo is not None: xxx diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py --- a/pypy/rlib/jit.py +++ b/pypy/rlib/jit.py @@ -888,24 +888,24 @@ logger - an instance of jit.metainterp.logger.LogOperations type - either 'loop', 'entry bridge' or 'bridge' looptoken - description of a loop - fail_descr_no - number of failing descr for bridges, -1 otherwise + fail_descr - fail descr or None asminfo - extra assembler information """ asminfo = None def __init__(self, jitdriver_sd, logger, looptoken, operations, type, - greenkey=None, fail_descr_no=-1): + greenkey=None, fail_descr=None): self.jitdriver_sd = jitdriver_sd self.logger = logger self.looptoken = looptoken self.operations = operations self.type = type if type == 'bridge': - assert fail_descr_no != -1 + assert fail_descr is not None else: assert greenkey is not None self.greenkey = greenkey - self.fail_descr_no = fail_descr_no + self.fail_descr = fail_descr def get_jitdriver(self): """ Return where the jitdriver on which the jitting started From noreply at buildbot.pypy.org Thu Jan 17 18:40:45 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 17 Jan 2013 18:40:45 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: missing free Message-ID: <20130117174045.681391C00BE@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60138:7dba5511c418 Date: 2013-01-17 19:40 +0200 http://bitbucket.org/pypy/pypy/changeset/7dba5511c418/ Log: missing free diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py --- a/pypy/jit/backend/x86/regalloc.py +++ b/pypy/jit/backend/x86/regalloc.py @@ -522,6 +522,7 @@ def _consider_binop(self, op): loc, argloc = self._consider_binop_part(op) self.Perform(op, [loc, argloc], loc) + self.possibly_free_var(op.getarg(0)) def _consider_lea(self, op, loc): argloc = self.loc(op.getarg(1)) From noreply at buildbot.pypy.org Thu Jan 17 19:08:57 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 17 Jan 2013 19:08:57 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fix fix fix Message-ID: <20130117180857.A0A471C12FC@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60139:02e616816493 Date: 2013-01-17 19:46 +0200 http://bitbucket.org/pypy/pypy/changeset/02e616816493/ Log: fix fix fix diff --git a/pypy/jit/metainterp/compile.py b/pypy/jit/metainterp/compile.py --- a/pypy/jit/metainterp/compile.py +++ b/pypy/jit/metainterp/compile.py @@ -309,7 +309,7 @@ def do_compile_bridge(metainterp_sd, faildescr, inputargs, operations, original_loop_token, log=True): - metainterp_sd.logger_ops.log_bridge(inputargs, operations, -2) + metainterp_sd.logger_ops.log_bridge(inputargs, operations, "compiling") return metainterp_sd.cpu.compile_bridge(faildescr, inputargs, operations, original_loop_token, log=log) @@ -402,7 +402,7 @@ ops_offset = asminfo.ops_offset else: ops_offset = None - metainterp_sd.logger_ops.log_bridge(inputargs, operations, faildescr, + metainterp_sd.logger_ops.log_bridge(inputargs, operations, None, faildescr, ops_offset) # #if metainterp_sd.warmrunnerdesc is not None: # for tests diff --git a/pypy/jit/metainterp/logger.py b/pypy/jit/metainterp/logger.py --- a/pypy/jit/metainterp/logger.py +++ b/pypy/jit/metainterp/logger.py @@ -30,20 +30,19 @@ debug_stop("jit-log-opt-loop") return logops - def log_bridge(self, inputargs, operations, descr=None, ops_offset=None): - # XXX great idea to pass number = -1 or -2 to mean stuff - return - if number == -1: + def log_bridge(self, inputargs, operations, extra=None, + descr=None, ops_offset=None): + if extra == "noopt": debug_start("jit-log-noopt-bridge") logops = self._log_operations(inputargs, operations, ops_offset) debug_stop("jit-log-noopt-bridge") - elif number == -2: + elif extra == "compiling": debug_start("jit-log-compiling-bridge") logops = self._log_operations(inputargs, operations, ops_offset) debug_stop("jit-log-compiling-bridge") else: debug_start("jit-log-opt-bridge") - debug_print("# bridge out of Guard", number, + debug_print("# bridge out of Guard", compute_unique_id(descr), "with", len(operations), "ops") logops = self._log_operations(inputargs, operations, ops_offset) debug_stop("jit-log-opt-bridge") diff --git a/pypy/jit/metainterp/test/test_logger.py b/pypy/jit/metainterp/test/test_logger.py --- a/pypy/jit/metainterp/test/test_logger.py +++ b/pypy/jit/metainterp/test/test_logger.py @@ -1,16 +1,17 @@ -import sys + +import re from pypy.rlib import debug from pypy.jit.tool.oparser import pure_parse from pypy.jit.metainterp import logger from pypy.jit.metainterp.typesystem import llhelper from StringIO import StringIO from pypy.jit.metainterp.optimizeopt.util import equaloplists -from pypy.jit.metainterp.history import AbstractDescr, JitCellToken, BasicFailDescr +from pypy.jit.metainterp.history import AbstractDescr, JitCellToken, BasicFailDescr, BasicFinalDescr from pypy.jit.backend.model import AbstractCPU class Descr(AbstractDescr): - pass + final_descr = False def capturing(func, *args, **kwds): log_stream = StringIO() @@ -108,9 +109,10 @@ inp = ''' [] guard_not_invalidated(descr=descr) [] - finish() + finish(descr=finaldescr) ''' - loop = pure_parse(inp, namespace={'descr': Descr()}) + loop = pure_parse(inp, namespace={'descr': Descr(), + 'finaldescr': BasicFinalDescr()}) logger = Logger(self.make_metainterp_sd()) output = logger.log_loop(loop, {'descr': Descr()}) assert 'guard_not_invalidated(descr=' in output @@ -164,7 +166,7 @@ loop = pure_parse(inp, namespace=namespace) logger = Logger(self.make_metainterp_sd(), guard_number=True) output = logger.log_loop(loop) - assert output.splitlines()[-1] == "guard_true(i0, descr=) [i0]" + assert re.match("guard_true\(i0, descr=\) \[i0\]", output.splitlines()[-1]) pure_parse(output) logger = Logger(self.make_metainterp_sd(), guard_number=False) @@ -198,7 +200,8 @@ def test_intro_bridge(self): bare_logger = logger.Logger(self.make_metainterp_sd()) output = capturing(bare_logger.log_bridge, [], [], 3) - assert output.splitlines()[0] == "# bridge out of Guard 3 with 0 ops" + assert re.match("# bridge out of Guard \d+ with 0 ops", + output.splitlines()[0]) pure_parse(output) def test_repr_single_op(self): From noreply at buildbot.pypy.org Thu Jan 17 19:08:58 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 17 Jan 2013 19:08:58 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: cleanup I think\ Message-ID: <20130117180858.C50C61C12FC@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60140:51eb72e9c8fc Date: 2013-01-17 20:08 +0200 http://bitbucket.org/pypy/pypy/changeset/51eb72e9c8fc/ Log: cleanup I think\ \ diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py --- a/pypy/jit/backend/x86/regalloc.py +++ b/pypy/jit/backend/x86/regalloc.py @@ -242,6 +242,8 @@ var = op.getarg(i) if var is not None: # xxx kludgy self.possibly_free_var(var) + if op.result: + self.possibly_free_var(op.result) def possibly_free_vars(self, vars): for var in vars: @@ -349,8 +351,6 @@ self.xrm.position += 1 self.assembler.regalloc_perform_with_guard(op, guard_op, faillocs, arglocs, result_loc) - if op.result is not None: - self.possibly_free_var(op.result) self.possibly_free_vars(guard_op.getfailargs()) def perform_guard(self, guard_op, arglocs, result_loc): @@ -413,8 +413,7 @@ self._consider_force_spill(op) else: oplist[op.getopnum()](self, op) - if op.result is not None: - self.possibly_free_var(op.result) + self.possibly_free_vars_for_op(op) self.rm._check_invariants() self.xrm._check_invariants() i += 1 @@ -445,7 +444,6 @@ def _consider_guard(self, op): loc = self.rm.make_sure_var_in_reg(op.getarg(0)) self.perform_guard(op, [loc], None) - self.rm.possibly_free_var(op.getarg(0)) consider_guard_true = _consider_guard consider_guard_false = _consider_guard @@ -463,8 +461,6 @@ else: locs = [imm(fail_descr)] self.Perform(op, locs, None) - if op.numargs() == 1: - self.possibly_free_var(op.getarg(0)) def consider_guard_no_exception(self, op): self.perform_guard(op, [], None) @@ -490,7 +486,6 @@ else: resloc = None self.perform_guard(op, [loc, loc1], resloc) - self.rm.possibly_free_vars_for_op(op) self.rm.possibly_free_var(box) consider_guard_no_overflow = consider_guard_no_exception @@ -500,14 +495,12 @@ x = self.make_sure_var_in_reg(op.getarg(0)) y = self.loc(op.getarg(1)) self.perform_guard(op, [x, y], None) - self.possibly_free_vars_for_op(op) def consider_guard_class(self, op): assert isinstance(op.getarg(0), Box) x = self.rm.make_sure_var_in_reg(op.getarg(0)) y = self.loc(op.getarg(1)) self.perform_guard(op, [x, y], None) - self.rm.possibly_free_vars_for_op(op) consider_guard_nonnull_class = consider_guard_class @@ -522,7 +515,6 @@ def _consider_binop(self, op): loc, argloc = self._consider_binop_part(op) self.Perform(op, [loc, argloc], loc) - self.possibly_free_var(op.getarg(0)) def _consider_lea(self, op, loc): argloc = self.loc(op.getarg(1)) @@ -575,7 +567,6 @@ args = op.getarglist() loc1 = self.rm.force_result_in_reg(op.result, op.getarg(0), args) self.Perform(op, [loc1, loc2], loc1) - self.rm.possibly_free_vars_for_op(op) consider_int_rshift = consider_int_lshift consider_uint_rshift = consider_int_lshift @@ -591,7 +582,6 @@ assert l0 is eax assert l1 is ecx assert l2 is resultreg - self.rm.possibly_free_vars_for_op(op) self.rm.possibly_free_var(tmpvar) def consider_int_mod(self, op): @@ -640,7 +630,6 @@ args = op.getarglist() loc0 = self.xrm.force_result_in_reg(op.result, op.getarg(0), args) self.Perform(op, [loc0, loc1], loc0) - self.xrm.possibly_free_vars_for_op(op) consider_float_add = _consider_float_op consider_float_sub = _consider_float_op @@ -674,7 +663,6 @@ def _consider_float_unary_op(self, op): loc0 = self.xrm.force_result_in_reg(op.result, op.getarg(0)) self.Perform(op, [loc0], loc0) - self.xrm.possibly_free_var(op.getarg(0)) consider_float_neg = _consider_float_unary_op consider_float_abs = _consider_float_unary_op @@ -683,13 +671,11 @@ loc0 = self.xrm.make_sure_var_in_reg(op.getarg(0)) loc1 = self.rm.force_allocate_reg(op.result) self.Perform(op, [loc0], loc1) - self.xrm.possibly_free_var(op.getarg(0)) def consider_cast_int_to_float(self, op): loc0 = self.rm.make_sure_var_in_reg(op.getarg(0)) loc1 = self.xrm.force_allocate_reg(op.result) self.Perform(op, [loc0], loc1) - self.rm.possibly_free_var(op.getarg(0)) def consider_cast_float_to_singlefloat(self, op): loc0 = self.xrm.make_sure_var_in_reg(op.getarg(0)) @@ -707,26 +693,22 @@ loc0 = self.xrm.make_sure_var_in_reg(op.getarg(0)) loc1 = self.rm.force_allocate_reg(op.result) self.Perform(op, [loc0], loc1) - self.xrm.possibly_free_var(op.getarg(0)) else: arg0 = op.getarg(0) loc0 = self.xrm.loc(arg0) loc1 = self.xrm.force_allocate_reg(op.result, forbidden_vars=[arg0]) self.Perform(op, [loc0], loc1) - self.xrm.possibly_free_var(arg0) def consider_convert_longlong_bytes_to_float(self, op): if longlong.is_64_bit: loc0 = self.rm.make_sure_var_in_reg(op.getarg(0)) loc1 = self.xrm.force_allocate_reg(op.result) self.Perform(op, [loc0], loc1) - self.rm.possibly_free_var(op.getarg(0)) else: arg0 = op.getarg(0) loc0 = self.xrm.make_sure_var_in_reg(arg0) loc1 = self.xrm.force_allocate_reg(op.result, forbidden_vars=[arg0]) self.Perform(op, [loc0], loc1) - self.xrm.possibly_free_var(arg0) def _consider_llong_binop_xx(self, op): # must force both arguments into xmm registers, because we don't @@ -736,7 +718,6 @@ loc1 = self.load_xmm_aligned_16_bytes(args[1]) loc0 = self.xrm.force_result_in_reg(op.result, args[0], args) self.PerformLLong(op, [loc0, loc1], loc0) - self.xrm.possibly_free_vars(args) def _consider_llong_eq_ne_xx(self, op): # must force both arguments into xmm registers, because we don't @@ -750,7 +731,6 @@ self.xrm.possibly_free_var(tmpxvar) loc0 = self.rm.force_allocate_reg(op.result, need_lower_byte=True) self.PerformLLong(op, [loc1, loc2, loc3], loc0) - self.xrm.possibly_free_vars(args) def _maybe_consider_llong_lt(self, op): # XXX just a special case for now @@ -773,7 +753,6 @@ loc1 = self.xrm.loc(op.getarg(1)) loc0 = self.rm.force_allocate_reg(op.result) self.PerformLLong(op, [loc1], loc0) - self.xrm.possibly_free_var(op.getarg(1)) def _loc_of_const_longlong(self, value64): c = ConstFloat(value64) @@ -792,19 +771,16 @@ loc2 = self.xrm.force_allocate_reg(tmpxvar, [op.result]) self.xrm.possibly_free_var(tmpxvar) self.PerformLLong(op, [loc1, loc2], loc0) - self.rm.possibly_free_var(box) def _consider_llong_from_uint(self, op): assert IS_X86_32 loc0 = self.xrm.force_allocate_reg(op.result) loc1 = self.rm.make_sure_var_in_reg(op.getarg(1)) self.PerformLLong(op, [loc1], loc0) - self.rm.possibly_free_vars_for_op(op) def _consider_math_sqrt(self, op): loc0 = self.xrm.force_result_in_reg(op.result, op.getarg(1)) self.PerformMath(op, [loc0], loc0) - self.xrm.possibly_free_var(op.getarg(1)) def _call(self, op, arglocs, force_store=[], guard_not_forced_op=None): # we need to save registers on the stack: @@ -852,7 +828,6 @@ self._call(op, [imm(size), sign_loc] + [self.loc(op.getarg(i)) for i in range(op.numargs())], guard_not_forced_op=guard_not_forced_op) - self.possibly_free_vars(op.getarglist()) def consider_call(self, op): effectinfo = op.getdescr().get_extra_info() @@ -904,7 +879,6 @@ xxx self._call(op, [frame_loc, self.loc(op.getarg(0))], guard_not_forced_op=guard_op) - self.possibly_free_var(op.getarg(0)) def consider_cond_call_gc_wb(self, op): assert op.result is None @@ -916,7 +890,6 @@ arglocs = [self.rm.make_sure_var_in_reg(op.getarg(i), args) for i in range(N)] self.PerformDiscard(op, arglocs) - self.rm.possibly_free_vars_for_op(op) consider_cond_call_gc_wb_array = consider_cond_call_gc_wb @@ -952,7 +925,6 @@ base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args) value_loc = self.make_sure_var_in_reg(op.getarg(1), args, need_lower_byte=need_lower_byte) - self.possibly_free_vars(args) self.PerformDiscard(op, [base_loc, ofs_loc, size_loc, value_loc]) consider_setfield_raw = consider_setfield_gc @@ -998,7 +970,6 @@ ofs_loc = self.rm.make_sure_var_in_reg(op.getarg(1), args) value_loc = self.rm.make_sure_var_in_reg(op.getarg(2), args, need_lower_byte=True) - self.rm.possibly_free_vars_for_op(op) self.PerformDiscard(op, [base_loc, ofs_loc, value_loc]) consider_unicodesetitem = consider_strsetitem @@ -1014,7 +985,6 @@ value_loc = self.make_sure_var_in_reg(op.getarg(2), args, need_lower_byte=need_lower_byte) ofs_loc = self.rm.make_sure_var_in_reg(op.getarg(1), args) - self.possibly_free_vars(args) self.PerformDiscard(op, [base_loc, ofs_loc, value_loc, imm(itemsize), imm(ofs)]) From noreply at buildbot.pypy.org Thu Jan 17 19:11:22 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 17 Jan 2013 19:11:22 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: avoid reallocating in the "equal" case Message-ID: <20130117181122.408691C1167@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60141:a4fa587b75f0 Date: 2013-01-17 20:10 +0200 http://bitbucket.org/pypy/pypy/changeset/a4fa587b75f0/ Log: avoid reallocating in the "equal" case diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -678,7 +678,7 @@ self.mc.CMP_bi(ofs - base_ofs, 0xffffff) stack_check_cmp_ofs = self.mc.get_relative_pos() - 4 assert not IS_X86_32 - self.mc.J_il8(rx86.Conditions['G'], 0) + self.mc.J_il8(rx86.Conditions['GE'], 0) jg_location = self.mc.get_relative_pos() self.mc.CALL(imm(self._stack_check_failure)) # patch the JG above From noreply at buildbot.pypy.org Thu Jan 17 20:56:42 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 17 Jan 2013 20:56:42 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fix maybe? for now commit with debug checks Message-ID: <20130117195642.3FF861C12FC@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60142:0d6f2ebbb86c Date: 2013-01-17 21:56 +0200 http://bitbucket.org/pypy/pypy/changeset/0d6f2ebbb86c/ Log: fix maybe? for now commit with debug checks diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py --- a/pypy/jit/backend/llsupport/gc.py +++ b/pypy/jit/backend/llsupport/gc.py @@ -118,10 +118,12 @@ descrs = JitFrameDescrs() descrs.arraydescr = cpu.arraydescrof(jitframe.JITFRAME) for name in ['jf_descr', 'jf_guard_exc', 'jf_force_descr', - 'jf_frame_info', 'jf_gcpattern']: + 'jf_frame_info', 'jf_gcpattern', 'jf_gcmap']: setattr(descrs, name, cpu.fielddescrof(jitframe.JITFRAME, name)) descrs.jfi_frame_depth = cpu.fielddescrof(jitframe.JITFRAMEINFO, 'jfi_frame_depth') + descrs.jfi_gcmap = cpu.fielddescrof(jitframe.JITFRAMEINFO, + 'jfi_gcmap') return descrs def getarraydescr_for_frame(self, type, index): diff --git a/pypy/jit/backend/llsupport/jitframe.py b/pypy/jit/backend/llsupport/jitframe.py --- a/pypy/jit/backend/llsupport/jitframe.py +++ b/pypy/jit/backend/llsupport/jitframe.py @@ -15,6 +15,7 @@ # gcindexlist is a list of indexes of GC ptrs # in the actual array jf_frame of JITFRAME ('jfi_gcmap', lltype.Ptr(GCMAP)), + ('counter', lltype.Signed), ) NULLFRAMEINFO = lltype.nullptr(JITFRAMEINFO) diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py --- a/pypy/jit/backend/llsupport/llmodel.py +++ b/pypy/jit/backend/llsupport/llmodel.py @@ -52,12 +52,13 @@ pass def _setup_frame_realloc(self): - FUNC_TP = lltype.Ptr(lltype.FuncType([llmemory.GCREF], + FUNC_TP = lltype.Ptr(lltype.FuncType([llmemory.GCREF, lltype.Signed], llmemory.GCREF)) - def realloc_frame(frame): + def realloc_frame(frame, size): frame = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, frame) new_frame = frame.copy() + assert frame.jf_frame_info.jfi_frame_depth >= size # XXX now we know, rewrite this # we need to do this, because we're not sure what things # are GC pointers and which ones are not diff --git a/pypy/jit/backend/llsupport/rewrite.py b/pypy/jit/backend/llsupport/rewrite.py --- a/pypy/jit/backend/llsupport/rewrite.py +++ b/pypy/jit/backend/llsupport/rewrite.py @@ -8,7 +8,7 @@ from pypy.jit.backend.llsupport.descr import SizeDescr, ArrayDescr from pypy.jit.backend.llsupport import jitframe from pypy.rpython.lltypesystem import lltype, llmemory - +from pypy.rlib import rgc class GcRewriterAssembler(object): # This class performs the following rewrites on the list of operations: diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -78,6 +78,7 @@ self.propagate_exception_path = 0 self.gcrootmap_retaddr_forced = 0 self.teardown() + self.counter = 0 def set_debug(self, v): r = self._debug @@ -514,6 +515,8 @@ clt = CompiledLoopToken(self.cpu, looptoken.number) clt.frame_info = lltype.malloc(jitframe.JITFRAMEINFO) + clt.frame_info.counter = self.counter + self.counter += 1 clt.allgcrefs = [] clt.frame_info.jfi_frame_depth = 0 # for now looptoken.compiled_loop_token = clt @@ -530,8 +533,10 @@ # self._call_header_with_stack_check() clt._debug_nbargs = len(inputargs) - operations = regalloc.prepare_loop(inputargs, operations, - looptoken, clt.allgcrefs) + operations = regalloc.prepare_loop(inputargs, operations, looptoken, + clt.allgcrefs) + rgc._make_sure_does_not_move(clt.frame_info) + self._insert_frame_adjustment(clt.frame_info) looppos = self.mc.get_relative_pos() looptoken._x86_loop_code = looppos frame_depth = self._assemble(regalloc, inputargs, operations) @@ -589,7 +594,7 @@ operations, self.current_clt.allgcrefs, self.current_clt.frame_info) - stack_check_patch_ofs = self._check_frame_depth() + stack_check_patch_ofs, stack_check_patch_ofs2 = self._check_frame_depth() frame_depth = self._assemble(regalloc, inputargs, operations) codeendpos = self.mc.get_relative_pos() self.write_pending_failure_recoveries() @@ -610,8 +615,15 @@ frame_depth = max(self.current_clt.frame_info.jfi_frame_depth, frame_depth + JITFRAME_FIXED_SIZE) self._patch_stackadjust(stack_check_patch_ofs + rawstart, frame_depth) + self._patch_stackadjust(stack_check_patch_ofs2 + rawstart, frame_depth) self.fixup_target_tokens(rawstart) + print frame_depth + print self.current_clt + print self.current_clt.frame_info + print self.current_clt.frame_info.jfi_frame_depth + print self.current_clt.frame_info.counter self.current_clt.frame_info.jfi_frame_depth = frame_depth + print self.current_clt.frame_info.jfi_frame_depth self._update_gcmap(self.current_clt.frame_info, regalloc) self.teardown() # oprofile support @@ -680,12 +692,22 @@ assert not IS_X86_32 self.mc.J_il8(rx86.Conditions['GE'], 0) jg_location = self.mc.get_relative_pos() + self.mc.MOV_ri(esi.value, 0xffffff) + ofs = self.mc.get_relative_pos() - 4 self.mc.CALL(imm(self._stack_check_failure)) # patch the JG above offset = self.mc.get_relative_pos() - jg_location assert 0 < offset <= 127 self.mc.overwrite(jg_location-1, chr(offset)) - return stack_check_cmp_ofs + return stack_check_cmp_ofs, ofs + + def _insert_frame_adjustment(self, frame_info): + gcmap_ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') + frame_info_addr = rffi.cast(lltype.Signed, frame_info) + frame_info_ofs = self.cpu.get_ofs_of_frame_field('jf_frame_info') + jfi_gc_map_ofs = self.cpu.get_ofs_of_frame_field('jfi_gcmap') + self.mc.MOV_bi(gcmap_ofs, frame_info_addr + jfi_gc_map_ofs) + self.mc.MOV_bi(frame_info_ofs, frame_info_addr) def _patch_stackadjust(self, adr, allocated_depth): mc = codebuf.MachineCodeBlockWrapper() @@ -802,8 +824,6 @@ return frame_depth def _call_header(self): - # NB. the shape of the frame is hard-coded in get_basic_shape() too. - # Also, make sure this is consistent with FRAME_FIXED_SIZE. # XXX should be LEA? self.mc.SUB_ri(esp.value, FRAME_FIXED_SIZE * WORD) self.mc.MOV_sr(PASS_ON_MY_FRAME * WORD, ebp.value) @@ -1960,6 +1980,9 @@ # exit function self._call_footer() + def genop_discard_label(self, op, arglocs): + self._insert_frame_adjustment(self.current_clt.frame_info) + def implement_guard(self, guard_token, condition=None): # These jumps are patched later. if condition: diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py --- a/pypy/jit/backend/x86/regalloc.py +++ b/pypy/jit/backend/x86/regalloc.py @@ -1295,39 +1295,6 @@ # This operation is used only for testing self.force_spill_var(op.getarg(0)) - def get_mark_gc_roots(self, gcrootmap, use_copy_area=False): - shape = gcrootmap.get_basic_shape() - for v, val in self.fm.bindings.items(): - if (isinstance(v, BoxPtr) and self.rm.stays_alive(v)): - assert isinstance(val, StackLoc) - gcrootmap.add_frame_offset(shape, get_ebp_ofs(val.position)) - for v, reg in self.rm.reg_bindings.items(): - if reg is eax: - continue # ok to ignore this one - if (isinstance(v, BoxPtr) and self.rm.stays_alive(v)): - # - # The register 'reg' is alive across this call. - gcrootmap = self.assembler.cpu.gc_ll_descr.gcrootmap - if gcrootmap is None or not gcrootmap.is_shadow_stack: - # - # Asmgcc: if reg is a callee-save register, we can - # explicitly mark it as containing a BoxPtr. - if reg in self.rm.REGLOC_TO_GCROOTMAP_REG_INDEX: - gcrootmap.add_callee_save_reg( - shape, self.rm.REGLOC_TO_GCROOTMAP_REG_INDEX[reg]) - continue - # - # Else, 'use_copy_area' must be True (otherwise this BoxPtr - # should not be in a register). The copy area contains the - # real value of the register. - assert use_copy_area - assert reg in self.rm.REGLOC_TO_COPY_AREA_OFS - area_offset = self.rm.REGLOC_TO_COPY_AREA_OFS[reg] - gcrootmap.add_frame_offset(shape, area_offset) - # - return gcrootmap.compress_callshape(shape, - self.assembler.datablockwrapper) - def consider_force_token(self, op): # the FORCE_TOKEN operation returns directly 'ebp' self.rm.force_allocate_frame_reg(op.result) @@ -1378,9 +1345,12 @@ # end of the same loop, i.e. if what we are compiling is a single # loop that ends up jumping to this LABEL, then we can now provide # the hints about the expected position of the spilled variables. - jump_op = self.final_jump_op - if jump_op is not None and jump_op.getdescr() is descr: - self._compute_hint_frame_locations_from_descr(descr) + + self.PerformDiscard(op, []) + # XXX we never compile code like that? + #jump_op = self.final_jump_op + #if jump_op is not None and jump_op.getdescr() is descr: + # self._compute_hint_frame_locations_from_descr(descr) def consider_keepalive(self, op): pass diff --git a/pypy/jit/backend/x86/runner.py b/pypy/jit/backend/x86/runner.py --- a/pypy/jit/backend/x86/runner.py +++ b/pypy/jit/backend/x86/runner.py @@ -205,7 +205,10 @@ @specialize.arg(1) def get_ofs_of_frame_field(self, name): descrs = self.gc_ll_descr.getframedescrs(self) - base_ofs = self.unpack_arraydescr(descrs.arraydescr) + if name.startswith('jfi_'): + base_ofs = 0 # not relative to frame + else: + base_ofs = self.unpack_arraydescr(descrs.arraydescr) ofs = self.unpack_fielddescr(getattr(descrs, name)) return ofs - base_ofs From noreply at buildbot.pypy.org Thu Jan 17 20:58:26 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 17 Jan 2013 20:58:26 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: kill some prints, leave debugging on for now Message-ID: <20130117195826.D578E1C12FC@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60143:84877fe3c441 Date: 2013-01-17 21:58 +0200 http://bitbucket.org/pypy/pypy/changeset/84877fe3c441/ Log: kill some prints, leave debugging on for now diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -617,13 +617,7 @@ self._patch_stackadjust(stack_check_patch_ofs + rawstart, frame_depth) self._patch_stackadjust(stack_check_patch_ofs2 + rawstart, frame_depth) self.fixup_target_tokens(rawstart) - print frame_depth - print self.current_clt - print self.current_clt.frame_info - print self.current_clt.frame_info.jfi_frame_depth - print self.current_clt.frame_info.counter self.current_clt.frame_info.jfi_frame_depth = frame_depth - print self.current_clt.frame_info.jfi_frame_depth self._update_gcmap(self.current_clt.frame_info, regalloc) self.teardown() # oprofile support From noreply at buildbot.pypy.org Thu Jan 17 21:16:42 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Thu, 17 Jan 2013 21:16:42 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: hg merge default Message-ID: <20130117201642.3ABB11C00BE@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60144:8d03ae695f01 Date: 2013-01-17 21:08 +0100 http://bitbucket.org/pypy/pypy/changeset/8d03ae695f01/ Log: hg merge default diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py --- a/pypy/module/micronumpy/arrayimpl/concrete.py +++ b/pypy/module/micronumpy/arrayimpl/concrete.py @@ -9,7 +9,7 @@ from pypy.interpreter.error import OperationError, operationerrfmt from rpython.rtyper.lltypesystem import rffi, lltype from rpython.rlib import jit -from rpython.rlib.rawstorage import free_raw_storage +from rpython.rlib.rawstorage import free_raw_storage, RAW_STORAGE from rpython.rlib.debug import make_sure_not_resized class ConcreteArrayIterator(base.BaseArrayIterator): @@ -427,18 +427,18 @@ def get_storage_as_int(self, space): return rffi.cast(lltype.Signed, self.storage) -class ConcreteArray(BaseConcreteArray): - def __init__(self, shape, dtype, order, strides, backstrides): +class ConcreteArrayNotOwning(BaseConcreteArray): + def __init__(self, shape, dtype, order, strides, backstrides, storage): make_sure_not_resized(shape) make_sure_not_resized(strides) make_sure_not_resized(backstrides) self.shape = shape self.size = support.product(shape) * dtype.get_size() - self.storage = dtype.itemtype.malloc(self.size) self.order = order self.dtype = dtype self.strides = strides self.backstrides = backstrides + self.storage = storage def create_iter(self, shape=None): if shape is None or shape == self.get_shape(): @@ -451,14 +451,24 @@ def fill(self, box): self.dtype.fill(self.storage, box, 0, self.size) - def __del__(self): - free_raw_storage(self.storage, track_allocation=False) - def set_shape(self, space, new_shape): strides, backstrides = support.calc_strides(new_shape, self.dtype, self.order) return SliceArray(0, strides, backstrides, new_shape, self) +class ConcreteArray(ConcreteArrayNotOwning): + def __init__(self, shape, dtype, order, strides, backstrides): + # we allocate the actual storage later because we need to compute + # self.size first + null_storage = lltype.nullptr(RAW_STORAGE) + ConcreteArrayNotOwning.__init__(self, shape, dtype, order, strides, backstrides, + null_storage) + self.storage = dtype.itemtype.malloc(self.size) + + def __del__(self): + free_raw_storage(self.storage, track_allocation=False) + + class NonWritableArray(ConcreteArray): def descr_setitem(self, space, w_index, w_value): raise OperationError(space.w_RuntimeError, space.wrap( diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py --- a/pypy/module/micronumpy/base.py +++ b/pypy/module/micronumpy/base.py @@ -28,6 +28,16 @@ return W_NDimArray(impl) @staticmethod + def from_shape_and_storage(shape, storage, dtype, order='C'): + from pypy.module.micronumpy.arrayimpl import concrete + assert shape + strides, backstrides = calc_strides(shape, dtype, order) + impl = concrete.ConcreteArrayNotOwning(shape, dtype, order, strides, + backstrides, storage) + return W_NDimArray(impl) + + + @staticmethod def new_slice(offset, strides, backstrides, shape, parent, dtype=None): from pypy.module.micronumpy.arrayimpl import concrete diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -397,7 +397,6 @@ space.w_False])) return w_d - # --------------------- operations ---------------------------- def _unaryop_impl(ufunc_name): @@ -565,6 +564,20 @@ return W_NDimArray.new_scalar(space, dtype) return W_NDimArray.from_shape(shape, dtype) + at unwrap_spec(addr=int) +def descr__from_shape_and_storage(space, w_cls, w_shape, addr, w_dtype): + """ + Create an array from an existing buffer, given its address as int. + PyPy-only implementation detail. + """ + from pypy.rpython.lltypesystem import rffi + from pypy.rlib.rawstorage import RAW_STORAGE_PTR + shape = _find_shape(space, w_shape) + storage = rffi.cast(RAW_STORAGE_PTR, addr) + dtype = space.interp_w(interp_dtype.W_Dtype, + space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)) + return W_NDimArray.from_shape_and_storage(shape, storage, dtype) + W_NDimArray.typedef = TypeDef( "ndarray", __new__ = interp2app(descr_new_array), @@ -661,6 +674,8 @@ imag = GetSetProperty(W_NDimArray.descr_get_imag, W_NDimArray.descr_set_imag), __array_interface__ = GetSetProperty(W_NDimArray.descr_array_iface), + _from_shape_and_storage = interp2app(descr__from_shape_and_storage, + as_classmethod=True) ) @unwrap_spec(ndmin=int, copy=bool, subok=bool) diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -189,7 +189,23 @@ assert shape == [2] assert space.str_w(elems[0]) == "a" assert space.str_w(elems[1]) == "b" - + + def test_from_shape_and_storage(self): + from pypy.rlib.rawstorage import alloc_raw_storage, raw_storage_setitem + from pypy.rpython.lltypesystem import rffi + from pypy.module.micronumpy.interp_dtype import get_dtype_cache + storage = alloc_raw_storage(4, track_allocation=False, zero=True) + for i in range(4): + raw_storage_setitem(storage, i, rffi.cast(rffi.UCHAR, i)) + # + dtypes = get_dtype_cache(self.space) + w_array = W_NDimArray.from_shape_and_storage([2, 2], storage, dtypes.w_int8dtype) + def get(i, j): + return w_array.getitem(self.space, [i, j]).value + assert get(0, 0) == 0 + assert get(0, 1) == 1 + assert get(1, 0) == 2 + assert get(1, 1) == 3 class AppTestNumArray(BaseNumpyAppTest): def w_CustomIndexObject(self, index): @@ -2340,7 +2356,7 @@ s = repr(a) assert s.replace('\n', '') == \ "array(['abc', 'defg', 'ab'], dtype='|S4')" - + class AppTestPyPy(BaseNumpyAppTest): def setup_class(cls): @@ -2360,3 +2376,12 @@ assert a[0][1] == 2 a = _numpypy.array(([[[1, 2], [3, 4], [5, 6]]])) assert (a[0, 1] == [3, 4]).all() + + def test_from_shape_and_storage(self): + from _numpypy import array, ndarray + x = array([1, 2, 3, 4]) + addr, _ = x.__array_interface__['data'] + y = ndarray._from_shape_and_storage([2, 2], addr, x.dtype) + assert y[0, 1] == 2 + y[0, 1] = 42 + assert x[1] == 42 diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -5,7 +5,7 @@ from rpython.rlib.objectmodel import we_are_translated from rpython.rlib.debug import debug_start, debug_stop, debug_print from rpython.rlib import rstack -from rpython.rlib.jit import JitDebugInfo, Counters +from rpython.rlib.jit import JitDebugInfo, Counters, dont_look_inside from rpython.conftest import option from rpython.tool.sourcetools import func_with_new_name @@ -685,6 +685,7 @@ assert 0, "unreachable" @staticmethod + @dont_look_inside def force_now(cpu, token): # Called during a residual call from the assembler, if the code # actually needs to force one of the virtualrefs or the virtualizable. diff --git a/rpython/jit/metainterp/virtualref.py b/rpython/jit/metainterp/virtualref.py --- a/rpython/jit/metainterp/virtualref.py +++ b/rpython/jit/metainterp/virtualref.py @@ -169,4 +169,3 @@ # token == TOKEN_NONE and the vref was not forced: it's invalid raise InvalidVirtualRef return vref.forced - force_virtual._dont_inline_ = True From noreply at buildbot.pypy.org Thu Jan 17 21:16:43 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Thu, 17 Jan 2013 21:16:43 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Apparently these files were removed Message-ID: <20130117201643.9E2CD1C00BE@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60145:d55a0263448b Date: 2013-01-17 21:13 +0100 http://bitbucket.org/pypy/pypy/changeset/d55a0263448b/ Log: Apparently these files were removed diff --git a/rpython/jit/backend/x86/test/test_support.py b/rpython/jit/backend/x86/test/test_support.py deleted file mode 100644 --- a/rpython/jit/backend/x86/test/test_support.py +++ /dev/null @@ -1,21 +0,0 @@ - -from rpython.jit.backend.x86.support import values_array -from rpython.rtyper.lltypesystem import lltype, llmemory, rffi - -def test_values_array_signed(): - ar = values_array(lltype.Signed, 50) - adr = ar.get_addr_for_num(10) - rffi.cast(rffi.CArrayPtr(lltype.Signed), adr)[0] = 42 - assert ar.getitem(10) == 42 - ar.setitem(42, 38) - adr = ar.get_addr_for_num(42) - assert rffi.cast(rffi.CArrayPtr(lltype.Signed), adr)[0] == 38 - -def test_values_array_float(): - ar = values_array(lltype.Float, 50) - adr = ar.get_addr_for_num(10) - rffi.cast(rffi.CArrayPtr(lltype.Float), adr)[0] = 42.5 - assert ar.getitem(10) == 42.5 - ar.setitem(42, 38.5) - adr = ar.get_addr_for_num(42) - assert rffi.cast(rffi.CArrayPtr(lltype.Float), adr)[0] == 38.5 diff --git a/rpython/jit/metainterp/test/test_ztranslation.py b/rpython/jit/metainterp/test/test_ztranslation.py deleted file mode 100644 --- a/rpython/jit/metainterp/test/test_ztranslation.py +++ /dev/null @@ -1,154 +0,0 @@ -import py -from rpython.jit.metainterp.warmspot import rpython_ll_meta_interp, ll_meta_interp -from rpython.jit.backend.llgraph import runner -from rpython.rlib.jit import JitDriver, unroll_parameters, set_param -from rpython.rlib.jit import PARAMETERS, dont_look_inside, hint -from rpython.rlib.jit_hooks import boxint_new, resop_new, resop_getopnum -from rpython.jit.metainterp.jitprof import Profiler -from rpython.jit.metainterp.resoperation import rop -from rpython.rtyper.lltypesystem import lltype, llmemory - -class TranslationTest: - - CPUClass = None - type_system = None - - def test_stuff_translates(self): - # this is a basic test that tries to hit a number of features and their - # translation: - # - jitting of loops and bridges - # - virtualizables - # - set_param interface - # - profiler - # - full optimizer - # - jitdriver hooks - # - two JITs - # - string concatenation, slicing and comparison - # - jit hooks interface - - class Frame(object): - _virtualizable2_ = ['l[*]'] - - def __init__(self, i): - self = hint(self, fresh_virtualizable=True, - access_directly=True) - self.l = [i] - - class OtherFrame(object): - _virtualizable2_ = ['i', 'l[*]'] - - def __init__(self, i): - self = hint(self, fresh_virtualizable=True, - access_directly=True) - self.i = i - self.l = [float(i)] - - class JitCellCache: - entry = None - jitcellcache = JitCellCache() - def set_jitcell_at(entry): - jitcellcache.entry = entry - def get_jitcell_at(): - return jitcellcache.entry - def get_printable_location(): - return '(hello world)' - - jitdriver = JitDriver(greens = [], reds = ['total', 'frame'], - virtualizables = ['frame'], - get_jitcell_at=get_jitcell_at, - set_jitcell_at=set_jitcell_at, - get_printable_location=get_printable_location) - def f(i): - for param, defl in unroll_parameters: - set_param(jitdriver, param, defl) - set_param(jitdriver, "threshold", 3) - set_param(jitdriver, "trace_eagerness", 2) - total = 0 - frame = Frame(i) - while frame.l[0] > 3: - jitdriver.can_enter_jit(frame=frame, total=total) - jitdriver.jit_merge_point(frame=frame, total=total) - total += frame.l[0] - if frame.l[0] >= 20: - frame.l[0] -= 2 - frame.l[0] -= 1 - return total * 10 - # - myjitdriver2 = JitDriver(greens = ['g'], - reds = ['m', 's', 'f', 'float_s'], - virtualizables = ['f']) - def f2(g, m, x): - s = "" - f = OtherFrame(x) - float_s = 0.0 - while m > 0: - myjitdriver2.can_enter_jit(g=g, m=m, f=f, s=s, float_s=float_s) - myjitdriver2.jit_merge_point(g=g, m=m, f=f, s=s, - float_s=float_s) - s += 'xy' - if s[:2] == 'yz': - return -666 - m -= 1 - f.i += 3 - float_s += f.l[0] - return f.i - # - def main(i, j): - op = resop_new(rop.INT_ADD, [boxint_new(3), boxint_new(5)], - boxint_new(8)) - return f(i) - f2(i+j, i, j) + resop_getopnum(op) - res = ll_meta_interp(main, [40, 5], CPUClass=self.CPUClass, - type_system=self.type_system, - listops=True) - assert res == main(40, 5) - res = rpython_ll_meta_interp(main, [40, 5], - CPUClass=self.CPUClass, - type_system=self.type_system, - ProfilerClass=Profiler, - listops=True) - assert res == main(40, 5) - - def test_external_exception_handling_translates(self): - jitdriver = JitDriver(greens = [], reds = ['n', 'total']) - - @dont_look_inside - def f(x): - if x > 20: - return 2 - raise ValueError - @dont_look_inside - def g(x): - if x > 15: - raise ValueError - return 2 - def main(i): - set_param(jitdriver, "threshold", 3) - set_param(jitdriver, "trace_eagerness", 2) - total = 0 - n = i - while n > 3: - jitdriver.can_enter_jit(n=n, total=total) - jitdriver.jit_merge_point(n=n, total=total) - try: - total += f(n) - except ValueError: - total += 1 - try: - total += g(n) - except ValueError: - total -= 1 - n -= 1 - return total * 10 - res = ll_meta_interp(main, [40], CPUClass=self.CPUClass, - type_system=self.type_system) - assert res == main(40) - res = rpython_ll_meta_interp(main, [40], CPUClass=self.CPUClass, - type_system=self.type_system, - enable_opts='', - ProfilerClass=Profiler) - assert res == main(40) - -class TestTranslationLLtype(TranslationTest): - - CPUClass = runner.LLtypeCPU - type_system = 'lltype' From noreply at buildbot.pypy.org Thu Jan 17 21:16:44 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Thu, 17 Jan 2013 21:16:44 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed tests breaking Message-ID: <20130117201644.BE53B1C00BE@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60146:214f65e6d97c Date: 2013-01-17 21:16 +0100 http://bitbucket.org/pypy/pypy/changeset/214f65e6d97c/ Log: Fixed tests breaking diff --git a/rpython/conftest.py b/rpython/conftest.py --- a/rpython/conftest.py +++ b/rpython/conftest.py @@ -7,7 +7,7 @@ # i'll investigate differences in collection tommorow, for now, can you just # import the makemodule hook from pypy and add a comment that im responsible # for fixing? -from pypy.conftest import pytest_pycollect_makemodule +#from pypy.conftest import pytest_pycollect_makemodule cdir = realpath(join(dirname(__file__), 'translator', 'c')) cache_dir = realpath(join(dirname(__file__), '_cache')) From noreply at buildbot.pypy.org Thu Jan 17 21:45:20 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Thu, 17 Jan 2013 21:45:20 +0100 (CET) Subject: [pypy-commit] pypy py3k: kill _codecs.charbuffer_encode Message-ID: <20130117204520.844201C1167@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60147:297b96d813af Date: 2013-01-17 12:44 -0800 http://bitbucket.org/pypy/pypy/changeset/297b96d813af/ Log: kill _codecs.charbuffer_encode diff --git a/pypy/module/_codecs/__init__.py b/pypy/module/_codecs/__init__.py --- a/pypy/module/_codecs/__init__.py +++ b/pypy/module/_codecs/__init__.py @@ -72,7 +72,6 @@ 'utf_32_le_decode' : 'interp_codecs.utf_32_le_decode', 'utf_32_le_encode' : 'interp_codecs.utf_32_le_encode', 'utf_32_ex_decode' : 'interp_codecs.utf_32_ex_decode', - 'charbuffer_encode': 'interp_codecs.buffer_encode', 'readbuffer_encode': 'interp_codecs.buffer_encode', 'charmap_decode' : 'interp_codecs.charmap_decode', 'charmap_encode' : 'interp_codecs.charmap_encode', From noreply at buildbot.pypy.org Thu Jan 17 21:47:13 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Thu, 17 Jan 2013 21:47:13 +0100 (CET) Subject: [pypy-commit] pypy py3k: special builtin methods are an impl detail Message-ID: <20130117204713.6EFD91C1167@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60148:9b95b162ea88 Date: 2013-01-17 12:46 -0800 http://bitbucket.org/pypy/pypy/changeset/9b95b162ea88/ Log: special builtin methods are an impl detail diff --git a/lib-python/3.2/test/test_reprlib.py b/lib-python/3.2/test/test_reprlib.py --- a/lib-python/3.2/test/test_reprlib.py +++ b/lib-python/3.2/test/test_reprlib.py @@ -8,7 +8,7 @@ import shutil import unittest -from test.support import run_unittest +from test.support import check_impl_detail, run_unittest from reprlib import repr as r # Don't shadow builtin repr from reprlib import Repr from reprlib import recursive_repr @@ -138,8 +138,9 @@ # Functions eq(repr(hash), '') # Methods - self.assertTrue(repr(''.split).startswith( - '") + if check_impl_detail(pypy=False): + eq(repr(dict.items), "") # XXX member descriptors # XXX attribute descriptors # XXX slot descriptors From noreply at buildbot.pypy.org Thu Jan 17 21:51:39 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Thu, 17 Jan 2013 21:51:39 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed missing import Message-ID: <20130117205139.C97671C12FA@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60149:984aeff7a78d Date: 2013-01-17 21:49 +0100 http://bitbucket.org/pypy/pypy/changeset/984aeff7a78d/ Log: Fixed missing import diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -570,8 +570,8 @@ Create an array from an existing buffer, given its address as int. PyPy-only implementation detail. """ - from pypy.rpython.lltypesystem import rffi - from pypy.rlib.rawstorage import RAW_STORAGE_PTR + from rpython.rtyper.lltypesystem import rffi + from rpython.rlib.rawstorage import RAW_STORAGE_PTR shape = _find_shape(space, w_shape) storage = rffi.cast(RAW_STORAGE_PTR, addr) dtype = space.interp_w(interp_dtype.W_Dtype, From noreply at buildbot.pypy.org Thu Jan 17 21:51:40 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Thu, 17 Jan 2013 21:51:40 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: To fix later Message-ID: <20130117205140.F224C1C12FA@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60150:48db825bde1b Date: 2013-01-17 21:51 +0100 http://bitbucket.org/pypy/pypy/changeset/48db825bde1b/ Log: To fix later diff --git a/rpython/conftest.py b/rpython/conftest.py --- a/rpython/conftest.py +++ b/rpython/conftest.py @@ -7,7 +7,7 @@ # i'll investigate differences in collection tommorow, for now, can you just # import the makemodule hook from pypy and add a comment that im responsible # for fixing? -#from pypy.conftest import pytest_pycollect_makemodule +from pypy.conftest import pytest_pycollect_makemodule cdir = realpath(join(dirname(__file__), 'translator', 'c')) cache_dir = realpath(join(dirname(__file__), '_cache')) @@ -38,6 +38,15 @@ set_platform(value, None) def pytest_addoption(parser): + # XXX + # Aquana, i just ensured that with the pypy conftest we get all skips, + # i'll investigate differences in collection tommorow, for now, can you just + # import the makemodule hook from pypy and add a comment that im responsible + # for fixing? + from pypy.conftest import pytest_addoption + pytest_addoption(parser) + + group = parser.getgroup("rpython options") group.addoption('--view', action="store_true", dest="view", default=False, help="view translation tests' flow graphs with Pygame") From noreply at buildbot.pypy.org Thu Jan 17 21:52:24 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 17 Jan 2013 21:52:24 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: maaaaaybe? Message-ID: <20130117205224.437761C12FA@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60151:0805c40ffe53 Date: 2013-01-17 22:52 +0200 http://bitbucket.org/pypy/pypy/changeset/0805c40ffe53/ Log: maaaaaybe? diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -1909,7 +1909,7 @@ if withfloats: # Push all XMM regs ofs = len(gpr_reg_mgr_cls.all_regs) - for i in range(self.cpu.NUM_REGS): + for i in range(len(xmm_reg_mgr_cls.all_regs)): mc.MOVSD_bx((ofs + i) * WORD, i) def _pop_all_regs_from_frame(self, mc, withfloats): @@ -1919,7 +1919,7 @@ if withfloats: # Pop all XMM regs ofs = len(gpr_reg_mgr_cls.all_regs) - for i in range(self.cpu.NUM_REGS): + for i in range(len(xmm_reg_mgr_cls.all_regs)): mc.MOVSD_xb(i, (ofs + i) * WORD) def _build_failure_recovery(self, exc, withfloats=False): diff --git a/pypy/jit/backend/x86/test/test_gc_integration.py b/pypy/jit/backend/x86/test/test_gc_integration.py --- a/pypy/jit/backend/x86/test/test_gc_integration.py +++ b/pypy/jit/backend/x86/test/test_gc_integration.py @@ -14,27 +14,6 @@ CPU = getcpuclass() -class MockGcRootMap(object): - is_shadow_stack = False - def get_basic_shape(self): - return ['shape'] - def add_frame_offset(self, shape, offset): - shape.append(offset) - def add_callee_save_reg(self, shape, reg_index): - index_to_name = { 1: 'ebx', 2: 'esi', 3: 'edi' } - shape.append(index_to_name[reg_index]) - def compress_callshape(self, shape, datablockwrapper): - assert datablockwrapper == 'fakedatablockwrapper' - assert shape[0] == 'shape' - return ['compressed'] + shape[1:] - -class MockGcDescr(GcLLDescr_boehm): - gcrootmap = MockGcRootMap() - -class TestDirectGcIntegration(object): - def test_gcmap(self): - pass - class TestRegallocGcIntegration(BaseTestRegalloc): cpu = CPU(None, None) From noreply at buildbot.pypy.org Thu Jan 17 21:53:42 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 17 Jan 2013 21:53:42 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: er, we're not aligning frames here Message-ID: <20130117205342.48E2D1C12FA@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60152:d381f1ced67c Date: 2013-01-17 22:53 +0200 http://bitbucket.org/pypy/pypy/changeset/d381f1ced67c/ Log: er, we're not aligning frames here diff --git a/pypy/jit/backend/x86/arch.py b/pypy/jit/backend/x86/arch.py --- a/pypy/jit/backend/x86/arch.py +++ b/pypy/jit/backend/x86/arch.py @@ -48,12 +48,12 @@ SAVED_REGISTERS = 1 # range(1, 5) MY_COPY_OF_REGS = 5 # range(5, 9) XXX - JITFRAME_FIXED_SIZE = 29 # 13 GPR + 15 XMM + one word for alignment + JITFRAME_FIXED_SIZE = 28 # 13 GPR + 15 XMM else: # rbp + rbx + r12 + r13 + r14 + r15 + 12 extra words + return address = 19 FRAME_FIXED_SIZE = 19 PASS_ON_MY_FRAME = 12 - JITFRAME_FIXED_SIZE = 29 # 13 GPR + 15 XMM + one word for alignment + JITFRAME_FIXED_SIZE = 28 # 13 GPR + 15 XMM # "My copy of regs" has room for almost all registers, apart from eax and edx # which are used in the malloc itself. They are: From noreply at buildbot.pypy.org Thu Jan 17 21:56:02 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 17 Jan 2013 21:56:02 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: use scratch reg on 64bit Message-ID: <20130117205602.CF3621C12FA@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60153:407182a84d79 Date: 2013-01-17 22:55 +0200 http://bitbucket.org/pypy/pypy/changeset/407182a84d79/ Log: use scratch reg on 64bit diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -700,8 +700,16 @@ frame_info_addr = rffi.cast(lltype.Signed, frame_info) frame_info_ofs = self.cpu.get_ofs_of_frame_field('jf_frame_info') jfi_gc_map_ofs = self.cpu.get_ofs_of_frame_field('jfi_gcmap') - self.mc.MOV_bi(gcmap_ofs, frame_info_addr + jfi_gc_map_ofs) - self.mc.MOV_bi(frame_info_ofs, frame_info_addr) + if IS_X86_32: + self.mc.MOV_bi(gcmap_ofs, frame_info_addr + jfi_gc_map_ofs) + self.mc.MOV_bi(frame_info_ofs, frame_info_addr) + else: + self.mc.MOV_ri(X86_64_SCRATCH_REG.value, + frame_info_addr + jfi_gc_map_ofs) + self.mc.MOV_br(gcmap_ofs, X86_64_SCRATCH_REG.value) + self.mc.MOV_ri(X86_64_SCRATCH_REG.value, frame_info_addr) + self.mc.MOV_br(frame_info_ofs, X86_64_SCRATCH_REG.value) + def _patch_stackadjust(self, adr, allocated_depth): mc = codebuf.MachineCodeBlockWrapper() From noreply at buildbot.pypy.org Thu Jan 17 22:20:43 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 17 Jan 2013 22:20:43 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fix tests Message-ID: <20130117212043.849BF1C12FA@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60154:fa418e5683dd Date: 2013-01-17 23:04 +0200 http://bitbucket.org/pypy/pypy/changeset/fa418e5683dd/ Log: fix tests diff --git a/pypy/jit/backend/x86/test/test_runner.py b/pypy/jit/backend/x86/test/test_runner.py --- a/pypy/jit/backend/x86/test/test_runner.py +++ b/pypy/jit/backend/x86/test/test_runner.py @@ -33,8 +33,10 @@ # for the individual tests see # ====> ../../test/runner_test.py - add_loop_instructions = ['mov', 'add', 'test', 'je', 'jmp'] - bridge_loop_instructions = ['cmp', 'jg', 'mov', 'call', 'mov', 'jmp'] + add_loop_instructions = ['mov', 'movabs', 'mov', 'movabs', + 'mov', 'add', 'test', 'je', 'jmp'] + bridge_loop_instructions = ['cmp', 'jge', 'mov', 'mov', 'call', + 'mov', 'jmp'] def setup_method(self, meth): self.cpu = CPU(rtyper=None, stats=FakeStats()) From noreply at buildbot.pypy.org Thu Jan 17 22:20:44 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 17 Jan 2013 22:20:44 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: remove old tests. In fact we need the frame_adjustment just after label, since Message-ID: <20130117212044.BDF191C12FC@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60155:904242f8992d Date: 2013-01-17 23:20 +0200 http://bitbucket.org/pypy/pypy/changeset/904242f8992d/ Log: remove old tests. In fact we need the frame_adjustment just after label, since we can't jump to random places (and execute token takes care) diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -536,7 +536,6 @@ operations = regalloc.prepare_loop(inputargs, operations, looptoken, clt.allgcrefs) rgc._make_sure_does_not_move(clt.frame_info) - self._insert_frame_adjustment(clt.frame_info) looppos = self.mc.get_relative_pos() looptoken._x86_loop_code = looppos frame_depth = self._assemble(regalloc, inputargs, operations) @@ -696,6 +695,8 @@ return stack_check_cmp_ofs, ofs def _insert_frame_adjustment(self, frame_info): + # XXX note that this can be easily shifted to JUMP + # instead of LABEL, would be slightly faster gcmap_ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') frame_info_addr = rffi.cast(lltype.Signed, frame_info) frame_info_ofs = self.cpu.get_ofs_of_frame_field('jf_frame_info') diff --git a/pypy/jit/backend/x86/test/test_gc_integration.py b/pypy/jit/backend/x86/test/test_gc_integration.py --- a/pypy/jit/backend/x86/test/test_gc_integration.py +++ b/pypy/jit/backend/x86/test/test_gc_integration.py @@ -64,6 +64,11 @@ # this is a fairly CPU specific check all = len(gpr_reg_mgr_cls.all_regs) assert frame.jf_gcpattern == (1 << (all - 1)) | (1 << (all - 2)) + # the gcmap should contain three things, p0, p1 and p3 + assert len(frame.jf_gcmap) == 3 + for i in range(3): + assert frame.jf_gcmap[i] == frame.jf_frame_info.jfi_gcmap[i] + xxx def test_rewrite_constptr(self): ops = ''' @@ -266,155 +271,3 @@ s1ref = self.cpu.get_latest_value_ref(self.deadframe, i) s1 = lltype.cast_opaque_ptr(lltype.Ptr(S1), s1ref) assert s1 == getattr(s2, 's%d' % i) - - -class MockShadowStackRootMap(MockGcRootMap): - is_shadow_stack = True - MARKER_FRAME = 88 # this marker follows the frame addr - S1 = lltype.GcStruct('S1') - - def __init__(self): - self.addrs = lltype.malloc(rffi.CArray(lltype.Signed), 20, - flavor='raw') - # root_stack_top - self.addrs[0] = rffi.cast(lltype.Signed, self.addrs) + 3*WORD - # random stuff - self.addrs[1] = 123456 - self.addrs[2] = 654321 - self.check_initial_and_final_state() - self.callshapes = {} - self.should_see = [] - - def check_initial_and_final_state(self): - assert self.addrs[0] == rffi.cast(lltype.Signed, self.addrs) + 3*WORD - assert self.addrs[1] == 123456 - assert self.addrs[2] == 654321 - - def get_root_stack_top_addr(self): - return rffi.cast(lltype.Signed, self.addrs) - - def compress_callshape(self, shape, datablockwrapper): - assert shape[0] == 'shape' - return ['compressed'] + shape[1:] - - def write_callshape(self, mark, force_index): - assert mark[0] == 'compressed' - assert force_index not in self.callshapes - assert force_index == 42 + len(self.callshapes) - self.callshapes[force_index] = mark - - def hook_malloc_slowpath(self): - num_entries = self.addrs[0] - rffi.cast(lltype.Signed, self.addrs) - assert num_entries == 5*WORD # 3 initially, plus 2 by the asm frame - assert self.addrs[1] == 123456 # unchanged - assert self.addrs[2] == 654321 # unchanged - frame_addr = self.addrs[3] # pushed by the asm frame - assert self.addrs[4] == self.MARKER_FRAME # pushed by the asm frame - # - from pypy.jit.backend.x86.arch import FORCE_INDEX_OFS - addr = rffi.cast(rffi.CArrayPtr(lltype.Signed), - frame_addr + FORCE_INDEX_OFS) - force_index = addr[0] - assert force_index == 43 # in this test: the 2nd call_malloc_nursery - # - # The callshapes[43] saved above should list addresses both in the - # COPY_AREA and in the "normal" stack, where all the 16 values p1-p16 - # of test_save_regs_at_correct_place should have been stored. Here - # we replace them with new addresses, to emulate a moving GC. - shape = self.callshapes[force_index] - assert len(shape[1:]) == len(self.should_see) - new_objects = [None] * len(self.should_see) - for ofs in shape[1:]: - assert isinstance(ofs, int) # not a register at all here - addr = rffi.cast(rffi.CArrayPtr(lltype.Signed), frame_addr + ofs) - contains = addr[0] - for j in range(len(self.should_see)): - obj = self.should_see[j] - if contains == rffi.cast(lltype.Signed, obj): - assert new_objects[j] is None # duplicate? - break - else: - assert 0 # the value read from the stack looks random? - new_objects[j] = lltype.malloc(self.S1) - addr[0] = rffi.cast(lltype.Signed, new_objects[j]) - self.should_see[:] = new_objects - - -class TestMallocShadowStack(BaseTestRegalloc): - - def setup_method(self, method): - cpu = CPU(None, None) - cpu.gc_ll_descr = GCDescrFastpathMalloc() - cpu.gc_ll_descr.gcrootmap = MockShadowStackRootMap() - cpu.setup_once() - for i in range(42): - cpu.reserve_some_free_fail_descr_number() - self.cpu = cpu - - def test_save_regs_at_correct_place(self): - cpu = self.cpu - gc_ll_descr = cpu.gc_ll_descr - S1 = gc_ll_descr.gcrootmap.S1 - S2 = lltype.GcStruct('S2', ('s0', lltype.Ptr(S1)), - ('s1', lltype.Ptr(S1)), - ('s2', lltype.Ptr(S1)), - ('s3', lltype.Ptr(S1)), - ('s4', lltype.Ptr(S1)), - ('s5', lltype.Ptr(S1)), - ('s6', lltype.Ptr(S1)), - ('s7', lltype.Ptr(S1)), - ('s8', lltype.Ptr(S1)), - ('s9', lltype.Ptr(S1)), - ('s10', lltype.Ptr(S1)), - ('s11', lltype.Ptr(S1)), - ('s12', lltype.Ptr(S1)), - ('s13', lltype.Ptr(S1)), - ('s14', lltype.Ptr(S1)), - ('s15', lltype.Ptr(S1))) - self.namespace = self.namespace.copy() - for i in range(16): - self.namespace['ds%i' % i] = cpu.fielddescrof(S2, 's%d' % i) - ops = ''' - [i0, p0] - p1 = getfield_gc(p0, descr=ds0) - p2 = getfield_gc(p0, descr=ds1) - p3 = getfield_gc(p0, descr=ds2) - p4 = getfield_gc(p0, descr=ds3) - p5 = getfield_gc(p0, descr=ds4) - p6 = getfield_gc(p0, descr=ds5) - p7 = getfield_gc(p0, descr=ds6) - p8 = getfield_gc(p0, descr=ds7) - p9 = getfield_gc(p0, descr=ds8) - p10 = getfield_gc(p0, descr=ds9) - p11 = getfield_gc(p0, descr=ds10) - p12 = getfield_gc(p0, descr=ds11) - p13 = getfield_gc(p0, descr=ds12) - p14 = getfield_gc(p0, descr=ds13) - p15 = getfield_gc(p0, descr=ds14) - p16 = getfield_gc(p0, descr=ds15) - # - # now all registers are in use - p17 = call_malloc_nursery(40) - p18 = call_malloc_nursery(40) # overflow - # - guard_true(i0) [p1, p2, p3, p4, p5, p6, p7, p8, \ - p9, p10, p11, p12, p13, p14, p15, p16] - ''' - s2 = lltype.malloc(S2) - for i in range(16): - s1 = lltype.malloc(S1) - setattr(s2, 's%d' % i, s1) - gc_ll_descr.gcrootmap.should_see.append(s1) - s2ref = lltype.cast_opaque_ptr(llmemory.GCREF, s2) - # - self.interpret(ops, [0, s2ref]) - gc_ll_descr.check_nothing_in_nursery() - assert gc_ll_descr.calls == [40] - gc_ll_descr.gcrootmap.check_initial_and_final_state() - # check the returned pointers - for i in range(16): - s1ref = self.cpu.get_latest_value_ref(self.deadframe, i) - s1 = lltype.cast_opaque_ptr(lltype.Ptr(S1), s1ref) - for j in range(16): - assert s1 != getattr(s2, 's%d' % j) - assert s1 == gc_ll_descr.gcrootmap.should_see[i] From noreply at buildbot.pypy.org Thu Jan 17 22:53:53 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 17 Jan 2013 22:53:53 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: finish the test Message-ID: <20130117215353.3FED11C1353@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60156:d79c5eaa6c48 Date: 2013-01-17 23:25 +0200 http://bitbucket.org/pypy/pypy/changeset/d79c5eaa6c48/ Log: finish the test diff --git a/pypy/jit/backend/x86/test/test_gc_integration.py b/pypy/jit/backend/x86/test/test_gc_integration.py --- a/pypy/jit/backend/x86/test/test_gc_integration.py +++ b/pypy/jit/backend/x86/test/test_gc_integration.py @@ -64,11 +64,14 @@ # this is a fairly CPU specific check all = len(gpr_reg_mgr_cls.all_regs) assert frame.jf_gcpattern == (1 << (all - 1)) | (1 << (all - 2)) - # the gcmap should contain three things, p0, p1 and p3 - assert len(frame.jf_gcmap) == 3 - for i in range(3): + # the gcmap should contain three things, p0, p1 and p3, but + # p3 stays in a register and is only represented in gcpattern, + # while p0 is in both + assert len(frame.jf_gcmap) == 2 + for i in range(2): assert frame.jf_gcmap[i] == frame.jf_frame_info.jfi_gcmap[i] - xxx + assert frame.jf_gcmap[0] == 1 + assert frame.jf_gcmap[1] == 3 def test_rewrite_constptr(self): ops = ''' From noreply at buildbot.pypy.org Thu Jan 17 22:53:54 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 17 Jan 2013 22:53:54 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: no storing registers for free, I'm a moron Message-ID: <20130117215354.7CB741C1353@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60157:3f25d53e2d16 Date: 2013-01-17 23:53 +0200 http://bitbucket.org/pypy/pypy/changeset/3f25d53e2d16/ Log: no storing registers for free, I'm a moron diff --git a/pypy/jit/backend/llsupport/llmodel.py b/pypy/jit/backend/llsupport/llmodel.py --- a/pypy/jit/backend/llsupport/llmodel.py +++ b/pypy/jit/backend/llsupport/llmodel.py @@ -52,13 +52,12 @@ pass def _setup_frame_realloc(self): - FUNC_TP = lltype.Ptr(lltype.FuncType([llmemory.GCREF, lltype.Signed], + FUNC_TP = lltype.Ptr(lltype.FuncType([llmemory.GCREF], llmemory.GCREF)) - def realloc_frame(frame, size): + def realloc_frame(frame): frame = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, frame) new_frame = frame.copy() - assert frame.jf_frame_info.jfi_frame_depth >= size # XXX now we know, rewrite this # we need to do this, because we're not sure what things # are GC pointers and which ones are not diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -536,6 +536,7 @@ operations = regalloc.prepare_loop(inputargs, operations, looptoken, clt.allgcrefs) rgc._make_sure_does_not_move(clt.frame_info) + self._insert_frame_adjustment(clt.frame_info) looppos = self.mc.get_relative_pos() looptoken._x86_loop_code = looppos frame_depth = self._assemble(regalloc, inputargs, operations) @@ -593,7 +594,7 @@ operations, self.current_clt.allgcrefs, self.current_clt.frame_info) - stack_check_patch_ofs, stack_check_patch_ofs2 = self._check_frame_depth() + stack_check_patch_ofs = self._check_frame_depth() frame_depth = self._assemble(regalloc, inputargs, operations) codeendpos = self.mc.get_relative_pos() self.write_pending_failure_recoveries() @@ -614,7 +615,6 @@ frame_depth = max(self.current_clt.frame_info.jfi_frame_depth, frame_depth + JITFRAME_FIXED_SIZE) self._patch_stackadjust(stack_check_patch_ofs + rawstart, frame_depth) - self._patch_stackadjust(stack_check_patch_ofs2 + rawstart, frame_depth) self.fixup_target_tokens(rawstart) self.current_clt.frame_info.jfi_frame_depth = frame_depth self._update_gcmap(self.current_clt.frame_info, regalloc) @@ -685,14 +685,12 @@ assert not IS_X86_32 self.mc.J_il8(rx86.Conditions['GE'], 0) jg_location = self.mc.get_relative_pos() - self.mc.MOV_ri(esi.value, 0xffffff) - ofs = self.mc.get_relative_pos() - 4 self.mc.CALL(imm(self._stack_check_failure)) # patch the JG above offset = self.mc.get_relative_pos() - jg_location assert 0 < offset <= 127 self.mc.overwrite(jg_location-1, chr(offset)) - return stack_check_cmp_ofs, ofs + return stack_check_cmp_ofs def _insert_frame_adjustment(self, frame_info): # XXX note that this can be easily shifted to JUMP diff --git a/pypy/jit/backend/x86/test/test_gc_integration.py b/pypy/jit/backend/x86/test/test_gc_integration.py --- a/pypy/jit/backend/x86/test/test_gc_integration.py +++ b/pypy/jit/backend/x86/test/test_gc_integration.py @@ -73,6 +73,26 @@ assert frame.jf_gcmap[0] == 1 assert frame.jf_gcmap[1] == 3 + def test_label(self): + ops = ''' + [i0, p0, i1, p1] + label(i0, p0, i1, p1, descr=targettoken) + p3 = getfield_gc(p0, descr=fielddescr) + force_spill(p3) + guard_true(i0) [p0, i1, p1, p3] + finish() + ''' + s1 = lltype.malloc(self.S) + s2 = lltype.malloc(self.S) + s1.field = s2 + loop = self.interpret(ops, [0, s1, 1, s2]) + ops2 = ''' + [p0] + jump(1, p0, 1, p0, descr=targettoken) + ''' + self.interpret(ops2) + xxx + def test_rewrite_constptr(self): ops = ''' [] diff --git a/pypy/jit/backend/x86/test/test_regalloc.py b/pypy/jit/backend/x86/test/test_regalloc.py --- a/pypy/jit/backend/x86/test/test_regalloc.py +++ b/pypy/jit/backend/x86/test/test_regalloc.py @@ -145,8 +145,8 @@ namespace = locals().copy() type_system = 'lltype' - def parse(self, s, boxkinds=None): - return parse(s, self.cpu, self.namespace, + def parse(self, s, boxkinds=None, namespace=None): + return parse(s, self.cpu, namespace or self.namespace, type_system=self.type_system, boxkinds=boxkinds) From noreply at buildbot.pypy.org Thu Jan 17 23:08:14 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 17 Jan 2013 23:08:14 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fixes and tests Message-ID: <20130117220814.368521C1365@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60158:4cdddd8fa88e Date: 2013-01-18 00:07 +0200 http://bitbucket.org/pypy/pypy/changeset/4cdddd8fa88e/ Log: fixes and tests diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -520,6 +520,7 @@ clt.allgcrefs = [] clt.frame_info.jfi_frame_depth = 0 # for now looptoken.compiled_loop_token = clt + clt._debug_nbargs = len(inputargs) if not we_are_translated(): # Arguments should be unique assert len(set(inputargs)) == len(inputargs) @@ -532,7 +533,6 @@ regalloc = RegAlloc(self, self.cpu.translate_support_code) # self._call_header_with_stack_check() - clt._debug_nbargs = len(inputargs) operations = regalloc.prepare_loop(inputargs, operations, looptoken, clt.allgcrefs) rgc._make_sure_does_not_move(clt.frame_info) @@ -700,14 +700,15 @@ frame_info_ofs = self.cpu.get_ofs_of_frame_field('jf_frame_info') jfi_gc_map_ofs = self.cpu.get_ofs_of_frame_field('jfi_gcmap') if IS_X86_32: - self.mc.MOV_bi(gcmap_ofs, frame_info_addr + jfi_gc_map_ofs) self.mc.MOV_bi(frame_info_ofs, frame_info_addr) + XXX else: - self.mc.MOV_ri(X86_64_SCRATCH_REG.value, - frame_info_addr + jfi_gc_map_ofs) - self.mc.MOV_br(gcmap_ofs, X86_64_SCRATCH_REG.value) self.mc.MOV_ri(X86_64_SCRATCH_REG.value, frame_info_addr) self.mc.MOV_br(frame_info_ofs, X86_64_SCRATCH_REG.value) + self.mc.MOV_rm(X86_64_SCRATCH_REG.value, + (X86_64_SCRATCH_REG.value, + jfi_gc_map_ofs)) + self.mc.MOV_br(gcmap_ofs, X86_64_SCRATCH_REG.value) def _patch_stackadjust(self, adr, allocated_depth): @@ -2393,14 +2394,6 @@ op.getopname()) def closing_jump(self, target_token): - # The backend's logic assumes that the target code is in a piece of - # assembler that was also called with the same number of arguments, - # so that the locations [ebp+8..] of the input arguments are valid - # stack locations both before and after the jump. - my_nbargs = self.current_clt._debug_nbargs - target_nbargs = target_token._x86_clt._debug_nbargs - assert my_nbargs == target_nbargs - # target = target_token._x86_loop_code if target_token in self.target_tokens_currently_compiling: curpos = self.mc.get_relative_pos() + 5 diff --git a/pypy/jit/backend/x86/test/test_gc_integration.py b/pypy/jit/backend/x86/test/test_gc_integration.py --- a/pypy/jit/backend/x86/test/test_gc_integration.py +++ b/pypy/jit/backend/x86/test/test_gc_integration.py @@ -36,6 +36,7 @@ ptr0 = struct_ref targettoken = TargetToken() + targettoken2 = TargetToken() namespace = locals().copy() @@ -76,7 +77,7 @@ def test_label(self): ops = ''' [i0, p0, i1, p1] - label(i0, p0, i1, p1, descr=targettoken) + label(i0, p0, i1, p1, descr=targettoken2) p3 = getfield_gc(p0, descr=fielddescr) force_spill(p3) guard_true(i0) [p0, i1, p1, p3] @@ -85,13 +86,15 @@ s1 = lltype.malloc(self.S) s2 = lltype.malloc(self.S) s1.field = s2 - loop = self.interpret(ops, [0, s1, 1, s2]) + self.interpret(ops, [0, s1, 1, s2]) ops2 = ''' [p0] - jump(1, p0, 1, p0, descr=targettoken) + jump(1, p0, 1, p0, descr=targettoken2) ''' - self.interpret(ops2) - xxx + self.interpret(ops2, [s1]) + frame = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, self.deadframe) + assert len(frame.jf_gcmap) == 3 + assert [frame.jf_gcmap[i] for i in range(3)] == [1, 3, 4] def test_rewrite_constptr(self): ops = ''' From noreply at buildbot.pypy.org Fri Jan 18 00:28:49 2013 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 18 Jan 2013 00:28:49 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: fix one translation error Message-ID: <20130117232849.A4B3F1C1359@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: missing-ndarray-attributes Changeset: r60159:4061742c9b8f Date: 2013-01-18 01:16 +0200 http://bitbucket.org/pypy/pypy/changeset/4061742c9b8f/ Log: fix one translation error diff --git a/pypy/module/micronumpy/interp_flatiter.py b/pypy/module/micronumpy/interp_flatiter.py --- a/pypy/module/micronumpy/interp_flatiter.py +++ b/pypy/module/micronumpy/interp_flatiter.py @@ -9,11 +9,13 @@ like a real array for descr_eq and friends """ def __init__(self, base): - assert isinstance(base, W_NDimArray) - self.base = base + self._base = base self.dtype = base.get_dtype() self.shape = [base.get_size()] + def base(self): + return self._base + def get_shape(self): return self.shape @@ -25,7 +27,8 @@ def __init__(self, arr): self.base = arr # this is needed to support W_NDimArray interface - self.implementation = FakeArrayImplementation(self.base) + imp = FakeArrayImplementation(self.base) + self.implementation = imp self.reset() def reset(self): From noreply at buildbot.pypy.org Fri Jan 18 00:28:50 2013 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 18 Jan 2013 00:28:50 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: add some failing tests Message-ID: <20130117232850.DE8101C1359@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: missing-ndarray-attributes Changeset: r60160:75ac8c51f417 Date: 2013-01-18 01:27 +0200 http://bitbucket.org/pypy/pypy/changeset/75ac8c51f417/ Log: add some failing tests diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1634,7 +1634,29 @@ assert (a == [1, 256 + 2, 3]).all() assert (a.byteswap(True) == [0x0100, 0x0201, 0x0300]).all() assert (a == [0x0100, 0x0201, 0x0300]).all() - assert False, 'test float, float16, complex byteswap' + + a = array([1, -1, 1e300], dtype=float) + s1 = map(ord,a.tostring()) + s2 = map(ord, a.byteswap().tostring()) + assert s1[7::-1] == s2[:8] + assert s1[15:7:-1] == s2[8:16] + assert s1[:15:-1] == s2[16:] + + a = array([1+1e30j, -1, 1e10], dtype=complex) + s1 = map(ord,a.tostring()) + s2 = map(ord, a.byteswap().tostring()) + assert s1[7::-1] == s2[:8] + assert s1[15:7:-1] == s2[8:16] + assert s1[23:15:-1] == s2[16:24] + assert s1[31:23:-1] == s2[24:32] + assert s1[39:31:-1] == s2[32:40] + assert s1[:39:-1] == s2[40:] + + a = array([1, -1, 10000], dtype='float16') + s1 = map(ord,a.tostring()) + s2 = map(ord, a.byteswap().tostring()) + s3 = [s1[1], s1[0],s1[3], s1[2], s1[5], s1[4]] + assert s3 == s2 def test_clip(self): from _numpypy import array From noreply at buildbot.pypy.org Fri Jan 18 00:59:06 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 18 Jan 2013 00:59:06 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: work the slowpath malloc part Message-ID: <20130117235906.08CDC1C136E@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60161:2f2034d7228b Date: 2013-01-18 01:58 +0200 http://bitbucket.org/pypy/pypy/changeset/2f2034d7228b/ Log: work the slowpath malloc part diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -171,7 +171,7 @@ def _build_stack_check_failure(self): mc = codebuf.MachineCodeBlockWrapper() base_ofs = self.cpu.get_baseofs_of_frame_field() - self._push_all_regs_to_frame(mc, self.cpu.supports_floats) + self._push_all_regs_to_frame(mc, [], self.cpu.supports_floats) assert not IS_X86_32 # push first arg mc.LEA_rb(edi.value, -base_ofs) @@ -179,77 +179,41 @@ mc.CALL(imm(self.cpu.realloc_frame)) mc.ADD_ri(esp.value, WORD) mc.LEA_rm(ebp.value, (eax.value, base_ofs)) - self._pop_all_regs_from_frame(mc, self.cpu.supports_floats) + self._pop_all_regs_from_frame(mc, [], self.cpu.supports_floats) mc.RET() self._stack_check_failure = mc.materialize(self.cpu.asmmemmgr, []) def _build_malloc_slowpath(self): - xxx - # With asmgcc, we need two helpers, so that we can write two CALL - # instructions in assembler, with a mark_gc_roots in between. - # With shadowstack, this is not needed, so we produce a single helper. - gcrootmap = self.cpu.gc_ll_descr.gcrootmap - shadow_stack = (gcrootmap is not None and gcrootmap.is_shadow_stack) - # - # ---------- first helper for the slow path of malloc ---------- + """ While arriving on slowpath, we have a gcpattern on stack, + nursery_head in eax and the size in edx - eax + """ mc = codebuf.MachineCodeBlockWrapper() - if self.cpu.supports_floats: # save the XMM registers in - for i in range(self.cpu.NUM_REGS):# the *caller* frame, from esp+8 - mc.MOVSD_sx((WORD*2)+8*i, i) - mc.SUB_rr(edx.value, eax.value) # compute the size we want + self._push_all_regs_to_frame(mc, [eax, edi], self.cpu.supports_floats) + ofs = self.cpu.get_ofs_of_frame_field('jf_gcpattern') + # store the gc pattern + mc.MOV_rs(ecx.value, WORD) + mc.MOV_br(ofs, ecx.value) addr = self.cpu.gc_ll_descr.get_malloc_slowpath_addr() - # - # The registers to save in the copy area: with shadowstack, most - # registers need to be saved. With asmgcc, the callee-saved registers - # don't need to. - save_in_copy_area = gpr_reg_mgr_cls.REGLOC_TO_COPY_AREA_OFS.items() - if not shadow_stack: - save_in_copy_area = [(reg, ofs) for (reg, ofs) in save_in_copy_area - if reg not in gpr_reg_mgr_cls.REGLOC_TO_GCROOTMAP_REG_INDEX] - # - for reg, ofs in save_in_copy_area: - mc.MOV_br(ofs, reg.value) - # - if shadow_stack: - xxx - # ---- shadowstack ---- - mc.SUB_ri(esp.value, 16 - WORD) # stack alignment of 16 bytes - if IS_X86_32: - mc.MOV_sr(0, edx.value) # push argument - elif IS_X86_64: - mc.MOV_rr(edi.value, edx.value) - mc.CALL(imm(addr)) - mc.ADD_ri(esp.value, 16 - WORD) - else: - # ---- asmgcc ---- - if IS_X86_32: - mc.MOV_sr(WORD, edx.value) # save it as the new argument - elif IS_X86_64: - # rdi can be clobbered: its content was saved in the - # copy area of the stack - mc.MOV_rr(edi.value, edx.value) - mc.JMP(imm(addr)) # tail call to the real malloc - rawstart = mc.materialize(self.cpu.asmmemmgr, []) - self.malloc_slowpath1 = rawstart - # ---------- second helper for the slow path of malloc ---------- - mc = codebuf.MachineCodeBlockWrapper() - # - for reg, ofs in save_in_copy_area: - mc.MOV_rb(reg.value, ofs) - assert reg is not eax and reg is not edx - # - if self.cpu.supports_floats: # restore the XMM registers - for i in range(self.cpu.NUM_REGS):# from where they were saved - mc.MOVSD_xs(i, (WORD*2)+8*i) - # + # XXX investigate if we need to save callee-saved registers + # on the frame + mc.SUB_rr(edi.value, eax.value) # compute the size we want + assert not IS_X86_32 + # the arg is already in edi + if getattr(self.cpu.gc_ll_descr, 'passes_frame'): + base_ofs = self.cpu.get_baseofs_of_frame_field() + mc.LEA_rb(esi.value, -base_ofs) + mc.SUB_ri(esp.value, 16 - WORD) + mc.CALL(imm(addr)) + mc.ADD_ri(esp.value, 16 - WORD) # Note: we check this after the code above, just because the code # above is more than 127 bytes on 64-bits... + self._pop_all_regs_from_frame(mc, [eax, edi], self.cpu.supports_floats) mc.TEST_rr(eax.value, eax.value) mc.J_il8(rx86.Conditions['Z'], 0) # patched later jz_location = mc.get_relative_pos() # nursery_free_adr = self.cpu.gc_ll_descr.get_nursery_free_addr() - mc.MOV(edx, heap(nursery_free_adr)) # load this in EDX + mc.MOV(edi, heap(nursery_free_adr)) # load this in EDX mc.RET() # # If the slowpath malloc failed, we raise a MemoryError that @@ -263,7 +227,7 @@ mc.JMP(imm(self.propagate_exception_path)) # rawstart = mc.materialize(self.cpu.asmmemmgr, []) - self.malloc_slowpath2 = rawstart + self.malloc_slowpath = rawstart def _build_propagate_exception_path(self): if not self.cpu.propagate_exception_descr: @@ -536,7 +500,6 @@ operations = regalloc.prepare_loop(inputargs, operations, looptoken, clt.allgcrefs) rgc._make_sure_does_not_move(clt.frame_info) - self._insert_frame_adjustment(clt.frame_info) looppos = self.mc.get_relative_pos() looptoken._x86_loop_code = looppos frame_depth = self._assemble(regalloc, inputargs, operations) @@ -677,6 +640,11 @@ mc.copy_to_raw_memory(rawstart + pos_after_jz - 4) def _check_frame_depth(self): + """ check if the frame is of enough depth to follow this bridge. + Otherwise reallocate the frame in a helper. + There are other potential solutions + to that, but this one does not sound too bad. + """ descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) ofs = self.cpu.unpack_fielddescr(descrs.arraydescr.lendescr) base_ofs = self.cpu.get_baseofs_of_frame_field() @@ -693,6 +661,10 @@ return stack_check_cmp_ofs def _insert_frame_adjustment(self, frame_info): + """ Our frame might end up being different than what we expect. + Note that depth is fine (since bridges check that), but we need + to update gcmap + """ # XXX note that this can be easily shifted to JUMP # instead of LABEL, would be slightly faster gcmap_ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') @@ -1910,20 +1882,22 @@ def setup_failure_recovery(self): self.failure_recovery_code = [0, 0, 0, 0] - def _push_all_regs_to_frame(self, mc, withfloats): + def _push_all_regs_to_frame(self, mc, ignored_regs, withfloats): # Push all general purpose registers for i, gpr in enumerate(gpr_reg_mgr_cls.all_regs): - mc.MOV_br(i * WORD, gpr.value) + if gpr not in ignored_regs: + mc.MOV_br(i * WORD, gpr.value) if withfloats: # Push all XMM regs ofs = len(gpr_reg_mgr_cls.all_regs) for i in range(len(xmm_reg_mgr_cls.all_regs)): mc.MOVSD_bx((ofs + i) * WORD, i) - def _pop_all_regs_from_frame(self, mc, withfloats): + def _pop_all_regs_from_frame(self, mc, ignored_regs, withfloats): # Pop all general purpose registers for i, gpr in enumerate(gpr_reg_mgr_cls.all_regs): - mc.MOV_rb(gpr.value, i * WORD) + if gpr not in ignored_regs: + mc.MOV_rb(gpr.value, i * WORD) if withfloats: # Pop all XMM regs ofs = len(gpr_reg_mgr_cls.all_regs) @@ -1934,7 +1908,7 @@ mc = codebuf.MachineCodeBlockWrapper() self.mc = mc - self._push_all_regs_to_frame(mc, withfloats) + self._push_all_regs_to_frame(mc, [], withfloats) if exc: # We might have an exception pending. Load it into ebx @@ -2401,43 +2375,22 @@ else: self.mc.JMP(imm(target)) - def malloc_cond(self, nursery_free_adr, nursery_top_adr, size): + def malloc_cond(self, nursery_free_adr, nursery_top_adr, size, gcpattern): assert size & (WORD-1) == 0 # must be correctly aligned self.mc.MOV(eax, heap(nursery_free_adr)) - self.mc.LEA_rm(edx.value, (eax.value, size)) - self.mc.CMP(edx, heap(nursery_top_adr)) + self.mc.LEA_rm(edi.value, (eax.value, size)) + self.mc.CMP(edi, heap(nursery_top_adr)) self.mc.J_il8(rx86.Conditions['NA'], 0) # patched later jmp_adr = self.mc.get_relative_pos() - - # See comments in _build_malloc_slowpath for the - # details of the two helper functions that we are calling below. - # First, we need to call two of them and not just one because we - # need to have a mark_gc_roots() in between. Then the calling - # convention of slowpath_addr{1,2} are tweaked a lot to allow - # the code here to be just two CALLs: slowpath_addr1 gets the - # size of the object to allocate from (EDX-EAX) and returns the - # result in EAX; slowpath_addr2 additionally returns in EDX a - # copy of heap(nursery_free_adr), so that the final MOV below is - # a no-op. - - gcrootmap = self.cpu.gc_ll_descr.gcrootmap - shadow_stack = (gcrootmap is not None and gcrootmap.is_shadow_stack) - if not shadow_stack: - # there are two helpers to call only with asmgcc - slowpath_addr1 = self.malloc_slowpath1 - self.mc.CALL(imm(slowpath_addr1)) - slowpath_addr2 = self.malloc_slowpath2 - self.mc.CALL(imm(slowpath_addr2)) - - # reserve room for the argument to the real malloc and the - # saved XMM regs (on 32 bit: 8 * 2 words; on 64 bit: 16 * 1 - # word) - self._regalloc.needed_extra_stack_locations(1+16) - + # this is a 32bit gcpattern that describes where are GC roots + # in registers (which will be saved while doing slowpath) + # store it on top of the stack (we might not have a register free) + self.mc.MOV_si(0, gcpattern) + self.mc.CALL(imm(self.malloc_slowpath)) offset = self.mc.get_relative_pos() - jmp_adr assert 0 < offset <= 127 self.mc.overwrite(jmp_adr-1, chr(offset)) - self.mc.MOV(heap(nursery_free_adr), edx) + self.mc.MOV(heap(nursery_free_adr), edi) genop_discard_list = [Assembler386.not_implemented_op_discard] * rop._LAST genop_list = [Assembler386.not_implemented_op] * rop._LAST diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py --- a/pypy/jit/backend/x86/regalloc.py +++ b/pypy/jit/backend/x86/regalloc.py @@ -902,15 +902,18 @@ # We need edx as a temporary, but otherwise don't save any more # register. See comments in _build_malloc_slowpath(). tmp_box = TempBox() - self.rm.force_allocate_reg(tmp_box, selected_reg=edx) + self.rm.force_allocate_reg(tmp_box, selected_reg=edi) self.rm.possibly_free_var(tmp_box) # gc_ll_descr = self.assembler.cpu.gc_ll_descr + gcpattern = 0 + for box, reg in self.rm.reg_bindings.iteritems(): + if box.type == REF: + gcpattern |= (1 << self.rm.all_reg_indexes[reg.value]) self.assembler.malloc_cond( gc_ll_descr.get_nursery_free_addr(), gc_ll_descr.get_nursery_top_addr(), - size) - + size, gcpattern) def consider_setfield_gc(self, op): ofs, size, _ = unpack_fielddescr(op.getdescr()) diff --git a/pypy/jit/backend/x86/test/test_gc_integration.py b/pypy/jit/backend/x86/test/test_gc_integration.py --- a/pypy/jit/backend/x86/test/test_gc_integration.py +++ b/pypy/jit/backend/x86/test/test_gc_integration.py @@ -145,8 +145,9 @@ class GCDescrFastpathMalloc(GcLLDescription): gcrootmap = None write_barrier_descr = None + passes_frame = True - def __init__(self): + def __init__(self, callback): GcLLDescription.__init__(self, None) # create a nursery NTP = rffi.CArray(lltype.Char) @@ -158,7 +159,9 @@ self.addrs[0] = rffi.cast(lltype.Signed, self.nursery) self.addrs[1] = self.addrs[0] + 64 self.calls = [] - def malloc_slowpath(size): + def malloc_slowpath(size, frame): + if callback is not None: + callback(frame) if self.gcrootmap is not None: # hook self.gcrootmap.hook_malloc_slowpath() self.calls.append(size) @@ -167,7 +170,8 @@ self.addrs[0] = nadr + size return nadr self.generate_function('malloc_nursery', malloc_slowpath, - [lltype.Signed], lltype.Signed) + [lltype.Signed, jitframe.JITFRAMEPTR], + lltype.Signed) def get_nursery_free_addr(self): return rffi.cast(lltype.Signed, self.addrs) @@ -185,13 +189,18 @@ class TestMallocFastpath(BaseTestRegalloc): - def setup_method(self, method): + def teardown_method(self, method): + lltype.free(self.cpu.gc_ll_descr.addrs, flavor='raw') + lltype.free(self.cpu.gc_ll_descr.nursery, flavor='raw') + + def getcpu(self, callback): cpu = CPU(None, None) - cpu.gc_ll_descr = GCDescrFastpathMalloc() + cpu.gc_ll_descr = GCDescrFastpathMalloc(callback) cpu.setup_once() - self.cpu = cpu + return cpu def test_malloc_fastpath(self): + self.cpu = self.getcpu(None) ops = ''' [i0] p0 = call_malloc_nursery(16) @@ -203,7 +212,7 @@ # check the returned pointers gc_ll_descr = self.cpu.gc_ll_descr nurs_adr = rffi.cast(lltype.Signed, gc_ll_descr.nursery) - ref = lambda n: self.cpu.get_latest_value_ref(self.deadframe, n) + ref = lambda n: self.cpu.get_ref_value(self.deadframe, n) assert rffi.cast(lltype.Signed, ref(0)) == nurs_adr + 0 assert rffi.cast(lltype.Signed, ref(1)) == nurs_adr + 16 assert rffi.cast(lltype.Signed, ref(2)) == nurs_adr + 48 @@ -214,6 +223,13 @@ assert gc_ll_descr.calls == [] def test_malloc_slowpath(self): + def check(frame): + assert len(frame.jf_frame_info.jfi_gcmap) == 2 # 2 pointers + assert frame.jf_frame_info.jfi_gcmap[0] == 1 + assert frame.jf_frame_info.jfi_gcmap[1] == 2 + assert frame.jf_gcpattern == 0x2 + + self.cpu = self.getcpu(check) ops = ''' [i0] p0 = call_malloc_nursery(16) @@ -225,7 +241,7 @@ # check the returned pointers gc_ll_descr = self.cpu.gc_ll_descr nurs_adr = rffi.cast(lltype.Signed, gc_ll_descr.nursery) - ref = lambda n: self.cpu.get_latest_value_ref(self.deadframe, n) + ref = lambda n: self.cpu.get_ref_value(self.deadframe, n) assert rffi.cast(lltype.Signed, ref(0)) == nurs_adr + 0 assert rffi.cast(lltype.Signed, ref(1)) == nurs_adr + 16 assert rffi.cast(lltype.Signed, ref(2)) == nurs_adr + 0 @@ -236,6 +252,12 @@ assert gc_ll_descr.calls == [24] def test_save_regs_around_malloc(self): + def check(frame): + x = frame.jf_gcpattern + assert bin(x) == '0b1111111011111' + # all but two + + self.cpu = self.getcpu(check) S1 = lltype.GcStruct('S1') S2 = lltype.GcStruct('S2', ('s0', lltype.Ptr(S1)), ('s1', lltype.Ptr(S1)), @@ -294,6 +316,6 @@ assert gc_ll_descr.calls == [40] # check the returned pointers for i in range(16): - s1ref = self.cpu.get_latest_value_ref(self.deadframe, i) + s1ref = self.cpu.get_ref_value(self.deadframe, i) s1 = lltype.cast_opaque_ptr(lltype.Ptr(S1), s1ref) assert s1 == getattr(s2, 's%d' % i) From noreply at buildbot.pypy.org Fri Jan 18 01:16:59 2013 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 18 Jan 2013 01:16:59 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: pass byteswap tests Message-ID: <20130118001659.812A11C00BE@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: missing-ndarray-attributes Changeset: r60162:a626fbc58e73 Date: 2013-01-18 01:45 +0200 http://bitbucket.org/pypy/pypy/changeset/a626fbc58e73/ Log: pass byteswap tests diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -966,6 +966,12 @@ raw_storage_setitem(storage, i + offset, rffi.cast(self._STORAGE_T, hbits)) + def byteswap(self, w_v): + value = self.unbox(w_v) + hbits = float_pack(value,2) + swapped = byteswap(rffi.cast(self._STORAGE_T, hbits)) + return self.box(float_unpack(r_ulonglong(swapped), 2)) + class NonNativeFloat16(Float16): _attrs_ = () BoxType = interp_boxes.W_Float16Box @@ -1047,6 +1053,10 @@ def get_element_size(self): return 2 * rffi.sizeof(self._COMPONENTS_T) + def byteswap(self, w_v): + real, imag = self.unbox(w_v) + return self.box_complex(byteswap(real), byteswap(imag)) + @specialize.argtype(1) def box(self, value): return self.BoxType( From noreply at buildbot.pypy.org Fri Jan 18 01:17:01 2013 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 18 Jan 2013 01:17:01 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: merge default into branch Message-ID: <20130118001701.8CCEF1C00BE@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: missing-ndarray-attributes Changeset: r60163:9a095c7af768 Date: 2013-01-18 02:15 +0200 http://bitbucket.org/pypy/pypy/changeset/9a095c7af768/ Log: merge default into branch diff --git a/pypy/doc/test/test_whatsnew.py b/pypy/doc/test/test_whatsnew.py --- a/pypy/doc/test/test_whatsnew.py +++ b/pypy/doc/test/test_whatsnew.py @@ -29,7 +29,7 @@ merge() and \ branch(default)) and \ not branch(default)' % (startrev, endrev) - cmd = r"hg log -R '%s' -r '%s' --template '{branches}\n'" % (path, revset) + cmd = r'hg log -R "%s" -r "%s" --template "{branches}\n"' % (path, revset) out = getoutput(cmd) branches = set(map(str.strip, out.splitlines())) return branches diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -15,6 +15,9 @@ .. branch: numpypy-longdouble Long double support for numpypy +.. branch: numpypy-real-as-view +Convert real, imag from ufuncs to views. This involves the beginning of +view() functionality .. branch: signatures Improved RPython typing diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -178,6 +178,11 @@ else: oparg = 0 + # note: the structure of the code here is such that it makes + # (after translation) a big "if/elif" chain, which is then + # turned into a switch(). It starts here: even if the first + # one is not an "if" but a "while" the effect is the same. + while opcode == self.opcodedesc.EXTENDED_ARG.index: opcode = ord(co_code[next_instr]) if opcode < self.HAVE_ARGUMENT: @@ -226,6 +231,8 @@ 'END_FINALLY', 'JUMP_ABSOLUTE'): continue # opcodes implemented above + # the following "if" is part of the big switch described + # above. if opcode == opdesc.index: # dispatch to the opcode method meth = getattr(self, opdesc.methodname) diff --git a/pypy/jit/backend/test/calling_convention_test.py b/pypy/jit/backend/test/calling_convention_test.py --- a/pypy/jit/backend/test/calling_convention_test.py +++ b/pypy/jit/backend/test/calling_convention_test.py @@ -105,11 +105,13 @@ ops = '[%s]\n' % arguments ops += '%s\n' % spill_ops ops += 'f99 = call(ConstClass(func_ptr), %s, descr=calldescr)\n' % arguments - ops += 'finish(f99, %s)\n' % arguments + ops += 'i99 = same_as(0)\n' + ops += 'guard_true(i99) [f99, %s]\n' % arguments + ops += 'finish()\n' loop = parse(ops, namespace=locals()) looptoken = JitCellToken() - done_number = self.cpu.get_fail_descr_number(loop.operations[-1].getdescr()) + done_number = self.cpu.get_fail_descr_number(loop.operations[-2].getdescr()) self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) argvals, expected_result = self._prepare_args(args, floats, ints) diff --git a/pypy/jit/backend/test/runner_test.py b/pypy/jit/backend/test/runner_test.py --- a/pypy/jit/backend/test/runner_test.py +++ b/pypy/jit/backend/test/runner_test.py @@ -1192,7 +1192,8 @@ values.append(longlong.getfloatstorage(r.random())) # looptoken = JitCellToken() - faildescr = BasicFailDescr(42) + guarddescr = BasicFailDescr(42) + faildescr = BasicFailDescr(43) operations = [] retboxes = [] retvalues = [] @@ -1221,9 +1222,13 @@ retboxes.insert(kk, newbox) retvalues.insert(kk, y) # - operations.append( - ResOperation(rop.FINISH, retboxes, None, descr=faildescr) - ) + zero = BoxInt() + operations.extend([ + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=guarddescr), + ResOperation(rop.FINISH, [], None, descr=faildescr) + ]) + operations[-2].setfailargs(retboxes) print inputargs for op in operations: print op @@ -1344,12 +1349,15 @@ targettoken = TargetToken() faildescr1 = BasicFailDescr(1) faildescr2 = BasicFailDescr(2) + faildescr3 = BasicFailDescr(3) operations = [ ResOperation(rop.LABEL, fboxes, None, descr=targettoken), ResOperation(rop.FLOAT_LE, [fboxes[0], constfloat(9.2)], i2), ResOperation(rop.GUARD_TRUE, [i2], None, descr=faildescr1), - ResOperation(rop.FINISH, fboxes, None, descr=faildescr2), + ResOperation(rop.GUARD_FALSE, [i2], None, descr=faildescr2), + ResOperation(rop.FINISH, [], None, descr=faildescr3), ] + operations[-3].setfailargs(fboxes) operations[-2].setfailargs(fboxes) looptoken = JitCellToken() self.cpu.compile_loop(fboxes, operations, looptoken) @@ -1382,6 +1390,7 @@ py.test.skip("requires floats") fboxes = [BoxFloat() for i in range(3)] faildescr1 = BasicFailDescr(100) + faildescr2 = BasicFailDescr(102) loopops = """ [i0,f1, f2] f3 = float_add(f1, f2) @@ -1406,9 +1415,13 @@ assert longlong.getrealfloat(f2) == 0.75 assert longlong.getrealfloat(f3) == 133.0 + zero = BoxInt() bridgeops = [ - ResOperation(rop.FINISH, fboxes, None, descr=faildescr1), + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=faildescr1), + ResOperation(rop.FINISH, [], None, descr=faildescr2), ] + bridgeops[-2].setfailargs(fboxes[:]) self.cpu.compile_bridge(loop.operations[-2].getdescr(), fboxes, bridgeops, looptoken) args = [1, @@ -2002,7 +2015,7 @@ i1 = same_as(1) call(ConstClass(fptr), i0, descr=calldescr) p0 = guard_exception(ConstClass(xtp)) [i1] - finish(0, p0) + finish(p0) ''' FPTR = lltype.Ptr(lltype.FuncType([lltype.Signed], lltype.Void)) fptr = llhelper(FPTR, func) @@ -2023,8 +2036,7 @@ looptoken = JitCellToken() self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) deadframe = self.cpu.execute_token(looptoken, 1) - assert self.cpu.get_latest_value_int(deadframe, 0) == 0 - assert self.cpu.get_latest_value_ref(deadframe, 1) == xptr + assert self.cpu.get_latest_value_ref(deadframe, 0) == xptr excvalue = self.cpu.grab_exc_value(deadframe) assert not excvalue deadframe = self.cpu.execute_token(looptoken, 0) diff --git a/pypy/jit/backend/test/test_random.py b/pypy/jit/backend/test/test_random.py --- a/pypy/jit/backend/test/test_random.py +++ b/pypy/jit/backend/test/test_random.py @@ -26,6 +26,8 @@ class OperationBuilder(object): def __init__(self, cpu, loop, vars): self.cpu = cpu + if not hasattr(cpu, '_faildescr_keepalive'): + cpu._faildescr_keepalive = [] self.fakemetainterp = FakeMetaInterp() self.loop = loop self.intvars = [box for box in vars if isinstance(box, BoxInt)] @@ -206,6 +208,11 @@ if pytest.config.option.output: s.close() + def getfaildescr(self): + descr = BasicFailDescr() + self.cpu._faildescr_keepalive.append(descr) + return descr + class CannotProduceOperation(Exception): pass @@ -287,7 +294,7 @@ builder.intvars[:] = original_intvars else: op = ResOperation(rop.GUARD_NO_OVERFLOW, [], None) - op.setdescr(BasicFailDescr()) + op.setdescr(builder.getfaildescr()) op.setfailargs(fail_subset) builder.loop.operations.append(op) @@ -357,7 +364,7 @@ def produce_into(self, builder, r): op, passing = self.gen_guard(builder, r) builder.loop.operations.append(op) - op.setdescr(BasicFailDescr()) + op.setdescr(builder.getfaildescr()) op.setfailargs(builder.subset_of_intvars(r)) if not passing: builder.should_fail_by = op @@ -618,8 +625,9 @@ if v not in used_later: endvars.append(v) r.shuffle(endvars) + endvars = endvars[:1] loop.operations.append(ResOperation(rop.FINISH, endvars, None, - descr=BasicFailDescr())) + descr=builder.getfaildescr())) if builder.should_fail_by: self.should_fail_by = builder.should_fail_by self.guard_op = builder.guard_op @@ -712,7 +720,7 @@ else: op = ResOperation(rop.GUARD_EXCEPTION, [guard_op._exc_box], BoxPtr()) - op.setdescr(BasicFailDescr()) + op.setdescr(self.builder.getfaildescr()) op.setfailargs([]) return op diff --git a/pypy/jit/backend/x86/test/test_recompilation.py b/pypy/jit/backend/x86/test/test_recompilation.py --- a/pypy/jit/backend/x86/test/test_recompilation.py +++ b/pypy/jit/backend/x86/test/test_recompilation.py @@ -45,7 +45,8 @@ force_spill(i5) i8 = int_add(i7, 1) i9 = int_add(i8, 1) - finish(i3, i4, i5, i6, i7, i8, i9, descr=fdescr2) + guard_false(i3, descr=fdescr2) [i3, i4, i5, i6, i7, i8, i9] + finish() ''' bridge = self.attach_bridge(ops, loop, -2) descr = loop.operations[3].getdescr() diff --git a/pypy/jit/backend/x86/test/test_regalloc.py b/pypy/jit/backend/x86/test/test_regalloc.py --- a/pypy/jit/backend/x86/test/test_regalloc.py +++ b/pypy/jit/backend/x86/test/test_regalloc.py @@ -327,7 +327,7 @@ assert self.getint(0) == 0 bridge_ops = ''' [i0, i1] - finish(1, 2) + finish(2) ''' self.attach_bridge(bridge_ops, loop, 0) self.run(loop, 0, 1) @@ -428,7 +428,7 @@ ops = ''' [i0, i1, i2, i3, i4, i5, i6, i7] guard_value(i6, i1) [i0, i2, i3, i4, i5, i6] - finish(i0, i2, i3, i4, i5, i6) + finish(i0) ''' self.interpret(ops, [0, 0, 0, 0, 0, 0, 0, 0]) assert self.getint(0) == 0 @@ -438,14 +438,15 @@ [i0, i1, i2, i3, i4, i5, i6, i7, i8] i9 = same_as(0) guard_true(i0) [i9, i0, i1, i2, i3, i4, i5, i6, i7, i8] - finish(1, i0, i1, i2, i3, i4, i5, i6, i7, i8) + finish(1) ''' loop = self.interpret(ops, [0, 1, 2, 3, 4, 5, 6, 7, 8]) assert self.getint(0) == 0 bridge_ops = ''' [i9, i0, i1, i2, i3, i4, i5, i6, i7, i8] call(ConstClass(raising_fptr), 0, descr=raising_calldescr) - finish(i0, i1, i2, i3, i4, i5, i6, i7, i8) + guard_true(i9) [i0, i1, i2, i3, i4, i5, i6, i7, i8] + finish() ''' self.attach_bridge(bridge_ops, loop, 1) self.run(loop, 0, 1, 2, 3, 4, 5, 6, 7, 8) @@ -474,7 +475,8 @@ i1 = same_as(1) i2 = int_lt(i0, 100) guard_true(i3) [i1, i2] - finish(0, i2) + i4 = int_neg(i2) + finish(0) ''' self.interpret(ops, [0, 1]) assert self.getint(0) == 0 @@ -503,7 +505,8 @@ i15 = int_is_true(i5) i16 = int_is_true(i6) i17 = int_is_true(i7) - finish(i10, i11, i12, i13, i14, i15, i16, i17) + guard_true(i0) [i10, i11, i12, i13, i14, i15, i16, i17] + finish() ''' self.interpret(ops, [0, 42, 12, 0, 13, 0, 0, 3333]) assert self.getints(8) == [0, 1, 1, 0, 1, 0, 0, 1] @@ -517,7 +520,8 @@ i13 = int_eq(i5, i6) i14 = int_gt(i6, i2) i15 = int_ne(i2, i6) - finish(i10, i11, i12, i13, i14, i15) + guard_true(i0) [i10, i11, i12, i13, i14, i15] + finish() ''' self.interpret(ops, [0, 1, 2, 3, 4, 5, 6]) assert self.getints(6) == [1, 1, 0, 0, 1, 1] @@ -574,7 +578,9 @@ ops = ''' [f0, f1] f2 = float_add(f0, f1) - finish(f2, f0, f1) + i0 = same_as(0) + guard_true(i0) [f2, f0, f1] + finish() ''' self.interpret(ops, [3.0, 1.5]) assert self.getfloats(3) == [4.5, 3.0, 1.5] @@ -584,7 +590,9 @@ [f0, f1, f2, f3, f4, f5, f6, f7, f8] f9 = float_add(f0, f1) f10 = float_add(f8, 3.5) - finish(f9, f10, f2, f3, f4, f5, f6, f7, f8) + i0 = same_as(0) + guard_true(i0) [f9, f10, f2, f3, f4, f5, f6, f7, f8] + finish() ''' self.interpret(ops, [0.1, .2, .3, .4, .5, .6, .7, .8, .9]) assert self.getfloats(9) == [.1+.2, .9+3.5, .3, .4, .5, .6, .7, .8, .9] @@ -613,7 +621,8 @@ i7 = float_ne(f7, 0.0) i8 = float_ne(f8, 0.0) i9 = float_ne(f9, 0.0) - finish(i0, i1, i2, i3, i4, i5, i6, i7, i8, i9) + guard_true(i0) [i0, i1, i2, i3, i4, i5, i6, i7, i8, i9] + finish() ''' loop = self.interpret(ops, [0.0, .1, .2, .3, .4, .5, .6, .7, .8, .9]) assert self.getints(9) == [0, 1, 1, 1, 1, 1, 1, 1, 1] diff --git a/pypy/jit/backend/x86/test/test_regalloc2.py b/pypy/jit/backend/x86/test/test_regalloc2.py --- a/pypy/jit/backend/x86/test/test_regalloc2.py +++ b/pypy/jit/backend/x86/test/test_regalloc2.py @@ -11,13 +11,17 @@ v2 = BoxInt() v3 = BoxInt() v4 = BoxInt() + zero = BoxInt() inputargs = [v1] operations = [ ResOperation(rop.INT_ADD, [v1, v1], v2), ResOperation(rop.INT_INVERT, [v2], v3), ResOperation(rop.UINT_RSHIFT, [v1, ConstInt(3)], v4), - ResOperation(rop.FINISH, [v4, v3], None, descr=BasicFailDescr()), + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=BasicFailDescr()), + ResOperation(rop.FINISH, [], None, descr=BasicFailDescr()) ] + operations[-2].setfailargs([v4, v3]) cpu = CPU(None, None) cpu.setup_once() looptoken = JitCellToken() @@ -31,6 +35,7 @@ v2 = BoxInt() v3 = BoxInt() v4 = BoxInt() + zero = BoxInt() tmp5 = BoxInt() inputargs = [v1] operations = [ @@ -38,8 +43,11 @@ ResOperation(rop.INT_MUL, [v2, v1], v3), ResOperation(rop.INT_IS_TRUE, [v2], tmp5), ResOperation(rop.INT_IS_ZERO, [tmp5], v4), - ResOperation(rop.FINISH, [v4, v3, tmp5], None, descr=BasicFailDescr()), + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=BasicFailDescr()), + ResOperation(rop.FINISH, [], None, descr=BasicFailDescr()) ] + operations[-2].setfailargs([v4, v3, tmp5]) cpu = CPU(None, None) cpu.setup_once() looptoken = JitCellToken() @@ -90,6 +98,7 @@ v38 = BoxInt() v39 = BoxInt() v40 = BoxInt() + zero = BoxInt() tmp41 = BoxInt() tmp42 = BoxInt() tmp43 = BoxInt() @@ -134,8 +143,12 @@ ResOperation(rop.UINT_GT, [v33, ConstInt(-11)], v38), ResOperation(rop.INT_NEG, [v7], v39), ResOperation(rop.INT_GT, [v24, v32], v40), - ResOperation(rop.FINISH, [v40, v36, v37, v31, v16, v34, v35, v23, v22, v29, v14, v39, v30, v38], None, descr=BasicFailDescr()), + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=BasicFailDescr()), + ResOperation(rop.FINISH, [], None, descr=BasicFailDescr()) ] + operations[-2].setfailargs([v40, v36, v37, v31, v16, v34, v35, v23, + v22, v29, v14, v39, v30, v38]) cpu = CPU(None, None) cpu.setup_once() looptoken = JitCellToken() @@ -198,6 +211,7 @@ v38 = BoxInt() v39 = BoxInt() v40 = BoxInt() + zero = BoxInt() tmp41 = BoxInt() tmp42 = BoxInt() tmp43 = BoxInt() @@ -240,8 +254,13 @@ ResOperation(rop.INT_GT, [v4, v11], v38), ResOperation(rop.INT_LT, [v27, v22], v39), ResOperation(rop.INT_NEG, [v27], v40), - ResOperation(rop.FINISH, [v40, v10, v36, v26, v13, v30, v21, v33, v18, v25, v31, v32, v28, v29, v35, v38, v20, v39, v34, v23, v37], None, descr=BasicFailDescr()), + ResOperation(rop.SAME_AS, [ConstInt(0)], zero), + ResOperation(rop.GUARD_TRUE, [zero], None, descr=BasicFailDescr()), + ResOperation(rop.FINISH, [], None, descr=BasicFailDescr()) ] + operations[-2].setfailargs([v40, v10, v36, v26, v13, v30, v21, v33, + v18, v25, v31, v32, v28, v29, v35, v38, + v20, v39, v34, v23, v37]) cpu = CPU(None, None) cpu.setup_once() looptoken = JitCellToken() diff --git a/pypy/jit/backend/x86/test/test_runner.py b/pypy/jit/backend/x86/test/test_runner.py --- a/pypy/jit/backend/x86/test/test_runner.py +++ b/pypy/jit/backend/x86/test/test_runner.py @@ -499,9 +499,12 @@ i6, descr=calldescr), ResOperation(rop.GUARD_NOT_FORCED, [], None, descr=faildescr), - ResOperation(rop.FINISH, [i3, i4, i5, i6], None, - descr=BasicFailDescr(0)) + ResOperation(rop.GUARD_FALSE, [i3], None, + descr=BasicFailDescr(0)), + ResOperation(rop.FINISH, [], None, + descr=BasicFailDescr(1)) ] + ops[-2].setfailargs([i3, i4, i5, i6]) ops[1].setfailargs([]) ops[3].setfailargs([]) ops[5].setfailargs([]) diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py --- a/pypy/jit/codewriter/support.py +++ b/pypy/jit/codewriter/support.py @@ -10,7 +10,7 @@ from pypy.rpython.extregistry import ExtRegistryEntry from pypy.translator.simplify import get_funcobj from pypy.translator.unsimplify import split_block -from pypy.objspace.flow.model import Constant +from pypy.objspace.flow.model import Variable, Constant from pypy.translator.translator import TranslationContext from pypy.annotation.policy import AnnotatorPolicy from pypy.annotation import model as annmodel @@ -59,6 +59,45 @@ rtyper = annotate(func, values) return rtyper.annotator.translator.graphs[0] +def autodetect_jit_markers_redvars(graph): + # the idea is to find all the jit_merge_point and + # add all the variables across the links to the reds. + for block, op in graph.iterblockops(): + if op.opname == 'jit_marker': + jitdriver = op.args[1].value + if not jitdriver.autoreds: + continue + # if we want to support also can_enter_jit, we should find a + # way to detect a consistent set of red vars to pass *both* to + # jit_merge_point and can_enter_jit. The current simple + # solution doesn't work because can_enter_jit might be in + # another block, so the set of alive_v will be different. + methname = op.args[0].value + assert methname == 'jit_merge_point', ( + "reds='auto' is supported only for jit drivers which " + "calls only jit_merge_point. Found a call to %s" % methname) + # + # compute the set of live variables across the jit_marker + alive_v = set() + for link in block.exits: + alive_v.update(link.args) + alive_v.difference_update(link.getextravars()) + for op1 in block.operations[::-1]: + if op1 is op: + break # stop when the meet the jit_marker + alive_v.discard(op1.result) + alive_v.update(op1.args) + greens_v = op.args[2:] + reds_v = alive_v - set(greens_v) + reds_v = [v for v in reds_v if isinstance(v, Variable) and + v.concretetype is not lltype.Void] + reds_v = sort_vars(reds_v) + op.args.extend(reds_v) + if jitdriver.numreds is None: + jitdriver.numreds = len(reds_v) + else: + assert jitdriver.numreds == len(reds_v), 'inconsistent number of reds_v' + def split_before_jit_merge_point(graph, portalblock, portalopindex): """Split the block just before the 'jit_merge_point', making sure the input args are in the canonical order. diff --git a/pypy/jit/metainterp/compile.py b/pypy/jit/metainterp/compile.py --- a/pypy/jit/metainterp/compile.py +++ b/pypy/jit/metainterp/compile.py @@ -5,7 +5,7 @@ from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.debug import debug_start, debug_stop, debug_print from pypy.rlib import rstack -from pypy.rlib.jit import JitDebugInfo, Counters +from pypy.rlib.jit import JitDebugInfo, Counters, dont_look_inside from pypy.conftest import option from pypy.tool.sourcetools import func_with_new_name @@ -685,6 +685,7 @@ assert 0, "unreachable" @staticmethod + @dont_look_inside def force_now(cpu, token): # Called during a residual call from the assembler, if the code # actually needs to force one of the virtualrefs or the virtualizable. diff --git a/pypy/jit/metainterp/resoperation.py b/pypy/jit/metainterp/resoperation.py --- a/pypy/jit/metainterp/resoperation.py +++ b/pypy/jit/metainterp/resoperation.py @@ -353,6 +353,9 @@ def initarglist(self, args): self._args = args + if not we_are_translated() and \ + self.__class__.__name__.startswith('FINISH'): # XXX remove me + assert len(args) <= 1 # FINISH operations take 0 or 1 arg now def getarglist(self): return self._args diff --git a/pypy/jit/metainterp/test/test_fficall.py b/pypy/jit/metainterp/test/test_fficall.py --- a/pypy/jit/metainterp/test/test_fficall.py +++ b/pypy/jit/metainterp/test/test_fficall.py @@ -105,7 +105,7 @@ class TestFfiCall(FfiCallTests, LLJitMixin): - def test_jit_fii_vref(self): + def test_jit_ffi_vref(self): from pypy.rlib import clibffi from pypy.rlib.jit_libffi import jit_ffi_prep_cif, jit_ffi_call @@ -136,9 +136,10 @@ res = exb[3] lltype.free(exb, flavor='raw') # - lltype.free(atypes, flavor='raw') return res # res = self.interp_operations(f, []) lltype.free(cd, flavor='raw') assert res == math.sin(1.23) + + lltype.free(atypes, flavor='raw') diff --git a/pypy/jit/metainterp/test/test_warmspot.py b/pypy/jit/metainterp/test/test_warmspot.py --- a/pypy/jit/metainterp/test/test_warmspot.py +++ b/pypy/jit/metainterp/test/test_warmspot.py @@ -1,6 +1,6 @@ import py from pypy.jit.metainterp.warmspot import get_stats -from pypy.rlib.jit import JitDriver, set_param, unroll_safe +from pypy.rlib.jit import JitDriver, set_param, unroll_safe, jit_callback from pypy.jit.backend.llgraph import runner from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin @@ -383,6 +383,23 @@ assert res == expected self.check_resops(int_sub=2, int_mul=0, int_add=2) + def test_loop_automatic_reds_not_too_many_redvars(self): + myjitdriver = JitDriver(greens = ['m'], reds = 'auto') + def one(): + return 1 + def f(n, m): + res = 0 + while n > 0: + n -= one() + myjitdriver.jit_merge_point(m=m) + res += m*2 + return res + expected = f(21, 5) + res = self.meta_interp(f, [21, 5]) + assert res == expected + oplabel = get_stats().loops[0].operations[0] + assert len(oplabel.getarglist()) == 2 # 'n', 'res' in some order + def test_inline_jit_merge_point(self): # test that the machinery to inline jit_merge_points in callers # works. The final user does not need to mess manually with the @@ -520,41 +537,21 @@ def test_callback_jit_merge_point(self): - from pypy.rlib.objectmodel import register_around_callback_hook - from pypy.rpython.lltypesystem import lltype, rffi - from pypy.translator.tool.cbuild import ExternalCompilationInfo - - callback_jit_driver = JitDriver(greens = ['name'], reds = 'auto') - - def callback_merge_point(name): - callback_jit_driver.jit_merge_point(name=name) - - @callback_jit_driver.inline(callback_merge_point) - def callback_hook(name): - pass - + @jit_callback("testing") def callback(a, b): if a > b: return 1 return -1 - CB_TP = rffi.CCallback([lltype.Signed, lltype.Signed], lltype.Signed) - eci = ExternalCompilationInfo(includes=['stdlib.h']) - qsort = rffi.llexternal('qsort', - [rffi.VOIDP, lltype.Signed, lltype.Signed, - CB_TP], lltype.Void, compilation_info=eci) - ARR = rffi.CArray(lltype.Signed) + def main(): + total = 0 + for i in range(10): + total += callback(i, 2) + return total - def main(): - register_around_callback_hook(callback_hook) - raw = lltype.malloc(ARR, 10, flavor='raw') - for i in range(10): - raw[i] = 10 - i - qsort(raw, 10, rffi.sizeof(lltype.Signed), callback) - lltype.free(raw, flavor='raw') - - self.meta_interp(main, []) - self.check_trace_count(1) + res = self.meta_interp(main, []) + assert res == 7 - 3 + self.check_trace_count(2) class TestLLWarmspot(WarmspotTests, LLJitMixin): diff --git a/pypy/jit/metainterp/virtualref.py b/pypy/jit/metainterp/virtualref.py --- a/pypy/jit/metainterp/virtualref.py +++ b/pypy/jit/metainterp/virtualref.py @@ -169,4 +169,3 @@ # token == TOKEN_NONE and the vref was not forced: it's invalid raise InvalidVirtualRef return vref.forced - force_virtual._dont_inline_ = True diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py --- a/pypy/jit/metainterp/warmspot.py +++ b/pypy/jit/metainterp/warmspot.py @@ -347,62 +347,20 @@ self.jitdrivers_sd = [] graphs = self.translator.graphs for graph, block, pos in find_jit_merge_points(graphs): - self.autodetect_jit_markers_redvars(graph) + support.autodetect_jit_markers_redvars(graph) self.split_graph_and_record_jitdriver(graph, block, pos) # assert (len(set([jd.jitdriver for jd in self.jitdrivers_sd])) == len(self.jitdrivers_sd)), \ "there are multiple jit_merge_points with the same jitdriver" - def autodetect_jit_markers_redvars(self, graph): - # the idea is to find all the jit_merge_point and can_enter_jit and - # add all the variables across the links to the reds. - for block, op in graph.iterblockops(): - if op.opname == 'jit_marker': - jitdriver = op.args[1].value - if not jitdriver.autoreds: - continue - # if we want to support also can_enter_jit, we should find a - # way to detect a consistent set of red vars to pass *both* to - # jit_merge_point and can_enter_jit. The current simple - # solution doesn't work because can_enter_jit might be in - # another block, so the set of alive_v will be different. - methname = op.args[0].value - assert methname == 'jit_merge_point', ( - "reds='auto' is supported only for jit drivers which " - "calls only jit_merge_point. Found a call to %s" % methname) - # - # compute the set of live variables before the jit_marker - alive_v = set(block.inputargs) - for op1 in block.operations: - if op1 is op: - break # stop when the meet the jit_marker - if op1.result.concretetype != lltype.Void: - alive_v.add(op1.result) - greens_v = op.args[2:] - reds_v = alive_v - set(greens_v) - reds_v = [v for v in reds_v if v.concretetype is not lltype.Void] - reds_v = support.sort_vars(reds_v) - op.args.extend(reds_v) - if jitdriver.numreds is None: - jitdriver.numreds = len(reds_v) - else: - assert jitdriver.numreds == len(reds_v), 'inconsistent number of reds_v' - - def split_graph_and_record_jitdriver(self, graph, block, pos): op = block.operations[pos] jd = JitDriverStaticData() jd._jit_merge_point_in = graph args = op.args[2:] s_binding = self.translator.annotator.binding - if op.args[1].value.autoreds: - # _portal_args_s is used only by _make_hook_graph, but for now we - # declare the various set_jitcell_at, get_printable_location, - # etc. as incompatible with autoreds - jd._portal_args_s = None - else: - jd._portal_args_s = [s_binding(v) for v in args] + jd._portal_args_s = [s_binding(v) for v in args] graph = copygraph(graph) [jmpp] = find_jit_merge_points([graph]) graph.startblock = support.split_before_jit_merge_point(*jmpp) @@ -650,9 +608,10 @@ if func is None: return None # - assert not jitdriver_sd.jitdriver.autoreds, ( - "reds='auto' is not compatible with JitDriver hooks such as " - "{get,set}_jitcell_at, get_printable_location, confirm_enter_jit, etc.") + if not onlygreens: + assert not jitdriver_sd.jitdriver.autoreds, ( + "reds='auto' is not compatible with JitDriver hooks such as " + "confirm_enter_jit") extra_args_s = [] if s_first_arg is not None: extra_args_s.append(s_first_arg) @@ -717,6 +676,9 @@ jitdriver = op.args[1].value assert jitdriver in sublists, \ "can_enter_jit with no matching jit_merge_point" + assert not jitdriver.autoreds, ( + "can_enter_jit not supported with a jitdriver that " + "has reds='auto'") jd, sublist = sublists[jitdriver] origportalgraph = jd._jit_merge_point_in if graph is not origportalgraph: diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py --- a/pypy/module/_cffi_backend/ccallback.py +++ b/pypy/module/_cffi_backend/ccallback.py @@ -152,6 +152,7 @@ STDERR = 2 + at jit.jit_callback("CFFI") def invoke_callback(ffi_cif, ll_res, ll_args, ll_userdata): """ Callback specification. ffi_cif - something ffi specific, don't care diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py --- a/pypy/module/micronumpy/arrayimpl/concrete.py +++ b/pypy/module/micronumpy/arrayimpl/concrete.py @@ -11,7 +11,7 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.rlib import jit from pypy.rlib.rawstorage import free_raw_storage, raw_storage_getitem,\ - raw_storage_setitem + raw_storage_setitem, RAW_STORAGE from pypy.module.micronumpy.arrayimpl.sort import argsort_array from pypy.rlib.debug import make_sure_not_resized @@ -275,18 +275,19 @@ def get_buffer(self, space): return ArrayBuffer(self) -class ConcreteArray(BaseConcreteArray): - def __init__(self, shape, dtype, order, strides, backstrides): +class ConcreteArrayNotOwning(BaseConcreteArray): + def __init__(self, shape, dtype, order, strides, backstrides, storage): + make_sure_not_resized(shape) make_sure_not_resized(strides) make_sure_not_resized(backstrides) self.shape = shape self.size = support.product(shape) * dtype.get_size() - self.storage = dtype.itemtype.malloc(self.size) self.order = order self.dtype = dtype self.strides = strides self.backstrides = backstrides + self.storage = storage def create_iter(self, shape=None): if shape is None or shape == self.get_shape(): @@ -299,9 +300,6 @@ def fill(self, box): self.dtype.fill(self.storage, box, 0, self.size) - def __del__(self): - free_raw_storage(self.storage, track_allocation=False) - def set_shape(self, space, orig_array, new_shape): strides, backstrides = support.calc_strides(new_shape, self.dtype, self.order) @@ -319,6 +317,21 @@ def base(self): return None +class ConcreteArray(ConcreteArrayNotOwning): + def __init__(self, shape, dtype, order, strides, backstrides): + # we allocate the actual storage later because we need to compute + # self.size first + null_storage = lltype.nullptr(RAW_STORAGE) + ConcreteArrayNotOwning.__init__(self, shape, dtype, order, strides, backstrides, + null_storage) + self.storage = dtype.itemtype.malloc(self.size) + + def __del__(self): + free_raw_storage(self.storage, track_allocation=False) + + + + class NonWritableArray(ConcreteArray): def descr_setitem(self, space, orig_array, w_index, w_value): raise OperationError(space.w_RuntimeError, space.wrap( @@ -331,6 +344,7 @@ self.strides = strides self.backstrides = backstrides self.shape = shape + assert isinstance(parent, BaseConcreteArray) if isinstance(parent, SliceArray): parent = parent.parent # one level only self.parent = parent diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py --- a/pypy/module/micronumpy/base.py +++ b/pypy/module/micronumpy/base.py @@ -30,8 +30,17 @@ return W_NDimArray(impl) @staticmethod - def new_slice(offset, strides, backstrides, shape, parent, orig_arr, - dtype=None): + def from_shape_and_storage(shape, storage, dtype, order='C'): + from pypy.module.micronumpy.arrayimpl import concrete + assert shape + strides, backstrides = calc_strides(shape, dtype, order) + impl = concrete.ConcreteArrayNotOwning(shape, dtype, order, strides, + backstrides, storage) + return W_NDimArray(impl) + + + @staticmethod + def new_slice(offset, strides, backstrides, shape, parent, orig_arr, dtype=None): from pypy.module.micronumpy.arrayimpl import concrete impl = concrete.SliceArray(offset, strides, backstrides, shape, parent, diff --git a/pypy/module/micronumpy/interp_flatiter.py b/pypy/module/micronumpy/interp_flatiter.py --- a/pypy/module/micronumpy/interp_flatiter.py +++ b/pypy/module/micronumpy/interp_flatiter.py @@ -20,15 +20,14 @@ return self.shape def create_iter(self, shape=None): - assert isinstance(self.base, W_NDimArray) - return self.base.create_iter() + assert isinstance(self.base(), W_NDimArray) + return self.base().create_iter() class W_FlatIterator(W_NDimArray): def __init__(self, arr): self.base = arr # this is needed to support W_NDimArray interface - imp = FakeArrayImplementation(self.base) - self.implementation = imp + self.implementation = FakeArrayImplementation(self.base) self.reset() def reset(self): diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -561,7 +561,6 @@ raise OperationError(space.w_NotImplementedError, space.wrap( "view not implemented yet")) - # --------------------- operations ---------------------------- def _unaryop_impl(ufunc_name): @@ -735,6 +734,20 @@ return W_NDimArray.new_scalar(space, dtype) return W_NDimArray.from_shape(shape, dtype) + at unwrap_spec(addr=int) +def descr__from_shape_and_storage(space, w_cls, w_shape, addr, w_dtype): + """ + Create an array from an existing buffer, given its address as int. + PyPy-only implementation detail. + """ + from pypy.rpython.lltypesystem import rffi + from pypy.rlib.rawstorage import RAW_STORAGE_PTR + shape = _find_shape(space, w_shape) + storage = rffi.cast(RAW_STORAGE_PTR, addr) + dtype = space.interp_w(interp_dtype.W_Dtype, + space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)) + return W_NDimArray.from_shape_and_storage(shape, storage, dtype) + W_NDimArray.typedef = TypeDef( "ndarray", __new__ = interp2app(descr_new_array), @@ -846,6 +859,8 @@ ctypes = GetSetProperty(W_NDimArray.descr_get_ctypes), # XXX unimplemented __array_interface__ = GetSetProperty(W_NDimArray.descr_array_iface), + _from_shape_and_storage = interp2app(descr__from_shape_and_storage, + as_classmethod=True) ) @unwrap_spec(ndmin=int, copy=bool, subok=bool) diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -189,7 +189,23 @@ assert shape == [2] assert space.str_w(elems[0]) == "a" assert space.str_w(elems[1]) == "b" - + + def test_from_shape_and_storage(self): + from pypy.rlib.rawstorage import alloc_raw_storage, raw_storage_setitem + from pypy.rpython.lltypesystem import rffi + from pypy.module.micronumpy.interp_dtype import get_dtype_cache + storage = alloc_raw_storage(4, track_allocation=False, zero=True) + for i in range(4): + raw_storage_setitem(storage, i, rffi.cast(rffi.UCHAR, i)) + # + dtypes = get_dtype_cache(self.space) + w_array = W_NDimArray.from_shape_and_storage([2, 2], storage, dtypes.w_int8dtype) + def get(i, j): + return w_array.getitem(self.space, [i, j]).value + assert get(0, 0) == 0 + assert get(0, 1) == 1 + assert get(1, 0) == 2 + assert get(1, 1) == 3 class AppTestNumArray(BaseNumpyAppTest): def w_CustomIndexObject(self, index): @@ -2482,7 +2498,7 @@ s = repr(a) assert s.replace('\n', '') == \ "array(['abc', 'defg', 'ab'], dtype='|S4')" - + class AppTestPyPy(BaseNumpyAppTest): def setup_class(cls): @@ -2502,3 +2518,12 @@ assert a[0][1] == 2 a = _numpypy.array(([[[1, 2], [3, 4], [5, 6]]])) assert (a[0, 1] == [3, 4]).all() + + def test_from_shape_and_storage(self): + from _numpypy import array, ndarray + x = array([1, 2, 3, 4]) + addr, _ = x.__array_interface__['data'] + y = ndarray._from_shape_and_storage([2, 2], addr, x.dtype) + assert y[0, 1] == 2 + y[0, 1] = 42 + assert x[1] == 42 diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py --- a/pypy/module/pyexpat/interp_pyexpat.py +++ b/pypy/module/pyexpat/interp_pyexpat.py @@ -2,7 +2,7 @@ from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault from pypy.interpreter.error import OperationError -from pypy.rlib import rgc +from pypy.rlib import rgc, jit from pypy.rpython.lltypesystem import rffi, lltype from pypy.rpython.tool import rffi_platform from pypy.translator.tool.cbuild import ExternalCompilationInfo @@ -288,6 +288,7 @@ post_code = '' src = py.code.Source(""" + @jit.jit_callback('XML:%(name)s') def %(name)s_callback(%(first_arg)s, %(args)s): id = rffi.cast(lltype.Signed, %(ll_id)s) userdata = global_storage.get_object(id) diff --git a/pypy/module/pypyjit/interp_jit.py b/pypy/module/pypyjit/interp_jit.py --- a/pypy/module/pypyjit/interp_jit.py +++ b/pypy/module/pypyjit/interp_jit.py @@ -6,7 +6,7 @@ from pypy.tool.pairtype import extendabletype from pypy.rlib.rarithmetic import r_uint, intmask from pypy.rlib.jit import JitDriver, hint, we_are_jitted, dont_look_inside -from pypy.rlib import jit, objectmodel +from pypy.rlib import jit from pypy.rlib.jit import current_trace_length, unroll_parameters import pypy.interpreter.pyopcode # for side-effects from pypy.interpreter.error import OperationError, operationerrfmt @@ -97,15 +97,6 @@ is_being_profiled=self.is_being_profiled) return jumpto -callback_jit_driver = JitDriver(greens = ['name'], reds = 'auto') - -def callback_merge_point(name): - callback_jit_driver.jit_merge_point(name=name) - - at callback_jit_driver.inline(callback_merge_point) -def callback_hook(name): - pass - def _get_adapted_tick_counter(): # Normally, the tick counter is decremented by 100 for every # Python opcode. Here, to better support JIT compilation of diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py --- a/pypy/module/signal/interp_signal.py +++ b/pypy/module/signal/interp_signal.py @@ -40,6 +40,7 @@ includes = ['stdlib.h', 'src/signals.h'] if sys.platform != 'win32': includes.append('sys/time.h') +WIN32 = sys.platform == 'win32' cdir = py.path.local(autopath.pypydir).join('translator', 'c') @@ -236,7 +237,10 @@ None -- if an unknown handler is in effect (XXX UNIMPLEMENTED) anything else -- the callable Python object used as a handler """ - check_signum_in_range(space, signum) + if WIN32: + check_signum_exists(space, signum) + else: + check_signum_in_range(space, signum) action = space.check_signal_action if signum in action.handlers_w: return action.handlers_w[signum] diff --git a/pypy/module/signal/test/test_signal.py b/pypy/module/signal/test/test_signal.py --- a/pypy/module/signal/test/test_signal.py +++ b/pypy/module/signal/test/test_signal.py @@ -55,7 +55,7 @@ skip("requires os.kill() and os.getpid()") signal = self.signal # the signal module to test if not hasattr(signal, 'SIGUSR1'): - py.test.skip("requires SIGUSR1 in signal") + skip("requires SIGUSR1 in signal") signum = signal.SIGUSR1 received = [] @@ -156,6 +156,7 @@ import sys if sys.platform == 'win32': raises(ValueError, signal, 42, lambda *args: None) + raises(ValueError, signal, 7, lambda *args: None) else: signal(42, lambda *args: None) signal(42, SIG_DFL) diff --git a/pypy/module/thread/os_thread.py b/pypy/module/thread/os_thread.py --- a/pypy/module/thread/os_thread.py +++ b/pypy/module/thread/os_thread.py @@ -2,6 +2,7 @@ Thread support based on OS-level threads. """ +import os from pypy.module.thread import ll_thread as thread from pypy.module.thread.error import wrap_thread_error from pypy.interpreter.error import OperationError, operationerrfmt @@ -91,14 +92,18 @@ # run! try: bootstrapper.run(space, w_callable, args) - finally: - bootstrapper.nbthreads -= 1 - # clean up space.threadlocals to remove the ExecutionContext - # entry corresponding to the current thread + # done + except Exception, e: + # oups! last-level attempt to recover. try: - space.threadlocals.leave_thread(space) - finally: - thread.gc_thread_die() + STDERR = 2 + os.write(STDERR, "Thread exited with ") + os.write(STDERR, str(e)) + os.write(STDERR, "\n") + except OSError: + pass + bootstrapper.nbthreads -= 1 + thread.gc_thread_die() bootstrap = staticmethod(bootstrap) def acquire(space, w_callable, args): @@ -129,6 +134,9 @@ where = 'thread %d started by ' % ident e.write_unraisable(space, where, w_callable) e.clear(space) + # clean up space.threadlocals to remove the ExecutionContext + # entry corresponding to the current thread + space.threadlocals.leave_thread(space) run = staticmethod(run) bootstrapper = Bootstrapper() diff --git a/pypy/module/thread/test/test_gil.py b/pypy/module/thread/test/test_gil.py --- a/pypy/module/thread/test/test_gil.py +++ b/pypy/module/thread/test/test_gil.py @@ -61,8 +61,9 @@ def bootstrap(): try: runme() - finally: - thread.gc_thread_die() + except Exception, e: + assert 0 + thread.gc_thread_die() def f(): state.data = [] state.datalen1 = 0 diff --git a/pypy/rlib/clibffi.py b/pypy/rlib/clibffi.py --- a/pypy/rlib/clibffi.py +++ b/pypy/rlib/clibffi.py @@ -421,6 +421,7 @@ hints={'callback':True})) + at jit.jit_callback("CLIBFFI") def ll_callback(ffi_cif, ll_res, ll_args, ll_userdata): """ Callback specification. ffi_cif - something ffi specific, don't care diff --git a/pypy/rlib/jit.py b/pypy/rlib/jit.py --- a/pypy/rlib/jit.py +++ b/pypy/rlib/jit.py @@ -302,6 +302,32 @@ pass +def jit_callback(name): + """Use as a decorator for C callback functions, to insert a + jitdriver.jit_merge_point() at the start. Only for callbacks + that typically invoke more app-level Python code. + """ + def decorate(func): + from pypy.tool.sourcetools import compile2 + # + def get_printable_location(): + return name + jitdriver = JitDriver(get_printable_location=get_printable_location, + greens=[], reds='auto', name=name) + # + args = ','.join(['a%d' % i for i in range(func.func_code.co_argcount)]) + source = """def callback_with_jitdriver(%(args)s): + jitdriver.jit_merge_point() + return real_callback(%(args)s)""" % locals() + miniglobals = { + 'jitdriver': jitdriver, + 'real_callback': func, + } + exec compile2(source) in miniglobals + return miniglobals['callback_with_jitdriver'] + return decorate + + # ____________________________________________________________ # VRefs @@ -458,9 +484,8 @@ self.autoreds = True self.reds = [] self.numreds = None # see warmspot.autodetect_jit_markers_redvars - for hook in (get_jitcell_at, set_jitcell_at, get_printable_location, - confirm_enter_jit): - assert hook is None, "reds='auto' is not compatible with JitDriver hooks" + assert confirm_enter_jit is None, ( + "reds='auto' is not compatible with confirm_enter_jit") else: if reds is not None: self.reds = reds diff --git a/pypy/rlib/objectmodel.py b/pypy/rlib/objectmodel.py --- a/pypy/rlib/objectmodel.py +++ b/pypy/rlib/objectmodel.py @@ -595,16 +595,6 @@ llhelper(rffi.AroundFnPtr, before) llhelper(rffi.AroundFnPtr, after) -def register_around_callback_hook(hook): - """ Register a hook that's called before a callback from C calls RPython. - Primary usage is for JIT to have 'started from' hook. - """ - from pypy.rpython.lltypesystem import rffi - from pypy.rpython.annlowlevel import llhelper - - rffi.aroundstate.callback_hook = hook - llhelper(rffi.CallbackHookPtr, hook) - def is_in_callback(): from pypy.rpython.lltypesystem import rffi return rffi.stackcounter.stacks_counter > 1 diff --git a/pypy/rlib/runicode.py b/pypy/rlib/runicode.py --- a/pypy/rlib/runicode.py +++ b/pypy/rlib/runicode.py @@ -460,6 +460,10 @@ errorhandler=None, byteorder='little'): if size == 0: + if byteorder == 'native': + result = StringBuilder(2) + _STORECHAR(result, 0xFEFF, BYTEORDER) + return result.build() return "" result = StringBuilder(size * 2 + 2) @@ -621,6 +625,10 @@ errorhandler=None, byteorder='little'): if size == 0: + if byteorder == 'native': + result = StringBuilder(4) + _STORECHAR32(result, 0xFEFF, BYTEORDER) + return result.build() return "" result = StringBuilder(size * 4 + 4) diff --git a/pypy/rlib/test/test_runicode.py b/pypy/rlib/test/test_runicode.py --- a/pypy/rlib/test/test_runicode.py +++ b/pypy/rlib/test/test_runicode.py @@ -679,6 +679,11 @@ "utf-32 utf-32-be utf-32-le").split(): self.checkencode(uni, encoding) + def test_empty(self): + for encoding in ("utf-8 utf-16 utf-16-be utf-16-le " + "utf-32 utf-32-be utf-32-le").split(): + self.checkencode(u'', encoding) + def test_single_chars_utf8(self): # check every number of bytes per char for s in ["\xd7\x90", "\xd6\x96", "\xeb\x96\x95", "\xf0\x90\x91\x93"]: diff --git a/pypy/rpython/lltypesystem/rffi.py b/pypy/rpython/lltypesystem/rffi.py --- a/pypy/rpython/lltypesystem/rffi.py +++ b/pypy/rpython/lltypesystem/rffi.py @@ -1,19 +1,19 @@ import py from pypy.annotation import model as annmodel -from pypy.rpython.lltypesystem import lltype, rstr +from pypy.rpython.lltypesystem import lltype from pypy.rpython.lltypesystem import ll2ctypes -from pypy.rpython.lltypesystem.llmemory import cast_adr_to_ptr, cast_ptr_to_adr +from pypy.rpython.lltypesystem.llmemory import cast_ptr_to_adr from pypy.rpython.lltypesystem.llmemory import itemoffsetof, raw_memcopy from pypy.annotation.model import lltype_to_annotation from pypy.tool.sourcetools import func_with_new_name -from pypy.rlib.objectmodel import Symbolic, CDefinedIntSymbolic -from pypy.rlib.objectmodel import keepalive_until_here +from pypy.rlib.objectmodel import Symbolic +from pypy.rlib.objectmodel import keepalive_until_here, enforceargs from pypy.rlib import rarithmetic, rgc from pypy.rpython.extregistry import ExtRegistryEntry from pypy.rlib.unroll import unrolling_iterable from pypy.rpython.tool.rfficache import platform, sizeof_c_type from pypy.translator.tool.cbuild import ExternalCompilationInfo -from pypy.rpython.annlowlevel import llhelper, llstr +from pypy.rpython.annlowlevel import llhelper from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.rstring import StringBuilder, UnicodeBuilder, assert_str0 from pypy.rlib import jit @@ -279,17 +279,8 @@ callable_name = getattr(callable, '__name__', '?') if callbackholder is not None: callbackholder.callbacks[callable] = True - callable_name_descr = str(callable).replace('"', '\\"') args = ', '.join(['a%d' % i for i in range(len(TP.TO.ARGS))]) source = py.code.Source(r""" - def inner_wrapper(%(args)s): - if aroundstate is not None: - callback_hook = aroundstate.callback_hook - if callback_hook: - callback_hook(llstr("%(callable_name_descr)s")) - return callable(%(args)s) - inner_wrapper._never_inline_ = True - def wrapper(%(args)s): # no *args - no GIL for mallocing the tuple llop.gc_stack_bottom(lltype.Void) # marker for trackgcroot.py if aroundstate is not None: @@ -299,7 +290,7 @@ # from now on we hold the GIL stackcounter.stacks_counter += 1 try: - result = inner_wrapper(%(args)s) + result = callable(%(args)s) except Exception, e: os.write(2, "Warning: uncaught exception in callback: %%s %%s\n" %% @@ -321,7 +312,6 @@ miniglobals = locals().copy() miniglobals['Exception'] = Exception miniglobals['os'] = os - miniglobals['llstr'] = llstr miniglobals['we_are_translated'] = we_are_translated miniglobals['stackcounter'] = stackcounter exec source.compile() in miniglobals @@ -329,11 +319,8 @@ _make_wrapper_for._annspecialcase_ = 'specialize:memo' AroundFnPtr = lltype.Ptr(lltype.FuncType([], lltype.Void)) -CallbackHookPtr = lltype.Ptr(lltype.FuncType([lltype.Ptr(rstr.STR)], lltype.Void)) class AroundState: - callback_hook = None - def _cleanup_(self): self.before = None # or a regular RPython function self.after = None # or a regular RPython function @@ -795,6 +782,7 @@ # (char*, str, int, int) -> None @jit.dont_look_inside + @enforceargs(None, None, int, int) def str_from_buffer(raw_buf, gc_buf, allocated_size, needed_size): """ Converts from a pair returned by alloc_buffer to a high-level string. @@ -833,6 +821,7 @@ lltype.free(raw_buf, flavor='raw') # char* -> str, with an upper bound on the length in case there is no \x00 + @enforceargs(None, int) def charp2strn(cp, maxlen): b = builder_class(maxlen) i = 0 diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py --- a/pypy/rpython/memory/gctransform/framework.py +++ b/pypy/rpython/memory/gctransform/framework.py @@ -979,6 +979,7 @@ assert self.translator.config.translation.thread if hasattr(self.root_walker, 'thread_run_ptr'): livevars = self.push_roots(hop) + assert not livevars, "live GC var around %s!" % (hop.spaceop,) hop.genop("direct_call", [self.root_walker.thread_run_ptr]) self.pop_roots(hop, livevars) @@ -993,6 +994,7 @@ assert self.translator.config.translation.thread if hasattr(self.root_walker, 'thread_die_ptr'): livevars = self.push_roots(hop) + assert not livevars, "live GC var around %s!" % (hop.spaceop,) hop.genop("direct_call", [self.root_walker.thread_die_ptr]) self.pop_roots(hop, livevars) diff --git a/pypy/rpython/memory/gctransform/shadowstack.py b/pypy/rpython/memory/gctransform/shadowstack.py --- a/pypy/rpython/memory/gctransform/shadowstack.py +++ b/pypy/rpython/memory/gctransform/shadowstack.py @@ -326,6 +326,8 @@ self.gcdata.root_stack_top = llmemory.NULL # to detect missing restore def forget_current_state(self): + ll_assert(self.gcdata.root_stack_base == self.gcdata.root_stack_top, + "forget_current_state: shadowstack not empty!") if self.unused_full_stack: llmemory.raw_free(self.unused_full_stack) self.unused_full_stack = self.gcdata.root_stack_base diff --git a/pypy/translator/goal/targetpypystandalone.py b/pypy/translator/goal/targetpypystandalone.py --- a/pypy/translator/goal/targetpypystandalone.py +++ b/pypy/translator/goal/targetpypystandalone.py @@ -28,11 +28,6 @@ w_call_startup_gateway = space.wrap(gateway.interp2app(call_startup)) withjit = space.config.objspace.usemodules.pypyjit - if withjit: - from pypy.module.pypyjit.interp_jit import callback_hook - from pypy.rlib import objectmodel - objectmodel.register_around_callback_hook(callback_hook) - def entry_point(argv): if withjit: from pypy.jit.backend.hlinfo import highleveljitinfo From noreply at buildbot.pypy.org Fri Jan 18 01:41:33 2013 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 18 Jan 2013 01:41:33 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: use orig_array properly in get_real, get_imag Message-ID: <20130118004133.233501C00BE@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: missing-ndarray-attributes Changeset: r60164:334c093ab578 Date: 2013-01-18 02:41 +0200 http://bitbucket.org/pypy/pypy/changeset/334c093ab578/ Log: use orig_array properly in get_real, get_imag diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py --- a/pypy/module/micronumpy/arrayimpl/concrete.py +++ b/pypy/module/micronumpy/arrayimpl/concrete.py @@ -72,27 +72,27 @@ else: return None - def get_real(self): + def get_real(self, orig_array): strides = self.get_strides() backstrides = self.get_backstrides() if self.dtype.is_complex_type(): dtype = self.dtype.float_type return SliceArray(self.start, strides, backstrides, - self.get_shape(), self, self, dtype=dtype) + self.get_shape(), self, orig_array, dtype=dtype) return SliceArray(self.start, strides, backstrides, - self.get_shape(), self, self) + self.get_shape(), self, orig_array) - def get_imag(self): + def get_imag(self, orig_array): strides = self.get_strides() backstrides = self.get_backstrides() if self.dtype.is_complex_type(): dtype = self.dtype.float_type return SliceArray(self.start + dtype.get_size(), strides, - backstrides, self.get_shape(), self, self, dtype=dtype) + backstrides, self.get_shape(), self, orig_array, dtype=dtype) if self.dtype.is_flexible_type(): # numpy returns self for self.imag return SliceArray(self.start, strides, backstrides, - self.get_shape(), self, self) + self.get_shape(), self, orig_array) impl = NonWritableArray(self.get_shape(), self.dtype, self.order, strides, backstrides) impl.fill(self.dtype.box(0)) @@ -344,7 +344,6 @@ self.strides = strides self.backstrides = backstrides self.shape = shape - assert isinstance(parent, BaseConcreteArray) if isinstance(parent, SliceArray): parent = parent.parent # one level only self.parent = parent diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -240,10 +240,10 @@ return W_NDimArray(self.implementation.copy()) def descr_get_real(self, space): - return W_NDimArray(self.implementation.get_real()) + return W_NDimArray(self.implementation.get_real(self)) def descr_get_imag(self, space): - ret = self.implementation.get_imag() + ret = self.implementation.get_imag(self) if ret: return W_NDimArray(ret) raise OperationError(space.w_NotImplementedError, @@ -251,7 +251,7 @@ def descr_set_real(self, space, w_value): # copy (broadcast) values into self - tmp = self.implementation.get_real() + tmp = self.implementation.get_real(self) tmp.setslice(space, convert_to_array(space, w_value)) def descr_set_imag(self, space, w_value): @@ -259,7 +259,7 @@ if not self.get_dtype().is_complex_type(): raise OperationError(space.w_TypeError, space.wrap('array does not have imaginary part to set')) - tmp = self.implementation.get_imag() + tmp = self.implementation.get_imag(self) tmp.setslice(space, convert_to_array(space, w_value)) def descr_reshape(self, space, args_w): From noreply at buildbot.pypy.org Fri Jan 18 02:10:55 2013 From: noreply at buildbot.pypy.org (mattip) Date: Fri, 18 Jan 2013 02:10:55 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: add argsort test for sorting long runs Message-ID: <20130118011055.C15101C12FA@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: missing-ndarray-attributes Changeset: r60165:e6abb66fd78f Date: 2013-01-18 03:10 +0200 http://bitbucket.org/pypy/pypy/changeset/e6abb66fd78f/ Log: add argsort test for sorting long runs diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -2344,6 +2344,10 @@ from _numpypy import array a = array([[4, 2], [1, 3]]) assert (a.argsort() == [[1, 0], [0, 1]]).all() + #trigger timsort run mode + a = array(range(100) + range(100) + range(100)) + b = a.argsort() + assert (b[:3] == [0, 100, 200]).all() def test_argsort_axis(self): from _numpypy import array diff --git a/pypy/rlib/listsort.py b/pypy/rlib/listsort.py --- a/pypy/rlib/listsort.py +++ b/pypy/rlib/listsort.py @@ -517,6 +517,7 @@ def merge_force_collapse(self): p = self.pending while len(p) > 1: + xxx if len(p) >= 3 and p[-3].len < p[-1].len: self.merge_at(-3) else: From noreply at buildbot.pypy.org Fri Jan 18 10:00:12 2013 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Fri, 18 Jan 2013 10:00:12 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: work around pytest issue 251 Message-ID: <20130118090012.674B61C0724@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: split-rpython Changeset: r60166:812974e07064 Date: 2013-01-18 09:59 +0100 http://bitbucket.org/pypy/pypy/changeset/812974e07064/ Log: work around pytest issue 251 collection ignores test classes with __init__ for unittest support, but is silent about that diff --git a/rpython/conftest.py b/rpython/conftest.py --- a/rpython/conftest.py +++ b/rpython/conftest.py @@ -2,13 +2,6 @@ import py, pytest from rpython.tool import leakfinder -# XXX -# Aquana, i just ensured that with the pypy conftest we get all skips, -# i'll investigate differences in collection tommorow, for now, can you just -# import the makemodule hook from pypy and add a comment that im responsible -# for fixing? -from pypy.conftest import pytest_pycollect_makemodule - cdir = realpath(join(dirname(__file__), 'translator', 'c')) cache_dir = realpath(join(dirname(__file__), '_cache')) option = None @@ -38,15 +31,6 @@ set_platform(value, None) def pytest_addoption(parser): - # XXX - # Aquana, i just ensured that with the pypy conftest we get all skips, - # i'll investigate differences in collection tommorow, for now, can you just - # import the makemodule hook from pypy and add a comment that im responsible - # for fixing? - from pypy.conftest import pytest_addoption - pytest_addoption(parser) - - group = parser.getgroup("rpython options") group.addoption('--view', action="store_true", dest="view", default=False, help="view translation tests' flow graphs with Pygame") @@ -58,6 +42,17 @@ default=False, dest="viewloops", help="show only the compiled loops") + +def pytest_pycollect_makeitem(__multicall__,collector, name, obj): + res = __multicall__.execute() + # work around pytest issue 251 + import inspect + if res is None and inspect.isclass(obj) and \ + collector.classnamefilter(name): + return py.test.collect.Class(name, parent=collector) + return res + + def pytest_addhooks(pluginmanager): pluginmanager.register(LeakFinder()) From noreply at buildbot.pypy.org Fri Jan 18 11:42:48 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Fri, 18 Jan 2013 11:42:48 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed missing rename Message-ID: <20130118104248.799EC1C10B5@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60167:8bb2781a798b Date: 2013-01-17 22:38 +0100 http://bitbucket.org/pypy/pypy/changeset/8bb2781a798b/ Log: Fixed missing rename diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -191,8 +191,8 @@ assert space.str_w(elems[1]) == "b" def test_from_shape_and_storage(self): - from pypy.rlib.rawstorage import alloc_raw_storage, raw_storage_setitem - from pypy.rpython.lltypesystem import rffi + from rpython.rlib.rawstorage import alloc_raw_storage, raw_storage_setitem + from rpython.rtyper.lltypesystem import rffi from pypy.module.micronumpy.interp_dtype import get_dtype_cache storage = alloc_raw_storage(4, track_allocation=False, zero=True) for i in range(4): From noreply at buildbot.pypy.org Fri Jan 18 11:42:49 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Fri, 18 Jan 2013 11:42:49 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed test Message-ID: <20130118104249.C05E21C10B5@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60168:d0ef4dc66113 Date: 2013-01-18 11:19 +0100 http://bitbucket.org/pypy/pypy/changeset/d0ef4dc66113/ Log: Fixed test diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -191,8 +191,8 @@ assert space.str_w(elems[1]) == "b" def test_from_shape_and_storage(self): - from pypy.rlib.rawstorage import alloc_raw_storage, raw_storage_setitem - from pypy.rpython.lltypesystem import rffi + from rpython.rlib.rawstorage import alloc_raw_storage, raw_storage_setitem + from rpython.rtyper.lltypesystem import rffi from pypy.module.micronumpy.interp_dtype import get_dtype_cache storage = alloc_raw_storage(4, track_allocation=False, zero=True) for i in range(4): From noreply at buildbot.pypy.org Fri Jan 18 11:42:50 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Fri, 18 Jan 2013 11:42:50 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: call rpythons pytest_addoption in pypys Message-ID: <20130118104250.EA9051C10B5@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60169:b4732d7b026e Date: 2013-01-18 11:37 +0100 http://bitbucket.org/pypy/pypy/changeset/b4732d7b026e/ Log: call rpythons pytest_addoption in pypys diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -34,6 +34,9 @@ option = config.option def pytest_addoption(parser): + from rpython.conftest import pytest_addoption + pytest_addoption(parser) + group = parser.getgroup("pypy options") group.addoption('-A', '--runappdirect', action="store_true", default=False, dest="runappdirect", From noreply at buildbot.pypy.org Fri Jan 18 11:42:52 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Fri, 18 Jan 2013 11:42:52 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: merge? Message-ID: <20130118104252.1AD281C10B5@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60170:2c26eb0182a7 Date: 2013-01-18 11:41 +0100 http://bitbucket.org/pypy/pypy/changeset/2c26eb0182a7/ Log: merge? From noreply at buildbot.pypy.org Fri Jan 18 13:58:43 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Fri, 18 Jan 2013 13:58:43 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed test Message-ID: <20130118125843.1ACE41C10B5@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60171:1b9bd9092295 Date: 2013-01-18 13:57 +0100 http://bitbucket.org/pypy/pypy/changeset/1b9bd9092295/ Log: Fixed test diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py --- a/rpython/translator/c/test/test_standalone.py +++ b/rpython/translator/c/test/test_standalone.py @@ -35,7 +35,7 @@ else: cbuilder.generate_source() cbuilder.compile() - if option.view: + if option is not None and option.view: t.view() return t, cbuilder From noreply at buildbot.pypy.org Fri Jan 18 14:38:52 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Fri, 18 Jan 2013 14:38:52 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Fixed import Message-ID: <20130118133852.6CFF71C10CB@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60172:63789f745d8f Date: 2013-01-18 14:38 +0100 http://bitbucket.org/pypy/pypy/changeset/63789f745d8f/ Log: Fixed import diff --git a/rpython/tool/test/test_descriptor.py b/rpython/tool/test/test_descriptor.py --- a/rpython/tool/test/test_descriptor.py +++ b/rpython/tool/test/test_descriptor.py @@ -1,4 +1,4 @@ -rpython.tool.descriptor import InstanceMethod +from rpython.tool.descriptor import InstanceMethod class X(object): def f(self, *args, **kwds): diff --git a/rpython/tool/test/test_identitydict.py b/rpython/tool/test/test_identitydict.py --- a/rpython/tool/test/test_identitydict.py +++ b/rpython/tool/test/test_identitydict.py @@ -35,7 +35,7 @@ assert len(d) == 3 assert d[a] == 4 - raises(KeyError, d.__getitem__, []) + py.test.raises(KeyError, d.__getitem__, []) def test_keys(self): d = self.identity_dict() From noreply at buildbot.pypy.org Fri Jan 18 15:17:35 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 18 Jan 2013 15:17:35 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: an API I want from a custom tracer (that can be run directly too) Message-ID: <20130118141735.7BC231C10B0@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60173:c68a9032570c Date: 2013-01-18 16:17 +0200 http://bitbucket.org/pypy/pypy/changeset/c68a9032570c/ Log: an API I want from a custom tracer (that can be run directly too) diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py --- a/pypy/rpython/lltypesystem/lltype.py +++ b/pypy/rpython/lltypesystem/lltype.py @@ -361,7 +361,7 @@ Struct._install_extras(self, **kwds) def _attach_runtime_type_info_funcptr(self, funcptr, destrptr, - customtraceptr): + custom_trace_func): if self._runtime_type_info is None: raise TypeError("attachRuntimeTypeInfo: %r must have been built " "with the rtti=True argument" % (self,)) @@ -385,18 +385,8 @@ raise TypeError("expected a destructor function " "implementation, got: %s" % destrptr) self._runtime_type_info.destructor_funcptr = destrptr - if customtraceptr is not None: - from pypy.rpython.lltypesystem import llmemory - T = typeOf(customtraceptr) - if (not isinstance(T, Ptr) or - not isinstance(T.TO, FuncType) or - len(T.TO.ARGS) != 2 or - T.TO.RESULT != llmemory.Address or - T.TO.ARGS[0] != llmemory.Address or - T.TO.ARGS[1] != llmemory.Address): - raise TypeError("expected a custom trace function " - "implementation, got: %s" % customtraceptr) - self._runtime_type_info.custom_trace_funcptr = customtraceptr + if custom_trace_func is not None: + self._runtime_type_info.custom_trace_func = custom_trace_func class GcStruct(RttiStruct): _gckind = 'gc' @@ -2050,11 +2040,11 @@ return _ptr(PTRTYPE, oddint, solid=True) def attachRuntimeTypeInfo(GCSTRUCT, funcptr=None, destrptr=None, - customtraceptr=None): + custom_trace_func=None): if not isinstance(GCSTRUCT, RttiStruct): raise TypeError, "expected a RttiStruct: %s" % GCSTRUCT GCSTRUCT._attach_runtime_type_info_funcptr(funcptr, destrptr, - customtraceptr) + custom_trace_func) return _ptr(Ptr(RuntimeTypeInfo), GCSTRUCT._runtime_type_info) def getRuntimeTypeInfo(GCSTRUCT): diff --git a/pypy/rpython/memory/gc/base.py b/pypy/rpython/memory/gc/base.py --- a/pypy/rpython/memory/gc/base.py +++ b/pypy/rpython/memory/gc/base.py @@ -73,7 +73,6 @@ member_index, is_rpython_class, has_custom_trace, - get_custom_trace, fast_path_tracing): self.getfinalizer = getfinalizer self.getlightfinalizer = getlightfinalizer @@ -90,7 +89,6 @@ self.member_index = member_index self.is_rpython_class = is_rpython_class self.has_custom_trace = has_custom_trace - self.get_custom_trace = get_custom_trace self.fast_path_tracing = fast_path_tracing def get_member_index(self, type_id): @@ -211,6 +209,13 @@ i += 1 trace._annspecialcase_ = 'specialize:arg(2)' + def _call_custom_trace(self, obj, typeid, callback, arg): + def wrapper(item, arg): + if self.is_valid_gc_object(item): + callback(item, arg) + + self.custom_trace_funcs[typeid.index](obj, wrapper, arg) + def _trace_slow_path(self, obj, callback, arg): typeid = self.get_type_id(obj) if self.has_gcptr_in_varsize(typeid): @@ -228,14 +233,7 @@ item += itemlength length -= 1 if self.has_custom_trace(typeid): - generator = self.get_custom_trace(typeid) - item = llmemory.NULL - while True: - item = generator(obj, item) - if not item: - break - if self.points_to_valid_gc_object(item): - callback(item, arg) + self._call_custom_trace(obj, typeid, callback, arg) _trace_slow_path._annspecialcase_ = 'specialize:arg(2)' def trace_partial(self, obj, start, stop, callback, arg): diff --git a/pypy/rpython/memory/gctypelayout.py b/pypy/rpython/memory/gctypelayout.py --- a/pypy/rpython/memory/gctypelayout.py +++ b/pypy/rpython/memory/gctypelayout.py @@ -127,12 +127,6 @@ infobits = self.get(typeid).infobits return infobits & T_HAS_CUSTOM_TRACE != 0 - def q_get_custom_trace(self, typeid): - ll_assert(self.q_has_custom_trace(typeid), - "T_HAS_CUSTOM_TRACE missing") - typeinfo = self.get(typeid) - return typeinfo.finalizer_or_customtrace - def q_fast_path_tracing(self, typeid): # return True if none of the flags T_HAS_GCPTR_IN_VARSIZE, # T_IS_GCARRAY_OF_GCPTR or T_HAS_CUSTOM_TRACE is set @@ -159,7 +153,6 @@ self.q_member_index, self.q_is_rpython_class, self.q_has_custom_trace, - self.q_get_custom_trace, self.q_fast_path_tracing) @@ -204,10 +197,12 @@ infobits |= T_HAS_FINALIZER elif kind == 'light_finalizer': infobits |= T_HAS_FINALIZER | T_HAS_LIGHTWEIGHT_FINALIZER - elif kind == "custom_trace": - infobits |= T_HAS_CUSTOM_TRACE else: assert 0, kind + custom_trace_func = builder.get_custom_trace_func(TYPE) + if custom_trace_func is not None: + infobits |= T_HAS_CUSTOM_TRACE + builder.record_custom_trace(index, custom_trace_func) # if not TYPE._is_varsize(): info.fixedsize = llarena.round_up_for_allocation( @@ -378,16 +373,11 @@ if TYPE in self._special_funcptrs: return self._special_funcptrs[TYPE] fptr1, is_lightweight = self.make_finalizer_funcptr_for_type(TYPE) - fptr2 = self.make_custom_trace_funcptr_for_type(TYPE) - assert not (fptr1 and fptr2), ( - "type %r needs both a finalizer and a custom tracer" % (TYPE,)) if fptr1: if is_lightweight: kind_and_fptr = "light_finalizer", fptr1 else: kind_and_fptr = "finalizer", fptr1 - elif fptr2: - kind_and_fptr = "custom_trace", fptr2 else: kind_and_fptr = None self._special_funcptrs[TYPE] = kind_and_fptr diff --git a/pypy/rpython/memory/gcwrapper.py b/pypy/rpython/memory/gcwrapper.py --- a/pypy/rpython/memory/gcwrapper.py +++ b/pypy/rpython/memory/gcwrapper.py @@ -25,9 +25,12 @@ def prepare_graphs(self, flowgraphs): lltype2vtable = self.llinterp.typer.lltype2vtable + # only valid for direct layout builder + self.gc.custom_trace_funcs = {} layoutbuilder = DirectRunLayoutBuilder(self.gc.__class__, lltype2vtable, - self.llinterp) + self.llinterp, + self.gc.custom_trace_funcs) self.get_type_id = layoutbuilder.get_type_id layoutbuilder.initialize_gc_query_function(self.gc) @@ -200,9 +203,10 @@ class DirectRunLayoutBuilder(gctypelayout.TypeLayoutBuilder): - def __init__(self, GCClass, lltype2vtable, llinterp): + def __init__(self, GCClass, lltype2vtable, llinterp, custom_trace_funcs): self.llinterp = llinterp super(DirectRunLayoutBuilder, self).__init__(GCClass, lltype2vtable) + self.custom_trace_funcs = custom_trace_funcs def make_finalizer_funcptr_for_type(self, TYPE): from pypy.rpython.memory.gctransform.support import get_rtti @@ -227,11 +231,14 @@ return llmemory.NULL return llhelper(gctypelayout.GCData.FINALIZER_OR_CT, ll_finalizer), light - def make_custom_trace_funcptr_for_type(self, TYPE): + def record_custom_trace(self, tid, custom_trace_func): + self.custom_trace_funcs[tid] = custom_trace_func + + def get_custom_trace_func(self, TYPE): from pypy.rpython.memory.gctransform.support import get_rtti rtti = get_rtti(TYPE) - if rtti is not None and hasattr(rtti._obj, 'custom_trace_funcptr'): - return rtti._obj.custom_trace_funcptr + if rtti is not None and hasattr(rtti._obj, 'custom_trace_func'): + return rtti._obj.custom_trace_func else: return None diff --git a/pypy/rpython/memory/test/test_gc.py b/pypy/rpython/memory/test/test_gc.py --- a/pypy/rpython/memory/test/test_gc.py +++ b/pypy/rpython/memory/test/test_gc.py @@ -7,10 +7,11 @@ from pypy.rpython.lltypesystem import lltype from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.objectmodel import we_are_translated -from pypy.rlib.objectmodel import compute_unique_id +from pypy.rlib.objectmodel import compute_unique_id, specialize from pypy.rlib import rgc from pypy.rlib.rstring import StringBuilder from pypy.rlib.rarithmetic import LONG_BIT +from pypy.jit.backend.llsupport import jitframe WORD = LONG_BIT // 8 @@ -237,7 +238,6 @@ assert 160 <= res <= 165 def test_custom_trace(self): - from pypy.rpython.annlowlevel import llhelper from pypy.rpython.lltypesystem import llmemory from pypy.rpython.lltypesystem.llarena import ArenaError # @@ -245,15 +245,10 @@ ('y', llmemory.Address), rtti=True) T = lltype.GcStruct('T', ('z', lltype.Signed)) offset_of_x = llmemory.offsetof(S, 'x') - def customtrace(obj, prev): - if not prev: - return obj + offset_of_x - else: - return llmemory.NULL - CUSTOMTRACEFUNC = lltype.FuncType([llmemory.Address, llmemory.Address], - llmemory.Address) - customtraceptr = llhelper(lltype.Ptr(CUSTOMTRACEFUNC), customtrace) - lltype.attachRuntimeTypeInfo(S, customtraceptr=customtraceptr) + def customtrace(obj, callback, arg): + callback(obj + offset_of_x, arg) + + lltype.attachRuntimeTypeInfo(S, custom_trace_func=customtrace) # for attrname in ['x', 'y']: def setup(): From noreply at buildbot.pypy.org Fri Jan 18 15:45:49 2013 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Fri, 18 Jan 2013 15:45:49 +0100 (CET) Subject: [pypy-commit] pypy pytest: merge from default Message-ID: <20130118144549.88EC51C0DC1@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: pytest Changeset: r60174:1d88326d9eda Date: 2013-01-18 10:10 +0100 http://bitbucket.org/pypy/pypy/changeset/1d88326d9eda/ Log: merge from default diff --git a/pypy/jit/metainterp/compile.py b/pypy/jit/metainterp/compile.py --- a/pypy/jit/metainterp/compile.py +++ b/pypy/jit/metainterp/compile.py @@ -5,7 +5,7 @@ from pypy.rlib.objectmodel import we_are_translated from pypy.rlib.debug import debug_start, debug_stop, debug_print from pypy.rlib import rstack -from pypy.rlib.jit import JitDebugInfo, Counters +from pypy.rlib.jit import JitDebugInfo, Counters, dont_look_inside from pypy.conftest import option from pypy.tool.sourcetools import func_with_new_name @@ -685,6 +685,7 @@ assert 0, "unreachable" @staticmethod + @dont_look_inside def force_now(cpu, token): # Called during a residual call from the assembler, if the code # actually needs to force one of the virtualrefs or the virtualizable. diff --git a/pypy/jit/metainterp/virtualref.py b/pypy/jit/metainterp/virtualref.py --- a/pypy/jit/metainterp/virtualref.py +++ b/pypy/jit/metainterp/virtualref.py @@ -169,4 +169,3 @@ # token == TOKEN_NONE and the vref was not forced: it's invalid raise InvalidVirtualRef return vref.forced - force_virtual._dont_inline_ = True diff --git a/pypy/module/micronumpy/arrayimpl/concrete.py b/pypy/module/micronumpy/arrayimpl/concrete.py --- a/pypy/module/micronumpy/arrayimpl/concrete.py +++ b/pypy/module/micronumpy/arrayimpl/concrete.py @@ -9,7 +9,7 @@ from pypy.interpreter.error import OperationError, operationerrfmt from pypy.rpython.lltypesystem import rffi, lltype from pypy.rlib import jit -from pypy.rlib.rawstorage import free_raw_storage +from pypy.rlib.rawstorage import free_raw_storage, RAW_STORAGE from pypy.rlib.debug import make_sure_not_resized class ConcreteArrayIterator(base.BaseArrayIterator): @@ -427,18 +427,18 @@ def get_storage_as_int(self, space): return rffi.cast(lltype.Signed, self.storage) -class ConcreteArray(BaseConcreteArray): - def __init__(self, shape, dtype, order, strides, backstrides): +class ConcreteArrayNotOwning(BaseConcreteArray): + def __init__(self, shape, dtype, order, strides, backstrides, storage): make_sure_not_resized(shape) make_sure_not_resized(strides) make_sure_not_resized(backstrides) self.shape = shape self.size = support.product(shape) * dtype.get_size() - self.storage = dtype.itemtype.malloc(self.size) self.order = order self.dtype = dtype self.strides = strides self.backstrides = backstrides + self.storage = storage def create_iter(self, shape=None): if shape is None or shape == self.get_shape(): @@ -451,14 +451,24 @@ def fill(self, box): self.dtype.fill(self.storage, box, 0, self.size) - def __del__(self): - free_raw_storage(self.storage, track_allocation=False) - def set_shape(self, space, new_shape): strides, backstrides = support.calc_strides(new_shape, self.dtype, self.order) return SliceArray(0, strides, backstrides, new_shape, self) +class ConcreteArray(ConcreteArrayNotOwning): + def __init__(self, shape, dtype, order, strides, backstrides): + # we allocate the actual storage later because we need to compute + # self.size first + null_storage = lltype.nullptr(RAW_STORAGE) + ConcreteArrayNotOwning.__init__(self, shape, dtype, order, strides, backstrides, + null_storage) + self.storage = dtype.itemtype.malloc(self.size) + + def __del__(self): + free_raw_storage(self.storage, track_allocation=False) + + class NonWritableArray(ConcreteArray): def descr_setitem(self, space, w_index, w_value): raise OperationError(space.w_RuntimeError, space.wrap( diff --git a/pypy/module/micronumpy/base.py b/pypy/module/micronumpy/base.py --- a/pypy/module/micronumpy/base.py +++ b/pypy/module/micronumpy/base.py @@ -28,6 +28,16 @@ return W_NDimArray(impl) @staticmethod + def from_shape_and_storage(shape, storage, dtype, order='C'): + from pypy.module.micronumpy.arrayimpl import concrete + assert shape + strides, backstrides = calc_strides(shape, dtype, order) + impl = concrete.ConcreteArrayNotOwning(shape, dtype, order, strides, + backstrides, storage) + return W_NDimArray(impl) + + + @staticmethod def new_slice(offset, strides, backstrides, shape, parent, dtype=None): from pypy.module.micronumpy.arrayimpl import concrete diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -397,7 +397,6 @@ space.w_False])) return w_d - # --------------------- operations ---------------------------- def _unaryop_impl(ufunc_name): @@ -565,6 +564,20 @@ return W_NDimArray.new_scalar(space, dtype) return W_NDimArray.from_shape(shape, dtype) + at unwrap_spec(addr=int) +def descr__from_shape_and_storage(space, w_cls, w_shape, addr, w_dtype): + """ + Create an array from an existing buffer, given its address as int. + PyPy-only implementation detail. + """ + from pypy.rpython.lltypesystem import rffi + from pypy.rlib.rawstorage import RAW_STORAGE_PTR + shape = _find_shape(space, w_shape) + storage = rffi.cast(RAW_STORAGE_PTR, addr) + dtype = space.interp_w(interp_dtype.W_Dtype, + space.call_function(space.gettypefor(interp_dtype.W_Dtype), w_dtype)) + return W_NDimArray.from_shape_and_storage(shape, storage, dtype) + W_NDimArray.typedef = TypeDef( "ndarray", __new__ = interp2app(descr_new_array), @@ -661,6 +674,8 @@ imag = GetSetProperty(W_NDimArray.descr_get_imag, W_NDimArray.descr_set_imag), __array_interface__ = GetSetProperty(W_NDimArray.descr_array_iface), + _from_shape_and_storage = interp2app(descr__from_shape_and_storage, + as_classmethod=True) ) @unwrap_spec(ndmin=int, copy=bool, subok=bool) diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -189,7 +189,23 @@ assert shape == [2] assert space.str_w(elems[0]) == "a" assert space.str_w(elems[1]) == "b" - + + def test_from_shape_and_storage(self): + from pypy.rlib.rawstorage import alloc_raw_storage, raw_storage_setitem + from pypy.rpython.lltypesystem import rffi + from pypy.module.micronumpy.interp_dtype import get_dtype_cache + storage = alloc_raw_storage(4, track_allocation=False, zero=True) + for i in range(4): + raw_storage_setitem(storage, i, rffi.cast(rffi.UCHAR, i)) + # + dtypes = get_dtype_cache(self.space) + w_array = W_NDimArray.from_shape_and_storage([2, 2], storage, dtypes.w_int8dtype) + def get(i, j): + return w_array.getitem(self.space, [i, j]).value + assert get(0, 0) == 0 + assert get(0, 1) == 1 + assert get(1, 0) == 2 + assert get(1, 1) == 3 class AppTestNumArray(BaseNumpyAppTest): def w_CustomIndexObject(self, index): @@ -2340,7 +2356,7 @@ s = repr(a) assert s.replace('\n', '') == \ "array(['abc', 'defg', 'ab'], dtype='|S4')" - + class AppTestPyPy(BaseNumpyAppTest): def setup_class(cls): @@ -2360,3 +2376,12 @@ assert a[0][1] == 2 a = _numpypy.array(([[[1, 2], [3, 4], [5, 6]]])) assert (a[0, 1] == [3, 4]).all() + + def test_from_shape_and_storage(self): + from _numpypy import array, ndarray + x = array([1, 2, 3, 4]) + addr, _ = x.__array_interface__['data'] + y = ndarray._from_shape_and_storage([2, 2], addr, x.dtype) + assert y[0, 1] == 2 + y[0, 1] = 42 + assert x[1] == 42 diff --git a/pypy/rlib/runicode.py b/pypy/rlib/runicode.py --- a/pypy/rlib/runicode.py +++ b/pypy/rlib/runicode.py @@ -460,6 +460,10 @@ errorhandler=None, byteorder='little'): if size == 0: + if byteorder == 'native': + result = StringBuilder(2) + _STORECHAR(result, 0xFEFF, BYTEORDER) + return result.build() return "" result = StringBuilder(size * 2 + 2) @@ -621,6 +625,10 @@ errorhandler=None, byteorder='little'): if size == 0: + if byteorder == 'native': + result = StringBuilder(4) + _STORECHAR32(result, 0xFEFF, BYTEORDER) + return result.build() return "" result = StringBuilder(size * 4 + 4) diff --git a/pypy/rlib/test/test_runicode.py b/pypy/rlib/test/test_runicode.py --- a/pypy/rlib/test/test_runicode.py +++ b/pypy/rlib/test/test_runicode.py @@ -679,6 +679,11 @@ "utf-32 utf-32-be utf-32-le").split(): self.checkencode(uni, encoding) + def test_empty(self): + for encoding in ("utf-8 utf-16 utf-16-be utf-16-le " + "utf-32 utf-32-be utf-32-le").split(): + self.checkencode(u'', encoding) + def test_single_chars_utf8(self): # check every number of bytes per char for s in ["\xd7\x90", "\xd6\x96", "\xeb\x96\x95", "\xf0\x90\x91\x93"]: From noreply at buildbot.pypy.org Fri Jan 18 15:45:50 2013 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Fri, 18 Jan 2013 15:45:50 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: move expecttest to rpython, where it is used Message-ID: <20130118144550.D06661C0DC1@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: split-rpython Changeset: r60175:ebd8f08afe07 Date: 2013-01-18 15:45 +0100 http://bitbucket.org/pypy/pypy/changeset/ebd8f08afe07/ Log: move expecttest to rpython, where it is used diff --git a/pypy/tool/pytest/test/test_pytestsupport.py b/pypy/tool/pytest/test/test_pytestsupport.py --- a/pypy/tool/pytest/test/test_pytestsupport.py +++ b/pypy/tool/pytest/test/test_pytestsupport.py @@ -9,7 +9,6 @@ import os import sys import pypy -conftestpath = py.path.local(pypy.__file__).dirpath("conftest.py") pytest_plugins = "pytest_pytester" @@ -101,48 +100,6 @@ def test_equal(self): assert self.compute(3) == 5 -def test_expectcollect(testdir): - py.test.importorskip("pexpect") - conftestpath.copy(testdir.tmpdir) - sorter = testdir.inline_runsource(""" - class ExpectTestOne: - def test_one(self): - pass - """) - passed, skipped, failed = sorter.countoutcomes() - assert passed == 1 - -def test_safename(): - from pypy.tool.pytest.expecttest import ExpectTestMethod - - safe_name = ExpectTestMethod.safe_name - assert safe_name(['pypy', 'tool', 'test', 'test_pytestsupport.py', - 'ExpectTest', '()', 'test_one']) == \ - 'pypy_tool_test_test_pytestsupport_ExpectTest_paren_test_one' - -def test_safe_filename(testdir): - py.test.importorskip("pexpect") - conftestpath.copy(testdir.tmpdir) - sorter = testdir.inline_runsource(""" - class ExpectTestOne: - def test_one(self): - pass - """) - evlist = sorter.getcalls("pytest_runtest_makereport") - ev = [x for x in evlist if x.call.when == "call"][0] - print ev - sfn = ev.item.safe_filename() - print sfn - assert sfn == 'test_safe_filename_test_safe_filename_ExpectTestOne_paren_test_one_1.py' - -class ExpectTest: - def test_one(self): - import os - import sys - assert os.ttyname(sys.stdin.fileno()) - - def test_two(self): - import pypy def test_app_test_blow(testdir): conftestpath.copy(testdir.tmpdir) diff --git a/rpython/conftest.py b/rpython/conftest.py --- a/rpython/conftest.py +++ b/rpython/conftest.py @@ -2,6 +2,8 @@ import py, pytest from rpython.tool import leakfinder +pytest_plugins = 'rpython.tool.pytest.expecttest' + cdir = realpath(join(dirname(__file__), 'translator', 'c')) cache_dir = realpath(join(dirname(__file__), '_cache')) option = None diff --git a/rpython/tool/pytest/__init__.py b/rpython/tool/pytest/__init__.py new file mode 100644 diff --git a/pypy/tool/pytest/expecttest.py b/rpython/tool/pytest/expecttest.py rename from pypy/tool/pytest/expecttest.py rename to rpython/tool/pytest/expecttest.py --- a/pypy/tool/pytest/expecttest.py +++ b/rpython/tool/pytest/expecttest.py @@ -7,9 +7,10 @@ import py -import os, sys +import os +import sys from rpython.tool.udir import udir -from pypy.conftest import pypydir +from pypy.conftest import pypydir #XXX class ExpectTestMethod(py.test.collect.Function): @@ -71,4 +72,7 @@ except ImportError: py.test.skip("pexpect not found") - + at py.test.mark.tryfirst +def pytest_pycollect_makeitem(collector, name, obj): + if py.std.inspect.isclass(obj) and name.startswith('ExpectTest'): + return ExpectClassCollector(name, parent=collector) diff --git a/rpython/tool/pytest/test/test_expecttest.py b/rpython/tool/pytest/test/test_expecttest.py new file mode 100644 --- /dev/null +++ b/rpython/tool/pytest/test/test_expecttest.py @@ -0,0 +1,53 @@ +import py +import rpython + +pytest_plugins = "pytest_pytester" + +conftestpath = py.path.local(rpython.__file__).dirpath("conftest.py") + + +def test_expectcollect(testdir): + py.test.importorskip("pexpect") + conftestpath.copy(testdir.tmpdir) + sorter = testdir.inline_runsource(""" + class ExpectTestOne: + def test_one(self): + pass + """) + passed, skipped, failed = sorter.countoutcomes() + assert passed == 1 + + +def test_safename(): + from rpython.tool.pytest.expecttest import ExpectTestMethod + + safe_name = ExpectTestMethod.safe_name + assert safe_name(['pypy', 'tool', 'test', 'test_pytestsupport.py', + 'ExpectTest', '()', 'test_one']) == \ + 'pypy_tool_test_test_pytestsupport_ExpectTest_paren_test_one' + + +def test_safe_filename(testdir): + py.test.importorskip("pexpect") + conftestpath.copy(testdir.tmpdir) + sorter = testdir.inline_runsource(""" + class ExpectTestOne: + def test_one(self): + pass + """) + evlist = sorter.getcalls("pytest_runtest_makereport") + ev = [x for x in evlist if x.call.when == "call"][0] + print ev + sfn = ev.item.safe_filename() + print sfn + assert sfn == 'test_safe_filename_test_safe_filename_ExpectTestOne_paren_test_one_1.py' + + +class ExpectTest: + def test_one(self): + import os + import sys + assert os.ttyname(sys.stdin.fileno()) + + def test_two(self): + import pypy From noreply at buildbot.pypy.org Fri Jan 18 16:04:22 2013 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 18 Jan 2013 16:04:22 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: Do not turn inevitable either on raw_malloc, in addition to Message-ID: <20130118150422.A7D7A1C0DC1@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r60176:ffd1aad64887 Date: 2013-01-18 16:04 +0100 http://bitbucket.org/pypy/pypy/changeset/ffd1aad64887/ Log: Do not turn inevitable either on raw_malloc, in addition to malloc(flavor='raw'). diff --git a/pypy/translator/backendopt/writeanalyze.py b/pypy/translator/backendopt/writeanalyze.py --- a/pypy/translator/backendopt/writeanalyze.py +++ b/pypy/translator/backendopt/writeanalyze.py @@ -82,7 +82,7 @@ for op in block.operations: self.allvariables.add(op.result) if (op.opname == 'malloc' or op.opname == 'malloc_varsize' - or op.opname == 'new'): + or op.opname == 'new' or op.opname == 'raw_malloc'): continue elif op.opname in ('cast_pointer', 'same_as'): if self.is_fresh_malloc(op.args[0]): diff --git a/pypy/translator/stm/inevitable.py b/pypy/translator/stm/inevitable.py --- a/pypy/translator/stm/inevitable.py +++ b/pypy/translator/stm/inevitable.py @@ -71,6 +71,11 @@ # If the transaction is splitted, the remaining parts of the # CFG will always run in inevitable mode anyways. return not fresh_mallocs.is_fresh_malloc(op.args[0]) + # + if op.opname == 'raw_malloc': + return False # XXX: Produces memory leaks on aborts + if op.opname == 'raw_free': + return not fresh_mallocs.is_fresh_malloc(op.args[0]) # # Function calls diff --git a/pypy/translator/stm/test/test_inevitable.py b/pypy/translator/stm/test/test_inevitable.py --- a/pypy/translator/stm/test/test_inevitable.py +++ b/pypy/translator/stm/test/test_inevitable.py @@ -42,13 +42,13 @@ def test_unsupported_op(self): X = lltype.Struct('X', ('foo', lltype.Signed)) + addr = llmemory.raw_malloc(llmemory.sizeof(X)) def f1(): - addr = llmemory.raw_malloc(llmemory.sizeof(X)) llmemory.raw_free(addr) res = self.interpret_inevitable(f1, []) - assert res == 'raw_malloc' + assert res == 'raw_free' def test_raw_getfield(self): X = lltype.Struct('X', ('foo', lltype.Signed)) @@ -105,7 +105,7 @@ res = self.interpret_inevitable(f1, []) assert res is None - def test_raw_malloc(self): + def test_raw_malloc_1(self): X = lltype.Struct('X', ('foo', lltype.Signed)) def f1(): @@ -117,6 +117,18 @@ assert 0, """we do not turn inevitable before raw-mallocs which causes leaks on aborts""" + def test_raw_malloc_2(self): + X = lltype.Struct('X', ('foo', lltype.Signed)) + + def f1(): + addr = llmemory.raw_malloc(llmemory.sizeof(X)) + llmemory.raw_free(addr) + + res = self.interpret_inevitable(f1, []) + assert res is None + assert 0, """we do not turn inevitable before + raw-mallocs which causes leaks on aborts""" + def test_unknown_raw_free(self): X = lltype.Struct('X', ('foo', lltype.Signed)) def f2(p): From noreply at buildbot.pypy.org Fri Jan 18 16:17:19 2013 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Fri, 18 Jan 2013 16:17:19 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: kill remainders of expecttest in pypy Message-ID: <20130118151719.3488E1C10B0@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: split-rpython Changeset: r60177:2e610d355c7e Date: 2013-01-18 16:16 +0100 http://bitbucket.org/pypy/pypy/changeset/2e610d355c7e/ Log: kill remainders of expecttest in pypy also fixme notes for defered fixes to the expecttest plugin diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -113,16 +113,6 @@ if name.startswith('AppTest'): from pypy.tool.pytest.apptest import AppClassCollector return AppClassCollector(name, parent=self) - elif name.startswith('ExpectTest'): - if self.config.option.rundirect: - return py.test.collect.Class(name, parent=self) - from pypy.tool.pytest.expecttest import ExpectClassCollector - return ExpectClassCollector(name, parent=self) - # XXX todo - #elif name.startswith('AppExpectTest'): - # if option.rundirect: - # return AppClassCollector(name, parent=self) - # return AppExpectClassCollector(name, parent=self) else: from pypy.tool.pytest.inttest import IntClassCollector return IntClassCollector(name, parent=self) diff --git a/rpython/tool/pytest/expecttest.py b/rpython/tool/pytest/expecttest.py --- a/rpython/tool/pytest/expecttest.py +++ b/rpython/tool/pytest/expecttest.py @@ -72,7 +72,10 @@ except ImportError: py.test.skip("pexpect not found") + @py.test.mark.tryfirst def pytest_pycollect_makeitem(collector, name, obj): if py.std.inspect.isclass(obj) and name.startswith('ExpectTest'): + #XXX: in conftest we had a rundirect option + #XXX: kill expecttest for a more explicit way return ExpectClassCollector(name, parent=collector) From noreply at buildbot.pypy.org Fri Jan 18 16:42:44 2013 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 18 Jan 2013 16:42:44 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: Fix at least one of the ztranslated test: test_targetdemo. Message-ID: <20130118154244.41B7B1C10B0@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r60178:3fe775d4717c Date: 2013-01-18 16:42 +0100 http://bitbucket.org/pypy/pypy/changeset/3fe775d4717c/ Log: Fix at least one of the ztranslated test: test_targetdemo. diff --git a/pypy/translator/c/src/mem.c b/pypy/translator/c/src/mem.c --- a/pypy/translator/c/src/mem.c +++ b/pypy/translator/c/src/mem.c @@ -15,25 +15,37 @@ static struct pypy_debug_alloc_s *pypy_debug_alloc_list = NULL; +#ifdef RPY_STM +# include "src_stm/atomic_ops.h" +static volatile Unsigned pypy_debug_alloc_lock = 0; +#else +# define stm_lock_acquire(lock) /* nothing */ +# define stm_lock_release(lock) /* nothing */ +#endif + void pypy_debug_alloc_start(void *addr, const char *funcname) { struct pypy_debug_alloc_s *p = malloc(sizeof(struct pypy_debug_alloc_s)); RPyAssert(p, "out of memory"); - p->next = pypy_debug_alloc_list; p->addr = addr; p->funcname = funcname; + stm_lock_acquire(pypy_debug_alloc_lock); + p->next = pypy_debug_alloc_list; pypy_debug_alloc_list = p; + stm_lock_release(pypy_debug_alloc_lock); } void pypy_debug_alloc_stop(void *addr) { struct pypy_debug_alloc_s **p; + stm_lock_acquire(pypy_debug_alloc_lock); for (p = &pypy_debug_alloc_list; *p; p = &((*p)->next)) if ((*p)->addr == addr) { struct pypy_debug_alloc_s *dying; dying = *p; *p = dying->next; + stm_lock_release(pypy_debug_alloc_lock); free(dying); return; } diff --git a/pypy/translator/stm/src_stm/atomic_ops.h b/pypy/translator/stm/src_stm/atomic_ops.h --- a/pypy/translator/stm/src_stm/atomic_ops.h +++ b/pypy/translator/stm/src_stm/atomic_ops.h @@ -33,13 +33,13 @@ #else /* x86 (32 bits and 64 bits) */ static inline _Bool -bool_cas(volatile revision_t *ptr, revision_t old, revision_t _new) +bool_cas(volatile Unsigned *ptr, Unsigned old, Unsigned _new) { - revision_t prev; + Unsigned prev; #if defined(__amd64__) - assert(sizeof(revision_t) == 8); + assert(sizeof(Unsigned) == 8); #elif defined(__i386__) - assert(sizeof(revision_t) == 4); + assert(sizeof(Unsigned) == 4); #else # error "the custom version of bool_cas() is only for x86 or x86-64" #endif @@ -67,4 +67,11 @@ } +#define stm_lock_acquire(lock) \ + do { while (!bool_cas(&(lock), 0, 1)) spinloop(); } while (0) + +#define stm_lock_release(lock) \ + (lock) = 0; + + #endif /* _SRCSTM_ATOMIC_OPS_ */ diff --git a/pypy/translator/stm/test/test_ztranslated.py b/pypy/translator/stm/test/test_ztranslated.py --- a/pypy/translator/stm/test/test_ztranslated.py +++ b/pypy/translator/stm/test/test_ztranslated.py @@ -6,7 +6,7 @@ class TestNoGcSTMTranslated(NoGcCompiledSTMTests): - def test_targetdemo(self): + def test_nogc_targetdemo(self): t, cbuilder = self.compile(targetdemo2.entry_point) data, dataerr = cbuilder.cmdexec('4 100', err=True) assert 'check ok!' in data From noreply at buildbot.pypy.org Fri Jan 18 16:56:39 2013 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 18 Jan 2013 16:56:39 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: Another point Message-ID: <20130118155639.697A01C10B0@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r60179:712bc4b2fa75 Date: 2013-01-18 16:56 +0100 http://bitbucket.org/pypy/pypy/changeset/712bc4b2fa75/ Log: Another point diff --git a/TODO b/TODO --- a/TODO +++ b/TODO @@ -33,7 +33,7 @@ ------------------------------------------------------------ -GC: major collections +GC: major collections; call __del__() ------------------------------------------------------------ From noreply at buildbot.pypy.org Fri Jan 18 16:58:24 2013 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 18 Jan 2013 16:58:24 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: These two flags are not referenced from C. Message-ID: <20130118155824.534B61C10B0@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r60180:521cbc238824 Date: 2013-01-18 16:58 +0100 http://bitbucket.org/pypy/pypy/changeset/521cbc238824/ Log: These two flags are not referenced from C. diff --git a/pypy/rpython/memory/gc/stmgc.py b/pypy/rpython/memory/gc/stmgc.py --- a/pypy/rpython/memory/gc/stmgc.py +++ b/pypy/rpython/memory/gc/stmgc.py @@ -115,8 +115,8 @@ GCFLAG_NOT_WRITTEN = first_gcflag << 2 # keep in sync with et.h GCFLAG_LOCAL_COPY = first_gcflag << 3 # keep in sync with et.h GCFLAG_VISITED = first_gcflag << 4 # keep in sync with et.h -GCFLAG_HASH_FIELD = first_gcflag << 5 # keep in sync with et.h -GCFLAG_NEW_HASH = first_gcflag << 6 # keep in sync with et.h +GCFLAG_HASH_FIELD = first_gcflag << 5 +GCFLAG_NEW_HASH = first_gcflag << 6 GCFLAG_PREBUILT = GCFLAG_GLOBAL | GCFLAG_NOT_WRITTEN REV_INITIAL = r_uint(1) From noreply at buildbot.pypy.org Fri Jan 18 18:10:50 2013 From: noreply at buildbot.pypy.org (antocuni) Date: Fri, 18 Jan 2013 18:10:50 +0100 (CET) Subject: [pypy-commit] pypy default: add a writable __pypy_data__ attribute to ndarray: it will be useful to implement the numpy C API in pure python Message-ID: <20130118171050.C33271C0DC1@cobra.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: Changeset: r60181:f62b43e0d1b5 Date: 2013-01-18 17:04 +0000 http://bitbucket.org/pypy/pypy/changeset/f62b43e0d1b5/ Log: add a writable __pypy_data__ attribute to ndarray: it will be useful to implement the numpy C API in pure python diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -397,6 +397,16 @@ space.w_False])) return w_d + w_pypy_data = None + def fget___pypy_data__(self, space): + return self.w_pypy_data + + def fset___pypy_data__(self, space, w_data): + self.w_pypy_data = w_data + + def fdel___pypy_data__(self, space): + self.w_pypy_data = None + # --------------------- operations ---------------------------- def _unaryop_impl(ufunc_name): @@ -675,7 +685,10 @@ W_NDimArray.descr_set_imag), __array_interface__ = GetSetProperty(W_NDimArray.descr_array_iface), _from_shape_and_storage = interp2app(descr__from_shape_and_storage, - as_classmethod=True) + as_classmethod=True), + __pypy_data__ = GetSetProperty(W_NDimArray.fget___pypy_data__, + W_NDimArray.fset___pypy_data__, + W_NDimArray.fdel___pypy_data__), ) @unwrap_spec(ndmin=int, copy=bool, subok=bool) diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -2385,3 +2385,14 @@ assert y[0, 1] == 2 y[0, 1] = 42 assert x[1] == 42 + + def test___pypy_data__(self): + from _numpypy import array + x = array([1, 2, 3, 4]) + x.__pypy_data__ is None + obj = object() + x.__pypy_data__ = obj + assert x.__pypy_data__ is obj + del x.__pypy_data__ + assert x.__pypy_data__ is None + From noreply at buildbot.pypy.org Fri Jan 18 18:10:51 2013 From: noreply at buildbot.pypy.org (antocuni) Date: Fri, 18 Jan 2013 18:10:51 +0100 (CET) Subject: [pypy-commit] pypy default: numpy arrays should be weakreferenceable Message-ID: <20130118171051.E65D41C0DC1@cobra.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: Changeset: r60182:85218b7bd814 Date: 2013-01-18 17:10 +0000 http://bitbucket.org/pypy/pypy/changeset/85218b7bd814/ Log: numpy arrays should be weakreferenceable diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -1,6 +1,6 @@ from pypy.interpreter.error import operationerrfmt, OperationError -from pypy.interpreter.typedef import TypeDef, GetSetProperty +from pypy.interpreter.typedef import TypeDef, GetSetProperty, make_weakref_descr from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault from pypy.module.micronumpy.base import W_NDimArray, convert_to_array,\ ArrayArgumentException, issequence_w @@ -684,8 +684,9 @@ imag = GetSetProperty(W_NDimArray.descr_get_imag, W_NDimArray.descr_set_imag), __array_interface__ = GetSetProperty(W_NDimArray.descr_array_iface), - _from_shape_and_storage = interp2app(descr__from_shape_and_storage, - as_classmethod=True), + __weakref__ = make_weakref_descr(W_NDimArray), + _from_shape_and_storage = interp2app(descr__from_shape_and_storage, + as_classmethod=True), __pypy_data__ = GetSetProperty(W_NDimArray.fget___pypy_data__, W_NDimArray.fset___pypy_data__, W_NDimArray.fdel___pypy_data__), diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1615,6 +1615,14 @@ assert (b == [20, 1, 21, 3, 4]).all() raises(ValueError, "array([1, 2])[array([True, False, True])] = [1, 2, 3]") + def test_weakref(self): + import _weakref + from numpypy import array + a = array([1, 2, 3]) + assert _weakref.ref(a) + a = array(42) + assert _weakref.ref(a) + class AppTestMultiDim(BaseNumpyAppTest): def test_init(self): import _numpypy From noreply at buildbot.pypy.org Fri Jan 18 19:55:37 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Fri, 18 Jan 2013 19:55:37 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: hg merge default Message-ID: <20130118185537.13F4A1C12DE@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60183:777d30901063 Date: 2013-01-18 19:55 +0100 http://bitbucket.org/pypy/pypy/changeset/777d30901063/ Log: hg merge default diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -1,6 +1,6 @@ from pypy.interpreter.error import operationerrfmt, OperationError -from pypy.interpreter.typedef import TypeDef, GetSetProperty +from pypy.interpreter.typedef import TypeDef, GetSetProperty, make_weakref_descr from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault from pypy.module.micronumpy.base import W_NDimArray, convert_to_array,\ ArrayArgumentException, issequence_w @@ -397,6 +397,16 @@ space.w_False])) return w_d + w_pypy_data = None + def fget___pypy_data__(self, space): + return self.w_pypy_data + + def fset___pypy_data__(self, space, w_data): + self.w_pypy_data = w_data + + def fdel___pypy_data__(self, space): + self.w_pypy_data = None + # --------------------- operations ---------------------------- def _unaryop_impl(ufunc_name): @@ -674,8 +684,12 @@ imag = GetSetProperty(W_NDimArray.descr_get_imag, W_NDimArray.descr_set_imag), __array_interface__ = GetSetProperty(W_NDimArray.descr_array_iface), - _from_shape_and_storage = interp2app(descr__from_shape_and_storage, - as_classmethod=True) + __weakref__ = make_weakref_descr(W_NDimArray), + _from_shape_and_storage = interp2app(descr__from_shape_and_storage, + as_classmethod=True), + __pypy_data__ = GetSetProperty(W_NDimArray.fget___pypy_data__, + W_NDimArray.fset___pypy_data__, + W_NDimArray.fdel___pypy_data__), ) @unwrap_spec(ndmin=int, copy=bool, subok=bool) diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1615,6 +1615,14 @@ assert (b == [20, 1, 21, 3, 4]).all() raises(ValueError, "array([1, 2])[array([True, False, True])] = [1, 2, 3]") + def test_weakref(self): + import _weakref + from numpypy import array + a = array([1, 2, 3]) + assert _weakref.ref(a) + a = array(42) + assert _weakref.ref(a) + class AppTestMultiDim(BaseNumpyAppTest): def test_init(self): import _numpypy @@ -2385,3 +2393,14 @@ assert y[0, 1] == 2 y[0, 1] = 42 assert x[1] == 42 + + def test___pypy_data__(self): + from _numpypy import array + x = array([1, 2, 3, 4]) + x.__pypy_data__ is None + obj = object() + x.__pypy_data__ = obj + assert x.__pypy_data__ is obj + del x.__pypy_data__ + assert x.__pypy_data__ is None + From noreply at buildbot.pypy.org Fri Jan 18 21:31:31 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Fri, 18 Jan 2013 21:31:31 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Added conftestpath back to test_pytestsupport Message-ID: <20130118203131.DF29F1C12DB@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60184:6cf02861c5c4 Date: 2013-01-18 21:31 +0100 http://bitbucket.org/pypy/pypy/changeset/6cf02861c5c4/ Log: Added conftestpath back to test_pytestsupport diff --git a/pypy/tool/pytest/test/test_pytestsupport.py b/pypy/tool/pytest/test/test_pytestsupport.py --- a/pypy/tool/pytest/test/test_pytestsupport.py +++ b/pypy/tool/pytest/test/test_pytestsupport.py @@ -9,6 +9,7 @@ import os import sys import pypy +conftestpath = py.path.local(pypy.__file__).dirpath("conftest.py") pytest_plugins = "pytest_pytester" From noreply at buildbot.pypy.org Fri Jan 18 22:32:13 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Fri, 18 Jan 2013 22:32:13 +0100 (CET) Subject: [pypy-commit] pypy py3k: o add py3's inherent comparisons Message-ID: <20130118213213.6D7141C1138@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60185:950bba699a87 Date: 2013-01-18 13:28 -0800 http://bitbucket.org/pypy/pypy/changeset/950bba699a87/ Log: o add py3's inherent comparisons o avoid binding comparison descriptors when possible, as you can't bind descriptors to None itself! but it's also an optimization diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py --- a/pypy/objspace/descroperation.py +++ b/pypy/objspace/descroperation.py @@ -550,17 +550,26 @@ def _invoke_comparison(space, w_descr, w_obj1, w_obj2): if w_descr is not None: - try: - w_impl = space.get(w_descr, w_obj1) - except OperationError as e: - # see testForExceptionsRaisedInInstanceGetattr2 in - # test_class - if not e.match(space, space.w_AttributeError): - raise + # a special case for performance (see get_and_call_function) but + # also avoids binding via __get__ when unnecessary; in + # particular when w_obj1 is None, __get__(None, type(None)) + # won't actually bind =] + descr = space.interpclass_w(w_descr) + typ = type(descr) + if typ is Function or typ is FunctionWithFixedCode: + w_res = descr.funccall(w_obj1, w_obj2) else: - w_res = space.call_function(w_impl, w_obj2) - if _check_notimplemented(space, w_res): - return w_res + try: + w_impl = space.get(w_descr, w_obj1) + except OperationError as e: + # see testForExceptionsRaisedInInstanceGetattr2 in + # test_class + if not e.match(space, space.w_AttributeError): + raise + else: + w_res = space.call_function(w_impl, w_obj2) + if _check_notimplemented(space, w_res): + return w_res return None def _make_comparison_impl(symbol, specialnames): diff --git a/pypy/objspace/std/objecttype.py b/pypy/objspace/std/objecttype.py --- a/pypy/objspace/std/objecttype.py +++ b/pypy/objspace/std/objecttype.py @@ -205,6 +205,19 @@ reduce_1 = app.interphook('reduce_1') reduce_2 = app.interphook('reduce_2') +def descr__eq__(space, w_self, w_other): + if space.is_w(w_self, w_other): + return space.w_True + # Return NotImplemented instead of False, so if two objects are + # compared, both get a chance at the comparison (issue #1393) + return space.w_NotImplemented + +def descr__ne__(space, w_self, w_other): + return space.not_(space.eq(w_self, w_other)) + +def descr_richcompare(space, w_self, w_other): + return space.w_NotImplemented + # ____________________________________________________________ object_typedef = StdTypeDef("object", @@ -222,5 +235,11 @@ __format__ = gateway.interp2app(descr___format__), __subclasshook__ = gateway.interp2app(descr___subclasshook__, as_classmethod=True), + __eq__ = gateway.interp2app(descr__eq__), + __ne__ = gateway.interp2app(descr__ne__), + __le__ = gateway.interp2app(descr_richcompare), + __lt__ = gateway.interp2app(descr_richcompare), + __ge__ = gateway.interp2app(descr_richcompare), + __gt__ = gateway.interp2app(descr_richcompare), __init__ = gateway.interp2app(descr__init__), ) diff --git a/pypy/objspace/std/test/test_obj.py b/pypy/objspace/std/test/test_obj.py --- a/pypy/objspace/std/test/test_obj.py +++ b/pypy/objspace/std/test/test_obj.py @@ -248,6 +248,17 @@ y += 2 assert object.__hash__(x) == object.__hash__(y) + def test_richcompare(self): + o = object() + o2 = object() + assert o.__eq__(o) is True + assert o.__eq__(o2) is NotImplemented + assert o.__ne__(o) is False + assert o.__ne__(o2) is True + assert o.__le__(o2) is NotImplemented + assert o.__lt__(o2) is NotImplemented + assert o.__ge__(o2) is NotImplemented + assert o.__gt__(o2) is NotImplemented def test_isinstance_shortcut(): from pypy.objspace.std import objspace From noreply at buildbot.pypy.org Fri Jan 18 22:32:14 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Fri, 18 Jan 2013 22:32:14 +0100 (CET) Subject: [pypy-commit] pypy py3k: raw_unicode_escape_decode accepts unicode Message-ID: <20130118213214.C22921C1138@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60186:4739822aa049 Date: 2013-01-18 13:29 -0800 http://bitbucket.org/pypy/pypy/changeset/4739822aa049/ Log: raw_unicode_escape_decode accepts unicode diff --git a/pypy/module/_codecs/interp_codecs.py b/pypy/module/_codecs/interp_codecs.py --- a/pypy/module/_codecs/interp_codecs.py +++ b/pypy/module/_codecs/interp_codecs.py @@ -513,7 +513,6 @@ "utf_32_decode", "utf_32_be_decode", "utf_32_le_decode", - "raw_unicode_escape_decode", ]: make_decoder_wrapper(decoders) @@ -767,6 +766,21 @@ return space.newtuple([space.wrap(result), space.wrap(consumed)]) # ____________________________________________________________ +# Raw Unicode escape (accepts bytes or str) + + at unwrap_spec(string='bufferstr_or_u', errors='str_or_None', + w_final=WrappedDefault(False)) +def raw_unicode_escape_decode(space, string, errors="strict", w_final=None): + if errors is None: + errors = 'strict' + final = space.is_true(w_final) + state = space.fromcache(CodecState) + result, consumed = runicode.str_decode_raw_unicode_escape( + string, len(string), errors, + final, state.decode_error_handler) + return space.newtuple([space.wrap(result), space.wrap(consumed)]) + +# ____________________________________________________________ # Unicode-internal @unwrap_spec(errors='str_or_None') diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py --- a/pypy/module/_codecs/test/test_codecs.py +++ b/pypy/module/_codecs/test/test_codecs.py @@ -268,8 +268,10 @@ assert bytes.decode("unicode_internal") == "a" def test_raw_unicode_escape(self): + import _codecs assert str(b"\u0663", "raw-unicode-escape") == "\u0663" assert "\u0663".encode("raw-unicode-escape") == b"\u0663" + assert _codecs.raw_unicode_escape_decode(r"\u1234") == ("\u1234", 6) def test_escape_decode(self): import _codecs From noreply at buildbot.pypy.org Fri Jan 18 22:32:16 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Fri, 18 Jan 2013 22:32:16 +0100 (CET) Subject: [pypy-commit] pypy py3k: sys.long_info -> int_info Message-ID: <20130118213216.1F2631C1138@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60187:baaf834ac228 Date: 2013-01-18 13:29 -0800 http://bitbucket.org/pypy/pypy/changeset/baaf834ac228/ Log: sys.long_info -> int_info diff --git a/lib-python/3.2/test/test_sys.py b/lib-python/3.2/test/test_sys.py --- a/lib-python/3.2/test/test_sys.py +++ b/lib-python/3.2/test/test_sys.py @@ -425,9 +425,9 @@ self.assertEqual(sys.float_info.radix, 2) self.assertEqual(len(sys.int_info), 2) if test.support.check_impl_detail(cpython=True): - self.assertTrue(sys.long_info.bits_per_digit % 5 == 0) + self.assertTrue(sys.int_info.bits_per_digit % 5 == 0) else: - self.assertTrue(sys.long_info.bits_per_digit >= 1) + self.assertTrue(sys.int_info.bits_per_digit >= 1) self.assertTrue(sys.int_info.sizeof_digit >= 1) self.assertEqual(type(sys.int_info.bits_per_digit), int) self.assertEqual(type(sys.int_info.sizeof_digit), int) From noreply at buildbot.pypy.org Fri Jan 18 22:32:17 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Fri, 18 Jan 2013 22:32:17 +0100 (CET) Subject: [pypy-commit] pypy py3k: .pyo are an impl detail Message-ID: <20130118213217.597CB1C1138@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60188:7b97d4152ebc Date: 2013-01-18 13:29 -0800 http://bitbucket.org/pypy/pypy/changeset/7b97d4152ebc/ Log: .pyo are an impl detail diff --git a/lib-python/3.2/test/test_compileall.py b/lib-python/3.2/test/test_compileall.py --- a/lib-python/3.2/test/test_compileall.py +++ b/lib-python/3.2/test/test_compileall.py @@ -174,10 +174,11 @@ # Ensure that the default behavior of compileall's CLI is to create # PEP 3147 pyc/pyo files. + _pyo = 'pyo' if support.check_impl_detail() else 'pyc' for name, ext, switch in [ ('normal', 'pyc', []), - ('optimize', 'pyo', ['-O']), - ('doubleoptimize', 'pyo', ['-OO']), + ('optimize', _pyo, ['-O']), + ('doubleoptimize', _pyo, ['-OO']), ]: def f(self, ext=ext, switch=switch): script_helper.assert_python_ok(*(switch + From noreply at buildbot.pypy.org Fri Jan 18 22:32:18 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Fri, 18 Jan 2013 22:32:18 +0100 (CET) Subject: [pypy-commit] pypy py3k: PyPy ignores the optimize flag Message-ID: <20130118213218.929A61C1138@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60189:6fa9f0dc9942 Date: 2013-01-18 13:30 -0800 http://bitbucket.org/pypy/pypy/changeset/6fa9f0dc9942/ Log: PyPy ignores the optimize flag diff --git a/lib-python/3.2/test/test_imp.py b/lib-python/3.2/test/test_imp.py --- a/lib-python/3.2/test/test_imp.py +++ b/lib-python/3.2/test/test_imp.py @@ -216,6 +216,7 @@ imp.cache_from_source('/foo/bar/baz/qux.py', True), '/foo/bar/baz/__pycache__/qux.{}.pyc'.format(self.tag)) + @support.impl_detail("PyPy ignores the optimize flag", pypy=False) def test_cache_from_source_optimized(self): # Given the path to a .py file, return the path to its PEP 3147 # defined .pyo file (i.e. under __pycache__). @@ -228,6 +229,7 @@ os.sep.join(('__pycache__', 'foo.{}.pyc'.format(self.tag)))) + @support.impl_detail("PyPy ignores the optimize flag", pypy=False) def test_cache_from_source_override(self): # When debug_override is not None, it can be any true-ish or false-ish # value. From noreply at buildbot.pypy.org Sat Jan 19 09:47:13 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 19 Jan 2013 09:47:13 +0100 (CET) Subject: [pypy-commit] pypy default: remove unused import Message-ID: <20130119084713.AD5851C01D2@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r60190:0a7ee9d89700 Date: 2013-01-19 10:46 +0200 http://bitbucket.org/pypy/pypy/changeset/0a7ee9d89700/ Log: remove unused import diff --git a/pypy/rlib/rsre/test/test_zjit.py b/pypy/rlib/rsre/test/test_zjit.py --- a/pypy/rlib/rsre/test/test_zjit.py +++ b/pypy/rlib/rsre/test/test_zjit.py @@ -1,6 +1,5 @@ import py from pypy.jit.metainterp.test import support -from pypy.rlib.nonconst import NonConstant from pypy.rlib.rsre.test.test_match import get_code from pypy.rlib.rsre import rsre_core from pypy.rpython.lltypesystem import lltype From noreply at buildbot.pypy.org Sat Jan 19 09:49:49 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 19 Jan 2013 09:49:49 +0100 (CET) Subject: [pypy-commit] pypy imrpove-custom-gc-tracing: an API I want from a custom tracer (that can be run directly too) Message-ID: <20130119084949.37DAA1C01D2@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: imrpove-custom-gc-tracing Changeset: r60191:5bffe8e57968 Date: 2013-01-18 16:17 +0200 http://bitbucket.org/pypy/pypy/changeset/5bffe8e57968/ Log: an API I want from a custom tracer (that can be run directly too) diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py --- a/pypy/rpython/lltypesystem/lltype.py +++ b/pypy/rpython/lltypesystem/lltype.py @@ -361,7 +361,7 @@ Struct._install_extras(self, **kwds) def _attach_runtime_type_info_funcptr(self, funcptr, destrptr, - customtraceptr): + custom_trace_func): if self._runtime_type_info is None: raise TypeError("attachRuntimeTypeInfo: %r must have been built " "with the rtti=True argument" % (self,)) @@ -385,18 +385,8 @@ raise TypeError("expected a destructor function " "implementation, got: %s" % destrptr) self._runtime_type_info.destructor_funcptr = destrptr - if customtraceptr is not None: - from pypy.rpython.lltypesystem import llmemory - T = typeOf(customtraceptr) - if (not isinstance(T, Ptr) or - not isinstance(T.TO, FuncType) or - len(T.TO.ARGS) != 2 or - T.TO.RESULT != llmemory.Address or - T.TO.ARGS[0] != llmemory.Address or - T.TO.ARGS[1] != llmemory.Address): - raise TypeError("expected a custom trace function " - "implementation, got: %s" % customtraceptr) - self._runtime_type_info.custom_trace_funcptr = customtraceptr + if custom_trace_func is not None: + self._runtime_type_info.custom_trace_func = custom_trace_func class GcStruct(RttiStruct): _gckind = 'gc' @@ -2050,11 +2040,11 @@ return _ptr(PTRTYPE, oddint, solid=True) def attachRuntimeTypeInfo(GCSTRUCT, funcptr=None, destrptr=None, - customtraceptr=None): + custom_trace_func=None): if not isinstance(GCSTRUCT, RttiStruct): raise TypeError, "expected a RttiStruct: %s" % GCSTRUCT GCSTRUCT._attach_runtime_type_info_funcptr(funcptr, destrptr, - customtraceptr) + custom_trace_func) return _ptr(Ptr(RuntimeTypeInfo), GCSTRUCT._runtime_type_info) def getRuntimeTypeInfo(GCSTRUCT): diff --git a/pypy/rpython/memory/gc/base.py b/pypy/rpython/memory/gc/base.py --- a/pypy/rpython/memory/gc/base.py +++ b/pypy/rpython/memory/gc/base.py @@ -73,7 +73,6 @@ member_index, is_rpython_class, has_custom_trace, - get_custom_trace, fast_path_tracing): self.getfinalizer = getfinalizer self.getlightfinalizer = getlightfinalizer @@ -90,7 +89,6 @@ self.member_index = member_index self.is_rpython_class = is_rpython_class self.has_custom_trace = has_custom_trace - self.get_custom_trace = get_custom_trace self.fast_path_tracing = fast_path_tracing def get_member_index(self, type_id): @@ -211,6 +209,13 @@ i += 1 trace._annspecialcase_ = 'specialize:arg(2)' + def _call_custom_trace(self, obj, typeid, callback, arg): + def wrapper(item, arg): + if self.is_valid_gc_object(item): + callback(item, arg) + + self.custom_trace_funcs[typeid.index](obj, wrapper, arg) + def _trace_slow_path(self, obj, callback, arg): typeid = self.get_type_id(obj) if self.has_gcptr_in_varsize(typeid): @@ -228,14 +233,7 @@ item += itemlength length -= 1 if self.has_custom_trace(typeid): - generator = self.get_custom_trace(typeid) - item = llmemory.NULL - while True: - item = generator(obj, item) - if not item: - break - if self.points_to_valid_gc_object(item): - callback(item, arg) + self._call_custom_trace(obj, typeid, callback, arg) _trace_slow_path._annspecialcase_ = 'specialize:arg(2)' def trace_partial(self, obj, start, stop, callback, arg): diff --git a/pypy/rpython/memory/gctypelayout.py b/pypy/rpython/memory/gctypelayout.py --- a/pypy/rpython/memory/gctypelayout.py +++ b/pypy/rpython/memory/gctypelayout.py @@ -127,12 +127,6 @@ infobits = self.get(typeid).infobits return infobits & T_HAS_CUSTOM_TRACE != 0 - def q_get_custom_trace(self, typeid): - ll_assert(self.q_has_custom_trace(typeid), - "T_HAS_CUSTOM_TRACE missing") - typeinfo = self.get(typeid) - return typeinfo.finalizer_or_customtrace - def q_fast_path_tracing(self, typeid): # return True if none of the flags T_HAS_GCPTR_IN_VARSIZE, # T_IS_GCARRAY_OF_GCPTR or T_HAS_CUSTOM_TRACE is set @@ -159,7 +153,6 @@ self.q_member_index, self.q_is_rpython_class, self.q_has_custom_trace, - self.q_get_custom_trace, self.q_fast_path_tracing) @@ -204,10 +197,12 @@ infobits |= T_HAS_FINALIZER elif kind == 'light_finalizer': infobits |= T_HAS_FINALIZER | T_HAS_LIGHTWEIGHT_FINALIZER - elif kind == "custom_trace": - infobits |= T_HAS_CUSTOM_TRACE else: assert 0, kind + custom_trace_func = builder.get_custom_trace_func(TYPE) + if custom_trace_func is not None: + infobits |= T_HAS_CUSTOM_TRACE + builder.record_custom_trace(index, custom_trace_func) # if not TYPE._is_varsize(): info.fixedsize = llarena.round_up_for_allocation( @@ -378,16 +373,11 @@ if TYPE in self._special_funcptrs: return self._special_funcptrs[TYPE] fptr1, is_lightweight = self.make_finalizer_funcptr_for_type(TYPE) - fptr2 = self.make_custom_trace_funcptr_for_type(TYPE) - assert not (fptr1 and fptr2), ( - "type %r needs both a finalizer and a custom tracer" % (TYPE,)) if fptr1: if is_lightweight: kind_and_fptr = "light_finalizer", fptr1 else: kind_and_fptr = "finalizer", fptr1 - elif fptr2: - kind_and_fptr = "custom_trace", fptr2 else: kind_and_fptr = None self._special_funcptrs[TYPE] = kind_and_fptr diff --git a/pypy/rpython/memory/gcwrapper.py b/pypy/rpython/memory/gcwrapper.py --- a/pypy/rpython/memory/gcwrapper.py +++ b/pypy/rpython/memory/gcwrapper.py @@ -25,9 +25,12 @@ def prepare_graphs(self, flowgraphs): lltype2vtable = self.llinterp.typer.lltype2vtable + # only valid for direct layout builder + self.gc.custom_trace_funcs = {} layoutbuilder = DirectRunLayoutBuilder(self.gc.__class__, lltype2vtable, - self.llinterp) + self.llinterp, + self.gc.custom_trace_funcs) self.get_type_id = layoutbuilder.get_type_id layoutbuilder.initialize_gc_query_function(self.gc) @@ -200,9 +203,10 @@ class DirectRunLayoutBuilder(gctypelayout.TypeLayoutBuilder): - def __init__(self, GCClass, lltype2vtable, llinterp): + def __init__(self, GCClass, lltype2vtable, llinterp, custom_trace_funcs): self.llinterp = llinterp super(DirectRunLayoutBuilder, self).__init__(GCClass, lltype2vtable) + self.custom_trace_funcs = custom_trace_funcs def make_finalizer_funcptr_for_type(self, TYPE): from pypy.rpython.memory.gctransform.support import get_rtti @@ -227,11 +231,14 @@ return llmemory.NULL return llhelper(gctypelayout.GCData.FINALIZER_OR_CT, ll_finalizer), light - def make_custom_trace_funcptr_for_type(self, TYPE): + def record_custom_trace(self, tid, custom_trace_func): + self.custom_trace_funcs[tid] = custom_trace_func + + def get_custom_trace_func(self, TYPE): from pypy.rpython.memory.gctransform.support import get_rtti rtti = get_rtti(TYPE) - if rtti is not None and hasattr(rtti._obj, 'custom_trace_funcptr'): - return rtti._obj.custom_trace_funcptr + if rtti is not None and hasattr(rtti._obj, 'custom_trace_func'): + return rtti._obj.custom_trace_func else: return None diff --git a/pypy/rpython/memory/test/test_gc.py b/pypy/rpython/memory/test/test_gc.py --- a/pypy/rpython/memory/test/test_gc.py +++ b/pypy/rpython/memory/test/test_gc.py @@ -7,10 +7,11 @@ from pypy.rpython.lltypesystem import lltype from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.objectmodel import we_are_translated -from pypy.rlib.objectmodel import compute_unique_id +from pypy.rlib.objectmodel import compute_unique_id, specialize from pypy.rlib import rgc from pypy.rlib.rstring import StringBuilder from pypy.rlib.rarithmetic import LONG_BIT +from pypy.jit.backend.llsupport import jitframe WORD = LONG_BIT // 8 @@ -237,7 +238,6 @@ assert 160 <= res <= 165 def test_custom_trace(self): - from pypy.rpython.annlowlevel import llhelper from pypy.rpython.lltypesystem import llmemory from pypy.rpython.lltypesystem.llarena import ArenaError # @@ -245,15 +245,10 @@ ('y', llmemory.Address), rtti=True) T = lltype.GcStruct('T', ('z', lltype.Signed)) offset_of_x = llmemory.offsetof(S, 'x') - def customtrace(obj, prev): - if not prev: - return obj + offset_of_x - else: - return llmemory.NULL - CUSTOMTRACEFUNC = lltype.FuncType([llmemory.Address, llmemory.Address], - llmemory.Address) - customtraceptr = llhelper(lltype.Ptr(CUSTOMTRACEFUNC), customtrace) - lltype.attachRuntimeTypeInfo(S, customtraceptr=customtraceptr) + def customtrace(obj, callback, arg): + callback(obj + offset_of_x, arg) + + lltype.attachRuntimeTypeInfo(S, custom_trace_func=customtrace) # for attrname in ['x', 'y']: def setup(): From noreply at buildbot.pypy.org Sat Jan 19 10:32:37 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 19 Jan 2013 10:32:37 +0100 (CET) Subject: [pypy-commit] pypy imrpove-custom-gc-tracing: work in progress not sure where to go from here Message-ID: <20130119093237.C21B21C02D9@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: imrpove-custom-gc-tracing Changeset: r60192:b103c0c9fec5 Date: 2013-01-19 11:32 +0200 http://bitbucket.org/pypy/pypy/changeset/b103c0c9fec5/ Log: work in progress not sure where to go from here diff --git a/pypy/rpython/lltypesystem/lloperation.py b/pypy/rpython/lltypesystem/lloperation.py --- a/pypy/rpython/lltypesystem/lloperation.py +++ b/pypy/rpython/lltypesystem/lloperation.py @@ -500,6 +500,7 @@ 'gc_gcflag_extra' : LLOp(), 'gc_add_memory_pressure': LLOp(), 'gc_set_extra_threshold': LLOp(canrun=True, canmallocgc=True), + 'gc_call_custom_trace': LLOp(canrun=True), # cannot malloc # ------- JIT & GC interaction, only for some GCs ---------- diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py --- a/pypy/rpython/lltypesystem/opimpl.py +++ b/pypy/rpython/lltypesystem/opimpl.py @@ -647,6 +647,9 @@ def op_gc_set_extra_threshold(threshold): pass +def op_gc_call_custom_trace(gc, *args): + gc._call_custom_trace(*args) + def op_shrink_array(array, smallersize): return False diff --git a/pypy/rpython/memory/gc/base.py b/pypy/rpython/memory/gc/base.py --- a/pypy/rpython/memory/gc/base.py +++ b/pypy/rpython/memory/gc/base.py @@ -6,6 +6,7 @@ from pypy.rpython.memory.support import get_address_stack, get_address_deque from pypy.rpython.memory.support import AddressDict, null_address_dict from pypy.rpython.lltypesystem.llmemory import NULL, raw_malloc_usage +from pypy.rlib.objectmodel import specialize TYPEID_MAP = lltype.GcStruct('TYPEID_MAP', ('count', lltype.Signed), ('size', lltype.Signed), @@ -210,11 +211,13 @@ trace._annspecialcase_ = 'specialize:arg(2)' def _call_custom_trace(self, obj, typeid, callback, arg): - def wrapper(item, arg): - if self.is_valid_gc_object(item): - callback(item, arg) - - self.custom_trace_funcs[typeid.index](obj, wrapper, arg) + #def wrapper(item, arg): + # if self.is_valid_gc_object(item): + # callback(item, arg) + + for obj_typeid, func in self.custom_trace_funcs: + if typeid == obj_typeid: + func(obj, callback, arg) def _trace_slow_path(self, obj, callback, arg): typeid = self.get_type_id(obj) @@ -233,7 +236,10 @@ item += itemlength length -= 1 if self.has_custom_trace(typeid): - self._call_custom_trace(obj, typeid, callback, arg) + # an obscure hack to flow those only when we + # actually have all typeids + llop.gc_call_custom_trace(lltype.Void, self, obj, typeid, callback, + arg) _trace_slow_path._annspecialcase_ = 'specialize:arg(2)' def trace_partial(self, obj, start, stop, callback, arg): diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py --- a/pypy/rpython/memory/gctransform/framework.py +++ b/pypy/rpython/memory/gctransform/framework.py @@ -888,6 +888,11 @@ source_addr, dest_addr] + op.args[2:], resultvar=op.result) + def gct_gc_call_custom_trace(self, hop): + args = hop.spaceop.args + args[3].concretetype + hop.genop('direct_call', []) + def gct_weakref_create(self, hop): op = hop.spaceop @@ -1216,6 +1221,7 @@ else: lltype2vtable = None self.translator = translator + self.custom_trace_funcs = [] super(TransformerLayoutBuilder, self).__init__(GCClass, lltype2vtable) def has_finalizer(self, TYPE): @@ -1229,7 +1235,7 @@ def has_custom_trace(self, TYPE): rtti = get_rtti(TYPE) - return rtti is not None and getattr(rtti._obj, 'custom_trace_funcptr', + return rtti is not None and getattr(rtti._obj, 'custom_trace_func', None) def make_finalizer_funcptr_for_type(self, TYPE): @@ -1249,16 +1255,14 @@ light = not FinalizerAnalyzer(self.translator).analyze_light_finalizer(g) return fptr, light - def make_custom_trace_funcptr_for_type(self, TYPE): + def get_custom_trace_func(self, TYPE): if not self.has_custom_trace(TYPE): return None rtti = get_rtti(TYPE) - fptr = rtti._obj.custom_trace_funcptr - if not hasattr(fptr._obj, 'graph'): - ll_func = fptr._obj._callable - fptr = self.transformer.annotate_finalizer(ll_func, - [llmemory.Address, llmemory.Address], llmemory.Address) - return fptr + return rtti._obj.custom_trace_func + + def record_custom_trace(self, typeid, custom_trace_func): + self.custom_trace_funcs.append((typeid, custom_trace_func)) def gen_zero_gc_pointers(TYPE, v, llops, previous_steps=None): diff --git a/pypy/rpython/memory/gctransform/transform.py b/pypy/rpython/memory/gctransform/transform.py --- a/pypy/rpython/memory/gctransform/transform.py +++ b/pypy/rpython/memory/gctransform/transform.py @@ -103,6 +103,7 @@ self.minimalgctransformer = self.MinimalGCTransformer(self) else: self.minimalgctransformer = None + self.call_custom_trace_cache = {} def get_lltype_of_exception_value(self): exceptiondata = self.translator.rtyper.getexceptiondata() diff --git a/pypy/rpython/memory/gctypelayout.py b/pypy/rpython/memory/gctypelayout.py --- a/pypy/rpython/memory/gctypelayout.py +++ b/pypy/rpython/memory/gctypelayout.py @@ -183,7 +183,7 @@ "invalid type_id") -def encode_type_shape(builder, info, TYPE, index): +def encode_type_shape(builder, info, TYPE, index, type_id): """Encode the shape of the TYPE into the TYPE_INFO structure 'info'.""" offsets = offsets_to_gc_pointers(TYPE) infobits = index @@ -202,7 +202,7 @@ custom_trace_func = builder.get_custom_trace_func(TYPE) if custom_trace_func is not None: infobits |= T_HAS_CUSTOM_TRACE - builder.record_custom_trace(index, custom_trace_func) + builder.record_custom_trace(type_id, custom_trace_func) # if not TYPE._is_varsize(): info.fixedsize = llarena.round_up_for_allocation( @@ -296,9 +296,10 @@ info = fullinfo.header type_id = self.type_info_group.add_member(fullinfo) if self.can_encode_type_shape: - encode_type_shape(self, info, TYPE, type_id.index) + encode_type_shape(self, info, TYPE, type_id.index, type_id) else: - self._pending_type_shapes.append((info, TYPE, type_id.index)) + self._pending_type_shapes.append((info, TYPE, type_id.index, + type_id)) # store it self.id_of_type[TYPE] = type_id self.add_vtable_after_typeinfo(TYPE) @@ -341,8 +342,8 @@ def encode_type_shapes_now(self): if not self.can_encode_type_shape: self.can_encode_type_shape = True - for info, TYPE, index in self._pending_type_shapes: - encode_type_shape(self, info, TYPE, index) + for info, TYPE, index, type_id in self._pending_type_shapes: + encode_type_shape(self, info, TYPE, index, type_id) del self._pending_type_shapes def delay_encoding(self): diff --git a/pypy/rpython/memory/gcwrapper.py b/pypy/rpython/memory/gcwrapper.py --- a/pypy/rpython/memory/gcwrapper.py +++ b/pypy/rpython/memory/gcwrapper.py @@ -26,7 +26,7 @@ def prepare_graphs(self, flowgraphs): lltype2vtable = self.llinterp.typer.lltype2vtable # only valid for direct layout builder - self.gc.custom_trace_funcs = {} + self.gc.custom_trace_funcs = [] layoutbuilder = DirectRunLayoutBuilder(self.gc.__class__, lltype2vtable, self.llinterp, @@ -232,7 +232,7 @@ return llhelper(gctypelayout.GCData.FINALIZER_OR_CT, ll_finalizer), light def record_custom_trace(self, tid, custom_trace_func): - self.custom_trace_funcs[tid] = custom_trace_func + self.custom_trace_funcs.append((tid, custom_trace_func)) def get_custom_trace_func(self, TYPE): from pypy.rpython.memory.gctransform.support import get_rtti diff --git a/pypy/rpython/memory/test/test_transformed_gc.py b/pypy/rpython/memory/test/test_transformed_gc.py --- a/pypy/rpython/memory/test/test_transformed_gc.py +++ b/pypy/rpython/memory/test/test_transformed_gc.py @@ -382,21 +382,13 @@ assert 160 <= res <= 165 def define_custom_trace(cls): - from pypy.rpython.annlowlevel import llhelper - from pypy.rpython.lltypesystem import llmemory - # + S = lltype.GcStruct('S', ('x', llmemory.Address), rtti=True) T = lltype.GcStruct('T', ('z', lltype.Signed)) offset_of_x = llmemory.offsetof(S, 'x') - def customtrace(obj, prev): - if not prev: - return obj + offset_of_x - else: - return llmemory.NULL - CUSTOMTRACEFUNC = lltype.FuncType([llmemory.Address, llmemory.Address], - llmemory.Address) - customtraceptr = llhelper(lltype.Ptr(CUSTOMTRACEFUNC), customtrace) - lltype.attachRuntimeTypeInfo(S, customtraceptr=customtraceptr) + def customtrace(obj, callback, arg): + callback(obj + offset_of_x, arg) + lltype.attachRuntimeTypeInfo(S, custom_trace_func=customtrace) # def setup(): s1 = lltype.malloc(S) From noreply at buildbot.pypy.org Sat Jan 19 11:46:46 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 19 Jan 2013 11:46:46 +0100 (CET) Subject: [pypy-commit] pypy imrpove-custom-gc-tracing: a bit of shuffling before giving up Message-ID: <20130119104646.393431C0DC1@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: imrpove-custom-gc-tracing Changeset: r60193:7d6bb887766f Date: 2013-01-19 11:46 +0200 http://bitbucket.org/pypy/pypy/changeset/7d6bb887766f/ Log: a bit of shuffling before giving up diff --git a/pypy/rpython/lltypesystem/opimpl.py b/pypy/rpython/lltypesystem/opimpl.py --- a/pypy/rpython/lltypesystem/opimpl.py +++ b/pypy/rpython/lltypesystem/opimpl.py @@ -648,7 +648,7 @@ pass def op_gc_call_custom_trace(gc, *args): - gc._call_custom_trace(*args) + gc._call_custom_trace_body(*args) def op_shrink_array(array, smallersize): return False diff --git a/pypy/rpython/memory/gc/base.py b/pypy/rpython/memory/gc/base.py --- a/pypy/rpython/memory/gc/base.py +++ b/pypy/rpython/memory/gc/base.py @@ -210,7 +210,7 @@ i += 1 trace._annspecialcase_ = 'specialize:arg(2)' - def _call_custom_trace(self, obj, typeid, callback, arg): + def _call_custom_trace_body(self, obj, typeid, callback, arg): #def wrapper(item, arg): # if self.is_valid_gc_object(item): # callback(item, arg) @@ -219,6 +219,12 @@ if typeid == obj_typeid: func(obj, callback, arg) + + @specialize.arg(3) + def _call_custom_trace(self, obj, typeid, callback, arg): + llop.gc_call_custom_trace(lltype.Void, self, obj, typeid, callback, + arg) + def _trace_slow_path(self, obj, callback, arg): typeid = self.get_type_id(obj) if self.has_gcptr_in_varsize(typeid): @@ -238,8 +244,7 @@ if self.has_custom_trace(typeid): # an obscure hack to flow those only when we # actually have all typeids - llop.gc_call_custom_trace(lltype.Void, self, obj, typeid, callback, - arg) + self._call_custom_trace(obj, typeid, callback, arg) _trace_slow_path._annspecialcase_ = 'specialize:arg(2)' def trace_partial(self, obj, start, stop, callback, arg): diff --git a/pypy/rpython/memory/gctransform/framework.py b/pypy/rpython/memory/gctransform/framework.py --- a/pypy/rpython/memory/gctransform/framework.py +++ b/pypy/rpython/memory/gctransform/framework.py @@ -889,6 +889,8 @@ resultvar=op.result) def gct_gc_call_custom_trace(self, hop): + import pdb + pdb.set_trace() args = hop.spaceop.args args[3].concretetype hop.genop('direct_call', []) From noreply at buildbot.pypy.org Sat Jan 19 11:46:47 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 19 Jan 2013 11:46:47 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: Backed out changeset c68a9032570c. Too much of a mess Message-ID: <20130119104647.627DE1C0DC1@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60194:1fff0bac1857 Date: 2013-01-19 11:47 +0200 http://bitbucket.org/pypy/pypy/changeset/1fff0bac1857/ Log: Backed out changeset c68a9032570c. Too much of a mess diff --git a/pypy/rpython/lltypesystem/lltype.py b/pypy/rpython/lltypesystem/lltype.py --- a/pypy/rpython/lltypesystem/lltype.py +++ b/pypy/rpython/lltypesystem/lltype.py @@ -361,7 +361,7 @@ Struct._install_extras(self, **kwds) def _attach_runtime_type_info_funcptr(self, funcptr, destrptr, - custom_trace_func): + customtraceptr): if self._runtime_type_info is None: raise TypeError("attachRuntimeTypeInfo: %r must have been built " "with the rtti=True argument" % (self,)) @@ -385,8 +385,18 @@ raise TypeError("expected a destructor function " "implementation, got: %s" % destrptr) self._runtime_type_info.destructor_funcptr = destrptr - if custom_trace_func is not None: - self._runtime_type_info.custom_trace_func = custom_trace_func + if customtraceptr is not None: + from pypy.rpython.lltypesystem import llmemory + T = typeOf(customtraceptr) + if (not isinstance(T, Ptr) or + not isinstance(T.TO, FuncType) or + len(T.TO.ARGS) != 2 or + T.TO.RESULT != llmemory.Address or + T.TO.ARGS[0] != llmemory.Address or + T.TO.ARGS[1] != llmemory.Address): + raise TypeError("expected a custom trace function " + "implementation, got: %s" % customtraceptr) + self._runtime_type_info.custom_trace_funcptr = customtraceptr class GcStruct(RttiStruct): _gckind = 'gc' @@ -2040,11 +2050,11 @@ return _ptr(PTRTYPE, oddint, solid=True) def attachRuntimeTypeInfo(GCSTRUCT, funcptr=None, destrptr=None, - custom_trace_func=None): + customtraceptr=None): if not isinstance(GCSTRUCT, RttiStruct): raise TypeError, "expected a RttiStruct: %s" % GCSTRUCT GCSTRUCT._attach_runtime_type_info_funcptr(funcptr, destrptr, - custom_trace_func) + customtraceptr) return _ptr(Ptr(RuntimeTypeInfo), GCSTRUCT._runtime_type_info) def getRuntimeTypeInfo(GCSTRUCT): diff --git a/pypy/rpython/memory/gc/base.py b/pypy/rpython/memory/gc/base.py --- a/pypy/rpython/memory/gc/base.py +++ b/pypy/rpython/memory/gc/base.py @@ -73,6 +73,7 @@ member_index, is_rpython_class, has_custom_trace, + get_custom_trace, fast_path_tracing): self.getfinalizer = getfinalizer self.getlightfinalizer = getlightfinalizer @@ -89,6 +90,7 @@ self.member_index = member_index self.is_rpython_class = is_rpython_class self.has_custom_trace = has_custom_trace + self.get_custom_trace = get_custom_trace self.fast_path_tracing = fast_path_tracing def get_member_index(self, type_id): @@ -209,13 +211,6 @@ i += 1 trace._annspecialcase_ = 'specialize:arg(2)' - def _call_custom_trace(self, obj, typeid, callback, arg): - def wrapper(item, arg): - if self.is_valid_gc_object(item): - callback(item, arg) - - self.custom_trace_funcs[typeid.index](obj, wrapper, arg) - def _trace_slow_path(self, obj, callback, arg): typeid = self.get_type_id(obj) if self.has_gcptr_in_varsize(typeid): @@ -233,7 +228,14 @@ item += itemlength length -= 1 if self.has_custom_trace(typeid): - self._call_custom_trace(obj, typeid, callback, arg) + generator = self.get_custom_trace(typeid) + item = llmemory.NULL + while True: + item = generator(obj, item) + if not item: + break + if self.points_to_valid_gc_object(item): + callback(item, arg) _trace_slow_path._annspecialcase_ = 'specialize:arg(2)' def trace_partial(self, obj, start, stop, callback, arg): diff --git a/pypy/rpython/memory/gctypelayout.py b/pypy/rpython/memory/gctypelayout.py --- a/pypy/rpython/memory/gctypelayout.py +++ b/pypy/rpython/memory/gctypelayout.py @@ -127,6 +127,12 @@ infobits = self.get(typeid).infobits return infobits & T_HAS_CUSTOM_TRACE != 0 + def q_get_custom_trace(self, typeid): + ll_assert(self.q_has_custom_trace(typeid), + "T_HAS_CUSTOM_TRACE missing") + typeinfo = self.get(typeid) + return typeinfo.finalizer_or_customtrace + def q_fast_path_tracing(self, typeid): # return True if none of the flags T_HAS_GCPTR_IN_VARSIZE, # T_IS_GCARRAY_OF_GCPTR or T_HAS_CUSTOM_TRACE is set @@ -153,6 +159,7 @@ self.q_member_index, self.q_is_rpython_class, self.q_has_custom_trace, + self.q_get_custom_trace, self.q_fast_path_tracing) @@ -197,12 +204,10 @@ infobits |= T_HAS_FINALIZER elif kind == 'light_finalizer': infobits |= T_HAS_FINALIZER | T_HAS_LIGHTWEIGHT_FINALIZER + elif kind == "custom_trace": + infobits |= T_HAS_CUSTOM_TRACE else: assert 0, kind - custom_trace_func = builder.get_custom_trace_func(TYPE) - if custom_trace_func is not None: - infobits |= T_HAS_CUSTOM_TRACE - builder.record_custom_trace(index, custom_trace_func) # if not TYPE._is_varsize(): info.fixedsize = llarena.round_up_for_allocation( @@ -373,11 +378,16 @@ if TYPE in self._special_funcptrs: return self._special_funcptrs[TYPE] fptr1, is_lightweight = self.make_finalizer_funcptr_for_type(TYPE) + fptr2 = self.make_custom_trace_funcptr_for_type(TYPE) + assert not (fptr1 and fptr2), ( + "type %r needs both a finalizer and a custom tracer" % (TYPE,)) if fptr1: if is_lightweight: kind_and_fptr = "light_finalizer", fptr1 else: kind_and_fptr = "finalizer", fptr1 + elif fptr2: + kind_and_fptr = "custom_trace", fptr2 else: kind_and_fptr = None self._special_funcptrs[TYPE] = kind_and_fptr diff --git a/pypy/rpython/memory/gcwrapper.py b/pypy/rpython/memory/gcwrapper.py --- a/pypy/rpython/memory/gcwrapper.py +++ b/pypy/rpython/memory/gcwrapper.py @@ -25,12 +25,9 @@ def prepare_graphs(self, flowgraphs): lltype2vtable = self.llinterp.typer.lltype2vtable - # only valid for direct layout builder - self.gc.custom_trace_funcs = {} layoutbuilder = DirectRunLayoutBuilder(self.gc.__class__, lltype2vtable, - self.llinterp, - self.gc.custom_trace_funcs) + self.llinterp) self.get_type_id = layoutbuilder.get_type_id layoutbuilder.initialize_gc_query_function(self.gc) @@ -203,10 +200,9 @@ class DirectRunLayoutBuilder(gctypelayout.TypeLayoutBuilder): - def __init__(self, GCClass, lltype2vtable, llinterp, custom_trace_funcs): + def __init__(self, GCClass, lltype2vtable, llinterp): self.llinterp = llinterp super(DirectRunLayoutBuilder, self).__init__(GCClass, lltype2vtable) - self.custom_trace_funcs = custom_trace_funcs def make_finalizer_funcptr_for_type(self, TYPE): from pypy.rpython.memory.gctransform.support import get_rtti @@ -231,14 +227,11 @@ return llmemory.NULL return llhelper(gctypelayout.GCData.FINALIZER_OR_CT, ll_finalizer), light - def record_custom_trace(self, tid, custom_trace_func): - self.custom_trace_funcs[tid] = custom_trace_func - - def get_custom_trace_func(self, TYPE): + def make_custom_trace_funcptr_for_type(self, TYPE): from pypy.rpython.memory.gctransform.support import get_rtti rtti = get_rtti(TYPE) - if rtti is not None and hasattr(rtti._obj, 'custom_trace_func'): - return rtti._obj.custom_trace_func + if rtti is not None and hasattr(rtti._obj, 'custom_trace_funcptr'): + return rtti._obj.custom_trace_funcptr else: return None diff --git a/pypy/rpython/memory/test/test_gc.py b/pypy/rpython/memory/test/test_gc.py --- a/pypy/rpython/memory/test/test_gc.py +++ b/pypy/rpython/memory/test/test_gc.py @@ -7,11 +7,10 @@ from pypy.rpython.lltypesystem import lltype from pypy.rpython.lltypesystem.lloperation import llop from pypy.rlib.objectmodel import we_are_translated -from pypy.rlib.objectmodel import compute_unique_id, specialize +from pypy.rlib.objectmodel import compute_unique_id from pypy.rlib import rgc from pypy.rlib.rstring import StringBuilder from pypy.rlib.rarithmetic import LONG_BIT -from pypy.jit.backend.llsupport import jitframe WORD = LONG_BIT // 8 @@ -238,6 +237,7 @@ assert 160 <= res <= 165 def test_custom_trace(self): + from pypy.rpython.annlowlevel import llhelper from pypy.rpython.lltypesystem import llmemory from pypy.rpython.lltypesystem.llarena import ArenaError # @@ -245,10 +245,15 @@ ('y', llmemory.Address), rtti=True) T = lltype.GcStruct('T', ('z', lltype.Signed)) offset_of_x = llmemory.offsetof(S, 'x') - def customtrace(obj, callback, arg): - callback(obj + offset_of_x, arg) - - lltype.attachRuntimeTypeInfo(S, custom_trace_func=customtrace) + def customtrace(obj, prev): + if not prev: + return obj + offset_of_x + else: + return llmemory.NULL + CUSTOMTRACEFUNC = lltype.FuncType([llmemory.Address, llmemory.Address], + llmemory.Address) + customtraceptr = llhelper(lltype.Ptr(CUSTOMTRACEFUNC), customtrace) + lltype.attachRuntimeTypeInfo(S, customtraceptr=customtraceptr) # for attrname in ['x', 'y']: def setup(): From noreply at buildbot.pypy.org Sat Jan 19 11:46:48 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 19 Jan 2013 11:46:48 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: add TODO Message-ID: <20130119104648.901371C0DC1@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60195:5ee42fd8c3c3 Date: 2013-01-19 11:57 +0200 http://bitbucket.org/pypy/pypy/changeset/5ee42fd8c3c3/ Log: add TODO diff --git a/pypy/TODO b/pypy/TODO new file mode 100644 --- /dev/null +++ b/pypy/TODO @@ -0,0 +1,5 @@ + +* 32bit x86 +* ARM +* shadowstack + asmgcc +* kill jit2gc on translator \ No newline at end of file From noreply at buildbot.pypy.org Sat Jan 19 11:46:49 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 19 Jan 2013 11:46:49 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: implement a custom tracer abomination Message-ID: <20130119104649.DDA4F1C0DC1@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60196:d6f5e379ab27 Date: 2013-01-19 12:46 +0200 http://bitbucket.org/pypy/pypy/changeset/d6f5e379ab27/ Log: implement a custom tracer abomination diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py --- a/pypy/jit/backend/llsupport/gc.py +++ b/pypy/jit/backend/llsupport/gc.py @@ -143,7 +143,6 @@ kind = 'boehm' moving_gc = False round_up = False - gcrootmap = None write_barrier_descr = None fielddescr_tid = None str_type_id = 0 @@ -300,8 +299,9 @@ else: assert self.translate_support_code,"required with the framework GC" self._check_valid_gc() - self._make_gcrootmap() self._make_layoutbuilder() + self.gcrootfindername = self.gcdescr.config.translation.gcrootfinder + assert self.gcrootfindername in ['asmgcc', 'shadowstack'] self._setup_gcclass() self._setup_tid() self._setup_write_barrier() @@ -322,17 +322,6 @@ raise NotImplementedError("--gc=%s not implemented with the JIT" % (self.gcdescr.config.translation.gc,)) - def _make_gcrootmap(self): - # to find roots in the assembler, make a GcRootMap - name = self.gcdescr.config.translation.gcrootfinder - try: - cls = globals()['GcRootMap_' + name] - except KeyError: - raise NotImplementedError("--gcrootfinder=%s not implemented" - " with the JIT" % (name,)) - gcrootmap = cls(self.gcdescr) - self.gcrootmap = gcrootmap - def _make_layoutbuilder(self): # make a TransformerLayoutBuilder and save it on the translator # where it can be fished and reused by the FrameworkGCTransformer @@ -340,8 +329,8 @@ translator = self.translator self.layoutbuilder = framework.TransformerLayoutBuilder(translator) self.layoutbuilder.delay_encoding() + # XXX this can probably die horrible death translator._jit2gc = {'layoutbuilder': self.layoutbuilder} - self.gcrootmap.add_jit2gc_hooks(translator._jit2gc) def _setup_gcclass(self): from pypy.rpython.memory.gcheader import GCHeaderBuilder @@ -492,7 +481,7 @@ return rffi.cast(lltype.Signed, nurs_top_addr) def initialize(self): - self.gcrootmap.initialize() + pass def init_size_descr(self, S, descr): if self.layoutbuilder is not None: diff --git a/pypy/jit/backend/llsupport/jitframe.py b/pypy/jit/backend/llsupport/jitframe.py --- a/pypy/jit/backend/llsupport/jitframe.py +++ b/pypy/jit/backend/llsupport/jitframe.py @@ -1,4 +1,8 @@ from pypy.rpython.lltypesystem import lltype, llmemory +from pypy.rpython.annlowlevel import llhelper +from pypy.rlib.objectmodel import specialize + +STATICSIZE = 0 # patch from the assembler backend # this is an info that only depends on the assembler executed, copied from # compiled loop token (in fact we could use this as a compiled loop token @@ -15,7 +19,6 @@ # gcindexlist is a list of indexes of GC ptrs # in the actual array jf_frame of JITFRAME ('jfi_gcmap', lltype.Ptr(GCMAP)), - ('counter', lltype.Signed), ) NULLFRAMEINFO = lltype.nullptr(JITFRAMEINFO) @@ -58,6 +61,8 @@ # RPython code that finishes the function with an exception, the # exception is not stored there, but is simply kept as a variable there) ('jf_guard_exc', llmemory.GCREF), + # absolutely useless field used to make up for tracing hooks inflexibilities + ('jf_gc_trace_state', lltype.Signed), # the actual frame ('jf_frame', lltype.Array(lltype.Signed)), # note that we keep length field, because it's crucial to have the data @@ -65,7 +70,66 @@ adtmeths = { 'allocate': jitframe_allocate, 'copy': jitframe_copy, - } + }, + rtti = True, ) + at specialize.memo() +def getofs(name): + return llmemory.offsetof(JITFRAME, name) + +GCMAPLENGTHOFS = llmemory.arraylengthoffset(GCMAP) +GCMAPBASEOFS = llmemory.itemoffsetof(GCMAP, 0) +BASEITEMOFS = llmemory.itemoffsetof(JITFRAME.jf_frame, 0) +SIGN_SIZE = llmemory.sizeof(lltype.Signed) + +def jitframe_trace(obj_addr, prev): + if prev == llmemory.NULL: + (obj_addr + getofs('jf_gc_trace_state')).signed[0] = 0 + return obj_addr + getofs('jf_frame_info') + fld = (obj_addr + getofs('jf_gc_trace_state')).signed[0] + state = fld & 0xff + no = fld >> 8 + if state == 0: + (obj_addr + getofs('jf_gc_trace_state')).signed[0] = 1 + return obj_addr + getofs('jf_descr') + elif state == 1: + (obj_addr + getofs('jf_gc_trace_state')).signed[0] = 2 + return obj_addr + getofs('jf_force_descr') + elif state == 2: + (obj_addr + getofs('jf_gc_trace_state')).signed[0] = 3 + return obj_addr + getofs('jf_gcmap') + elif state == 3: + (obj_addr + getofs('jf_gc_trace_state')).signed[0] = 4 + return obj_addr + getofs('jf_savedata') + elif state == 4: + (obj_addr + getofs('jf_gc_trace_state')).signed[0] = 5 + return obj_addr + getofs('jf_guard_exc') + elif state == 5: + # bit pattern + gcpat = (obj_addr + getofs('jf_gcpattern')).signed[0] + while no < STATICSIZE and gcpat & (1 << no) == 0: + no += 1 + if no != STATICSIZE: + newstate = 5 | ((no + 1) << 8) + (obj_addr + getofs('jf_gc_trace_state')).signed[0] = newstate + return obj_addr + getofs('jf_frame') + BASEITEMOFS + SIGN_SIZE * no + state = 6 + no = 0 + assert state == 6 + gcmap = (obj_addr + getofs('jf_gcmap')).address[0] + gcmaplen = (gcmap + GCMAPLENGTHOFS).signed[0] + if no >= gcmaplen: + return llmemory.NULL + index = (gcmap + GCMAPBASEOFS + SIGN_SIZE * no).signed[0] + STATICSIZE + newstate = 6 | ((no + 1) << 8) + (obj_addr + getofs('jf_gc_trace_state')).signed[0] = newstate + return obj_addr + getofs('jf_frame') + BASEITEMOFS + SIGN_SIZE * index + +CUSTOMTRACEFUNC = lltype.FuncType([llmemory.Address, llmemory.Address], + llmemory.Address) +jitframe_trace_ptr = llhelper(lltype.Ptr(CUSTOMTRACEFUNC), jitframe_trace) + +lltype.attachRuntimeTypeInfo(JITFRAME, customtraceptr=jitframe_trace_ptr) + JITFRAMEPTR = lltype.Ptr(JITFRAME) diff --git a/pypy/jit/backend/llsupport/test/test_gc.py b/pypy/jit/backend/llsupport/test/test_gc.py --- a/pypy/jit/backend/llsupport/test/test_gc.py +++ b/pypy/jit/backend/llsupport/test/test_gc.py @@ -1,20 +1,16 @@ -import random -from pypy.rpython.lltypesystem import lltype, llmemory, rffi, rstr +from pypy.rpython.lltypesystem import lltype, llmemory, rstr from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.annlowlevel import llhelper -from pypy.jit.backend.llsupport.descr import * -from pypy.jit.backend.llsupport.gc import * +from pypy.jit.backend.llsupport import jitframe, gc, descr from pypy.jit.backend.llsupport import symbolic from pypy.jit.metainterp.gc import get_description from pypy.jit.metainterp.history import BoxPtr, BoxInt, ConstPtr -from pypy.jit.metainterp.resoperation import get_deep_immutable_oplist -from pypy.jit.tool.oparser import parse -from pypy.rpython.lltypesystem.rclass import OBJECT, OBJECT_VTABLE -from pypy.jit.metainterp.optimizeopt.util import equaloplists +from pypy.jit.metainterp.resoperation import get_deep_immutable_oplist, rop,\ + ResOperation from pypy.rlib.rarithmetic import is_valid_int def test_boehm(): - gc_ll_descr = GcLLDescr_boehm(None, None, None) + gc_ll_descr = gc.GcLLDescr_boehm(None, None, None) # record = [] prev_malloc_fn_ptr = gc_ll_descr.malloc_fn_ptr @@ -26,13 +22,13 @@ # # ---------- gc_malloc ---------- S = lltype.GcStruct('S', ('x', lltype.Signed)) - sizedescr = get_size_descr(gc_ll_descr, S) + sizedescr = descr.get_size_descr(gc_ll_descr, S) p = gc_ll_descr.gc_malloc(sizedescr) assert record == [(sizedescr.size, p)] del record[:] # ---------- gc_malloc_array ---------- A = lltype.GcArray(lltype.Signed) - arraydescr = get_array_descr(gc_ll_descr, A) + arraydescr = descr.get_array_descr(gc_ll_descr, A) p = gc_ll_descr.gc_malloc_array(10, arraydescr) assert record == [(arraydescr.basesize + 10 * arraydescr.itemsize, p)] @@ -52,199 +48,6 @@ # ____________________________________________________________ -class TestGcRootMapAsmGcc: - - def test_make_shapes(self): - def frame_pos(n): - return -4*(4+n) - gcrootmap = GcRootMap_asmgcc() - gcrootmap.is_64_bit = False - num1 = frame_pos(-5) - num1a = num1|2 - num2 = frame_pos(55) - num2a = ((-num2|3) >> 7) | 128 - num2b = (-num2|3) & 127 - shape = gcrootmap.get_basic_shape() - gcrootmap.add_frame_offset(shape, num1) - gcrootmap.add_frame_offset(shape, num2) - assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a]) - gcrootmap.add_callee_save_reg(shape, 1) - assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, - 4]) - gcrootmap.add_callee_save_reg(shape, 2) - assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, - 4, 8]) - gcrootmap.add_callee_save_reg(shape, 3) - assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, - 4, 8, 12]) - gcrootmap.add_callee_save_reg(shape, 4) - assert shape == map(chr, [6, 7, 11, 15, 2, 0, num1a, num2b, num2a, - 4, 8, 12, 16]) - - def test_compress_callshape(self): - class FakeDataBlockWrapper: - def malloc_aligned(self, size, alignment): - assert alignment == 1 # here - assert size == 4 - return rffi.cast(lltype.Signed, p) - datablockwrapper = FakeDataBlockWrapper() - p = lltype.malloc(rffi.CArray(lltype.Char), 4, immortal=True) - gcrootmap = GcRootMap_asmgcc() - shape = ['a', 'b', 'c', 'd'] - gcrootmap.compress_callshape(shape, datablockwrapper) - assert p[0] == 'd' - assert p[1] == 'c' - assert p[2] == 'b' - assert p[3] == 'a' - - def test_put_basic(self): - gcrootmap = GcRootMap_asmgcc() - retaddr = 1234567890 - shapeaddr = 51627384 - gcrootmap.put(retaddr, shapeaddr) - assert gcrootmap._gcmap[0] == retaddr - assert gcrootmap._gcmap[1] == shapeaddr - p = rffi.cast(rffi.SIGNEDP, gcrootmap.gcmapstart()) - assert p[0] == retaddr - assert (gcrootmap.gcmapend() == - gcrootmap.gcmapstart() + rffi.sizeof(lltype.Signed) * 2) - - def test_put_resize(self): - # the same as before, but enough times to trigger a few resizes - gcrootmap = GcRootMap_asmgcc() - for i in range(700): - shapeaddr = i * 100 + 1 - retaddr = 123456789 + i - gcrootmap.put(retaddr, shapeaddr) - for i in range(700): - assert gcrootmap._gcmap[i*2+0] == 123456789 + i - assert gcrootmap._gcmap[i*2+1] == i * 100 + 1 - - def test_remove_nulls(self): - expected = [] - def check(): - assert gcrootmap._gcmap_curlength == len(expected) * 2 - for i, (a, b) in enumerate(expected): - assert gcrootmap._gcmap[i*2] == a - assert gcrootmap._gcmap[i*2+1] == b - # - gcrootmap = GcRootMap_asmgcc() - for i in range(700): - shapeaddr = i * 100 # 0 if i == 0 - retaddr = 123456789 + i - gcrootmap.put(retaddr, shapeaddr) - if shapeaddr != 0: - expected.append((retaddr, shapeaddr)) - # at the first resize, the 0 should be removed - check() - for repeat in range(10): - # now clear up half the entries - assert len(expected) == 699 - for i in range(0, len(expected), 2): - gcrootmap._gcmap[i*2+1] = 0 - gcrootmap._gcmap_deadentries += 1 - expected = expected[1::2] - assert gcrootmap._gcmap_deadentries*6 > gcrootmap._gcmap_maxlength - # check that we can again insert 350 entries without a resize - oldgcmap = gcrootmap._gcmap - for i in range(0, 699, 2): - gcrootmap.put(515151 + i + repeat, 626262 + i) - expected.append((515151 + i + repeat, 626262 + i)) - assert gcrootmap._gcmap == oldgcmap - check() - - def test_freeing_block(self): - from pypy.jit.backend.llsupport import gc - class Asmgcroot: - arrayitemsize = 2 * llmemory.sizeof(llmemory.Address) - sort_count = 0 - def sort_gcmap(self, gcmapstart, gcmapend): - self.sort_count += 1 - def binary_search(self, gcmapstart, gcmapend, startaddr): - i = 0 - while (i < gcrootmap._gcmap_curlength//2 and - gcrootmap._gcmap[i*2] < startaddr): - i += 1 - if i > 0: - i -= 1 - assert 0 <= i < gcrootmap._gcmap_curlength//2 - p = rffi.cast(rffi.CArrayPtr(llmemory.Address), gcmapstart) - p = rffi.ptradd(p, 2*i) - return llmemory.cast_ptr_to_adr(p) - saved = gc.asmgcroot - try: - gc.asmgcroot = Asmgcroot() - # - gcrootmap = GcRootMap_asmgcc() - gcrootmap._gcmap = lltype.malloc(gcrootmap.GCMAP_ARRAY, - 1400, flavor='raw', - immortal=True) - for i in range(700): - gcrootmap._gcmap[i*2] = 1200000 + i - gcrootmap._gcmap[i*2+1] = i * 100 + 1 - assert gcrootmap._gcmap_deadentries == 0 - assert gc.asmgcroot.sort_count == 0 - gcrootmap._gcmap_maxlength = 1400 - gcrootmap._gcmap_curlength = 1400 - gcrootmap._gcmap_sorted = False - # - gcrootmap.freeing_block(1200000 - 100, 1200000) - assert gcrootmap._gcmap_deadentries == 0 - assert gc.asmgcroot.sort_count == 1 - # - gcrootmap.freeing_block(1200000 + 100, 1200000 + 200) - assert gcrootmap._gcmap_deadentries == 100 - assert gc.asmgcroot.sort_count == 1 - for i in range(700): - if 100 <= i < 200: - expected = 0 - else: - expected = i * 100 + 1 - assert gcrootmap._gcmap[i*2] == 1200000 + i - assert gcrootmap._gcmap[i*2+1] == expected - # - gcrootmap.freeing_block(1200000 + 650, 1200000 + 750) - assert gcrootmap._gcmap_deadentries == 150 - assert gc.asmgcroot.sort_count == 1 - for i in range(700): - if 100 <= i < 200 or 650 <= i: - expected = 0 - else: - expected = i * 100 + 1 - assert gcrootmap._gcmap[i*2] == 1200000 + i - assert gcrootmap._gcmap[i*2+1] == expected - # - finally: - gc.asmgcroot = saved - - -class TestGcRootMapShadowStack: - class FakeGcDescr: - force_index_ofs = 92 - - def test_make_shapes(self): - gcrootmap = GcRootMap_shadowstack(self.FakeGcDescr()) - shape = gcrootmap.get_basic_shape() - gcrootmap.add_frame_offset(shape, 16) - gcrootmap.add_frame_offset(shape, -24) - assert shape == [16, -24] - - def test_compress_callshape(self): - class FakeDataBlockWrapper: - def malloc_aligned(self, size, alignment): - assert alignment == 4 # even on 64-bits - assert size == 12 # 4*3, even on 64-bits - return rffi.cast(lltype.Signed, p) - datablockwrapper = FakeDataBlockWrapper() - p = lltype.malloc(rffi.CArray(rffi.INT), 3, immortal=True) - gcrootmap = GcRootMap_shadowstack(self.FakeGcDescr()) - shape = [16, -24] - gcrootmap.compress_callshape(shape, datablockwrapper) - assert rffi.cast(lltype.Signed, p[0]) == 16 - assert rffi.cast(lltype.Signed, p[1]) == -24 - assert rffi.cast(lltype.Signed, p[2]) == 0 - - class FakeLLOp(object): def __init__(self): self.record = [] @@ -324,10 +127,9 @@ return 43 gcdescr = get_description(config_) - translator = FakeTranslator() llop1 = FakeLLOp() - gc_ll_descr = GcLLDescr_framework(gcdescr, FakeTranslator(), None, - llop1) + gc_ll_descr = gc.GcLLDescr_framework(gcdescr, FakeTranslator(), None, + llop1) gc_ll_descr.initialize() llop1.gcheaderbuilder = gc_ll_descr.gcheaderbuilder self.llop1 = llop1 @@ -348,7 +150,7 @@ def test_gc_malloc(self): S = lltype.GcStruct('S', ('x', lltype.Signed)) - sizedescr = get_size_descr(self.gc_ll_descr, S) + sizedescr = descr.get_size_descr(self.gc_ll_descr, S) p = self.gc_ll_descr.gc_malloc(sizedescr) assert lltype.typeOf(p) == llmemory.GCREF assert self.llop1.record == [("fixedsize", repr(sizedescr.size), @@ -356,7 +158,7 @@ def test_gc_malloc_array(self): A = lltype.GcArray(lltype.Signed) - arraydescr = get_array_descr(self.gc_ll_descr, A) + arraydescr = descr.get_array_descr(self.gc_ll_descr, A) p = self.gc_ll_descr.gc_malloc_array(10, arraydescr) assert self.llop1.record == [("varsize", arraydescr.tid, 10, repr(arraydescr.basesize), @@ -408,7 +210,7 @@ gc_ll_descr = self.gc_ll_descr llop1 = self.llop1 # - rewriter = GcRewriterAssembler(gc_ll_descr, None) + rewriter = gc.GcRewriterAssembler(gc_ll_descr, None) newops = rewriter.newops v_base = BoxPtr() v_value = BoxPtr() @@ -462,3 +264,44 @@ class TestFrameworkMiniMark(TestFramework): gc = 'minimark' + +def test_custom_tracer(): + def indexof(no): + return (frame_adr + jitframe.getofs('jf_frame') + + jitframe.BASEITEMOFS + jitframe.SIGN_SIZE * no) + + PREV_STATICSIZE = jitframe.STATICSIZE + try: + jitframe.STATICSIZE = 3 + frame_info = lltype.malloc(jitframe.JITFRAMEINFO) + frame = lltype.malloc(jitframe.JITFRAME, 15) + frame.jf_frame_info = frame_info + frame.jf_gcmap = lltype.malloc(jitframe.GCMAP, 4) + frame.jf_gcmap[0] = 5 + frame.jf_gcmap[1] = 7 + frame.jf_gcmap[2] = 8 + frame.jf_gcmap[3] = 10 + frame.jf_gcpattern = 1 | 4 + frame_adr = llmemory.cast_ptr_to_adr(frame) + all_addrs = [] + next = jitframe.jitframe_trace(frame_adr, llmemory.NULL) + while next: + all_addrs.append(next) + next = jitframe.jitframe_trace(frame_adr, next) + counter = 0 + for name in jitframe.JITFRAME._names: + TP = getattr(jitframe.JITFRAME, name) + if isinstance(TP, lltype.Ptr): # only GC pointers + assert all_addrs[counter] == frame_adr + jitframe.getofs(name) + counter += 1 + # gcpattern + assert all_addrs[6] == indexof(0) + assert all_addrs[7] == indexof(2) + assert all_addrs[8] == indexof(3 + 5) + assert all_addrs[9] == indexof(3 + 7) + assert all_addrs[10] == indexof(3 + 8) + assert all_addrs[11] == indexof(3 + 10) + assert len(all_addrs) == 6 + 4 + 2 + # 6 static fields, 4 addresses from gcmap, 2 from gcpattern + finally: + jitframe.STATICSIZE = PREV_STATICSIZE diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -78,7 +78,6 @@ self.propagate_exception_path = 0 self.gcrootmap_retaddr_forced = 0 self.teardown() - self.counter = 0 def set_debug(self, v): r = self._debug @@ -479,8 +478,6 @@ clt = CompiledLoopToken(self.cpu, looptoken.number) clt.frame_info = lltype.malloc(jitframe.JITFRAMEINFO) - clt.frame_info.counter = self.counter - self.counter += 1 clt.allgcrefs = [] clt.frame_info.jfi_frame_depth = 0 # for now looptoken.compiled_loop_token = clt diff --git a/pypy/jit/backend/x86/runner.py b/pypy/jit/backend/x86/runner.py --- a/pypy/jit/backend/x86/runner.py +++ b/pypy/jit/backend/x86/runner.py @@ -16,6 +16,8 @@ from pypy.jit.backend.llsupport.symbolic import WORD from pypy.jit.backend.llsupport.descr import ArrayDescr, FLAG_POINTER,\ FLAG_FLOAT + +jitframe.STATICSIZE = JITFRAME_FIXED_SIZE import sys diff --git a/pypy/rpython/lltypesystem/llmemory.py b/pypy/rpython/lltypesystem/llmemory.py --- a/pypy/rpython/lltypesystem/llmemory.py +++ b/pypy/rpython/lltypesystem/llmemory.py @@ -5,7 +5,7 @@ # sizeof, offsetof import weakref -from pypy.rlib.objectmodel import Symbolic +from pypy.rlib.objectmodel import Symbolic, specialize from pypy.rpython.lltypesystem import lltype from pypy.tool.uid import uid from pypy.rlib.rarithmetic import is_valid_int @@ -369,14 +369,15 @@ # ____________________________________________________________ + at specialize.memo() def _sizeof_none(TYPE): assert not TYPE._is_varsize() return ItemOffset(TYPE) _sizeof_none._annspecialcase_ = 'specialize:memo' + at specialize.memo() def _internal_array_field(TYPE): return TYPE._arrayfld, TYPE._flds[TYPE._arrayfld] -_internal_array_field._annspecialcase_ = 'specialize:memo' def _sizeof_int(TYPE, n): if isinstance(TYPE, lltype.Struct): @@ -385,6 +386,7 @@ else: raise Exception("don't know how to take the size of a %r"%TYPE) + at specialize.arg(0) def sizeof(TYPE, n=None): if n is None: return _sizeof_none(TYPE) @@ -392,19 +394,23 @@ return itemoffsetof(TYPE) + _sizeof_none(TYPE.OF) * n else: return _sizeof_int(TYPE, n) -sizeof._annspecialcase_ = 'specialize:arg(0)' + at specialize.memo() def offsetof(TYPE, fldname): assert fldname in TYPE._flds return FieldOffset(TYPE, fldname) -offsetof._annspecialcase_ = 'specialize:memo' + at specialize.memo() def itemoffsetof(TYPE, n=0): result = ArrayItemsOffset(TYPE) if n != 0: result += ItemOffset(TYPE.OF) * n return result -itemoffsetof._annspecialcase_ = 'specialize:memo' + + at specialize.memo() +def arraylengthoffset(TYPE): + return ArrayLengthOffset(TYPE) + # ------------------------------------------------------------- class fakeaddress(object): From noreply at buildbot.pypy.org Sat Jan 19 11:51:25 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 19 Jan 2013 11:51:25 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: no asserts Message-ID: <20130119105125.7D5281C0DC1@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60197:ae26b2dd2e89 Date: 2013-01-19 12:50 +0200 http://bitbucket.org/pypy/pypy/changeset/ae26b2dd2e89/ Log: no asserts diff --git a/pypy/jit/backend/llsupport/jitframe.py b/pypy/jit/backend/llsupport/jitframe.py --- a/pypy/jit/backend/llsupport/jitframe.py +++ b/pypy/jit/backend/llsupport/jitframe.py @@ -1,6 +1,7 @@ from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.annlowlevel import llhelper from pypy.rlib.objectmodel import specialize +from pypy.rlib.debug import ll_assert STATICSIZE = 0 # patch from the assembler backend @@ -116,7 +117,7 @@ return obj_addr + getofs('jf_frame') + BASEITEMOFS + SIGN_SIZE * no state = 6 no = 0 - assert state == 6 + ll_assert(state == 6, "invalid tracer state") gcmap = (obj_addr + getofs('jf_gcmap')).address[0] gcmaplen = (gcmap + GCMAPLENGTHOFS).signed[0] if no >= gcmaplen: From noreply at buildbot.pypy.org Sat Jan 19 12:17:54 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 19 Jan 2013 12:17:54 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: cleanup imports Message-ID: <20130119111754.8DFE41C02D9@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60199:99202d717462 Date: 2013-01-19 13:06 +0200 http://bitbucket.org/pypy/pypy/changeset/99202d717462/ Log: cleanup imports diff --git a/pypy/jit/backend/x86/test/test_assembler.py b/pypy/jit/backend/x86/test/test_assembler.py --- a/pypy/jit/backend/x86/test/test_assembler.py +++ b/pypy/jit/backend/x86/test/test_assembler.py @@ -1,14 +1,8 @@ from pypy.jit.backend.x86.regloc import * -from pypy.jit.backend.x86.assembler import Assembler386 -from pypy.jit.backend.x86.regalloc import X86FrameManager, get_ebp_ofs -from pypy.jit.metainterp.history import BoxInt, BoxPtr, BoxFloat, ConstFloat -from pypy.jit.metainterp.history import INT, REF, FLOAT -from pypy.rlib.rarithmetic import intmask -from pypy.rpython.lltypesystem import lltype, llmemory, rffi -from pypy.jit.backend.x86.arch import WORD, IS_X86_32, IS_X86_64 +from pypy.jit.metainterp.history import ConstFloat +from pypy.jit.metainterp.history import INT, FLOAT +from pypy.rpython.lltypesystem import llmemory, rffi from pypy.jit.backend.detect_cpu import getcpuclass -from pypy.jit.backend.x86.regalloc import X86RegisterManager, X86_64_RegisterManager, X86XMMRegisterManager, X86_64_XMMRegisterManager -from pypy.jit.backend.llsupport import jitframe from pypy.jit.codewriter import longlong import ctypes import py From noreply at buildbot.pypy.org Sat Jan 19 12:17:53 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 19 Jan 2013 12:17:53 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: slow progress; Message-ID: <20130119111753.3EE6E1C01D2@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60198:9fa11e930749 Date: 2013-01-19 13:04 +0200 http://bitbucket.org/pypy/pypy/changeset/9fa11e930749/ Log: slow progress; diff --git a/pypy/jit/backend/llsupport/asmmemmgr.py b/pypy/jit/backend/llsupport/asmmemmgr.py --- a/pypy/jit/backend/llsupport/asmmemmgr.py +++ b/pypy/jit/backend/llsupport/asmmemmgr.py @@ -303,7 +303,7 @@ if self.gcroot_markers is not None: assert gcrootmap is not None for pos, mark in self.gcroot_markers: - gcrootmap.put(rawstart + pos, mark) + gcrootmap.register_asm_addr(rawstart + pos, mark) return rawstart def _become_a_plain_block_builder(self): diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py --- a/pypy/jit/backend/llsupport/gc.py +++ b/pypy/jit/backend/llsupport/gc.py @@ -70,13 +70,14 @@ return False def has_write_barrier_class(self): return None - def freeing_block(self, start, stop): - pass def get_nursery_free_addr(self): raise NotImplementedError def get_nursery_top_addr(self): raise NotImplementedError + def freeing_block(self, rawstart, rawstop): + pass + def gc_malloc(self, sizedescr): """Blackhole: do a 'bh_new'. Also used for 'bh_new_with_vtable', with the vtable pointer set manually afterwards.""" @@ -145,6 +146,7 @@ round_up = False write_barrier_descr = None fielddescr_tid = None + gcrootmap = None str_type_id = 0 unicode_type_id = 0 get_malloc_slowpath_addr = None @@ -224,10 +226,20 @@ arraydescr.itemsize, arraydescr.lendescr.offset) - # ____________________________________________________________ # All code below is for the hybrid or minimark GC +class GcRootMap_asmgcc(object): + is_shadow_stack = False + + def register_asm_addr(self, start, mark): + pass + +class GcRootMap_shadowstack(object): + is_shadow_stack = True + + def register_asm_addr(self, start, mark): + pass class WriteBarrierDescr(AbstractDescr): def __init__(self, gc_ll_descr): @@ -300,14 +312,24 @@ assert self.translate_support_code,"required with the framework GC" self._check_valid_gc() self._make_layoutbuilder() - self.gcrootfindername = self.gcdescr.config.translation.gcrootfinder - assert self.gcrootfindername in ['asmgcc', 'shadowstack'] + self._make_gcrootmap() self._setup_gcclass() self._setup_tid() self._setup_write_barrier() self._setup_str() self._make_functions(really_not_translated) + def _make_gcrootmap(self): + # to find roots in the assembler, make a GcRootMap + name = self.gcdescr.config.translation.gcrootfinder + try: + cls = globals()['GcRootMap_' + name] + except KeyError: + raise NotImplementedError("--gcrootfinder=%s not implemented" + " with the JIT" % (name,)) + gcrootmap = cls(self.gcdescr) + self.gcrootmap = gcrootmap + def _initialize_for_tests(self): self.layoutbuilder = None self.fielddescr_tid = AbstractDescr() @@ -481,7 +503,7 @@ return rffi.cast(lltype.Signed, nurs_top_addr) def initialize(self): - pass + self.gcrootmap.initialize() def init_size_descr(self, S, descr): if self.layoutbuilder is not None: @@ -518,9 +540,6 @@ def has_write_barrier_class(self): return WriteBarrierDescr - def freeing_block(self, start, stop): - self.gcrootmap.freeing_block(start, stop) - def get_malloc_slowpath_addr(self): return self.get_malloc_fn_addr('malloc_nursery') diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -815,7 +815,6 @@ if self.stack_check_slowpath == 0: pass # no stack check (e.g. not translated) else: - xxx endaddr, lengthaddr, _ = self.cpu.insert_stack_check() self.mc.MOV(eax, heap(endaddr)) # MOV eax, [start] self.mc.SUB(eax, esp) # SUB eax, current diff --git a/pypy/jit/backend/x86/test/test_runner.py b/pypy/jit/backend/x86/test/test_runner.py --- a/pypy/jit/backend/x86/test/test_runner.py +++ b/pypy/jit/backend/x86/test/test_runner.py @@ -13,10 +13,7 @@ from pypy.jit.metainterp.executor import execute from pypy.jit.backend.test.runner_test import LLtypeBackendTest from pypy.jit.tool.oparser import parse -from pypy.tool.udir import udir import ctypes -import sys -import os CPU = getcpuclass() @@ -33,9 +30,9 @@ # for the individual tests see # ====> ../../test/runner_test.py - add_loop_instructions = ['mov', 'movabs', 'mov', 'movabs', + add_loop_instructions = ['mov', 'mov', 'mov', 'mov', 'mov', 'add', 'test', 'je', 'jmp'] - bridge_loop_instructions = ['cmp', 'jge', 'mov', 'mov', 'call', + bridge_loop_instructions = ['cmp', 'jge', 'mov', 'call', 'mov', 'jmp'] def setup_method(self, meth): From noreply at buildbot.pypy.org Sat Jan 19 12:17:55 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 19 Jan 2013 12:17:55 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: shadowstack support, disable stackchecking for now Message-ID: <20130119111755.C21861C01D2@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60200:4bf8cd0f1d34 Date: 2013-01-19 13:17 +0200 http://bitbucket.org/pypy/pypy/changeset/4bf8cd0f1d34/ Log: shadowstack support, disable stackchecking for now diff --git a/pypy/jit/backend/llsupport/gc.py b/pypy/jit/backend/llsupport/gc.py --- a/pypy/jit/backend/llsupport/gc.py +++ b/pypy/jit/backend/llsupport/gc.py @@ -241,6 +241,10 @@ def register_asm_addr(self, start, mark): pass + def get_root_stack_top_addr(self): + rst_addr = llop.gc_adr_of_root_stack_top(llmemory.Address) + return rffi.cast(lltype.Signed, rst_addr) + class WriteBarrierDescr(AbstractDescr): def __init__(self, gc_ll_descr): self.llop1 = gc_ll_descr.llop1 diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -253,8 +253,8 @@ def _build_stack_check_slowpath(self): _, _, slowpathaddr = self.cpu.insert_stack_check() - if slowpathaddr == 0 or not self.cpu.propagate_exception_descr: - return # no stack check (for tests, or non-translated) + #if slowpathaddr == 0 or not self.cpu.propagate_exception_descr: + return # no stack check (for tests, or non-translated) xxx # # make a "function" that is called immediately at the start of @@ -852,23 +852,23 @@ self.mc.MOV_ri(r13.value, rst) # MOV r13, rootstacktop self.mc.MOV_rm(eax.value, (r13.value, 0)) # MOV eax, [r13] # - MARKER = gcrootmap.MARKER_FRAME - self.mc.LEA_rm(ebx.value, (eax.value, 2*WORD)) # LEA ebx, [eax+2*WORD] - self.mc.MOV_mi((eax.value, WORD), MARKER) # MOV [eax+WORD], MARKER - self.mc.MOV_mr((eax.value, 0), ebp.value) # MOV [eax], ebp - # + if IS_X86_64: + self.mc.MOV_mr((eax.value, 0), edi.value) # MOV [eax], edi + else: + xxx + self.mc.ADD_ri(eax.value, WORD) if rx86.fits_in_32bits(rst): - self.mc.MOV_jr(rst, ebx.value) # MOV [rootstacktop], ebx + self.mc.MOV_jr(rst, eax.value) # MOV [rootstacktop], eax else: - self.mc.MOV_mr((r13.value, 0), ebx.value) # MOV [r13], ebx + self.mc.MOV_mr((r13.value, 0), eax.value) # MOV [r13], eax def _call_footer_shadowstack(self, gcrootmap): rst = gcrootmap.get_root_stack_top_addr() if rx86.fits_in_32bits(rst): - self.mc.SUB_ji8(rst, 2*WORD) # SUB [rootstacktop], 2*WORD + self.mc.SUB_ji8(rst, WORD) # SUB [rootstacktop], WORD else: self.mc.MOV_ri(ebx.value, rst) # MOV ebx, rootstacktop - self.mc.SUB_mi8((ebx.value, 0), 2*WORD) # SUB [ebx], 2*WORD + self.mc.SUB_mi8((ebx.value, 0), WORD) # SUB [ebx], WORD def redirect_call_assembler(self, oldlooptoken, newlooptoken): # some minimal sanity checking From noreply at buildbot.pypy.org Sat Jan 19 14:25:39 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 19 Jan 2013 14:25:39 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fish virtualizable Message-ID: <20130119132539.495421C02D9@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60201:054ad90ac866 Date: 2013-01-19 15:25 +0200 http://bitbucket.org/pypy/pypy/changeset/054ad90ac866/ Log: fish virtualizable diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -2167,8 +2167,13 @@ # Write a call to the target assembler # we need to allocate the frame, keep in sync with runner's # execute_token + jd = descr.outermost_jitdriver_sd + if jd.index_of_virtualizable >= 0: + vloc = stack(jd.index_of_virtualizable, REF) + else: + vloc = imm(0) self._emit_call(imm(descr._x86_function_addr), - [argloc], 0, tmp=eax) + [argloc, vloc], 0, tmp=eax) if op.result is None: assert result_loc is None value = self.cpu.done_with_this_frame_descr_void @@ -2194,7 +2199,6 @@ je_location = self.mc.get_relative_pos() # # Path A: use assembler_helper_adr - jd = descr.outermost_jitdriver_sd assert jd is not None asm_helper_adr = self.cpu.cast_adr_to_int(jd.assembler_helper_adr) self._emit_call(imm(asm_helper_adr), @@ -2212,13 +2216,13 @@ # # Reset the vable token --- XXX really too much special logic here:-( if jd.index_of_virtualizable >= 0: - xxx from pypy.jit.backend.llsupport.descr import FieldDescr fielddescr = jd.vable_token_descr assert isinstance(fielddescr, FieldDescr) - ofs = fielddescr.offset - self.mc.MOV(edx, arglocs[1]) - self.mc.MOV_mi((edx.value, ofs), 0) + vtoken_ofs = fielddescr.offset + vable_ofs = (jd.index_of_virtualizable + JITFRAME_FIXED_SIZE) * WORD + self.mc.MOV_rb(edx.value, vable_ofs) + self.mc.MOV_mi((edx.value, vtoken_ofs), 0) # in the line above, TOKEN_NONE = 0 # if op.result is not None: @@ -2429,6 +2433,9 @@ def raw_stack(offset, type=INT): return RawStackLoc(offset, type) +def stack(index, type): + return StackLoc(index, get_ebp_ofs(index), type) + def heap(addr): return AddressLoc(ImmedLoc(addr), imm0, 0, 0) diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py --- a/pypy/jit/backend/x86/regalloc.py +++ b/pypy/jit/backend/x86/regalloc.py @@ -870,9 +870,6 @@ def consider_call_assembler(self, op, guard_op): descr = op.getdescr() assert isinstance(descr, JitCellToken) - jd = descr.outermost_jitdriver_sd - # we attached a frame as a first arg, move index of virtualizable by one - vable_index = jd.index_of_virtualizable self.rm._sync_var(op.getarg(0)) frame_loc = self.fm.loc(op.getarg(0)) if vable_index >= 0: diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py --- a/pypy/jit/metainterp/warmspot.py +++ b/pypy/jit/metainterp/warmspot.py @@ -925,7 +925,6 @@ def assembler_call_helper(deadframe, virtualizableref): fail_descr = self.cpu.get_latest_descr(deadframe) if vinfo is not None: - xxx virtualizable = lltype.cast_opaque_ptr( vinfo.VTYPEPTR, virtualizableref) vinfo.reset_vable_token(virtualizable) From noreply at buildbot.pypy.org Sat Jan 19 14:29:32 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 19 Jan 2013 14:29:32 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fix test_resume Message-ID: <20130119132932.592601C02D9@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60202:5b4da4c7bf61 Date: 2013-01-19 15:29 +0200 http://bitbucket.org/pypy/pypy/changeset/5b4da4c7bf61/ Log: fix test_resume diff --git a/pypy/jit/metainterp/test/test_resume.py b/pypy/jit/metainterp/test/test_resume.py --- a/pypy/jit/metainterp/test/test_resume.py +++ b/pypy/jit/metainterp/test/test_resume.py @@ -103,13 +103,13 @@ CONST_NULL = ConstPtr(gcrefnull) def __init__(self, values): self.values = values - def get_latest_value_int(self, deadframe, index): + def get_int_value(self, deadframe, index): assert deadframe == "deadframe" return self.values[index] - def get_latest_value_ref(self, deadframe, index): + def get_ref_value(self, deadframe, index): assert deadframe == "deadframe" return self.values[index] - def get_latest_value_float(self, deadframe, index): + def get_float_value(self, deadframe, index): assert deadframe == "deadframe" return self.values[index] @@ -240,6 +240,7 @@ rd_numb = [] rd_consts = [] rd_pendingfields = None + rd_count = 0 class FakeMetainterp(object): _already_allocated_resume_virtuals = None cpu = None From noreply at buildbot.pypy.org Sat Jan 19 14:34:08 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 19 Jan 2013 14:34:08 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fix this test Message-ID: <20130119133408.E52191C02D9@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60203:c85600087e9a Date: 2013-01-19 15:33 +0200 http://bitbucket.org/pypy/pypy/changeset/c85600087e9a/ Log: fix this test diff --git a/pypy/jit/metainterp/test/test_virtualref.py b/pypy/jit/metainterp/test/test_virtualref.py --- a/pypy/jit/metainterp/test/test_virtualref.py +++ b/pypy/jit/metainterp/test/test_virtualref.py @@ -114,9 +114,8 @@ # try reloading from blackhole.py's point of view from pypy.jit.metainterp.resume import ResumeDataDirectReader cpu = self.metainterp.cpu - cpu.get_latest_value_count = lambda df: len(guard_op.getfailargs()) - cpu.get_latest_value_int = lambda df,i:guard_op.getfailargs()[i].getint() - cpu.get_latest_value_ref = lambda df,i:guard_op.getfailargs()[i].getref_base() + cpu.get_int_value = lambda df,i:guard_op.getfailargs()[i].getint() + cpu.get_ref_value = lambda df,i:guard_op.getfailargs()[i].getref_base() cpu.clear_latest_values = lambda count: None class FakeMetaInterpSd: callinfocollection = None From noreply at buildbot.pypy.org Sat Jan 19 14:35:54 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 19 Jan 2013 14:35:54 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: oops Message-ID: <20130119133554.DAAA81C02D9@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60204:4f19c69ba01d Date: 2013-01-19 15:35 +0200 http://bitbucket.org/pypy/pypy/changeset/4f19c69ba01d/ Log: oops diff --git a/pypy/jit/backend/x86/regalloc.py b/pypy/jit/backend/x86/regalloc.py --- a/pypy/jit/backend/x86/regalloc.py +++ b/pypy/jit/backend/x86/regalloc.py @@ -872,8 +872,6 @@ assert isinstance(descr, JitCellToken) self.rm._sync_var(op.getarg(0)) frame_loc = self.fm.loc(op.getarg(0)) - if vable_index >= 0: - xxx self._call(op, [frame_loc, self.loc(op.getarg(0))], guard_not_forced_op=guard_op) From noreply at buildbot.pypy.org Sat Jan 19 15:03:36 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 19 Jan 2013 15:03:36 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: rpythonize runner Message-ID: <20130119140336.48EB71C02D9@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60205:57a25033f068 Date: 2013-01-19 15:41 +0200 http://bitbucket.org/pypy/pypy/changeset/57a25033f068/ Log: rpythonize runner diff --git a/pypy/jit/backend/x86/runner.py b/pypy/jit/backend/x86/runner.py --- a/pypy/jit/backend/x86/runner.py +++ b/pypy/jit/backend/x86/runner.py @@ -1,4 +1,5 @@ import py +from pypy.rlib.unroll import unrolling_iterable from pypy.rpython.lltypesystem import lltype, llmemory, rffi from pypy.rpython.lltypesystem.lloperation import llop from pypy.rpython.llinterp import LLInterpreter @@ -108,7 +109,10 @@ def make_execute_token(self, *ARGS): FUNCPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF], llmemory.GCREF)) - # + + lst = [(i, history.getkind(ARG)[0]) for i, ARG in enumerate(ARGS)] + kinds = unrolling_iterable(lst) + def execute_token(executable_token, *args): clt = executable_token.compiled_loop_token assert len(args) == clt._debug_nbargs @@ -124,16 +128,17 @@ prev_interpreter = LLInterpreter.current_interpreter LLInterpreter.current_interpreter = self.debug_ll_interpreter try: - # XXX RPythonize num = JITFRAME_FIXED_SIZE * WORD - for arg in args: - if isinstance(arg, int): + for i, kind in kinds: + arg = args[i] + if kind == history.INT: self.set_int_value(frame, num, arg) - elif isinstance(arg, float): + elif kind == history.FLOAT: self.set_float_value(frame, num, arg) if IS_X86_32: num += WORD else: + assert kind == history.REF self.set_ref_value(frame, num, arg) num += WORD ll_frame = func(ll_frame) From noreply at buildbot.pypy.org Sat Jan 19 15:03:37 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 19 Jan 2013 15:03:37 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fix this test a bit Message-ID: <20130119140337.80EA01C02D9@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60206:fbb2583b6e4d Date: 2013-01-19 16:03 +0200 http://bitbucket.org/pypy/pypy/changeset/fbb2583b6e4d/ Log: fix this test a bit diff --git a/pypy/jit/backend/test/calling_convention_test.py b/pypy/jit/backend/test/calling_convention_test.py --- a/pypy/jit/backend/test/calling_convention_test.py +++ b/pypy/jit/backend/test/calling_convention_test.py @@ -1,21 +1,11 @@ -from pypy.jit.metainterp.history import (AbstractFailDescr, - AbstractDescr, - BasicFailDescr, - BoxInt, Box, BoxPtr, - JitCellToken, - ConstInt, ConstPtr, - BoxObj, Const, - ConstObj, BoxFloat, ConstFloat) -from pypy.jit.metainterp.resoperation import ResOperation, rop -from pypy.jit.metainterp.typesystem import deref +from pypy.jit.metainterp.history import BasicFinalDescr, BoxInt, JitCellToken,\ + ConstInt, BoxFloat, ConstFloat +from pypy.jit.metainterp.resoperation import rop from pypy.jit.codewriter.effectinfo import EffectInfo from pypy.jit.tool.oparser import parse -from pypy.rpython.lltypesystem import lltype, llmemory, rstr, rffi, rclass -from pypy.rpython.ootypesystem import ootype +from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.annlowlevel import llhelper -from pypy.rpython.llinterp import LLException from pypy.jit.codewriter import heaptracker, longlong -from pypy.rlib.rarithmetic import intmask from pypy.jit.backend.detect_cpu import getcpuclass from pypy.jit.backend.test.runner_test import Runner import py @@ -111,12 +101,11 @@ loop = parse(ops, namespace=locals()) looptoken = JitCellToken() - done_number = self.cpu.get_fail_descr_number(loop.operations[-2].getdescr()) self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) argvals, expected_result = self._prepare_args(args, floats, ints) deadframe = self.cpu.execute_token(looptoken, *argvals) - x = longlong.getrealfloat(cpu.get_latest_value_float(deadframe, 0)) + x = longlong.getrealfloat(cpu.get_float_value(deadframe, 0)) assert abs(x - expected_result) < 0.0001 def test_call_aligned_with_imm_values(self): @@ -208,8 +197,8 @@ if not cpu.supports_floats: py.test.skip('requires floats') - fdescr3 = BasicFailDescr(3) - fdescr4 = BasicFailDescr(4) + fdescr3 = BasicFinalDescr(3) + fdescr4 = BasicFinalDescr(4) def assembler_helper(failindex, virtualizable): assert 0, 'should not be called, but was with failindex (%d)' % failindex @@ -255,13 +244,13 @@ called_loop = parse(called_ops, namespace=locals()) called_looptoken = JitCellToken() called_looptoken.outermost_jitdriver_sd = FakeJitDriverSD() - done_number = self.cpu.get_fail_descr_number(called_loop.operations[-1].getdescr()) + done_descr = called_loop.operations[-1].getdescr() self.cpu.compile_loop(called_loop.inputargs, called_loop.operations, called_looptoken) argvals, expected_result = self._prepare_args(args, floats, ints) deadframe = cpu.execute_token(called_looptoken, *argvals) - assert cpu.get_latest_descr(deadframe).identifier == 3 - t = longlong.getrealfloat(cpu.get_latest_value_float(deadframe, 0)) + assert cpu.get_latest_descr(deadframe) == fdescr3 + t = longlong.getrealfloat(cpu.get_float_value(deadframe, 0)) assert abs(t - expected_result) < 0.0001 ARGS = [] @@ -282,7 +271,7 @@ ''' % (arguments, arguments) loop = parse(ops, namespace=locals()) # we want to take the fast path - self.cpu.done_with_this_frame_float_v = done_number + self.cpu.done_with_this_frame_descr_float = done_descr try: othertoken = JitCellToken() self.cpu.compile_loop(loop.inputargs, loop.operations, othertoken) @@ -291,38 +280,37 @@ argvals, _ = self._prepare_args(args, floats, ints) deadframe = cpu.execute_token(othertoken, *argvals) x = longlong.getrealfloat( - cpu.get_latest_value_float(deadframe, 0)) - assert cpu.get_latest_descr(deadframe).identifier == 4 + cpu.get_float_value(deadframe, 0)) + assert cpu.get_latest_descr(deadframe) == fdescr4 assert abs(x - expected_result) < 0.0001 finally: - del self.cpu.done_with_this_frame_float_v + del self.cpu.done_with_this_frame_descr_float def test_call_with_imm_values_bug_constint0(self): - from pypy.rlib.libffi import types - cpu = self.cpu + cpu = self.cpu - I = lltype.Signed - ints = [7, 11, 23, 13, -42, 0, 0, 9] + I = lltype.Signed + ints = [7, 11, 23, 13, -42, 0, 0, 9] - def func(*args): - for i in range(len(args)): - assert args[i] == ints[i] - return sum(args) + def func(*args): + for i in range(len(args)): + assert args[i] == ints[i] + return sum(args) - result = sum(ints) - args = [I] * len(ints) - argslist = [ConstInt(i) for i in ints] - FUNC = self.FuncType(args, I) - FPTR = self.Ptr(FUNC) - func_ptr = llhelper(FPTR, func) - calldescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo.MOST_GENERAL) - funcbox = self.get_funcbox(cpu, func_ptr) + result = sum(ints) + args = [I] * len(ints) + argslist = [ConstInt(i) for i in ints] + FUNC = self.FuncType(args, I) + FPTR = self.Ptr(FUNC) + func_ptr = llhelper(FPTR, func) + calldescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo.MOST_GENERAL) + funcbox = self.get_funcbox(cpu, func_ptr) - res = self.execute_operation(rop.CALL, - [funcbox] + argslist, - 'int', descr=calldescr) - assert res.value == result + res = self.execute_operation(rop.CALL, + [funcbox] + argslist, + 'int', descr=calldescr) + assert res.value == result def test_call_with_singlefloats(self): diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -936,13 +936,17 @@ # move a ConstFloatLoc directly to a StackLoc, as two MOVs # (even on x86-64, because the immediates are encoded as 32 bits) assert isinstance(from_loc, ConstFloatLoc) - assert isinstance(to_loc, StackLoc) low_part = rffi.cast(rffi.CArrayPtr(rffi.INT), from_loc.value)[0] high_part = rffi.cast(rffi.CArrayPtr(rffi.INT), from_loc.value)[1] low_part = intmask(low_part) high_part = intmask(high_part) - self.mc.MOV32_bi(to_loc.value, low_part) - self.mc.MOV32_bi(to_loc.value + 4, high_part) + if isinstance(to_loc, RawStackLoc): + self.mc.MOV32_bi(to_loc.value, low_part) + self.mc.MOV32_bi(to_loc.value + 4, high_part) + else: + assert isinstance(to_loc, RawEspLoc) + self.mc.MOV32_si(to_loc.value, low_part) + self.mc.MOV32_si(to_loc.value + 4, high_part) def regalloc_perform(self, op, arglocs, resloc): genop_list[op.getopnum()](self, op, arglocs, resloc) @@ -1196,6 +1200,11 @@ # Load the singlefloat arguments from main regs or stack to xmm regs if singlefloats is not None: for src, dst in singlefloats: + if isinstance(dst, RawEspLoc) and isinstance(src, RawStackLoc): + # XXX too much special logic + self.mc.MOV32(X86_64_SCRATCH_REG, src) + self.mc.MOV32(dst, X86_64_SCRATCH_REG) + continue if isinstance(src, ImmedLoc): self.mc.MOV(X86_64_SCRATCH_REG, src) src = X86_64_SCRATCH_REG From noreply at buildbot.pypy.org Sat Jan 19 15:06:11 2013 From: noreply at buildbot.pypy.org (hakanardo) Date: Sat, 19 Jan 2013 15:06:11 +0100 (CET) Subject: [pypy-commit] pypy jit-usable_retrace_3: unneeded Message-ID: <20130119140611.D78131C02D9@cobra.cs.uni-duesseldorf.de> Author: Hakan Ardo Branch: jit-usable_retrace_3 Changeset: r60207:1c04768e7bb0 Date: 2013-01-14 20:38 +0100 http://bitbucket.org/pypy/pypy/changeset/1c04768e7bb0/ Log: unneeded diff --git a/pypy/jit/metainterp/test/test_random_loops.py b/pypy/jit/metainterp/test/test_random_loops.py --- a/pypy/jit/metainterp/test/test_random_loops.py +++ b/pypy/jit/metainterp/test/test_random_loops.py @@ -189,7 +189,7 @@ def random_loop(self, max_stmts_per_block=10, max_levels=5): self.max_stmts_per_block = max_stmts_per_block self.levels = max_levels - return '{{' + self.block() + '1}1}' + return '{{' + self.block() + '}}' class BaseTests(RandomLoopBase): @@ -229,7 +229,7 @@ self.check('09-A{aa+A}', max_back_jumps=100) def test_failure1(self): - self.check('{{7d<(37>(c6-A{a6>n(x)5b<()(bb+Bea-A){7d>n(x)1e+A7d-B3d+E35+Eac-B2b<(04+Cxe8+D9d+C)(){c6n(x)a5=(2c+E)(xxe9-A)ec>(b7-A40+Ce0-Cx06-A)(xc0-Ad2-A1e+B0d+Bx09+B0b-D)}be-Ab7=(3a+Ab7-Dde<(x))(ec+Cd4+D5d<(x10+A)(cb-Cb9-C))59<(2b-C00-Ab0<(bc-Bde+Bxa2-E4d+A))(e7+Eb3<(c4-Bxx)59-C7c+Aa8>(96-Exce+D3e-Ex)db+A76-Cdb-Bdc=(2b-A))}d2+D)(27-C26+B39+A9d+Bcc+Dab+Cdd=(ae+Eee-Ex{5d>n(x)}d0<(2a+C41+Dca+C6a>(47-Ae9-Eb7+Cx60+D63-C)bb+D9e+Ec5=(d2-Cxaa-D84-Ced+B20+A4c+B)65+A59-E)ed<(cd+B{a8=n(x)ed+Bxxd8+A}ba-D{66=n(x)a4+D30+Acd+C1c+A6c+Ac7-Dxc0+B}x)4b-Dx)e4-D{aa=n(x)44-Eac>(3e+Aae+A12<(a4-A)99<(ce-C4a+A7b+Dxdd+E)(c5+A10-D63-Eda-E22-E85+Dea+E)x)ed+Dbb=(e5-A9e-C)(x{can(x)}e5+Ba5+B}c9=()c0=(7b-Ae1+Daa+Aed+C)(91-C))3d=(a2>(1e-Cba=(2c-A)21+C6b>(aa-Bxd5-Bc7-E)ab<(8c-Ba8-Cxea-D4e+Ae8+Dc5+Ade+B)x33+B5a-C17+D)2a+B{ebn(x)x6a+Cb4+A27-Eab-C35-E25+B1d+C}cc+Aae-D}cc+C61-C68-D)(xe4>(e4+Db4-Cxdb=(xc7-Bcd-Bde-A00+A39+E9c+Dc9+B24-B){2b>n(x)x4d+E8d+Dbb+Ce7-Bxea+B54+A}80-A43<(89-B)(1c+Cx7b+Ae2+E02-C84+Exd0+A))(3d+Eb3-Ecd-A{8c=n(x)37-Bb0+Axxee+C11-C2c-De6-D46+D}cb+E)84+B05-Cbd+D)ad>(b6-C7d+Ae4-B4a+E)34-Cx9a=({c4>n(x)dd+B72<(a9+A)(b0-Db8-Dac-Bba-Excb+Ce2+Bx)ba-Bda+Cce-Cx}78+E97=(aa+Dx67+Dbc+B{de>n(x)b8-C14-Cx})c4+Cac+Ex80+C))xca=({55(de+Ax)da-B{d6>n(x)8d+A36-Bx8d-Bx49+Bbc-D88+Bx}x26-Da7-E)({9c>n(x)}bc<(0a-Cx61+Cce+A54-Ce7+A21-C9d+D)(6d-E)x{7e=n(x)6d-Ce6-Cx}5c>(bd+Ed5+Cc0-Dc2+C93-B)x69>(d9-Ac8+B8c-Dx94-E76+Ax)(ba+D)b6>(bb+Ebb-A5e+Bc3-D6c-E))54>(95+B46-D{15(xb6+Ec6-Aac-Bxc2-Ax99-C))b0-E58>(x0e+Bxa4+D5e>(7e-Dde-B1c+Bba-B8a-D)(35-Cce-C4c-C90-C1e+B9a+Be7+C1d-E)45>(xb9+A91+Ecc-Aac-B91+Cd1-De3+Ddd-B)c0+A0a>(45+Cbd+Bdc+Db4-D7c+C29+A7a+B60+A))(5d>(xx6e-Dxea+C)(c2-Cad+D36-Cx)61+E20-C9e-Cc2+Cce+Ad0+B{4b=n(x)})x{a1=n(x)e8<(x35-Dd3-Bd6-Bea+Eac-Acc+B79+Eaa-C)}x)xc7>(6b-C{57>n(x)e2>(1a+A97+Dxcc-E7b+A5d-B82+C2b-B)0a+Be1-Cbc+Ca2-E01+B{41>n(x)e1+Dc6+B47+Beb+Ac2+Bxab+D}}d3-D0e+E0d+Bae=(c7+Ce3-Bc3<(d2+E68+Bbc-Cdc+B1b-Ea2-Ac8-E)45-B43+Aac-E)xbe+C)dc=({ee>n(x)a4-Axd2+B1b+B}aa+Exxxb4>()(cb-C{4d(a0-Dab+A8c+Ddd-Edc+A)(xce-D)de=(d6+E83+Cxxc6-Axx)(0b+D78+Cx07-Eda+Ax1d-Eba+A)e6+E)de-E1c+Aae=(20-B25+Dba>(xxx71-A2b+Dc7+E){e9=n(x)7b+Ecd+Bda-Eec-A4b+De7-Ae3-D}ae+C7c+Aaa+Bx)86+B)}{cc=n(x)6b-B{cd(cc+D7e-Bc0+Bxde-A2c-B5d-C)(ba+D2b-Axxx07+Ca4-C)eb-C23-D7e<(xb8+Edd-De1+Add+C35-Dxx)x)(87-Ccc+Ee2=(9a-E78-C)()cb-A)39+D0b-Aa4>(ca+A)c6=({9c(bb-E22-D)x5e-C11<()9b=(6d-Bcd+A9e-E2a-E38+B4e-C)xaa-B{82=n(x)ae-B6a-Ab7+Ca9-A}08+C)d7+Cxcb-A}6c-B9c+D9c-E})3d+Cd4>(xab<()()3e+Ebd-Ba0-B{6c(63+Eb5+Bxec<(a1+E4e+Axdc-C)ea+Eaa=(x37-Edd-Cb4+B)(43-Ae0-Bx7e-C94+A)e8-Bd1-Da8-D)())(ce=(6c-B10+C7a+A3d+Ax19+Aca+B)(ed+A)1d>()d1+Ex43+A9d+Eb5+A)}8d+Ae9-C)(3d-D6c+Ee9<(29-C4c=({e9(4c+B)8a-Dc0-E82+E50+D}a1+D98=(be-B)3a-Ae9+De2-Add-E)35-D1}1}', [13, 40, 35, 44, 76], 1000) + self.check('{{7d<(37>(c6-A{a6>n(x)5b<()(bb+Bea-A){7d>n(x)1e+A7d-B3d+E35+Eac-B2b<(04+Cxe8+D9d+C)(){c6n(x)a5=(2c+E)(xxe9-A)ec>(b7-A40+Ce0-Cx06-A)(xc0-Ad2-A1e+B0d+Bx09+B0b-D)}be-Ab7=(3a+Ab7-Dde<(x))(ec+Cd4+D5d<(x10+A)(cb-Cb9-C))59<(2b-C00-Ab0<(bc-Bde+Bxa2-E4d+A))(e7+Eb3<(c4-Bxx)59-C7c+Aa8>(96-Exce+D3e-Ex)db+A76-Cdb-Bdc=(2b-A))}d2+D)(27-C26+B39+A9d+Bcc+Dab+Cdd=(ae+Eee-Ex{5d>n(x)}d0<(2a+C41+Dca+C6a>(47-Ae9-Eb7+Cx60+D63-C)bb+D9e+Ec5=(d2-Cxaa-D84-Ced+B20+A4c+B)65+A59-E)ed<(cd+B{a8=n(x)ed+Bxxd8+A}ba-D{66=n(x)a4+D30+Acd+C1c+A6c+Ac7-Dxc0+B}x)4b-Dx)e4-D{aa=n(x)44-Eac>(3e+Aae+A12<(a4-A)99<(ce-C4a+A7b+Dxdd+E)(c5+A10-D63-Eda-E22-E85+Dea+E)x)ed+Dbb=(e5-A9e-C)(x{can(x)}e5+Ba5+B}c9=()c0=(7b-Ae1+Daa+Aed+C)(91-C))3d=(a2>(1e-Cba=(2c-A)21+C6b>(aa-Bxd5-Bc7-E)ab<(8c-Ba8-Cxea-D4e+Ae8+Dc5+Ade+B)x33+B5a-C17+D)2a+B{ebn(x)x6a+Cb4+A27-Eab-C35-E25+B1d+C}cc+Aae-D}cc+C61-C68-D)(xe4>(e4+Db4-Cxdb=(xc7-Bcd-Bde-A00+A39+E9c+Dc9+B24-B){2b>n(x)x4d+E8d+Dbb+Ce7-Bxea+B54+A}80-A43<(89-B)(1c+Cx7b+Ae2+E02-C84+Exd0+A))(3d+Eb3-Ecd-A{8c=n(x)37-Bb0+Axxee+C11-C2c-De6-D46+D}cb+E)84+B05-Cbd+D)ad>(b6-C7d+Ae4-B4a+E)34-Cx9a=({c4>n(x)dd+B72<(a9+A)(b0-Db8-Dac-Bba-Excb+Ce2+Bx)ba-Bda+Cce-Cx}78+E97=(aa+Dx67+Dbc+B{de>n(x)b8-C14-Cx})c4+Cac+Ex80+C))xca=({55(de+Ax)da-B{d6>n(x)8d+A36-Bx8d-Bx49+Bbc-D88+Bx}x26-Da7-E)({9c>n(x)}bc<(0a-Cx61+Cce+A54-Ce7+A21-C9d+D)(6d-E)x{7e=n(x)6d-Ce6-Cx}5c>(bd+Ed5+Cc0-Dc2+C93-B)x69>(d9-Ac8+B8c-Dx94-E76+Ax)(ba+D)b6>(bb+Ebb-A5e+Bc3-D6c-E))54>(95+B46-D{15(xb6+Ec6-Aac-Bxc2-Ax99-C))b0-E58>(x0e+Bxa4+D5e>(7e-Dde-B1c+Bba-B8a-D)(35-Cce-C4c-C90-C1e+B9a+Be7+C1d-E)45>(xb9+A91+Ecc-Aac-B91+Cd1-De3+Ddd-B)c0+A0a>(45+Cbd+Bdc+Db4-D7c+C29+A7a+B60+A))(5d>(xx6e-Dxea+C)(c2-Cad+D36-Cx)61+E20-C9e-Cc2+Cce+Ad0+B{4b=n(x)})x{a1=n(x)e8<(x35-Dd3-Bd6-Bea+Eac-Acc+B79+Eaa-C)}x)xc7>(6b-C{57>n(x)e2>(1a+A97+Dxcc-E7b+A5d-B82+C2b-B)0a+Be1-Cbc+Ca2-E01+B{41>n(x)e1+Dc6+B47+Beb+Ac2+Bxab+D}}d3-D0e+E0d+Bae=(c7+Ce3-Bc3<(d2+E68+Bbc-Cdc+B1b-Ea2-Ac8-E)45-B43+Aac-E)xbe+C)dc=({ee>n(x)a4-Axd2+B1b+B}aa+Exxxb4>()(cb-C{4d(a0-Dab+A8c+Ddd-Edc+A)(xce-D)de=(d6+E83+Cxxc6-Axx)(0b+D78+Cx07-Eda+Ax1d-Eba+A)e6+E)de-E1c+Aae=(20-B25+Dba>(xxx71-A2b+Dc7+E){e9=n(x)7b+Ecd+Bda-Eec-A4b+De7-Ae3-D}ae+C7c+Aaa+Bx)86+B)}{cc=n(x)6b-B{cd(cc+D7e-Bc0+Bxde-A2c-B5d-C)(ba+D2b-Axxx07+Ca4-C)eb-C23-D7e<(xb8+Edd-De1+Add+C35-Dxx)x)(87-Ccc+Ee2=(9a-E78-C)()cb-A)39+D0b-Aa4>(ca+A)c6=({9c(bb-E22-D)x5e-C11<()9b=(6d-Bcd+A9e-E2a-E38+B4e-C)xaa-B{82=n(x)ae-B6a-Ab7+Ca9-A}08+C)d7+Cxcb-A}6c-B9c+D9c-E})3d+Cd4>(xab<()()3e+Ebd-Ba0-B{6c(63+Eb5+Bxec<(a1+E4e+Axdc-C)ea+Eaa=(x37-Edd-Cb4+B)(43-Ae0-Bx7e-C94+A)e8-Bd1-Da8-D)())(ce=(6c-B10+C7a+A3d+Ax19+Aca+B)(ed+A)1d>()d1+Ex43+A9d+Eb5+A)}8d+Ae9-C)(3d-D6c+Ee9<(29-C4c=({e9(4c+B)8a-Dc0-E82+E50+D}a1+D98=(be-B)3a-Ae9+De2-Add-E)35-D}}', [13, 40, 35, 44, 76], 1000) def test_random_bytecode(self): for i in xrange(1000): From noreply at buildbot.pypy.org Sat Jan 19 15:06:13 2013 From: noreply at buildbot.pypy.org (hakanardo) Date: Sat, 19 Jan 2013 15:06:13 +0100 (CET) Subject: [pypy-commit] pypy jit-usable_retrace_3: more specific version of test_random_loops.test_failure1 Message-ID: <20130119140613.2073C1C02D9@cobra.cs.uni-duesseldorf.de> Author: Hakan Ardo Branch: jit-usable_retrace_3 Changeset: r60208:7694bc8ca631 Date: 2013-01-19 14:36 +0100 http://bitbucket.org/pypy/pypy/changeset/7694bc8ca631/ Log: more specific version of test_random_loops.test_failure1 diff --git a/pypy/jit/metainterp/test/test_virtualstate.py b/pypy/jit/metainterp/test/test_virtualstate.py --- a/pypy/jit/metainterp/test/test_virtualstate.py +++ b/pypy/jit/metainterp/test/test_virtualstate.py @@ -1316,6 +1316,26 @@ self.setfield(o.node2, o.descr1, o.const_int0) self.combine([o.node2], [o.node1], [Virtual(o.node_class, {o.descr1: Unknown})]) + def test_boxed_int_unassigned_zero1(self): + o = self.optimizer + self.setfield(o.node1, o.descr1, o.const_int1) + self.combine([o.node1], [o.node1], [Virtual(o.node_class, {o.descr1: Const(1)})]) + + def test_boxed_int_unassigned_zero2(self): + o = self.optimizer + self.setfield(o.node1, o.descr1, o.const_int1) + self.combine([o.node2], [o.node2], [Virtual(o.node_class, {})]) + + def test_boxed_int_unassigned_zero3(self): + o = self.optimizer + self.setfield(o.node1, o.descr1, o.const_int1) + self.combine([o.node1], [o.node2], [Virtual(o.node_class, {o.descr1: Unknown})]) + + def test_boxed_int_unassigned_zero4(self): + o = self.optimizer + self.setfield(o.node1, o.descr1, o.const_int1) + self.combine([o.node2], [o.node1], [Virtual(o.node_class, {o.descr1: Unknown})]) + def test_three_boxed_int_zero(self): o = self.optimizer for consts1 in itertools.permutations([o.const_int0, o.const_int1, o.const_int2]): From noreply at buildbot.pypy.org Sat Jan 19 15:06:14 2013 From: noreply at buildbot.pypy.org (hakanardo) Date: Sat, 19 Jan 2013 15:06:14 +0100 (CET) Subject: [pypy-commit] pypy jit-usable_retrace_3: Rearange code to make value.getfiled always return a propert OptValue Message-ID: <20130119140614.51F661C02D9@cobra.cs.uni-duesseldorf.de> Author: Hakan Ardo Branch: jit-usable_retrace_3 Changeset: r60209:db653f81ea07 Date: 2013-01-19 15:03 +0100 http://bitbucket.org/pypy/pypy/changeset/db653f81ea07/ Log: Rearange code to make value.getfiled always return a propert OptValue diff --git a/pypy/jit/metainterp/optimizeopt/virtualize.py b/pypy/jit/metainterp/optimizeopt/virtualize.py --- a/pypy/jit/metainterp/optimizeopt/virtualize.py +++ b/pypy/jit/metainterp/optimizeopt/virtualize.py @@ -77,8 +77,15 @@ self._fields = {} self._cached_sorted_fields = None - def getfield(self, ofs, default): - return self._fields.get(ofs, default) + def getfield(self, ofs, optimizer): + assert optimizer + fieldvalue = self._fields.get(ofs, None) + if fieldvalue is None: + fieldvalue = optimizer.new_const(ofs) + return fieldvalue + + def field_is_set(self, ofs): + return ofs in self._fields def setfield(self, ofs, fieldvalue): assert isinstance(fieldvalue, optimizer.OptValue) @@ -442,16 +449,13 @@ # If this is an immutable field (as indicated by op.is_always_pure()) # then it's safe to reuse the virtual's field, even if it has been # forced, because it should never be written to again. - if value.is_forced_virtual() and op.is_always_pure(): - fieldvalue = value.getfield(op.getdescr(), None) - if fieldvalue is not None: - self.make_equal_to(op.result, fieldvalue) - return + if value.is_forced_virtual() and op.is_always_pure() and value.field_is_set(op.getdescr()): + fieldvalue = value.getfield(op.getdescr(), self.optimizer) + self.make_equal_to(op.result, fieldvalue) + return if value.is_virtual(): assert isinstance(value, AbstractVirtualValue) - fieldvalue = value.getfield(op.getdescr(), None) - if fieldvalue is None: - fieldvalue = self.optimizer.new_const(op.getdescr()) + fieldvalue = value.getfield(op.getdescr(), self.optimizer) self.make_equal_to(op.result, fieldvalue) else: value.ensure_nonnull() diff --git a/pypy/jit/metainterp/optimizeopt/virtualstate.py b/pypy/jit/metainterp/optimizeopt/virtualstate.py --- a/pypy/jit/metainterp/optimizeopt/virtualstate.py +++ b/pypy/jit/metainterp/optimizeopt/virtualstate.py @@ -140,32 +140,32 @@ while i < len(self.fielddescrs) and j < len(other.fielddescrs): if other.fielddescrs[j] is self.fielddescrs[i]: new_field_value = self.fieldstate[i].make_guardable_generalization_of(other.fieldstate[j], - value.getfield(self.fielddescrs[i], None), optimizer) + value.getfield(self.fielddescrs[i], optimizer), optimizer) if new_field_value: value.setfield(self.fielddescrs[i], new_field_value) i += 1 j += 1 elif self.fielddescrs[i].sort_key() < other.fielddescrs[j].sort_key(): new_field_value = self.fieldstate[i].make_guardable_generalization_of_null( - value.getfield(self.fielddescrs[i], None), optimizer) + value.getfield(self.fielddescrs[i], optimizer), optimizer) if new_field_value: value.setfield(self.fielddescrs[i], new_field_value) i += 1 else: new_field_value = other.fieldstate[j].make_guardable_generalization_of_null( - value.getfield(other.fielddescrs[j], None), optimizer) + value.getfield(other.fielddescrs[j], optimizer), optimizer) if new_field_value: value.setfield(other.fielddescrs[j], new_field_value) j += 1 while i < len(self.fielddescrs): new_field_value = self.fieldstate[i].make_guardable_generalization_of_null( - value.getfield(self.fielddescrs[i], None), optimizer) + value.getfield(self.fielddescrs[i], optimizer), optimizer) if new_field_value: value.setfield(self.fielddescrs[i], new_field_value) i += 1 while j < len(other.fielddescrs): new_field_value = other.fieldstate[j].make_guardable_generalization_of_null( - value.getfield(other.fielddescrs[j], None), optimizer) + value.getfield(other.fielddescrs[j], optimizer), optimizer) if new_field_value: value.setfield(other.fielddescrs[j], new_field_value) j += 1 @@ -188,9 +188,7 @@ if not value.is_virtual(): raise BadVirtualState for i in range(len(self.fielddescrs)): - v = value.getfield(self.fielddescrs[i], None) - if v is None: - v = optimizer.new_const(self.fielddescrs[i]) + v = value.getfield(self.fielddescrs[i], optimizer) s = self.fieldstate[i] if s.position > self.position: s.enum_forced_boxes(boxes, v, optimizer, doforce) diff --git a/pypy/jit/metainterp/test/test_virtualstate.py b/pypy/jit/metainterp/test/test_virtualstate.py --- a/pypy/jit/metainterp/test/test_virtualstate.py +++ b/pypy/jit/metainterp/test/test_virtualstate.py @@ -13,7 +13,7 @@ from pypy.jit.metainterp.history import TreeLoop, JitCellToken from pypy.jit.metainterp.optimizeopt.test.test_optimizeopt import FakeMetaInterpStaticData from pypy.jit.metainterp.resoperation import ResOperation, rop -from pypy.jit.metainterp.optimizeopt.optimizer import LEVEL_UNKNOWN, LEVEL_CONSTANT, Optimizer +from pypy.jit.metainterp.optimizeopt.optimizer import LEVEL_UNKNOWN, LEVEL_CONSTANT, Optimizer, CVAL_ZERO import itertools class TestBasic: @@ -1205,6 +1205,10 @@ def make_equal_to(self, box, value, replace=False): self.values[box] = value + + def new_const(self, descr): + assert isinstance(descr, FakeDescr) + return CVAL_ZERO optearlyforce = None opaque_pointers = {} From noreply at buildbot.pypy.org Sat Jan 19 15:07:37 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 19 Jan 2013 15:07:37 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fix test_jump Message-ID: <20130119140737.F2E421C02D9@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60210:d3bdf7d158b5 Date: 2013-01-19 16:07 +0200 http://bitbucket.org/pypy/pypy/changeset/d3bdf7d158b5/ Log: fix test_jump diff --git a/pypy/TODO b/pypy/TODO --- a/pypy/TODO +++ b/pypy/TODO @@ -2,4 +2,5 @@ * 32bit x86 * ARM * shadowstack + asmgcc -* kill jit2gc on translator \ No newline at end of file +* kill jit2gc on translator +* fix test_singlefloats in test_calling_conventions \ No newline at end of file diff --git a/pypy/jit/backend/x86/regloc.py b/pypy/jit/backend/x86/regloc.py --- a/pypy/jit/backend/x86/regloc.py +++ b/pypy/jit/backend/x86/regloc.py @@ -97,6 +97,8 @@ return self.type == FLOAT class StackLoc(RawStackLoc): + _immutable_ = True + def __init__(self, position, ebp_offset, type): from pypy.jit.backend.x86.arch import JITFRAME_FIXED_SIZE, WORD @@ -106,7 +108,8 @@ assert ebp_offset >= 0 #assert not (0 <= ebp_offset < 8 + 8 * IS_X86_64) self.position = position - assert (position + JITFRAME_FIXED_SIZE) * WORD == ebp_offset + if position != 9999: + assert (position + JITFRAME_FIXED_SIZE) * WORD == ebp_offset self.value = ebp_offset # One of INT, REF, FLOAT self.type = type diff --git a/pypy/jit/backend/x86/test/test_jump.py b/pypy/jit/backend/x86/test/test_jump.py --- a/pypy/jit/backend/x86/test/test_jump.py +++ b/pypy/jit/backend/x86/test/test_jump.py @@ -389,11 +389,11 @@ def test_overflow_bug(): CASE = [ - (-144, -248), # \ cycle - (-248, -144), # / - (-488, -416), # \ two usages of -488 - (-488, -480), # / - (-488, -488), # - one self-application of -488 + (144, 248), # \ cycle + (248, 144), # / + (488, 416), # \ two usages of -488 + (488, 480), # / + (488, 488), # - one self-application of -488 ] class FakeAssembler: def regalloc_mov(self, src, dst): From noreply at buildbot.pypy.org Sat Jan 19 15:12:23 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 19 Jan 2013 15:12:23 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fix Message-ID: <20130119141223.CD6FC1C01D2@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60211:3658fb4c15a5 Date: 2013-01-19 16:12 +0200 http://bitbucket.org/pypy/pypy/changeset/3658fb4c15a5/ Log: fix diff --git a/pypy/jit/backend/x86/test/test_recompilation.py b/pypy/jit/backend/x86/test/test_recompilation.py --- a/pypy/jit/backend/x86/test/test_recompilation.py +++ b/pypy/jit/backend/x86/test/test_recompilation.py @@ -16,11 +16,11 @@ ops = ''' [i1] i3 = int_add(i1, 1) - finish(i3, descr=fdescr2) + finish(i3) ''' bridge = self.attach_bridge(ops, loop, -2) fail = self.run(loop, 0) - assert fail.identifier == 2 + assert fail == bridge.operations[-1].getdescr() assert self.getint(0) == 21 def test_compile_bridge_deeper(self): From noreply at buildbot.pypy.org Sat Jan 19 18:40:30 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 19 Jan 2013 18:40:30 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: Fixes Message-ID: <20130119174030.7C91B1C02D9@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60212:d0469f22d0e8 Date: 2013-01-19 19:40 +0200 http://bitbucket.org/pypy/pypy/changeset/d0469f22d0e8/ Log: Fixes diff --git a/pypy/jit/backend/x86/assembler.py b/pypy/jit/backend/x86/assembler.py --- a/pypy/jit/backend/x86/assembler.py +++ b/pypy/jit/backend/x86/assembler.py @@ -213,6 +213,8 @@ # nursery_free_adr = self.cpu.gc_ll_descr.get_nursery_free_addr() mc.MOV(edi, heap(nursery_free_adr)) # load this in EDX + # clear the gc pattern + mc.MOV_bi(ofs, imm(0)) mc.RET() # # If the slowpath malloc failed, we raise a MemoryError that @@ -2182,7 +2184,7 @@ else: vloc = imm(0) self._emit_call(imm(descr._x86_function_addr), - [argloc, vloc], 0, tmp=eax) + [argloc], 0, tmp=eax) if op.result is None: assert result_loc is None value = self.cpu.done_with_this_frame_descr_void @@ -2202,7 +2204,7 @@ value = rffi.cast(lltype.Signed, cast_instance_to_gcref(value)) base_ofs = self.cpu.get_baseofs_of_frame_field() ofs = self.cpu.get_ofs_of_frame_field('jf_descr') - self.mc.CMP_mi((eax.value, base_ofs + ofs), value) + self.mc.CMP_mi((eax.value, base_ofs + ofs), value) # patched later self.mc.J_il8(rx86.Conditions['E'], 0) # goto B if we get 'done_with_this_frame' je_location = self.mc.get_relative_pos() @@ -2211,7 +2213,7 @@ assert jd is not None asm_helper_adr = self.cpu.cast_adr_to_int(jd.assembler_helper_adr) self._emit_call(imm(asm_helper_adr), - [eax, imm0], 0, tmp=ecx) + [eax, vloc], 0, tmp=ecx) if IS_X86_32 and isinstance(result_loc, StackLoc) and result_loc.type == FLOAT: self.mc.FSTPL_b(result_loc.value) #else: result_loc is already either eax or None, checked below From noreply at buildbot.pypy.org Sat Jan 19 21:12:05 2013 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 19 Jan 2013 21:12:05 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: whoops Message-ID: <20130119201205.3AC081C01D2@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: missing-ndarray-attributes Changeset: r60213:59d998a98dc1 Date: 2013-01-18 13:15 +0200 http://bitbucket.org/pypy/pypy/changeset/59d998a98dc1/ Log: whoops diff --git a/pypy/rlib/listsort.py b/pypy/rlib/listsort.py --- a/pypy/rlib/listsort.py +++ b/pypy/rlib/listsort.py @@ -517,7 +517,6 @@ def merge_force_collapse(self): p = self.pending while len(p) > 1: - xxx if len(p) >= 3 and p[-3].len < p[-1].len: self.merge_at(-3) else: From noreply at buildbot.pypy.org Sat Jan 19 21:12:06 2013 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 19 Jan 2013 21:12:06 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: implement arg_gettitem_slice Message-ID: <20130119201206.BD3FC1C02D9@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: missing-ndarray-attributes Changeset: r60214:beb9be493502 Date: 2013-01-19 22:11 +0200 http://bitbucket.org/pypy/pypy/changeset/beb9be493502/ Log: implement arg_gettitem_slice diff --git a/pypy/module/micronumpy/arrayimpl/sort.py b/pypy/module/micronumpy/arrayimpl/sort.py --- a/pypy/module/micronumpy/arrayimpl/sort.py +++ b/pypy/module/micronumpy/arrayimpl/sort.py @@ -6,8 +6,8 @@ from pypy.rpython.lltypesystem import rffi, lltype from pypy.rlib.listsort import make_timsort_class from pypy.rlib.objectmodel import specialize -from pypy.rlib.rawstorage import raw_storage_getitem, raw_storage_setitem - +from pypy.rlib.rawstorage import raw_storage_getitem, raw_storage_setitem, \ + free_raw_storage, alloc_raw_storage from pypy.interpreter.error import OperationError from pypy.module.micronumpy.base import W_NDimArray from pypy.module.micronumpy import interp_dtype, types @@ -29,7 +29,6 @@ self.size = size self.values = values self.indexes = indexes - self.start = start def getitem(self, item): v = raw_storage_getitem(TP, self.values, item * self.stride_size @@ -44,6 +43,18 @@ self.start, rffi.cast(TP, item[0])) raw_storage_setitem(self.indexes, idx * self.index_stride_size + self.index_start, item[1]) + class ArgArrayRepWithStorage(ArgArrayRepresentation): + def __init__(self, index_stride_size, stride_size, size): + start = 0 + dtype = interp_dtype.get_dtype_cache(space).w_longdtype + self.indexes = dtype.itemtype.malloc(size*dtype.get_size()) + self.values = alloc_raw_storage(size*rffi.sizeof(TP), track_allocation=False) + ArgArrayRepresentation.__init__(self, index_stride_size, stride_size, + size, self.values, self.indexes, start, start) + + def __del__(self): + free_raw_storage(self.indexes, track_allocation=False) + free_raw_storage(self.values, track_allocation=False) def arg_getitem(lst, item): return lst.getitem(item) @@ -55,7 +66,11 @@ return lst.size def arg_getitem_slice(lst, start, stop): - xxx + retval = ArgArrayRepWithStorage(lst.index_stride_size, lst.stride_size, + stop-start) + for i in range(stop-start): + retval.setitem(i, lst.getitem(i+start)) + return retval def arg_lt(a, b): return a[0] < b[0] diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -2344,7 +2344,10 @@ from _numpypy import array a = array([[4, 2], [1, 3]]) assert (a.argsort() == [[1, 0], [0, 1]]).all() - #trigger timsort run mode + a = array(range(10) + range(10) + range(10)) + b = a.argsort() + assert (b[:3] == [0, 10, 20]).all() + #trigger timsort 'run' mode which calls arg_getitem_slice a = array(range(100) + range(100) + range(100)) b = a.argsort() assert (b[:3] == [0, 100, 200]).all() From noreply at buildbot.pypy.org Sat Jan 19 21:26:05 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Sat, 19 Jan 2013 21:26:05 +0100 (CET) Subject: [pypy-commit] pypy inline-virtualref: Begin hacking, doesn't work at all. Message-ID: <20130119202605.4FD941C02D9@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: inline-virtualref Changeset: r60215:a62651601098 Date: 2013-01-19 12:25 -0800 http://bitbucket.org/pypy/pypy/changeset/a62651601098/ Log: Begin hacking, doesn't work at all. diff --git a/pypy/jit/codewriter/jtransform.py b/pypy/jit/codewriter/jtransform.py --- a/pypy/jit/codewriter/jtransform.py +++ b/pypy/jit/codewriter/jtransform.py @@ -1745,13 +1745,6 @@ assert False, 'unsupported oopspec: %s' % oopspec_name return self._handle_oopspec_call(op, args, oopspecindex, extraeffect) - def rewrite_op_jit_force_virtual(self, op): - return self._do_builtin_call(op) - - def rewrite_op_jit_is_virtual(self, op): - raise Exception, ( - "'vref.virtual' should not be used from jit-visible code") - def rewrite_op_jit_force_virtualizable(self, op): # this one is for virtualizables vinfo = self.get_vinfo(op.args[0]) diff --git a/pypy/jit/codewriter/support.py b/pypy/jit/codewriter/support.py --- a/pypy/jit/codewriter/support.py +++ b/pypy/jit/codewriter/support.py @@ -224,9 +224,6 @@ def _ll_1_gc_id(ptr): return llop.gc_id(lltype.Signed, ptr) -def _ll_1_jit_force_virtual(inst): - return llop.jit_force_virtual(lltype.typeOf(inst), inst) - def _ll_2_int_floordiv_ovf_zer(x, y): if y == 0: diff --git a/pypy/jit/metainterp/virtualref.py b/pypy/jit/metainterp/virtualref.py --- a/pypy/jit/metainterp/virtualref.py +++ b/pypy/jit/metainterp/virtualref.py @@ -1,8 +1,11 @@ +from pypy.annotation import model as annmodel +from pypy.jit.codewriter import heaptracker +from pypy.jit.metainterp import history +from pypy.rlib.jit import InvalidVirtualRef +from pypy.rpython.annlowlevel import MixLevelHelperAnnotator +from pypy.rpython.lltypesystem import lltype, llmemory, rclass from pypy.rpython.rmodel import inputconst, log -from pypy.rpython.lltypesystem import lltype, llmemory, rclass -from pypy.jit.metainterp import history -from pypy.jit.codewriter import heaptracker -from pypy.rlib.jit import InvalidVirtualRef + class VirtualRefInfo: @@ -125,30 +128,32 @@ # ____________________________________________________________ def get_force_virtual_fnptr(self): - # def force_virtual_if_necessary(inst): if not inst or inst.typeptr != self.jit_virtual_ref_vtable: return inst # common, fast case return self.force_virtual(inst) - # + FUNC = lltype.FuncType([rclass.OBJECTPTR], rclass.OBJECTPTR) - funcptr = self.warmrunnerdesc.helper_func( - lltype.Ptr(FUNC), - force_virtual_if_necessary) - return inputconst(lltype.typeOf(funcptr), funcptr) + args_s = [annmodel.lltype_to_annotation(v) for v in FUNC.ARGS] + s_result = annmodel.lltype_to_annotation(FUNC.RESULT) + mixlevelann = MixLevelHelperAnnotator(self.warmrunnerdesc.rtyper) + c_func = mixlevelann.constfunc(force_virtual_if_necessary, args_s, s_result) + mixlevelann.finish() + return inputconst(lltype.typeOf(c_func.value), c_func.value) def get_is_virtual_fnptr(self): - # def is_virtual(inst): if not inst: return False return inst.typeptr == self.jit_virtual_ref_vtable - # + FUNC = lltype.FuncType([rclass.OBJECTPTR], lltype.Bool) funcptr = self.warmrunnerdesc.helper_func(lltype.Ptr(FUNC), is_virtual) return inputconst(lltype.typeOf(funcptr), funcptr) def force_virtual(self, inst): + from pypy.jit.metainterp.compile import ResumeGuardForcedDescr + vref = lltype.cast_pointer(lltype.Ptr(self.JIT_VIRTUAL_REF), inst) token = vref.virtual_token if token != self.TOKEN_NONE: @@ -161,7 +166,6 @@ vref.virtual_token = self.TOKEN_NONE else: assert not vref.forced - from pypy.jit.metainterp.compile import ResumeGuardForcedDescr ResumeGuardForcedDescr.force_now(self.cpu, token) assert vref.virtual_token == self.TOKEN_NONE assert vref.forced diff --git a/pypy/jit/metainterp/warmspot.py b/pypy/jit/metainterp/warmspot.py --- a/pypy/jit/metainterp/warmspot.py +++ b/pypy/jit/metainterp/warmspot.py @@ -218,10 +218,10 @@ verbose = False # not self.cpu.translate_support_code self.rewrite_access_helpers() + self.rewrite_force_virtual(vrefinfo) self.codewriter.make_jitcodes(verbose=verbose) self.rewrite_can_enter_jits() self.rewrite_set_param_and_get_stats() - self.rewrite_force_virtual(vrefinfo) self.rewrite_force_quasi_immutable() self.add_finish() self.metainterp_sd.finish_setup(self.codewriter) From noreply at buildbot.pypy.org Sat Jan 19 21:32:21 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Sat, 19 Jan 2013 21:32:21 +0100 (CET) Subject: [pypy-commit] pypy inline-virtualref: simplify, doesn't fix anything Message-ID: <20130119203221.E19781C02D9@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: inline-virtualref Changeset: r60216:7484e3c524fd Date: 2013-01-19 12:32 -0800 http://bitbucket.org/pypy/pypy/changeset/7484e3c524fd/ Log: simplify, doesn't fix anything diff --git a/pypy/jit/metainterp/virtualref.py b/pypy/jit/metainterp/virtualref.py --- a/pypy/jit/metainterp/virtualref.py +++ b/pypy/jit/metainterp/virtualref.py @@ -139,7 +139,7 @@ mixlevelann = MixLevelHelperAnnotator(self.warmrunnerdesc.rtyper) c_func = mixlevelann.constfunc(force_virtual_if_necessary, args_s, s_result) mixlevelann.finish() - return inputconst(lltype.typeOf(c_func.value), c_func.value) + return c_func def get_is_virtual_fnptr(self): def is_virtual(inst): From noreply at buildbot.pypy.org Sat Jan 19 21:56:15 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Sat, 19 Jan 2013 21:56:15 +0100 (CET) Subject: [pypy-commit] pypy py3k: woops, fix translation Message-ID: <20130119205615.D3CB91C02D9@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60217:daa01948e092 Date: 2013-01-19 12:55 -0800 http://bitbucket.org/pypy/pypy/changeset/daa01948e092/ Log: woops, fix translation diff --git a/pypy/objspace/descroperation.py b/pypy/objspace/descroperation.py --- a/pypy/objspace/descroperation.py +++ b/pypy/objspace/descroperation.py @@ -566,6 +566,7 @@ # test_class if not e.match(space, space.w_AttributeError): raise + return None else: w_res = space.call_function(w_impl, w_obj2) if _check_notimplemented(space, w_res): From noreply at buildbot.pypy.org Sat Jan 19 21:59:24 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 19 Jan 2013 21:59:24 +0100 (CET) Subject: [pypy-commit] pypy default: Fix (forgot this file, same fix as cb0037c8f67c) Message-ID: <20130119205924.BC2CD1C02D9@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r60218:0d36011c0987 Date: 2013-01-19 21:59 +0100 http://bitbucket.org/pypy/pypy/changeset/0d36011c0987/ Log: Fix (forgot this file, same fix as cb0037c8f67c) diff --git a/pypy/jit/backend/test/test_ll_random.py b/pypy/jit/backend/test/test_ll_random.py --- a/pypy/jit/backend/test/test_ll_random.py +++ b/pypy/jit/backend/test/test_ll_random.py @@ -4,7 +4,6 @@ from pypy.jit.metainterp.resoperation import ResOperation, rop from pypy.jit.metainterp.history import ConstInt, ConstPtr from pypy.jit.metainterp.history import BoxPtr, BoxInt -from pypy.jit.metainterp.history import BasicFailDescr from pypy.jit.codewriter import heaptracker from pypy.jit.codewriter.effectinfo import EffectInfo from pypy.rpython.annlowlevel import llhelper @@ -556,7 +555,7 @@ descr = self.getcalldescr(builder, TP) self.put(builder, args, descr) op = ResOperation(rop.GUARD_NO_EXCEPTION, [], None, - descr=BasicFailDescr()) + descr=builder.getfaildescr()) op.setfailargs(fail_subset) builder.loop.operations.append(op) @@ -578,7 +577,7 @@ _, vtableptr = builder.get_random_structure_type_and_vtable(r) exc_box = ConstAddr(llmemory.cast_ptr_to_adr(vtableptr), builder.cpu) op = ResOperation(rop.GUARD_EXCEPTION, [exc_box], BoxPtr(), - descr=BasicFailDescr()) + descr=builder.getfaildescr()) op.setfailargs(builder.subset_of_intvars(r)) op._exc_box = None builder.should_fail_by = op @@ -599,7 +598,7 @@ self.put(builder, args, descr) exc_box = ConstAddr(llmemory.cast_ptr_to_adr(exc), builder.cpu) op = ResOperation(rop.GUARD_EXCEPTION, [exc_box], BoxPtr(), - descr=BasicFailDescr()) + descr=builder.getfaildescr()) op.setfailargs(fail_subset) builder.loop.operations.append(op) @@ -615,7 +614,7 @@ descr = self.getcalldescr(builder, TP) self.put(builder, args, descr) op = ResOperation(rop.GUARD_NO_EXCEPTION, [], BoxPtr(), - descr=BasicFailDescr()) + descr=builder.getfaildescr()) op._exc_box = ConstAddr(llmemory.cast_ptr_to_adr(exc), builder.cpu) op.setfailargs(builder.subset_of_intvars(r)) builder.should_fail_by = op @@ -639,7 +638,7 @@ break other_box = ConstAddr(llmemory.cast_ptr_to_adr(vtableptr), builder.cpu) op = ResOperation(rop.GUARD_EXCEPTION, [other_box], BoxPtr(), - descr=BasicFailDescr()) + descr=builder.getfaildescr()) op._exc_box = ConstAddr(llmemory.cast_ptr_to_adr(exc), builder.cpu) op.setfailargs(builder.subset_of_intvars(r)) builder.should_fail_by = op From noreply at buildbot.pypy.org Sat Jan 19 22:28:13 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Sat, 19 Jan 2013 22:28:13 +0100 (CET) Subject: [pypy-commit] pypy inline-virtualref: finally write a test Message-ID: <20130119212813.D1A4A1C01D2@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: inline-virtualref Changeset: r60219:6d12a7821174 Date: 2013-01-19 13:27 -0800 http://bitbucket.org/pypy/pypy/changeset/6d12a7821174/ Log: finally write a test diff --git a/pypy/jit/metainterp/test/test_virtualref.py b/pypy/jit/metainterp/test/test_virtualref.py --- a/pypy/jit/metainterp/test/test_virtualref.py +++ b/pypy/jit/metainterp/test/test_virtualref.py @@ -656,6 +656,31 @@ res = self.meta_interp(f, [10]) assert res == 0 + def test_force_virtual_vref(self): + myjitdriver = JitDriver(greens=[], reds=['n', 'ec']) + + class ExecutionContext(object): + pass + + class Frame(object): + pass + + def f(n): + ec = ExecutionContext() + while n > 0: + myjitdriver.jit_merge_point(n=n, ec=ec) + frame = Frame() + ec.topframeref = virtual_ref(frame) + ec.topframeref() + frame_vref = ec.topframeref + ec.topframeref = vref_None + virtual_ref_finish(frame_vref, frame) + n -= 1 + return n + res = self.meta_interp(f, [10]) + assert res == 0 + self.check_resops({'int_sub': 1, 'int_gt': 1, 'jump': 1}) + class TestLLtype(VRefTests, LLJitMixin): pass From noreply at buildbot.pypy.org Sat Jan 19 22:46:30 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Sat, 19 Jan 2013 22:46:30 +0100 (CET) Subject: [pypy-commit] pypy inline-virtualref: hacks so this test will run, it sitll doesn't run Message-ID: <20130119214630.A8D4C1C01D2@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: inline-virtualref Changeset: r60220:12e302fb1c07 Date: 2013-01-19 13:46 -0800 http://bitbucket.org/pypy/pypy/changeset/12e302fb1c07/ Log: hacks so this test will run, it sitll doesn't run diff --git a/pypy/jit/metainterp/test/support.py b/pypy/jit/metainterp/test/support.py --- a/pypy/jit/metainterp/test/support.py +++ b/pypy/jit/metainterp/test/support.py @@ -1,5 +1,7 @@ +import sys -import py, sys +import py + from pypy.rpython.lltypesystem import lltype, llmemory from pypy.rpython.ootypesystem import ootype from pypy.jit.backend.llgraph import runner @@ -13,10 +15,16 @@ from pypy.translator.backendopt.all import backend_optimizations -def _get_jitcodes(testself, CPUClass, func, values, type_system, - supports_longlong=False, translationoptions={}, **kwds): +def _get_rtyper(func, values, type_system, translationoptions={}): from pypy.jit.codewriter import support + func._jit_unroll_safe_ = True + return support.annotate(func, values, type_system=type_system, + translationoptions=translationoptions) + + +def _get_jitcodes(testself, CPUClass, rtyper, supports_longlong=False, + make_jitcodes=True, **kwds): class FakeJitCell(object): __product_token = None def get_procedure_token(self): @@ -47,11 +55,7 @@ if kwds.pop('disable_optimizations', False): FakeWarmRunnerState.enable_opts = {} - func._jit_unroll_safe_ = True - rtyper = support.annotate(func, values, type_system=type_system, - translationoptions=translationoptions) graphs = rtyper.annotator.translator.graphs - testself.all_graphs = graphs result_kind = history.getkind(graphs[0].getreturnvar().concretetype)[0] class FakeJitDriverSD: @@ -79,8 +83,9 @@ FakeJitDriverSD.warmstate = testself.warmrunnerstate if hasattr(testself, 'finish_setup_for_interp_operations'): testself.finish_setup_for_interp_operations() - # - cw.make_jitcodes(verbose=True) + + if make_jitcodes: + cw.make_jitcodes(verbose=True) def _run_with_blackhole(testself, args): from pypy.jit.metainterp.blackhole import BlackholeInterpBuilder @@ -209,7 +214,8 @@ def interp_operations(self, f, args, **kwds): # get the JitCodes for the function f - _get_jitcodes(self, self.CPUClass, f, args, self.type_system, **kwds) + rtyper = _get_rtyper(f, args, self.type_system) + _get_jitcodes(self, self.CPUClass, rtyper, **kwds) # try to run it with blackhole.py result1 = _run_with_blackhole(self, args) # try to run it with pyjitpl.py diff --git a/pypy/jit/metainterp/test/test_virtualref.py b/pypy/jit/metainterp/test/test_virtualref.py --- a/pypy/jit/metainterp/test/test_virtualref.py +++ b/pypy/jit/metainterp/test/test_virtualref.py @@ -1,11 +1,11 @@ import py -from pypy.rpython.lltypesystem import lltype, llmemory, lloperation +from pypy.rpython.lltypesystem import lltype, lloperation from pypy.rpython.exceptiondata import UnknownException from pypy.rlib.jit import JitDriver, dont_look_inside, vref_None from pypy.rlib.jit import virtual_ref, virtual_ref_finish, InvalidVirtualRef from pypy.rlib.jit import non_virtual_ref from pypy.rlib.objectmodel import compute_unique_id -from pypy.jit.metainterp.test.support import LLJitMixin, OOJitMixin, _get_jitcodes +from pypy.jit.metainterp.test.support import LLJitMixin, _get_jitcodes, _get_rtyper from pypy.jit.metainterp.resoperation import rop from pypy.jit.metainterp.virtualref import VirtualRefInfo @@ -21,21 +21,23 @@ def test_rewrite_graphs(self): class X: pass + def fn(): x = X() vref = virtual_ref(x) x1 = vref() # jit_force_virtual virtual_ref_finish(vref, x) # - _get_jitcodes(self, self.CPUClass, fn, [], self.type_system) - graph = self.all_graphs[0] + rtyper = _get_rtyper(fn, [], self.type_system) + graph = rtyper.annotator.translator.graphs[0] + _get_jitcodes(self, self.CPUClass, rtyper, make_jitcodes=False) + self.vrefinfo.replace_force_virtual_with_call([graph]) assert graph.name == 'fn' - self.vrefinfo.replace_force_virtual_with_call([graph]) - # + def check_call(op, fname): assert op.opname == 'direct_call' assert op.args[0].value._obj._name == fname - # + ops = [op for block, op in graph.iterblockops()] check_call(ops[-3], 'virtual_ref') check_call(ops[-2], 'force_virtual_if_necessary') diff --git a/pypy/jit/metainterp/virtualref.py b/pypy/jit/metainterp/virtualref.py --- a/pypy/jit/metainterp/virtualref.py +++ b/pypy/jit/metainterp/virtualref.py @@ -1,14 +1,14 @@ from pypy.annotation import model as annmodel from pypy.jit.codewriter import heaptracker from pypy.jit.metainterp import history +from pypy.objspace.flow.model import Constant from pypy.rlib.jit import InvalidVirtualRef from pypy.rpython.annlowlevel import MixLevelHelperAnnotator from pypy.rpython.lltypesystem import lltype, llmemory, rclass from pypy.rpython.rmodel import inputconst, log -class VirtualRefInfo: - +class VirtualRefInfo(object): def __init__(self, warmrunnerdesc): self.warmrunnerdesc = warmrunnerdesc self.cpu = warmrunnerdesc.cpu From noreply at buildbot.pypy.org Sat Jan 19 23:21:19 2013 From: noreply at buildbot.pypy.org (mattip) Date: Sat, 19 Jan 2013 23:21:19 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: why is this needed? Message-ID: <20130119222119.4E9411C01D2@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: missing-ndarray-attributes Changeset: r60221:c158cea62fa8 Date: 2013-01-20 00:20 +0200 http://bitbucket.org/pypy/pypy/changeset/c158cea62fa8/ Log: why is this needed? diff --git a/pypy/rlib/rarithmetic.py b/pypy/rlib/rarithmetic.py --- a/pypy/rlib/rarithmetic.py +++ b/pypy/rlib/rarithmetic.py @@ -132,6 +132,7 @@ return r_longlonglong(n) def widen(n): + assert not isinstance(n, r_longfloat) from pypy.rpython.lltypesystem import lltype if _should_widen_type(lltype.typeOf(n)): return intmask(n) From noreply at buildbot.pypy.org Sun Jan 20 10:13:50 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sun, 20 Jan 2013 10:13:50 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Moved /goal/test2 to /pypy/interpreter/ Message-ID: <20130120091350.B84421C00F9@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: split-rpython Changeset: r60222:1603b693f4c2 Date: 2013-01-20 10:13 +0100 http://bitbucket.org/pypy/pypy/changeset/1603b693f4c2/ Log: Moved /goal/test2 to /pypy/interpreter/ diff --git a/goal/test2/__init__.py b/pypy/interpreter/test2/__init__.py rename from goal/test2/__init__.py rename to pypy/interpreter/test2/__init__.py diff --git a/goal/test2/mymodule.py b/pypy/interpreter/test2/mymodule.py rename from goal/test2/mymodule.py rename to pypy/interpreter/test2/mymodule.py diff --git a/goal/test2/test_app_main.py b/pypy/interpreter/test2/test_app_main.py rename from goal/test2/test_app_main.py rename to pypy/interpreter/test2/test_app_main.py diff --git a/goal/test2/test_targetpypy.py b/pypy/interpreter/test2/test_targetpypy.py rename from goal/test2/test_targetpypy.py rename to pypy/interpreter/test2/test_targetpypy.py From noreply at buildbot.pypy.org Sun Jan 20 10:18:59 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 20 Jan 2013 10:18:59 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Since we run stuff in the subprocess we can just kill expect here Message-ID: <20130120091859.5D4511C00F9@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: split-rpython Changeset: r60223:d4f5fc22f903 Date: 2013-01-20 11:06 +0200 http://bitbucket.org/pypy/pypy/changeset/d4f5fc22f903/ Log: Since we run stuff in the subprocess we can just kill expect here diff --git a/pypy/module/_minimal_curses/test/test_curses.py b/pypy/module/_minimal_curses/test/test_curses.py --- a/pypy/module/_minimal_curses/test/test_curses.py +++ b/pypy/module/_minimal_curses/test/test_curses.py @@ -71,7 +71,7 @@ child = self.spawn(['--withmod-_minimal_curses', str(f)]) child.expect('ok!') -class ExpectTestCCurses(object): +class TestCCurses(object): """ Test compiled version """ def test_csetupterm(self): From noreply at buildbot.pypy.org Sun Jan 20 10:19:00 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 20 Jan 2013 10:19:00 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: Another gross hack bites the dust. Message-ID: <20130120091900.94BF91C00F9@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: split-rpython Changeset: r60224:02d7d2319c9e Date: 2013-01-20 11:18 +0200 http://bitbucket.org/pypy/pypy/changeset/02d7d2319c9e/ Log: Another gross hack bites the dust. diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -101,11 +101,6 @@ return self.accept_regular_test() if name.startswith('AppTest'): return True - if name.startswith('ExpectTest'): - return True - #XXX todo - #if name.startswith('AppExpectTest'): - # return True return False def makeitem(self, name, obj): diff --git a/rpython/rtyper/module/test/test_ll_os.py b/rpython/rtyper/module/test/test_ll_os.py --- a/rpython/rtyper/module/test/test_ll_os.py +++ b/rpython/rtyper/module/test/test_ll_os.py @@ -1,9 +1,10 @@ import os -from py.path import local from rpython.tool.udir import udir from rpython.translator.c.test.test_genc import compile -from rpython.rtyper.module import ll_os #has side effect of registering functions +from rpython.rtyper.module import ll_os +#has side effect of registering functions +from rpython.tool.pytest.expecttest import ExpectTest from rpython.rtyper import extregistry import errno @@ -11,7 +12,6 @@ import py def getllimpl(fn): - from rpython.rtyper.module import ll_os return extregistry.lookup(fn).lltypeimpl def test_access(): @@ -276,25 +276,27 @@ assert f(-1) == False -class ExpectTestOs: +class TestOsExpect(ExpectTest): def setup_class(cls): if not hasattr(os, 'ttyname'): py.test.skip("no ttyname") def test_ttyname(self): - import os - import py - from rpython.rtyper.test.test_llinterp import interpret + def f(): + import os + import py + from rpython.rtyper.test.test_llinterp import interpret - def ll_to_string(s): - return ''.join(s.chars) - - def f(num): - try: - return os.ttyname(num) - except OSError: - return '' + def ll_to_string(s): + return ''.join(s.chars) - assert ll_to_string(interpret(f, [0])) == f(0) - assert ll_to_string(interpret(f, [338])) == '' + def f(num): + try: + return os.ttyname(num) + except OSError: + return '' + assert ll_to_string(interpret(f, [0])) == f(0) + assert ll_to_string(interpret(f, [338])) == '' + + self.run_test(f) diff --git a/rpython/rtyper/module/test/test_ll_termios.py b/rpython/rtyper/module/test/test_ll_termios.py --- a/rpython/rtyper/module/test/test_ll_termios.py +++ b/rpython/rtyper/module/test/test_ll_termios.py @@ -1,5 +1,6 @@ import py, re, sys from rpython.tool.udir import udir +from rpython.tool.pytest.expecttest import ExpectTest # tests here are run as snippets through a pexpected python subprocess def setup_module(mod): @@ -88,15 +89,17 @@ fn = compile(runs_tcall, [], backendopt=False) self.run(fn, "ok") -class ExpectTestTermios(object): +class TestTermios(ExpectTest): def test_tcsetattr_icanon(self): - from rpython.rlib import rtermios - import termios - def check(fd, when, attributes): - count = len([i for i in attributes[-1] if isinstance(i, int)]) - assert count == 2 - termios.tcsetattr = check - attr = list(rtermios.tcgetattr(2)) - attr[3] |= termios.ICANON - rtermios.tcsetattr(2, termios.TCSANOW, attr) + def f(): + from rpython.rlib import rtermios + import termios + def check(fd, when, attributes): + count = len([i for i in attributes[-1] if isinstance(i, int)]) + assert count == 2 + termios.tcsetattr = check + attr = list(rtermios.tcgetattr(2)) + attr[3] |= termios.ICANON + rtermios.tcsetattr(2, termios.TCSANOW, attr) + self.run_test(f) diff --git a/rpython/tool/pytest/expecttest.py b/rpython/tool/pytest/expecttest.py --- a/rpython/tool/pytest/expecttest.py +++ b/rpython/tool/pytest/expecttest.py @@ -1,36 +1,11 @@ -# Collects and executes "Expect" tests. -# -# Classes which names start with "ExpectTest", are started in a -# separate process, and monitored by the pexpect module. This allows -# execution of dangerous code, which messes with the terminal for -# example. +# Executes "Expect" tests. import py -import os import sys -from rpython.tool.udir import udir -from pypy.conftest import pypydir #XXX +from tempfile import NamedTemporaryFile - -class ExpectTestMethod(py.test.collect.Function): - @staticmethod - def safe_name(target): - s = "_".join(target) - s = s.replace("()", "paren") - s = s.replace(".py", "") - s = s.replace(".", "_") - s = s.replace(os.sep, "_") - return s - - def safe_filename(self): - name = self.safe_name(self.listnames()) - num = 0 - while udir.join(name + '.py').check(): - num += 1 - name = self.safe_name(self.listnames()) + "_" + str(num) - return name + '.py' - +class ExpectTest(object): def _spawn(self, *args, **kwds): import pexpect kwds.setdefault('timeout', 600) @@ -41,41 +16,22 @@ def spawn(self, argv): return self._spawn(sys.executable, argv) - def runtest(self): - target = self.obj - import pexpect - source = py.code.Source(target)[1:].deindent() - filename = self.safe_filename() + def run_test(self, func): + try: + import pexpect + except ImportError: + py.test.skip("pexpect not found") + source = py.code.Source(func)[1:].deindent() + tmpfile = NamedTemporaryFile(suffix='.py') + fname = tmpfile.name + dir_to_insert = py.path.local(__file__).join('..', '..', '..', '..') source.lines = ['import sys', - 'sys.path.insert(0, %s)' % repr(os.path.dirname(pypydir)) + 'sys.path.insert(0, %s)' % repr(str(dir_to_insert)) ] + source.lines - source.lines.append('print "%s ok!"' % filename) - f = udir.join(filename) + source.lines.append('print "%s ok!"' % fname) + f = py.path.local(fname) f.write(source) # run target in the guarded environment child = self.spawn([str(f)]) import re - child.expect(re.escape(filename + " ok!")) - - -class ExpectClassInstance(py.test.collect.Instance): - Function = ExpectTestMethod - - -class ExpectClassCollector(py.test.collect.Class): - Instance = ExpectClassInstance - - def setup(self): - super(ExpectClassCollector, self).setup() - try: - import pexpect - except ImportError: - py.test.skip("pexpect not found") - - - at py.test.mark.tryfirst -def pytest_pycollect_makeitem(collector, name, obj): - if py.std.inspect.isclass(obj) and name.startswith('ExpectTest'): - #XXX: in conftest we had a rundirect option - #XXX: kill expecttest for a more explicit way - return ExpectClassCollector(name, parent=collector) + child.expect(re.escape(fname + " ok!")) diff --git a/rpython/tool/pytest/test/test_expecttest.py b/rpython/tool/pytest/test/test_expecttest.py --- a/rpython/tool/pytest/test/test_expecttest.py +++ b/rpython/tool/pytest/test/test_expecttest.py @@ -1,53 +1,9 @@ -import py -import rpython +from rpython.tool.pytest.expecttest import ExpectTest -pytest_plugins = "pytest_pytester" - -conftestpath = py.path.local(rpython.__file__).dirpath("conftest.py") - - -def test_expectcollect(testdir): - py.test.importorskip("pexpect") - conftestpath.copy(testdir.tmpdir) - sorter = testdir.inline_runsource(""" - class ExpectTestOne: - def test_one(self): - pass - """) - passed, skipped, failed = sorter.countoutcomes() - assert passed == 1 - - -def test_safename(): - from rpython.tool.pytest.expecttest import ExpectTestMethod - - safe_name = ExpectTestMethod.safe_name - assert safe_name(['pypy', 'tool', 'test', 'test_pytestsupport.py', - 'ExpectTest', '()', 'test_one']) == \ - 'pypy_tool_test_test_pytestsupport_ExpectTest_paren_test_one' - - -def test_safe_filename(testdir): - py.test.importorskip("pexpect") - conftestpath.copy(testdir.tmpdir) - sorter = testdir.inline_runsource(""" - class ExpectTestOne: - def test_one(self): - pass - """) - evlist = sorter.getcalls("pytest_runtest_makereport") - ev = [x for x in evlist if x.call.when == "call"][0] - print ev - sfn = ev.item.safe_filename() - print sfn - assert sfn == 'test_safe_filename_test_safe_filename_ExpectTestOne_paren_test_one_1.py' - - -class ExpectTest: +class TestExpect(ExpectTest): def test_one(self): - import os - import sys - assert os.ttyname(sys.stdin.fileno()) - - def test_two(self): - import pypy + def func(): + import os + import sys + assert os.ttyname(sys.stdin.fileno()) + self.run_test(func) From noreply at buildbot.pypy.org Sun Jan 20 10:19:01 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 20 Jan 2013 10:19:01 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: merge Message-ID: <20130120091901.CF4F91C00F9@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: split-rpython Changeset: r60225:da2336fdc01f Date: 2013-01-20 11:18 +0200 http://bitbucket.org/pypy/pypy/changeset/da2336fdc01f/ Log: merge diff --git a/goal/test2/__init__.py b/pypy/interpreter/test2/__init__.py rename from goal/test2/__init__.py rename to pypy/interpreter/test2/__init__.py diff --git a/goal/test2/mymodule.py b/pypy/interpreter/test2/mymodule.py rename from goal/test2/mymodule.py rename to pypy/interpreter/test2/mymodule.py diff --git a/goal/test2/test_app_main.py b/pypy/interpreter/test2/test_app_main.py rename from goal/test2/test_app_main.py rename to pypy/interpreter/test2/test_app_main.py diff --git a/goal/test2/test_targetpypy.py b/pypy/interpreter/test2/test_targetpypy.py rename from goal/test2/test_targetpypy.py rename to pypy/interpreter/test2/test_targetpypy.py From noreply at buildbot.pypy.org Sun Jan 20 11:03:06 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 20 Jan 2013 11:03:06 +0100 (CET) Subject: [pypy-commit] pypy default: Syntax test and fix, part 1. Message-ID: <20130120100306.CCFE71C02D9@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r60226:8a7426025407 Date: 2013-01-20 10:57 +0100 http://bitbucket.org/pypy/pypy/changeset/8a7426025407/ Log: Syntax test and fix, part 1. diff --git a/pypy/interpreter/astcompiler/asthelpers.py b/pypy/interpreter/astcompiler/asthelpers.py --- a/pypy/interpreter/astcompiler/asthelpers.py +++ b/pypy/interpreter/astcompiler/asthelpers.py @@ -43,7 +43,7 @@ if self.elts: for elt in self.elts: elt.set_context(ctx) - self.ctx = ctx + self.ctx = ctx class __extend__(ast.Attribute): diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py --- a/pypy/interpreter/astcompiler/test/test_compiler.py +++ b/pypy/interpreter/astcompiler/test/test_compiler.py @@ -782,6 +782,17 @@ source = "for x in y: pass\n" * 1000 compile_with_astcompiler(source, 'exec', self.space) + def test_assign_to_empty_list_1(self): + source = """if 1: + for i in range(5): + del [] + [] = () + [] = [] + [] = [] = [] + ok = 1 + """ + self.simple_test(source, 'ok', 1) + class AppTestCompiler: From noreply at buildbot.pypy.org Sun Jan 20 11:03:07 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 20 Jan 2013 11:03:07 +0100 (CET) Subject: [pypy-commit] pypy default: A second test, from issue #1364. Message-ID: <20130120100307.F32671C02D9@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r60227:0bbf95bbb3b1 Date: 2013-01-20 11:02 +0100 http://bitbucket.org/pypy/pypy/changeset/0bbf95bbb3b1/ Log: A second test, from issue #1364. diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py --- a/pypy/interpreter/astcompiler/test/test_compiler.py +++ b/pypy/interpreter/astcompiler/test/test_compiler.py @@ -793,6 +793,25 @@ """ self.simple_test(source, 'ok', 1) + def test_assign_to_empty_list_2(self): + source = """if 1: + for i in range(5): + try: [] = 1, 2, 3 + except ValueError: pass + else: raise AssertionError + try: [] = a = 1 + except TypeError: pass + else: raise AssertionError + try: [] = _ = iter(['foo']) + except ValueError: pass + else: raise AssertionError + try: [], _ = iter(['foo']), 1 + except ValueError: pass + else: raise AssertionError + ok = 1 + """ + self.simple_test(source, 'ok', 1) + class AppTestCompiler: From noreply at buildbot.pypy.org Sun Jan 20 11:50:43 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 20 Jan 2013 11:50:43 +0100 (CET) Subject: [pypy-commit] pypy split-rpython: close about-to-be-merged-branch Message-ID: <20130120105043.A7C7B1C1385@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: split-rpython Changeset: r60228:dd7c37f6318b Date: 2013-01-20 12:47 +0200 http://bitbucket.org/pypy/pypy/changeset/dd7c37f6318b/ Log: close about-to-be-merged-branch From noreply at buildbot.pypy.org Sun Jan 20 11:51:15 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 20 Jan 2013 11:51:15 +0100 (CET) Subject: [pypy-commit] pypy default: (Aquana) Merge split-rpython Message-ID: <20130120105115.51BDE1C1385@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r60229:5a31162e3a0a Date: 2013-01-20 12:48 +0200 http://bitbucket.org/pypy/pypy/changeset/5a31162e3a0a/ Log: (Aquana) Merge split-rpython This branch splits rpython and pypy into separate directories, hopefully separate repositories in the future. diff too long, truncating to 2000 out of 722771 lines diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -85,3 +85,4 @@ ^compiled ^.git/ ^release/ +^rpython/_cache$ diff --git a/dotviewer/test/test_translator.py b/dotviewer/test/test_translator.py --- a/dotviewer/test/test_translator.py +++ b/dotviewer/test/test_translator.py @@ -21,7 +21,7 @@ def test_annotated(): - from pypy.translator.interactive import Translation + from rpython.translator.interactive import Translation t = Translation(is_prime) t.annotate([int]) t.viewcg() diff --git a/goal/multibuild.py b/goal/multibuild.py new file mode 100644 --- /dev/null +++ b/goal/multibuild.py @@ -0,0 +1,127 @@ +from pypy.config.pypyoption import get_pypy_config +from rpython.translator.goal import translate +from rpython.translator.goal import targetpypystandalone +from rpython.translator.driver import TranslationDriver +import os, sys, traceback, random + +def longoptfromname(config, name): + from pypy.config.makerestdoc import get_cmdline + # begin horror + h, n = config._cfgimpl_get_home_by_path(name) + opt = getattr(h._cfgimpl_descr, n) + # end horror + cmdline = get_cmdline(opt.cmdline, name) + assert cmdline is not None + shortest_long_option = 'X'*1000 + for cmd in cmdline.split(): + if cmd.startswith('--') and len(cmd) < len(shortest_long_option): + shortest_long_option = cmd + return shortest_long_option + +def exe_name_from_options(config, opts): + from pypy.module.sys.version import svn_revision + + backend = config.translation.backend + if not backend: + backend = 'c' + rev = svn_revision() + + nameparts = [] + + for opt, v in opts.iteritems(): + if opt == 'translation.backend': + backend = v + optname = longoptfromname(config, opt).strip('-') + if v is False: + optname = 'no-' + optname + elif v is not True: + optname += '=' + str(v) + nameparts.append(optname) + + suffix = '' + if nameparts: + def k(s): + if s.startswith('no-'): + return s[3:] + else: + return s + nameparts.sort(key=k) + suffix = '-' + '-'.join(nameparts) + + return 'pypy-%s-%d%s'%(backend, rev, suffix) + +def _build(config, exe_name): + driver = TranslationDriver.from_targetspec( + targetpypystandalone.__dict__, + config=config) + driver.exe_name = exe_name + driver.compile() + +def build_pypy_with_options(basedir, opts): + config = get_pypy_config(translate.OVERRIDES, translating=True) + + try: + config.set(**opts) + except: + return exe_name_from_options(config, opts), "didn't configure" + + exe_name = os.path.join(basedir, exe_name_from_options(config, opts)) + + print exe_name, + sys.stdout.flush() + + pid = os.fork() + + if pid == 0: + logfile = open(exe_name + '-log', 'w') + davenull = os.open('/dev/null', os.O_RDONLY) + os.dup2(davenull, 0) + os.dup2(logfile.fileno(), 1) + os.dup2(logfile.fileno(), 2) + try: + try: + r = _build(config, exe_name) + finally: + logfile.close() + except: + os._exit(1) + else: + os._exit(0) + else: + pid, status = os.waitpid(pid, 0) + if status: + r = 'failed' + else: + r = 'succeeded' + print r + return exe_name, r + +def get_options(fname): + def gen_opts(sofar, remaining): + if not remaining: + yield sofar + else: + for (k, v) in remaining[0]: + d2 = sofar.copy() + d2[k] = v + for d in gen_opts(d2, remaining[1:]): + yield d + options = [] + for line in open(fname): + l = [] + optname, options_ = line.split(':') + options.append([(optname.strip(), eval(optval.strip())) for optval in options_.split(',')]) + return gen_opts({}, options) + + +if __name__ == '__main__': + basedir = sys.argv[1] + optionsfile = sys.argv[2] + results = [] + options = list(get_options(optionsfile)) + random.shuffle(options) + for opts in options: + results.append(build_pypy_with_options(basedir, opts)) + out = open(os.path.join(basedir, 'results'), 'w') + for exe, r in results: + print >>out, exe, r diff --git a/goal/targetnumpystandalone.py b/goal/targetnumpystandalone.py new file mode 100644 --- /dev/null +++ b/goal/targetnumpystandalone.py @@ -0,0 +1,43 @@ + +""" Usage: + +./targetnumpystandalone-c array_size + +Will execute a give numpy bytecode. Arrays will be ranges (in float) modulo 10, +constants would be consecutive starting from one. + +Bytecode should contain letters 'a' 'l' and 'f' so far and be correct +""" + +import time +from pypy.module.micronumpy.compile import numpy_compile +from rpython.jit.codewriter.policy import JitPolicy +from rpython.rtyper.annlowlevel import hlstr + +def entry_point(argv): + if len(argv) != 3: + print __doc__ + return 1 + try: + size = int(argv[2]) + except ValueError: + print "INVALID LITERAL FOR INT:", argv[2] + print __doc__ + return 3 + t0 = time.time() + main(argv[0], size) + print "bytecode:", argv[0], "size:", size + print "took:", time.time() - t0 + return 0 + +def main(bc, size): + if not isinstance(bc, str): + bc = hlstr(bc) # for tests + a = numpy_compile(bc, size) + a = a.compute() + +def target(*args): + return entry_point, None + +def jitpolicy(driver): + return JitPolicy() diff --git a/pypy/translator/goal/targetpypystandalone.py b/goal/targetpypystandalone.py rename from pypy/translator/goal/targetpypystandalone.py rename to goal/targetpypystandalone.py --- a/pypy/translator/goal/targetpypystandalone.py +++ b/goal/targetpypystandalone.py @@ -4,10 +4,11 @@ from pypy.interpreter import gateway from pypy.interpreter.error import OperationError -from pypy.translator.goal.ann_override import PyPyAnnotatorPolicy -from pypy.config.config import Config, to_optparse, make_dict, SUPPRESS_USAGE -from pypy.config.config import ConflictConfigError +from pypy.tool.ann_override import PyPyAnnotatorPolicy +from rpython.config.config import Config, to_optparse, make_dict, SUPPRESS_USAGE +from rpython.config.config import ConflictConfigError from pypy.tool.option import make_objspace +from pypy.conftest import pypydir thisdir = py.path.local(__file__).dirpath() @@ -30,7 +31,7 @@ def entry_point(argv): if withjit: - from pypy.jit.backend.hlinfo import highleveljitinfo + from rpython.jit.backend.hlinfo import highleveljitinfo highleveljitinfo.sys_executable = argv[0] #debug("entry point starting") @@ -42,7 +43,7 @@ # It works in Boehm and in the semispace or generational GCs # (but see comments in semispace.py:set_max_heap_size()). # At the moment this option exists mainly to support sandboxing. - from pypy.rlib import rgc + from rpython.rlib import rgc rgc.set_max_heap_size(int(argv[2])) argv = argv[:1] + argv[3:] try: @@ -207,7 +208,7 @@ # HACK: The ftruncate implementation in streamio.py which is used for the Win32 platform # is specific for the C backend and can't be generated on CLI or JVM. Because of that, # we have to patch it out. - from pypy.rlib import streamio + from rpython.rlib import streamio def ftruncate_win32_dummy(fd, size): pass def _setfd_binary_dummy(fd): pass streamio.ftruncate_win32 = ftruncate_win32_dummy @@ -227,7 +228,7 @@ space = make_objspace(config) # manually imports app_main.py - filename = os.path.join(this_dir, 'app_main.py') + filename = os.path.join(pypydir, 'interpreter', 'app_main.py') app = gateway.applevel(open(filename).read(), 'app_main.py', 'app_main') app.hidden_applevel = False w_dict = app.getwdict(space) diff --git a/lib_pypy/ctypes_config_cache/autopath.py b/lib_pypy/ctypes_config_cache/autopath.py deleted file mode 100644 --- a/lib_pypy/ctypes_config_cache/autopath.py +++ /dev/null @@ -1,131 +0,0 @@ -""" -self cloning, automatic path configuration - -copy this into any subdirectory of pypy from which scripts need -to be run, typically all of the test subdirs. -The idea is that any such script simply issues - - import autopath - -and this will make sure that the parent directory containing "pypy" -is in sys.path. - -If you modify the master "autopath.py" version (in pypy/tool/autopath.py) -you can directly run it which will copy itself on all autopath.py files -it finds under the pypy root directory. - -This module always provides these attributes: - - pypydir pypy root directory path - this_dir directory where this autopath.py resides - -""" - -def __dirinfo(part): - """ return (partdir, this_dir) and insert parent of partdir - into sys.path. If the parent directories don't have the part - an EnvironmentError is raised.""" - - import sys, os - try: - head = this_dir = os.path.realpath(os.path.dirname(__file__)) - except NameError: - head = this_dir = os.path.realpath(os.path.dirname(sys.argv[0])) - - error = None - while head: - partdir = head - head, tail = os.path.split(head) - if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') - if not os.path.exists(checkfile): - error = "Cannot find %r" % (os.path.normpath(checkfile),) - break - else: - error = "Cannot find the parent directory %r of the path %r" % ( - partdir, this_dir) - if not error: - # check for bogus end-of-line style (e.g. files checked out on - # Windows and moved to Unix) - f = open(__file__.replace('.pyc', '.py'), 'r') - data = f.read() - f.close() - if data.endswith('\r\n') or data.endswith('\r'): - error = ("Bad end-of-line style in the .py files. Typically " - "caused by a zip file or a checkout done on Windows and " - "moved to Unix or vice-versa.") - if error: - raise EnvironmentError("Invalid source tree - bogus checkout! " + - error) - - pypy_root = os.path.join(head, '') - try: - sys.path.remove(head) - except ValueError: - pass - sys.path.insert(0, head) - - munged = {} - for name, mod in sys.modules.items(): - if '.' in name: - continue - fn = getattr(mod, '__file__', None) - if not isinstance(fn, str): - continue - newname = os.path.splitext(os.path.basename(fn))[0] - if not newname.startswith(part + '.'): - continue - path = os.path.join(os.path.dirname(os.path.realpath(fn)), '') - if path.startswith(pypy_root) and newname != part: - modpaths = os.path.normpath(path[len(pypy_root):]).split(os.sep) - if newname != '__init__': - modpaths.append(newname) - modpath = '.'.join(modpaths) - if modpath not in sys.modules: - munged[modpath] = mod - - for name, mod in munged.iteritems(): - if name not in sys.modules: - sys.modules[name] = mod - if '.' in name: - prename = name[:name.rfind('.')] - postname = name[len(prename)+1:] - if prename not in sys.modules: - __import__(prename) - if not hasattr(sys.modules[prename], postname): - setattr(sys.modules[prename], postname, mod) - - return partdir, this_dir - -def __clone(): - """ clone master version of autopath.py into all subdirs """ - from os.path import join, walk - if not this_dir.endswith(join('pypy','tool')): - raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) - - - def sync_walker(arg, dirname, fnames): - if _myname in fnames: - fn = join(dirname, _myname) - f = open(fn, 'rwb+') - try: - if f.read() == arg: - print "checkok", fn - else: - print "syncing", fn - f = open(fn, 'w') - f.write(arg) - finally: - f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) - -_myname = 'autopath.py' - -# set guaranteed attributes - -pypydir, this_dir = __dirinfo('pypy') - -if __name__ == '__main__': - __clone() diff --git a/lib_pypy/ctypes_config_cache/dumpcache.py b/lib_pypy/ctypes_config_cache/dumpcache.py --- a/lib_pypy/ctypes_config_cache/dumpcache.py +++ b/lib_pypy/ctypes_config_cache/dumpcache.py @@ -1,6 +1,6 @@ import os from ctypes_configure import dumpcache -from pypy.jit.backend import detect_cpu +from rpython.jit.backend import detect_cpu def dumpcache2(basename, config): model = detect_cpu.autodetect_main_model_and_size() @@ -14,7 +14,7 @@ try: from __pypy__ import cpumodel except ImportError: - from pypy.jit.backend import detect_cpu + from rpython.jit.backend import detect_cpu cpumodel = detect_cpu.autodetect_main_model_and_size() # XXX relative import, should be removed together with # XXX the relative imports done e.g. by lib_pypy/pypy_test/test_hashlib diff --git a/lib_pypy/ctypes_config_cache/rebuild.py b/lib_pypy/ctypes_config_cache/rebuild.py --- a/lib_pypy/ctypes_config_cache/rebuild.py +++ b/lib_pypy/ctypes_config_cache/rebuild.py @@ -1,21 +1,15 @@ #! /usr/bin/env python # Run this script to rebuild all caches from the *.ctc.py files. -# hack: we cannot directly import autopath, as we are outside the pypy -# package. However, we pretend to be inside pypy/tool and manually run it, to -# get the correct path -import os.path -this_dir = os.path.dirname(__file__) -autopath_py = os.path.join(this_dir, '../../pypy/tool/autopath.py') -autopath_py = os.path.abspath(autopath_py) -execfile(autopath_py, dict(__name__='autopath', __file__=autopath_py)) +import os, sys -import os, sys +sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..'))) + import py _dirpath = os.path.dirname(__file__) or os.curdir -from pypy.tool.ansi_print import ansi_log +from rpython.tool.ansi_print import ansi_log log = py.log.Producer("ctypes_config_cache") py.log.setconsumer("ctypes_config_cache", ansi_log) @@ -31,7 +25,7 @@ sys.path[:] = path def try_rebuild(): - from pypy.jit.backend import detect_cpu + from rpython.jit.backend import detect_cpu model = detect_cpu.autodetect_main_model_and_size() # remove the files '_*_model_.py' left = {} diff --git a/lib_pypy/ctypes_config_cache/test/test_cache.py b/lib_pypy/ctypes_config_cache/test/test_cache.py --- a/lib_pypy/ctypes_config_cache/test/test_cache.py +++ b/lib_pypy/ctypes_config_cache/test/test_cache.py @@ -1,6 +1,6 @@ import py import sys, os -from pypy.tool.udir import udir +from rpython.tool.udir import udir dirpath = py.path.local(__file__).dirpath().dirpath() diff --git a/lib_pypy/pypy_test/test_dbm_extra.py b/lib_pypy/pypy_test/test_dbm_extra.py --- a/lib_pypy/pypy_test/test_dbm_extra.py +++ b/lib_pypy/pypy_test/test_dbm_extra.py @@ -1,6 +1,6 @@ from __future__ import absolute_import import py -from pypy.tool.udir import udir +from rpython.tool.udir import udir try: from lib_pypy import dbm except ImportError, e: diff --git a/lib_pypy/pypy_test/test_marshal_extra.py b/lib_pypy/pypy_test/test_marshal_extra.py --- a/lib_pypy/pypy_test/test_marshal_extra.py +++ b/lib_pypy/pypy_test/test_marshal_extra.py @@ -4,7 +4,7 @@ import marshal as cpy_marshal from lib_pypy import _marshal as marshal -from pypy.tool.udir import udir +from rpython.tool.udir import udir hello = "he" hello += "llo" diff --git a/pypy/annotation/annrpython.py b/pypy/annotation/annrpython.py deleted file mode 100644 --- a/pypy/annotation/annrpython.py +++ /dev/null @@ -1,676 +0,0 @@ -from __future__ import absolute_import - -import types -from pypy.tool.ansi_print import ansi_log -from pypy.tool.pairtype import pair -from pypy.tool.error import (format_blocked_annotation_error, - AnnotatorError, gather_error, ErrorWrapper) -from pypy.objspace.flow.model import (Variable, Constant, FunctionGraph, - c_last_exception, checkgraph) -from pypy.translator import simplify, transform -from pypy.annotation import model as annmodel, signature, unaryop, binaryop -from pypy.annotation.bookkeeper import Bookkeeper -import py -log = py.log.Producer("annrpython") -py.log.setconsumer("annrpython", ansi_log) - - -FAIL = object() - -class RPythonAnnotator(object): - """Block annotator for RPython. - See description in doc/translation.txt.""" - - def __init__(self, translator=None, policy=None, bookkeeper=None): - import pypy.rpython.ootypesystem.ooregistry # has side effects - import pypy.rpython.extfuncregistry # has side effects - import pypy.rlib.nonconst # has side effects - - if translator is None: - # interface for tests - from pypy.translator.translator import TranslationContext - translator = TranslationContext() - translator.annotator = self - self.translator = translator - self.pendingblocks = {} # map {block: graph-containing-it} - self.bindings = {} # map Variables to SomeValues - self.annotated = {} # set of blocks already seen - self.added_blocks = None # see processblock() below - self.links_followed = {} # set of links that have ever been followed - self.notify = {} # {block: {positions-to-reflow-from-when-done}} - self.fixed_graphs = {} # set of graphs not to annotate again - self.blocked_blocks = {} # set of {blocked_block: (graph, index)} - # --- the following information is recorded for debugging --- - self.blocked_graphs = {} # set of graphs that have blocked blocks - # --- end of debugging information --- - self.frozen = False - if policy is None: - from pypy.annotation.policy import AnnotatorPolicy - self.policy = AnnotatorPolicy() - else: - self.policy = policy - if bookkeeper is None: - bookkeeper = Bookkeeper(self) - self.bookkeeper = bookkeeper - - def __getstate__(self): - attrs = """translator pendingblocks bindings annotated links_followed - notify bookkeeper frozen policy added_blocks""".split() - ret = self.__dict__.copy() - for key, value in ret.items(): - if key not in attrs: - assert type(value) is dict, ( - "%r is not dict. please update %s.__getstate__" % - (key, self.__class__.__name__)) - ret[key] = {} - return ret - - #___ convenience high-level interface __________________ - - def build_types(self, function, input_arg_types, complete_now=True, - main_entry_point=False): - """Recursively build annotations about the specific entry point.""" - assert isinstance(function, types.FunctionType), "fix that!" - - from pypy.annotation.policy import AnnotatorPolicy - policy = AnnotatorPolicy() - # make input arguments and set their type - args_s = [self.typeannotation(t) for t in input_arg_types] - - # XXX hack - annmodel.TLS.check_str_without_nul = ( - self.translator.config.translation.check_str_without_nul) - - flowgraph, inputcells = self.get_call_parameters(function, args_s, policy) - if not isinstance(flowgraph, FunctionGraph): - assert isinstance(flowgraph, annmodel.SomeObject) - return flowgraph - - if main_entry_point: - self.translator.entry_point_graph = flowgraph - return self.build_graph_types(flowgraph, inputcells, complete_now=complete_now) - - def get_call_parameters(self, function, args_s, policy): - desc = self.bookkeeper.getdesc(function) - args = self.bookkeeper.build_args("simple_call", args_s[:]) - result = [] - def schedule(graph, inputcells): - result.append((graph, inputcells)) - return annmodel.s_ImpossibleValue - - prevpolicy = self.policy - self.policy = policy - self.bookkeeper.enter(None) - try: - desc.pycall(schedule, args, annmodel.s_ImpossibleValue) - finally: - self.bookkeeper.leave() - self.policy = prevpolicy - [(graph, inputcells)] = result - return graph, inputcells - - def annotate_helper(self, function, args_s, policy=None): - if policy is None: - from pypy.annotation.policy import AnnotatorPolicy - policy = AnnotatorPolicy() - graph, inputcells = self.get_call_parameters(function, args_s, policy) - self.build_graph_types(graph, inputcells, complete_now=False) - self.complete_helpers(policy) - return graph - - def complete_helpers(self, policy): - saved = self.policy, self.added_blocks - self.policy = policy - try: - self.added_blocks = {} - self.complete() - # invoke annotation simplifications for the new blocks - self.simplify(block_subset=self.added_blocks) - finally: - self.policy, self.added_blocks = saved - - def build_graph_types(self, flowgraph, inputcells, complete_now=True): - checkgraph(flowgraph) - - nbarg = len(flowgraph.getargs()) - if len(inputcells) != nbarg: - raise TypeError("%s expects %d args, got %d" %( - flowgraph, nbarg, len(inputcells))) - - # register the entry point - self.addpendinggraph(flowgraph, inputcells) - # recursively proceed until no more pending block is left - if complete_now: - self.complete() - return self.binding(flowgraph.getreturnvar(), None) - - def gettype(self, variable): - """Return the known type of a control flow graph variable, - defaulting to 'object'.""" - if isinstance(variable, Constant): - return type(variable.value) - elif isinstance(variable, Variable): - cell = self.bindings.get(variable) - if cell: - return cell.knowntype - else: - return object - else: - raise TypeError, ("Variable or Constant instance expected, " - "got %r" % (variable,)) - - def getuserclassdefinitions(self): - """Return a list of ClassDefs.""" - return self.bookkeeper.classdefs - - #___ medium-level interface ____________________________ - - def addpendinggraph(self, flowgraph, inputcells): - self.addpendingblock(flowgraph, flowgraph.startblock, inputcells) - - def addpendingblock(self, graph, block, cells): - """Register an entry point into block with the given input cells.""" - if graph in self.fixed_graphs: - # special case for annotating/rtyping in several phases: calling - # a graph that has already been rtyped. Safety-check the new - # annotations that are passed in, and don't annotate the old - # graph -- it's already low-level operations! - for a, s_newarg in zip(graph.getargs(), cells): - s_oldarg = self.binding(a) - assert annmodel.unionof(s_oldarg, s_newarg) == s_oldarg - else: - assert not self.frozen - for a in cells: - assert isinstance(a, annmodel.SomeObject) - if block not in self.annotated: - self.bindinputargs(graph, block, cells) - else: - self.mergeinputargs(graph, block, cells) - if not self.annotated[block]: - self.pendingblocks[block] = graph - - def complete(self): - """Process pending blocks until none is left.""" - while True: - while self.pendingblocks: - block, graph = self.pendingblocks.popitem() - self.processblock(graph, block) - self.policy.no_more_blocks_to_annotate(self) - if not self.pendingblocks: - break # finished - # make sure that the return variables of all graphs is annotated - if self.added_blocks is not None: - newgraphs = [self.annotated[block] for block in self.added_blocks] - newgraphs = dict.fromkeys(newgraphs) - got_blocked_blocks = False in newgraphs - else: - newgraphs = self.translator.graphs #all of them - got_blocked_blocks = False in self.annotated.values() - if got_blocked_blocks: - for graph in self.blocked_graphs.values(): - self.blocked_graphs[graph] = True - - blocked_blocks = [block for block, done in self.annotated.items() - if done is False] - assert len(blocked_blocks) == len(self.blocked_blocks) - - text = format_blocked_annotation_error(self, self.blocked_blocks) - #raise SystemExit() - raise AnnotatorError(text) - for graph in newgraphs: - v = graph.getreturnvar() - if v not in self.bindings: - self.setbinding(v, annmodel.s_ImpossibleValue) - # policy-dependent computation - self.bookkeeper.compute_at_fixpoint() - - def binding(self, arg, default=FAIL): - "Gives the SomeValue corresponding to the given Variable or Constant." - if isinstance(arg, Variable): - try: - return self.bindings[arg] - except KeyError: - if default is not FAIL: - return default - else: - raise - elif isinstance(arg, Constant): - #if arg.value is undefined_value: # undefined local variables - # return annmodel.s_ImpossibleValue - return self.bookkeeper.immutableconstant(arg) - else: - raise TypeError, 'Variable or Constant expected, got %r' % (arg,) - - def typeannotation(self, t): - return signature.annotation(t, self.bookkeeper) - - def setbinding(self, arg, s_value): - if arg in self.bindings: - assert s_value.contains(self.bindings[arg]) - self.bindings[arg] = s_value - - def transfer_binding(self, v_target, v_source): - assert v_source in self.bindings - self.bindings[v_target] = self.bindings[v_source] - - def warning(self, msg, pos=None): - if pos is None: - try: - pos = self.bookkeeper.position_key - except AttributeError: - pos = '?' - if pos != '?': - pos = self.whereami(pos) - - log.WARNING("%s/ %s" % (pos, msg)) - - - #___ interface for annotator.bookkeeper _______ - - def recursivecall(self, graph, whence, inputcells): - if isinstance(whence, tuple): - parent_graph, parent_block, parent_index = whence - tag = parent_block, parent_index - self.translator.update_call_graph(parent_graph, graph, tag) - # self.notify[graph.returnblock] is a dictionary of call - # points to this func which triggers a reflow whenever the - # return block of this graph has been analysed. - callpositions = self.notify.setdefault(graph.returnblock, {}) - if whence is not None: - if callable(whence): - def callback(): - whence(self, graph) - else: - callback = whence - callpositions[callback] = True - - # generalize the function's input arguments - self.addpendingblock(graph, graph.startblock, inputcells) - - # get the (current) return value - v = graph.getreturnvar() - try: - return self.bindings[v] - except KeyError: - # the function didn't reach any return statement so far. - # (some functions actually never do, they always raise exceptions) - return annmodel.s_ImpossibleValue - - def reflowfromposition(self, position_key): - graph, block, index = position_key - self.reflowpendingblock(graph, block) - - - #___ simplification (should be moved elsewhere?) _______ - - def simplify(self, block_subset=None, extra_passes=None): - # Generic simplifications - transform.transform_graph(self, block_subset=block_subset, - extra_passes=extra_passes) - if block_subset is None: - graphs = self.translator.graphs - else: - graphs = {} - for block in block_subset: - graph = self.annotated.get(block) - if graph: - graphs[graph] = True - for graph in graphs: - simplify.eliminate_empty_blocks(graph) - - - #___ flowing annotations in blocks _____________________ - - def processblock(self, graph, block): - # Important: this is not called recursively. - # self.flowin() can only issue calls to self.addpendingblock(). - # The analysis of a block can be in three states: - # * block not in self.annotated: - # never seen the block. - # * self.annotated[block] == False: - # the input variables of the block are in self.bindings but we - # still have to consider all the operations in the block. - # * self.annotated[block] == graph-containing-block: - # analysis done (at least until we find we must generalize the - # input variables). - - #print '* processblock', block, cells - self.annotated[block] = graph - if block in self.blocked_blocks: - del self.blocked_blocks[block] - try: - self.flowin(graph, block) - except BlockedInference, e: - self.annotated[block] = False # failed, hopefully temporarily - self.blocked_blocks[block] = (graph, e.opindex) - except Exception, e: - # hack for debug tools only - if not hasattr(e, '__annotator_block'): - setattr(e, '__annotator_block', block) - raise - - # The dict 'added_blocks' is used by rpython.annlowlevel to - # detect which are the new blocks that annotating an additional - # small helper creates. - if self.added_blocks is not None: - self.added_blocks[block] = True - - def reflowpendingblock(self, graph, block): - assert not self.frozen - assert graph not in self.fixed_graphs - self.pendingblocks[block] = graph - assert block in self.annotated - self.annotated[block] = False # must re-flow - self.blocked_blocks[block] = (graph, None) - - def bindinputargs(self, graph, block, inputcells): - # Create the initial bindings for the input args of a block. - assert len(block.inputargs) == len(inputcells) - for a, cell in zip(block.inputargs, inputcells): - self.setbinding(a, cell) - self.annotated[block] = False # must flowin. - self.blocked_blocks[block] = (graph, None) - - def mergeinputargs(self, graph, block, inputcells): - # Merge the new 'cells' with each of the block's existing input - # variables. - oldcells = [self.binding(a) for a in block.inputargs] - try: - unions = [annmodel.unionof(c1,c2) for c1, c2 in zip(oldcells,inputcells)] - except annmodel.UnionError, e: - e.args = e.args + ( - ErrorWrapper(gather_error(self, graph, block, None)),) - raise - # if the merged cells changed, we must redo the analysis - if unions != oldcells: - self.bindinputargs(graph, block, unions) - - def whereami(self, position_key): - graph, block, i = position_key - blk = "" - if block: - at = block.at() - if at: - blk = " block"+at - opid="" - if i is not None: - opid = " op=%d" % i - return repr(graph) + blk + opid - - def flowin(self, graph, block): - #print 'Flowing', block, [self.binding(a) for a in block.inputargs] - try: - for i in range(len(block.operations)): - try: - self.bookkeeper.enter((graph, block, i)) - self.consider_op(block, i) - finally: - self.bookkeeper.leave() - - except BlockedInference, e: - if (e.op is block.operations[-1] and - block.exitswitch == c_last_exception): - # this is the case where the last operation of the block will - # always raise an exception which is immediately caught by - # an exception handler. We then only follow the exceptional - # branches. - exits = [link for link in block.exits - if link.exitcase is not None] - - elif e.op.opname in ('simple_call', 'call_args', 'next'): - # XXX warning, keep the name of the call operations in sync - # with the flow object space. These are the operations for - # which it is fine to always raise an exception. We then - # swallow the BlockedInference and that's it. - # About 'next': see test_annotate_iter_empty_container(). - return - - else: - # other cases are problematic (but will hopefully be solved - # later by reflowing). Throw the BlockedInference up to - # processblock(). - raise - - except annmodel.HarmlesslyBlocked: - return - - else: - # dead code removal: don't follow all exits if the exitswitch - # is known - exits = block.exits - if isinstance(block.exitswitch, Variable): - s_exitswitch = self.bindings[block.exitswitch] - if s_exitswitch.is_constant(): - exits = [link for link in exits - if link.exitcase == s_exitswitch.const] - - # mapping (exitcase, variable) -> s_annotation - # that can be attached to booleans, exitswitches - knowntypedata = getattr(self.bindings.get(block.exitswitch), - "knowntypedata", {}) - - # filter out those exceptions which cannot - # occour for this specific, typed operation. - if block.exitswitch == c_last_exception: - op = block.operations[-1] - if op.opname in binaryop.BINARY_OPERATIONS: - arg1 = self.binding(op.args[0]) - arg2 = self.binding(op.args[1]) - binop = getattr(pair(arg1, arg2), op.opname, None) - can_only_throw = annmodel.read_can_only_throw(binop, arg1, arg2) - elif op.opname in unaryop.UNARY_OPERATIONS: - arg1 = self.binding(op.args[0]) - opname = op.opname - if opname == 'contains': opname = 'op_contains' - unop = getattr(arg1, opname, None) - can_only_throw = annmodel.read_can_only_throw(unop, arg1) - else: - can_only_throw = None - - if can_only_throw is not None: - candidates = can_only_throw - candidate_exits = exits - exits = [] - for link in candidate_exits: - case = link.exitcase - if case is None: - exits.append(link) - continue - covered = [c for c in candidates if issubclass(c, case)] - if covered: - exits.append(link) - candidates = [c for c in candidates if c not in covered] - - for link in exits: - in_except_block = False - - last_exception_var = link.last_exception # may be None for non-exception link - last_exc_value_var = link.last_exc_value # may be None for non-exception link - - if isinstance(link.exitcase, (types.ClassType, type)) \ - and issubclass(link.exitcase, py.builtin.BaseException): - assert last_exception_var and last_exc_value_var - last_exc_value_object = self.bookkeeper.valueoftype(link.exitcase) - last_exception_object = annmodel.SomeType() - if isinstance(last_exception_var, Constant): - last_exception_object.const = last_exception_var.value - last_exception_object.is_type_of = [last_exc_value_var] - - if isinstance(last_exception_var, Variable): - self.setbinding(last_exception_var, last_exception_object) - if isinstance(last_exc_value_var, Variable): - self.setbinding(last_exc_value_var, last_exc_value_object) - - last_exception_object = annmodel.SomeType() - if isinstance(last_exception_var, Constant): - last_exception_object.const = last_exception_var.value - #if link.exitcase is Exception: - # last_exc_value_object = annmodel.SomeObject() - #else: - last_exc_value_vars = [] - in_except_block = True - - ignore_link = False - cells = [] - renaming = {} - for a,v in zip(link.args,link.target.inputargs): - renaming.setdefault(a, []).append(v) - for a,v in zip(link.args,link.target.inputargs): - if a == last_exception_var: - assert in_except_block - cells.append(last_exception_object) - elif a == last_exc_value_var: - assert in_except_block - cells.append(last_exc_value_object) - last_exc_value_vars.append(v) - else: - cell = self.binding(a) - if (link.exitcase, a) in knowntypedata: - knownvarvalue = knowntypedata[(link.exitcase, a)] - cell = pair(cell, knownvarvalue).improve() - # ignore links that try to pass impossible values - if cell == annmodel.s_ImpossibleValue: - ignore_link = True - - if hasattr(cell,'is_type_of'): - renamed_is_type_of = [] - for v in cell.is_type_of: - new_vs = renaming.get(v,[]) - renamed_is_type_of += new_vs - assert cell.knowntype is type - newcell = annmodel.SomeType() - if cell.is_constant(): - newcell.const = cell.const - cell = newcell - cell.is_type_of = renamed_is_type_of - - if hasattr(cell, 'knowntypedata'): - renamed_knowntypedata = {} - for (value, v), s in cell.knowntypedata.items(): - new_vs = renaming.get(v, []) - for new_v in new_vs: - renamed_knowntypedata[value, new_v] = s - assert isinstance(cell, annmodel.SomeBool) - newcell = annmodel.SomeBool() - if cell.is_constant(): - newcell.const = cell.const - cell = newcell - cell.set_knowntypedata(renamed_knowntypedata) - - cells.append(cell) - - if ignore_link: - continue - - if in_except_block: - last_exception_object.is_type_of = last_exc_value_vars - - self.links_followed[link] = True - self.addpendingblock(graph, link.target, cells) - - if block in self.notify: - # reflow from certain positions when this block is done - for callback in self.notify[block]: - if isinstance(callback, tuple): - self.reflowfromposition(callback) # callback is a position - else: - callback() - - - #___ creating the annotations based on operations ______ - - def consider_op(self, block, opindex): - op = block.operations[opindex] - argcells = [self.binding(a) for a in op.args] - consider_meth = getattr(self,'consider_op_'+op.opname, - None) - if not consider_meth: - raise Exception,"unknown op: %r" % op - - # let's be careful about avoiding propagated SomeImpossibleValues - # to enter an op; the latter can result in violations of the - # more general results invariant: e.g. if SomeImpossibleValue enters is_ - # is_(SomeImpossibleValue, None) -> SomeBool - # is_(SomeInstance(not None), None) -> SomeBool(const=False) ... - # boom -- in the assert of setbinding() - for arg in argcells: - if isinstance(arg, annmodel.SomeImpossibleValue): - raise BlockedInference(self, op, opindex) - try: - resultcell = consider_meth(*argcells) - except Exception, e: - graph = self.bookkeeper.position_key[0] - e.args = e.args + ( - ErrorWrapper(gather_error(self, graph, block, opindex)),) - raise - if resultcell is None: - resultcell = self.noreturnvalue(op) - elif resultcell == annmodel.s_ImpossibleValue: - raise BlockedInference(self, op, opindex) # the operation cannot succeed - assert isinstance(resultcell, annmodel.SomeObject) - assert isinstance(op.result, Variable) - self.setbinding(op.result, resultcell) # bind resultcell to op.result - - def noreturnvalue(self, op): - return annmodel.s_ImpossibleValue # no return value (hook method) - - # XXX "contains" clash with SomeObject method - def consider_op_contains(self, seq, elem): - self.bookkeeper.count("contains", seq) - return seq.op_contains(elem) - - def consider_op_newtuple(self, *args): - return annmodel.SomeTuple(items = args) - - def consider_op_newlist(self, *args): - return self.bookkeeper.newlist(*args) - - def consider_op_newdict(self): - return self.bookkeeper.newdict() - - - def _registeroperations(cls, unary_ops, binary_ops): - # All unary operations - d = {} - for opname in unary_ops: - fnname = 'consider_op_' + opname - exec py.code.Source(""" -def consider_op_%s(self, arg, *args): - return arg.%s(*args) -""" % (opname, opname)).compile() in globals(), d - setattr(cls, fnname, d[fnname]) - # All binary operations - for opname in binary_ops: - fnname = 'consider_op_' + opname - exec py.code.Source(""" -def consider_op_%s(self, arg1, arg2, *args): - return pair(arg1,arg2).%s(*args) -""" % (opname, opname)).compile() in globals(), d - setattr(cls, fnname, d[fnname]) - _registeroperations = classmethod(_registeroperations) - -# register simple operations handling -RPythonAnnotator._registeroperations(unaryop.UNARY_OPERATIONS, binaryop.BINARY_OPERATIONS) - - -class BlockedInference(Exception): - """This exception signals the type inference engine that the situation - is currently blocked, and that it should try to progress elsewhere.""" - - def __init__(self, annotator, op, opindex): - self.annotator = annotator - try: - self.break_at = annotator.bookkeeper.position_key - except AttributeError: - self.break_at = None - self.op = op - self.opindex = opindex - - def __repr__(self): - if not self.break_at: - break_at = "?" - else: - break_at = self.annotator.whereami(self.break_at) - return "" %(break_at, self.op) - - __str__ = __repr__ diff --git a/pypy/annotation/binaryop.py b/pypy/annotation/binaryop.py deleted file mode 100644 --- a/pypy/annotation/binaryop.py +++ /dev/null @@ -1,1052 +0,0 @@ -""" -Binary operations between SomeValues. -""" - -import py -import operator -from pypy.tool.pairtype import pair, pairtype -from pypy.annotation.model import SomeObject, SomeInteger, SomeBool, s_Bool -from pypy.annotation.model import SomeString, SomeChar, SomeList, SomeDict -from pypy.annotation.model import SomeUnicodeCodePoint, SomeUnicodeString -from pypy.annotation.model import SomeTuple, SomeImpossibleValue, s_ImpossibleValue -from pypy.annotation.model import SomeInstance, SomeBuiltin, SomeIterator -from pypy.annotation.model import SomePBC, SomeFloat, s_None, SomeByteArray -from pypy.annotation.model import SomeExternalObject, SomeWeakRef -from pypy.annotation.model import SomeAddress, SomeTypedAddressAccess -from pypy.annotation.model import SomeSingleFloat, SomeLongFloat, SomeType -from pypy.annotation.model import unionof, UnionError, missing_operation -from pypy.annotation.model import TLS -from pypy.annotation.model import read_can_only_throw -from pypy.annotation.model import add_knowntypedata, merge_knowntypedata -from pypy.annotation.model import SomeGenericCallable -from pypy.annotation.bookkeeper import getbookkeeper -from pypy.objspace.flow.model import Variable, Constant -from pypy.rlib import rarithmetic -from pypy.tool.error import AnnotatorError - -# convenience only! -def immutablevalue(x): - return getbookkeeper().immutablevalue(x) - -# XXX unify this with ObjSpace.MethodTable -BINARY_OPERATIONS = set(['add', 'sub', 'mul', 'div', 'mod', - 'truediv', 'floordiv', 'divmod', - 'and_', 'or_', 'xor', - 'lshift', 'rshift', - 'getitem', 'setitem', 'delitem', - 'getitem_idx', 'getitem_key', 'getitem_idx_key', - 'inplace_add', 'inplace_sub', 'inplace_mul', - 'inplace_truediv', 'inplace_floordiv', 'inplace_div', - 'inplace_mod', - 'inplace_lshift', 'inplace_rshift', - 'inplace_and', 'inplace_or', 'inplace_xor', - 'lt', 'le', 'eq', 'ne', 'gt', 'ge', 'is_', 'cmp', - 'coerce', - ] - +[opname+'_ovf' for opname in - """add sub mul floordiv div mod lshift - """.split() - ]) - -for opname in BINARY_OPERATIONS: - missing_operation(pairtype(SomeObject, SomeObject), opname) - -class __extend__(pairtype(SomeObject, SomeObject)): - - def union((obj1, obj2)): - raise UnionError(obj1, obj2) - - # inplace_xxx ---> xxx by default - def inplace_add((obj1, obj2)): return pair(obj1, obj2).add() - def inplace_sub((obj1, obj2)): return pair(obj1, obj2).sub() - def inplace_mul((obj1, obj2)): return pair(obj1, obj2).mul() - def inplace_truediv((obj1, obj2)): return pair(obj1, obj2).truediv() - def inplace_floordiv((obj1, obj2)): return pair(obj1, obj2).floordiv() - def inplace_div((obj1, obj2)): return pair(obj1, obj2).div() - def inplace_mod((obj1, obj2)): return pair(obj1, obj2).mod() - def inplace_lshift((obj1, obj2)): return pair(obj1, obj2).lshift() - def inplace_rshift((obj1, obj2)): return pair(obj1, obj2).rshift() - def inplace_and((obj1, obj2)): return pair(obj1, obj2).and_() - def inplace_or((obj1, obj2)): return pair(obj1, obj2).or_() - def inplace_xor((obj1, obj2)): return pair(obj1, obj2).xor() - - for name, func in locals().items(): - if name.startswith('inplace_'): - func.can_only_throw = [] - - inplace_div.can_only_throw = [ZeroDivisionError] - inplace_truediv.can_only_throw = [ZeroDivisionError] - inplace_floordiv.can_only_throw = [ZeroDivisionError] - inplace_mod.can_only_throw = [ZeroDivisionError] - - def lt((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const < obj2.const) - else: - getbookkeeper().count("non_int_comp", obj1, obj2) - return s_Bool - - def le((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const <= obj2.const) - else: - getbookkeeper().count("non_int_comp", obj1, obj2) - return s_Bool - - def eq((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const == obj2.const) - else: - getbookkeeper().count("non_int_eq", obj1, obj2) - return s_Bool - - def ne((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const != obj2.const) - else: - getbookkeeper().count("non_int_eq", obj1, obj2) - return s_Bool - - def gt((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const > obj2.const) - else: - getbookkeeper().count("non_int_comp", obj1, obj2) - return s_Bool - - def ge((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const >= obj2.const) - else: - getbookkeeper().count("non_int_comp", obj1, obj2) - return s_Bool - - def cmp((obj1, obj2)): - getbookkeeper().count("cmp", obj1, obj2) - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(cmp(obj1.const, obj2.const)) - else: - return SomeInteger() - - def is_((obj1, obj2)): - r = SomeBool() - if obj2.is_constant(): - if obj1.is_constant(): - r.const = obj1.const is obj2.const - if obj2.const is None and not obj1.can_be_none(): - r.const = False - elif obj1.is_constant(): - if obj1.const is None and not obj2.can_be_none(): - r.const = False - # XXX HACK HACK HACK - # XXX HACK HACK HACK - # XXX HACK HACK HACK - bk = getbookkeeper() - if bk is not None: # for testing - knowntypedata = {} - fn, block, i = bk.position_key - - annotator = bk.annotator - op = block.operations[i] - assert op.opname == "is_" - assert len(op.args) == 2 - - def bind(src_obj, tgt_obj, tgt_arg): - if hasattr(tgt_obj, 'is_type_of') and src_obj.is_constant(): - add_knowntypedata(knowntypedata, True, tgt_obj.is_type_of, - bk.valueoftype(src_obj.const)) - - assert annotator.binding(op.args[tgt_arg]) == tgt_obj - add_knowntypedata(knowntypedata, True, [op.args[tgt_arg]], src_obj) - - nonnone_obj = tgt_obj - if src_obj.is_constant() and src_obj.const is None and tgt_obj.can_be_none(): - nonnone_obj = tgt_obj.nonnoneify() - - add_knowntypedata(knowntypedata, False, [op.args[tgt_arg]], nonnone_obj) - - bind(obj2, obj1, 0) - bind(obj1, obj2, 1) - r.set_knowntypedata(knowntypedata) - - return r - - def divmod((obj1, obj2)): - getbookkeeper().count("divmod", obj1, obj2) - return SomeTuple([pair(obj1, obj2).div(), pair(obj1, obj2).mod()]) - - def coerce((obj1, obj2)): - getbookkeeper().count("coerce", obj1, obj2) - return pair(obj1, obj2).union() # reasonable enough - - # approximation of an annotation intersection, the result should be the annotation obj or - # the intersection of obj and improvement - def improve((obj, improvement)): - if not improvement.contains(obj) and obj.contains(improvement): - return improvement - else: - return obj - - # checked getitems - - def _getitem_can_only_throw(s_c1, s_o2): - impl = pair(s_c1, s_o2).getitem - return read_can_only_throw(impl, s_c1, s_o2) - - def getitem_idx_key((s_c1, s_o2)): - impl = pair(s_c1, s_o2).getitem - return impl() - getitem_idx_key.can_only_throw = _getitem_can_only_throw - - getitem_idx = getitem_idx_key - getitem_key = getitem_idx_key - - -class __extend__(pairtype(SomeType, SomeType)): - - def union((obj1, obj2)): - result = SomeType() - is_type_of1 = getattr(obj1, 'is_type_of', None) - is_type_of2 = getattr(obj2, 'is_type_of', None) - if obj1.is_immutable_constant() and obj2.is_immutable_constant() and obj1.const == obj2.const: - result.const = obj1.const - is_type_of = {} - if is_type_of1: - for v in is_type_of1: - is_type_of[v] = True - if is_type_of2: - for v in is_type_of2: - is_type_of[v] = True - if is_type_of: - result.is_type_of = is_type_of.keys() - else: - if is_type_of1 and is_type_of1 == is_type_of2: - result.is_type_of = is_type_of1 - return result - - -# cloning a function with identical code, for the can_only_throw attribute -def _clone(f, can_only_throw = None): - newfunc = type(f)(f.func_code, f.func_globals, f.func_name, - f.func_defaults, f.func_closure) - if can_only_throw is not None: - newfunc.can_only_throw = can_only_throw - return newfunc - -class __extend__(pairtype(SomeInteger, SomeInteger)): - # unsignedness is considered a rare and contagious disease - - def union((int1, int2)): - if int1.unsigned == int2.unsigned: - knowntype = rarithmetic.compute_restype(int1.knowntype, int2.knowntype) - else: - t1 = int1.knowntype - if t1 is bool: - t1 = int - t2 = int2.knowntype - if t2 is bool: - t2 = int - - if t2 is int: - if int2.nonneg == False: - raise UnionError, "Merging %s and a possibly negative int is not allowed" % t1 - knowntype = t1 - elif t1 is int: - if int1.nonneg == False: - raise UnionError, "Merging %s and a possibly negative int is not allowed" % t2 - knowntype = t2 - else: - raise UnionError, "Merging these types (%s, %s) is not supported" % (t1, t2) - return SomeInteger(nonneg=int1.nonneg and int2.nonneg, - knowntype=knowntype) - - or_ = xor = add = mul = _clone(union, []) - add_ovf = mul_ovf = _clone(union, [OverflowError]) - div = floordiv = mod = _clone(union, [ZeroDivisionError]) - div_ovf= floordiv_ovf = mod_ovf = _clone(union, [ZeroDivisionError, OverflowError]) - - def truediv((int1, int2)): - return SomeFloat() - truediv.can_only_throw = [ZeroDivisionError] - truediv_ovf = _clone(truediv, [ZeroDivisionError, OverflowError]) - - inplace_div = div - inplace_truediv = truediv - - def sub((int1, int2)): - knowntype = rarithmetic.compute_restype(int1.knowntype, int2.knowntype) - return SomeInteger(knowntype=knowntype) - sub.can_only_throw = [] - sub_ovf = _clone(sub, [OverflowError]) - - def and_((int1, int2)): - knowntype = rarithmetic.compute_restype(int1.knowntype, int2.knowntype) - return SomeInteger(nonneg=int1.nonneg or int2.nonneg, - knowntype=knowntype) - and_.can_only_throw = [] - - def lshift((int1, int2)): - if isinstance(int1, SomeBool): - return SomeInteger() - else: - return SomeInteger(knowntype=int1.knowntype) - lshift.can_only_throw = [] - lshift_ovf = _clone(lshift, [OverflowError]) - - def rshift((int1, int2)): - if isinstance(int1, SomeBool): - return SomeInteger(nonneg=True) - else: - return SomeInteger(nonneg=int1.nonneg, knowntype=int1.knowntype) - rshift.can_only_throw = [] - - def _compare_helper((int1, int2), opname, operation): - r = SomeBool() - if int1.is_immutable_constant() and int2.is_immutable_constant(): - r.const = operation(int1.const, int2.const) - # - # The rest of the code propagates nonneg information between - # the two arguments. - # - # Doing the right thing when int1 or int2 change from signed - # to unsigned (r_uint) is almost impossible. See test_intcmp_bug. - # Instead, we only deduce constrains on the operands in the - # case where they are both signed. In other words, if y is - # nonneg then "assert x>=y" will let the annotator know that - # x is nonneg too, but it will not work if y is unsigned. - # - if not (rarithmetic.signedtype(int1.knowntype) and - rarithmetic.signedtype(int2.knowntype)): - return r - knowntypedata = {} - # XXX HACK HACK HACK - fn, block, i = getbookkeeper().position_key - op = block.operations[i] - assert op.opname == opname - assert len(op.args) == 2 - def tointtype(int0): - if int0.knowntype is bool: - return int - return int0.knowntype - if int1.nonneg and isinstance(op.args[1], Variable): - case = opname in ('lt', 'le', 'eq') - - add_knowntypedata(knowntypedata, case, [op.args[1]], - SomeInteger(nonneg=True, knowntype=tointtype(int2))) - if int2.nonneg and isinstance(op.args[0], Variable): - case = opname in ('gt', 'ge', 'eq') - add_knowntypedata(knowntypedata, case, [op.args[0]], - SomeInteger(nonneg=True, knowntype=tointtype(int1))) - r.set_knowntypedata(knowntypedata) - # a special case for 'x < 0' or 'x >= 0', - # where 0 is a flow graph Constant - # (in this case we are sure that it cannot become a r_uint later) - if (isinstance(op.args[1], Constant) and - type(op.args[1].value) is int and # filter out Symbolics - op.args[1].value == 0): - if int1.nonneg: - if opname == 'lt': - r.const = False - if opname == 'ge': - r.const = True - return r - - def lt(intint): return intint._compare_helper('lt', operator.lt) - def le(intint): return intint._compare_helper('le', operator.le) - def eq(intint): return intint._compare_helper('eq', operator.eq) - def ne(intint): return intint._compare_helper('ne', operator.ne) - def gt(intint): return intint._compare_helper('gt', operator.gt) - def ge(intint): return intint._compare_helper('ge', operator.ge) - - -class __extend__(pairtype(SomeBool, SomeBool)): - - def union((boo1, boo2)): - s = SomeBool() - if getattr(boo1, 'const', -1) == getattr(boo2, 'const', -2): - s.const = boo1.const - if hasattr(boo1, 'knowntypedata') and \ - hasattr(boo2, 'knowntypedata'): - ktd = merge_knowntypedata(boo1.knowntypedata, boo2.knowntypedata) - s.set_knowntypedata(ktd) - return s - - def and_((boo1, boo2)): - s = SomeBool() - if boo1.is_constant(): - if not boo1.const: - s.const = False - else: - return boo2 - if boo2.is_constant(): - if not boo2.const: - s.const = False - return s - - def or_((boo1, boo2)): - s = SomeBool() - if boo1.is_constant(): - if boo1.const: - s.const = True - else: - return boo2 - if boo2.is_constant(): - if boo2.const: - s.const = True - return s - - def xor((boo1, boo2)): - s = SomeBool() - if boo1.is_constant() and boo2.is_constant(): - s.const = boo1.const ^ boo2.const - return s - -class __extend__(pairtype(SomeString, SomeString)): - - def union((str1, str2)): - can_be_None = str1.can_be_None or str2.can_be_None - no_nul = str1.no_nul and str2.no_nul - return SomeString(can_be_None=can_be_None, no_nul=no_nul) - - def add((str1, str2)): - # propagate const-ness to help getattr(obj, 'prefix' + const_name) - result = SomeString(no_nul=str1.no_nul and str2.no_nul) - if str1.is_immutable_constant() and str2.is_immutable_constant(): - result.const = str1.const + str2.const - return result - -class __extend__(pairtype(SomeByteArray, SomeByteArray)): - def union((b1, b2)): - can_be_None = b1.can_be_None or b2.can_be_None - return SomeByteArray(can_be_None=can_be_None) - - def add((b1, b2)): - result = SomeByteArray() - if b1.is_immutable_constant() and b2.is_immutable_constant(): - result.const = b1.const + b2.const - return result - -class __extend__(pairtype(SomeByteArray, SomeInteger)): - def getitem((s_b, s_i)): - return SomeInteger() - - def setitem((s_b, s_i), s_i2): - assert isinstance(s_i2, SomeInteger) - -class __extend__(pairtype(SomeString, SomeByteArray), - pairtype(SomeByteArray, SomeString), - pairtype(SomeChar, SomeByteArray), - pairtype(SomeByteArray, SomeChar)): - def add((b1, b2)): - result = SomeByteArray() - if b1.is_immutable_constant() and b2.is_immutable_constant(): - result.const = b1.const + b2.const - return result - -class __extend__(pairtype(SomeChar, SomeChar)): - - def union((chr1, chr2)): - return SomeChar() - - -class __extend__(pairtype(SomeChar, SomeUnicodeCodePoint), - pairtype(SomeUnicodeCodePoint, SomeChar)): - def union((uchr1, uchr2)): - return SomeUnicodeCodePoint() - -class __extend__(pairtype(SomeUnicodeCodePoint, SomeUnicodeCodePoint)): - def union((uchr1, uchr2)): - return SomeUnicodeCodePoint() - - def add((chr1, chr2)): - return SomeUnicodeString() - -class __extend__(pairtype(SomeString, SomeUnicodeString), - pairtype(SomeUnicodeString, SomeString)): - def mod((str, unistring)): - raise NotImplementedError( - "string formatting mixing strings and unicode not supported") - - -class __extend__(pairtype(SomeString, SomeTuple), - pairtype(SomeUnicodeString, SomeTuple)): - def mod((s_string, s_tuple)): - is_string = isinstance(s_string, SomeString) - is_unicode = isinstance(s_string, SomeUnicodeString) - assert is_string or is_unicode - for s_item in s_tuple.items: - if (is_unicode and isinstance(s_item, (SomeChar, SomeString)) or - is_string and isinstance(s_item, (SomeUnicodeCodePoint, - SomeUnicodeString))): - raise NotImplementedError( - "string formatting mixing strings and unicode not supported") - getbookkeeper().count('strformat', s_string, s_tuple) - no_nul = s_string.no_nul - for s_item in s_tuple.items: - if isinstance(s_item, SomeFloat): - pass # or s_item is a subclass, like SomeInteger - elif (isinstance(s_item, SomeString) or - isinstance(s_item, SomeUnicodeString)) and s_item.no_nul: - pass - else: - no_nul = False - break - return s_string.__class__(no_nul=no_nul) - - -class __extend__(pairtype(SomeString, SomeObject), - pairtype(SomeUnicodeString, SomeObject)): - - def mod((s_string, args)): - getbookkeeper().count('strformat', s_string, args) - return s_string.__class__() - -class __extend__(pairtype(SomeFloat, SomeFloat)): - - def union((flt1, flt2)): - return SomeFloat() - - add = sub = mul = union - - def div((flt1, flt2)): - return SomeFloat() - div.can_only_throw = [] - truediv = div - - # repeat these in order to copy the 'can_only_throw' attribute - inplace_div = div - inplace_truediv = truediv - - -class __extend__(pairtype(SomeSingleFloat, SomeSingleFloat)): - - def union((flt1, flt2)): - return SomeSingleFloat() - - -class __extend__(pairtype(SomeLongFloat, SomeLongFloat)): - - def union((flt1, flt2)): - return SomeLongFloat() - - -class __extend__(pairtype(SomeList, SomeList)): - - def union((lst1, lst2)): - return SomeList(lst1.listdef.union(lst2.listdef)) - - def add((lst1, lst2)): - return lst1.listdef.offspring(lst2.listdef) - - def eq((lst1, lst2)): - lst1.listdef.agree(lst2.listdef) - return s_Bool - ne = eq - - -class __extend__(pairtype(SomeList, SomeObject)): - - def inplace_add((lst1, obj2)): - lst1.method_extend(obj2) - return lst1 - inplace_add.can_only_throw = [] - - def inplace_mul((lst1, obj2)): - lst1.listdef.resize() - return lst1 - inplace_mul.can_only_throw = [] - -class __extend__(pairtype(SomeTuple, SomeTuple)): - - def union((tup1, tup2)): - if len(tup1.items) != len(tup2.items): - raise UnionError("cannot take the union of a tuple of length %d " - "and a tuple of length %d" % (len(tup1.items), - len(tup2.items))) - else: - unions = [unionof(x,y) for x,y in zip(tup1.items, tup2.items)] - return SomeTuple(items = unions) - - def add((tup1, tup2)): - return SomeTuple(items = tup1.items + tup2.items) - - def eq(tup1tup2): - tup1tup2.union() - return s_Bool - ne = eq - - def lt((tup1, tup2)): - raise Exception("unsupported: (...) < (...)") - def le((tup1, tup2)): - raise Exception("unsupported: (...) <= (...)") - def gt((tup1, tup2)): - raise Exception("unsupported: (...) > (...)") - def ge((tup1, tup2)): - raise Exception("unsupported: (...) >= (...)") - - -class __extend__(pairtype(SomeDict, SomeDict)): - - def union((dic1, dic2)): - return SomeDict(dic1.dictdef.union(dic2.dictdef)) - - -class __extend__(pairtype(SomeDict, SomeObject)): - - def _can_only_throw(dic1, *ignore): - if dic1.dictdef.dictkey.custom_eq_hash: - return None - return [KeyError] - - def getitem((dic1, obj2)): - getbookkeeper().count("dict_getitem", dic1) - dic1.dictdef.generalize_key(obj2) - return dic1.dictdef.read_value() - getitem.can_only_throw = _can_only_throw - - def setitem((dic1, obj2), s_value): - getbookkeeper().count("dict_setitem", dic1) - dic1.dictdef.generalize_key(obj2) - dic1.dictdef.generalize_value(s_value) - setitem.can_only_throw = _can_only_throw - - def delitem((dic1, obj2)): - getbookkeeper().count("dict_delitem", dic1) - dic1.dictdef.generalize_key(obj2) - delitem.can_only_throw = _can_only_throw - - -class __extend__(pairtype(SomeTuple, SomeInteger)): - - def getitem((tup1, int2)): - if int2.is_immutable_constant(): - try: - return tup1.items[int2.const] - except IndexError: - return s_ImpossibleValue - else: - getbookkeeper().count("tuple_random_getitem", tup1) - return unionof(*tup1.items) - getitem.can_only_throw = [IndexError] - - -class __extend__(pairtype(SomeList, SomeInteger)): - - def mul((lst1, int2)): - return lst1.listdef.offspring() - - def getitem((lst1, int2)): - getbookkeeper().count("list_getitem", int2) - return lst1.listdef.read_item() - getitem.can_only_throw = [] - - getitem_key = getitem - - def getitem_idx((lst1, int2)): - getbookkeeper().count("list_getitem", int2) - return lst1.listdef.read_item() - getitem_idx.can_only_throw = [IndexError] - - getitem_idx_key = getitem_idx - - def setitem((lst1, int2), s_value): - getbookkeeper().count("list_setitem", int2) - lst1.listdef.mutate() - lst1.listdef.generalize(s_value) - setitem.can_only_throw = [IndexError] - - def delitem((lst1, int2)): - getbookkeeper().count("list_delitem", int2) - lst1.listdef.resize() - delitem.can_only_throw = [IndexError] - -class __extend__(pairtype(SomeString, SomeInteger)): - - def getitem((str1, int2)): - getbookkeeper().count("str_getitem", int2) - return SomeChar() - getitem.can_only_throw = [] - - getitem_key = getitem - - def getitem_idx((str1, int2)): - getbookkeeper().count("str_getitem", int2) - return SomeChar() - getitem_idx.can_only_throw = [IndexError] - - getitem_idx_key = getitem_idx - - def mul((str1, int2)): # xxx do we want to support this - getbookkeeper().count("str_mul", str1, int2) - return SomeString(no_nul=str1.no_nul) - -class __extend__(pairtype(SomeUnicodeString, SomeInteger)): - def getitem((str1, int2)): - getbookkeeper().count("str_getitem", int2) - return SomeUnicodeCodePoint() - getitem.can_only_throw = [] - - getitem_key = getitem - - def getitem_idx((str1, int2)): - getbookkeeper().count("str_getitem", int2) - return SomeUnicodeCodePoint() - getitem_idx.can_only_throw = [IndexError] - - getitem_idx_key = getitem_idx - - def mul((str1, int2)): # xxx do we want to support this - getbookkeeper().count("str_mul", str1, int2) - return SomeUnicodeString() - -class __extend__(pairtype(SomeInteger, SomeString), - pairtype(SomeInteger, SomeUnicodeString)): - - def mul((int1, str2)): # xxx do we want to support this - getbookkeeper().count("str_mul", str2, int1) - return str2.basestringclass() - -class __extend__(pairtype(SomeUnicodeCodePoint, SomeUnicodeString), - pairtype(SomeUnicodeString, SomeUnicodeCodePoint), - pairtype(SomeUnicodeString, SomeUnicodeString)): - def union((str1, str2)): - return SomeUnicodeString(can_be_None=str1.can_be_none() or - str2.can_be_none()) - - def add((str1, str2)): - # propagate const-ness to help getattr(obj, 'prefix' + const_name) - result = SomeUnicodeString() - if str1.is_immutable_constant() and str2.is_immutable_constant(): - result.const = str1.const + str2.const - return result - -class __extend__(pairtype(SomeInteger, SomeList)): - - def mul((int1, lst2)): - return lst2.listdef.offspring() - - -class __extend__(pairtype(SomeInstance, SomeInstance)): - - def union((ins1, ins2)): - if ins1.classdef is None or ins2.classdef is None: - # special case only - basedef = None - else: - basedef = ins1.classdef.commonbase(ins2.classdef) - if basedef is None: - raise UnionError(ins1, ins2) - flags = ins1.flags - if flags: - flags = flags.copy() - for key, value in flags.items(): - if key not in ins2.flags or ins2.flags[key] != value: - del flags[key] - return SomeInstance(basedef, - can_be_None=ins1.can_be_None or ins2.can_be_None, - flags=flags) - - def improve((ins1, ins2)): - if ins1.classdef is None: - resdef = ins2.classdef - elif ins2.classdef is None: - resdef = ins1.classdef - else: - basedef = ins1.classdef.commonbase(ins2.classdef) - if basedef is ins1.classdef: - resdef = ins2.classdef - elif basedef is ins2.classdef: - resdef = ins1.classdef - else: - if ins1.can_be_None and ins2.can_be_None: - return s_None - else: - return s_ImpossibleValue - res = SomeInstance(resdef, can_be_None=ins1.can_be_None and ins2.can_be_None) - if ins1.contains(res) and ins2.contains(res): - return res # fine - else: - # this case can occur in the presence of 'const' attributes, - # which we should try to preserve. Fall-back... - thistype = pairtype(SomeInstance, SomeInstance) - return super(thistype, pair(ins1, ins2)).improve() - - -class __extend__(pairtype(SomeIterator, SomeIterator)): - - def union((iter1, iter2)): - s_cont = unionof(iter1.s_container, iter2.s_container) - if iter1.variant != iter2.variant: - raise UnionError("merging incompatible iterators variants") - return SomeIterator(s_cont, *iter1.variant) - - -class __extend__(pairtype(SomeBuiltin, SomeBuiltin)): - - def union((bltn1, bltn2)): - if (bltn1.analyser != bltn2.analyser or - bltn1.methodname != bltn2.methodname or - bltn1.s_self is None or bltn2.s_self is None): - raise UnionError("cannot merge two different builtin functions " - "or methods:\n %r\n %r" % (bltn1, bltn2)) - s_self = unionof(bltn1.s_self, bltn2.s_self) - return SomeBuiltin(bltn1.analyser, s_self, methodname=bltn1.methodname) - -class __extend__(pairtype(SomePBC, SomePBC)): - - def union((pbc1, pbc2)): - d = pbc1.descriptions.copy() - d.update(pbc2.descriptions) - return SomePBC(d, can_be_None = pbc1.can_be_None or pbc2.can_be_None) - - def is_((pbc1, pbc2)): - thistype = pairtype(SomePBC, SomePBC) - s = super(thistype, pair(pbc1, pbc2)).is_() - if not s.is_constant(): - if not pbc1.can_be_None or not pbc2.can_be_None: - for desc in pbc1.descriptions: - if desc in pbc2.descriptions: - break - else: - s.const = False # no common desc in the two sets - return s - -class __extend__(pairtype(SomeGenericCallable, SomePBC)): - def union((gencall, pbc)): - for desc in pbc.descriptions: - unique_key = desc - bk = desc.bookkeeper - s_result = bk.emulate_pbc_call(unique_key, pbc, gencall.args_s) - s_result = unionof(s_result, gencall.s_result) - assert gencall.s_result.contains(s_result) - return gencall - -class __extend__(pairtype(SomePBC, SomeGenericCallable)): - def union((pbc, gencall)): - return pair(gencall, pbc).union() - -class __extend__(pairtype(SomeImpossibleValue, SomeObject)): - def union((imp1, obj2)): - return obj2 From noreply at buildbot.pypy.org Sun Jan 20 11:51:16 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 20 Jan 2013 11:51:16 +0100 (CET) Subject: [pypy-commit] pypy default: merge Message-ID: <20130120105116.7BDD61C1385@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r60230:4cd8686d3e2d Date: 2013-01-20 12:50 +0200 http://bitbucket.org/pypy/pypy/changeset/4cd8686d3e2d/ Log: merge diff --git a/pypy/interpreter/astcompiler/asthelpers.py b/pypy/interpreter/astcompiler/asthelpers.py --- a/pypy/interpreter/astcompiler/asthelpers.py +++ b/pypy/interpreter/astcompiler/asthelpers.py @@ -43,7 +43,7 @@ if self.elts: for elt in self.elts: elt.set_context(ctx) - self.ctx = ctx + self.ctx = ctx class __extend__(ast.Attribute): diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py --- a/pypy/interpreter/astcompiler/test/test_compiler.py +++ b/pypy/interpreter/astcompiler/test/test_compiler.py @@ -782,6 +782,36 @@ source = "for x in y: pass\n" * 1000 compile_with_astcompiler(source, 'exec', self.space) + def test_assign_to_empty_list_1(self): + source = """if 1: + for i in range(5): + del [] + [] = () + [] = [] + [] = [] = [] + ok = 1 + """ + self.simple_test(source, 'ok', 1) + + def test_assign_to_empty_list_2(self): + source = """if 1: + for i in range(5): + try: [] = 1, 2, 3 + except ValueError: pass + else: raise AssertionError + try: [] = a = 1 + except TypeError: pass + else: raise AssertionError + try: [] = _ = iter(['foo']) + except ValueError: pass + else: raise AssertionError + try: [], _ = iter(['foo']), 1 + except ValueError: pass + else: raise AssertionError + ok = 1 + """ + self.simple_test(source, 'ok', 1) + class AppTestCompiler: diff --git a/rpython/jit/backend/test/test_ll_random.py b/rpython/jit/backend/test/test_ll_random.py --- a/rpython/jit/backend/test/test_ll_random.py +++ b/rpython/jit/backend/test/test_ll_random.py @@ -3,8 +3,7 @@ from rpython.jit.backend.test import test_random from rpython.jit.metainterp.resoperation import ResOperation, rop from rpython.jit.metainterp.history import ConstInt, ConstPtr -from rpython.jit.metainterp.history import BoxPtr, BoxInt -from rpython.jit.metainterp.history import BasicFailDescr +from rpython.jit.metainterp.history import BoxPtr from rpython.jit.codewriter import heaptracker from rpython.jit.codewriter.effectinfo import EffectInfo from rpython.rtyper.annlowlevel import llhelper @@ -440,7 +439,7 @@ def produce_into(self, builder, r): v_string = self.get_string(builder, r) v_index = builder.get_index(len(v_string.getref(self.ptr).chars), r) - v_result = builder.do(self.opnum, [v_string, v_index]) + builder.do(self.opnum, [v_string, v_index]) class AbstractSetItemOperation(AbstractStringOperation): def produce_into(self, builder, r): @@ -556,7 +555,7 @@ descr = self.getcalldescr(builder, TP) self.put(builder, args, descr) op = ResOperation(rop.GUARD_NO_EXCEPTION, [], None, - descr=BasicFailDescr()) + descr=builder.getfaildescr()) op.setfailargs(fail_subset) builder.loop.operations.append(op) @@ -578,7 +577,7 @@ _, vtableptr = builder.get_random_structure_type_and_vtable(r) exc_box = ConstAddr(llmemory.cast_ptr_to_adr(vtableptr), builder.cpu) op = ResOperation(rop.GUARD_EXCEPTION, [exc_box], BoxPtr(), - descr=BasicFailDescr()) + descr=builder.getfaildescr()) op.setfailargs(builder.subset_of_intvars(r)) op._exc_box = None builder.should_fail_by = op @@ -599,7 +598,7 @@ self.put(builder, args, descr) exc_box = ConstAddr(llmemory.cast_ptr_to_adr(exc), builder.cpu) op = ResOperation(rop.GUARD_EXCEPTION, [exc_box], BoxPtr(), - descr=BasicFailDescr()) + descr=builder.getfaildescr()) op.setfailargs(fail_subset) builder.loop.operations.append(op) @@ -615,7 +614,7 @@ descr = self.getcalldescr(builder, TP) self.put(builder, args, descr) op = ResOperation(rop.GUARD_NO_EXCEPTION, [], BoxPtr(), - descr=BasicFailDescr()) + descr=builder.getfaildescr()) op._exc_box = ConstAddr(llmemory.cast_ptr_to_adr(exc), builder.cpu) op.setfailargs(builder.subset_of_intvars(r)) builder.should_fail_by = op @@ -639,7 +638,7 @@ break other_box = ConstAddr(llmemory.cast_ptr_to_adr(vtableptr), builder.cpu) op = ResOperation(rop.GUARD_EXCEPTION, [other_box], BoxPtr(), - descr=BasicFailDescr()) + descr=builder.getfaildescr()) op._exc_box = ConstAddr(llmemory.cast_ptr_to_adr(exc), builder.cpu) op.setfailargs(builder.subset_of_intvars(r)) builder.should_fail_by = op From noreply at buildbot.pypy.org Sun Jan 20 12:21:21 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 20 Jan 2013 12:21:21 +0100 (CET) Subject: [pypy-commit] pypy default: Move signature back to the interpreter Message-ID: <20130120112121.82FB61C00F9@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r60231:dfc78c7fd8d9 Date: 2013-01-20 13:21 +0200 http://bitbucket.org/pypy/pypy/changeset/dfc78c7fd8d9/ Log: Move signature back to the interpreter diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py --- a/pypy/interpreter/gateway.py +++ b/pypy/interpreter/gateway.py @@ -15,7 +15,7 @@ from pypy.interpreter.eval import Code from pypy.interpreter.argument import Arguments -from rpython.rtyper.signature import Signature +from pypy.interpreter.signature import Signature from pypy.interpreter.baseobjspace import (W_Root, ObjSpace, Wrappable, SpaceCache, DescrMismatch) from pypy.interpreter.error import OperationError diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -7,7 +7,7 @@ import dis, imp, struct, types, new, sys from pypy.interpreter import eval -from rpython.rtyper.signature import Signature +from pypy.interpreter.signature import Signature from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import unwrap_spec from pypy.interpreter.astcompiler.consts import ( diff --git a/rpython/rtyper/signature.py b/pypy/interpreter/signature.py rename from rpython/rtyper/signature.py rename to pypy/interpreter/signature.py diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py --- a/pypy/interpreter/test/test_argument.py +++ b/pypy/interpreter/test/test_argument.py @@ -2,7 +2,7 @@ import py from pypy.interpreter.argument import (Arguments, ArgErr, ArgErrUnknownKwds, ArgErrMultipleValues, ArgErrCount) -from rpython.rtyper.signature import Signature +from pypy.interpreter.signature import Signature from pypy.interpreter.error import OperationError diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py --- a/pypy/interpreter/test/test_gateway.py +++ b/pypy/interpreter/test/test_gateway.py @@ -3,7 +3,7 @@ from pypy.interpreter import gateway, argument from pypy.interpreter.gateway import ObjSpace, W_Root, WrappedDefault -from rpython.rtyper.signature import Signature +from pypy.interpreter.signature import Signature import py import sys diff --git a/pypy/module/oracle/interp_pool.py b/pypy/module/oracle/interp_pool.py --- a/pypy/module/oracle/interp_pool.py +++ b/pypy/module/oracle/interp_pool.py @@ -4,8 +4,8 @@ from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.interpreter.typedef import interp_attrproperty, interp_attrproperty_w from pypy.interpreter.error import OperationError +from pypy.interpreter.signature import Signature from rpython.rtyper.lltypesystem import rffi, lltype -from rpython.rtyper.signature import Signature from pypy.module.oracle import roci, config from pypy.module.oracle import interp_error, interp_environ diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py --- a/pypy/objspace/std/bytearrayobject.py +++ b/pypy/objspace/std/bytearrayobject.py @@ -20,12 +20,12 @@ from pypy.objspace.std import slicetype from pypy.interpreter import gateway from pypy.interpreter.buffer import RWBuffer +from pypy.interpreter.signature import Signature from pypy.objspace.std.bytearraytype import ( makebytearraydata_w, getbytevalue, new_bytearray ) from rpython.tool.sourcetools import func_with_new_name -from rpython.rtyper.signature import Signature class W_BytearrayObject(W_Object): diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py --- a/pypy/objspace/std/dictmultiobject.py +++ b/pypy/objspace/std/dictmultiobject.py @@ -3,14 +3,12 @@ from pypy.objspace.std.register_all import register_all from pypy.objspace.std.settype import set_typedef as settypedef from pypy.objspace.std.frozensettype import frozenset_typedef as frozensettypedef -from pypy.interpreter import gateway from pypy.interpreter.error import OperationError, operationerrfmt +from pypy.interpreter.signature import Signature -from rpython.rlib.objectmodel import r_dict, we_are_translated, specialize,\ - newlist_hint +from rpython.rlib.objectmodel import r_dict, specialize, newlist_hint from rpython.rlib.debug import mark_dict_non_null from rpython.tool.sourcetools import func_with_new_name -from rpython.rtyper.signature import Signature from rpython.rlib import rerased from rpython.rlib import jit diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py --- a/pypy/objspace/std/listobject.py +++ b/pypy/objspace/std/listobject.py @@ -8,11 +8,11 @@ from pypy.objspace.std.sliceobject import W_SliceObject, normalize_simple_slice from pypy.objspace.std import slicetype from pypy.interpreter import gateway, baseobjspace +from pypy.interpreter.signature import Signature from rpython.rlib.objectmodel import (instantiate, newlist_hint, specialize, resizelist_hint) from rpython.rlib.listsort import make_timsort_class from rpython.rlib import rerased, jit, debug -from rpython.rtyper.signature import Signature from rpython.tool.sourcetools import func_with_new_name UNROLL_CUTOFF = 5 diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py --- a/pypy/objspace/std/setobject.py +++ b/pypy/objspace/std/setobject.py @@ -1,20 +1,20 @@ from pypy.objspace.std.model import registerimplementation, W_Object from pypy.objspace.std.register_all import register_all -from rpython.rlib.objectmodel import r_dict -from rpython.rlib.rarithmetic import intmask, r_uint from pypy.interpreter.error import OperationError from pypy.interpreter import gateway from pypy.objspace.std.settype import set_typedef as settypedef from pypy.objspace.std.frozensettype import frozenset_typedef as frozensettypedef -from rpython.rlib import rerased -from rpython.rlib.objectmodel import instantiate -from rpython.rtyper.signature import Signature +from pypy.interpreter.signature import Signature from pypy.interpreter.generator import GeneratorIterator from pypy.objspace.std.listobject import W_ListObject from pypy.objspace.std.intobject import W_IntObject from pypy.objspace.std.stringobject import W_StringObject from pypy.objspace.std.unicodeobject import W_UnicodeObject +from rpython.rlib.objectmodel import r_dict +from rpython.rlib.rarithmetic import intmask, r_uint +from rpython.rlib import rerased + class W_BaseSetObject(W_Object): typedef = None diff --git a/rpython/rtyper/normalizecalls.py b/rpython/rtyper/normalizecalls.py --- a/rpython/rtyper/normalizecalls.py +++ b/rpython/rtyper/normalizecalls.py @@ -1,5 +1,5 @@ from rpython.annotator import model as annmodel, description -from rpython.rtyper.signature import Signature +from rpython.flowspace.argument import Signature from rpython.flowspace.model import (Variable, Constant, Block, Link, checkgraph, FunctionGraph, SpaceOperation) from rpython.rlib.objectmodel import ComputedIntSymbolic From noreply at buildbot.pypy.org Sun Jan 20 12:26:14 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 20 Jan 2013 12:26:14 +0100 (CET) Subject: [pypy-commit] pypy default: shuffle stuff around, move rpython to rpython/bin Message-ID: <20130120112614.2CA1A1C0237@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r60232:e84c29970081 Date: 2013-01-20 13:25 +0200 http://bitbucket.org/pypy/pypy/changeset/e84c29970081/ Log: shuffle stuff around, move rpython to rpython/bin diff --git a/goal/multibuild.py b/pypy/goal/multibuild.py rename from goal/multibuild.py rename to pypy/goal/multibuild.py diff --git a/goal/targetnumpystandalone.py b/pypy/goal/targetnumpystandalone.py rename from goal/targetnumpystandalone.py rename to pypy/goal/targetnumpystandalone.py diff --git a/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py rename from goal/targetpypystandalone.py rename to pypy/goal/targetpypystandalone.py diff --git a/pypy/bin/rpython b/rpython/bin/rpython rename from pypy/bin/rpython rename to rpython/bin/rpython From noreply at buildbot.pypy.org Sun Jan 20 14:46:42 2013 From: noreply at buildbot.pypy.org (amauryfa) Date: Sun, 20 Jan 2013 14:46:42 +0100 (CET) Subject: [pypy-commit] pypy py3k: hg merge default Message-ID: <20130120134642.2FD3E1C0237@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r60233:21ef66d8d225 Date: 2013-01-20 14:43 +0100 http://bitbucket.org/pypy/pypy/changeset/21ef66d8d225/ Log: hg merge default This contains the large change made by rpython- split. Tested with "bin/pyinteractive.py --allworking" diff too long, truncating to 2000 out of 755607 lines diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -85,3 +85,4 @@ ^compiled ^.git/ ^release/ +^rpython/_cache$ diff --git a/dotviewer/test/test_translator.py b/dotviewer/test/test_translator.py --- a/dotviewer/test/test_translator.py +++ b/dotviewer/test/test_translator.py @@ -21,7 +21,7 @@ def test_annotated(): - from pypy.translator.interactive import Translation + from rpython.translator.interactive import Translation t = Translation(is_prime) t.annotate([int]) t.viewcg() diff --git a/goal/multibuild.py b/goal/multibuild.py new file mode 100644 --- /dev/null +++ b/goal/multibuild.py @@ -0,0 +1,127 @@ +from pypy.config.pypyoption import get_pypy_config +from rpython.translator.goal import translate +from rpython.translator.goal import targetpypystandalone +from rpython.translator.driver import TranslationDriver +import os, sys, traceback, random + +def longoptfromname(config, name): + from pypy.config.makerestdoc import get_cmdline + # begin horror + h, n = config._cfgimpl_get_home_by_path(name) + opt = getattr(h._cfgimpl_descr, n) + # end horror + cmdline = get_cmdline(opt.cmdline, name) + assert cmdline is not None + shortest_long_option = 'X'*1000 + for cmd in cmdline.split(): + if cmd.startswith('--') and len(cmd) < len(shortest_long_option): + shortest_long_option = cmd + return shortest_long_option + +def exe_name_from_options(config, opts): + from pypy.module.sys.version import svn_revision + + backend = config.translation.backend + if not backend: + backend = 'c' + rev = svn_revision() + + nameparts = [] + + for opt, v in opts.iteritems(): + if opt == 'translation.backend': + backend = v + optname = longoptfromname(config, opt).strip('-') + if v is False: + optname = 'no-' + optname + elif v is not True: + optname += '=' + str(v) + nameparts.append(optname) + + suffix = '' + if nameparts: + def k(s): + if s.startswith('no-'): + return s[3:] + else: + return s + nameparts.sort(key=k) + suffix = '-' + '-'.join(nameparts) + + return 'pypy-%s-%d%s'%(backend, rev, suffix) + +def _build(config, exe_name): + driver = TranslationDriver.from_targetspec( + targetpypystandalone.__dict__, + config=config) + driver.exe_name = exe_name + driver.compile() + +def build_pypy_with_options(basedir, opts): + config = get_pypy_config(translate.OVERRIDES, translating=True) + + try: + config.set(**opts) + except: + return exe_name_from_options(config, opts), "didn't configure" + + exe_name = os.path.join(basedir, exe_name_from_options(config, opts)) + + print exe_name, + sys.stdout.flush() + + pid = os.fork() + + if pid == 0: + logfile = open(exe_name + '-log', 'w') + davenull = os.open('/dev/null', os.O_RDONLY) + os.dup2(davenull, 0) + os.dup2(logfile.fileno(), 1) + os.dup2(logfile.fileno(), 2) + try: + try: + r = _build(config, exe_name) + finally: + logfile.close() + except: + os._exit(1) + else: + os._exit(0) + else: + pid, status = os.waitpid(pid, 0) + if status: + r = 'failed' + else: + r = 'succeeded' + print r + return exe_name, r + +def get_options(fname): + def gen_opts(sofar, remaining): + if not remaining: + yield sofar + else: + for (k, v) in remaining[0]: + d2 = sofar.copy() + d2[k] = v + for d in gen_opts(d2, remaining[1:]): + yield d + options = [] + for line in open(fname): + l = [] + optname, options_ = line.split(':') + options.append([(optname.strip(), eval(optval.strip())) for optval in options_.split(',')]) + return gen_opts({}, options) + + +if __name__ == '__main__': + basedir = sys.argv[1] + optionsfile = sys.argv[2] + results = [] + options = list(get_options(optionsfile)) + random.shuffle(options) + for opts in options: + results.append(build_pypy_with_options(basedir, opts)) + out = open(os.path.join(basedir, 'results'), 'w') + for exe, r in results: + print >>out, exe, r diff --git a/goal/targetnumpystandalone.py b/goal/targetnumpystandalone.py new file mode 100644 --- /dev/null +++ b/goal/targetnumpystandalone.py @@ -0,0 +1,43 @@ + +""" Usage: + +./targetnumpystandalone-c array_size + +Will execute a give numpy bytecode. Arrays will be ranges (in float) modulo 10, +constants would be consecutive starting from one. + +Bytecode should contain letters 'a' 'l' and 'f' so far and be correct +""" + +import time +from pypy.module.micronumpy.compile import numpy_compile +from rpython.jit.codewriter.policy import JitPolicy +from rpython.rtyper.annlowlevel import hlstr + +def entry_point(argv): + if len(argv) != 3: + print __doc__ + return 1 + try: + size = int(argv[2]) + except ValueError: + print "INVALID LITERAL FOR INT:", argv[2] + print __doc__ + return 3 + t0 = time.time() + main(argv[0], size) + print "bytecode:", argv[0], "size:", size + print "took:", time.time() - t0 + return 0 + +def main(bc, size): + if not isinstance(bc, str): + bc = hlstr(bc) # for tests + a = numpy_compile(bc, size) + a = a.compute() + +def target(*args): + return entry_point, None + +def jitpolicy(driver): + return JitPolicy() diff --git a/goal/targetpypystandalone.py b/goal/targetpypystandalone.py new file mode 100644 --- /dev/null +++ b/goal/targetpypystandalone.py @@ -0,0 +1,246 @@ +import py + +import os, sys + +from pypy.interpreter import gateway +from pypy.interpreter.error import OperationError +from pypy.tool.ann_override import PyPyAnnotatorPolicy +from rpython.config.config import Config, to_optparse, make_dict, SUPPRESS_USAGE +from rpython.config.config import ConflictConfigError +from pypy.tool.option import make_objspace +from pypy.conftest import pypydir + +thisdir = py.path.local(__file__).dirpath() + +try: + this_dir = os.path.dirname(__file__) +except NameError: + this_dir = os.path.dirname(sys.argv[0]) + +def debug(msg): + os.write(2, "debug: " + msg + '\n') + +# __________ Entry point __________ + +def create_entry_point(space, w_dict): + w_entry_point = space.getitem(w_dict, space.wrap('entry_point')) + w_run_toplevel = space.getitem(w_dict, space.wrap('run_toplevel')) + w_call_finish_gateway = space.wrap(gateway.interp2app(call_finish)) + w_call_startup_gateway = space.wrap(gateway.interp2app(call_startup)) + withjit = space.config.objspace.usemodules.pypyjit + + def entry_point(argv): + if withjit: + from rpython.jit.backend.hlinfo import highleveljitinfo + highleveljitinfo.sys_executable = argv[0] + + #debug("entry point starting") + #for arg in argv: + # debug(" argv -> " + arg) + if len(argv) > 2 and argv[1] == '--heapsize': + # Undocumented option, handled at interp-level. + # It has silently no effect with some GCs. + # It works in Boehm and in the semispace or generational GCs + # (but see comments in semispace.py:set_max_heap_size()). + # At the moment this option exists mainly to support sandboxing. + from rpython.rlib import rgc + rgc.set_max_heap_size(int(argv[2])) + argv = argv[:1] + argv[3:] + try: + try: + space.call_function(w_run_toplevel, w_call_startup_gateway) + w_executable = space.wrap(argv[0]) + w_argv = space.newlist([space.wrap(s) for s in argv[1:]]) + w_exitcode = space.call_function(w_entry_point, w_executable, w_argv) + exitcode = space.int_w(w_exitcode) + # try to pull it all in + ## from pypy.interpreter import main, interactive, error + ## con = interactive.PyPyConsole(space) + ## con.interact() + except OperationError, e: + debug("OperationError:") + debug(" operror-type: " + e.w_type.getname(space)) + debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space)))) + return 1 + finally: + try: + space.call_function(w_run_toplevel, w_call_finish_gateway) + except OperationError, e: + debug("OperationError:") + debug(" operror-type: " + e.w_type.getname(space)) + debug(" operror-value: " + space.str_w(space.str(e.get_w_value(space)))) + return 1 + return exitcode + return entry_point + +def call_finish(space): + space.finish() + +def call_startup(space): + space.startup() + +# _____ Define and setup target ___ + +# for now this will do for option handling + +class PyPyTarget(object): + + usage = SUPPRESS_USAGE + + take_options = True + + def opt_parser(self, config): + parser = to_optparse(config, useoptions=["objspace.*"], + parserkwargs={'usage': self.usage}) + return parser + + def handle_config(self, config, translateconfig): + if (not translateconfig.help and + translateconfig._cfgimpl_value_owners['opt'] == 'default'): + raise Exception("You have to specify the --opt level.\n" + "Try --opt=2 or --opt=jit, or equivalently -O2 or -Ojit .") + self.translateconfig = translateconfig + # set up the objspace optimizations based on the --opt argument + from pypy.config.pypyoption import set_pypy_opt_level + set_pypy_opt_level(config, translateconfig.opt) + + # as of revision 27081, multimethod.py uses the InstallerVersion1 by default + # because it is much faster both to initialize and run on top of CPython. + # The InstallerVersion2 is optimized for making a translator-friendly + # structure for low level backends. However, InstallerVersion1 is still + # preferable for high level backends, so we patch here. + + from pypy.objspace.std import multimethod + if config.objspace.std.multimethods == 'mrd': + assert multimethod.InstallerVersion1.instance_counter == 0,\ + 'The wrong Installer version has already been instatiated' + multimethod.Installer = multimethod.InstallerVersion2 + elif config.objspace.std.multimethods == 'doubledispatch': + # don't rely on the default, set again here + assert multimethod.InstallerVersion2.instance_counter == 0,\ + 'The wrong Installer version has already been instatiated' + multimethod.Installer = multimethod.InstallerVersion1 + + def print_help(self, config): + self.opt_parser(config).print_help() + + def get_additional_config_options(self): + from pypy.config.pypyoption import pypy_optiondescription + return pypy_optiondescription + + def target(self, driver, args): + driver.exe_name = 'pypy-%(backend)s' + + config = driver.config + parser = self.opt_parser(config) + + parser.parse_args(args) + + # expose the following variables to ease debugging + global space, entry_point + + if config.objspace.allworkingmodules: + from pypy.config.pypyoption import enable_allworkingmodules + enable_allworkingmodules(config) + if config.objspace.translationmodules: + from pypy.config.pypyoption import enable_translationmodules + enable_translationmodules(config) + + ## if config.translation.type_system == 'ootype': + ## config.objspace.usemodules.suggest(rbench=True) + + config.translation.suggest(check_str_without_nul=True) + + if config.translation.thread: + config.objspace.usemodules.thread = True + elif config.objspace.usemodules.thread: + try: + config.translation.thread = True + except ConflictConfigError: + # If --allworkingmodules is given, we reach this point + # if threads cannot be enabled (e.g. they conflict with + # something else). In this case, we can try setting the + # usemodules.thread option to False again. It will + # cleanly fail if that option was set to True by the + # command-line directly instead of via --allworkingmodules. + config.objspace.usemodules.thread = False + + if config.translation.continuation: + config.objspace.usemodules._continuation = True + elif config.objspace.usemodules._continuation: + try: + config.translation.continuation = True + except ConflictConfigError: + # Same as above: try to auto-disable the _continuation + # module if translation.continuation cannot be enabled + config.objspace.usemodules._continuation = False + + if not config.translation.rweakref: + config.objspace.usemodules._weakref = False + + if config.translation.jit: + config.objspace.usemodules.pypyjit = True + elif config.objspace.usemodules.pypyjit: + config.translation.jit = True + + if config.translation.backend == "cli": + config.objspace.usemodules.clr = True + # XXX did it ever work? + #elif config.objspace.usemodules.clr: + # config.translation.backend == "cli" + + if config.translation.sandbox: + config.objspace.usepycfiles = False + + config.translating = True + + import translate + translate.log_config(config.objspace, "PyPy config object") + + # obscure hack to stuff the translation options into the translated PyPy + import pypy.module.sys + options = make_dict(config) + wrapstr = 'space.wrap(%r)' % (options) + pypy.module.sys.Module.interpleveldefs['pypy_translation_info'] = wrapstr + + if config.translation.backend in ["cli", "jvm"] and sys.platform == "win32": + # HACK: The ftruncate implementation in streamio.py which is used for the Win32 platform + # is specific for the C backend and can't be generated on CLI or JVM. Because of that, + # we have to patch it out. + from rpython.rlib import streamio + def ftruncate_win32_dummy(fd, size): pass + def _setfd_binary_dummy(fd): pass + streamio.ftruncate_win32 = ftruncate_win32_dummy + streamio._setfd_binary = _setfd_binary_dummy + + return self.get_entry_point(config) + + def jitpolicy(self, driver): + from pypy.module.pypyjit.policy import PyPyJitPolicy, pypy_hooks + return PyPyJitPolicy(pypy_hooks) + + def get_entry_point(self, config): + from pypy.tool.lib_pypy import import_from_lib_pypy + rebuild = import_from_lib_pypy('ctypes_config_cache/rebuild') + rebuild.try_rebuild() + + space = make_objspace(config) + + # manually imports app_main.py + filename = os.path.join(pypydir, 'interpreter', 'app_main.py') + app = gateway.applevel(open(filename).read(), 'app_main.py', 'app_main') + app.hidden_applevel = False + w_dict = app.getwdict(space) + entry_point = create_entry_point(space, w_dict) + + return entry_point, None, PyPyAnnotatorPolicy(single_space = space) + + def interface(self, ns): + for name in ['take_options', 'handle_config', 'print_help', 'target', + 'jitpolicy', 'get_entry_point', + 'get_additional_config_options']: + ns[name] = getattr(self, name) + + +PyPyTarget().interface(globals()) + diff --git a/lib_pypy/ctypes_config_cache/autopath.py b/lib_pypy/ctypes_config_cache/autopath.py deleted file mode 100644 --- a/lib_pypy/ctypes_config_cache/autopath.py +++ /dev/null @@ -1,131 +0,0 @@ -""" -self cloning, automatic path configuration - -copy this into any subdirectory of pypy from which scripts need -to be run, typically all of the test subdirs. -The idea is that any such script simply issues - - import autopath - -and this will make sure that the parent directory containing "pypy" -is in sys.path. - -If you modify the master "autopath.py" version (in pypy/tool/autopath.py) -you can directly run it which will copy itself on all autopath.py files -it finds under the pypy root directory. - -This module always provides these attributes: - - pypydir pypy root directory path - this_dir directory where this autopath.py resides - -""" - -def __dirinfo(part): - """ return (partdir, this_dir) and insert parent of partdir - into sys.path. If the parent directories don't have the part - an EnvironmentError is raised.""" - - import sys, os - try: - head = this_dir = os.path.realpath(os.path.dirname(__file__)) - except NameError: - head = this_dir = os.path.realpath(os.path.dirname(sys.argv[0])) - - error = None - while head: - partdir = head - head, tail = os.path.split(head) - if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') - if not os.path.exists(checkfile): - error = "Cannot find %r" % (os.path.normpath(checkfile),) - break - else: - error = "Cannot find the parent directory %r of the path %r" % ( - partdir, this_dir) - if not error: - # check for bogus end-of-line style (e.g. files checked out on - # Windows and moved to Unix) - f = open(__file__.replace('.pyc', '.py'), 'r') - data = f.read() - f.close() - if data.endswith('\r\n') or data.endswith('\r'): - error = ("Bad end-of-line style in the .py files. Typically " - "caused by a zip file or a checkout done on Windows and " - "moved to Unix or vice-versa.") - if error: - raise EnvironmentError("Invalid source tree - bogus checkout! " + - error) - - pypy_root = os.path.join(head, '') - try: - sys.path.remove(head) - except ValueError: - pass - sys.path.insert(0, head) - - munged = {} - for name, mod in sys.modules.items(): - if '.' in name: - continue - fn = getattr(mod, '__file__', None) - if not isinstance(fn, str): - continue - newname = os.path.splitext(os.path.basename(fn))[0] - if not newname.startswith(part + '.'): - continue - path = os.path.join(os.path.dirname(os.path.realpath(fn)), '') - if path.startswith(pypy_root) and newname != part: - modpaths = os.path.normpath(path[len(pypy_root):]).split(os.sep) - if newname != '__init__': - modpaths.append(newname) - modpath = '.'.join(modpaths) - if modpath not in sys.modules: - munged[modpath] = mod - - for name, mod in munged.iteritems(): - if name not in sys.modules: - sys.modules[name] = mod - if '.' in name: - prename = name[:name.rfind('.')] - postname = name[len(prename)+1:] - if prename not in sys.modules: - __import__(prename) - if not hasattr(sys.modules[prename], postname): - setattr(sys.modules[prename], postname, mod) - - return partdir, this_dir - -def __clone(): - """ clone master version of autopath.py into all subdirs """ - from os.path import join, walk - if not this_dir.endswith(join('pypy','tool')): - raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) - - - def sync_walker(arg, dirname, fnames): - if _myname in fnames: - fn = join(dirname, _myname) - f = open(fn, 'rwb+') - try: - if f.read() == arg: - print "checkok", fn - else: - print "syncing", fn - f = open(fn, 'w') - f.write(arg) - finally: - f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) - -_myname = 'autopath.py' - -# set guaranteed attributes - -pypydir, this_dir = __dirinfo('pypy') - -if __name__ == '__main__': - __clone() diff --git a/lib_pypy/ctypes_config_cache/dumpcache.py b/lib_pypy/ctypes_config_cache/dumpcache.py --- a/lib_pypy/ctypes_config_cache/dumpcache.py +++ b/lib_pypy/ctypes_config_cache/dumpcache.py @@ -1,6 +1,6 @@ import os from ctypes_configure import dumpcache -from pypy.jit.backend import detect_cpu +from rpython.jit.backend import detect_cpu def dumpcache2(basename, config): model = detect_cpu.autodetect_main_model_and_size() @@ -14,7 +14,7 @@ try: from __pypy__ import cpumodel except ImportError: - from pypy.jit.backend import detect_cpu + from rpython.jit.backend import detect_cpu cpumodel = detect_cpu.autodetect_main_model_and_size() # XXX relative import, should be removed together with # XXX the relative imports done e.g. by lib_pypy/pypy_test/test_hashlib diff --git a/lib_pypy/ctypes_config_cache/rebuild.py b/lib_pypy/ctypes_config_cache/rebuild.py --- a/lib_pypy/ctypes_config_cache/rebuild.py +++ b/lib_pypy/ctypes_config_cache/rebuild.py @@ -1,21 +1,15 @@ #! /usr/bin/env python # Run this script to rebuild all caches from the *.ctc.py files. -# hack: we cannot directly import autopath, as we are outside the pypy -# package. However, we pretend to be inside pypy/tool and manually run it, to -# get the correct path -import os.path -this_dir = os.path.dirname(__file__) -autopath_py = os.path.join(this_dir, '../../pypy/tool/autopath.py') -autopath_py = os.path.abspath(autopath_py) -execfile(autopath_py, dict(__name__='autopath', __file__=autopath_py)) +import os, sys -import os, sys +sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..'))) + import py _dirpath = os.path.dirname(__file__) or os.curdir -from pypy.tool.ansi_print import ansi_log +from rpython.tool.ansi_print import ansi_log log = py.log.Producer("ctypes_config_cache") py.log.setconsumer("ctypes_config_cache", ansi_log) @@ -31,7 +25,7 @@ sys.path[:] = path def try_rebuild(): - from pypy.jit.backend import detect_cpu + from rpython.jit.backend import detect_cpu model = detect_cpu.autodetect_main_model_and_size() # remove the files '_*_model_.py' left = {} diff --git a/lib_pypy/ctypes_config_cache/test/test_cache.py b/lib_pypy/ctypes_config_cache/test/test_cache.py --- a/lib_pypy/ctypes_config_cache/test/test_cache.py +++ b/lib_pypy/ctypes_config_cache/test/test_cache.py @@ -1,6 +1,6 @@ import py import sys, os -from pypy.tool.udir import udir +from rpython.tool.udir import udir dirpath = py.path.local(__file__).dirpath().dirpath() diff --git a/lib_pypy/pypy_test/test_dbm_extra.py b/lib_pypy/pypy_test/test_dbm_extra.py --- a/lib_pypy/pypy_test/test_dbm_extra.py +++ b/lib_pypy/pypy_test/test_dbm_extra.py @@ -1,5 +1,5 @@ import py -from pypy.tool.udir import udir +from rpython.tool.udir import udir try: from lib_pypy import dbm except ImportError as e: diff --git a/lib_pypy/pypy_test/test_marshal_extra.py b/lib_pypy/pypy_test/test_marshal_extra.py --- a/lib_pypy/pypy_test/test_marshal_extra.py +++ b/lib_pypy/pypy_test/test_marshal_extra.py @@ -3,7 +3,7 @@ import marshal as cpy_marshal from lib_pypy import _marshal as marshal -from pypy.tool.udir import udir +from rpython.tool.udir import udir hello = "he" hello += "llo" diff --git a/pypy/annotation/annrpython.py b/pypy/annotation/annrpython.py deleted file mode 100644 --- a/pypy/annotation/annrpython.py +++ /dev/null @@ -1,676 +0,0 @@ -from __future__ import absolute_import - -import types -from pypy.tool.ansi_print import ansi_log -from pypy.tool.pairtype import pair -from pypy.tool.error import (format_blocked_annotation_error, - AnnotatorError, gather_error, ErrorWrapper) -from pypy.objspace.flow.model import (Variable, Constant, FunctionGraph, - c_last_exception, checkgraph) -from pypy.translator import simplify, transform -from pypy.annotation import model as annmodel, signature, unaryop, binaryop -from pypy.annotation.bookkeeper import Bookkeeper -import py -log = py.log.Producer("annrpython") -py.log.setconsumer("annrpython", ansi_log) - - -FAIL = object() - -class RPythonAnnotator(object): - """Block annotator for RPython. - See description in doc/translation.txt.""" - - def __init__(self, translator=None, policy=None, bookkeeper=None): - import pypy.rpython.ootypesystem.ooregistry # has side effects - import pypy.rpython.extfuncregistry # has side effects - import pypy.rlib.nonconst # has side effects - - if translator is None: - # interface for tests - from pypy.translator.translator import TranslationContext - translator = TranslationContext() - translator.annotator = self - self.translator = translator - self.pendingblocks = {} # map {block: graph-containing-it} - self.bindings = {} # map Variables to SomeValues - self.annotated = {} # set of blocks already seen - self.added_blocks = None # see processblock() below - self.links_followed = {} # set of links that have ever been followed - self.notify = {} # {block: {positions-to-reflow-from-when-done}} - self.fixed_graphs = {} # set of graphs not to annotate again - self.blocked_blocks = {} # set of {blocked_block: (graph, index)} - # --- the following information is recorded for debugging --- - self.blocked_graphs = {} # set of graphs that have blocked blocks - # --- end of debugging information --- - self.frozen = False - if policy is None: - from pypy.annotation.policy import AnnotatorPolicy - self.policy = AnnotatorPolicy() - else: - self.policy = policy - if bookkeeper is None: - bookkeeper = Bookkeeper(self) - self.bookkeeper = bookkeeper - - def __getstate__(self): - attrs = """translator pendingblocks bindings annotated links_followed - notify bookkeeper frozen policy added_blocks""".split() - ret = self.__dict__.copy() - for key, value in ret.items(): - if key not in attrs: - assert type(value) is dict, ( - "%r is not dict. please update %s.__getstate__" % - (key, self.__class__.__name__)) - ret[key] = {} - return ret - - #___ convenience high-level interface __________________ - - def build_types(self, function, input_arg_types, complete_now=True, - main_entry_point=False): - """Recursively build annotations about the specific entry point.""" - assert isinstance(function, types.FunctionType), "fix that!" - - from pypy.annotation.policy import AnnotatorPolicy - policy = AnnotatorPolicy() - # make input arguments and set their type - args_s = [self.typeannotation(t) for t in input_arg_types] - - # XXX hack - annmodel.TLS.check_str_without_nul = ( - self.translator.config.translation.check_str_without_nul) - - flowgraph, inputcells = self.get_call_parameters(function, args_s, policy) - if not isinstance(flowgraph, FunctionGraph): - assert isinstance(flowgraph, annmodel.SomeObject) - return flowgraph - - if main_entry_point: - self.translator.entry_point_graph = flowgraph - return self.build_graph_types(flowgraph, inputcells, complete_now=complete_now) - - def get_call_parameters(self, function, args_s, policy): - desc = self.bookkeeper.getdesc(function) - args = self.bookkeeper.build_args("simple_call", args_s[:]) - result = [] - def schedule(graph, inputcells): - result.append((graph, inputcells)) - return annmodel.s_ImpossibleValue - - prevpolicy = self.policy - self.policy = policy - self.bookkeeper.enter(None) - try: - desc.pycall(schedule, args, annmodel.s_ImpossibleValue) - finally: - self.bookkeeper.leave() - self.policy = prevpolicy - [(graph, inputcells)] = result - return graph, inputcells - - def annotate_helper(self, function, args_s, policy=None): - if policy is None: - from pypy.annotation.policy import AnnotatorPolicy - policy = AnnotatorPolicy() - graph, inputcells = self.get_call_parameters(function, args_s, policy) - self.build_graph_types(graph, inputcells, complete_now=False) - self.complete_helpers(policy) - return graph - - def complete_helpers(self, policy): - saved = self.policy, self.added_blocks - self.policy = policy - try: - self.added_blocks = {} - self.complete() - # invoke annotation simplifications for the new blocks - self.simplify(block_subset=self.added_blocks) - finally: - self.policy, self.added_blocks = saved - - def build_graph_types(self, flowgraph, inputcells, complete_now=True): - checkgraph(flowgraph) - - nbarg = len(flowgraph.getargs()) - if len(inputcells) != nbarg: - raise TypeError("%s expects %d args, got %d" %( - flowgraph, nbarg, len(inputcells))) - - # register the entry point - self.addpendinggraph(flowgraph, inputcells) - # recursively proceed until no more pending block is left - if complete_now: - self.complete() - return self.binding(flowgraph.getreturnvar(), None) - - def gettype(self, variable): - """Return the known type of a control flow graph variable, - defaulting to 'object'.""" - if isinstance(variable, Constant): - return type(variable.value) - elif isinstance(variable, Variable): - cell = self.bindings.get(variable) - if cell: - return cell.knowntype - else: - return object - else: - raise TypeError, ("Variable or Constant instance expected, " - "got %r" % (variable,)) - - def getuserclassdefinitions(self): - """Return a list of ClassDefs.""" - return self.bookkeeper.classdefs - - #___ medium-level interface ____________________________ - - def addpendinggraph(self, flowgraph, inputcells): - self.addpendingblock(flowgraph, flowgraph.startblock, inputcells) - - def addpendingblock(self, graph, block, cells): - """Register an entry point into block with the given input cells.""" - if graph in self.fixed_graphs: - # special case for annotating/rtyping in several phases: calling - # a graph that has already been rtyped. Safety-check the new - # annotations that are passed in, and don't annotate the old - # graph -- it's already low-level operations! - for a, s_newarg in zip(graph.getargs(), cells): - s_oldarg = self.binding(a) - assert annmodel.unionof(s_oldarg, s_newarg) == s_oldarg - else: - assert not self.frozen - for a in cells: - assert isinstance(a, annmodel.SomeObject) - if block not in self.annotated: - self.bindinputargs(graph, block, cells) - else: - self.mergeinputargs(graph, block, cells) - if not self.annotated[block]: - self.pendingblocks[block] = graph - - def complete(self): - """Process pending blocks until none is left.""" - while True: - while self.pendingblocks: - block, graph = self.pendingblocks.popitem() - self.processblock(graph, block) - self.policy.no_more_blocks_to_annotate(self) - if not self.pendingblocks: - break # finished - # make sure that the return variables of all graphs is annotated - if self.added_blocks is not None: - newgraphs = [self.annotated[block] for block in self.added_blocks] - newgraphs = dict.fromkeys(newgraphs) - got_blocked_blocks = False in newgraphs - else: - newgraphs = self.translator.graphs #all of them - got_blocked_blocks = False in self.annotated.values() - if got_blocked_blocks: - for graph in self.blocked_graphs.values(): - self.blocked_graphs[graph] = True - - blocked_blocks = [block for block, done in self.annotated.items() - if done is False] - assert len(blocked_blocks) == len(self.blocked_blocks) - - text = format_blocked_annotation_error(self, self.blocked_blocks) - #raise SystemExit() - raise AnnotatorError(text) - for graph in newgraphs: - v = graph.getreturnvar() - if v not in self.bindings: - self.setbinding(v, annmodel.s_ImpossibleValue) - # policy-dependent computation - self.bookkeeper.compute_at_fixpoint() - - def binding(self, arg, default=FAIL): - "Gives the SomeValue corresponding to the given Variable or Constant." - if isinstance(arg, Variable): - try: - return self.bindings[arg] - except KeyError: - if default is not FAIL: - return default - else: - raise - elif isinstance(arg, Constant): - #if arg.value is undefined_value: # undefined local variables - # return annmodel.s_ImpossibleValue - return self.bookkeeper.immutableconstant(arg) - else: - raise TypeError, 'Variable or Constant expected, got %r' % (arg,) - - def typeannotation(self, t): - return signature.annotation(t, self.bookkeeper) - - def setbinding(self, arg, s_value): - if arg in self.bindings: - assert s_value.contains(self.bindings[arg]) - self.bindings[arg] = s_value - - def transfer_binding(self, v_target, v_source): - assert v_source in self.bindings - self.bindings[v_target] = self.bindings[v_source] - - def warning(self, msg, pos=None): - if pos is None: - try: - pos = self.bookkeeper.position_key - except AttributeError: - pos = '?' - if pos != '?': - pos = self.whereami(pos) - - log.WARNING("%s/ %s" % (pos, msg)) - - - #___ interface for annotator.bookkeeper _______ - - def recursivecall(self, graph, whence, inputcells): - if isinstance(whence, tuple): - parent_graph, parent_block, parent_index = whence - tag = parent_block, parent_index - self.translator.update_call_graph(parent_graph, graph, tag) - # self.notify[graph.returnblock] is a dictionary of call - # points to this func which triggers a reflow whenever the - # return block of this graph has been analysed. - callpositions = self.notify.setdefault(graph.returnblock, {}) - if whence is not None: - if callable(whence): - def callback(): - whence(self, graph) - else: - callback = whence - callpositions[callback] = True - - # generalize the function's input arguments - self.addpendingblock(graph, graph.startblock, inputcells) - - # get the (current) return value - v = graph.getreturnvar() - try: - return self.bindings[v] - except KeyError: - # the function didn't reach any return statement so far. - # (some functions actually never do, they always raise exceptions) - return annmodel.s_ImpossibleValue - - def reflowfromposition(self, position_key): - graph, block, index = position_key - self.reflowpendingblock(graph, block) - - - #___ simplification (should be moved elsewhere?) _______ - - def simplify(self, block_subset=None, extra_passes=None): - # Generic simplifications - transform.transform_graph(self, block_subset=block_subset, - extra_passes=extra_passes) - if block_subset is None: - graphs = self.translator.graphs - else: - graphs = {} - for block in block_subset: - graph = self.annotated.get(block) - if graph: - graphs[graph] = True - for graph in graphs: - simplify.eliminate_empty_blocks(graph) - - - #___ flowing annotations in blocks _____________________ - - def processblock(self, graph, block): - # Important: this is not called recursively. - # self.flowin() can only issue calls to self.addpendingblock(). - # The analysis of a block can be in three states: - # * block not in self.annotated: - # never seen the block. - # * self.annotated[block] == False: - # the input variables of the block are in self.bindings but we - # still have to consider all the operations in the block. - # * self.annotated[block] == graph-containing-block: - # analysis done (at least until we find we must generalize the - # input variables). - - #print '* processblock', block, cells - self.annotated[block] = graph - if block in self.blocked_blocks: - del self.blocked_blocks[block] - try: - self.flowin(graph, block) - except BlockedInference, e: - self.annotated[block] = False # failed, hopefully temporarily - self.blocked_blocks[block] = (graph, e.opindex) - except Exception, e: - # hack for debug tools only - if not hasattr(e, '__annotator_block'): - setattr(e, '__annotator_block', block) - raise - - # The dict 'added_blocks' is used by rpython.annlowlevel to - # detect which are the new blocks that annotating an additional - # small helper creates. - if self.added_blocks is not None: - self.added_blocks[block] = True - - def reflowpendingblock(self, graph, block): - assert not self.frozen - assert graph not in self.fixed_graphs - self.pendingblocks[block] = graph - assert block in self.annotated - self.annotated[block] = False # must re-flow - self.blocked_blocks[block] = (graph, None) - - def bindinputargs(self, graph, block, inputcells): - # Create the initial bindings for the input args of a block. - assert len(block.inputargs) == len(inputcells) - for a, cell in zip(block.inputargs, inputcells): - self.setbinding(a, cell) - self.annotated[block] = False # must flowin. - self.blocked_blocks[block] = (graph, None) - - def mergeinputargs(self, graph, block, inputcells): - # Merge the new 'cells' with each of the block's existing input - # variables. - oldcells = [self.binding(a) for a in block.inputargs] - try: - unions = [annmodel.unionof(c1,c2) for c1, c2 in zip(oldcells,inputcells)] - except annmodel.UnionError, e: - e.args = e.args + ( - ErrorWrapper(gather_error(self, graph, block, None)),) - raise - # if the merged cells changed, we must redo the analysis - if unions != oldcells: - self.bindinputargs(graph, block, unions) - - def whereami(self, position_key): - graph, block, i = position_key - blk = "" - if block: - at = block.at() - if at: - blk = " block"+at - opid="" - if i is not None: - opid = " op=%d" % i - return repr(graph) + blk + opid - - def flowin(self, graph, block): - #print 'Flowing', block, [self.binding(a) for a in block.inputargs] - try: - for i in range(len(block.operations)): - try: - self.bookkeeper.enter((graph, block, i)) - self.consider_op(block, i) - finally: - self.bookkeeper.leave() - - except BlockedInference, e: - if (e.op is block.operations[-1] and - block.exitswitch == c_last_exception): - # this is the case where the last operation of the block will - # always raise an exception which is immediately caught by - # an exception handler. We then only follow the exceptional - # branches. - exits = [link for link in block.exits - if link.exitcase is not None] - - elif e.op.opname in ('simple_call', 'call_args', 'next'): - # XXX warning, keep the name of the call operations in sync - # with the flow object space. These are the operations for - # which it is fine to always raise an exception. We then - # swallow the BlockedInference and that's it. - # About 'next': see test_annotate_iter_empty_container(). - return - - else: - # other cases are problematic (but will hopefully be solved - # later by reflowing). Throw the BlockedInference up to - # processblock(). - raise - - except annmodel.HarmlesslyBlocked: - return - - else: - # dead code removal: don't follow all exits if the exitswitch - # is known - exits = block.exits - if isinstance(block.exitswitch, Variable): - s_exitswitch = self.bindings[block.exitswitch] - if s_exitswitch.is_constant(): - exits = [link for link in exits - if link.exitcase == s_exitswitch.const] - - # mapping (exitcase, variable) -> s_annotation - # that can be attached to booleans, exitswitches - knowntypedata = getattr(self.bindings.get(block.exitswitch), - "knowntypedata", {}) - - # filter out those exceptions which cannot - # occour for this specific, typed operation. - if block.exitswitch == c_last_exception: - op = block.operations[-1] - if op.opname in binaryop.BINARY_OPERATIONS: - arg1 = self.binding(op.args[0]) - arg2 = self.binding(op.args[1]) - binop = getattr(pair(arg1, arg2), op.opname, None) - can_only_throw = annmodel.read_can_only_throw(binop, arg1, arg2) - elif op.opname in unaryop.UNARY_OPERATIONS: - arg1 = self.binding(op.args[0]) - opname = op.opname - if opname == 'contains': opname = 'op_contains' - unop = getattr(arg1, opname, None) - can_only_throw = annmodel.read_can_only_throw(unop, arg1) - else: - can_only_throw = None - - if can_only_throw is not None: - candidates = can_only_throw - candidate_exits = exits - exits = [] - for link in candidate_exits: - case = link.exitcase - if case is None: - exits.append(link) - continue - covered = [c for c in candidates if issubclass(c, case)] - if covered: - exits.append(link) - candidates = [c for c in candidates if c not in covered] - - for link in exits: - in_except_block = False - - last_exception_var = link.last_exception # may be None for non-exception link - last_exc_value_var = link.last_exc_value # may be None for non-exception link - - if isinstance(link.exitcase, (types.ClassType, type)) \ - and issubclass(link.exitcase, py.builtin.BaseException): - assert last_exception_var and last_exc_value_var - last_exc_value_object = self.bookkeeper.valueoftype(link.exitcase) - last_exception_object = annmodel.SomeType() - if isinstance(last_exception_var, Constant): - last_exception_object.const = last_exception_var.value - last_exception_object.is_type_of = [last_exc_value_var] - - if isinstance(last_exception_var, Variable): - self.setbinding(last_exception_var, last_exception_object) - if isinstance(last_exc_value_var, Variable): - self.setbinding(last_exc_value_var, last_exc_value_object) - - last_exception_object = annmodel.SomeType() - if isinstance(last_exception_var, Constant): - last_exception_object.const = last_exception_var.value - #if link.exitcase is Exception: - # last_exc_value_object = annmodel.SomeObject() - #else: - last_exc_value_vars = [] - in_except_block = True - - ignore_link = False - cells = [] - renaming = {} - for a,v in zip(link.args,link.target.inputargs): - renaming.setdefault(a, []).append(v) - for a,v in zip(link.args,link.target.inputargs): - if a == last_exception_var: - assert in_except_block - cells.append(last_exception_object) - elif a == last_exc_value_var: - assert in_except_block - cells.append(last_exc_value_object) - last_exc_value_vars.append(v) - else: - cell = self.binding(a) - if (link.exitcase, a) in knowntypedata: - knownvarvalue = knowntypedata[(link.exitcase, a)] - cell = pair(cell, knownvarvalue).improve() - # ignore links that try to pass impossible values - if cell == annmodel.s_ImpossibleValue: - ignore_link = True - - if hasattr(cell,'is_type_of'): - renamed_is_type_of = [] - for v in cell.is_type_of: - new_vs = renaming.get(v,[]) - renamed_is_type_of += new_vs - assert cell.knowntype is type - newcell = annmodel.SomeType() - if cell.is_constant(): - newcell.const = cell.const - cell = newcell - cell.is_type_of = renamed_is_type_of - - if hasattr(cell, 'knowntypedata'): - renamed_knowntypedata = {} - for (value, v), s in cell.knowntypedata.items(): - new_vs = renaming.get(v, []) - for new_v in new_vs: - renamed_knowntypedata[value, new_v] = s - assert isinstance(cell, annmodel.SomeBool) - newcell = annmodel.SomeBool() - if cell.is_constant(): - newcell.const = cell.const - cell = newcell - cell.set_knowntypedata(renamed_knowntypedata) - - cells.append(cell) - - if ignore_link: - continue - - if in_except_block: - last_exception_object.is_type_of = last_exc_value_vars - - self.links_followed[link] = True - self.addpendingblock(graph, link.target, cells) - - if block in self.notify: - # reflow from certain positions when this block is done - for callback in self.notify[block]: - if isinstance(callback, tuple): - self.reflowfromposition(callback) # callback is a position - else: - callback() - - - #___ creating the annotations based on operations ______ - - def consider_op(self, block, opindex): - op = block.operations[opindex] - argcells = [self.binding(a) for a in op.args] - consider_meth = getattr(self,'consider_op_'+op.opname, - None) - if not consider_meth: - raise Exception,"unknown op: %r" % op - - # let's be careful about avoiding propagated SomeImpossibleValues - # to enter an op; the latter can result in violations of the - # more general results invariant: e.g. if SomeImpossibleValue enters is_ - # is_(SomeImpossibleValue, None) -> SomeBool - # is_(SomeInstance(not None), None) -> SomeBool(const=False) ... - # boom -- in the assert of setbinding() - for arg in argcells: - if isinstance(arg, annmodel.SomeImpossibleValue): - raise BlockedInference(self, op, opindex) - try: - resultcell = consider_meth(*argcells) - except Exception, e: - graph = self.bookkeeper.position_key[0] - e.args = e.args + ( - ErrorWrapper(gather_error(self, graph, block, opindex)),) - raise - if resultcell is None: - resultcell = self.noreturnvalue(op) - elif resultcell == annmodel.s_ImpossibleValue: - raise BlockedInference(self, op, opindex) # the operation cannot succeed - assert isinstance(resultcell, annmodel.SomeObject) - assert isinstance(op.result, Variable) - self.setbinding(op.result, resultcell) # bind resultcell to op.result - - def noreturnvalue(self, op): - return annmodel.s_ImpossibleValue # no return value (hook method) - - # XXX "contains" clash with SomeObject method - def consider_op_contains(self, seq, elem): - self.bookkeeper.count("contains", seq) - return seq.op_contains(elem) - - def consider_op_newtuple(self, *args): - return annmodel.SomeTuple(items = args) - - def consider_op_newlist(self, *args): - return self.bookkeeper.newlist(*args) - - def consider_op_newdict(self): - return self.bookkeeper.newdict() - - - def _registeroperations(cls, unary_ops, binary_ops): - # All unary operations - d = {} - for opname in unary_ops: - fnname = 'consider_op_' + opname - exec py.code.Source(""" -def consider_op_%s(self, arg, *args): - return arg.%s(*args) -""" % (opname, opname)).compile() in globals(), d - setattr(cls, fnname, d[fnname]) - # All binary operations - for opname in binary_ops: - fnname = 'consider_op_' + opname - exec py.code.Source(""" -def consider_op_%s(self, arg1, arg2, *args): - return pair(arg1,arg2).%s(*args) -""" % (opname, opname)).compile() in globals(), d - setattr(cls, fnname, d[fnname]) - _registeroperations = classmethod(_registeroperations) - -# register simple operations handling -RPythonAnnotator._registeroperations(unaryop.UNARY_OPERATIONS, binaryop.BINARY_OPERATIONS) - - -class BlockedInference(Exception): - """This exception signals the type inference engine that the situation - is currently blocked, and that it should try to progress elsewhere.""" - - def __init__(self, annotator, op, opindex): - self.annotator = annotator - try: - self.break_at = annotator.bookkeeper.position_key - except AttributeError: - self.break_at = None - self.op = op - self.opindex = opindex - - def __repr__(self): - if not self.break_at: - break_at = "?" - else: - break_at = self.annotator.whereami(self.break_at) - return "" %(break_at, self.op) - - __str__ = __repr__ diff --git a/pypy/annotation/binaryop.py b/pypy/annotation/binaryop.py deleted file mode 100644 --- a/pypy/annotation/binaryop.py +++ /dev/null @@ -1,1053 +0,0 @@ -""" -Binary operations between SomeValues. -""" - -import py -import operator -from pypy.tool.pairtype import pair, pairtype -from pypy.annotation.model import SomeObject, SomeInteger, SomeBool, s_Bool -from pypy.annotation.model import SomeString, SomeChar, SomeList, SomeDict -from pypy.annotation.model import SomeUnicodeCodePoint, SomeUnicodeString -from pypy.annotation.model import SomeTuple, SomeImpossibleValue, s_ImpossibleValue -from pypy.annotation.model import SomeInstance, SomeBuiltin, SomeIterator -from pypy.annotation.model import SomePBC, SomeFloat, s_None, SomeByteArray -from pypy.annotation.model import SomeExternalObject, SomeWeakRef -from pypy.annotation.model import SomeAddress, SomeTypedAddressAccess -from pypy.annotation.model import SomeSingleFloat, SomeLongFloat, SomeType -from pypy.annotation.model import unionof, UnionError, missing_operation -from pypy.annotation.model import TLS -from pypy.annotation.model import read_can_only_throw -from pypy.annotation.model import add_knowntypedata, merge_knowntypedata -from pypy.annotation.model import SomeGenericCallable -from pypy.annotation.bookkeeper import getbookkeeper -from pypy.objspace.flow.model import Variable, Constant -from pypy.rlib import rarithmetic -from pypy.tool.error import AnnotatorError - -# convenience only! -def immutablevalue(x): - return getbookkeeper().immutablevalue(x) - -# XXX unify this with ObjSpace.MethodTable -BINARY_OPERATIONS = set(['add', 'sub', 'mul', 'div', 'mod', - 'truediv', 'floordiv', 'divmod', - 'and_', 'or_', 'xor', - 'lshift', 'rshift', - 'getitem', 'setitem', 'delitem', - 'getitem_idx', 'getitem_key', 'getitem_idx_key', - 'inplace_add', 'inplace_sub', 'inplace_mul', - 'inplace_truediv', 'inplace_floordiv', 'inplace_div', - 'inplace_mod', - 'inplace_lshift', 'inplace_rshift', - 'inplace_and', 'inplace_or', 'inplace_xor', - 'lt', 'le', 'eq', 'ne', 'gt', 'ge', 'is_', 'cmp', - 'coerce', - ] - +[opname+'_ovf' for opname in - """add sub mul floordiv div mod lshift - """.split() - ]) - -for opname in BINARY_OPERATIONS: - missing_operation(pairtype(SomeObject, SomeObject), opname) - -class __extend__(pairtype(SomeObject, SomeObject)): - - def union((obj1, obj2)): - raise UnionError(obj1, obj2) - - # inplace_xxx ---> xxx by default - def inplace_add((obj1, obj2)): return pair(obj1, obj2).add() - def inplace_sub((obj1, obj2)): return pair(obj1, obj2).sub() - def inplace_mul((obj1, obj2)): return pair(obj1, obj2).mul() - def inplace_truediv((obj1, obj2)): return pair(obj1, obj2).truediv() - def inplace_floordiv((obj1, obj2)): return pair(obj1, obj2).floordiv() - def inplace_div((obj1, obj2)): return pair(obj1, obj2).div() - def inplace_mod((obj1, obj2)): return pair(obj1, obj2).mod() - def inplace_lshift((obj1, obj2)): return pair(obj1, obj2).lshift() - def inplace_rshift((obj1, obj2)): return pair(obj1, obj2).rshift() - def inplace_and((obj1, obj2)): return pair(obj1, obj2).and_() - def inplace_or((obj1, obj2)): return pair(obj1, obj2).or_() - def inplace_xor((obj1, obj2)): return pair(obj1, obj2).xor() - - for name, func in locals().items(): - if name.startswith('inplace_'): - func.can_only_throw = [] - - inplace_div.can_only_throw = [ZeroDivisionError] - inplace_truediv.can_only_throw = [ZeroDivisionError] - inplace_floordiv.can_only_throw = [ZeroDivisionError] - inplace_mod.can_only_throw = [ZeroDivisionError] - - def lt((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const < obj2.const) - else: - getbookkeeper().count("non_int_comp", obj1, obj2) - return s_Bool - - def le((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const <= obj2.const) - else: - getbookkeeper().count("non_int_comp", obj1, obj2) - return s_Bool - - def eq((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const == obj2.const) - else: - getbookkeeper().count("non_int_eq", obj1, obj2) - return s_Bool - - def ne((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const != obj2.const) - else: - getbookkeeper().count("non_int_eq", obj1, obj2) - return s_Bool - - def gt((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const > obj2.const) - else: - getbookkeeper().count("non_int_comp", obj1, obj2) - return s_Bool - - def ge((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const >= obj2.const) - else: - getbookkeeper().count("non_int_comp", obj1, obj2) - return s_Bool - - def cmp((obj1, obj2)): - getbookkeeper().count("cmp", obj1, obj2) - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(cmp(obj1.const, obj2.const)) - else: - return SomeInteger() - - def is_((obj1, obj2)): - r = SomeBool() - if obj2.is_constant(): - if obj1.is_constant(): - r.const = obj1.const is obj2.const - if obj2.const is None and not obj1.can_be_none(): - r.const = False - elif obj1.is_constant(): - if obj1.const is None and not obj2.can_be_none(): - r.const = False - # XXX HACK HACK HACK - # XXX HACK HACK HACK - # XXX HACK HACK HACK - bk = getbookkeeper() - if bk is not None: # for testing - knowntypedata = {} - fn, block, i = bk.position_key - - annotator = bk.annotator - op = block.operations[i] - assert op.opname == "is_" - assert len(op.args) == 2 - - def bind(src_obj, tgt_obj, tgt_arg): - if hasattr(tgt_obj, 'is_type_of') and src_obj.is_constant(): - add_knowntypedata(knowntypedata, True, tgt_obj.is_type_of, - bk.valueoftype(src_obj.const)) - - assert annotator.binding(op.args[tgt_arg]) == tgt_obj - add_knowntypedata(knowntypedata, True, [op.args[tgt_arg]], src_obj) - - nonnone_obj = tgt_obj - if src_obj.is_constant() and src_obj.const is None and tgt_obj.can_be_none(): - nonnone_obj = tgt_obj.nonnoneify() - - add_knowntypedata(knowntypedata, False, [op.args[tgt_arg]], nonnone_obj) - - bind(obj2, obj1, 0) - bind(obj1, obj2, 1) - r.set_knowntypedata(knowntypedata) - - return r - - def divmod((obj1, obj2)): - getbookkeeper().count("divmod", obj1, obj2) - return SomeTuple([pair(obj1, obj2).div(), pair(obj1, obj2).mod()]) - - def coerce((obj1, obj2)): - getbookkeeper().count("coerce", obj1, obj2) - return pair(obj1, obj2).union() # reasonable enough - - # approximation of an annotation intersection, the result should be the annotation obj or - # the intersection of obj and improvement - def improve((obj, improvement)): - if not improvement.contains(obj) and obj.contains(improvement): - return improvement - else: - return obj - - # checked getitems - - def _getitem_can_only_throw(s_c1, s_o2): - impl = pair(s_c1, s_o2).getitem - return read_can_only_throw(impl, s_c1, s_o2) - - def getitem_idx_key((s_c1, s_o2)): - impl = pair(s_c1, s_o2).getitem - return impl() - getitem_idx_key.can_only_throw = _getitem_can_only_throw - - getitem_idx = getitem_idx_key - getitem_key = getitem_idx_key - - -class __extend__(pairtype(SomeType, SomeType)): - - def union((obj1, obj2)): - result = SomeType() - is_type_of1 = getattr(obj1, 'is_type_of', None) - is_type_of2 = getattr(obj2, 'is_type_of', None) - if obj1.is_immutable_constant() and obj2.is_immutable_constant() and obj1.const == obj2.const: - result.const = obj1.const - is_type_of = {} - if is_type_of1: - for v in is_type_of1: - is_type_of[v] = True - if is_type_of2: - for v in is_type_of2: - is_type_of[v] = True - if is_type_of: - result.is_type_of = is_type_of.keys() - else: - if is_type_of1 and is_type_of1 == is_type_of2: - result.is_type_of = is_type_of1 - return result - - -# cloning a function with identical code, for the can_only_throw attribute -def _clone(f, can_only_throw = None): - newfunc = type(f)(f.func_code, f.func_globals, f.func_name, - f.func_defaults, f.func_closure) - if can_only_throw is not None: - newfunc.can_only_throw = can_only_throw - return newfunc - -class __extend__(pairtype(SomeInteger, SomeInteger)): - # unsignedness is considered a rare and contagious disease - - def union((int1, int2)): - if int1.unsigned == int2.unsigned: - knowntype = rarithmetic.compute_restype(int1.knowntype, int2.knowntype) - else: - t1 = int1.knowntype - if t1 is bool: - t1 = int - t2 = int2.knowntype - if t2 is bool: - t2 = int - - if t2 is int: - if int2.nonneg == False: - raise UnionError, "Merging %s and a possibly negative int is not allowed" % t1 - knowntype = t1 - elif t1 is int: - if int1.nonneg == False: - raise UnionError, "Merging %s and a possibly negative int is not allowed" % t2 - knowntype = t2 - else: - raise UnionError, "Merging these types (%s, %s) is not supported" % (t1, t2) - return SomeInteger(nonneg=int1.nonneg and int2.nonneg, - knowntype=knowntype) - - or_ = xor = add = mul = _clone(union, []) - add_ovf = mul_ovf = _clone(union, [OverflowError]) - div = floordiv = mod = _clone(union, [ZeroDivisionError]) - div_ovf= floordiv_ovf = mod_ovf = _clone(union, [ZeroDivisionError, OverflowError]) - - def truediv((int1, int2)): - return SomeFloat() - truediv.can_only_throw = [ZeroDivisionError] - truediv_ovf = _clone(truediv, [ZeroDivisionError, OverflowError]) - - inplace_div = div - inplace_truediv = truediv - - def sub((int1, int2)): - knowntype = rarithmetic.compute_restype(int1.knowntype, int2.knowntype) - return SomeInteger(knowntype=knowntype) - sub.can_only_throw = [] - sub_ovf = _clone(sub, [OverflowError]) - - def and_((int1, int2)): - knowntype = rarithmetic.compute_restype(int1.knowntype, int2.knowntype) - return SomeInteger(nonneg=int1.nonneg or int2.nonneg, - knowntype=knowntype) - and_.can_only_throw = [] - - def lshift((int1, int2)): - if isinstance(int1, SomeBool): - return SomeInteger() - else: - return SomeInteger(knowntype=int1.knowntype) - lshift.can_only_throw = [] - lshift_ovf = _clone(lshift, [OverflowError]) - - def rshift((int1, int2)): - if isinstance(int1, SomeBool): - return SomeInteger(nonneg=True) - else: - return SomeInteger(nonneg=int1.nonneg, knowntype=int1.knowntype) - rshift.can_only_throw = [] - - def _compare_helper((int1, int2), opname, operation): - r = SomeBool() - if int1.is_immutable_constant() and int2.is_immutable_constant(): - r.const = operation(int1.const, int2.const) - # - # The rest of the code propagates nonneg information between - # the two arguments. - # - # Doing the right thing when int1 or int2 change from signed - # to unsigned (r_uint) is almost impossible. See test_intcmp_bug. - # Instead, we only deduce constrains on the operands in the - # case where they are both signed. In other words, if y is - # nonneg then "assert x>=y" will let the annotator know that - # x is nonneg too, but it will not work if y is unsigned. - # - if not (rarithmetic.signedtype(int1.knowntype) and - rarithmetic.signedtype(int2.knowntype)): - return r - knowntypedata = {} - # XXX HACK HACK HACK - fn, block, i = getbookkeeper().position_key - op = block.operations[i] - assert op.opname == opname - assert len(op.args) == 2 - def tointtype(int0): - if int0.knowntype is bool: - return int - return int0.knowntype - if int1.nonneg and isinstance(op.args[1], Variable): - case = opname in ('lt', 'le', 'eq') - - add_knowntypedata(knowntypedata, case, [op.args[1]], - SomeInteger(nonneg=True, knowntype=tointtype(int2))) - if int2.nonneg and isinstance(op.args[0], Variable): - case = opname in ('gt', 'ge', 'eq') - add_knowntypedata(knowntypedata, case, [op.args[0]], - SomeInteger(nonneg=True, knowntype=tointtype(int1))) - r.set_knowntypedata(knowntypedata) - # a special case for 'x < 0' or 'x >= 0', - # where 0 is a flow graph Constant - # (in this case we are sure that it cannot become a r_uint later) - if (isinstance(op.args[1], Constant) and - type(op.args[1].value) is int and # filter out Symbolics - op.args[1].value == 0): - if int1.nonneg: - if opname == 'lt': - r.const = False - if opname == 'ge': - r.const = True - return r - - def lt(intint): return intint._compare_helper('lt', operator.lt) - def le(intint): return intint._compare_helper('le', operator.le) - def eq(intint): return intint._compare_helper('eq', operator.eq) - def ne(intint): return intint._compare_helper('ne', operator.ne) - def gt(intint): return intint._compare_helper('gt', operator.gt) - def ge(intint): return intint._compare_helper('ge', operator.ge) - - -class __extend__(pairtype(SomeBool, SomeBool)): - - def union((boo1, boo2)): - s = SomeBool() - if getattr(boo1, 'const', -1) == getattr(boo2, 'const', -2): - s.const = boo1.const - if hasattr(boo1, 'knowntypedata') and \ - hasattr(boo2, 'knowntypedata'): - ktd = merge_knowntypedata(boo1.knowntypedata, boo2.knowntypedata) - s.set_knowntypedata(ktd) - return s - - def and_((boo1, boo2)): - s = SomeBool() - if boo1.is_constant(): - if not boo1.const: - s.const = False - else: - return boo2 - if boo2.is_constant(): - if not boo2.const: - s.const = False - return s - - def or_((boo1, boo2)): - s = SomeBool() - if boo1.is_constant(): - if boo1.const: - s.const = True - else: - return boo2 - if boo2.is_constant(): - if boo2.const: - s.const = True - return s - - def xor((boo1, boo2)): - s = SomeBool() - if boo1.is_constant() and boo2.is_constant(): - s.const = boo1.const ^ boo2.const - return s - -class __extend__(pairtype(SomeString, SomeString)): - - def union((str1, str2)): - can_be_None = str1.can_be_None or str2.can_be_None - no_nul = str1.no_nul and str2.no_nul - return SomeString(can_be_None=can_be_None, no_nul=no_nul) - - def add((str1, str2)): - # propagate const-ness to help getattr(obj, 'prefix' + const_name) - result = SomeString(no_nul=str1.no_nul and str2.no_nul) - if str1.is_immutable_constant() and str2.is_immutable_constant(): - result.const = str1.const + str2.const - return result - -class __extend__(pairtype(SomeByteArray, SomeByteArray)): - def union((b1, b2)): - can_be_None = b1.can_be_None or b2.can_be_None - return SomeByteArray(can_be_None=can_be_None) - - def add((b1, b2)): - result = SomeByteArray() - if b1.is_immutable_constant() and b2.is_immutable_constant(): - result.const = b1.const + b2.const - return result - -class __extend__(pairtype(SomeByteArray, SomeInteger)): - def getitem((s_b, s_i)): - return SomeInteger() - - def setitem((s_b, s_i), s_i2): - assert isinstance(s_i2, SomeInteger) - -class __extend__(pairtype(SomeString, SomeByteArray), - pairtype(SomeByteArray, SomeString), - pairtype(SomeChar, SomeByteArray), - pairtype(SomeByteArray, SomeChar)): - def add((b1, b2)): - result = SomeByteArray() - if b1.is_immutable_constant() and b2.is_immutable_constant(): - result.const = b1.const + b2.const - return result - -class __extend__(pairtype(SomeChar, SomeChar)): - - def union((chr1, chr2)): - no_nul = chr1.no_nul and chr2.no_nul - return SomeChar(no_nul=no_nul) - - -class __extend__(pairtype(SomeChar, SomeUnicodeCodePoint), - pairtype(SomeUnicodeCodePoint, SomeChar)): - def union((uchr1, uchr2)): - return SomeUnicodeCodePoint() - -class __extend__(pairtype(SomeUnicodeCodePoint, SomeUnicodeCodePoint)): - def union((uchr1, uchr2)): - return SomeUnicodeCodePoint() - - def add((chr1, chr2)): - return SomeUnicodeString() - -class __extend__(pairtype(SomeString, SomeUnicodeString), - pairtype(SomeUnicodeString, SomeString)): - def mod((str, unistring)): - raise NotImplementedError( - "string formatting mixing strings and unicode not supported") - - -class __extend__(pairtype(SomeString, SomeTuple), - pairtype(SomeUnicodeString, SomeTuple)): - def mod((s_string, s_tuple)): - is_string = isinstance(s_string, SomeString) - is_unicode = isinstance(s_string, SomeUnicodeString) - assert is_string or is_unicode - for s_item in s_tuple.items: - if (is_unicode and isinstance(s_item, (SomeChar, SomeString)) or - is_string and isinstance(s_item, (SomeUnicodeCodePoint, - SomeUnicodeString))): - raise NotImplementedError( - "string formatting mixing strings and unicode not supported") - getbookkeeper().count('strformat', s_string, s_tuple) - no_nul = s_string.no_nul - for s_item in s_tuple.items: - if isinstance(s_item, SomeFloat): - pass # or s_item is a subclass, like SomeInteger - elif (isinstance(s_item, SomeString) or - isinstance(s_item, SomeUnicodeString)) and s_item.no_nul: - pass - else: - no_nul = False - break - return s_string.__class__(no_nul=no_nul) - - -class __extend__(pairtype(SomeString, SomeObject), - pairtype(SomeUnicodeString, SomeObject)): - - def mod((s_string, args)): - getbookkeeper().count('strformat', s_string, args) - return s_string.__class__() - -class __extend__(pairtype(SomeFloat, SomeFloat)): - - def union((flt1, flt2)): - return SomeFloat() - - add = sub = mul = union - - def div((flt1, flt2)): - return SomeFloat() - div.can_only_throw = [] - truediv = div - - # repeat these in order to copy the 'can_only_throw' attribute - inplace_div = div - inplace_truediv = truediv - - -class __extend__(pairtype(SomeSingleFloat, SomeSingleFloat)): - - def union((flt1, flt2)): - return SomeSingleFloat() - - -class __extend__(pairtype(SomeLongFloat, SomeLongFloat)): - - def union((flt1, flt2)): - return SomeLongFloat() - - -class __extend__(pairtype(SomeList, SomeList)): - - def union((lst1, lst2)): - return SomeList(lst1.listdef.union(lst2.listdef)) - - def add((lst1, lst2)): - return lst1.listdef.offspring(lst2.listdef) - - def eq((lst1, lst2)): - lst1.listdef.agree(lst2.listdef) - return s_Bool - ne = eq - - -class __extend__(pairtype(SomeList, SomeObject)): - - def inplace_add((lst1, obj2)): - lst1.method_extend(obj2) - return lst1 - inplace_add.can_only_throw = [] - - def inplace_mul((lst1, obj2)): - lst1.listdef.resize() - return lst1 - inplace_mul.can_only_throw = [] - -class __extend__(pairtype(SomeTuple, SomeTuple)): - - def union((tup1, tup2)): - if len(tup1.items) != len(tup2.items): - raise UnionError("cannot take the union of a tuple of length %d " - "and a tuple of length %d" % (len(tup1.items), - len(tup2.items))) - else: - unions = [unionof(x,y) for x,y in zip(tup1.items, tup2.items)] - return SomeTuple(items = unions) - - def add((tup1, tup2)): - return SomeTuple(items = tup1.items + tup2.items) - - def eq(tup1tup2): - tup1tup2.union() - return s_Bool - ne = eq - - def lt((tup1, tup2)): - raise Exception("unsupported: (...) < (...)") - def le((tup1, tup2)): - raise Exception("unsupported: (...) <= (...)") - def gt((tup1, tup2)): - raise Exception("unsupported: (...) > (...)") - def ge((tup1, tup2)): - raise Exception("unsupported: (...) >= (...)") - - -class __extend__(pairtype(SomeDict, SomeDict)): - - def union((dic1, dic2)): - return SomeDict(dic1.dictdef.union(dic2.dictdef)) - - -class __extend__(pairtype(SomeDict, SomeObject)): - - def _can_only_throw(dic1, *ignore): - if dic1.dictdef.dictkey.custom_eq_hash: - return None - return [KeyError] - - def getitem((dic1, obj2)): - getbookkeeper().count("dict_getitem", dic1) - dic1.dictdef.generalize_key(obj2) - return dic1.dictdef.read_value() - getitem.can_only_throw = _can_only_throw - - def setitem((dic1, obj2), s_value): - getbookkeeper().count("dict_setitem", dic1) - dic1.dictdef.generalize_key(obj2) - dic1.dictdef.generalize_value(s_value) - setitem.can_only_throw = _can_only_throw - - def delitem((dic1, obj2)): - getbookkeeper().count("dict_delitem", dic1) - dic1.dictdef.generalize_key(obj2) - delitem.can_only_throw = _can_only_throw - - -class __extend__(pairtype(SomeTuple, SomeInteger)): - - def getitem((tup1, int2)): - if int2.is_immutable_constant(): - try: - return tup1.items[int2.const] - except IndexError: - return s_ImpossibleValue - else: - getbookkeeper().count("tuple_random_getitem", tup1) - return unionof(*tup1.items) - getitem.can_only_throw = [IndexError] - - -class __extend__(pairtype(SomeList, SomeInteger)): - - def mul((lst1, int2)): From noreply at buildbot.pypy.org Sun Jan 20 14:53:16 2013 From: noreply at buildbot.pypy.org (amauryfa) Date: Sun, 20 Jan 2013 14:53:16 +0100 (CET) Subject: [pypy-commit] pypy py3k: hg merge default again Message-ID: <20130120135316.2E12A1C0237@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r60234:5bbf664124e9 Date: 2013-01-20 14:52 +0100 http://bitbucket.org/pypy/pypy/changeset/5bbf664124e9/ Log: hg merge default again diff --git a/goal/multibuild.py b/pypy/goal/multibuild.py rename from goal/multibuild.py rename to pypy/goal/multibuild.py diff --git a/goal/targetnumpystandalone.py b/pypy/goal/targetnumpystandalone.py rename from goal/targetnumpystandalone.py rename to pypy/goal/targetnumpystandalone.py diff --git a/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py rename from goal/targetpypystandalone.py rename to pypy/goal/targetpypystandalone.py diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py --- a/pypy/interpreter/gateway.py +++ b/pypy/interpreter/gateway.py @@ -15,7 +15,7 @@ from pypy.interpreter.eval import Code from pypy.interpreter.argument import Arguments -from rpython.rtyper.signature import Signature +from pypy.interpreter.signature import Signature from pypy.interpreter.baseobjspace import (W_Root, ObjSpace, Wrappable, SpaceCache, DescrMismatch) from pypy.interpreter.error import OperationError diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -7,7 +7,7 @@ import dis, imp, struct, types, new, sys from pypy.interpreter import eval -from rpython.rtyper.signature import Signature +from pypy.interpreter.signature import Signature from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import unwrap_spec from pypy.interpreter.astcompiler.consts import ( diff --git a/rpython/rtyper/signature.py b/pypy/interpreter/signature.py rename from rpython/rtyper/signature.py rename to pypy/interpreter/signature.py diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py --- a/pypy/interpreter/test/test_argument.py +++ b/pypy/interpreter/test/test_argument.py @@ -2,7 +2,7 @@ import py from pypy.interpreter.argument import (Arguments, ArgErr, ArgErrUnknownKwds, ArgErrMultipleValues, ArgErrCount) -from rpython.rtyper.signature import Signature +from pypy.interpreter.signature import Signature from pypy.interpreter.error import OperationError diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py --- a/pypy/interpreter/test/test_gateway.py +++ b/pypy/interpreter/test/test_gateway.py @@ -3,7 +3,7 @@ from pypy.interpreter import gateway, argument from pypy.interpreter.gateway import ObjSpace, W_Root, WrappedDefault -from rpython.rtyper.signature import Signature +from pypy.interpreter.signature import Signature import py import sys diff --git a/pypy/module/oracle/interp_pool.py b/pypy/module/oracle/interp_pool.py --- a/pypy/module/oracle/interp_pool.py +++ b/pypy/module/oracle/interp_pool.py @@ -4,8 +4,8 @@ from pypy.interpreter.typedef import TypeDef, GetSetProperty from pypy.interpreter.typedef import interp_attrproperty, interp_attrproperty_w from pypy.interpreter.error import OperationError +from pypy.interpreter.signature import Signature from rpython.rtyper.lltypesystem import rffi, lltype -from rpython.rtyper.signature import Signature from pypy.module.oracle import roci, config from pypy.module.oracle import interp_error, interp_environ diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py --- a/pypy/objspace/std/dictmultiobject.py +++ b/pypy/objspace/std/dictmultiobject.py @@ -3,14 +3,12 @@ from pypy.objspace.std.register_all import register_all from pypy.objspace.std.settype import set_typedef as settypedef from pypy.objspace.std.frozensettype import frozenset_typedef as frozensettypedef -from pypy.interpreter import gateway from pypy.interpreter.error import OperationError, operationerrfmt +from pypy.interpreter.signature import Signature -from rpython.rlib.objectmodel import r_dict, we_are_translated, specialize,\ - newlist_hint +from rpython.rlib.objectmodel import r_dict, specialize, newlist_hint from rpython.rlib.debug import mark_dict_non_null from rpython.tool.sourcetools import func_with_new_name -from rpython.rtyper.signature import Signature from rpython.rlib import rerased from rpython.rlib import jit diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py --- a/pypy/objspace/std/listobject.py +++ b/pypy/objspace/std/listobject.py @@ -8,11 +8,11 @@ from pypy.objspace.std.sliceobject import W_SliceObject, normalize_simple_slice from pypy.objspace.std import slicetype from pypy.interpreter import gateway, baseobjspace +from pypy.interpreter.signature import Signature from rpython.rlib.objectmodel import (instantiate, newlist_hint, specialize, resizelist_hint) from rpython.rlib.listsort import make_timsort_class from rpython.rlib import rerased, jit, debug -from rpython.rtyper.signature import Signature from rpython.tool.sourcetools import func_with_new_name UNROLL_CUTOFF = 5 diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py --- a/pypy/objspace/std/setobject.py +++ b/pypy/objspace/std/setobject.py @@ -1,20 +1,20 @@ from pypy.objspace.std.model import registerimplementation, W_Object from pypy.objspace.std.register_all import register_all -from rpython.rlib.objectmodel import r_dict -from rpython.rlib.rarithmetic import intmask, r_uint from pypy.interpreter.error import OperationError from pypy.interpreter import gateway from pypy.objspace.std.settype import set_typedef as settypedef from pypy.objspace.std.frozensettype import frozenset_typedef as frozensettypedef -from rpython.rlib import rerased -from rpython.rlib.objectmodel import instantiate -from rpython.rtyper.signature import Signature +from pypy.interpreter.signature import Signature from pypy.interpreter.generator import GeneratorIterator from pypy.objspace.std.listobject import W_ListObject from pypy.objspace.std.intobject import W_IntObject from pypy.objspace.std.stringobject import W_StringObject from pypy.objspace.std.unicodeobject import W_UnicodeObject +from rpython.rlib.objectmodel import r_dict +from rpython.rlib.rarithmetic import intmask, r_uint +from rpython.rlib import rerased + class W_BaseSetObject(W_Object): typedef = None diff --git a/pypy/bin/rpython b/rpython/bin/rpython rename from pypy/bin/rpython rename to rpython/bin/rpython diff --git a/rpython/rtyper/normalizecalls.py b/rpython/rtyper/normalizecalls.py --- a/rpython/rtyper/normalizecalls.py +++ b/rpython/rtyper/normalizecalls.py @@ -1,5 +1,5 @@ from rpython.annotator import model as annmodel, description -from rpython.rtyper.signature import Signature +from rpython.flowspace.argument import Signature from rpython.flowspace.model import (Variable, Constant, Block, Link, checkgraph, FunctionGraph, SpaceOperation) from rpython.rlib.objectmodel import ComputedIntSymbolic From noreply at buildbot.pypy.org Sun Jan 20 15:13:44 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 20 Jan 2013 15:13:44 +0100 (CET) Subject: [pypy-commit] pypy default: shuffle stuff around Message-ID: <20130120141344.3F3381C00F9@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r60235:8b3fd7dd838c Date: 2013-01-20 16:13 +0200 http://bitbucket.org/pypy/pypy/changeset/8b3fd7dd838c/ Log: shuffle stuff around diff --git a/pypy/module/unicodedata/interp_ucd.py b/pypy/module/unicodedata/interp_ucd.py --- a/pypy/module/unicodedata/interp_ucd.py +++ b/pypy/module/unicodedata/interp_ucd.py @@ -9,6 +9,7 @@ from rpython.rlib.objectmodel import we_are_translated from rpython.rlib.runicode import MAXUNICODE from rpython.rlib.unicodedata import unicodedb_5_2_0, unicodedb_3_2_0 +from rpython.rlib.unicodedata.ucd import code_to_unichr, ORD import sys @@ -30,25 +31,6 @@ # The functions below are subtly different from the ones in runicode.py. # When PyPy implements Python 3 they should be merged. -def UNICHR(c): - if c <= sys.maxunicode and c <= MAXUNICODE: - return unichr(c) - else: - c -= 0x10000 - return (unichr(0xD800 + (c >> 10)) + - unichr(0xDC00 + (c & 0x03FF))) - -def ORD(u): - assert isinstance(u, unicode) - if len(u) == 1: - return ord(u[0]) - elif len(u) == 2: - ch1 = ord(u[0]) - ch2 = ord(u[1]) - if 0xD800 <= ch1 <= 0xDBFF and 0xDC00 <= ch2 <= 0xDFFF: - return (((ch1 - 0xD800) << 10) | (ch2 - 0xDC00)) + 0x10000 - raise ValueError - if MAXUNICODE > 0xFFFF: # Target is wide build def unichr_to_code_w(space, w_unichr): @@ -69,12 +51,6 @@ 'need a single Unicode character as parameter')) return space.int_w(space.ord(w_unichr)) - def code_to_unichr(code): - if not we_are_translated() and sys.maxunicode == 0xFFFF: - # Host CPython is narrow build, generate surrogates - return UNICHR(code) - else: - return unichr(code) else: # Target is narrow build def unichr_to_code_w(space, w_unichr): @@ -97,10 +73,6 @@ raise OperationError(space.w_TypeError, space.wrap( 'need a single Unicode character as parameter')) - def code_to_unichr(code): - # generate surrogates for large codes - return UNICHR(code) - class UCD(Wrappable): def __init__(self, unicodedb): diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py --- a/rpython/rlib/runicode.py +++ b/rpython/rlib/runicode.py @@ -1,49 +1,9 @@ import sys -from rpython.rlib.bitmanipulation import splitter -from rpython.rtyper.lltypesystem import lltype, rffi -from rpython.rlib.objectmodel import we_are_translated, specialize, enforceargs +from rpython.rlib.objectmodel import specialize, enforceargs from rpython.rlib.rstring import StringBuilder, UnicodeBuilder from rpython.rlib.rarithmetic import r_uint, intmask from rpython.rlib.unicodedata import unicodedb - -if rffi.sizeof(lltype.UniChar) == 4: - MAXUNICODE = 0x10ffff -else: - MAXUNICODE = 0xffff -BYTEORDER = sys.byteorder - -if MAXUNICODE > sys.maxunicode: - # A version of unichr which allows codes outside the BMP - # even on narrow unicode builds. - # It will be used when interpreting code on top of a UCS2 CPython, - # when sizeof(wchar_t) == 4. - # Note that Python3 uses a similar implementation. - def UNICHR(c): - assert not we_are_translated() - if c <= sys.maxunicode or c > MAXUNICODE: - return unichr(c) - else: - c -= 0x10000 - return (unichr(0xD800 + (c >> 10)) + - unichr(0xDC00 + (c & 0x03FF))) - UNICHR._flowspace_rewrite_directly_as_ = unichr - # ^^^ NB.: for translation, it's essential to use this hack instead - # of calling unichr() from UNICHR(), because unichr() detects if there - # is a "try:except ValueError" immediately around it. - - def ORD(u): - assert not we_are_translated() - if isinstance(u, unicode) and len(u) == 2: - ch1 = ord(u[0]) - ch2 = ord(u[1]) - if 0xD800 <= ch1 <= 0xDBFF and 0xDC00 <= ch2 <= 0xDFFF: - return (((ch1 - 0xD800) << 10) | (ch2 - 0xDC00)) + 0x10000 - return ord(u) - ORD._flowspace_rewrite_directly_as_ = ord - -else: - UNICHR = unichr - ORD = ord +from rpython.rlib.unicodedata.ucd import MAXUNICODE, UNICHR, BYTEORDER def default_unicode_error_decode(errors, encoding, msg, s, @@ -446,16 +406,6 @@ result.append(r) return result.build(), pos, bo -def _STORECHAR(result, CH, byteorder): - hi = chr(((CH) >> 8) & 0xff) - lo = chr((CH) & 0xff) - if byteorder == 'little': - result.append(lo) - result.append(hi) - else: - result.append(hi) - result.append(lo) - def unicode_encode_utf_16_helper(s, size, errors, errorhandler=None, byteorder='little'): diff --git a/pypy/module/unicodedata/test_interp_ucd.py b/rpython/rlib/unicodedata/test/test_ucd.py rename from pypy/module/unicodedata/test_interp_ucd.py rename to rpython/rlib/unicodedata/test/test_ucd.py --- a/pypy/module/unicodedata/test_interp_ucd.py +++ b/rpython/rlib/unicodedata/test/test_ucd.py @@ -1,6 +1,6 @@ from rpython.rtyper.test.tool import BaseRtypingTest, LLRtypeMixin from rpython.rlib.unicodedata import unicodedb_5_2_0 -from pypy.module.unicodedata.interp_ucd import code_to_unichr +from rpython.rlib.unicodedata.ucd import code_to_unichr class TestTranslated(BaseRtypingTest, LLRtypeMixin): diff --git a/rpython/rlib/unicodedata/test/test_unicodedata.py b/rpython/rlib/unicodedata/test/test_unicodedata.py --- a/rpython/rlib/unicodedata/test/test_unicodedata.py +++ b/rpython/rlib/unicodedata/test/test_unicodedata.py @@ -1,109 +1,6 @@ import py from rpython.rlib.unicodedata import unicodedb_3_2_0, unicodedb_5_2_0 -class AppTestUnicodeData: - spaceconfig = dict(usemodules=('unicodedata',)) - - def test_hangul_syllables(self): - import unicodedata - # Test all leading, vowel and trailing jamo - # but not every combination of them. - for code, name in ((0xAC00, 'HANGUL SYLLABLE GA'), - (0xAE69, 'HANGUL SYLLABLE GGAEG'), - (0xB0D2, 'HANGUL SYLLABLE NYAGG'), - (0xB33B, 'HANGUL SYLLABLE DYAEGS'), - (0xB5A4, 'HANGUL SYLLABLE DDEON'), - (0xB80D, 'HANGUL SYLLABLE RENJ'), - (0xBA76, 'HANGUL SYLLABLE MYEONH'), - (0xBCDF, 'HANGUL SYLLABLE BYED'), - (0xBF48, 'HANGUL SYLLABLE BBOL'), - (0xC1B1, 'HANGUL SYLLABLE SWALG'), - (0xC41A, 'HANGUL SYLLABLE SSWAELM'), - (0xC683, 'HANGUL SYLLABLE OELB'), - (0xC8EC, 'HANGUL SYLLABLE JYOLS'), - (0xCB55, 'HANGUL SYLLABLE JJULT'), - (0xCDBE, 'HANGUL SYLLABLE CWEOLP'), - (0xD027, 'HANGUL SYLLABLE KWELH'), - (0xD290, 'HANGUL SYLLABLE TWIM'), - (0xD4F9, 'HANGUL SYLLABLE PYUB'), - (0xD762, 'HANGUL SYLLABLE HEUBS'), - (0xAE27, 'HANGUL SYLLABLE GYIS'), - (0xB090, 'HANGUL SYLLABLE GGISS'), - (0xB0AD, 'HANGUL SYLLABLE NANG'), - (0xB316, 'HANGUL SYLLABLE DAEJ'), - (0xB57F, 'HANGUL SYLLABLE DDYAC'), - (0xB7E8, 'HANGUL SYLLABLE RYAEK'), - (0xBA51, 'HANGUL SYLLABLE MEOT'), - (0xBCBA, 'HANGUL SYLLABLE BEP'), - (0xBF23, 'HANGUL SYLLABLE BBYEOH'), - (0xD7A3, 'HANGUL SYLLABLE HIH')): - assert unicodedata.name(unichr(code)) == name - assert unicodedata.lookup(name) == unichr(code) - # Test outside the range - py.test.raises(ValueError, unicodedata.name, unichr(0xAC00 - 1)) - py.test.raises(ValueError, unicodedata.name, unichr(0xD7A3 + 1)) - - def test_cjk(self): - import sys - import unicodedata - cases = ((0x3400, 0x4DB5), - (0x4E00, 0x9FA5)) - if unicodedata.unidata_version >= "5": # don't know the exact limit - cases = ((0x3400, 0x4DB5), - (0x4E00, 0x9FCB), - (0x20000, 0x2A6D6), - (0x2A700, 0x2B734)) - elif unicodedata.unidata_version >= "4.1": - cases = ((0x3400, 0x4DB5), - (0x4E00, 0x9FBB), - (0x20000, 0x2A6D6)) - for first, last in cases: - # Test at and inside the boundary - for i in (first, first + 1, last - 1, last): - charname = 'CJK UNIFIED IDEOGRAPH-%X'%i - char = ('\\U%08X' % i).decode('unicode-escape') - assert unicodedata.name(char) == charname - assert unicodedata.lookup(charname) == char - # Test outside the boundary - for i in first - 1, last + 1: - charname = 'CJK UNIFIED IDEOGRAPH-%X'%i - char = ('\\U%08X' % i).decode('unicode-escape') - try: - unicodedata.name(char) - except ValueError, e: - assert e.message == 'no such name' - py.test.raises(KeyError, unicodedata.lookup, charname) - - def test_bug_1704793(self): # from CPython - import unicodedata - assert unicodedata.lookup("GOTHIC LETTER FAIHU") == u'\U00010346' - - def test_normalize(self): - import unicodedata - py.test.raises(TypeError, unicodedata.normalize, 'x') - - def test_normalize_wide(self): - import sys, unicodedata - if sys.maxunicode < 0x10ffff: - skip("requires a 'wide' python build.") - assert unicodedata.normalize('NFC', u'\U000110a5\U000110ba') == u'\U000110ab' - - def test_linebreaks(self): - linebreaks = (0x0a, 0x0b, 0x0c, 0x0d, 0x85, - 0x1c, 0x1d, 0x1e, 0x2028, 0x2029) - for i in linebreaks: - for j in range(-2, 3): - lines = (unichr(i + j) + u'A').splitlines() - if i + j in linebreaks: - assert len(lines) == 2 - else: - assert len(lines) == 1 - - def test_mirrored(self): - import unicodedata - # For no reason, unicodedata.mirrored() returns an int, not a bool - assert repr(unicodedata.mirrored(u' ')) == '0' - class TestUnicodeData(object): def setup_class(cls): import random, unicodedata diff --git a/rpython/rlib/unicodedata/ucd.py b/rpython/rlib/unicodedata/ucd.py new file mode 100644 --- /dev/null +++ b/rpython/rlib/unicodedata/ucd.py @@ -0,0 +1,87 @@ + +import sys +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rlib.objectmodel import we_are_translated + + +if rffi.sizeof(lltype.UniChar) == 4: + MAXUNICODE = 0x10ffff +else: + MAXUNICODE = 0xffff + +BYTEORDER = sys.byteorder + +if MAXUNICODE > sys.maxunicode: + # A version of unichr which allows codes outside the BMP + # even on narrow unicode builds. + # It will be used when interpreting code on top of a UCS2 CPython, + # when sizeof(wchar_t) == 4. + # Note that Python3 uses a similar implementation. + def UNICHR(c): + assert not we_are_translated() + if c <= sys.maxunicode or c > MAXUNICODE: + return unichr(c) + else: + c -= 0x10000 + return (unichr(0xD800 + (c >> 10)) + + unichr(0xDC00 + (c & 0x03FF))) + UNICHR._flowspace_rewrite_directly_as_ = unichr + # ^^^ NB.: for translation, it's essential to use this hack instead + # of calling unichr() from UNICHR(), because unichr() detects if there + # is a "try:except ValueError" immediately around it. + + def ORD(u): + assert not we_are_translated() + if isinstance(u, unicode) and len(u) == 2: + ch1 = ord(u[0]) + ch2 = ord(u[1]) + if 0xD800 <= ch1 <= 0xDBFF and 0xDC00 <= ch2 <= 0xDFFF: + return (((ch1 - 0xD800) << 10) | (ch2 - 0xDC00)) + 0x10000 + return ord(u) + ORD._flowspace_rewrite_directly_as_ = ord + +else: + UNICHR = unichr + ORD = ord + +if MAXUNICODE > 0xFFFF: + def code_to_unichr(code): + if not we_are_translated() and sys.maxunicode == 0xFFFF: + # Host CPython is narrow build, generate surrogates + return UNICHR(code) + else: + return unichr(code) +else: + def code_to_unichr(code): + # generate surrogates for large codes + return UNICHR(code) + + +def UNICHR(c): + if c <= sys.maxunicode and c <= MAXUNICODE: + return unichr(c) + else: + c -= 0x10000 + return (unichr(0xD800 + (c >> 10)) + + unichr(0xDC00 + (c & 0x03FF))) + +def ORD(u): + assert isinstance(u, unicode) + if len(u) == 1: + return ord(u[0]) + elif len(u) == 2: + ch1 = ord(u[0]) + ch2 = ord(u[1]) + if 0xD800 <= ch1 <= 0xDBFF and 0xDC00 <= ch2 <= 0xDFFF: + return (((ch1 - 0xD800) << 10) | (ch2 - 0xDC00)) + 0x10000 + raise ValueError + +def _STORECHAR(result, CH, byteorder): + hi = chr(((CH) >> 8) & 0xff) + lo = chr((CH) & 0xff) + if byteorder == 'little': + result.append(lo) + result.append(hi) + else: + result.append(hi) + result.append(lo) From noreply at buildbot.pypy.org Sun Jan 20 15:16:44 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 20 Jan 2013 15:16:44 +0100 (CET) Subject: [pypy-commit] pypy default: make rpython insert stuff on path Message-ID: <20130120141644.D7F281C00F9@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r60236:f3801ca4d8b7 Date: 2013-01-20 16:16 +0200 http://bitbucket.org/pypy/pypy/changeset/f3801ca4d8b7/ Log: make rpython insert stuff on path diff --git a/rpython/bin/rpython b/rpython/bin/rpython --- a/rpython/bin/rpython +++ b/rpython/bin/rpython @@ -7,7 +7,8 @@ run with --help for more information """ -import sys +import sys, os +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) from rpython.translator.goal.translate import main # no implicit targets From noreply at buildbot.pypy.org Sun Jan 20 15:18:47 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 20 Jan 2013 15:18:47 +0100 (CET) Subject: [pypy-commit] buildbot default: use new translate paths Message-ID: <20130120141847.311831C00F9@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r734:7717f489459b Date: 2013-01-20 16:18 +0200 http://bitbucket.org/pypy/buildbot/changeset/7717f489459b/ Log: use new translate paths diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -128,12 +128,12 @@ description = ["translating"] descriptionDone = ["translation"] - command = ["translate.py", "--batch"] + command = ["../../rpython/bin/rpython", "--batch"] translationTarget = "targetpypystandalone" haltOnFailure = True def __init__(self, translationArgs, targetArgs, - workdir="build/pypy/translator/goal", + workdir="build/pypy/goal", interpreter='pypy', *a, **kw): add_args = {'translationArgs': translationArgs, From noreply at buildbot.pypy.org Sun Jan 20 15:18:48 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 20 Jan 2013 15:18:48 +0100 (CET) Subject: [pypy-commit] buildbot default: merge Message-ID: <20130120141848.561B81C00F9@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r735:5aea4cf91fcc Date: 2013-01-20 16:18 +0200 http://bitbucket.org/pypy/buildbot/changeset/5aea4cf91fcc/ Log: merge diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -367,7 +367,7 @@ timeout=kwargs.get('timeout', 4000) self.addStep(PytestCmd( - description="pytest", + description="pytest pypy", command=["python", "testrunner/runner.py", "--logfile=testrun.log", "--config=pypy/testrunner_cfg.py", @@ -379,6 +379,19 @@ env={"PYTHONPATH": ['.'], "PYPYCHERRYPICK": cherrypick})) + self.addStep(PytestCmd( + description="pytest rpython", + command=["python", "testrunner/runner.py", + "--logfile=testrun.log", + "--config=pypy/testrunner_cfg.py", + "--config=~/machine_cfg.py", + "--root=rpython", "--timeout=%s" % (timeout,) + ] + ["--config=%s" % cfg for cfg in extra_cfgs], + logfiles={'pytestLog': 'testrun.log'}, + timeout=timeout, + env={"PYTHONPATH": ['.'], + "PYPYCHERRYPICK": cherrypick})) + class Translated(factory.BuildFactory): From noreply at buildbot.pypy.org Sun Jan 20 15:19:45 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 20 Jan 2013 15:19:45 +0100 (CET) Subject: [pypy-commit] pypy default: add a missing file Message-ID: <20130120141945.7E9BD1C00F9@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r60237:320d0abbca64 Date: 2013-01-20 16:19 +0200 http://bitbucket.org/pypy/pypy/changeset/320d0abbca64/ Log: add a missing file diff --git a/pypy/module/unicodedata/test_unicodedata.py b/pypy/module/unicodedata/test_unicodedata.py new file mode 100644 --- /dev/null +++ b/pypy/module/unicodedata/test_unicodedata.py @@ -0,0 +1,103 @@ + +class AppTestUnicodeData: + spaceconfig = dict(usemodules=('unicodedata',)) + + def test_hangul_syllables(self): + import unicodedata + # Test all leading, vowel and trailing jamo + # but not every combination of them. + for code, name in ((0xAC00, 'HANGUL SYLLABLE GA'), + (0xAE69, 'HANGUL SYLLABLE GGAEG'), + (0xB0D2, 'HANGUL SYLLABLE NYAGG'), + (0xB33B, 'HANGUL SYLLABLE DYAEGS'), + (0xB5A4, 'HANGUL SYLLABLE DDEON'), + (0xB80D, 'HANGUL SYLLABLE RENJ'), + (0xBA76, 'HANGUL SYLLABLE MYEONH'), + (0xBCDF, 'HANGUL SYLLABLE BYED'), + (0xBF48, 'HANGUL SYLLABLE BBOL'), + (0xC1B1, 'HANGUL SYLLABLE SWALG'), + (0xC41A, 'HANGUL SYLLABLE SSWAELM'), + (0xC683, 'HANGUL SYLLABLE OELB'), + (0xC8EC, 'HANGUL SYLLABLE JYOLS'), + (0xCB55, 'HANGUL SYLLABLE JJULT'), + (0xCDBE, 'HANGUL SYLLABLE CWEOLP'), + (0xD027, 'HANGUL SYLLABLE KWELH'), + (0xD290, 'HANGUL SYLLABLE TWIM'), + (0xD4F9, 'HANGUL SYLLABLE PYUB'), + (0xD762, 'HANGUL SYLLABLE HEUBS'), + (0xAE27, 'HANGUL SYLLABLE GYIS'), + (0xB090, 'HANGUL SYLLABLE GGISS'), + (0xB0AD, 'HANGUL SYLLABLE NANG'), + (0xB316, 'HANGUL SYLLABLE DAEJ'), + (0xB57F, 'HANGUL SYLLABLE DDYAC'), + (0xB7E8, 'HANGUL SYLLABLE RYAEK'), + (0xBA51, 'HANGUL SYLLABLE MEOT'), + (0xBCBA, 'HANGUL SYLLABLE BEP'), + (0xBF23, 'HANGUL SYLLABLE BBYEOH'), + (0xD7A3, 'HANGUL SYLLABLE HIH')): + assert unicodedata.name(unichr(code)) == name + assert unicodedata.lookup(name) == unichr(code) + # Test outside the range + raises(ValueError, unicodedata.name, unichr(0xAC00 - 1)) + raises(ValueError, unicodedata.name, unichr(0xD7A3 + 1)) + + def test_cjk(self): + import sys + import unicodedata + cases = ((0x3400, 0x4DB5), + (0x4E00, 0x9FA5)) + if unicodedata.unidata_version >= "5": # don't know the exact limit + cases = ((0x3400, 0x4DB5), + (0x4E00, 0x9FCB), + (0x20000, 0x2A6D6), + (0x2A700, 0x2B734)) + elif unicodedata.unidata_version >= "4.1": + cases = ((0x3400, 0x4DB5), + (0x4E00, 0x9FBB), + (0x20000, 0x2A6D6)) + for first, last in cases: + # Test at and inside the boundary + for i in (first, first + 1, last - 1, last): + charname = 'CJK UNIFIED IDEOGRAPH-%X'%i + char = ('\\U%08X' % i).decode('unicode-escape') + assert unicodedata.name(char) == charname + assert unicodedata.lookup(charname) == char + # Test outside the boundary + for i in first - 1, last + 1: + charname = 'CJK UNIFIED IDEOGRAPH-%X'%i + char = ('\\U%08X' % i).decode('unicode-escape') + try: + unicodedata.name(char) + except ValueError, e: + assert e.message == 'no such name' + raises(KeyError, unicodedata.lookup, charname) + + def test_bug_1704793(self): # from CPython + import unicodedata + assert unicodedata.lookup("GOTHIC LETTER FAIHU") == u'\U00010346' + + def test_normalize(self): + import unicodedata + raises(TypeError, unicodedata.normalize, 'x') + + def test_normalize_wide(self): + import sys, unicodedata + if sys.maxunicode < 0x10ffff: + skip("requires a 'wide' python build.") + assert unicodedata.normalize('NFC', u'\U000110a5\U000110ba') == u'\U000110ab' + + def test_linebreaks(self): + linebreaks = (0x0a, 0x0b, 0x0c, 0x0d, 0x85, + 0x1c, 0x1d, 0x1e, 0x2028, 0x2029) + for i in linebreaks: + for j in range(-2, 3): + lines = (unichr(i + j) + u'A').splitlines() + if i + j in linebreaks: + assert len(lines) == 2 + else: + assert len(lines) == 1 + + def test_mirrored(self): + import unicodedata + # For no reason, unicodedata.mirrored() returns an int, not a bool + assert repr(unicodedata.mirrored(u' ')) == '0' From noreply at buildbot.pypy.org Sun Jan 20 15:28:27 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 20 Jan 2013 15:28:27 +0100 (CET) Subject: [pypy-commit] pypy default: oops fix Message-ID: <20130120142827.3655B1C00F9@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r60238:3706381fd356 Date: 2013-01-20 16:28 +0200 http://bitbucket.org/pypy/pypy/changeset/3706381fd356/ Log: oops fix diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py --- a/rpython/rlib/runicode.py +++ b/rpython/rlib/runicode.py @@ -1,5 +1,5 @@ import sys -from rpython.rlib.objectmodel import specialize, enforceargs +from rpython.rlib.objectmodel import specialize, we_are_translated from rpython.rlib.rstring import StringBuilder, UnicodeBuilder from rpython.rlib.rarithmetic import r_uint, intmask from rpython.rlib.unicodedata import unicodedb From noreply at buildbot.pypy.org Sun Jan 20 15:59:23 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 20 Jan 2013 15:59:23 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: painfully merge default Message-ID: <20130120145923.337421C0237@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60239:00b3d0dff2d6 Date: 2013-01-20 16:58 +0200 http://bitbucket.org/pypy/pypy/changeset/00b3d0dff2d6/ Log: painfully merge default diff too long, truncating to 2000 out of 702651 lines diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -85,3 +85,4 @@ ^compiled ^.git/ ^release/ +^rpython/_cache$ diff --git a/dotviewer/test/test_translator.py b/dotviewer/test/test_translator.py --- a/dotviewer/test/test_translator.py +++ b/dotviewer/test/test_translator.py @@ -21,7 +21,7 @@ def test_annotated(): - from pypy.translator.interactive import Translation + from rpython.translator.interactive import Translation t = Translation(is_prime) t.annotate([int]) t.viewcg() diff --git a/lib_pypy/ctypes_config_cache/autopath.py b/lib_pypy/ctypes_config_cache/autopath.py deleted file mode 100644 --- a/lib_pypy/ctypes_config_cache/autopath.py +++ /dev/null @@ -1,131 +0,0 @@ -""" -self cloning, automatic path configuration - -copy this into any subdirectory of pypy from which scripts need -to be run, typically all of the test subdirs. -The idea is that any such script simply issues - - import autopath - -and this will make sure that the parent directory containing "pypy" -is in sys.path. - -If you modify the master "autopath.py" version (in pypy/tool/autopath.py) -you can directly run it which will copy itself on all autopath.py files -it finds under the pypy root directory. - -This module always provides these attributes: - - pypydir pypy root directory path - this_dir directory where this autopath.py resides - -""" - -def __dirinfo(part): - """ return (partdir, this_dir) and insert parent of partdir - into sys.path. If the parent directories don't have the part - an EnvironmentError is raised.""" - - import sys, os - try: - head = this_dir = os.path.realpath(os.path.dirname(__file__)) - except NameError: - head = this_dir = os.path.realpath(os.path.dirname(sys.argv[0])) - - error = None - while head: - partdir = head - head, tail = os.path.split(head) - if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') - if not os.path.exists(checkfile): - error = "Cannot find %r" % (os.path.normpath(checkfile),) - break - else: - error = "Cannot find the parent directory %r of the path %r" % ( - partdir, this_dir) - if not error: - # check for bogus end-of-line style (e.g. files checked out on - # Windows and moved to Unix) - f = open(__file__.replace('.pyc', '.py'), 'r') - data = f.read() - f.close() - if data.endswith('\r\n') or data.endswith('\r'): - error = ("Bad end-of-line style in the .py files. Typically " - "caused by a zip file or a checkout done on Windows and " - "moved to Unix or vice-versa.") - if error: - raise EnvironmentError("Invalid source tree - bogus checkout! " + - error) - - pypy_root = os.path.join(head, '') - try: - sys.path.remove(head) - except ValueError: - pass - sys.path.insert(0, head) - - munged = {} - for name, mod in sys.modules.items(): - if '.' in name: - continue - fn = getattr(mod, '__file__', None) - if not isinstance(fn, str): - continue - newname = os.path.splitext(os.path.basename(fn))[0] - if not newname.startswith(part + '.'): - continue - path = os.path.join(os.path.dirname(os.path.realpath(fn)), '') - if path.startswith(pypy_root) and newname != part: - modpaths = os.path.normpath(path[len(pypy_root):]).split(os.sep) - if newname != '__init__': - modpaths.append(newname) - modpath = '.'.join(modpaths) - if modpath not in sys.modules: - munged[modpath] = mod - - for name, mod in munged.iteritems(): - if name not in sys.modules: - sys.modules[name] = mod - if '.' in name: - prename = name[:name.rfind('.')] - postname = name[len(prename)+1:] - if prename not in sys.modules: - __import__(prename) - if not hasattr(sys.modules[prename], postname): - setattr(sys.modules[prename], postname, mod) - - return partdir, this_dir - -def __clone(): - """ clone master version of autopath.py into all subdirs """ - from os.path import join, walk - if not this_dir.endswith(join('pypy','tool')): - raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) - - - def sync_walker(arg, dirname, fnames): - if _myname in fnames: - fn = join(dirname, _myname) - f = open(fn, 'rwb+') - try: - if f.read() == arg: - print "checkok", fn - else: - print "syncing", fn - f = open(fn, 'w') - f.write(arg) - finally: - f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) - -_myname = 'autopath.py' - -# set guaranteed attributes - -pypydir, this_dir = __dirinfo('pypy') - -if __name__ == '__main__': - __clone() diff --git a/lib_pypy/ctypes_config_cache/dumpcache.py b/lib_pypy/ctypes_config_cache/dumpcache.py --- a/lib_pypy/ctypes_config_cache/dumpcache.py +++ b/lib_pypy/ctypes_config_cache/dumpcache.py @@ -1,6 +1,6 @@ import os from ctypes_configure import dumpcache -from pypy.jit.backend import detect_cpu +from rpython.jit.backend import detect_cpu def dumpcache2(basename, config): model = detect_cpu.autodetect_main_model_and_size() @@ -14,7 +14,7 @@ try: from __pypy__ import cpumodel except ImportError: - from pypy.jit.backend import detect_cpu + from rpython.jit.backend import detect_cpu cpumodel = detect_cpu.autodetect_main_model_and_size() # XXX relative import, should be removed together with # XXX the relative imports done e.g. by lib_pypy/pypy_test/test_hashlib diff --git a/lib_pypy/ctypes_config_cache/rebuild.py b/lib_pypy/ctypes_config_cache/rebuild.py --- a/lib_pypy/ctypes_config_cache/rebuild.py +++ b/lib_pypy/ctypes_config_cache/rebuild.py @@ -1,21 +1,15 @@ #! /usr/bin/env python # Run this script to rebuild all caches from the *.ctc.py files. -# hack: we cannot directly import autopath, as we are outside the pypy -# package. However, we pretend to be inside pypy/tool and manually run it, to -# get the correct path -import os.path -this_dir = os.path.dirname(__file__) -autopath_py = os.path.join(this_dir, '../../pypy/tool/autopath.py') -autopath_py = os.path.abspath(autopath_py) -execfile(autopath_py, dict(__name__='autopath', __file__=autopath_py)) +import os, sys -import os, sys +sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..'))) + import py _dirpath = os.path.dirname(__file__) or os.curdir -from pypy.tool.ansi_print import ansi_log +from rpython.tool.ansi_print import ansi_log log = py.log.Producer("ctypes_config_cache") py.log.setconsumer("ctypes_config_cache", ansi_log) @@ -31,7 +25,7 @@ sys.path[:] = path def try_rebuild(): - from pypy.jit.backend import detect_cpu + from rpython.jit.backend import detect_cpu model = detect_cpu.autodetect_main_model_and_size() # remove the files '_*_model_.py' left = {} diff --git a/lib_pypy/ctypes_config_cache/test/test_cache.py b/lib_pypy/ctypes_config_cache/test/test_cache.py --- a/lib_pypy/ctypes_config_cache/test/test_cache.py +++ b/lib_pypy/ctypes_config_cache/test/test_cache.py @@ -1,6 +1,6 @@ import py import sys, os -from pypy.tool.udir import udir +from rpython.tool.udir import udir dirpath = py.path.local(__file__).dirpath().dirpath() diff --git a/lib_pypy/pypy_test/test_dbm_extra.py b/lib_pypy/pypy_test/test_dbm_extra.py --- a/lib_pypy/pypy_test/test_dbm_extra.py +++ b/lib_pypy/pypy_test/test_dbm_extra.py @@ -1,6 +1,6 @@ from __future__ import absolute_import import py -from pypy.tool.udir import udir +from rpython.tool.udir import udir try: from lib_pypy import dbm except ImportError, e: diff --git a/lib_pypy/pypy_test/test_marshal_extra.py b/lib_pypy/pypy_test/test_marshal_extra.py --- a/lib_pypy/pypy_test/test_marshal_extra.py +++ b/lib_pypy/pypy_test/test_marshal_extra.py @@ -4,7 +4,7 @@ import marshal as cpy_marshal from lib_pypy import _marshal as marshal -from pypy.tool.udir import udir +from rpython.tool.udir import udir hello = "he" hello += "llo" diff --git a/pypy/annotation/annrpython.py b/pypy/annotation/annrpython.py deleted file mode 100644 --- a/pypy/annotation/annrpython.py +++ /dev/null @@ -1,676 +0,0 @@ -from __future__ import absolute_import - -import types -from pypy.tool.ansi_print import ansi_log -from pypy.tool.pairtype import pair -from pypy.tool.error import (format_blocked_annotation_error, - AnnotatorError, gather_error, ErrorWrapper) -from pypy.objspace.flow.model import (Variable, Constant, FunctionGraph, - c_last_exception, checkgraph) -from pypy.translator import simplify, transform -from pypy.annotation import model as annmodel, signature, unaryop, binaryop -from pypy.annotation.bookkeeper import Bookkeeper -import py -log = py.log.Producer("annrpython") -py.log.setconsumer("annrpython", ansi_log) - - -FAIL = object() - -class RPythonAnnotator(object): - """Block annotator for RPython. - See description in doc/translation.txt.""" - - def __init__(self, translator=None, policy=None, bookkeeper=None): - import pypy.rpython.ootypesystem.ooregistry # has side effects - import pypy.rpython.extfuncregistry # has side effects - import pypy.rlib.nonconst # has side effects - - if translator is None: - # interface for tests - from pypy.translator.translator import TranslationContext - translator = TranslationContext() - translator.annotator = self - self.translator = translator - self.pendingblocks = {} # map {block: graph-containing-it} - self.bindings = {} # map Variables to SomeValues - self.annotated = {} # set of blocks already seen - self.added_blocks = None # see processblock() below - self.links_followed = {} # set of links that have ever been followed - self.notify = {} # {block: {positions-to-reflow-from-when-done}} - self.fixed_graphs = {} # set of graphs not to annotate again - self.blocked_blocks = {} # set of {blocked_block: (graph, index)} - # --- the following information is recorded for debugging --- - self.blocked_graphs = {} # set of graphs that have blocked blocks - # --- end of debugging information --- - self.frozen = False - if policy is None: - from pypy.annotation.policy import AnnotatorPolicy - self.policy = AnnotatorPolicy() - else: - self.policy = policy - if bookkeeper is None: - bookkeeper = Bookkeeper(self) - self.bookkeeper = bookkeeper - - def __getstate__(self): - attrs = """translator pendingblocks bindings annotated links_followed - notify bookkeeper frozen policy added_blocks""".split() - ret = self.__dict__.copy() - for key, value in ret.items(): - if key not in attrs: - assert type(value) is dict, ( - "%r is not dict. please update %s.__getstate__" % - (key, self.__class__.__name__)) - ret[key] = {} - return ret - - #___ convenience high-level interface __________________ - - def build_types(self, function, input_arg_types, complete_now=True, - main_entry_point=False): - """Recursively build annotations about the specific entry point.""" - assert isinstance(function, types.FunctionType), "fix that!" - - from pypy.annotation.policy import AnnotatorPolicy - policy = AnnotatorPolicy() - # make input arguments and set their type - args_s = [self.typeannotation(t) for t in input_arg_types] - - # XXX hack - annmodel.TLS.check_str_without_nul = ( - self.translator.config.translation.check_str_without_nul) - - flowgraph, inputcells = self.get_call_parameters(function, args_s, policy) - if not isinstance(flowgraph, FunctionGraph): - assert isinstance(flowgraph, annmodel.SomeObject) - return flowgraph - - if main_entry_point: - self.translator.entry_point_graph = flowgraph - return self.build_graph_types(flowgraph, inputcells, complete_now=complete_now) - - def get_call_parameters(self, function, args_s, policy): - desc = self.bookkeeper.getdesc(function) - args = self.bookkeeper.build_args("simple_call", args_s[:]) - result = [] - def schedule(graph, inputcells): - result.append((graph, inputcells)) - return annmodel.s_ImpossibleValue - - prevpolicy = self.policy - self.policy = policy - self.bookkeeper.enter(None) - try: - desc.pycall(schedule, args, annmodel.s_ImpossibleValue) - finally: - self.bookkeeper.leave() - self.policy = prevpolicy - [(graph, inputcells)] = result - return graph, inputcells - - def annotate_helper(self, function, args_s, policy=None): - if policy is None: - from pypy.annotation.policy import AnnotatorPolicy - policy = AnnotatorPolicy() - graph, inputcells = self.get_call_parameters(function, args_s, policy) - self.build_graph_types(graph, inputcells, complete_now=False) - self.complete_helpers(policy) - return graph - - def complete_helpers(self, policy): - saved = self.policy, self.added_blocks - self.policy = policy - try: - self.added_blocks = {} - self.complete() - # invoke annotation simplifications for the new blocks - self.simplify(block_subset=self.added_blocks) - finally: - self.policy, self.added_blocks = saved - - def build_graph_types(self, flowgraph, inputcells, complete_now=True): - checkgraph(flowgraph) - - nbarg = len(flowgraph.getargs()) - if len(inputcells) != nbarg: - raise TypeError("%s expects %d args, got %d" %( - flowgraph, nbarg, len(inputcells))) - - # register the entry point - self.addpendinggraph(flowgraph, inputcells) - # recursively proceed until no more pending block is left - if complete_now: - self.complete() - return self.binding(flowgraph.getreturnvar(), None) - - def gettype(self, variable): - """Return the known type of a control flow graph variable, - defaulting to 'object'.""" - if isinstance(variable, Constant): - return type(variable.value) - elif isinstance(variable, Variable): - cell = self.bindings.get(variable) - if cell: - return cell.knowntype - else: - return object - else: - raise TypeError, ("Variable or Constant instance expected, " - "got %r" % (variable,)) - - def getuserclassdefinitions(self): - """Return a list of ClassDefs.""" - return self.bookkeeper.classdefs - - #___ medium-level interface ____________________________ - - def addpendinggraph(self, flowgraph, inputcells): - self.addpendingblock(flowgraph, flowgraph.startblock, inputcells) - - def addpendingblock(self, graph, block, cells): - """Register an entry point into block with the given input cells.""" - if graph in self.fixed_graphs: - # special case for annotating/rtyping in several phases: calling - # a graph that has already been rtyped. Safety-check the new - # annotations that are passed in, and don't annotate the old - # graph -- it's already low-level operations! - for a, s_newarg in zip(graph.getargs(), cells): - s_oldarg = self.binding(a) - assert annmodel.unionof(s_oldarg, s_newarg) == s_oldarg - else: - assert not self.frozen - for a in cells: - assert isinstance(a, annmodel.SomeObject) - if block not in self.annotated: - self.bindinputargs(graph, block, cells) - else: - self.mergeinputargs(graph, block, cells) - if not self.annotated[block]: - self.pendingblocks[block] = graph - - def complete(self): - """Process pending blocks until none is left.""" - while True: - while self.pendingblocks: - block, graph = self.pendingblocks.popitem() - self.processblock(graph, block) - self.policy.no_more_blocks_to_annotate(self) - if not self.pendingblocks: - break # finished - # make sure that the return variables of all graphs is annotated - if self.added_blocks is not None: - newgraphs = [self.annotated[block] for block in self.added_blocks] - newgraphs = dict.fromkeys(newgraphs) - got_blocked_blocks = False in newgraphs - else: - newgraphs = self.translator.graphs #all of them - got_blocked_blocks = False in self.annotated.values() - if got_blocked_blocks: - for graph in self.blocked_graphs.values(): - self.blocked_graphs[graph] = True - - blocked_blocks = [block for block, done in self.annotated.items() - if done is False] - assert len(blocked_blocks) == len(self.blocked_blocks) - - text = format_blocked_annotation_error(self, self.blocked_blocks) - #raise SystemExit() - raise AnnotatorError(text) - for graph in newgraphs: - v = graph.getreturnvar() - if v not in self.bindings: - self.setbinding(v, annmodel.s_ImpossibleValue) - # policy-dependent computation - self.bookkeeper.compute_at_fixpoint() - - def binding(self, arg, default=FAIL): - "Gives the SomeValue corresponding to the given Variable or Constant." - if isinstance(arg, Variable): - try: - return self.bindings[arg] - except KeyError: - if default is not FAIL: - return default - else: - raise - elif isinstance(arg, Constant): - #if arg.value is undefined_value: # undefined local variables - # return annmodel.s_ImpossibleValue - return self.bookkeeper.immutableconstant(arg) - else: - raise TypeError, 'Variable or Constant expected, got %r' % (arg,) - - def typeannotation(self, t): - return signature.annotation(t, self.bookkeeper) - - def setbinding(self, arg, s_value): - if arg in self.bindings: - assert s_value.contains(self.bindings[arg]) - self.bindings[arg] = s_value - - def transfer_binding(self, v_target, v_source): - assert v_source in self.bindings - self.bindings[v_target] = self.bindings[v_source] - - def warning(self, msg, pos=None): - if pos is None: - try: - pos = self.bookkeeper.position_key - except AttributeError: - pos = '?' - if pos != '?': - pos = self.whereami(pos) - - log.WARNING("%s/ %s" % (pos, msg)) - - - #___ interface for annotator.bookkeeper _______ - - def recursivecall(self, graph, whence, inputcells): - if isinstance(whence, tuple): - parent_graph, parent_block, parent_index = whence - tag = parent_block, parent_index - self.translator.update_call_graph(parent_graph, graph, tag) - # self.notify[graph.returnblock] is a dictionary of call - # points to this func which triggers a reflow whenever the - # return block of this graph has been analysed. - callpositions = self.notify.setdefault(graph.returnblock, {}) - if whence is not None: - if callable(whence): - def callback(): - whence(self, graph) - else: - callback = whence - callpositions[callback] = True - - # generalize the function's input arguments - self.addpendingblock(graph, graph.startblock, inputcells) - - # get the (current) return value - v = graph.getreturnvar() - try: - return self.bindings[v] - except KeyError: - # the function didn't reach any return statement so far. - # (some functions actually never do, they always raise exceptions) - return annmodel.s_ImpossibleValue - - def reflowfromposition(self, position_key): - graph, block, index = position_key - self.reflowpendingblock(graph, block) - - - #___ simplification (should be moved elsewhere?) _______ - - def simplify(self, block_subset=None, extra_passes=None): - # Generic simplifications - transform.transform_graph(self, block_subset=block_subset, - extra_passes=extra_passes) - if block_subset is None: - graphs = self.translator.graphs - else: - graphs = {} - for block in block_subset: - graph = self.annotated.get(block) - if graph: - graphs[graph] = True - for graph in graphs: - simplify.eliminate_empty_blocks(graph) - - - #___ flowing annotations in blocks _____________________ - - def processblock(self, graph, block): - # Important: this is not called recursively. - # self.flowin() can only issue calls to self.addpendingblock(). - # The analysis of a block can be in three states: - # * block not in self.annotated: - # never seen the block. - # * self.annotated[block] == False: - # the input variables of the block are in self.bindings but we - # still have to consider all the operations in the block. - # * self.annotated[block] == graph-containing-block: - # analysis done (at least until we find we must generalize the - # input variables). - - #print '* processblock', block, cells - self.annotated[block] = graph - if block in self.blocked_blocks: - del self.blocked_blocks[block] - try: - self.flowin(graph, block) - except BlockedInference, e: - self.annotated[block] = False # failed, hopefully temporarily - self.blocked_blocks[block] = (graph, e.opindex) - except Exception, e: - # hack for debug tools only - if not hasattr(e, '__annotator_block'): - setattr(e, '__annotator_block', block) - raise - - # The dict 'added_blocks' is used by rpython.annlowlevel to - # detect which are the new blocks that annotating an additional - # small helper creates. - if self.added_blocks is not None: - self.added_blocks[block] = True - - def reflowpendingblock(self, graph, block): - assert not self.frozen - assert graph not in self.fixed_graphs - self.pendingblocks[block] = graph - assert block in self.annotated - self.annotated[block] = False # must re-flow - self.blocked_blocks[block] = (graph, None) - - def bindinputargs(self, graph, block, inputcells): - # Create the initial bindings for the input args of a block. - assert len(block.inputargs) == len(inputcells) - for a, cell in zip(block.inputargs, inputcells): - self.setbinding(a, cell) - self.annotated[block] = False # must flowin. - self.blocked_blocks[block] = (graph, None) - - def mergeinputargs(self, graph, block, inputcells): - # Merge the new 'cells' with each of the block's existing input - # variables. - oldcells = [self.binding(a) for a in block.inputargs] - try: - unions = [annmodel.unionof(c1,c2) for c1, c2 in zip(oldcells,inputcells)] - except annmodel.UnionError, e: - e.args = e.args + ( - ErrorWrapper(gather_error(self, graph, block, None)),) - raise - # if the merged cells changed, we must redo the analysis - if unions != oldcells: - self.bindinputargs(graph, block, unions) - - def whereami(self, position_key): - graph, block, i = position_key - blk = "" - if block: - at = block.at() - if at: - blk = " block"+at - opid="" - if i is not None: - opid = " op=%d" % i - return repr(graph) + blk + opid - - def flowin(self, graph, block): - #print 'Flowing', block, [self.binding(a) for a in block.inputargs] - try: - for i in range(len(block.operations)): - try: - self.bookkeeper.enter((graph, block, i)) - self.consider_op(block, i) - finally: - self.bookkeeper.leave() - - except BlockedInference, e: - if (e.op is block.operations[-1] and - block.exitswitch == c_last_exception): - # this is the case where the last operation of the block will - # always raise an exception which is immediately caught by - # an exception handler. We then only follow the exceptional - # branches. - exits = [link for link in block.exits - if link.exitcase is not None] - - elif e.op.opname in ('simple_call', 'call_args', 'next'): - # XXX warning, keep the name of the call operations in sync - # with the flow object space. These are the operations for - # which it is fine to always raise an exception. We then - # swallow the BlockedInference and that's it. - # About 'next': see test_annotate_iter_empty_container(). - return - - else: - # other cases are problematic (but will hopefully be solved - # later by reflowing). Throw the BlockedInference up to - # processblock(). - raise - - except annmodel.HarmlesslyBlocked: - return - - else: - # dead code removal: don't follow all exits if the exitswitch - # is known - exits = block.exits - if isinstance(block.exitswitch, Variable): - s_exitswitch = self.bindings[block.exitswitch] - if s_exitswitch.is_constant(): - exits = [link for link in exits - if link.exitcase == s_exitswitch.const] - - # mapping (exitcase, variable) -> s_annotation - # that can be attached to booleans, exitswitches - knowntypedata = getattr(self.bindings.get(block.exitswitch), - "knowntypedata", {}) - - # filter out those exceptions which cannot - # occour for this specific, typed operation. - if block.exitswitch == c_last_exception: - op = block.operations[-1] - if op.opname in binaryop.BINARY_OPERATIONS: - arg1 = self.binding(op.args[0]) - arg2 = self.binding(op.args[1]) - binop = getattr(pair(arg1, arg2), op.opname, None) - can_only_throw = annmodel.read_can_only_throw(binop, arg1, arg2) - elif op.opname in unaryop.UNARY_OPERATIONS: - arg1 = self.binding(op.args[0]) - opname = op.opname - if opname == 'contains': opname = 'op_contains' - unop = getattr(arg1, opname, None) - can_only_throw = annmodel.read_can_only_throw(unop, arg1) - else: - can_only_throw = None - - if can_only_throw is not None: - candidates = can_only_throw - candidate_exits = exits - exits = [] - for link in candidate_exits: - case = link.exitcase - if case is None: - exits.append(link) - continue - covered = [c for c in candidates if issubclass(c, case)] - if covered: - exits.append(link) - candidates = [c for c in candidates if c not in covered] - - for link in exits: - in_except_block = False - - last_exception_var = link.last_exception # may be None for non-exception link - last_exc_value_var = link.last_exc_value # may be None for non-exception link - - if isinstance(link.exitcase, (types.ClassType, type)) \ - and issubclass(link.exitcase, py.builtin.BaseException): - assert last_exception_var and last_exc_value_var - last_exc_value_object = self.bookkeeper.valueoftype(link.exitcase) - last_exception_object = annmodel.SomeType() - if isinstance(last_exception_var, Constant): - last_exception_object.const = last_exception_var.value - last_exception_object.is_type_of = [last_exc_value_var] - - if isinstance(last_exception_var, Variable): - self.setbinding(last_exception_var, last_exception_object) - if isinstance(last_exc_value_var, Variable): - self.setbinding(last_exc_value_var, last_exc_value_object) - - last_exception_object = annmodel.SomeType() - if isinstance(last_exception_var, Constant): - last_exception_object.const = last_exception_var.value - #if link.exitcase is Exception: - # last_exc_value_object = annmodel.SomeObject() - #else: - last_exc_value_vars = [] - in_except_block = True - - ignore_link = False - cells = [] - renaming = {} - for a,v in zip(link.args,link.target.inputargs): - renaming.setdefault(a, []).append(v) - for a,v in zip(link.args,link.target.inputargs): - if a == last_exception_var: - assert in_except_block - cells.append(last_exception_object) - elif a == last_exc_value_var: - assert in_except_block - cells.append(last_exc_value_object) - last_exc_value_vars.append(v) - else: - cell = self.binding(a) - if (link.exitcase, a) in knowntypedata: - knownvarvalue = knowntypedata[(link.exitcase, a)] - cell = pair(cell, knownvarvalue).improve() - # ignore links that try to pass impossible values - if cell == annmodel.s_ImpossibleValue: - ignore_link = True - - if hasattr(cell,'is_type_of'): - renamed_is_type_of = [] - for v in cell.is_type_of: - new_vs = renaming.get(v,[]) - renamed_is_type_of += new_vs - assert cell.knowntype is type - newcell = annmodel.SomeType() - if cell.is_constant(): - newcell.const = cell.const - cell = newcell - cell.is_type_of = renamed_is_type_of - - if hasattr(cell, 'knowntypedata'): - renamed_knowntypedata = {} - for (value, v), s in cell.knowntypedata.items(): - new_vs = renaming.get(v, []) - for new_v in new_vs: - renamed_knowntypedata[value, new_v] = s - assert isinstance(cell, annmodel.SomeBool) - newcell = annmodel.SomeBool() - if cell.is_constant(): - newcell.const = cell.const - cell = newcell - cell.set_knowntypedata(renamed_knowntypedata) - - cells.append(cell) - - if ignore_link: - continue - - if in_except_block: - last_exception_object.is_type_of = last_exc_value_vars - - self.links_followed[link] = True - self.addpendingblock(graph, link.target, cells) - - if block in self.notify: - # reflow from certain positions when this block is done - for callback in self.notify[block]: - if isinstance(callback, tuple): - self.reflowfromposition(callback) # callback is a position - else: - callback() - - - #___ creating the annotations based on operations ______ - - def consider_op(self, block, opindex): - op = block.operations[opindex] - argcells = [self.binding(a) for a in op.args] - consider_meth = getattr(self,'consider_op_'+op.opname, - None) - if not consider_meth: - raise Exception,"unknown op: %r" % op - - # let's be careful about avoiding propagated SomeImpossibleValues - # to enter an op; the latter can result in violations of the - # more general results invariant: e.g. if SomeImpossibleValue enters is_ - # is_(SomeImpossibleValue, None) -> SomeBool - # is_(SomeInstance(not None), None) -> SomeBool(const=False) ... - # boom -- in the assert of setbinding() - for arg in argcells: - if isinstance(arg, annmodel.SomeImpossibleValue): - raise BlockedInference(self, op, opindex) - try: - resultcell = consider_meth(*argcells) - except Exception, e: - graph = self.bookkeeper.position_key[0] - e.args = e.args + ( - ErrorWrapper(gather_error(self, graph, block, opindex)),) - raise - if resultcell is None: - resultcell = self.noreturnvalue(op) - elif resultcell == annmodel.s_ImpossibleValue: - raise BlockedInference(self, op, opindex) # the operation cannot succeed - assert isinstance(resultcell, annmodel.SomeObject) - assert isinstance(op.result, Variable) - self.setbinding(op.result, resultcell) # bind resultcell to op.result - - def noreturnvalue(self, op): - return annmodel.s_ImpossibleValue # no return value (hook method) - - # XXX "contains" clash with SomeObject method - def consider_op_contains(self, seq, elem): - self.bookkeeper.count("contains", seq) - return seq.op_contains(elem) - - def consider_op_newtuple(self, *args): - return annmodel.SomeTuple(items = args) - - def consider_op_newlist(self, *args): - return self.bookkeeper.newlist(*args) - - def consider_op_newdict(self): - return self.bookkeeper.newdict() - - - def _registeroperations(cls, unary_ops, binary_ops): - # All unary operations - d = {} - for opname in unary_ops: - fnname = 'consider_op_' + opname - exec py.code.Source(""" -def consider_op_%s(self, arg, *args): - return arg.%s(*args) -""" % (opname, opname)).compile() in globals(), d - setattr(cls, fnname, d[fnname]) - # All binary operations - for opname in binary_ops: - fnname = 'consider_op_' + opname - exec py.code.Source(""" -def consider_op_%s(self, arg1, arg2, *args): - return pair(arg1,arg2).%s(*args) -""" % (opname, opname)).compile() in globals(), d - setattr(cls, fnname, d[fnname]) - _registeroperations = classmethod(_registeroperations) - -# register simple operations handling -RPythonAnnotator._registeroperations(unaryop.UNARY_OPERATIONS, binaryop.BINARY_OPERATIONS) - - -class BlockedInference(Exception): - """This exception signals the type inference engine that the situation - is currently blocked, and that it should try to progress elsewhere.""" - - def __init__(self, annotator, op, opindex): - self.annotator = annotator - try: - self.break_at = annotator.bookkeeper.position_key - except AttributeError: - self.break_at = None - self.op = op - self.opindex = opindex - - def __repr__(self): - if not self.break_at: - break_at = "?" - else: - break_at = self.annotator.whereami(self.break_at) - return "" %(break_at, self.op) - - __str__ = __repr__ diff --git a/pypy/annotation/binaryop.py b/pypy/annotation/binaryop.py deleted file mode 100644 --- a/pypy/annotation/binaryop.py +++ /dev/null @@ -1,1052 +0,0 @@ -""" -Binary operations between SomeValues. -""" - -import py -import operator -from pypy.tool.pairtype import pair, pairtype -from pypy.annotation.model import SomeObject, SomeInteger, SomeBool, s_Bool -from pypy.annotation.model import SomeString, SomeChar, SomeList, SomeDict -from pypy.annotation.model import SomeUnicodeCodePoint, SomeUnicodeString -from pypy.annotation.model import SomeTuple, SomeImpossibleValue, s_ImpossibleValue -from pypy.annotation.model import SomeInstance, SomeBuiltin, SomeIterator -from pypy.annotation.model import SomePBC, SomeFloat, s_None, SomeByteArray -from pypy.annotation.model import SomeExternalObject, SomeWeakRef -from pypy.annotation.model import SomeAddress, SomeTypedAddressAccess -from pypy.annotation.model import SomeSingleFloat, SomeLongFloat, SomeType -from pypy.annotation.model import unionof, UnionError, missing_operation -from pypy.annotation.model import TLS -from pypy.annotation.model import read_can_only_throw -from pypy.annotation.model import add_knowntypedata, merge_knowntypedata -from pypy.annotation.model import SomeGenericCallable -from pypy.annotation.bookkeeper import getbookkeeper -from pypy.objspace.flow.model import Variable, Constant -from pypy.rlib import rarithmetic -from pypy.tool.error import AnnotatorError - -# convenience only! -def immutablevalue(x): - return getbookkeeper().immutablevalue(x) - -# XXX unify this with ObjSpace.MethodTable -BINARY_OPERATIONS = set(['add', 'sub', 'mul', 'div', 'mod', - 'truediv', 'floordiv', 'divmod', - 'and_', 'or_', 'xor', - 'lshift', 'rshift', - 'getitem', 'setitem', 'delitem', - 'getitem_idx', 'getitem_key', 'getitem_idx_key', - 'inplace_add', 'inplace_sub', 'inplace_mul', - 'inplace_truediv', 'inplace_floordiv', 'inplace_div', - 'inplace_mod', - 'inplace_lshift', 'inplace_rshift', - 'inplace_and', 'inplace_or', 'inplace_xor', - 'lt', 'le', 'eq', 'ne', 'gt', 'ge', 'is_', 'cmp', - 'coerce', - ] - +[opname+'_ovf' for opname in - """add sub mul floordiv div mod lshift - """.split() - ]) - -for opname in BINARY_OPERATIONS: - missing_operation(pairtype(SomeObject, SomeObject), opname) - -class __extend__(pairtype(SomeObject, SomeObject)): - - def union((obj1, obj2)): - raise UnionError(obj1, obj2) - - # inplace_xxx ---> xxx by default - def inplace_add((obj1, obj2)): return pair(obj1, obj2).add() - def inplace_sub((obj1, obj2)): return pair(obj1, obj2).sub() - def inplace_mul((obj1, obj2)): return pair(obj1, obj2).mul() - def inplace_truediv((obj1, obj2)): return pair(obj1, obj2).truediv() - def inplace_floordiv((obj1, obj2)): return pair(obj1, obj2).floordiv() - def inplace_div((obj1, obj2)): return pair(obj1, obj2).div() - def inplace_mod((obj1, obj2)): return pair(obj1, obj2).mod() - def inplace_lshift((obj1, obj2)): return pair(obj1, obj2).lshift() - def inplace_rshift((obj1, obj2)): return pair(obj1, obj2).rshift() - def inplace_and((obj1, obj2)): return pair(obj1, obj2).and_() - def inplace_or((obj1, obj2)): return pair(obj1, obj2).or_() - def inplace_xor((obj1, obj2)): return pair(obj1, obj2).xor() - - for name, func in locals().items(): - if name.startswith('inplace_'): - func.can_only_throw = [] - - inplace_div.can_only_throw = [ZeroDivisionError] - inplace_truediv.can_only_throw = [ZeroDivisionError] - inplace_floordiv.can_only_throw = [ZeroDivisionError] - inplace_mod.can_only_throw = [ZeroDivisionError] - - def lt((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const < obj2.const) - else: - getbookkeeper().count("non_int_comp", obj1, obj2) - return s_Bool - - def le((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const <= obj2.const) - else: - getbookkeeper().count("non_int_comp", obj1, obj2) - return s_Bool - - def eq((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const == obj2.const) - else: - getbookkeeper().count("non_int_eq", obj1, obj2) - return s_Bool - - def ne((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const != obj2.const) - else: - getbookkeeper().count("non_int_eq", obj1, obj2) - return s_Bool - - def gt((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const > obj2.const) - else: - getbookkeeper().count("non_int_comp", obj1, obj2) - return s_Bool - - def ge((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const >= obj2.const) - else: - getbookkeeper().count("non_int_comp", obj1, obj2) - return s_Bool - - def cmp((obj1, obj2)): - getbookkeeper().count("cmp", obj1, obj2) - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(cmp(obj1.const, obj2.const)) - else: - return SomeInteger() - - def is_((obj1, obj2)): - r = SomeBool() - if obj2.is_constant(): - if obj1.is_constant(): - r.const = obj1.const is obj2.const - if obj2.const is None and not obj1.can_be_none(): - r.const = False - elif obj1.is_constant(): - if obj1.const is None and not obj2.can_be_none(): - r.const = False - # XXX HACK HACK HACK - # XXX HACK HACK HACK - # XXX HACK HACK HACK - bk = getbookkeeper() - if bk is not None: # for testing - knowntypedata = {} - fn, block, i = bk.position_key - - annotator = bk.annotator - op = block.operations[i] - assert op.opname == "is_" - assert len(op.args) == 2 - - def bind(src_obj, tgt_obj, tgt_arg): - if hasattr(tgt_obj, 'is_type_of') and src_obj.is_constant(): - add_knowntypedata(knowntypedata, True, tgt_obj.is_type_of, - bk.valueoftype(src_obj.const)) - - assert annotator.binding(op.args[tgt_arg]) == tgt_obj - add_knowntypedata(knowntypedata, True, [op.args[tgt_arg]], src_obj) - - nonnone_obj = tgt_obj - if src_obj.is_constant() and src_obj.const is None and tgt_obj.can_be_none(): - nonnone_obj = tgt_obj.nonnoneify() - - add_knowntypedata(knowntypedata, False, [op.args[tgt_arg]], nonnone_obj) - - bind(obj2, obj1, 0) - bind(obj1, obj2, 1) - r.set_knowntypedata(knowntypedata) - - return r - - def divmod((obj1, obj2)): - getbookkeeper().count("divmod", obj1, obj2) - return SomeTuple([pair(obj1, obj2).div(), pair(obj1, obj2).mod()]) - - def coerce((obj1, obj2)): - getbookkeeper().count("coerce", obj1, obj2) - return pair(obj1, obj2).union() # reasonable enough - - # approximation of an annotation intersection, the result should be the annotation obj or - # the intersection of obj and improvement - def improve((obj, improvement)): - if not improvement.contains(obj) and obj.contains(improvement): - return improvement - else: - return obj - - # checked getitems - - def _getitem_can_only_throw(s_c1, s_o2): - impl = pair(s_c1, s_o2).getitem - return read_can_only_throw(impl, s_c1, s_o2) - - def getitem_idx_key((s_c1, s_o2)): - impl = pair(s_c1, s_o2).getitem - return impl() - getitem_idx_key.can_only_throw = _getitem_can_only_throw - - getitem_idx = getitem_idx_key - getitem_key = getitem_idx_key - - -class __extend__(pairtype(SomeType, SomeType)): - - def union((obj1, obj2)): - result = SomeType() - is_type_of1 = getattr(obj1, 'is_type_of', None) - is_type_of2 = getattr(obj2, 'is_type_of', None) - if obj1.is_immutable_constant() and obj2.is_immutable_constant() and obj1.const == obj2.const: - result.const = obj1.const - is_type_of = {} - if is_type_of1: - for v in is_type_of1: - is_type_of[v] = True - if is_type_of2: - for v in is_type_of2: - is_type_of[v] = True - if is_type_of: - result.is_type_of = is_type_of.keys() - else: - if is_type_of1 and is_type_of1 == is_type_of2: - result.is_type_of = is_type_of1 - return result - - -# cloning a function with identical code, for the can_only_throw attribute -def _clone(f, can_only_throw = None): - newfunc = type(f)(f.func_code, f.func_globals, f.func_name, - f.func_defaults, f.func_closure) - if can_only_throw is not None: - newfunc.can_only_throw = can_only_throw - return newfunc - -class __extend__(pairtype(SomeInteger, SomeInteger)): - # unsignedness is considered a rare and contagious disease - - def union((int1, int2)): - if int1.unsigned == int2.unsigned: - knowntype = rarithmetic.compute_restype(int1.knowntype, int2.knowntype) - else: - t1 = int1.knowntype - if t1 is bool: - t1 = int - t2 = int2.knowntype - if t2 is bool: - t2 = int - - if t2 is int: - if int2.nonneg == False: - raise UnionError, "Merging %s and a possibly negative int is not allowed" % t1 - knowntype = t1 - elif t1 is int: - if int1.nonneg == False: - raise UnionError, "Merging %s and a possibly negative int is not allowed" % t2 - knowntype = t2 - else: - raise UnionError, "Merging these types (%s, %s) is not supported" % (t1, t2) - return SomeInteger(nonneg=int1.nonneg and int2.nonneg, - knowntype=knowntype) - - or_ = xor = add = mul = _clone(union, []) - add_ovf = mul_ovf = _clone(union, [OverflowError]) - div = floordiv = mod = _clone(union, [ZeroDivisionError]) - div_ovf= floordiv_ovf = mod_ovf = _clone(union, [ZeroDivisionError, OverflowError]) - - def truediv((int1, int2)): - return SomeFloat() - truediv.can_only_throw = [ZeroDivisionError] - truediv_ovf = _clone(truediv, [ZeroDivisionError, OverflowError]) - - inplace_div = div - inplace_truediv = truediv - - def sub((int1, int2)): - knowntype = rarithmetic.compute_restype(int1.knowntype, int2.knowntype) - return SomeInteger(knowntype=knowntype) - sub.can_only_throw = [] - sub_ovf = _clone(sub, [OverflowError]) - - def and_((int1, int2)): - knowntype = rarithmetic.compute_restype(int1.knowntype, int2.knowntype) - return SomeInteger(nonneg=int1.nonneg or int2.nonneg, - knowntype=knowntype) - and_.can_only_throw = [] - - def lshift((int1, int2)): - if isinstance(int1, SomeBool): - return SomeInteger() - else: - return SomeInteger(knowntype=int1.knowntype) - lshift.can_only_throw = [] - lshift_ovf = _clone(lshift, [OverflowError]) - - def rshift((int1, int2)): - if isinstance(int1, SomeBool): - return SomeInteger(nonneg=True) - else: - return SomeInteger(nonneg=int1.nonneg, knowntype=int1.knowntype) - rshift.can_only_throw = [] - - def _compare_helper((int1, int2), opname, operation): - r = SomeBool() - if int1.is_immutable_constant() and int2.is_immutable_constant(): - r.const = operation(int1.const, int2.const) - # - # The rest of the code propagates nonneg information between - # the two arguments. - # - # Doing the right thing when int1 or int2 change from signed - # to unsigned (r_uint) is almost impossible. See test_intcmp_bug. - # Instead, we only deduce constrains on the operands in the - # case where they are both signed. In other words, if y is - # nonneg then "assert x>=y" will let the annotator know that - # x is nonneg too, but it will not work if y is unsigned. - # - if not (rarithmetic.signedtype(int1.knowntype) and - rarithmetic.signedtype(int2.knowntype)): - return r - knowntypedata = {} - # XXX HACK HACK HACK - fn, block, i = getbookkeeper().position_key - op = block.operations[i] - assert op.opname == opname - assert len(op.args) == 2 - def tointtype(int0): - if int0.knowntype is bool: - return int - return int0.knowntype - if int1.nonneg and isinstance(op.args[1], Variable): - case = opname in ('lt', 'le', 'eq') - - add_knowntypedata(knowntypedata, case, [op.args[1]], - SomeInteger(nonneg=True, knowntype=tointtype(int2))) - if int2.nonneg and isinstance(op.args[0], Variable): - case = opname in ('gt', 'ge', 'eq') - add_knowntypedata(knowntypedata, case, [op.args[0]], - SomeInteger(nonneg=True, knowntype=tointtype(int1))) - r.set_knowntypedata(knowntypedata) - # a special case for 'x < 0' or 'x >= 0', - # where 0 is a flow graph Constant - # (in this case we are sure that it cannot become a r_uint later) - if (isinstance(op.args[1], Constant) and - type(op.args[1].value) is int and # filter out Symbolics - op.args[1].value == 0): - if int1.nonneg: - if opname == 'lt': - r.const = False - if opname == 'ge': - r.const = True - return r - - def lt(intint): return intint._compare_helper('lt', operator.lt) - def le(intint): return intint._compare_helper('le', operator.le) - def eq(intint): return intint._compare_helper('eq', operator.eq) - def ne(intint): return intint._compare_helper('ne', operator.ne) - def gt(intint): return intint._compare_helper('gt', operator.gt) - def ge(intint): return intint._compare_helper('ge', operator.ge) - - -class __extend__(pairtype(SomeBool, SomeBool)): - - def union((boo1, boo2)): - s = SomeBool() - if getattr(boo1, 'const', -1) == getattr(boo2, 'const', -2): - s.const = boo1.const - if hasattr(boo1, 'knowntypedata') and \ - hasattr(boo2, 'knowntypedata'): - ktd = merge_knowntypedata(boo1.knowntypedata, boo2.knowntypedata) - s.set_knowntypedata(ktd) - return s - - def and_((boo1, boo2)): - s = SomeBool() - if boo1.is_constant(): - if not boo1.const: - s.const = False - else: - return boo2 - if boo2.is_constant(): - if not boo2.const: - s.const = False - return s - - def or_((boo1, boo2)): - s = SomeBool() - if boo1.is_constant(): - if boo1.const: - s.const = True - else: - return boo2 - if boo2.is_constant(): - if boo2.const: - s.const = True - return s - - def xor((boo1, boo2)): - s = SomeBool() - if boo1.is_constant() and boo2.is_constant(): - s.const = boo1.const ^ boo2.const - return s - -class __extend__(pairtype(SomeString, SomeString)): - - def union((str1, str2)): - can_be_None = str1.can_be_None or str2.can_be_None - no_nul = str1.no_nul and str2.no_nul - return SomeString(can_be_None=can_be_None, no_nul=no_nul) - - def add((str1, str2)): - # propagate const-ness to help getattr(obj, 'prefix' + const_name) - result = SomeString(no_nul=str1.no_nul and str2.no_nul) - if str1.is_immutable_constant() and str2.is_immutable_constant(): - result.const = str1.const + str2.const - return result - -class __extend__(pairtype(SomeByteArray, SomeByteArray)): - def union((b1, b2)): - can_be_None = b1.can_be_None or b2.can_be_None - return SomeByteArray(can_be_None=can_be_None) - - def add((b1, b2)): - result = SomeByteArray() - if b1.is_immutable_constant() and b2.is_immutable_constant(): - result.const = b1.const + b2.const - return result - -class __extend__(pairtype(SomeByteArray, SomeInteger)): - def getitem((s_b, s_i)): - return SomeInteger() - - def setitem((s_b, s_i), s_i2): - assert isinstance(s_i2, SomeInteger) - -class __extend__(pairtype(SomeString, SomeByteArray), - pairtype(SomeByteArray, SomeString), - pairtype(SomeChar, SomeByteArray), - pairtype(SomeByteArray, SomeChar)): - def add((b1, b2)): - result = SomeByteArray() - if b1.is_immutable_constant() and b2.is_immutable_constant(): - result.const = b1.const + b2.const - return result - -class __extend__(pairtype(SomeChar, SomeChar)): - - def union((chr1, chr2)): - return SomeChar() - - -class __extend__(pairtype(SomeChar, SomeUnicodeCodePoint), - pairtype(SomeUnicodeCodePoint, SomeChar)): - def union((uchr1, uchr2)): - return SomeUnicodeCodePoint() - -class __extend__(pairtype(SomeUnicodeCodePoint, SomeUnicodeCodePoint)): - def union((uchr1, uchr2)): - return SomeUnicodeCodePoint() - - def add((chr1, chr2)): - return SomeUnicodeString() - -class __extend__(pairtype(SomeString, SomeUnicodeString), - pairtype(SomeUnicodeString, SomeString)): - def mod((str, unistring)): - raise NotImplementedError( - "string formatting mixing strings and unicode not supported") - - -class __extend__(pairtype(SomeString, SomeTuple), - pairtype(SomeUnicodeString, SomeTuple)): - def mod((s_string, s_tuple)): - is_string = isinstance(s_string, SomeString) - is_unicode = isinstance(s_string, SomeUnicodeString) - assert is_string or is_unicode - for s_item in s_tuple.items: - if (is_unicode and isinstance(s_item, (SomeChar, SomeString)) or - is_string and isinstance(s_item, (SomeUnicodeCodePoint, - SomeUnicodeString))): - raise NotImplementedError( - "string formatting mixing strings and unicode not supported") - getbookkeeper().count('strformat', s_string, s_tuple) - no_nul = s_string.no_nul - for s_item in s_tuple.items: - if isinstance(s_item, SomeFloat): - pass # or s_item is a subclass, like SomeInteger - elif (isinstance(s_item, SomeString) or - isinstance(s_item, SomeUnicodeString)) and s_item.no_nul: - pass - else: - no_nul = False - break - return s_string.__class__(no_nul=no_nul) - - -class __extend__(pairtype(SomeString, SomeObject), - pairtype(SomeUnicodeString, SomeObject)): - - def mod((s_string, args)): - getbookkeeper().count('strformat', s_string, args) - return s_string.__class__() - -class __extend__(pairtype(SomeFloat, SomeFloat)): - - def union((flt1, flt2)): - return SomeFloat() - - add = sub = mul = union - - def div((flt1, flt2)): - return SomeFloat() - div.can_only_throw = [] - truediv = div - - # repeat these in order to copy the 'can_only_throw' attribute - inplace_div = div - inplace_truediv = truediv - - -class __extend__(pairtype(SomeSingleFloat, SomeSingleFloat)): - - def union((flt1, flt2)): - return SomeSingleFloat() - - -class __extend__(pairtype(SomeLongFloat, SomeLongFloat)): - - def union((flt1, flt2)): - return SomeLongFloat() - - -class __extend__(pairtype(SomeList, SomeList)): - - def union((lst1, lst2)): - return SomeList(lst1.listdef.union(lst2.listdef)) - - def add((lst1, lst2)): - return lst1.listdef.offspring(lst2.listdef) - - def eq((lst1, lst2)): - lst1.listdef.agree(lst2.listdef) - return s_Bool - ne = eq - - -class __extend__(pairtype(SomeList, SomeObject)): - - def inplace_add((lst1, obj2)): - lst1.method_extend(obj2) - return lst1 - inplace_add.can_only_throw = [] - - def inplace_mul((lst1, obj2)): - lst1.listdef.resize() - return lst1 - inplace_mul.can_only_throw = [] - -class __extend__(pairtype(SomeTuple, SomeTuple)): - - def union((tup1, tup2)): - if len(tup1.items) != len(tup2.items): - raise UnionError("cannot take the union of a tuple of length %d " - "and a tuple of length %d" % (len(tup1.items), - len(tup2.items))) - else: - unions = [unionof(x,y) for x,y in zip(tup1.items, tup2.items)] - return SomeTuple(items = unions) - - def add((tup1, tup2)): - return SomeTuple(items = tup1.items + tup2.items) - - def eq(tup1tup2): - tup1tup2.union() - return s_Bool - ne = eq - - def lt((tup1, tup2)): - raise Exception("unsupported: (...) < (...)") - def le((tup1, tup2)): - raise Exception("unsupported: (...) <= (...)") - def gt((tup1, tup2)): - raise Exception("unsupported: (...) > (...)") - def ge((tup1, tup2)): - raise Exception("unsupported: (...) >= (...)") - - -class __extend__(pairtype(SomeDict, SomeDict)): - - def union((dic1, dic2)): - return SomeDict(dic1.dictdef.union(dic2.dictdef)) - - -class __extend__(pairtype(SomeDict, SomeObject)): - - def _can_only_throw(dic1, *ignore): - if dic1.dictdef.dictkey.custom_eq_hash: - return None - return [KeyError] - - def getitem((dic1, obj2)): - getbookkeeper().count("dict_getitem", dic1) - dic1.dictdef.generalize_key(obj2) - return dic1.dictdef.read_value() - getitem.can_only_throw = _can_only_throw - - def setitem((dic1, obj2), s_value): - getbookkeeper().count("dict_setitem", dic1) - dic1.dictdef.generalize_key(obj2) - dic1.dictdef.generalize_value(s_value) - setitem.can_only_throw = _can_only_throw - - def delitem((dic1, obj2)): - getbookkeeper().count("dict_delitem", dic1) - dic1.dictdef.generalize_key(obj2) - delitem.can_only_throw = _can_only_throw - - -class __extend__(pairtype(SomeTuple, SomeInteger)): - - def getitem((tup1, int2)): - if int2.is_immutable_constant(): - try: - return tup1.items[int2.const] - except IndexError: - return s_ImpossibleValue - else: - getbookkeeper().count("tuple_random_getitem", tup1) - return unionof(*tup1.items) - getitem.can_only_throw = [IndexError] - - -class __extend__(pairtype(SomeList, SomeInteger)): - - def mul((lst1, int2)): - return lst1.listdef.offspring() - - def getitem((lst1, int2)): - getbookkeeper().count("list_getitem", int2) - return lst1.listdef.read_item() - getitem.can_only_throw = [] - - getitem_key = getitem - - def getitem_idx((lst1, int2)): - getbookkeeper().count("list_getitem", int2) - return lst1.listdef.read_item() - getitem_idx.can_only_throw = [IndexError] - - getitem_idx_key = getitem_idx - - def setitem((lst1, int2), s_value): - getbookkeeper().count("list_setitem", int2) - lst1.listdef.mutate() - lst1.listdef.generalize(s_value) - setitem.can_only_throw = [IndexError] - - def delitem((lst1, int2)): - getbookkeeper().count("list_delitem", int2) - lst1.listdef.resize() - delitem.can_only_throw = [IndexError] - -class __extend__(pairtype(SomeString, SomeInteger)): - - def getitem((str1, int2)): - getbookkeeper().count("str_getitem", int2) - return SomeChar() - getitem.can_only_throw = [] - - getitem_key = getitem - - def getitem_idx((str1, int2)): - getbookkeeper().count("str_getitem", int2) - return SomeChar() - getitem_idx.can_only_throw = [IndexError] - - getitem_idx_key = getitem_idx - - def mul((str1, int2)): # xxx do we want to support this - getbookkeeper().count("str_mul", str1, int2) - return SomeString(no_nul=str1.no_nul) - -class __extend__(pairtype(SomeUnicodeString, SomeInteger)): - def getitem((str1, int2)): - getbookkeeper().count("str_getitem", int2) - return SomeUnicodeCodePoint() - getitem.can_only_throw = [] - - getitem_key = getitem - - def getitem_idx((str1, int2)): - getbookkeeper().count("str_getitem", int2) - return SomeUnicodeCodePoint() - getitem_idx.can_only_throw = [IndexError] - - getitem_idx_key = getitem_idx - - def mul((str1, int2)): # xxx do we want to support this - getbookkeeper().count("str_mul", str1, int2) - return SomeUnicodeString() - -class __extend__(pairtype(SomeInteger, SomeString), - pairtype(SomeInteger, SomeUnicodeString)): - - def mul((int1, str2)): # xxx do we want to support this - getbookkeeper().count("str_mul", str2, int1) - return str2.basestringclass() - -class __extend__(pairtype(SomeUnicodeCodePoint, SomeUnicodeString), - pairtype(SomeUnicodeString, SomeUnicodeCodePoint), - pairtype(SomeUnicodeString, SomeUnicodeString)): - def union((str1, str2)): - return SomeUnicodeString(can_be_None=str1.can_be_none() or - str2.can_be_none()) - - def add((str1, str2)): - # propagate const-ness to help getattr(obj, 'prefix' + const_name) - result = SomeUnicodeString() - if str1.is_immutable_constant() and str2.is_immutable_constant(): - result.const = str1.const + str2.const - return result - -class __extend__(pairtype(SomeInteger, SomeList)): - - def mul((int1, lst2)): - return lst2.listdef.offspring() - - -class __extend__(pairtype(SomeInstance, SomeInstance)): - - def union((ins1, ins2)): - if ins1.classdef is None or ins2.classdef is None: - # special case only - basedef = None - else: - basedef = ins1.classdef.commonbase(ins2.classdef) - if basedef is None: - raise UnionError(ins1, ins2) - flags = ins1.flags - if flags: - flags = flags.copy() - for key, value in flags.items(): - if key not in ins2.flags or ins2.flags[key] != value: - del flags[key] - return SomeInstance(basedef, - can_be_None=ins1.can_be_None or ins2.can_be_None, - flags=flags) - - def improve((ins1, ins2)): - if ins1.classdef is None: - resdef = ins2.classdef - elif ins2.classdef is None: - resdef = ins1.classdef - else: - basedef = ins1.classdef.commonbase(ins2.classdef) - if basedef is ins1.classdef: - resdef = ins2.classdef - elif basedef is ins2.classdef: - resdef = ins1.classdef - else: - if ins1.can_be_None and ins2.can_be_None: - return s_None - else: - return s_ImpossibleValue - res = SomeInstance(resdef, can_be_None=ins1.can_be_None and ins2.can_be_None) - if ins1.contains(res) and ins2.contains(res): - return res # fine - else: - # this case can occur in the presence of 'const' attributes, - # which we should try to preserve. Fall-back... - thistype = pairtype(SomeInstance, SomeInstance) - return super(thistype, pair(ins1, ins2)).improve() - - -class __extend__(pairtype(SomeIterator, SomeIterator)): - - def union((iter1, iter2)): - s_cont = unionof(iter1.s_container, iter2.s_container) - if iter1.variant != iter2.variant: - raise UnionError("merging incompatible iterators variants") - return SomeIterator(s_cont, *iter1.variant) - - -class __extend__(pairtype(SomeBuiltin, SomeBuiltin)): - - def union((bltn1, bltn2)): - if (bltn1.analyser != bltn2.analyser or - bltn1.methodname != bltn2.methodname or - bltn1.s_self is None or bltn2.s_self is None): - raise UnionError("cannot merge two different builtin functions " - "or methods:\n %r\n %r" % (bltn1, bltn2)) - s_self = unionof(bltn1.s_self, bltn2.s_self) - return SomeBuiltin(bltn1.analyser, s_self, methodname=bltn1.methodname) - -class __extend__(pairtype(SomePBC, SomePBC)): - - def union((pbc1, pbc2)): - d = pbc1.descriptions.copy() - d.update(pbc2.descriptions) - return SomePBC(d, can_be_None = pbc1.can_be_None or pbc2.can_be_None) - - def is_((pbc1, pbc2)): - thistype = pairtype(SomePBC, SomePBC) - s = super(thistype, pair(pbc1, pbc2)).is_() - if not s.is_constant(): - if not pbc1.can_be_None or not pbc2.can_be_None: - for desc in pbc1.descriptions: - if desc in pbc2.descriptions: - break - else: - s.const = False # no common desc in the two sets - return s - -class __extend__(pairtype(SomeGenericCallable, SomePBC)): - def union((gencall, pbc)): - for desc in pbc.descriptions: - unique_key = desc - bk = desc.bookkeeper - s_result = bk.emulate_pbc_call(unique_key, pbc, gencall.args_s) - s_result = unionof(s_result, gencall.s_result) - assert gencall.s_result.contains(s_result) - return gencall - -class __extend__(pairtype(SomePBC, SomeGenericCallable)): - def union((pbc, gencall)): - return pair(gencall, pbc).union() - -class __extend__(pairtype(SomeImpossibleValue, SomeObject)): - def union((imp1, obj2)): - return obj2 - -class __extend__(pairtype(SomeObject, SomeImpossibleValue)): - def union((obj1, imp2)): - return obj1 - -# mixing Nones with other objects - -def _make_none_union(classname, constructor_args='', glob=None): - if glob is None: - glob = globals() - loc = locals() - source = py.code.Source(""" - class __extend__(pairtype(%(classname)s, SomePBC)): - def union((obj, pbc)): - if pbc.isNone(): - return %(classname)s(%(constructor_args)s) - else: - return SomeObject() - - class __extend__(pairtype(SomePBC, %(classname)s)): - def union((pbc, obj)): - if pbc.isNone(): - return %(classname)s(%(constructor_args)s) - else: - return SomeObject() - """ % loc) - exec source.compile() in glob - -_make_none_union('SomeInstance', 'classdef=obj.classdef, can_be_None=True') -_make_none_union('SomeString', 'no_nul=obj.no_nul, can_be_None=True') -_make_none_union('SomeUnicodeString', 'can_be_None=True') -_make_none_union('SomeList', 'obj.listdef') -_make_none_union('SomeDict', 'obj.dictdef') -_make_none_union('SomeExternalObject', 'obj.knowntype') -_make_none_union('SomeWeakRef', 'obj.classdef') - -# getitem on SomePBCs, in particular None fails - -class __extend__(pairtype(SomePBC, SomeObject)): - def getitem((pbc, o)): - if not pbc.isNone(): - raise AnnotatorError("getitem on %r" % pbc) - return s_ImpossibleValue - - def setitem((pbc, o), s_value): - if not pbc.isNone(): - raise AnnotatorError("setitem on %r" % pbc) - -class __extend__(pairtype(SomePBC, SomeString)): - def add((pbc, o)): - if not pbc.isNone(): - raise AnnotatorError('add on %r' % pbc) - return s_ImpossibleValue - -class __extend__(pairtype(SomeString, SomePBC)): - def add((o, pbc)): - if not pbc.isNone(): - raise AnnotatorError('add on %r' % pbc) - return s_ImpossibleValue - -class __extend__(pairtype(SomeExternalObject, SomeExternalObject)): - def union((ext1, ext2)): - if ext1.knowntype == ext2.knowntype: - return SomeExternalObject(ext1.knowntype) - return SomeObject() - -# ____________________________________________________________ -# annotation of low-level types -from pypy.annotation.model import SomePtr, SomeOOInstance, SomeOOClass -from pypy.annotation.model import SomeOOObject -from pypy.annotation.model import ll_to_annotation, annotation_to_lltype -from pypy.rpython.ootypesystem import ootype - -_make_none_union('SomeOOInstance', 'ootype=obj.ootype, can_be_None=True') - -class __extend__(pairtype(SomePtr, SomePtr)): - def union((p1, p2)): - assert p1.ll_ptrtype == p2.ll_ptrtype,("mixing of incompatible pointer types: %r, %r" % - (p1.ll_ptrtype, p2.ll_ptrtype)) - return SomePtr(p1.ll_ptrtype) - -class __extend__(pairtype(SomePtr, SomeInteger)): - - def getitem((p, int1)): - example = p.ll_ptrtype._example() - try: - v = example[0] - except IndexError: - return None # impossible value, e.g. FixedSizeArray(0) - return ll_to_annotation(v) - getitem.can_only_throw = [] - - def setitem((p, int1), s_value): # just doing checking - example = p.ll_ptrtype._example() - if example[0] is not None: # ignore Void s_value - v_lltype = annotation_to_lltype(s_value) - example[0] = v_lltype._defl() - setitem.can_only_throw = [] - -class __extend__(pairtype(SomePtr, SomeObject)): - def union((p, obj)): - assert False, ("mixing pointer type %r with something else %r" % (p.ll_ptrtype, obj)) - - def getitem((p, obj)): - assert False,"ptr %r getitem index not an int: %r" % (p.ll_ptrtype, obj) - - def setitem((p, obj), s_value): - assert False,"ptr %r setitem index not an int: %r" % (p.ll_ptrtype, obj) - -class __extend__(pairtype(SomeObject, SomePtr)): - def union((obj, p2)): - return pair(p2, obj).union() - - -class __extend__(pairtype(SomeOOInstance, SomeOOInstance)): - def union((r1, r2)): - common = ootype.commonBaseclass(r1.ootype, r2.ootype) - assert common is not None, 'Mixing of incompatible instances %r, %r' %(r1.ootype, r2.ootype) - return SomeOOInstance(common, can_be_None=r1.can_be_None or r2.can_be_None) - -class __extend__(pairtype(SomeOOClass, SomeOOClass)): - def union((r1, r2)): - if r1.ootype is None: - common = r2.ootype - elif r2.ootype is None: - common = r1.ootype - elif r1.ootype == r2.ootype: - common = r1.ootype - elif isinstance(r1.ootype, ootype.Instance) and isinstance(r2.ootype, ootype.Instance): - common = ootype.commonBaseclass(r1.ootype, r2.ootype) - assert common is not None, ('Mixing of incompatible classes %r, %r' - % (r1.ootype, r2.ootype)) - else: - common = ootype.Object - return SomeOOClass(common) - -class __extend__(pairtype(SomeOOInstance, SomeObject)): - def union((r, obj)): - assert False, ("mixing reference type %r with something else %r" % (r.ootype, obj)) - -class __extend__(pairtype(SomeObject, SomeOOInstance)): - def union((obj, r2)): - return pair(r2, obj).union() - -class __extend__(pairtype(SomeOOObject, SomeOOObject)): - def union((r1, r2)): - assert r1.ootype is ootype.Object and r2.ootype is ootype.Object - return SomeOOObject() - -#_________________________________________ -# weakrefs - -class __extend__(pairtype(SomeWeakRef, SomeWeakRef)): - def union((s_wrf1, s_wrf2)): - if s_wrf1.classdef is None: - basedef = s_wrf2.classdef # s_wrf1 is known to be dead - elif s_wrf2.classdef is None: - basedef = s_wrf1.classdef # s_wrf2 is known to be dead - else: - basedef = s_wrf1.classdef.commonbase(s_wrf2.classdef) - if basedef is None: # no common base class! complain... - return SomeObject() - return SomeWeakRef(basedef) - -#_________________________________________ -# memory addresses - -class __extend__(pairtype(SomeAddress, SomeAddress)): - def union((s_addr1, s_addr2)): - return SomeAddress() - - def sub((s_addr1, s_addr2)): - if s_addr1.is_null_address() and s_addr2.is_null_address(): - return getbookkeeper().immutablevalue(0) - return SomeInteger() - - def is_((s_addr1, s_addr2)): - assert False, "comparisons with is not supported by addresses" - -class __extend__(pairtype(SomeTypedAddressAccess, SomeTypedAddressAccess)): - def union((s_taa1, s_taa2)): - assert s_taa1.type == s_taa2.type - return s_taa1 - -class __extend__(pairtype(SomeTypedAddressAccess, SomeInteger)): - def getitem((s_taa, s_int)): - from pypy.annotation.model import lltype_to_annotation - return lltype_to_annotation(s_taa.type) - getitem.can_only_throw = [] - - def setitem((s_taa, s_int), s_value): - from pypy.annotation.model import annotation_to_lltype - assert annotation_to_lltype(s_value) is s_taa.type - setitem.can_only_throw = [] - - -class __extend__(pairtype(SomeAddress, SomeInteger)): - def add((s_addr, s_int)): - return SomeAddress() - - def sub((s_addr, s_int)): - return SomeAddress() - -class __extend__(pairtype(SomeAddress, SomeImpossibleValue)): - # need to override this specifically to hide the 'raise UnionError' - # of pairtype(SomeAddress, SomeObject). - def union((s_addr, s_imp)): - return s_addr - -class __extend__(pairtype(SomeImpossibleValue, SomeAddress)): - # need to override this specifically to hide the 'raise UnionError' - # of pairtype(SomeObject, SomeAddress). - def union((s_imp, s_addr)): - return s_addr - -class __extend__(pairtype(SomeAddress, SomeObject)): - def union((s_addr, s_obj)): - raise UnionError, "union of address and anything else makes no sense" - -class __extend__(pairtype(SomeObject, SomeAddress)): - def union((s_obj, s_addr)): - raise UnionError, "union of address and anything else makes no sense" diff --git a/pypy/annotation/bookkeeper.py b/pypy/annotation/bookkeeper.py deleted file mode 100644 --- a/pypy/annotation/bookkeeper.py +++ /dev/null @@ -1,797 +0,0 @@ -""" -The Bookkeeper class. -""" - -from __future__ import absolute_import - -import sys, types, inspect, weakref - -from pypy.objspace.flow.model import Constant From noreply at buildbot.pypy.org Sun Jan 20 16:00:51 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 20 Jan 2013 16:00:51 +0100 (CET) Subject: [pypy-commit] pypy default: err, remove unnecessary changes Message-ID: <20130120150051.DDA351C0237@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r60240:291b1440912f Date: 2013-01-20 17:00 +0200 http://bitbucket.org/pypy/pypy/changeset/291b1440912f/ Log: err, remove unnecessary changes diff --git a/pypy/module/unicodedata/interp_ucd.py b/pypy/module/unicodedata/interp_ucd.py --- a/pypy/module/unicodedata/interp_ucd.py +++ b/pypy/module/unicodedata/interp_ucd.py @@ -9,7 +9,7 @@ from rpython.rlib.objectmodel import we_are_translated from rpython.rlib.runicode import MAXUNICODE from rpython.rlib.unicodedata import unicodedb_5_2_0, unicodedb_3_2_0 -from rpython.rlib.unicodedata.ucd import code_to_unichr, ORD +from rpython.rlib.runicode import code_to_unichr, ORD import sys diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py --- a/rpython/rlib/runicode.py +++ b/rpython/rlib/runicode.py @@ -3,9 +3,91 @@ from rpython.rlib.rstring import StringBuilder, UnicodeBuilder from rpython.rlib.rarithmetic import r_uint, intmask from rpython.rlib.unicodedata import unicodedb -from rpython.rlib.unicodedata.ucd import MAXUNICODE, UNICHR, BYTEORDER +from rpython.rtyper.lltypesystem import lltype, rffi +if rffi.sizeof(lltype.UniChar) == 4: + MAXUNICODE = 0x10ffff +else: + MAXUNICODE = 0xffff + +BYTEORDER = sys.byteorder + +if MAXUNICODE > sys.maxunicode: + # A version of unichr which allows codes outside the BMP + # even on narrow unicode builds. + # It will be used when interpreting code on top of a UCS2 CPython, + # when sizeof(wchar_t) == 4. + # Note that Python3 uses a similar implementation. + def UNICHR(c): + assert not we_are_translated() + if c <= sys.maxunicode or c > MAXUNICODE: + return unichr(c) + else: + c -= 0x10000 + return (unichr(0xD800 + (c >> 10)) + + unichr(0xDC00 + (c & 0x03FF))) + UNICHR._flowspace_rewrite_directly_as_ = unichr + # ^^^ NB.: for translation, it's essential to use this hack instead + # of calling unichr() from UNICHR(), because unichr() detects if there + # is a "try:except ValueError" immediately around it. + + def ORD(u): + assert not we_are_translated() + if isinstance(u, unicode) and len(u) == 2: + ch1 = ord(u[0]) + ch2 = ord(u[1]) + if 0xD800 <= ch1 <= 0xDBFF and 0xDC00 <= ch2 <= 0xDFFF: + return (((ch1 - 0xD800) << 10) | (ch2 - 0xDC00)) + 0x10000 + return ord(u) + ORD._flowspace_rewrite_directly_as_ = ord + +else: + UNICHR = unichr + ORD = ord + +if MAXUNICODE > 0xFFFF: + def code_to_unichr(code): + if not we_are_translated() and sys.maxunicode == 0xFFFF: + # Host CPython is narrow build, generate surrogates + return UNICHR(code) + else: + return unichr(code) +else: + def code_to_unichr(code): + # generate surrogates for large codes + return UNICHR(code) + + +def UNICHR(c): + if c <= sys.maxunicode and c <= MAXUNICODE: + return unichr(c) + else: + c -= 0x10000 + return (unichr(0xD800 + (c >> 10)) + + unichr(0xDC00 + (c & 0x03FF))) + +def ORD(u): + assert isinstance(u, unicode) + if len(u) == 1: + return ord(u[0]) + elif len(u) == 2: + ch1 = ord(u[0]) + ch2 = ord(u[1]) + if 0xD800 <= ch1 <= 0xDBFF and 0xDC00 <= ch2 <= 0xDFFF: + return (((ch1 - 0xD800) << 10) | (ch2 - 0xDC00)) + 0x10000 + raise ValueError + +def _STORECHAR(result, CH, byteorder): + hi = chr(((CH) >> 8) & 0xff) + lo = chr((CH) & 0xff) + if byteorder == 'little': + result.append(lo) + result.append(hi) + else: + result.append(hi) + result.append(lo) + def default_unicode_error_decode(errors, encoding, msg, s, startingpos, endingpos): if errors == 'replace': diff --git a/rpython/rlib/unicodedata/ucd.py b/rpython/rlib/unicodedata/ucd.py deleted file mode 100644 --- a/rpython/rlib/unicodedata/ucd.py +++ /dev/null @@ -1,87 +0,0 @@ - -import sys -from rpython.rtyper.lltypesystem import lltype, rffi -from rpython.rlib.objectmodel import we_are_translated - - -if rffi.sizeof(lltype.UniChar) == 4: - MAXUNICODE = 0x10ffff -else: - MAXUNICODE = 0xffff - -BYTEORDER = sys.byteorder - -if MAXUNICODE > sys.maxunicode: - # A version of unichr which allows codes outside the BMP - # even on narrow unicode builds. - # It will be used when interpreting code on top of a UCS2 CPython, - # when sizeof(wchar_t) == 4. - # Note that Python3 uses a similar implementation. - def UNICHR(c): - assert not we_are_translated() - if c <= sys.maxunicode or c > MAXUNICODE: - return unichr(c) - else: - c -= 0x10000 - return (unichr(0xD800 + (c >> 10)) + - unichr(0xDC00 + (c & 0x03FF))) - UNICHR._flowspace_rewrite_directly_as_ = unichr - # ^^^ NB.: for translation, it's essential to use this hack instead - # of calling unichr() from UNICHR(), because unichr() detects if there - # is a "try:except ValueError" immediately around it. - - def ORD(u): - assert not we_are_translated() - if isinstance(u, unicode) and len(u) == 2: - ch1 = ord(u[0]) - ch2 = ord(u[1]) - if 0xD800 <= ch1 <= 0xDBFF and 0xDC00 <= ch2 <= 0xDFFF: - return (((ch1 - 0xD800) << 10) | (ch2 - 0xDC00)) + 0x10000 - return ord(u) - ORD._flowspace_rewrite_directly_as_ = ord - -else: - UNICHR = unichr - ORD = ord - -if MAXUNICODE > 0xFFFF: - def code_to_unichr(code): - if not we_are_translated() and sys.maxunicode == 0xFFFF: - # Host CPython is narrow build, generate surrogates - return UNICHR(code) - else: - return unichr(code) -else: - def code_to_unichr(code): - # generate surrogates for large codes - return UNICHR(code) - - -def UNICHR(c): - if c <= sys.maxunicode and c <= MAXUNICODE: - return unichr(c) - else: - c -= 0x10000 - return (unichr(0xD800 + (c >> 10)) + - unichr(0xDC00 + (c & 0x03FF))) - -def ORD(u): - assert isinstance(u, unicode) - if len(u) == 1: - return ord(u[0]) - elif len(u) == 2: - ch1 = ord(u[0]) - ch2 = ord(u[1]) - if 0xD800 <= ch1 <= 0xDBFF and 0xDC00 <= ch2 <= 0xDFFF: - return (((ch1 - 0xD800) << 10) | (ch2 - 0xDC00)) + 0x10000 - raise ValueError - -def _STORECHAR(result, CH, byteorder): - hi = chr(((CH) >> 8) & 0xff) - lo = chr((CH) & 0xff) - if byteorder == 'little': - result.append(lo) - result.append(hi) - else: - result.append(hi) - result.append(lo) From noreply at buildbot.pypy.org Sun Jan 20 16:01:23 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 20 Jan 2013 16:01:23 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: merge default Message-ID: <20130120150123.1DDCF1C0237@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60241:53ce6999a6b1 Date: 2013-01-20 17:01 +0200 http://bitbucket.org/pypy/pypy/changeset/53ce6999a6b1/ Log: merge default diff --git a/pypy/module/unicodedata/interp_ucd.py b/pypy/module/unicodedata/interp_ucd.py --- a/pypy/module/unicodedata/interp_ucd.py +++ b/pypy/module/unicodedata/interp_ucd.py @@ -9,7 +9,7 @@ from rpython.rlib.objectmodel import we_are_translated from rpython.rlib.runicode import MAXUNICODE from rpython.rlib.unicodedata import unicodedb_5_2_0, unicodedb_3_2_0 -from rpython.rlib.unicodedata.ucd import code_to_unichr, ORD +from rpython.rlib.runicode import code_to_unichr, ORD import sys diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py --- a/rpython/rlib/runicode.py +++ b/rpython/rlib/runicode.py @@ -3,9 +3,91 @@ from rpython.rlib.rstring import StringBuilder, UnicodeBuilder from rpython.rlib.rarithmetic import r_uint, intmask from rpython.rlib.unicodedata import unicodedb -from rpython.rlib.unicodedata.ucd import MAXUNICODE, UNICHR, BYTEORDER +from rpython.rtyper.lltypesystem import lltype, rffi +if rffi.sizeof(lltype.UniChar) == 4: + MAXUNICODE = 0x10ffff +else: + MAXUNICODE = 0xffff + +BYTEORDER = sys.byteorder + +if MAXUNICODE > sys.maxunicode: + # A version of unichr which allows codes outside the BMP + # even on narrow unicode builds. + # It will be used when interpreting code on top of a UCS2 CPython, + # when sizeof(wchar_t) == 4. + # Note that Python3 uses a similar implementation. + def UNICHR(c): + assert not we_are_translated() + if c <= sys.maxunicode or c > MAXUNICODE: + return unichr(c) + else: + c -= 0x10000 + return (unichr(0xD800 + (c >> 10)) + + unichr(0xDC00 + (c & 0x03FF))) + UNICHR._flowspace_rewrite_directly_as_ = unichr + # ^^^ NB.: for translation, it's essential to use this hack instead + # of calling unichr() from UNICHR(), because unichr() detects if there + # is a "try:except ValueError" immediately around it. + + def ORD(u): + assert not we_are_translated() + if isinstance(u, unicode) and len(u) == 2: + ch1 = ord(u[0]) + ch2 = ord(u[1]) + if 0xD800 <= ch1 <= 0xDBFF and 0xDC00 <= ch2 <= 0xDFFF: + return (((ch1 - 0xD800) << 10) | (ch2 - 0xDC00)) + 0x10000 + return ord(u) + ORD._flowspace_rewrite_directly_as_ = ord + +else: + UNICHR = unichr + ORD = ord + +if MAXUNICODE > 0xFFFF: + def code_to_unichr(code): + if not we_are_translated() and sys.maxunicode == 0xFFFF: + # Host CPython is narrow build, generate surrogates + return UNICHR(code) + else: + return unichr(code) +else: + def code_to_unichr(code): + # generate surrogates for large codes + return UNICHR(code) + + +def UNICHR(c): + if c <= sys.maxunicode and c <= MAXUNICODE: + return unichr(c) + else: + c -= 0x10000 + return (unichr(0xD800 + (c >> 10)) + + unichr(0xDC00 + (c & 0x03FF))) + +def ORD(u): + assert isinstance(u, unicode) + if len(u) == 1: + return ord(u[0]) + elif len(u) == 2: + ch1 = ord(u[0]) + ch2 = ord(u[1]) + if 0xD800 <= ch1 <= 0xDBFF and 0xDC00 <= ch2 <= 0xDFFF: + return (((ch1 - 0xD800) << 10) | (ch2 - 0xDC00)) + 0x10000 + raise ValueError + +def _STORECHAR(result, CH, byteorder): + hi = chr(((CH) >> 8) & 0xff) + lo = chr((CH) & 0xff) + if byteorder == 'little': + result.append(lo) + result.append(hi) + else: + result.append(hi) + result.append(lo) + def default_unicode_error_decode(errors, encoding, msg, s, startingpos, endingpos): if errors == 'replace': diff --git a/rpython/rlib/unicodedata/ucd.py b/rpython/rlib/unicodedata/ucd.py deleted file mode 100644 --- a/rpython/rlib/unicodedata/ucd.py +++ /dev/null @@ -1,5 +0,0 @@ - -import sys -from rpython.rtyper.lltypesystem import lltype, rffi -from rpython.rlib.objectmodel import we_are_translated - From noreply at buildbot.pypy.org Sun Jan 20 16:33:08 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 20 Jan 2013 16:33:08 +0100 (CET) Subject: [pypy-commit] pypy default: remove strange copies for now. I think some explanations is needed Message-ID: <20130120153308.31C051C02D9@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r60242:727405791c06 Date: 2013-01-20 17:32 +0200 http://bitbucket.org/pypy/pypy/changeset/727405791c06/ Log: remove strange copies for now. I think some explanations is needed diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py --- a/rpython/rlib/runicode.py +++ b/rpython/rlib/runicode.py @@ -58,26 +58,6 @@ # generate surrogates for large codes return UNICHR(code) - -def UNICHR(c): - if c <= sys.maxunicode and c <= MAXUNICODE: - return unichr(c) - else: - c -= 0x10000 - return (unichr(0xD800 + (c >> 10)) + - unichr(0xDC00 + (c & 0x03FF))) - -def ORD(u): - assert isinstance(u, unicode) - if len(u) == 1: - return ord(u[0]) - elif len(u) == 2: - ch1 = ord(u[0]) - ch2 = ord(u[1]) - if 0xD800 <= ch1 <= 0xDBFF and 0xDC00 <= ch2 <= 0xDFFF: - return (((ch1 - 0xD800) << 10) | (ch2 - 0xDC00)) + 0x10000 - raise ValueError - def _STORECHAR(result, CH, byteorder): hi = chr(((CH) >> 8) & 0xff) lo = chr((CH) & 0xff) From noreply at buildbot.pypy.org Sun Jan 20 19:59:10 2013 From: noreply at buildbot.pypy.org (hakanardo) Date: Sun, 20 Jan 2013 19:59:10 +0100 (CET) Subject: [pypy-commit] pypy jit-usable_retrace_3: second failure found Message-ID: <20130120185910.2B36F1C1305@cobra.cs.uni-duesseldorf.de> Author: Hakan Ardo Branch: jit-usable_retrace_3 Changeset: r60243:44f28ed8c749 Date: 2013-01-20 11:39 +0100 http://bitbucket.org/pypy/pypy/changeset/44f28ed8c749/ Log: second failure found diff --git a/pypy/jit/metainterp/test/test_random_loops.py b/pypy/jit/metainterp/test/test_random_loops.py --- a/pypy/jit/metainterp/test/test_random_loops.py +++ b/pypy/jit/metainterp/test/test_random_loops.py @@ -32,7 +32,7 @@ class RandomLoopBase(object): def check(self, bytecode, args=(0,0,0,0,0), max_back_jumps=-1, **kwargs): - print 'check:', bytecode, args, max_back_jumps + print "check('%s'," % bytecode, args, ',%d)' % max_back_jumps bytecode = re.subn('\s', '', bytecode)[0] offsets = self.offsets(bytecode) assert len(args) == 5 @@ -231,6 +231,9 @@ def test_failure1(self): self.check('{{7d<(37>(c6-A{a6>n(x)5b<()(bb+Bea-A){7d>n(x)1e+A7d-B3d+E35+Eac-B2b<(04+Cxe8+D9d+C)(){c6n(x)a5=(2c+E)(xxe9-A)ec>(b7-A40+Ce0-Cx06-A)(xc0-Ad2-A1e+B0d+Bx09+B0b-D)}be-Ab7=(3a+Ab7-Dde<(x))(ec+Cd4+D5d<(x10+A)(cb-Cb9-C))59<(2b-C00-Ab0<(bc-Bde+Bxa2-E4d+A))(e7+Eb3<(c4-Bxx)59-C7c+Aa8>(96-Exce+D3e-Ex)db+A76-Cdb-Bdc=(2b-A))}d2+D)(27-C26+B39+A9d+Bcc+Dab+Cdd=(ae+Eee-Ex{5d>n(x)}d0<(2a+C41+Dca+C6a>(47-Ae9-Eb7+Cx60+D63-C)bb+D9e+Ec5=(d2-Cxaa-D84-Ced+B20+A4c+B)65+A59-E)ed<(cd+B{a8=n(x)ed+Bxxd8+A}ba-D{66=n(x)a4+D30+Acd+C1c+A6c+Ac7-Dxc0+B}x)4b-Dx)e4-D{aa=n(x)44-Eac>(3e+Aae+A12<(a4-A)99<(ce-C4a+A7b+Dxdd+E)(c5+A10-D63-Eda-E22-E85+Dea+E)x)ed+Dbb=(e5-A9e-C)(x{can(x)}e5+Ba5+B}c9=()c0=(7b-Ae1+Daa+Aed+C)(91-C))3d=(a2>(1e-Cba=(2c-A)21+C6b>(aa-Bxd5-Bc7-E)ab<(8c-Ba8-Cxea-D4e+Ae8+Dc5+Ade+B)x33+B5a-C17+D)2a+B{ebn(x)x6a+Cb4+A27-Eab-C35-E25+B1d+C}cc+Aae-D}cc+C61-C68-D)(xe4>(e4+Db4-Cxdb=(xc7-Bcd-Bde-A00+A39+E9c+Dc9+B24-B){2b>n(x)x4d+E8d+Dbb+Ce7-Bxea+B54+A}80-A43<(89-B)(1c+Cx7b+Ae2+E02-C84+Exd0+A))(3d+Eb3-Ecd-A{8c=n(x)37-Bb0+Axxee+C11-C2c-De6-D46+D}cb+E)84+B05-Cbd+D)ad>(b6-C7d+Ae4-B4a+E)34-Cx9a=({c4>n(x)dd+B72<(a9+A)(b0-Db8-Dac-Bba-Excb+Ce2+Bx)ba-Bda+Cce-Cx}78+E97=(aa+Dx67+Dbc+B{de>n(x)b8-C14-Cx})c4+Cac+Ex80+C))xca=({55(de+Ax)da-B{d6>n(x)8d+A36-Bx8d-Bx49+Bbc-D88+Bx}x26-Da7-E)({9c>n(x)}bc<(0a-Cx61+Cce+A54-Ce7+A21-C9d+D)(6d-E)x{7e=n(x)6d-Ce6-Cx}5c>(bd+Ed5+Cc0-Dc2+C93-B)x69>(d9-Ac8+B8c-Dx94-E76+Ax)(ba+D)b6>(bb+Ebb-A5e+Bc3-D6c-E))54>(95+B46-D{15(xb6+Ec6-Aac-Bxc2-Ax99-C))b0-E58>(x0e+Bxa4+D5e>(7e-Dde-B1c+Bba-B8a-D)(35-Cce-C4c-C90-C1e+B9a+Be7+C1d-E)45>(xb9+A91+Ecc-Aac-B91+Cd1-De3+Ddd-B)c0+A0a>(45+Cbd+Bdc+Db4-D7c+C29+A7a+B60+A))(5d>(xx6e-Dxea+C)(c2-Cad+D36-Cx)61+E20-C9e-Cc2+Cce+Ad0+B{4b=n(x)})x{a1=n(x)e8<(x35-Dd3-Bd6-Bea+Eac-Acc+B79+Eaa-C)}x)xc7>(6b-C{57>n(x)e2>(1a+A97+Dxcc-E7b+A5d-B82+C2b-B)0a+Be1-Cbc+Ca2-E01+B{41>n(x)e1+Dc6+B47+Beb+Ac2+Bxab+D}}d3-D0e+E0d+Bae=(c7+Ce3-Bc3<(d2+E68+Bbc-Cdc+B1b-Ea2-Ac8-E)45-B43+Aac-E)xbe+C)dc=({ee>n(x)a4-Axd2+B1b+B}aa+Exxxb4>()(cb-C{4d(a0-Dab+A8c+Ddd-Edc+A)(xce-D)de=(d6+E83+Cxxc6-Axx)(0b+D78+Cx07-Eda+Ax1d-Eba+A)e6+E)de-E1c+Aae=(20-B25+Dba>(xxx71-A2b+Dc7+E){e9=n(x)7b+Ecd+Bda-Eec-A4b+De7-Ae3-D}ae+C7c+Aaa+Bx)86+B)}{cc=n(x)6b-B{cd(cc+D7e-Bc0+Bxde-A2c-B5d-C)(ba+D2b-Axxx07+Ca4-C)eb-C23-D7e<(xb8+Edd-De1+Add+C35-Dxx)x)(87-Ccc+Ee2=(9a-E78-C)()cb-A)39+D0b-Aa4>(ca+A)c6=({9c(bb-E22-D)x5e-C11<()9b=(6d-Bcd+A9e-E2a-E38+B4e-C)xaa-B{82=n(x)ae-B6a-Ab7+Ca9-A}08+C)d7+Cxcb-A}6c-B9c+D9c-E})3d+Cd4>(xab<()()3e+Ebd-Ba0-B{6c(63+Eb5+Bxec<(a1+E4e+Axdc-C)ea+Eaa=(x37-Edd-Cb4+B)(43-Ae0-Bx7e-C94+A)e8-Bd1-Da8-D)())(ce=(6c-B10+C7a+A3d+Ax19+Aca+B)(ed+A)1d>()d1+Ex43+A9d+Eb5+A)}8d+Ae9-C)(3d-D6c+Ee9<(29-C4c=({e9(4c+B)8a-Dc0-E82+E50+D}a1+D98=(be-B)3a-Ae9+De2-Add-E)35-D}}', [13, 40, 35, 44, 76], 1000) + def test_failure2(self): + self.check('{{d2-Bab-D48<(e3-E0d+Eac-B1d>(9b>(25>(14+Da9-Eb9-B{05=n(x)bb+Cb2+Ee6+C}35-Acd+E)(c4-Exaa-A{ce=n(x)}0c+E)7e<(0b-B85-Dc9-Bb2-Dcc<(x11+A9a+Cx)(xxde+Da8+B)6b-Ada-D)aa+D))73+D{dc=n(x)d9-Bea+A8e=(81<(e8-D)(94+B8c+D9d-E{ee=n(x)a0+Dx43-Bx}a0+B56+Ca7-C4e=(01-C4c+Dxa7-B))3a-Cae-B50+Cb3-Aee+D4b-Bb5+B)}ed+Ea6>(ab+Abb-C86+B{b9n(x){b1>n(x)21+C}3e=(db+Excd-Dx21+C)48-B}{6a(x{0b Author: Hakan Ardo Branch: jit-usable_retrace_3 Changeset: r60244:a606c6ca4a15 Date: 2013-01-20 15:42 +0100 http://bitbucket.org/pypy/pypy/changeset/a606c6ca4a15/ Log: earlier error diff --git a/pypy/jit/metainterp/test/test_random_loops.py b/pypy/jit/metainterp/test/test_random_loops.py --- a/pypy/jit/metainterp/test/test_random_loops.py +++ b/pypy/jit/metainterp/test/test_random_loops.py @@ -232,7 +232,7 @@ self.check('{{7d<(37>(c6-A{a6>n(x)5b<()(bb+Bea-A){7d>n(x)1e+A7d-B3d+E35+Eac-B2b<(04+Cxe8+D9d+C)(){c6n(x)a5=(2c+E)(xxe9-A)ec>(b7-A40+Ce0-Cx06-A)(xc0-Ad2-A1e+B0d+Bx09+B0b-D)}be-Ab7=(3a+Ab7-Dde<(x))(ec+Cd4+D5d<(x10+A)(cb-Cb9-C))59<(2b-C00-Ab0<(bc-Bde+Bxa2-E4d+A))(e7+Eb3<(c4-Bxx)59-C7c+Aa8>(96-Exce+D3e-Ex)db+A76-Cdb-Bdc=(2b-A))}d2+D)(27-C26+B39+A9d+Bcc+Dab+Cdd=(ae+Eee-Ex{5d>n(x)}d0<(2a+C41+Dca+C6a>(47-Ae9-Eb7+Cx60+D63-C)bb+D9e+Ec5=(d2-Cxaa-D84-Ced+B20+A4c+B)65+A59-E)ed<(cd+B{a8=n(x)ed+Bxxd8+A}ba-D{66=n(x)a4+D30+Acd+C1c+A6c+Ac7-Dxc0+B}x)4b-Dx)e4-D{aa=n(x)44-Eac>(3e+Aae+A12<(a4-A)99<(ce-C4a+A7b+Dxdd+E)(c5+A10-D63-Eda-E22-E85+Dea+E)x)ed+Dbb=(e5-A9e-C)(x{can(x)}e5+Ba5+B}c9=()c0=(7b-Ae1+Daa+Aed+C)(91-C))3d=(a2>(1e-Cba=(2c-A)21+C6b>(aa-Bxd5-Bc7-E)ab<(8c-Ba8-Cxea-D4e+Ae8+Dc5+Ade+B)x33+B5a-C17+D)2a+B{ebn(x)x6a+Cb4+A27-Eab-C35-E25+B1d+C}cc+Aae-D}cc+C61-C68-D)(xe4>(e4+Db4-Cxdb=(xc7-Bcd-Bde-A00+A39+E9c+Dc9+B24-B){2b>n(x)x4d+E8d+Dbb+Ce7-Bxea+B54+A}80-A43<(89-B)(1c+Cx7b+Ae2+E02-C84+Exd0+A))(3d+Eb3-Ecd-A{8c=n(x)37-Bb0+Axxee+C11-C2c-De6-D46+D}cb+E)84+B05-Cbd+D)ad>(b6-C7d+Ae4-B4a+E)34-Cx9a=({c4>n(x)dd+B72<(a9+A)(b0-Db8-Dac-Bba-Excb+Ce2+Bx)ba-Bda+Cce-Cx}78+E97=(aa+Dx67+Dbc+B{de>n(x)b8-C14-Cx})c4+Cac+Ex80+C))xca=({55(de+Ax)da-B{d6>n(x)8d+A36-Bx8d-Bx49+Bbc-D88+Bx}x26-Da7-E)({9c>n(x)}bc<(0a-Cx61+Cce+A54-Ce7+A21-C9d+D)(6d-E)x{7e=n(x)6d-Ce6-Cx}5c>(bd+Ed5+Cc0-Dc2+C93-B)x69>(d9-Ac8+B8c-Dx94-E76+Ax)(ba+D)b6>(bb+Ebb-A5e+Bc3-D6c-E))54>(95+B46-D{15(xb6+Ec6-Aac-Bxc2-Ax99-C))b0-E58>(x0e+Bxa4+D5e>(7e-Dde-B1c+Bba-B8a-D)(35-Cce-C4c-C90-C1e+B9a+Be7+C1d-E)45>(xb9+A91+Ecc-Aac-B91+Cd1-De3+Ddd-B)c0+A0a>(45+Cbd+Bdc+Db4-D7c+C29+A7a+B60+A))(5d>(xx6e-Dxea+C)(c2-Cad+D36-Cx)61+E20-C9e-Cc2+Cce+Ad0+B{4b=n(x)})x{a1=n(x)e8<(x35-Dd3-Bd6-Bea+Eac-Acc+B79+Eaa-C)}x)xc7>(6b-C{57>n(x)e2>(1a+A97+Dxcc-E7b+A5d-B82+C2b-B)0a+Be1-Cbc+Ca2-E01+B{41>n(x)e1+Dc6+B47+Beb+Ac2+Bxab+D}}d3-D0e+E0d+Bae=(c7+Ce3-Bc3<(d2+E68+Bbc-Cdc+B1b-Ea2-Ac8-E)45-B43+Aac-E)xbe+C)dc=({ee>n(x)a4-Axd2+B1b+B}aa+Exxxb4>()(cb-C{4d(a0-Dab+A8c+Ddd-Edc+A)(xce-D)de=(d6+E83+Cxxc6-Axx)(0b+D78+Cx07-Eda+Ax1d-Eba+A)e6+E)de-E1c+Aae=(20-B25+Dba>(xxx71-A2b+Dc7+E){e9=n(x)7b+Ecd+Bda-Eec-A4b+De7-Ae3-D}ae+C7c+Aaa+Bx)86+B)}{cc=n(x)6b-B{cd(cc+D7e-Bc0+Bxde-A2c-B5d-C)(ba+D2b-Axxx07+Ca4-C)eb-C23-D7e<(xb8+Edd-De1+Add+C35-Dxx)x)(87-Ccc+Ee2=(9a-E78-C)()cb-A)39+D0b-Aa4>(ca+A)c6=({9c(bb-E22-D)x5e-C11<()9b=(6d-Bcd+A9e-E2a-E38+B4e-C)xaa-B{82=n(x)ae-B6a-Ab7+Ca9-A}08+C)d7+Cxcb-A}6c-B9c+D9c-E})3d+Cd4>(xab<()()3e+Ebd-Ba0-B{6c(63+Eb5+Bxec<(a1+E4e+Axdc-C)ea+Eaa=(x37-Edd-Cb4+B)(43-Ae0-Bx7e-C94+A)e8-Bd1-Da8-D)())(ce=(6c-B10+C7a+A3d+Ax19+Aca+B)(ed+A)1d>()d1+Ex43+A9d+Eb5+A)}8d+Ae9-C)(3d-D6c+Ee9<(29-C4c=({e9(4c+B)8a-Dc0-E82+E50+D}a1+D98=(be-B)3a-Ae9+De2-Add-E)35-D}}', [13, 40, 35, 44, 76], 1000) def test_failure2(self): - self.check('{{d2-Bab-D48<(e3-E0d+Eac-B1d>(9b>(25>(14+Da9-Eb9-B{05=n(x)bb+Cb2+Ee6+C}35-Acd+E)(c4-Exaa-A{ce=n(x)}0c+E)7e<(0b-B85-Dc9-Bb2-Dcc<(x11+A9a+Cx)(xxde+Da8+B)6b-Ada-D)aa+D))73+D{dc=n(x)d9-Bea+A8e=(81<(e8-D)(94+B8c+D9d-E{ee=n(x)a0+Dx43-Bx}a0+B56+Ca7-C4e=(01-C4c+Dxa7-B))3a-Cae-B50+Cb3-Aee+D4b-Bb5+B)}ed+Ea6>(ab+Abb-C86+B{b9n(x){b1>n(x)21+C}3e=(db+Excd-Dx21+C)48-B}{6a(x{0b(9b>(25>(14+Da9-Eb9-B{05=n(x)bb+Cb2+Ee6+C}35-Acd+E)(c4-Exaa-A{ce=n(x)}0c+E)7e<(0b-B85-Dc9-Bb2-Dcc<(x11+A9a+Cx)(xxde+Da8+B)6b-Ada-D)aa+D))73+D{dc=n(x)d9-Bea+A8e=(81<(e8-D)(94+B8c+D9d-E{ee=n(x)a0+Dx43-Bx}a0+B56+Ca7-C4e=(01-C4c+Dxa7-B))3a-Cae-B50+Cb3-Aee+D4b-Bb5+B)}ed+Ea6>(ab+Abb-C86+B{b9n(x){b1>n(x)21+C}3e=(db+Excd-Dx21+C)48-B}{6a(x{0b Author: Hakan Ardo Branch: jit-usable_retrace_3 Changeset: r60245:8e58b8431a3a Date: 2013-01-20 19:56 +0100 http://bitbucket.org/pypy/pypy/changeset/8e58b8431a3a/ Log: use overflow check instead to get a well defined (but silly) behaviour diff --git a/pypy/jit/metainterp/test/test_random_loops.py b/pypy/jit/metainterp/test/test_random_loops.py --- a/pypy/jit/metainterp/test/test_random_loops.py +++ b/pypy/jit/metainterp/test/test_random_loops.py @@ -1,7 +1,7 @@ from pypy.jit.metainterp.test.support import LLJitMixin from pypy.rlib.jit import JitDriver from random import choice, randrange -from pypy.rlib.rarithmetic import intmask +from pypy.rlib.rarithmetic import ovfcheck import re class IntBox(object): @@ -12,10 +12,16 @@ return self.val def add(self, other): - return IntBox(intmask(self.value() + other.value())) + try: + return IntBox(ovfcheck(self.value() + other.value())) + except OverflowError: + return IntBox(42) def sub(self, other): - return IntBox(intmask(self.value() - other.value())) + try: + return IntBox(ovfcheck(self.value() - other.value())) + except OverflowError: + return IntBox(-42) def gt(self, other): return IntBox(self.value() > other.value()) From noreply at buildbot.pypy.org Sun Jan 20 20:04:07 2013 From: noreply at buildbot.pypy.org (mattip) Date: Sun, 20 Jan 2013 20:04:07 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: back out c158cea62fa8 Message-ID: <20130120190407.87F761C1305@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: missing-ndarray-attributes Changeset: r60246:bef758981c7f Date: 2013-01-20 18:10 +0200 http://bitbucket.org/pypy/pypy/changeset/bef758981c7f/ Log: back out c158cea62fa8 diff --git a/pypy/rlib/rarithmetic.py b/pypy/rlib/rarithmetic.py --- a/pypy/rlib/rarithmetic.py +++ b/pypy/rlib/rarithmetic.py @@ -132,7 +132,6 @@ return r_longlonglong(n) def widen(n): - assert not isinstance(n, r_longfloat) from pypy.rpython.lltypesystem import lltype if _should_widen_type(lltype.typeOf(n)): return intmask(n) From noreply at buildbot.pypy.org Sun Jan 20 20:04:12 2013 From: noreply at buildbot.pypy.org (mattip) Date: Sun, 20 Jan 2013 20:04:12 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: failing test for byteswap on longfloat Message-ID: <20130120190414.4E34D1C1306@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: missing-ndarray-attributes Changeset: r60247:a9de6c414b4d Date: 2013-01-20 21:03 +0200 http://bitbucket.org/pypy/pypy/changeset/a9de6c414b4d/ Log: failing test for byteswap on longfloat diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1674,6 +1674,12 @@ s3 = [s1[1], s1[0],s1[3], s1[2], s1[5], s1[4]] assert s3 == s2 + a = array([1, -1, 10000], dtype='longfloat') + s1 = map(ord,a.tostring()) + s2 = map(ord, a.byteswap().tostring()) + n = a.dtype.itemsize + assert s1[n-1] == s2[0] + def test_clip(self): from _numpypy import array a = array([1, 2, 17, -3, 12]) diff --git a/pypy/rlib/rarithmetic.py b/pypy/rlib/rarithmetic.py --- a/pypy/rlib/rarithmetic.py +++ b/pypy/rlib/rarithmetic.py @@ -623,6 +623,8 @@ is_float = True T = rffi.LONGLONG arg = float2longlong(arg) + elif T == lltype.LongFloat: + raise NotImplemented('cannot byteswap LongFloat') else: # we cannot do arithmetics on small ints arg = widen(arg) From noreply at buildbot.pypy.org Sun Jan 20 20:48:13 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 20 Jan 2013 20:48:13 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fix the merge Message-ID: <20130120194813.AA5401C00F9@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60248:73672862e880 Date: 2013-01-20 21:47 +0200 http://bitbucket.org/pypy/pypy/changeset/73672862e880/ Log: fix the merge diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py --- a/rpython/jit/backend/llsupport/gc.py +++ b/rpython/jit/backend/llsupport/gc.py @@ -232,12 +232,18 @@ class GcRootMap_asmgcc(object): is_shadow_stack = False + def __init__(self, gcdescr): + pass + def register_asm_addr(self, start, mark): pass class GcRootMap_shadowstack(object): is_shadow_stack = True + def __init__(self, gcdescr): + pass + def register_asm_addr(self, start, mark): pass @@ -507,7 +513,8 @@ return rffi.cast(lltype.Signed, nurs_top_addr) def initialize(self): - self.gcrootmap.initialize() + pass + #self.gcrootmap.initialize() def init_size_descr(self, S, descr): if self.layoutbuilder is not None: diff --git a/rpython/jit/backend/llsupport/test/test_asmmemmgr.py b/rpython/jit/backend/llsupport/test/test_asmmemmgr.py --- a/rpython/jit/backend/llsupport/test/test_asmmemmgr.py +++ b/rpython/jit/backend/llsupport/test/test_asmmemmgr.py @@ -155,7 +155,7 @@ def test_insert_gcroot_marker(self): puts = [] class FakeGcRootMap: - def put(self, retaddr, mark): + def register_asm_addr(self, retaddr, mark): puts.append((retaddr, mark)) # mc = BlockBuilderMixin() diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py --- a/rpython/jit/backend/llsupport/test/test_rewrite.py +++ b/rpython/jit/backend/llsupport/test/test_rewrite.py @@ -1,6 +1,6 @@ from rpython.jit.backend.llsupport.descr import get_size_descr,\ get_field_descr, get_array_descr, ArrayDescr, FieldDescr,\ - SizeDescrWithVTable + SizeDescrWithVTable, get_interiorfield_descr from rpython.jit.backend.llsupport.gc import GcLLDescr_boehm,\ GcLLDescr_framework from rpython.jit.backend.llsupport import jitframe @@ -9,7 +9,8 @@ from rpython.jit.metainterp.optimizeopt.util import equaloplists from rpython.jit.codewriter.heaptracker import register_known_gctype from rpython.jit.metainterp.history import JitCellToken, FLOAT -from rpython.rtyper.lltypesystem import lltype +from rpython.rtyper.lltypesystem import lltype, rclass, llmemory +from rpython.jit.backend.x86.arch import WORD class Evaluator(object): def __init__(self, scope): diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py --- a/rpython/jit/backend/test/runner_test.py +++ b/rpython/jit/backend/test/runner_test.py @@ -12,10 +12,9 @@ from rpython.jit.metainterp.typesystem import deref from rpython.jit.codewriter.effectinfo import EffectInfo from rpython.jit.tool.oparser import parse -from rpython.rpython.lltypesystem import lltype, llmemory, rstr, rffi, rclass -from rpython.rpython.ootypesystem import ootype -from rpython.rpython.annlowlevel import llhelper, cast_instance_to_gcref -from rpython.rpython.llinterp import LLException +from rpython.rtyper.lltypesystem import lltype, llmemory, rstr, rffi, rclass +from rpython.rtyper.annlowlevel import llhelper +from rpython.rtyper.llinterp import LLException from rpython.jit.codewriter import heaptracker, longlong from rpython.rlib import longlong2float from rpython.rlib.rarithmetic import intmask, is_valid_int diff --git a/rpython/rtyper/annlowlevel.py b/rpython/rtyper/annlowlevel.py --- a/rpython/rtyper/annlowlevel.py +++ b/rpython/rtyper/annlowlevel.py @@ -3,15 +3,15 @@ """ from rpython.tool.sourcetools import valid_identifier -from rpython.annotation import model as annmodel -from rpython.annotation.policy import AnnotatorPolicy, Sig -from rpython.annotation.specialize import flatten_star_args -from rpython.rpython.lltypesystem import lltype, llmemory -from rpython.rpython.ootypesystem import ootype -from rpython.rpython import extregistry -from rpython.objspace.flow.model import Constant +from rpython.annotator import model as annmodel +from rpython.annotator.policy import AnnotatorPolicy, Sig +from rpython.annotator.specialize import flatten_star_args +from rpython.rtyper.lltypesystem import lltype, llmemory +from rpython.rtyper.ootypesystem import ootype +from rpython.rtyper import extregistry +from rpython.flowspace.model import Constant from rpython.translator.simplify import get_functype -from rpython.rpython.rmodel import warning +from rpython.rtyper.rmodel import warning from rpython.rlib.objectmodel import specialize class KeyComp(object): From noreply at buildbot.pypy.org Sun Jan 20 20:52:32 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 20 Jan 2013 20:52:32 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: more merge fixes Message-ID: <20130120195232.CBD4C1C00F9@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60249:811d6ab9fc7d Date: 2013-01-20 21:52 +0200 http://bitbucket.org/pypy/pypy/changeset/811d6ab9fc7d/ Log: more merge fixes diff --git a/rpython/jit/backend/test/calling_convention_test.py b/rpython/jit/backend/test/calling_convention_test.py --- a/rpython/jit/backend/test/calling_convention_test.py +++ b/rpython/jit/backend/test/calling_convention_test.py @@ -4,8 +4,8 @@ from rpython.jit.metainterp.resoperation import rop from rpython.jit.codewriter.effectinfo import EffectInfo from rpython.jit.tool.oparser import parse -from rpython.rpython.lltypesystem import lltype, llmemory -from rpython.rpython.annlowlevel import llhelper +from rpython.rtyper.lltypesystem import lltype, llmemory +from rpython.rtyper.annlowlevel import llhelper from rpython.jit.codewriter import heaptracker, longlong from rpython.jit.backend.detect_cpu import getcpuclass from rpython.jit.backend.test.runner_test import Runner @@ -289,12 +289,7 @@ def test_call_with_imm_values_bug_constint0(self): -<<<<<<< local cpu = self.cpu -======= - from rpython.rlib.libffi import types - cpu = self.cpu ->>>>>>> other I = lltype.Signed ints = [7, 11, 23, 13, -42, 0, 0, 9] diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -4,9 +4,9 @@ from rpython.jit.metainterp.history import Const, Box, BoxInt, ConstInt from rpython.jit.metainterp.history import AbstractFailDescr, INT, REF, FLOAT from rpython.jit.metainterp.history import JitCellToken -from rpython.rpython.lltypesystem import lltype, rffi, rstr, llmemory -from rpython.rpython.lltypesystem.lloperation import llop -from rpython.rpython.annlowlevel import llhelper, cast_instance_to_gcref +from rpython.rtyper.lltypesystem import lltype, rffi, rstr, llmemory +from rpython.rtyper.lltypesystem.lloperation import llop +from rpython.rtyper.annlowlevel import llhelper, cast_instance_to_gcref from rpython.rlib.jit import AsmInfo from rpython.jit.backend.model import CompiledLoopToken from rpython.jit.backend.x86.regalloc import (RegAlloc, get_ebp_ofs, _get_scale, @@ -2091,7 +2091,7 @@ # used by arglocs for the *next* call, then trouble; for now we # will just push/pop them. xxx - from rpython.rpython.memory.gctransform import asmgcroot + from rpython.rtyper.memory.gctransform import asmgcroot css = self._regalloc.close_stack_struct if css == 0: use_words = (2 + max(asmgcroot.INDEX_OF_EBP, diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -8,8 +8,8 @@ BoxFloat, INT, REF, FLOAT, TargetToken, JitCellToken) from rpython.jit.backend.x86.regloc import * -from rpython.rpython.lltypesystem import lltype, rffi, rstr -from rpython.rpython.annlowlevel import cast_instance_to_gcref +from rpython.rtyper.lltypesystem import lltype, rffi, rstr +from rpython.rtyper.annlowlevel import cast_instance_to_gcref from rpython.rlib.objectmodel import we_are_translated from rpython.rlib import rgc from rpython.jit.backend.llsupport import symbolic diff --git a/rpython/jit/backend/x86/test/test_gc_integration.py b/rpython/jit/backend/x86/test/test_gc_integration.py --- a/rpython/jit/backend/x86/test/test_gc_integration.py +++ b/rpython/jit/backend/x86/test/test_gc_integration.py @@ -319,158 +319,3 @@ s1ref = self.cpu.get_ref_value(self.deadframe, i) s1 = lltype.cast_opaque_ptr(lltype.Ptr(S1), s1ref) assert s1 == getattr(s2, 's%d' % i) -<<<<<<< local -======= - - -class MockShadowStackRootMap(MockGcRootMap): - is_shadow_stack = True - MARKER_FRAME = 88 # this marker follows the frame addr - S1 = lltype.GcStruct('S1') - - def __init__(self): - self.addrs = lltype.malloc(rffi.CArray(lltype.Signed), 20, - flavor='raw') - # root_stack_top - self.addrs[0] = rffi.cast(lltype.Signed, self.addrs) + 3*WORD - # random stuff - self.addrs[1] = 123456 - self.addrs[2] = 654321 - self.check_initial_and_final_state() - self.callshapes = {} - self.should_see = [] - - def check_initial_and_final_state(self): - assert self.addrs[0] == rffi.cast(lltype.Signed, self.addrs) + 3*WORD - assert self.addrs[1] == 123456 - assert self.addrs[2] == 654321 - - def get_root_stack_top_addr(self): - return rffi.cast(lltype.Signed, self.addrs) - - def compress_callshape(self, shape, datablockwrapper): - assert shape[0] == 'shape' - return ['compressed'] + shape[1:] - - def write_callshape(self, mark, force_index): - assert mark[0] == 'compressed' - assert force_index not in self.callshapes - assert force_index == 42 + len(self.callshapes) - self.callshapes[force_index] = mark - - def hook_malloc_slowpath(self): - num_entries = self.addrs[0] - rffi.cast(lltype.Signed, self.addrs) - assert num_entries == 5*WORD # 3 initially, plus 2 by the asm frame - assert self.addrs[1] == 123456 # unchanged - assert self.addrs[2] == 654321 # unchanged - frame_addr = self.addrs[3] # pushed by the asm frame - assert self.addrs[4] == self.MARKER_FRAME # pushed by the asm frame - # - from rpython.jit.backend.x86.arch import FORCE_INDEX_OFS - addr = rffi.cast(rffi.CArrayPtr(lltype.Signed), - frame_addr + FORCE_INDEX_OFS) - force_index = addr[0] - assert force_index == 43 # in this test: the 2nd call_malloc_nursery - # - # The callshapes[43] saved above should list addresses both in the - # COPY_AREA and in the "normal" stack, where all the 16 values p1-p16 - # of test_save_regs_at_correct_place should have been stored. Here - # we replace them with new addresses, to emulate a moving GC. - shape = self.callshapes[force_index] - assert len(shape[1:]) == len(self.should_see) - new_objects = [None] * len(self.should_see) - for ofs in shape[1:]: - assert isinstance(ofs, int) # not a register at all here - addr = rffi.cast(rffi.CArrayPtr(lltype.Signed), frame_addr + ofs) - contains = addr[0] - for j in range(len(self.should_see)): - obj = self.should_see[j] - if contains == rffi.cast(lltype.Signed, obj): - assert new_objects[j] is None # duplicate? - break - else: - assert 0 # the value read from the stack looks random? - new_objects[j] = lltype.malloc(self.S1) - addr[0] = rffi.cast(lltype.Signed, new_objects[j]) - self.should_see[:] = new_objects - - -class TestMallocShadowStack(BaseTestRegalloc): - - def setup_method(self, method): - cpu = CPU(None, None) - cpu.gc_ll_descr = GCDescrFastpathMalloc() - cpu.gc_ll_descr.gcrootmap = MockShadowStackRootMap() - cpu.setup_once() - for i in range(42): - cpu.reserve_some_free_fail_descr_number() - self.cpu = cpu - - def test_save_regs_at_correct_place(self): - cpu = self.cpu - gc_ll_descr = cpu.gc_ll_descr - S1 = gc_ll_descr.gcrootmap.S1 - S2 = lltype.GcStruct('S2', ('s0', lltype.Ptr(S1)), - ('s1', lltype.Ptr(S1)), - ('s2', lltype.Ptr(S1)), - ('s3', lltype.Ptr(S1)), - ('s4', lltype.Ptr(S1)), - ('s5', lltype.Ptr(S1)), - ('s6', lltype.Ptr(S1)), - ('s7', lltype.Ptr(S1)), - ('s8', lltype.Ptr(S1)), - ('s9', lltype.Ptr(S1)), - ('s10', lltype.Ptr(S1)), - ('s11', lltype.Ptr(S1)), - ('s12', lltype.Ptr(S1)), - ('s13', lltype.Ptr(S1)), - ('s14', lltype.Ptr(S1)), - ('s15', lltype.Ptr(S1))) - self.namespace = self.namespace.copy() - for i in range(16): - self.namespace['ds%i' % i] = cpu.fielddescrof(S2, 's%d' % i) - ops = ''' - [i0, p0] - p1 = getfield_gc(p0, descr=ds0) - p2 = getfield_gc(p0, descr=ds1) - p3 = getfield_gc(p0, descr=ds2) - p4 = getfield_gc(p0, descr=ds3) - p5 = getfield_gc(p0, descr=ds4) - p6 = getfield_gc(p0, descr=ds5) - p7 = getfield_gc(p0, descr=ds6) - p8 = getfield_gc(p0, descr=ds7) - p9 = getfield_gc(p0, descr=ds8) - p10 = getfield_gc(p0, descr=ds9) - p11 = getfield_gc(p0, descr=ds10) - p12 = getfield_gc(p0, descr=ds11) - p13 = getfield_gc(p0, descr=ds12) - p14 = getfield_gc(p0, descr=ds13) - p15 = getfield_gc(p0, descr=ds14) - p16 = getfield_gc(p0, descr=ds15) - # - # now all registers are in use - p17 = call_malloc_nursery(40) - p18 = call_malloc_nursery(40) # overflow - # - guard_true(i0) [p1, p2, p3, p4, p5, p6, p7, p8, \ - p9, p10, p11, p12, p13, p14, p15, p16] - ''' - s2 = lltype.malloc(S2) - for i in range(16): - s1 = lltype.malloc(S1) - setattr(s2, 's%d' % i, s1) - gc_ll_descr.gcrootmap.should_see.append(s1) - s2ref = lltype.cast_opaque_ptr(llmemory.GCREF, s2) - # - self.interpret(ops, [0, s2ref]) - gc_ll_descr.check_nothing_in_nursery() - assert gc_ll_descr.calls == [40] - gc_ll_descr.gcrootmap.check_initial_and_final_state() - # check the returned pointers - for i in range(16): - s1ref = self.cpu.get_latest_value_ref(self.deadframe, i) - s1 = lltype.cast_opaque_ptr(lltype.Ptr(S1), s1ref) - for j in range(16): - assert s1 != getattr(s2, 's%d' % j) - assert s1 == gc_ll_descr.gcrootmap.should_see[i] ->>>>>>> other From noreply at buildbot.pypy.org Sun Jan 20 21:25:54 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 20 Jan 2013 21:25:54 +0100 (CET) Subject: [pypy-commit] pypy default: simplify Message-ID: <20130120202554.EB8E61C1306@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r60250:8648b3444808 Date: 2013-01-20 22:25 +0200 http://bitbucket.org/pypy/pypy/changeset/8648b3444808/ Log: simplify diff --git a/rpython/rlib/parsing/test/test_pcre_regtest.py b/rpython/rlib/parsing/test/test_pcre_regtest.py --- a/rpython/rlib/parsing/test/test_pcre_regtest.py +++ b/rpython/rlib/parsing/test/test_pcre_regtest.py @@ -84,8 +84,7 @@ from rpython.rlib.parsing.regexparse import make_runner, unescape import string import re -import os -this_dir = py.path.local(os.path.realpath(os.path.dirname(__file__))) +this_dir = py.path.local(__file__).join('..') #py.test.skip("Still in progress") diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py --- a/rpython/translator/c/genc.py +++ b/rpython/translator/c/genc.py @@ -131,7 +131,7 @@ self.secondary_entrypoints = secondary_entrypoints def get_eci(self): - pypy_include_dir = py.path.local(os.path.realpath(os.path.dirname(__file__))) + pypy_include_dir = py.path.local(__file__).join('..') include_dirs = [pypy_include_dir] return ExternalCompilationInfo(include_dirs=include_dirs) @@ -753,7 +753,7 @@ defines['PYPY_LONGLONG_BIT'] = LONGLONG_BIT def add_extra_files(eci): - srcdir = py.path.local(os.path.realpath(os.path.dirname(__file__))).join('src') + srcdir = py.path.local(__file__).join('..', 'src') files = [ srcdir / 'entrypoint.c', # ifdef PYPY_STANDALONE srcdir / 'allocator.c', # ifdef PYPY_STANDALONE From noreply at buildbot.pypy.org Sun Jan 20 21:26:41 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 20 Jan 2013 21:26:41 +0100 (CET) Subject: [pypy-commit] pypy default: fix Message-ID: <20130120202641.A840A1C1306@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r60251:cd09ce3faa2d Date: 2013-01-20 22:26 +0200 http://bitbucket.org/pypy/pypy/changeset/cd09ce3faa2d/ Log: fix diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py --- a/rpython/translator/platform/posix.py +++ b/rpython/translator/platform/posix.py @@ -5,7 +5,7 @@ from rpython.translator.platform import Platform, log, _run_subprocess import rpython -rpydir = os.path.dirname(rpython.__file__) +rpydir = str(py.path.local(rpython.__file__).join('..')) class BasePosix(Platform): exe_ext = '' diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -7,7 +7,7 @@ from rpython.translator.platform import Platform, posix import rpython -rpydir = os.path.dirname(rpython.__file__) +rpydir = str(py.path.local(rpython.__file__).join('..')) def _get_compiler_type(cc, x64_flag): import subprocess From noreply at buildbot.pypy.org Sun Jan 20 21:44:55 2013 From: noreply at buildbot.pypy.org (mattip) Date: Sun, 20 Jan 2013 21:44:55 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: implement byteswap for longfloat Message-ID: <20130120204455.86D031C0237@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: missing-ndarray-attributes Changeset: r60252:54e9819b3e75 Date: 2013-01-20 22:44 +0200 http://bitbucket.org/pypy/pypy/changeset/54e9819b3e75/ Log: implement byteswap for longfloat diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -14,7 +14,7 @@ from pypy.rpython.lltypesystem import lltype, rffi from pypy.rlib.rstruct.runpack import runpack from pypy.rlib.rstruct.nativefmttable import native_is_bigendian -from pypy.rlib.rstruct.ieee import (float_pack, float_unpack, +from pypy.rlib.rstruct.ieee import (float_pack, float_unpack, pack_float, unpack_float, unpack_float128) from pypy.tool.sourcetools import func_with_new_name from pypy.rlib import jit @@ -1539,6 +1539,12 @@ fval = unpack_float128(s, native_is_bigendian) return self.box(fval) + def byteswap(self, w_v): + value = self.unbox(w_v) + result = StringBuilder(12) + pack_float(result, value, 12, not native_is_bigendian) + return self.box(unpack_float128(result.build(), native_is_bigendian)) + class NonNativeFloat96(Float96): pass @@ -1566,6 +1572,12 @@ fval = unpack_float128(s, native_is_bigendian) return self.box(fval) + def byteswap(self, w_v): + value = self.unbox(w_v) + result = StringBuilder(16) + pack_float(result, value, 16, not native_is_bigendian) + return self.box(unpack_float128(result.build(), native_is_bigendian)) + class NonNativeFloat128(Float128): pass diff --git a/pypy/rlib/rarithmetic.py b/pypy/rlib/rarithmetic.py --- a/pypy/rlib/rarithmetic.py +++ b/pypy/rlib/rarithmetic.py @@ -624,7 +624,7 @@ T = rffi.LONGLONG arg = float2longlong(arg) elif T == lltype.LongFloat: - raise NotImplemented('cannot byteswap LongFloat') + assert False else: # we cannot do arithmetics on small ints arg = widen(arg) diff --git a/pypy/rlib/rstruct/ieee.py b/pypy/rlib/rstruct/ieee.py --- a/pypy/rlib/rstruct/ieee.py +++ b/pypy/rlib/rstruct/ieee.py @@ -184,14 +184,14 @@ sign = r_ulonglong(sign) return ((sign << BITS - 1) | (exp << MANT_DIG - 1)) | mant -def float_pack80(x): - """Convert a Python float x into two 64-bit unsigned integers +def float_pack80(_x): + """Convert a Python float or longfloat x into two 64-bit unsigned integers with 80 bit extended representation.""" MIN_EXP = -16381 MAX_EXP = 16384 MANT_DIG = 64 BITS = 80 - + x = float(_x) #longfloat not really supported sign = rfloat.copysign(1.0, x) < 0.0 if not rfloat.isfinite(x): if rfloat.isinf(x): From noreply at buildbot.pypy.org Sun Jan 20 22:13:53 2013 From: noreply at buildbot.pypy.org (amauryfa) Date: Sun, 20 Jan 2013 22:13:53 +0100 (CET) Subject: [pypy-commit] pypy default: Correctly skip a test when host CPython is a narrow unicode build. Message-ID: <20130120211353.91E521C00F9@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: Changeset: r60253:5eb1d1eb91cb Date: 2013-01-20 22:12 +0100 http://bitbucket.org/pypy/pypy/changeset/5eb1d1eb91cb/ Log: Correctly skip a test when host CPython is a narrow unicode build. diff --git a/pypy/module/unicodedata/test_unicodedata.py b/pypy/module/unicodedata/test_unicodedata.py --- a/pypy/module/unicodedata/test_unicodedata.py +++ b/pypy/module/unicodedata/test_unicodedata.py @@ -1,3 +1,5 @@ +import py +import sys class AppTestUnicodeData: spaceconfig = dict(usemodules=('unicodedata',)) @@ -80,10 +82,10 @@ import unicodedata raises(TypeError, unicodedata.normalize, 'x') + @py.test.mark.skipif("sys.maxunicode < 0x10ffff", + reason="requires a 'wide' python build.") def test_normalize_wide(self): - import sys, unicodedata - if sys.maxunicode < 0x10ffff: - skip("requires a 'wide' python build.") + import unicodedata assert unicodedata.normalize('NFC', u'\U000110a5\U000110ba') == u'\U000110ab' def test_linebreaks(self): From noreply at buildbot.pypy.org Sun Jan 20 22:13:54 2013 From: noreply at buildbot.pypy.org (amauryfa) Date: Sun, 20 Jan 2013 22:13:54 +0100 (CET) Subject: [pypy-commit] pypy default: Move test file into test dir Message-ID: <20130120211354.CF4141C1306@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: Changeset: r60254:c2fd68fccd78 Date: 2013-01-20 22:13 +0100 http://bitbucket.org/pypy/pypy/changeset/c2fd68fccd78/ Log: Move test file into test dir diff --git a/pypy/module/unicodedata/test_unicodedata.py b/pypy/module/unicodedata/test/test_unicodedata.py rename from pypy/module/unicodedata/test_unicodedata.py rename to pypy/module/unicodedata/test/test_unicodedata.py From noreply at buildbot.pypy.org Sun Jan 20 22:44:33 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 20 Jan 2013 22:44:33 +0100 (CET) Subject: [pypy-commit] buildbot default: fix paths Message-ID: <20130120214433.C51401C0705@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r736:101f88ecf10a Date: 2013-01-20 23:44 +0200 http://bitbucket.org/pypy/buildbot/changeset/101f88ecf10a/ Log: fix paths diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py --- a/bot2/pypybuildbot/builds.py +++ b/bot2/pypybuildbot/builds.py @@ -326,7 +326,7 @@ factory.addStep(PytestCmd( description="lib-python test", command=prefix + ["python", "pypy/test_all.py", - "--pypy=pypy/translator/goal/pypy-c", + "--pypy=pypy/goal/pypy-c", "--timeout=3600", "--resultlog=cpython.log", "lib-python"], timeout=4000, @@ -339,16 +339,16 @@ factory.addStep(PytestCmd( description="pypyjit tests", command=prefix + ["python", "pypy/test_all.py", - "--pypy=pypy/translator/goal/pypy-c", + "--pypy=pypy/goal/pypy-c", "--resultlog=pypyjit.log", "pypy/module/pypyjit/test"], logfiles={'pytestLog': 'pypyjit.log'})) # # "new" test_pypy_c if platform == 'win32': - cmd = r'pypy\translator\goal\pypy-c' + cmd = r'pypy\goal\pypy-c' else: - cmd = 'pypy/translator/goal/pypy-c' + cmd = 'pypy/goal/pypy-c' factory.addStep(PytestCmd( description="pypyjit tests", command=prefix + [cmd, "pypy/test_all.py", @@ -479,7 +479,7 @@ # copy pypy-c to the expected location within the pypy source checkout self.addStep(ShellCmd( description="move pypy-c", - command=['cp', '-v', 'pypy-c/bin/pypy', 'build/pypy/translator/goal/pypy-c'], + command=['cp', '-v', 'pypy-c/bin/pypy', 'build/pypy/goal/pypy-c'], workdir='.')) # copy generated and copied header files to build/include self.addStep(ShellCmd( @@ -552,7 +552,7 @@ ) ) if host == 'tannit': - pypy_c_rel = 'build/pypy/translator/goal/pypy-c' + pypy_c_rel = 'build/pypy/goal/pypy-c' self.addStep(ShellCmd( env={'PYTHONPATH': './benchmarks/lib/jinja2'}, description="measure numpy compatibility", @@ -565,7 +565,7 @@ slavesrc="numpy-compat.html", masterdest=WithProperties(resfile), workdir=".")) - pypy_c_rel = "../build/pypy/translator/goal/pypy-c" + pypy_c_rel = "../build/pypy/goal/pypy-c" self.addStep(ShellCmd( # this step needs exclusive access to the CPU locks=[lock.access('exclusive')], From noreply at buildbot.pypy.org Sun Jan 20 22:53:20 2013 From: noreply at buildbot.pypy.org (amauryfa) Date: Sun, 20 Jan 2013 22:53:20 +0100 (CET) Subject: [pypy-commit] pypy py3k: Translation fixes. Message-ID: <20130120215320.DB8651C1305@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r60255:a8477e4eaa94 Date: 2013-01-20 17:50 +0100 http://bitbucket.org/pypy/pypy/changeset/a8477e4eaa94/ Log: Translation fixes. diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -1274,7 +1274,7 @@ return buffer.as_str() def bufferstr0_new_w(self, w_obj): - from pypy.rlib import rstring + from rpython.rlib import rstring result = self.bufferstr_new_w(w_obj) if '\x00' in result: raise OperationError(self.w_TypeError, self.wrap( diff --git a/pypy/module/_posixsubprocess/interp_subprocess.py b/pypy/module/_posixsubprocess/interp_subprocess.py --- a/pypy/module/_posixsubprocess/interp_subprocess.py +++ b/pypy/module/_posixsubprocess/interp_subprocess.py @@ -1,15 +1,13 @@ -from pypy.translator.goal import autopath -from pypy.rpython.lltypesystem import rffi, lltype, llmemory +from rpython.rtyper.lltypesystem import rffi, lltype, llmemory from pypy.module.posix.interp_posix import fsencode_w, run_fork_hooks from pypy.interpreter.gateway import unwrap_spec from pypy.interpreter.error import ( OperationError, exception_from_errno, wrap_oserror) -from pypy.translator.tool.cbuild import ExternalCompilationInfo +from rpython.translator.tool.cbuild import ExternalCompilationInfo import py import os -pypydir = py.path.local(autopath.pypydir) -thisdir = pypydir.join('module', '_posixsubprocess') +thisdir = py.path.local(__file__).dirpath() eci = ExternalCompilationInfo( separate_module_files=[thisdir.join('_posixsubprocess.c')], diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -19,6 +19,7 @@ # XXX Hack to seperate rpython and pypy def addr_as_object(addr, fd, space): + from rpython.rlib import _rsocket_rffi as _c if isinstance(addr, rsocket.INETAddress): return space.newtuple([space.wrap(addr.get_host()), space.wrap(addr.get_port())]) @@ -34,7 +35,7 @@ space.wrap(addr.get_hatype()), space.wrap(addr.get_addr())]) elif rsocket.HAS_AF_UNIX and isinstance(addr, rsocket.UNIXAddress): - path = self.get_path() + path = addr.get_path() if _c.linux and len(path) > 0 and path[0] == '\x00': # Linux abstract namespace return space.wrapbytes(path) @@ -45,7 +46,6 @@ space.wrap(addr.get_groups())]) # If we don't know the address family, don't raise an # exception -- return it as a tuple. - from rpython.rlib import _rsocket_rffi as _c a = addr.lock() family = rffi.cast(lltype.Signed, a.c_sa_family) datalen = addr.addrlen - rsocket.offsetof(_c.sockaddr, 'c_sa_data') @@ -163,7 +163,7 @@ try: fd, addr = self.accept() return space.newtuple([space.wrap(fd), - addr.as_object(addr, fd, space)]) + addr_as_object(addr, fd, space)]) except SocketError, e: raise converted_error(space, e) diff --git a/pypy/module/array/reconstructor.py b/pypy/module/array/reconstructor.py --- a/pypy/module/array/reconstructor.py +++ b/pypy/module/array/reconstructor.py @@ -5,9 +5,9 @@ from pypy.interpreter.gateway import unwrap_spec from pypy.interpreter.error import OperationError from pypy.interpreter.argument import Arguments -from pypy.rlib import runicode, rbigint -from pypy.rlib.rstruct import ieee -from pypy.rpython.lltypesystem import rffi +from rpython.rlib import runicode, rbigint +from rpython.rlib.rstruct import ieee +from rpython.rtyper.lltypesystem import rffi from pypy.module.array import interp_array diff --git a/pypy/tool/pytest/apptest.py b/pypy/tool/pytest/apptest.py --- a/pypy/tool/pytest/apptest.py +++ b/pypy/tool/pytest/apptest.py @@ -11,10 +11,10 @@ from pypy.interpreter.gateway import app2interp_temp from pypy.interpreter.error import OperationError from pypy.interpreter.function import Method -from pypy.tool import runsubprocess +from rpython.tool import runsubprocess from pypy.tool.pytest import appsupport from pypy.tool.pytest.objspace import gettestobjspace -from pypy.tool.udir import udir +from rpython.tool.udir import udir from pypy.conftest import PyPyClassCollector from inspect import getmro From noreply at buildbot.pypy.org Sun Jan 20 22:53:22 2013 From: noreply at buildbot.pypy.org (amauryfa) Date: Sun, 20 Jan 2013 22:53:22 +0100 (CET) Subject: [pypy-commit] pypy py3k: hg merge default Message-ID: <20130120215322.2D0DD1C1305@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r60256:4b63836b7e97 Date: 2013-01-20 20:27 +0100 http://bitbucket.org/pypy/pypy/changeset/4b63836b7e97/ Log: hg merge default diff --git a/pypy/module/unicodedata/interp_ucd.py b/pypy/module/unicodedata/interp_ucd.py --- a/pypy/module/unicodedata/interp_ucd.py +++ b/pypy/module/unicodedata/interp_ucd.py @@ -9,6 +9,7 @@ from rpython.rlib.objectmodel import we_are_translated from rpython.rlib.runicode import MAXUNICODE from rpython.rlib.unicodedata import unicodedb_5_2_0, unicodedb_3_2_0 +from rpython.rlib.runicode import code_to_unichr, ORD import sys @@ -30,25 +31,6 @@ # The functions below are subtly different from the ones in runicode.py. # When PyPy implements Python 3 they should be merged. -def UNICHR(c): - if c <= sys.maxunicode and c <= MAXUNICODE: - return unichr(c) - else: - c -= 0x10000 - return (unichr(0xD800 + (c >> 10)) + - unichr(0xDC00 + (c & 0x03FF))) - -def ORD(u): - assert isinstance(u, unicode) - if len(u) == 1: - return ord(u[0]) - elif len(u) == 2: - ch1 = ord(u[0]) - ch2 = ord(u[1]) - if 0xD800 <= ch1 <= 0xDBFF and 0xDC00 <= ch2 <= 0xDFFF: - return (((ch1 - 0xD800) << 10) | (ch2 - 0xDC00)) + 0x10000 - raise ValueError - if MAXUNICODE > 0xFFFF: # Target is wide build def unichr_to_code_w(space, w_unichr): @@ -69,12 +51,6 @@ 'need a single Unicode character as parameter')) return space.int_w(space.ord(w_unichr)) - def code_to_unichr(code): - if not we_are_translated() and sys.maxunicode == 0xFFFF: - # Host CPython is narrow build, generate surrogates - return UNICHR(code) - else: - return unichr(code) else: # Target is narrow build def unichr_to_code_w(space, w_unichr): @@ -97,10 +73,6 @@ raise OperationError(space.w_TypeError, space.wrap( 'need a single Unicode character as parameter')) - def code_to_unichr(code): - # generate surrogates for large codes - return UNICHR(code) - class UCD(Wrappable): def __init__(self, unicodedb): diff --git a/pypy/module/unicodedata/test_unicodedata.py b/pypy/module/unicodedata/test_unicodedata.py new file mode 100644 --- /dev/null +++ b/pypy/module/unicodedata/test_unicodedata.py @@ -0,0 +1,103 @@ + +class AppTestUnicodeData: + spaceconfig = dict(usemodules=('unicodedata',)) + + def test_hangul_syllables(self): + import unicodedata + # Test all leading, vowel and trailing jamo + # but not every combination of them. + for code, name in ((0xAC00, 'HANGUL SYLLABLE GA'), + (0xAE69, 'HANGUL SYLLABLE GGAEG'), + (0xB0D2, 'HANGUL SYLLABLE NYAGG'), + (0xB33B, 'HANGUL SYLLABLE DYAEGS'), + (0xB5A4, 'HANGUL SYLLABLE DDEON'), + (0xB80D, 'HANGUL SYLLABLE RENJ'), + (0xBA76, 'HANGUL SYLLABLE MYEONH'), + (0xBCDF, 'HANGUL SYLLABLE BYED'), + (0xBF48, 'HANGUL SYLLABLE BBOL'), + (0xC1B1, 'HANGUL SYLLABLE SWALG'), + (0xC41A, 'HANGUL SYLLABLE SSWAELM'), + (0xC683, 'HANGUL SYLLABLE OELB'), + (0xC8EC, 'HANGUL SYLLABLE JYOLS'), + (0xCB55, 'HANGUL SYLLABLE JJULT'), + (0xCDBE, 'HANGUL SYLLABLE CWEOLP'), + (0xD027, 'HANGUL SYLLABLE KWELH'), + (0xD290, 'HANGUL SYLLABLE TWIM'), + (0xD4F9, 'HANGUL SYLLABLE PYUB'), + (0xD762, 'HANGUL SYLLABLE HEUBS'), + (0xAE27, 'HANGUL SYLLABLE GYIS'), + (0xB090, 'HANGUL SYLLABLE GGISS'), + (0xB0AD, 'HANGUL SYLLABLE NANG'), + (0xB316, 'HANGUL SYLLABLE DAEJ'), + (0xB57F, 'HANGUL SYLLABLE DDYAC'), + (0xB7E8, 'HANGUL SYLLABLE RYAEK'), + (0xBA51, 'HANGUL SYLLABLE MEOT'), + (0xBCBA, 'HANGUL SYLLABLE BEP'), + (0xBF23, 'HANGUL SYLLABLE BBYEOH'), + (0xD7A3, 'HANGUL SYLLABLE HIH')): + assert unicodedata.name(chr(code)) == name + assert unicodedata.lookup(name) == chr(code) + # Test outside the range + raises(ValueError, unicodedata.name, chr(0xAC00 - 1)) + raises(ValueError, unicodedata.name, chr(0xD7A3 + 1)) + + def test_cjk(self): + import sys + import unicodedata + cases = ((0x3400, 0x4DB5), + (0x4E00, 0x9FA5)) + if unicodedata.unidata_version >= "5": # don't know the exact limit + cases = ((0x3400, 0x4DB5), + (0x4E00, 0x9FCB), + (0x20000, 0x2A6D6), + (0x2A700, 0x2B734)) + elif unicodedata.unidata_version >= "4.1": + cases = ((0x3400, 0x4DB5), + (0x4E00, 0x9FBB), + (0x20000, 0x2A6D6)) + for first, last in cases: + # Test at and inside the boundary + for i in (first, first + 1, last - 1, last): + charname = 'CJK UNIFIED IDEOGRAPH-%X'%i + char = chr(i) + assert unicodedata.name(char) == charname + assert unicodedata.lookup(charname) == char + # Test outside the boundary + for i in first - 1, last + 1: + charname = 'CJK UNIFIED IDEOGRAPH-%X'%i + char = chr(i) + try: + unicodedata.name(char) + except ValueError as e: + assert e.message == 'no such name' + raises(KeyError, unicodedata.lookup, charname) + + def test_bug_1704793(self): # from CPython + import unicodedata + assert unicodedata.lookup("GOTHIC LETTER FAIHU") == '\U00010346' + + def test_normalize(self): + import unicodedata + raises(TypeError, unicodedata.normalize, 'x') + + def test_normalize_wide(self): + import sys, unicodedata + if sys.maxunicode < 0x10ffff: + skip("requires a 'wide' python build.") + assert unicodedata.normalize('NFC', '\U000110a5\U000110ba') == '\U000110ab' + + def test_linebreaks(self): + linebreaks = (0x0a, 0x0b, 0x0c, 0x0d, 0x85, + 0x1c, 0x1d, 0x1e, 0x2028, 0x2029) + for i in linebreaks: + for j in range(-2, 3): + lines = (chr(i + j) + 'A').splitlines() + if i + j in linebreaks: + assert len(lines) == 2 + else: + assert len(lines) == 1 + + def test_mirrored(self): + import unicodedata + # For no reason, unicodedata.mirrored() returns an int, not a bool + assert repr(unicodedata.mirrored(' ')) == '0' diff --git a/rpython/bin/rpython b/rpython/bin/rpython --- a/rpython/bin/rpython +++ b/rpython/bin/rpython @@ -7,7 +7,8 @@ run with --help for more information """ -import sys +import sys, os +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) from rpython.translator.goal.translate import main # no implicit targets diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py --- a/rpython/rlib/runicode.py +++ b/rpython/rlib/runicode.py @@ -1,15 +1,16 @@ import sys -from rpython.rlib.bitmanipulation import splitter -from rpython.rtyper.lltypesystem import lltype, rffi -from rpython.rlib.objectmodel import we_are_translated, specialize, enforceargs +from rpython.rlib.objectmodel import specialize, we_are_translated from rpython.rlib.rstring import StringBuilder, UnicodeBuilder from rpython.rlib.rarithmetic import r_uint, intmask from rpython.rlib.unicodedata import unicodedb +from rpython.rtyper.lltypesystem import lltype, rffi + if rffi.sizeof(lltype.UniChar) == 4: MAXUNICODE = 0x10ffff else: MAXUNICODE = 0xffff + BYTEORDER = sys.byteorder if MAXUNICODE > sys.maxunicode: @@ -45,6 +46,27 @@ UNICHR = unichr ORD = ord +if MAXUNICODE > 0xFFFF: + def code_to_unichr(code): + if not we_are_translated() and sys.maxunicode == 0xFFFF: + # Host CPython is narrow build, generate surrogates + return UNICHR(code) + else: + return unichr(code) +else: + def code_to_unichr(code): + # generate surrogates for large codes + return UNICHR(code) + +def _STORECHAR(result, CH, byteorder): + hi = chr(((CH) >> 8) & 0xff) + lo = chr((CH) & 0xff) + if byteorder == 'little': + result.append(lo) + result.append(hi) + else: + result.append(hi) + result.append(lo) def default_unicode_error_decode(errors, encoding, msg, s, startingpos, endingpos): @@ -446,16 +468,6 @@ result.append(r) return result.build(), pos, bo -def _STORECHAR(result, CH, byteorder): - hi = chr(((CH) >> 8) & 0xff) - lo = chr((CH) & 0xff) - if byteorder == 'little': - result.append(lo) - result.append(hi) - else: - result.append(hi) - result.append(lo) - def unicode_encode_utf_16_helper(s, size, errors, errorhandler=None, byteorder='little'): diff --git a/pypy/module/unicodedata/test_interp_ucd.py b/rpython/rlib/unicodedata/test/test_ucd.py rename from pypy/module/unicodedata/test_interp_ucd.py rename to rpython/rlib/unicodedata/test/test_ucd.py --- a/pypy/module/unicodedata/test_interp_ucd.py +++ b/rpython/rlib/unicodedata/test/test_ucd.py @@ -1,6 +1,6 @@ from rpython.rtyper.test.tool import BaseRtypingTest, LLRtypeMixin from rpython.rlib.unicodedata import unicodedb_5_2_0 -from pypy.module.unicodedata.interp_ucd import code_to_unichr +from rpython.rlib.unicodedata.ucd import code_to_unichr class TestTranslated(BaseRtypingTest, LLRtypeMixin): diff --git a/rpython/rlib/unicodedata/test/test_unicodedata.py b/rpython/rlib/unicodedata/test/test_unicodedata.py --- a/rpython/rlib/unicodedata/test/test_unicodedata.py +++ b/rpython/rlib/unicodedata/test/test_unicodedata.py @@ -1,109 +1,6 @@ import py from rpython.rlib.unicodedata import unicodedb_3_2_0, unicodedb_5_2_0 -class AppTestUnicodeData: - spaceconfig = dict(usemodules=('unicodedata',)) - - def test_hangul_syllables(self): - import unicodedata - # Test all leading, vowel and trailing jamo - # but not every combination of them. - for code, name in ((0xAC00, 'HANGUL SYLLABLE GA'), - (0xAE69, 'HANGUL SYLLABLE GGAEG'), - (0xB0D2, 'HANGUL SYLLABLE NYAGG'), - (0xB33B, 'HANGUL SYLLABLE DYAEGS'), - (0xB5A4, 'HANGUL SYLLABLE DDEON'), - (0xB80D, 'HANGUL SYLLABLE RENJ'), - (0xBA76, 'HANGUL SYLLABLE MYEONH'), - (0xBCDF, 'HANGUL SYLLABLE BYED'), - (0xBF48, 'HANGUL SYLLABLE BBOL'), - (0xC1B1, 'HANGUL SYLLABLE SWALG'), - (0xC41A, 'HANGUL SYLLABLE SSWAELM'), - (0xC683, 'HANGUL SYLLABLE OELB'), - (0xC8EC, 'HANGUL SYLLABLE JYOLS'), - (0xCB55, 'HANGUL SYLLABLE JJULT'), - (0xCDBE, 'HANGUL SYLLABLE CWEOLP'), - (0xD027, 'HANGUL SYLLABLE KWELH'), - (0xD290, 'HANGUL SYLLABLE TWIM'), - (0xD4F9, 'HANGUL SYLLABLE PYUB'), - (0xD762, 'HANGUL SYLLABLE HEUBS'), - (0xAE27, 'HANGUL SYLLABLE GYIS'), - (0xB090, 'HANGUL SYLLABLE GGISS'), - (0xB0AD, 'HANGUL SYLLABLE NANG'), - (0xB316, 'HANGUL SYLLABLE DAEJ'), - (0xB57F, 'HANGUL SYLLABLE DDYAC'), - (0xB7E8, 'HANGUL SYLLABLE RYAEK'), - (0xBA51, 'HANGUL SYLLABLE MEOT'), - (0xBCBA, 'HANGUL SYLLABLE BEP'), - (0xBF23, 'HANGUL SYLLABLE BBYEOH'), - (0xD7A3, 'HANGUL SYLLABLE HIH')): - assert unicodedata.name(chr(code)) == name - assert unicodedata.lookup(name) == chr(code) - # Test outside the range - py.test.raises(ValueError, unicodedata.name, chr(0xAC00 - 1)) - py.test.raises(ValueError, unicodedata.name, chr(0xD7A3 + 1)) - - def test_cjk(self): - import sys - import unicodedata - cases = ((0x3400, 0x4DB5), - (0x4E00, 0x9FA5)) - if unicodedata.unidata_version >= "5": # don't know the exact limit - cases = ((0x3400, 0x4DB5), - (0x4E00, 0x9FCB), - (0x20000, 0x2A6D6), - (0x2A700, 0x2B734)) - elif unicodedata.unidata_version >= "4.1": - cases = ((0x3400, 0x4DB5), - (0x4E00, 0x9FBB), - (0x20000, 0x2A6D6)) - for first, last in cases: - # Test at and inside the boundary - for i in (first, first + 1, last - 1, last): - charname = 'CJK UNIFIED IDEOGRAPH-%X'%i - char = chr(i) - assert unicodedata.name(char) == charname - assert unicodedata.lookup(charname) == char - # Test outside the boundary - for i in first - 1, last + 1: - charname = 'CJK UNIFIED IDEOGRAPH-%X'%i - char = chr(i) - try: - unicodedata.name(char) - except ValueError as e: - assert e.message == 'no such name' - py.test.raises(KeyError, unicodedata.lookup, charname) - - def test_bug_1704793(self): # from CPython - import unicodedata - assert unicodedata.lookup("GOTHIC LETTER FAIHU") == '\U00010346' - - def test_normalize(self): - import unicodedata - py.test.raises(TypeError, unicodedata.normalize, 'x') - - def test_normalize_wide(self): - import sys, unicodedata - if sys.maxunicode < 0x10ffff: - skip("requires a 'wide' python build.") - assert unicodedata.normalize('NFC', '\U000110a5\U000110ba') == '\U000110ab' - - def test_linebreaks(self): - linebreaks = (0x0a, 0x0b, 0x0c, 0x0d, 0x85, - 0x1c, 0x1d, 0x1e, 0x2028, 0x2029) - for i in linebreaks: - for j in range(-2, 3): - lines = (chr(i + j) + 'A').splitlines() - if i + j in linebreaks: - assert len(lines) == 2 - else: - assert len(lines) == 1 - - def test_mirrored(self): - import unicodedata - # For no reason, unicodedata.mirrored() returns an int, not a bool - assert repr(unicodedata.mirrored(' ')) == '0' - class TestUnicodeData(object): def setup_class(cls): import random, unicodedata From noreply at buildbot.pypy.org Mon Jan 21 00:32:05 2013 From: noreply at buildbot.pypy.org (antocuni) Date: Mon, 21 Jan 2013 00:32:05 +0100 (CET) Subject: [pypy-commit] pypy default: expose the .strides attribute at applevel (read-only for now) Message-ID: <20130120233205.455631C13B9@cobra.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: Changeset: r60257:54f55ff51273 Date: 2013-01-20 16:16 +0100 http://bitbucket.org/pypy/pypy/changeset/54f55ff51273/ Log: expose the .strides attribute at applevel (read-only for now) diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -40,6 +40,10 @@ self.implementation = self.implementation.set_shape(space, get_shape_from_iterable(space, self.get_size(), w_new_shape)) + def descr_get_strides(self, space): + strides = self.implementation.get_strides() + return space.newtuple([space.wrap(i) for i in strides]) + def get_dtype(self): return self.implementation.dtype @@ -645,6 +649,7 @@ dtype = GetSetProperty(W_NDimArray.descr_get_dtype), shape = GetSetProperty(W_NDimArray.descr_get_shape, W_NDimArray.descr_set_shape), + strides = GetSetProperty(W_NDimArray.descr_get_strides), ndim = GetSetProperty(W_NDimArray.descr_get_ndim), size = GetSetProperty(W_NDimArray.descr_get_size), itemsize = GetSetProperty(W_NDimArray.descr_get_itemsize), diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -684,6 +684,12 @@ assert a.reshape([1]).shape == (1,) raises(ValueError, "a.reshape(3)") + def test_strides(self): + from _numpypy import array + a = array([[1.0, 2.0], + [3.0, 4.0]]) + assert a.strides == (16, 8) + def test_add(self): from _numpypy import array a = array(range(5)) From noreply at buildbot.pypy.org Mon Jan 21 00:32:06 2013 From: noreply at buildbot.pypy.org (antocuni) Date: Mon, 21 Jan 2013 00:32:06 +0100 (CET) Subject: [pypy-commit] pypy default: implement .strides for scalars Message-ID: <20130120233206.9CD781C13B9@cobra.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: Changeset: r60258:c08bafe19961 Date: 2013-01-20 16:21 +0100 http://bitbucket.org/pypy/pypy/changeset/c08bafe19961/ Log: implement .strides for scalars diff --git a/pypy/module/micronumpy/arrayimpl/scalar.py b/pypy/module/micronumpy/arrayimpl/scalar.py --- a/pypy/module/micronumpy/arrayimpl/scalar.py +++ b/pypy/module/micronumpy/arrayimpl/scalar.py @@ -34,6 +34,9 @@ def get_shape(self): return [] + def get_strides(self): + return () + def create_iter(self, shape=None): return ScalarIterator(self.value) diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -689,6 +689,12 @@ a = array([[1.0, 2.0], [3.0, 4.0]]) assert a.strides == (16, 8) + assert a[1:].strides == (16, 8) + + def test_strides_scalar(self): + from _numpypy import array + a = array(42) + assert a.strides == () def test_add(self): from _numpypy import array From noreply at buildbot.pypy.org Mon Jan 21 00:32:07 2013 From: noreply at buildbot.pypy.org (antocuni) Date: Mon, 21 Jan 2013 00:32:07 +0100 (CET) Subject: [pypy-commit] pypy default: rpython fix Message-ID: <20130120233207.CB49F1C13B9@cobra.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: Changeset: r60259:84d97d39a964 Date: 2013-01-20 23:10 +0100 http://bitbucket.org/pypy/pypy/changeset/84d97d39a964/ Log: rpython fix diff --git a/pypy/module/micronumpy/arrayimpl/scalar.py b/pypy/module/micronumpy/arrayimpl/scalar.py --- a/pypy/module/micronumpy/arrayimpl/scalar.py +++ b/pypy/module/micronumpy/arrayimpl/scalar.py @@ -35,7 +35,7 @@ return [] def get_strides(self): - return () + return [] def create_iter(self, shape=None): return ScalarIterator(self.value) From noreply at buildbot.pypy.org Mon Jan 21 00:32:36 2013 From: noreply at buildbot.pypy.org (antocuni) Date: Mon, 21 Jan 2013 00:32:36 +0100 (CET) Subject: [pypy-commit] pypy default: merge Message-ID: <20130120233236.25D271C13B9@cobra.cs.uni-duesseldorf.de> Author: Antonio Cuni Branch: Changeset: r60260:68d5bc1decb5 Date: 2013-01-21 00:35 +0100 http://bitbucket.org/pypy/pypy/changeset/68d5bc1decb5/ Log: merge diff too long, truncating to 2000 out of 728245 lines diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -85,3 +85,4 @@ ^compiled ^.git/ ^release/ +^rpython/_cache$ diff --git a/dotviewer/test/test_translator.py b/dotviewer/test/test_translator.py --- a/dotviewer/test/test_translator.py +++ b/dotviewer/test/test_translator.py @@ -21,7 +21,7 @@ def test_annotated(): - from pypy.translator.interactive import Translation + from rpython.translator.interactive import Translation t = Translation(is_prime) t.annotate([int]) t.viewcg() diff --git a/lib_pypy/ctypes_config_cache/autopath.py b/lib_pypy/ctypes_config_cache/autopath.py deleted file mode 100644 --- a/lib_pypy/ctypes_config_cache/autopath.py +++ /dev/null @@ -1,131 +0,0 @@ -""" -self cloning, automatic path configuration - -copy this into any subdirectory of pypy from which scripts need -to be run, typically all of the test subdirs. -The idea is that any such script simply issues - - import autopath - -and this will make sure that the parent directory containing "pypy" -is in sys.path. - -If you modify the master "autopath.py" version (in pypy/tool/autopath.py) -you can directly run it which will copy itself on all autopath.py files -it finds under the pypy root directory. - -This module always provides these attributes: - - pypydir pypy root directory path - this_dir directory where this autopath.py resides - -""" - -def __dirinfo(part): - """ return (partdir, this_dir) and insert parent of partdir - into sys.path. If the parent directories don't have the part - an EnvironmentError is raised.""" - - import sys, os - try: - head = this_dir = os.path.realpath(os.path.dirname(__file__)) - except NameError: - head = this_dir = os.path.realpath(os.path.dirname(sys.argv[0])) - - error = None - while head: - partdir = head - head, tail = os.path.split(head) - if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') - if not os.path.exists(checkfile): - error = "Cannot find %r" % (os.path.normpath(checkfile),) - break - else: - error = "Cannot find the parent directory %r of the path %r" % ( - partdir, this_dir) - if not error: - # check for bogus end-of-line style (e.g. files checked out on - # Windows and moved to Unix) - f = open(__file__.replace('.pyc', '.py'), 'r') - data = f.read() - f.close() - if data.endswith('\r\n') or data.endswith('\r'): - error = ("Bad end-of-line style in the .py files. Typically " - "caused by a zip file or a checkout done on Windows and " - "moved to Unix or vice-versa.") - if error: - raise EnvironmentError("Invalid source tree - bogus checkout! " + - error) - - pypy_root = os.path.join(head, '') - try: - sys.path.remove(head) - except ValueError: - pass - sys.path.insert(0, head) - - munged = {} - for name, mod in sys.modules.items(): - if '.' in name: - continue - fn = getattr(mod, '__file__', None) - if not isinstance(fn, str): - continue - newname = os.path.splitext(os.path.basename(fn))[0] - if not newname.startswith(part + '.'): - continue - path = os.path.join(os.path.dirname(os.path.realpath(fn)), '') - if path.startswith(pypy_root) and newname != part: - modpaths = os.path.normpath(path[len(pypy_root):]).split(os.sep) - if newname != '__init__': - modpaths.append(newname) - modpath = '.'.join(modpaths) - if modpath not in sys.modules: - munged[modpath] = mod - - for name, mod in munged.iteritems(): - if name not in sys.modules: - sys.modules[name] = mod - if '.' in name: - prename = name[:name.rfind('.')] - postname = name[len(prename)+1:] - if prename not in sys.modules: - __import__(prename) - if not hasattr(sys.modules[prename], postname): - setattr(sys.modules[prename], postname, mod) - - return partdir, this_dir - -def __clone(): - """ clone master version of autopath.py into all subdirs """ - from os.path import join, walk - if not this_dir.endswith(join('pypy','tool')): - raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) - - - def sync_walker(arg, dirname, fnames): - if _myname in fnames: - fn = join(dirname, _myname) - f = open(fn, 'rwb+') - try: - if f.read() == arg: - print "checkok", fn - else: - print "syncing", fn - f = open(fn, 'w') - f.write(arg) - finally: - f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) - -_myname = 'autopath.py' - -# set guaranteed attributes - -pypydir, this_dir = __dirinfo('pypy') - -if __name__ == '__main__': - __clone() diff --git a/lib_pypy/ctypes_config_cache/dumpcache.py b/lib_pypy/ctypes_config_cache/dumpcache.py --- a/lib_pypy/ctypes_config_cache/dumpcache.py +++ b/lib_pypy/ctypes_config_cache/dumpcache.py @@ -1,6 +1,6 @@ import os from ctypes_configure import dumpcache -from pypy.jit.backend import detect_cpu +from rpython.jit.backend import detect_cpu def dumpcache2(basename, config): model = detect_cpu.autodetect_main_model_and_size() @@ -14,7 +14,7 @@ try: from __pypy__ import cpumodel except ImportError: - from pypy.jit.backend import detect_cpu + from rpython.jit.backend import detect_cpu cpumodel = detect_cpu.autodetect_main_model_and_size() # XXX relative import, should be removed together with # XXX the relative imports done e.g. by lib_pypy/pypy_test/test_hashlib diff --git a/lib_pypy/ctypes_config_cache/rebuild.py b/lib_pypy/ctypes_config_cache/rebuild.py --- a/lib_pypy/ctypes_config_cache/rebuild.py +++ b/lib_pypy/ctypes_config_cache/rebuild.py @@ -1,21 +1,15 @@ #! /usr/bin/env python # Run this script to rebuild all caches from the *.ctc.py files. -# hack: we cannot directly import autopath, as we are outside the pypy -# package. However, we pretend to be inside pypy/tool and manually run it, to -# get the correct path -import os.path -this_dir = os.path.dirname(__file__) -autopath_py = os.path.join(this_dir, '../../pypy/tool/autopath.py') -autopath_py = os.path.abspath(autopath_py) -execfile(autopath_py, dict(__name__='autopath', __file__=autopath_py)) +import os, sys -import os, sys +sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..'))) + import py _dirpath = os.path.dirname(__file__) or os.curdir -from pypy.tool.ansi_print import ansi_log +from rpython.tool.ansi_print import ansi_log log = py.log.Producer("ctypes_config_cache") py.log.setconsumer("ctypes_config_cache", ansi_log) @@ -31,7 +25,7 @@ sys.path[:] = path def try_rebuild(): - from pypy.jit.backend import detect_cpu + from rpython.jit.backend import detect_cpu model = detect_cpu.autodetect_main_model_and_size() # remove the files '_*_model_.py' left = {} diff --git a/lib_pypy/ctypes_config_cache/test/test_cache.py b/lib_pypy/ctypes_config_cache/test/test_cache.py --- a/lib_pypy/ctypes_config_cache/test/test_cache.py +++ b/lib_pypy/ctypes_config_cache/test/test_cache.py @@ -1,6 +1,6 @@ import py import sys, os -from pypy.tool.udir import udir +from rpython.tool.udir import udir dirpath = py.path.local(__file__).dirpath().dirpath() diff --git a/lib_pypy/pypy_test/test_dbm_extra.py b/lib_pypy/pypy_test/test_dbm_extra.py --- a/lib_pypy/pypy_test/test_dbm_extra.py +++ b/lib_pypy/pypy_test/test_dbm_extra.py @@ -1,6 +1,6 @@ from __future__ import absolute_import import py -from pypy.tool.udir import udir +from rpython.tool.udir import udir try: from lib_pypy import dbm except ImportError, e: diff --git a/lib_pypy/pypy_test/test_marshal_extra.py b/lib_pypy/pypy_test/test_marshal_extra.py --- a/lib_pypy/pypy_test/test_marshal_extra.py +++ b/lib_pypy/pypy_test/test_marshal_extra.py @@ -4,7 +4,7 @@ import marshal as cpy_marshal from lib_pypy import _marshal as marshal -from pypy.tool.udir import udir +from rpython.tool.udir import udir hello = "he" hello += "llo" diff --git a/pypy/annotation/annrpython.py b/pypy/annotation/annrpython.py deleted file mode 100644 --- a/pypy/annotation/annrpython.py +++ /dev/null @@ -1,676 +0,0 @@ -from __future__ import absolute_import - -import types -from pypy.tool.ansi_print import ansi_log -from pypy.tool.pairtype import pair -from pypy.tool.error import (format_blocked_annotation_error, - AnnotatorError, gather_error, ErrorWrapper) -from pypy.objspace.flow.model import (Variable, Constant, FunctionGraph, - c_last_exception, checkgraph) -from pypy.translator import simplify, transform -from pypy.annotation import model as annmodel, signature, unaryop, binaryop -from pypy.annotation.bookkeeper import Bookkeeper -import py -log = py.log.Producer("annrpython") -py.log.setconsumer("annrpython", ansi_log) - - -FAIL = object() - -class RPythonAnnotator(object): - """Block annotator for RPython. - See description in doc/translation.txt.""" - - def __init__(self, translator=None, policy=None, bookkeeper=None): - import pypy.rpython.ootypesystem.ooregistry # has side effects - import pypy.rpython.extfuncregistry # has side effects - import pypy.rlib.nonconst # has side effects - - if translator is None: - # interface for tests - from pypy.translator.translator import TranslationContext - translator = TranslationContext() - translator.annotator = self - self.translator = translator - self.pendingblocks = {} # map {block: graph-containing-it} - self.bindings = {} # map Variables to SomeValues - self.annotated = {} # set of blocks already seen - self.added_blocks = None # see processblock() below - self.links_followed = {} # set of links that have ever been followed - self.notify = {} # {block: {positions-to-reflow-from-when-done}} - self.fixed_graphs = {} # set of graphs not to annotate again - self.blocked_blocks = {} # set of {blocked_block: (graph, index)} - # --- the following information is recorded for debugging --- - self.blocked_graphs = {} # set of graphs that have blocked blocks - # --- end of debugging information --- - self.frozen = False - if policy is None: - from pypy.annotation.policy import AnnotatorPolicy - self.policy = AnnotatorPolicy() - else: - self.policy = policy - if bookkeeper is None: - bookkeeper = Bookkeeper(self) - self.bookkeeper = bookkeeper - - def __getstate__(self): - attrs = """translator pendingblocks bindings annotated links_followed - notify bookkeeper frozen policy added_blocks""".split() - ret = self.__dict__.copy() - for key, value in ret.items(): - if key not in attrs: - assert type(value) is dict, ( - "%r is not dict. please update %s.__getstate__" % - (key, self.__class__.__name__)) - ret[key] = {} - return ret - - #___ convenience high-level interface __________________ - - def build_types(self, function, input_arg_types, complete_now=True, - main_entry_point=False): - """Recursively build annotations about the specific entry point.""" - assert isinstance(function, types.FunctionType), "fix that!" - - from pypy.annotation.policy import AnnotatorPolicy - policy = AnnotatorPolicy() - # make input arguments and set their type - args_s = [self.typeannotation(t) for t in input_arg_types] - - # XXX hack - annmodel.TLS.check_str_without_nul = ( - self.translator.config.translation.check_str_without_nul) - - flowgraph, inputcells = self.get_call_parameters(function, args_s, policy) - if not isinstance(flowgraph, FunctionGraph): - assert isinstance(flowgraph, annmodel.SomeObject) - return flowgraph - - if main_entry_point: - self.translator.entry_point_graph = flowgraph - return self.build_graph_types(flowgraph, inputcells, complete_now=complete_now) - - def get_call_parameters(self, function, args_s, policy): - desc = self.bookkeeper.getdesc(function) - args = self.bookkeeper.build_args("simple_call", args_s[:]) - result = [] - def schedule(graph, inputcells): - result.append((graph, inputcells)) - return annmodel.s_ImpossibleValue - - prevpolicy = self.policy - self.policy = policy - self.bookkeeper.enter(None) - try: - desc.pycall(schedule, args, annmodel.s_ImpossibleValue) - finally: - self.bookkeeper.leave() - self.policy = prevpolicy - [(graph, inputcells)] = result - return graph, inputcells - - def annotate_helper(self, function, args_s, policy=None): - if policy is None: - from pypy.annotation.policy import AnnotatorPolicy - policy = AnnotatorPolicy() - graph, inputcells = self.get_call_parameters(function, args_s, policy) - self.build_graph_types(graph, inputcells, complete_now=False) - self.complete_helpers(policy) - return graph - - def complete_helpers(self, policy): - saved = self.policy, self.added_blocks - self.policy = policy - try: - self.added_blocks = {} - self.complete() - # invoke annotation simplifications for the new blocks - self.simplify(block_subset=self.added_blocks) - finally: - self.policy, self.added_blocks = saved - - def build_graph_types(self, flowgraph, inputcells, complete_now=True): - checkgraph(flowgraph) - - nbarg = len(flowgraph.getargs()) - if len(inputcells) != nbarg: - raise TypeError("%s expects %d args, got %d" %( - flowgraph, nbarg, len(inputcells))) - - # register the entry point - self.addpendinggraph(flowgraph, inputcells) - # recursively proceed until no more pending block is left - if complete_now: - self.complete() - return self.binding(flowgraph.getreturnvar(), None) - - def gettype(self, variable): - """Return the known type of a control flow graph variable, - defaulting to 'object'.""" - if isinstance(variable, Constant): - return type(variable.value) - elif isinstance(variable, Variable): - cell = self.bindings.get(variable) - if cell: - return cell.knowntype - else: - return object - else: - raise TypeError, ("Variable or Constant instance expected, " - "got %r" % (variable,)) - - def getuserclassdefinitions(self): - """Return a list of ClassDefs.""" - return self.bookkeeper.classdefs - - #___ medium-level interface ____________________________ - - def addpendinggraph(self, flowgraph, inputcells): - self.addpendingblock(flowgraph, flowgraph.startblock, inputcells) - - def addpendingblock(self, graph, block, cells): - """Register an entry point into block with the given input cells.""" - if graph in self.fixed_graphs: - # special case for annotating/rtyping in several phases: calling - # a graph that has already been rtyped. Safety-check the new - # annotations that are passed in, and don't annotate the old - # graph -- it's already low-level operations! - for a, s_newarg in zip(graph.getargs(), cells): - s_oldarg = self.binding(a) - assert annmodel.unionof(s_oldarg, s_newarg) == s_oldarg - else: - assert not self.frozen - for a in cells: - assert isinstance(a, annmodel.SomeObject) - if block not in self.annotated: - self.bindinputargs(graph, block, cells) - else: - self.mergeinputargs(graph, block, cells) - if not self.annotated[block]: - self.pendingblocks[block] = graph - - def complete(self): - """Process pending blocks until none is left.""" - while True: - while self.pendingblocks: - block, graph = self.pendingblocks.popitem() - self.processblock(graph, block) - self.policy.no_more_blocks_to_annotate(self) - if not self.pendingblocks: - break # finished - # make sure that the return variables of all graphs is annotated - if self.added_blocks is not None: - newgraphs = [self.annotated[block] for block in self.added_blocks] - newgraphs = dict.fromkeys(newgraphs) - got_blocked_blocks = False in newgraphs - else: - newgraphs = self.translator.graphs #all of them - got_blocked_blocks = False in self.annotated.values() - if got_blocked_blocks: - for graph in self.blocked_graphs.values(): - self.blocked_graphs[graph] = True - - blocked_blocks = [block for block, done in self.annotated.items() - if done is False] - assert len(blocked_blocks) == len(self.blocked_blocks) - - text = format_blocked_annotation_error(self, self.blocked_blocks) - #raise SystemExit() - raise AnnotatorError(text) - for graph in newgraphs: - v = graph.getreturnvar() - if v not in self.bindings: - self.setbinding(v, annmodel.s_ImpossibleValue) - # policy-dependent computation - self.bookkeeper.compute_at_fixpoint() - - def binding(self, arg, default=FAIL): - "Gives the SomeValue corresponding to the given Variable or Constant." - if isinstance(arg, Variable): - try: - return self.bindings[arg] - except KeyError: - if default is not FAIL: - return default - else: - raise - elif isinstance(arg, Constant): - #if arg.value is undefined_value: # undefined local variables - # return annmodel.s_ImpossibleValue - return self.bookkeeper.immutableconstant(arg) - else: - raise TypeError, 'Variable or Constant expected, got %r' % (arg,) - - def typeannotation(self, t): - return signature.annotation(t, self.bookkeeper) - - def setbinding(self, arg, s_value): - if arg in self.bindings: - assert s_value.contains(self.bindings[arg]) - self.bindings[arg] = s_value - - def transfer_binding(self, v_target, v_source): - assert v_source in self.bindings - self.bindings[v_target] = self.bindings[v_source] - - def warning(self, msg, pos=None): - if pos is None: - try: - pos = self.bookkeeper.position_key - except AttributeError: - pos = '?' - if pos != '?': - pos = self.whereami(pos) - - log.WARNING("%s/ %s" % (pos, msg)) - - - #___ interface for annotator.bookkeeper _______ - - def recursivecall(self, graph, whence, inputcells): - if isinstance(whence, tuple): - parent_graph, parent_block, parent_index = whence - tag = parent_block, parent_index - self.translator.update_call_graph(parent_graph, graph, tag) - # self.notify[graph.returnblock] is a dictionary of call - # points to this func which triggers a reflow whenever the - # return block of this graph has been analysed. - callpositions = self.notify.setdefault(graph.returnblock, {}) - if whence is not None: - if callable(whence): - def callback(): - whence(self, graph) - else: - callback = whence - callpositions[callback] = True - - # generalize the function's input arguments - self.addpendingblock(graph, graph.startblock, inputcells) - - # get the (current) return value - v = graph.getreturnvar() - try: - return self.bindings[v] - except KeyError: - # the function didn't reach any return statement so far. - # (some functions actually never do, they always raise exceptions) - return annmodel.s_ImpossibleValue - - def reflowfromposition(self, position_key): - graph, block, index = position_key - self.reflowpendingblock(graph, block) - - - #___ simplification (should be moved elsewhere?) _______ - - def simplify(self, block_subset=None, extra_passes=None): - # Generic simplifications - transform.transform_graph(self, block_subset=block_subset, - extra_passes=extra_passes) - if block_subset is None: - graphs = self.translator.graphs - else: - graphs = {} - for block in block_subset: - graph = self.annotated.get(block) - if graph: - graphs[graph] = True - for graph in graphs: - simplify.eliminate_empty_blocks(graph) - - - #___ flowing annotations in blocks _____________________ - - def processblock(self, graph, block): - # Important: this is not called recursively. - # self.flowin() can only issue calls to self.addpendingblock(). - # The analysis of a block can be in three states: - # * block not in self.annotated: - # never seen the block. - # * self.annotated[block] == False: - # the input variables of the block are in self.bindings but we - # still have to consider all the operations in the block. - # * self.annotated[block] == graph-containing-block: - # analysis done (at least until we find we must generalize the - # input variables). - - #print '* processblock', block, cells - self.annotated[block] = graph - if block in self.blocked_blocks: - del self.blocked_blocks[block] - try: - self.flowin(graph, block) - except BlockedInference, e: - self.annotated[block] = False # failed, hopefully temporarily - self.blocked_blocks[block] = (graph, e.opindex) - except Exception, e: - # hack for debug tools only - if not hasattr(e, '__annotator_block'): - setattr(e, '__annotator_block', block) - raise - - # The dict 'added_blocks' is used by rpython.annlowlevel to - # detect which are the new blocks that annotating an additional - # small helper creates. - if self.added_blocks is not None: - self.added_blocks[block] = True - - def reflowpendingblock(self, graph, block): - assert not self.frozen - assert graph not in self.fixed_graphs - self.pendingblocks[block] = graph - assert block in self.annotated - self.annotated[block] = False # must re-flow - self.blocked_blocks[block] = (graph, None) - - def bindinputargs(self, graph, block, inputcells): - # Create the initial bindings for the input args of a block. - assert len(block.inputargs) == len(inputcells) - for a, cell in zip(block.inputargs, inputcells): - self.setbinding(a, cell) - self.annotated[block] = False # must flowin. - self.blocked_blocks[block] = (graph, None) - - def mergeinputargs(self, graph, block, inputcells): - # Merge the new 'cells' with each of the block's existing input - # variables. - oldcells = [self.binding(a) for a in block.inputargs] - try: - unions = [annmodel.unionof(c1,c2) for c1, c2 in zip(oldcells,inputcells)] - except annmodel.UnionError, e: - e.args = e.args + ( - ErrorWrapper(gather_error(self, graph, block, None)),) - raise - # if the merged cells changed, we must redo the analysis - if unions != oldcells: - self.bindinputargs(graph, block, unions) - - def whereami(self, position_key): - graph, block, i = position_key - blk = "" - if block: - at = block.at() - if at: - blk = " block"+at - opid="" - if i is not None: - opid = " op=%d" % i - return repr(graph) + blk + opid - - def flowin(self, graph, block): - #print 'Flowing', block, [self.binding(a) for a in block.inputargs] - try: - for i in range(len(block.operations)): - try: - self.bookkeeper.enter((graph, block, i)) - self.consider_op(block, i) - finally: - self.bookkeeper.leave() - - except BlockedInference, e: - if (e.op is block.operations[-1] and - block.exitswitch == c_last_exception): - # this is the case where the last operation of the block will - # always raise an exception which is immediately caught by - # an exception handler. We then only follow the exceptional - # branches. - exits = [link for link in block.exits - if link.exitcase is not None] - - elif e.op.opname in ('simple_call', 'call_args', 'next'): - # XXX warning, keep the name of the call operations in sync - # with the flow object space. These are the operations for - # which it is fine to always raise an exception. We then - # swallow the BlockedInference and that's it. - # About 'next': see test_annotate_iter_empty_container(). - return - - else: - # other cases are problematic (but will hopefully be solved - # later by reflowing). Throw the BlockedInference up to - # processblock(). - raise - - except annmodel.HarmlesslyBlocked: - return - - else: - # dead code removal: don't follow all exits if the exitswitch - # is known - exits = block.exits - if isinstance(block.exitswitch, Variable): - s_exitswitch = self.bindings[block.exitswitch] - if s_exitswitch.is_constant(): - exits = [link for link in exits - if link.exitcase == s_exitswitch.const] - - # mapping (exitcase, variable) -> s_annotation - # that can be attached to booleans, exitswitches - knowntypedata = getattr(self.bindings.get(block.exitswitch), - "knowntypedata", {}) - - # filter out those exceptions which cannot - # occour for this specific, typed operation. - if block.exitswitch == c_last_exception: - op = block.operations[-1] - if op.opname in binaryop.BINARY_OPERATIONS: - arg1 = self.binding(op.args[0]) - arg2 = self.binding(op.args[1]) - binop = getattr(pair(arg1, arg2), op.opname, None) - can_only_throw = annmodel.read_can_only_throw(binop, arg1, arg2) - elif op.opname in unaryop.UNARY_OPERATIONS: - arg1 = self.binding(op.args[0]) - opname = op.opname - if opname == 'contains': opname = 'op_contains' - unop = getattr(arg1, opname, None) - can_only_throw = annmodel.read_can_only_throw(unop, arg1) - else: - can_only_throw = None - - if can_only_throw is not None: - candidates = can_only_throw - candidate_exits = exits - exits = [] - for link in candidate_exits: - case = link.exitcase - if case is None: - exits.append(link) - continue - covered = [c for c in candidates if issubclass(c, case)] - if covered: - exits.append(link) - candidates = [c for c in candidates if c not in covered] - - for link in exits: - in_except_block = False - - last_exception_var = link.last_exception # may be None for non-exception link - last_exc_value_var = link.last_exc_value # may be None for non-exception link - - if isinstance(link.exitcase, (types.ClassType, type)) \ - and issubclass(link.exitcase, py.builtin.BaseException): - assert last_exception_var and last_exc_value_var - last_exc_value_object = self.bookkeeper.valueoftype(link.exitcase) - last_exception_object = annmodel.SomeType() - if isinstance(last_exception_var, Constant): - last_exception_object.const = last_exception_var.value - last_exception_object.is_type_of = [last_exc_value_var] - - if isinstance(last_exception_var, Variable): - self.setbinding(last_exception_var, last_exception_object) - if isinstance(last_exc_value_var, Variable): - self.setbinding(last_exc_value_var, last_exc_value_object) - - last_exception_object = annmodel.SomeType() - if isinstance(last_exception_var, Constant): - last_exception_object.const = last_exception_var.value - #if link.exitcase is Exception: - # last_exc_value_object = annmodel.SomeObject() - #else: - last_exc_value_vars = [] - in_except_block = True - - ignore_link = False - cells = [] - renaming = {} - for a,v in zip(link.args,link.target.inputargs): - renaming.setdefault(a, []).append(v) - for a,v in zip(link.args,link.target.inputargs): - if a == last_exception_var: - assert in_except_block - cells.append(last_exception_object) - elif a == last_exc_value_var: - assert in_except_block - cells.append(last_exc_value_object) - last_exc_value_vars.append(v) - else: - cell = self.binding(a) - if (link.exitcase, a) in knowntypedata: - knownvarvalue = knowntypedata[(link.exitcase, a)] - cell = pair(cell, knownvarvalue).improve() - # ignore links that try to pass impossible values - if cell == annmodel.s_ImpossibleValue: - ignore_link = True - - if hasattr(cell,'is_type_of'): - renamed_is_type_of = [] - for v in cell.is_type_of: - new_vs = renaming.get(v,[]) - renamed_is_type_of += new_vs - assert cell.knowntype is type - newcell = annmodel.SomeType() - if cell.is_constant(): - newcell.const = cell.const - cell = newcell - cell.is_type_of = renamed_is_type_of - - if hasattr(cell, 'knowntypedata'): - renamed_knowntypedata = {} - for (value, v), s in cell.knowntypedata.items(): - new_vs = renaming.get(v, []) - for new_v in new_vs: - renamed_knowntypedata[value, new_v] = s - assert isinstance(cell, annmodel.SomeBool) - newcell = annmodel.SomeBool() - if cell.is_constant(): - newcell.const = cell.const - cell = newcell - cell.set_knowntypedata(renamed_knowntypedata) - - cells.append(cell) - - if ignore_link: - continue - - if in_except_block: - last_exception_object.is_type_of = last_exc_value_vars - - self.links_followed[link] = True - self.addpendingblock(graph, link.target, cells) - - if block in self.notify: - # reflow from certain positions when this block is done - for callback in self.notify[block]: - if isinstance(callback, tuple): - self.reflowfromposition(callback) # callback is a position - else: - callback() - - - #___ creating the annotations based on operations ______ - - def consider_op(self, block, opindex): - op = block.operations[opindex] - argcells = [self.binding(a) for a in op.args] - consider_meth = getattr(self,'consider_op_'+op.opname, - None) - if not consider_meth: - raise Exception,"unknown op: %r" % op - - # let's be careful about avoiding propagated SomeImpossibleValues - # to enter an op; the latter can result in violations of the - # more general results invariant: e.g. if SomeImpossibleValue enters is_ - # is_(SomeImpossibleValue, None) -> SomeBool - # is_(SomeInstance(not None), None) -> SomeBool(const=False) ... - # boom -- in the assert of setbinding() - for arg in argcells: - if isinstance(arg, annmodel.SomeImpossibleValue): - raise BlockedInference(self, op, opindex) - try: - resultcell = consider_meth(*argcells) - except Exception, e: - graph = self.bookkeeper.position_key[0] - e.args = e.args + ( - ErrorWrapper(gather_error(self, graph, block, opindex)),) - raise - if resultcell is None: - resultcell = self.noreturnvalue(op) - elif resultcell == annmodel.s_ImpossibleValue: - raise BlockedInference(self, op, opindex) # the operation cannot succeed - assert isinstance(resultcell, annmodel.SomeObject) - assert isinstance(op.result, Variable) - self.setbinding(op.result, resultcell) # bind resultcell to op.result - - def noreturnvalue(self, op): - return annmodel.s_ImpossibleValue # no return value (hook method) - - # XXX "contains" clash with SomeObject method - def consider_op_contains(self, seq, elem): - self.bookkeeper.count("contains", seq) - return seq.op_contains(elem) - - def consider_op_newtuple(self, *args): - return annmodel.SomeTuple(items = args) - - def consider_op_newlist(self, *args): - return self.bookkeeper.newlist(*args) - - def consider_op_newdict(self): - return self.bookkeeper.newdict() - - - def _registeroperations(cls, unary_ops, binary_ops): - # All unary operations - d = {} - for opname in unary_ops: - fnname = 'consider_op_' + opname - exec py.code.Source(""" -def consider_op_%s(self, arg, *args): - return arg.%s(*args) -""" % (opname, opname)).compile() in globals(), d - setattr(cls, fnname, d[fnname]) - # All binary operations - for opname in binary_ops: - fnname = 'consider_op_' + opname - exec py.code.Source(""" -def consider_op_%s(self, arg1, arg2, *args): - return pair(arg1,arg2).%s(*args) -""" % (opname, opname)).compile() in globals(), d - setattr(cls, fnname, d[fnname]) - _registeroperations = classmethod(_registeroperations) - -# register simple operations handling -RPythonAnnotator._registeroperations(unaryop.UNARY_OPERATIONS, binaryop.BINARY_OPERATIONS) - - -class BlockedInference(Exception): - """This exception signals the type inference engine that the situation - is currently blocked, and that it should try to progress elsewhere.""" - - def __init__(self, annotator, op, opindex): - self.annotator = annotator - try: - self.break_at = annotator.bookkeeper.position_key - except AttributeError: - self.break_at = None - self.op = op - self.opindex = opindex - - def __repr__(self): - if not self.break_at: - break_at = "?" - else: - break_at = self.annotator.whereami(self.break_at) - return "" %(break_at, self.op) - - __str__ = __repr__ diff --git a/pypy/annotation/binaryop.py b/pypy/annotation/binaryop.py deleted file mode 100644 --- a/pypy/annotation/binaryop.py +++ /dev/null @@ -1,1052 +0,0 @@ -""" -Binary operations between SomeValues. -""" - -import py -import operator -from pypy.tool.pairtype import pair, pairtype -from pypy.annotation.model import SomeObject, SomeInteger, SomeBool, s_Bool -from pypy.annotation.model import SomeString, SomeChar, SomeList, SomeDict -from pypy.annotation.model import SomeUnicodeCodePoint, SomeUnicodeString -from pypy.annotation.model import SomeTuple, SomeImpossibleValue, s_ImpossibleValue -from pypy.annotation.model import SomeInstance, SomeBuiltin, SomeIterator -from pypy.annotation.model import SomePBC, SomeFloat, s_None, SomeByteArray -from pypy.annotation.model import SomeExternalObject, SomeWeakRef -from pypy.annotation.model import SomeAddress, SomeTypedAddressAccess -from pypy.annotation.model import SomeSingleFloat, SomeLongFloat, SomeType -from pypy.annotation.model import unionof, UnionError, missing_operation -from pypy.annotation.model import TLS -from pypy.annotation.model import read_can_only_throw -from pypy.annotation.model import add_knowntypedata, merge_knowntypedata -from pypy.annotation.model import SomeGenericCallable -from pypy.annotation.bookkeeper import getbookkeeper -from pypy.objspace.flow.model import Variable, Constant -from pypy.rlib import rarithmetic -from pypy.tool.error import AnnotatorError - -# convenience only! -def immutablevalue(x): - return getbookkeeper().immutablevalue(x) - -# XXX unify this with ObjSpace.MethodTable -BINARY_OPERATIONS = set(['add', 'sub', 'mul', 'div', 'mod', - 'truediv', 'floordiv', 'divmod', - 'and_', 'or_', 'xor', - 'lshift', 'rshift', - 'getitem', 'setitem', 'delitem', - 'getitem_idx', 'getitem_key', 'getitem_idx_key', - 'inplace_add', 'inplace_sub', 'inplace_mul', - 'inplace_truediv', 'inplace_floordiv', 'inplace_div', - 'inplace_mod', - 'inplace_lshift', 'inplace_rshift', - 'inplace_and', 'inplace_or', 'inplace_xor', - 'lt', 'le', 'eq', 'ne', 'gt', 'ge', 'is_', 'cmp', - 'coerce', - ] - +[opname+'_ovf' for opname in - """add sub mul floordiv div mod lshift - """.split() - ]) - -for opname in BINARY_OPERATIONS: - missing_operation(pairtype(SomeObject, SomeObject), opname) - -class __extend__(pairtype(SomeObject, SomeObject)): - - def union((obj1, obj2)): - raise UnionError(obj1, obj2) - - # inplace_xxx ---> xxx by default - def inplace_add((obj1, obj2)): return pair(obj1, obj2).add() - def inplace_sub((obj1, obj2)): return pair(obj1, obj2).sub() - def inplace_mul((obj1, obj2)): return pair(obj1, obj2).mul() - def inplace_truediv((obj1, obj2)): return pair(obj1, obj2).truediv() - def inplace_floordiv((obj1, obj2)): return pair(obj1, obj2).floordiv() - def inplace_div((obj1, obj2)): return pair(obj1, obj2).div() - def inplace_mod((obj1, obj2)): return pair(obj1, obj2).mod() - def inplace_lshift((obj1, obj2)): return pair(obj1, obj2).lshift() - def inplace_rshift((obj1, obj2)): return pair(obj1, obj2).rshift() - def inplace_and((obj1, obj2)): return pair(obj1, obj2).and_() - def inplace_or((obj1, obj2)): return pair(obj1, obj2).or_() - def inplace_xor((obj1, obj2)): return pair(obj1, obj2).xor() - - for name, func in locals().items(): - if name.startswith('inplace_'): - func.can_only_throw = [] - - inplace_div.can_only_throw = [ZeroDivisionError] - inplace_truediv.can_only_throw = [ZeroDivisionError] - inplace_floordiv.can_only_throw = [ZeroDivisionError] - inplace_mod.can_only_throw = [ZeroDivisionError] - - def lt((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const < obj2.const) - else: - getbookkeeper().count("non_int_comp", obj1, obj2) - return s_Bool - - def le((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const <= obj2.const) - else: - getbookkeeper().count("non_int_comp", obj1, obj2) - return s_Bool - - def eq((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const == obj2.const) - else: - getbookkeeper().count("non_int_eq", obj1, obj2) - return s_Bool - - def ne((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const != obj2.const) - else: - getbookkeeper().count("non_int_eq", obj1, obj2) - return s_Bool - - def gt((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const > obj2.const) - else: - getbookkeeper().count("non_int_comp", obj1, obj2) - return s_Bool - - def ge((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const >= obj2.const) - else: - getbookkeeper().count("non_int_comp", obj1, obj2) - return s_Bool - - def cmp((obj1, obj2)): - getbookkeeper().count("cmp", obj1, obj2) - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(cmp(obj1.const, obj2.const)) - else: - return SomeInteger() - - def is_((obj1, obj2)): - r = SomeBool() - if obj2.is_constant(): - if obj1.is_constant(): - r.const = obj1.const is obj2.const - if obj2.const is None and not obj1.can_be_none(): - r.const = False - elif obj1.is_constant(): - if obj1.const is None and not obj2.can_be_none(): - r.const = False - # XXX HACK HACK HACK - # XXX HACK HACK HACK - # XXX HACK HACK HACK - bk = getbookkeeper() - if bk is not None: # for testing - knowntypedata = {} - fn, block, i = bk.position_key - - annotator = bk.annotator - op = block.operations[i] - assert op.opname == "is_" - assert len(op.args) == 2 - - def bind(src_obj, tgt_obj, tgt_arg): - if hasattr(tgt_obj, 'is_type_of') and src_obj.is_constant(): - add_knowntypedata(knowntypedata, True, tgt_obj.is_type_of, - bk.valueoftype(src_obj.const)) - - assert annotator.binding(op.args[tgt_arg]) == tgt_obj - add_knowntypedata(knowntypedata, True, [op.args[tgt_arg]], src_obj) - - nonnone_obj = tgt_obj - if src_obj.is_constant() and src_obj.const is None and tgt_obj.can_be_none(): - nonnone_obj = tgt_obj.nonnoneify() - - add_knowntypedata(knowntypedata, False, [op.args[tgt_arg]], nonnone_obj) - - bind(obj2, obj1, 0) - bind(obj1, obj2, 1) - r.set_knowntypedata(knowntypedata) - - return r - - def divmod((obj1, obj2)): - getbookkeeper().count("divmod", obj1, obj2) - return SomeTuple([pair(obj1, obj2).div(), pair(obj1, obj2).mod()]) - - def coerce((obj1, obj2)): - getbookkeeper().count("coerce", obj1, obj2) - return pair(obj1, obj2).union() # reasonable enough - - # approximation of an annotation intersection, the result should be the annotation obj or - # the intersection of obj and improvement - def improve((obj, improvement)): - if not improvement.contains(obj) and obj.contains(improvement): - return improvement - else: - return obj - - # checked getitems - - def _getitem_can_only_throw(s_c1, s_o2): - impl = pair(s_c1, s_o2).getitem - return read_can_only_throw(impl, s_c1, s_o2) - - def getitem_idx_key((s_c1, s_o2)): - impl = pair(s_c1, s_o2).getitem - return impl() - getitem_idx_key.can_only_throw = _getitem_can_only_throw - - getitem_idx = getitem_idx_key - getitem_key = getitem_idx_key - - -class __extend__(pairtype(SomeType, SomeType)): - - def union((obj1, obj2)): - result = SomeType() - is_type_of1 = getattr(obj1, 'is_type_of', None) - is_type_of2 = getattr(obj2, 'is_type_of', None) - if obj1.is_immutable_constant() and obj2.is_immutable_constant() and obj1.const == obj2.const: - result.const = obj1.const - is_type_of = {} - if is_type_of1: - for v in is_type_of1: - is_type_of[v] = True - if is_type_of2: - for v in is_type_of2: - is_type_of[v] = True - if is_type_of: - result.is_type_of = is_type_of.keys() - else: - if is_type_of1 and is_type_of1 == is_type_of2: - result.is_type_of = is_type_of1 - return result - - -# cloning a function with identical code, for the can_only_throw attribute -def _clone(f, can_only_throw = None): - newfunc = type(f)(f.func_code, f.func_globals, f.func_name, - f.func_defaults, f.func_closure) - if can_only_throw is not None: - newfunc.can_only_throw = can_only_throw - return newfunc - -class __extend__(pairtype(SomeInteger, SomeInteger)): - # unsignedness is considered a rare and contagious disease - - def union((int1, int2)): - if int1.unsigned == int2.unsigned: - knowntype = rarithmetic.compute_restype(int1.knowntype, int2.knowntype) - else: - t1 = int1.knowntype - if t1 is bool: - t1 = int - t2 = int2.knowntype - if t2 is bool: - t2 = int - - if t2 is int: - if int2.nonneg == False: - raise UnionError, "Merging %s and a possibly negative int is not allowed" % t1 - knowntype = t1 - elif t1 is int: - if int1.nonneg == False: - raise UnionError, "Merging %s and a possibly negative int is not allowed" % t2 - knowntype = t2 - else: - raise UnionError, "Merging these types (%s, %s) is not supported" % (t1, t2) - return SomeInteger(nonneg=int1.nonneg and int2.nonneg, - knowntype=knowntype) - - or_ = xor = add = mul = _clone(union, []) - add_ovf = mul_ovf = _clone(union, [OverflowError]) - div = floordiv = mod = _clone(union, [ZeroDivisionError]) - div_ovf= floordiv_ovf = mod_ovf = _clone(union, [ZeroDivisionError, OverflowError]) - - def truediv((int1, int2)): - return SomeFloat() - truediv.can_only_throw = [ZeroDivisionError] - truediv_ovf = _clone(truediv, [ZeroDivisionError, OverflowError]) - - inplace_div = div - inplace_truediv = truediv - - def sub((int1, int2)): - knowntype = rarithmetic.compute_restype(int1.knowntype, int2.knowntype) - return SomeInteger(knowntype=knowntype) - sub.can_only_throw = [] - sub_ovf = _clone(sub, [OverflowError]) - - def and_((int1, int2)): - knowntype = rarithmetic.compute_restype(int1.knowntype, int2.knowntype) - return SomeInteger(nonneg=int1.nonneg or int2.nonneg, - knowntype=knowntype) - and_.can_only_throw = [] - - def lshift((int1, int2)): - if isinstance(int1, SomeBool): - return SomeInteger() - else: - return SomeInteger(knowntype=int1.knowntype) - lshift.can_only_throw = [] - lshift_ovf = _clone(lshift, [OverflowError]) - - def rshift((int1, int2)): - if isinstance(int1, SomeBool): - return SomeInteger(nonneg=True) - else: - return SomeInteger(nonneg=int1.nonneg, knowntype=int1.knowntype) - rshift.can_only_throw = [] - - def _compare_helper((int1, int2), opname, operation): - r = SomeBool() - if int1.is_immutable_constant() and int2.is_immutable_constant(): - r.const = operation(int1.const, int2.const) - # - # The rest of the code propagates nonneg information between - # the two arguments. - # - # Doing the right thing when int1 or int2 change from signed - # to unsigned (r_uint) is almost impossible. See test_intcmp_bug. - # Instead, we only deduce constrains on the operands in the - # case where they are both signed. In other words, if y is - # nonneg then "assert x>=y" will let the annotator know that - # x is nonneg too, but it will not work if y is unsigned. - # - if not (rarithmetic.signedtype(int1.knowntype) and - rarithmetic.signedtype(int2.knowntype)): - return r - knowntypedata = {} - # XXX HACK HACK HACK - fn, block, i = getbookkeeper().position_key - op = block.operations[i] - assert op.opname == opname - assert len(op.args) == 2 - def tointtype(int0): - if int0.knowntype is bool: - return int - return int0.knowntype - if int1.nonneg and isinstance(op.args[1], Variable): - case = opname in ('lt', 'le', 'eq') - - add_knowntypedata(knowntypedata, case, [op.args[1]], - SomeInteger(nonneg=True, knowntype=tointtype(int2))) - if int2.nonneg and isinstance(op.args[0], Variable): - case = opname in ('gt', 'ge', 'eq') - add_knowntypedata(knowntypedata, case, [op.args[0]], - SomeInteger(nonneg=True, knowntype=tointtype(int1))) - r.set_knowntypedata(knowntypedata) - # a special case for 'x < 0' or 'x >= 0', - # where 0 is a flow graph Constant - # (in this case we are sure that it cannot become a r_uint later) - if (isinstance(op.args[1], Constant) and - type(op.args[1].value) is int and # filter out Symbolics - op.args[1].value == 0): - if int1.nonneg: - if opname == 'lt': - r.const = False - if opname == 'ge': - r.const = True - return r - - def lt(intint): return intint._compare_helper('lt', operator.lt) - def le(intint): return intint._compare_helper('le', operator.le) - def eq(intint): return intint._compare_helper('eq', operator.eq) - def ne(intint): return intint._compare_helper('ne', operator.ne) - def gt(intint): return intint._compare_helper('gt', operator.gt) - def ge(intint): return intint._compare_helper('ge', operator.ge) - - -class __extend__(pairtype(SomeBool, SomeBool)): - - def union((boo1, boo2)): - s = SomeBool() - if getattr(boo1, 'const', -1) == getattr(boo2, 'const', -2): - s.const = boo1.const - if hasattr(boo1, 'knowntypedata') and \ - hasattr(boo2, 'knowntypedata'): - ktd = merge_knowntypedata(boo1.knowntypedata, boo2.knowntypedata) - s.set_knowntypedata(ktd) - return s - - def and_((boo1, boo2)): - s = SomeBool() - if boo1.is_constant(): - if not boo1.const: - s.const = False - else: - return boo2 - if boo2.is_constant(): - if not boo2.const: - s.const = False - return s - - def or_((boo1, boo2)): - s = SomeBool() - if boo1.is_constant(): - if boo1.const: - s.const = True - else: - return boo2 - if boo2.is_constant(): - if boo2.const: - s.const = True - return s - - def xor((boo1, boo2)): - s = SomeBool() - if boo1.is_constant() and boo2.is_constant(): - s.const = boo1.const ^ boo2.const - return s - -class __extend__(pairtype(SomeString, SomeString)): - - def union((str1, str2)): - can_be_None = str1.can_be_None or str2.can_be_None - no_nul = str1.no_nul and str2.no_nul - return SomeString(can_be_None=can_be_None, no_nul=no_nul) - - def add((str1, str2)): - # propagate const-ness to help getattr(obj, 'prefix' + const_name) - result = SomeString(no_nul=str1.no_nul and str2.no_nul) - if str1.is_immutable_constant() and str2.is_immutable_constant(): - result.const = str1.const + str2.const - return result - -class __extend__(pairtype(SomeByteArray, SomeByteArray)): - def union((b1, b2)): - can_be_None = b1.can_be_None or b2.can_be_None - return SomeByteArray(can_be_None=can_be_None) - - def add((b1, b2)): - result = SomeByteArray() - if b1.is_immutable_constant() and b2.is_immutable_constant(): - result.const = b1.const + b2.const - return result - -class __extend__(pairtype(SomeByteArray, SomeInteger)): - def getitem((s_b, s_i)): - return SomeInteger() - - def setitem((s_b, s_i), s_i2): - assert isinstance(s_i2, SomeInteger) - -class __extend__(pairtype(SomeString, SomeByteArray), - pairtype(SomeByteArray, SomeString), - pairtype(SomeChar, SomeByteArray), - pairtype(SomeByteArray, SomeChar)): - def add((b1, b2)): - result = SomeByteArray() - if b1.is_immutable_constant() and b2.is_immutable_constant(): - result.const = b1.const + b2.const - return result - -class __extend__(pairtype(SomeChar, SomeChar)): - - def union((chr1, chr2)): - return SomeChar() - - -class __extend__(pairtype(SomeChar, SomeUnicodeCodePoint), - pairtype(SomeUnicodeCodePoint, SomeChar)): - def union((uchr1, uchr2)): - return SomeUnicodeCodePoint() - -class __extend__(pairtype(SomeUnicodeCodePoint, SomeUnicodeCodePoint)): - def union((uchr1, uchr2)): - return SomeUnicodeCodePoint() - - def add((chr1, chr2)): - return SomeUnicodeString() - -class __extend__(pairtype(SomeString, SomeUnicodeString), - pairtype(SomeUnicodeString, SomeString)): - def mod((str, unistring)): - raise NotImplementedError( - "string formatting mixing strings and unicode not supported") - - -class __extend__(pairtype(SomeString, SomeTuple), - pairtype(SomeUnicodeString, SomeTuple)): - def mod((s_string, s_tuple)): - is_string = isinstance(s_string, SomeString) - is_unicode = isinstance(s_string, SomeUnicodeString) - assert is_string or is_unicode - for s_item in s_tuple.items: - if (is_unicode and isinstance(s_item, (SomeChar, SomeString)) or - is_string and isinstance(s_item, (SomeUnicodeCodePoint, - SomeUnicodeString))): - raise NotImplementedError( - "string formatting mixing strings and unicode not supported") - getbookkeeper().count('strformat', s_string, s_tuple) - no_nul = s_string.no_nul - for s_item in s_tuple.items: - if isinstance(s_item, SomeFloat): - pass # or s_item is a subclass, like SomeInteger - elif (isinstance(s_item, SomeString) or - isinstance(s_item, SomeUnicodeString)) and s_item.no_nul: - pass - else: - no_nul = False - break - return s_string.__class__(no_nul=no_nul) - - -class __extend__(pairtype(SomeString, SomeObject), - pairtype(SomeUnicodeString, SomeObject)): - - def mod((s_string, args)): - getbookkeeper().count('strformat', s_string, args) - return s_string.__class__() - -class __extend__(pairtype(SomeFloat, SomeFloat)): - - def union((flt1, flt2)): - return SomeFloat() - - add = sub = mul = union - - def div((flt1, flt2)): - return SomeFloat() - div.can_only_throw = [] - truediv = div - - # repeat these in order to copy the 'can_only_throw' attribute - inplace_div = div - inplace_truediv = truediv - - -class __extend__(pairtype(SomeSingleFloat, SomeSingleFloat)): - - def union((flt1, flt2)): - return SomeSingleFloat() - - -class __extend__(pairtype(SomeLongFloat, SomeLongFloat)): - - def union((flt1, flt2)): - return SomeLongFloat() - - -class __extend__(pairtype(SomeList, SomeList)): - - def union((lst1, lst2)): - return SomeList(lst1.listdef.union(lst2.listdef)) - - def add((lst1, lst2)): - return lst1.listdef.offspring(lst2.listdef) - - def eq((lst1, lst2)): - lst1.listdef.agree(lst2.listdef) - return s_Bool - ne = eq - - -class __extend__(pairtype(SomeList, SomeObject)): - - def inplace_add((lst1, obj2)): - lst1.method_extend(obj2) - return lst1 - inplace_add.can_only_throw = [] - - def inplace_mul((lst1, obj2)): - lst1.listdef.resize() - return lst1 - inplace_mul.can_only_throw = [] - -class __extend__(pairtype(SomeTuple, SomeTuple)): - - def union((tup1, tup2)): - if len(tup1.items) != len(tup2.items): - raise UnionError("cannot take the union of a tuple of length %d " - "and a tuple of length %d" % (len(tup1.items), - len(tup2.items))) - else: - unions = [unionof(x,y) for x,y in zip(tup1.items, tup2.items)] - return SomeTuple(items = unions) - - def add((tup1, tup2)): - return SomeTuple(items = tup1.items + tup2.items) - - def eq(tup1tup2): - tup1tup2.union() - return s_Bool - ne = eq - - def lt((tup1, tup2)): - raise Exception("unsupported: (...) < (...)") - def le((tup1, tup2)): - raise Exception("unsupported: (...) <= (...)") - def gt((tup1, tup2)): - raise Exception("unsupported: (...) > (...)") - def ge((tup1, tup2)): - raise Exception("unsupported: (...) >= (...)") - - -class __extend__(pairtype(SomeDict, SomeDict)): - - def union((dic1, dic2)): - return SomeDict(dic1.dictdef.union(dic2.dictdef)) - - -class __extend__(pairtype(SomeDict, SomeObject)): - - def _can_only_throw(dic1, *ignore): - if dic1.dictdef.dictkey.custom_eq_hash: - return None - return [KeyError] - - def getitem((dic1, obj2)): - getbookkeeper().count("dict_getitem", dic1) - dic1.dictdef.generalize_key(obj2) - return dic1.dictdef.read_value() - getitem.can_only_throw = _can_only_throw - - def setitem((dic1, obj2), s_value): - getbookkeeper().count("dict_setitem", dic1) - dic1.dictdef.generalize_key(obj2) - dic1.dictdef.generalize_value(s_value) - setitem.can_only_throw = _can_only_throw - - def delitem((dic1, obj2)): - getbookkeeper().count("dict_delitem", dic1) - dic1.dictdef.generalize_key(obj2) - delitem.can_only_throw = _can_only_throw - - -class __extend__(pairtype(SomeTuple, SomeInteger)): - - def getitem((tup1, int2)): - if int2.is_immutable_constant(): - try: - return tup1.items[int2.const] - except IndexError: - return s_ImpossibleValue - else: - getbookkeeper().count("tuple_random_getitem", tup1) - return unionof(*tup1.items) - getitem.can_only_throw = [IndexError] - - -class __extend__(pairtype(SomeList, SomeInteger)): - - def mul((lst1, int2)): - return lst1.listdef.offspring() - - def getitem((lst1, int2)): - getbookkeeper().count("list_getitem", int2) - return lst1.listdef.read_item() - getitem.can_only_throw = [] - - getitem_key = getitem - - def getitem_idx((lst1, int2)): - getbookkeeper().count("list_getitem", int2) - return lst1.listdef.read_item() - getitem_idx.can_only_throw = [IndexError] - - getitem_idx_key = getitem_idx - - def setitem((lst1, int2), s_value): - getbookkeeper().count("list_setitem", int2) - lst1.listdef.mutate() - lst1.listdef.generalize(s_value) - setitem.can_only_throw = [IndexError] - - def delitem((lst1, int2)): - getbookkeeper().count("list_delitem", int2) - lst1.listdef.resize() - delitem.can_only_throw = [IndexError] - -class __extend__(pairtype(SomeString, SomeInteger)): - - def getitem((str1, int2)): - getbookkeeper().count("str_getitem", int2) - return SomeChar() - getitem.can_only_throw = [] - - getitem_key = getitem - - def getitem_idx((str1, int2)): - getbookkeeper().count("str_getitem", int2) - return SomeChar() - getitem_idx.can_only_throw = [IndexError] - - getitem_idx_key = getitem_idx - - def mul((str1, int2)): # xxx do we want to support this - getbookkeeper().count("str_mul", str1, int2) - return SomeString(no_nul=str1.no_nul) - -class __extend__(pairtype(SomeUnicodeString, SomeInteger)): - def getitem((str1, int2)): - getbookkeeper().count("str_getitem", int2) - return SomeUnicodeCodePoint() - getitem.can_only_throw = [] - - getitem_key = getitem - - def getitem_idx((str1, int2)): - getbookkeeper().count("str_getitem", int2) - return SomeUnicodeCodePoint() - getitem_idx.can_only_throw = [IndexError] - - getitem_idx_key = getitem_idx - - def mul((str1, int2)): # xxx do we want to support this - getbookkeeper().count("str_mul", str1, int2) - return SomeUnicodeString() - -class __extend__(pairtype(SomeInteger, SomeString), - pairtype(SomeInteger, SomeUnicodeString)): - - def mul((int1, str2)): # xxx do we want to support this - getbookkeeper().count("str_mul", str2, int1) - return str2.basestringclass() - -class __extend__(pairtype(SomeUnicodeCodePoint, SomeUnicodeString), - pairtype(SomeUnicodeString, SomeUnicodeCodePoint), - pairtype(SomeUnicodeString, SomeUnicodeString)): - def union((str1, str2)): - return SomeUnicodeString(can_be_None=str1.can_be_none() or - str2.can_be_none()) - - def add((str1, str2)): - # propagate const-ness to help getattr(obj, 'prefix' + const_name) - result = SomeUnicodeString() - if str1.is_immutable_constant() and str2.is_immutable_constant(): - result.const = str1.const + str2.const - return result - -class __extend__(pairtype(SomeInteger, SomeList)): - - def mul((int1, lst2)): - return lst2.listdef.offspring() - - -class __extend__(pairtype(SomeInstance, SomeInstance)): - - def union((ins1, ins2)): - if ins1.classdef is None or ins2.classdef is None: - # special case only - basedef = None - else: - basedef = ins1.classdef.commonbase(ins2.classdef) - if basedef is None: - raise UnionError(ins1, ins2) - flags = ins1.flags - if flags: - flags = flags.copy() - for key, value in flags.items(): - if key not in ins2.flags or ins2.flags[key] != value: - del flags[key] - return SomeInstance(basedef, - can_be_None=ins1.can_be_None or ins2.can_be_None, - flags=flags) - - def improve((ins1, ins2)): - if ins1.classdef is None: - resdef = ins2.classdef - elif ins2.classdef is None: - resdef = ins1.classdef - else: - basedef = ins1.classdef.commonbase(ins2.classdef) - if basedef is ins1.classdef: - resdef = ins2.classdef - elif basedef is ins2.classdef: - resdef = ins1.classdef - else: - if ins1.can_be_None and ins2.can_be_None: - return s_None - else: - return s_ImpossibleValue - res = SomeInstance(resdef, can_be_None=ins1.can_be_None and ins2.can_be_None) - if ins1.contains(res) and ins2.contains(res): - return res # fine - else: - # this case can occur in the presence of 'const' attributes, - # which we should try to preserve. Fall-back... - thistype = pairtype(SomeInstance, SomeInstance) - return super(thistype, pair(ins1, ins2)).improve() - - -class __extend__(pairtype(SomeIterator, SomeIterator)): - - def union((iter1, iter2)): - s_cont = unionof(iter1.s_container, iter2.s_container) - if iter1.variant != iter2.variant: - raise UnionError("merging incompatible iterators variants") - return SomeIterator(s_cont, *iter1.variant) - - -class __extend__(pairtype(SomeBuiltin, SomeBuiltin)): - - def union((bltn1, bltn2)): - if (bltn1.analyser != bltn2.analyser or - bltn1.methodname != bltn2.methodname or - bltn1.s_self is None or bltn2.s_self is None): - raise UnionError("cannot merge two different builtin functions " - "or methods:\n %r\n %r" % (bltn1, bltn2)) - s_self = unionof(bltn1.s_self, bltn2.s_self) - return SomeBuiltin(bltn1.analyser, s_self, methodname=bltn1.methodname) - -class __extend__(pairtype(SomePBC, SomePBC)): - - def union((pbc1, pbc2)): - d = pbc1.descriptions.copy() - d.update(pbc2.descriptions) - return SomePBC(d, can_be_None = pbc1.can_be_None or pbc2.can_be_None) - - def is_((pbc1, pbc2)): - thistype = pairtype(SomePBC, SomePBC) - s = super(thistype, pair(pbc1, pbc2)).is_() - if not s.is_constant(): - if not pbc1.can_be_None or not pbc2.can_be_None: - for desc in pbc1.descriptions: - if desc in pbc2.descriptions: - break - else: - s.const = False # no common desc in the two sets - return s - -class __extend__(pairtype(SomeGenericCallable, SomePBC)): - def union((gencall, pbc)): - for desc in pbc.descriptions: - unique_key = desc - bk = desc.bookkeeper - s_result = bk.emulate_pbc_call(unique_key, pbc, gencall.args_s) - s_result = unionof(s_result, gencall.s_result) - assert gencall.s_result.contains(s_result) - return gencall - -class __extend__(pairtype(SomePBC, SomeGenericCallable)): - def union((pbc, gencall)): - return pair(gencall, pbc).union() - -class __extend__(pairtype(SomeImpossibleValue, SomeObject)): - def union((imp1, obj2)): - return obj2 - -class __extend__(pairtype(SomeObject, SomeImpossibleValue)): - def union((obj1, imp2)): - return obj1 - -# mixing Nones with other objects - -def _make_none_union(classname, constructor_args='', glob=None): - if glob is None: - glob = globals() - loc = locals() - source = py.code.Source(""" - class __extend__(pairtype(%(classname)s, SomePBC)): - def union((obj, pbc)): - if pbc.isNone(): - return %(classname)s(%(constructor_args)s) - else: - return SomeObject() - - class __extend__(pairtype(SomePBC, %(classname)s)): - def union((pbc, obj)): - if pbc.isNone(): - return %(classname)s(%(constructor_args)s) - else: - return SomeObject() - """ % loc) - exec source.compile() in glob - -_make_none_union('SomeInstance', 'classdef=obj.classdef, can_be_None=True') -_make_none_union('SomeString', 'no_nul=obj.no_nul, can_be_None=True') -_make_none_union('SomeUnicodeString', 'can_be_None=True') -_make_none_union('SomeList', 'obj.listdef') -_make_none_union('SomeDict', 'obj.dictdef') -_make_none_union('SomeExternalObject', 'obj.knowntype') -_make_none_union('SomeWeakRef', 'obj.classdef') - -# getitem on SomePBCs, in particular None fails - -class __extend__(pairtype(SomePBC, SomeObject)): - def getitem((pbc, o)): - if not pbc.isNone(): - raise AnnotatorError("getitem on %r" % pbc) - return s_ImpossibleValue - - def setitem((pbc, o), s_value): - if not pbc.isNone(): - raise AnnotatorError("setitem on %r" % pbc) - -class __extend__(pairtype(SomePBC, SomeString)): - def add((pbc, o)): - if not pbc.isNone(): - raise AnnotatorError('add on %r' % pbc) - return s_ImpossibleValue - -class __extend__(pairtype(SomeString, SomePBC)): - def add((o, pbc)): - if not pbc.isNone(): - raise AnnotatorError('add on %r' % pbc) - return s_ImpossibleValue - -class __extend__(pairtype(SomeExternalObject, SomeExternalObject)): - def union((ext1, ext2)): - if ext1.knowntype == ext2.knowntype: - return SomeExternalObject(ext1.knowntype) - return SomeObject() - -# ____________________________________________________________ -# annotation of low-level types -from pypy.annotation.model import SomePtr, SomeOOInstance, SomeOOClass -from pypy.annotation.model import SomeOOObject -from pypy.annotation.model import ll_to_annotation, annotation_to_lltype -from pypy.rpython.ootypesystem import ootype - -_make_none_union('SomeOOInstance', 'ootype=obj.ootype, can_be_None=True') - -class __extend__(pairtype(SomePtr, SomePtr)): - def union((p1, p2)): - assert p1.ll_ptrtype == p2.ll_ptrtype,("mixing of incompatible pointer types: %r, %r" % - (p1.ll_ptrtype, p2.ll_ptrtype)) - return SomePtr(p1.ll_ptrtype) - -class __extend__(pairtype(SomePtr, SomeInteger)): - - def getitem((p, int1)): - example = p.ll_ptrtype._example() - try: - v = example[0] - except IndexError: - return None # impossible value, e.g. FixedSizeArray(0) - return ll_to_annotation(v) - getitem.can_only_throw = [] - - def setitem((p, int1), s_value): # just doing checking - example = p.ll_ptrtype._example() - if example[0] is not None: # ignore Void s_value - v_lltype = annotation_to_lltype(s_value) - example[0] = v_lltype._defl() - setitem.can_only_throw = [] - -class __extend__(pairtype(SomePtr, SomeObject)): - def union((p, obj)): - assert False, ("mixing pointer type %r with something else %r" % (p.ll_ptrtype, obj)) - - def getitem((p, obj)): - assert False,"ptr %r getitem index not an int: %r" % (p.ll_ptrtype, obj) - - def setitem((p, obj), s_value): - assert False,"ptr %r setitem index not an int: %r" % (p.ll_ptrtype, obj) - -class __extend__(pairtype(SomeObject, SomePtr)): - def union((obj, p2)): - return pair(p2, obj).union() - - -class __extend__(pairtype(SomeOOInstance, SomeOOInstance)): - def union((r1, r2)): - common = ootype.commonBaseclass(r1.ootype, r2.ootype) - assert common is not None, 'Mixing of incompatible instances %r, %r' %(r1.ootype, r2.ootype) - return SomeOOInstance(common, can_be_None=r1.can_be_None or r2.can_be_None) - -class __extend__(pairtype(SomeOOClass, SomeOOClass)): - def union((r1, r2)): - if r1.ootype is None: - common = r2.ootype - elif r2.ootype is None: - common = r1.ootype - elif r1.ootype == r2.ootype: - common = r1.ootype - elif isinstance(r1.ootype, ootype.Instance) and isinstance(r2.ootype, ootype.Instance): - common = ootype.commonBaseclass(r1.ootype, r2.ootype) - assert common is not None, ('Mixing of incompatible classes %r, %r' - % (r1.ootype, r2.ootype)) - else: - common = ootype.Object - return SomeOOClass(common) - -class __extend__(pairtype(SomeOOInstance, SomeObject)): - def union((r, obj)): - assert False, ("mixing reference type %r with something else %r" % (r.ootype, obj)) - -class __extend__(pairtype(SomeObject, SomeOOInstance)): - def union((obj, r2)): - return pair(r2, obj).union() - -class __extend__(pairtype(SomeOOObject, SomeOOObject)): - def union((r1, r2)): - assert r1.ootype is ootype.Object and r2.ootype is ootype.Object - return SomeOOObject() - -#_________________________________________ -# weakrefs - -class __extend__(pairtype(SomeWeakRef, SomeWeakRef)): - def union((s_wrf1, s_wrf2)): - if s_wrf1.classdef is None: - basedef = s_wrf2.classdef # s_wrf1 is known to be dead - elif s_wrf2.classdef is None: - basedef = s_wrf1.classdef # s_wrf2 is known to be dead - else: - basedef = s_wrf1.classdef.commonbase(s_wrf2.classdef) - if basedef is None: # no common base class! complain... - return SomeObject() - return SomeWeakRef(basedef) - -#_________________________________________ -# memory addresses - -class __extend__(pairtype(SomeAddress, SomeAddress)): - def union((s_addr1, s_addr2)): - return SomeAddress() - - def sub((s_addr1, s_addr2)): - if s_addr1.is_null_address() and s_addr2.is_null_address(): - return getbookkeeper().immutablevalue(0) - return SomeInteger() - - def is_((s_addr1, s_addr2)): - assert False, "comparisons with is not supported by addresses" - -class __extend__(pairtype(SomeTypedAddressAccess, SomeTypedAddressAccess)): - def union((s_taa1, s_taa2)): - assert s_taa1.type == s_taa2.type - return s_taa1 - -class __extend__(pairtype(SomeTypedAddressAccess, SomeInteger)): - def getitem((s_taa, s_int)): - from pypy.annotation.model import lltype_to_annotation - return lltype_to_annotation(s_taa.type) - getitem.can_only_throw = [] - - def setitem((s_taa, s_int), s_value): - from pypy.annotation.model import annotation_to_lltype - assert annotation_to_lltype(s_value) is s_taa.type - setitem.can_only_throw = [] - - -class __extend__(pairtype(SomeAddress, SomeInteger)): - def add((s_addr, s_int)): - return SomeAddress() - - def sub((s_addr, s_int)): - return SomeAddress() - -class __extend__(pairtype(SomeAddress, SomeImpossibleValue)): - # need to override this specifically to hide the 'raise UnionError' - # of pairtype(SomeAddress, SomeObject). - def union((s_addr, s_imp)): - return s_addr - -class __extend__(pairtype(SomeImpossibleValue, SomeAddress)): - # need to override this specifically to hide the 'raise UnionError' - # of pairtype(SomeObject, SomeAddress). - def union((s_imp, s_addr)): - return s_addr - -class __extend__(pairtype(SomeAddress, SomeObject)): - def union((s_addr, s_obj)): - raise UnionError, "union of address and anything else makes no sense" - -class __extend__(pairtype(SomeObject, SomeAddress)): - def union((s_obj, s_addr)): - raise UnionError, "union of address and anything else makes no sense" diff --git a/pypy/annotation/bookkeeper.py b/pypy/annotation/bookkeeper.py deleted file mode 100644 --- a/pypy/annotation/bookkeeper.py +++ /dev/null @@ -1,797 +0,0 @@ -""" -The Bookkeeper class. -""" - -from __future__ import absolute_import - -import sys, types, inspect, weakref - -from pypy.objspace.flow.model import Constant From noreply at buildbot.pypy.org Mon Jan 21 01:45:25 2013 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Mon, 21 Jan 2013 01:45:25 +0100 (CET) Subject: [pypy-commit] pypy pytest: move viewerplugin to rpython Message-ID: <20130121004525.DDA5F1C0237@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: pytest Changeset: r60262:c162c8583faf Date: 2013-01-21 01:44 +0100 http://bitbucket.org/pypy/pypy/changeset/c162c8583faf/ Log: move viewerplugin to rpython diff --git a/pytest.ini b/pytest.ini --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,4 @@ [pytest] addopts = --assert=reinterp -rf - -p pypy.tool.pytest.viewerplugin + -p rpython.tool.pytest.viewerplugin diff --git a/pypy/tool/pytest/viewerplugin.py b/rpython/tool/pytest/viewerplugin.py rename from pypy/tool/pytest/viewerplugin.py rename to rpython/tool/pytest/viewerplugin.py diff --git a/pypy/tool/pytest/test/test_viewerplugin.py b/rpython/tool/test/test_viewerplugin.py rename from pypy/tool/pytest/test/test_viewerplugin.py rename to rpython/tool/test/test_viewerplugin.py --- a/pypy/tool/pytest/test/test_viewerplugin.py +++ b/rpython/tool/test/test_viewerplugin.py @@ -1,10 +1,10 @@ import sys import pytest -from pypy.tool.pytest import viewerplugin +from rpython.tool.pytest import viewerplugin -class mock: +class mock(object): view = False - + @staticmethod def execute(): pass From noreply at buildbot.pypy.org Mon Jan 21 01:45:24 2013 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Mon, 21 Jan 2013 01:45:24 +0100 (CET) Subject: [pypy-commit] pypy pytest: merge from default after rpython split Message-ID: <20130121004524.B05661C00F9@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: pytest Changeset: r60261:089b0f6d70c1 Date: 2013-01-21 01:30 +0100 http://bitbucket.org/pypy/pypy/changeset/089b0f6d70c1/ Log: merge from default after rpython split diff too long, truncating to 2000 out of 728198 lines diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -87,3 +87,4 @@ ^compiled ^.git/ ^release/ +^rpython/_cache$ diff --git a/dotviewer/test/test_translator.py b/dotviewer/test/test_translator.py --- a/dotviewer/test/test_translator.py +++ b/dotviewer/test/test_translator.py @@ -20,7 +20,7 @@ def test_annotated(): - from pypy.translator.interactive import Translation + from rpython.translator.interactive import Translation t = Translation(is_prime) t.annotate([int]) t.viewcg() diff --git a/lib_pypy/ctypes_config_cache/autopath.py b/lib_pypy/ctypes_config_cache/autopath.py deleted file mode 100644 --- a/lib_pypy/ctypes_config_cache/autopath.py +++ /dev/null @@ -1,131 +0,0 @@ -""" -self cloning, automatic path configuration - -copy this into any subdirectory of pypy from which scripts need -to be run, typically all of the test subdirs. -The idea is that any such script simply issues - - import autopath - -and this will make sure that the parent directory containing "pypy" -is in sys.path. - -If you modify the master "autopath.py" version (in pypy/tool/autopath.py) -you can directly run it which will copy itself on all autopath.py files -it finds under the pypy root directory. - -This module always provides these attributes: - - pypydir pypy root directory path - this_dir directory where this autopath.py resides - -""" - -def __dirinfo(part): - """ return (partdir, this_dir) and insert parent of partdir - into sys.path. If the parent directories don't have the part - an EnvironmentError is raised.""" - - import sys, os - try: - head = this_dir = os.path.realpath(os.path.dirname(__file__)) - except NameError: - head = this_dir = os.path.realpath(os.path.dirname(sys.argv[0])) - - error = None - while head: - partdir = head - head, tail = os.path.split(head) - if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') - if not os.path.exists(checkfile): - error = "Cannot find %r" % (os.path.normpath(checkfile),) - break - else: - error = "Cannot find the parent directory %r of the path %r" % ( - partdir, this_dir) - if not error: - # check for bogus end-of-line style (e.g. files checked out on - # Windows and moved to Unix) - f = open(__file__.replace('.pyc', '.py'), 'r') - data = f.read() - f.close() - if data.endswith('\r\n') or data.endswith('\r'): - error = ("Bad end-of-line style in the .py files. Typically " - "caused by a zip file or a checkout done on Windows and " - "moved to Unix or vice-versa.") - if error: - raise EnvironmentError("Invalid source tree - bogus checkout! " + - error) - - pypy_root = os.path.join(head, '') - try: - sys.path.remove(head) - except ValueError: - pass - sys.path.insert(0, head) - - munged = {} - for name, mod in sys.modules.items(): - if '.' in name: - continue - fn = getattr(mod, '__file__', None) - if not isinstance(fn, str): - continue - newname = os.path.splitext(os.path.basename(fn))[0] - if not newname.startswith(part + '.'): - continue - path = os.path.join(os.path.dirname(os.path.realpath(fn)), '') - if path.startswith(pypy_root) and newname != part: - modpaths = os.path.normpath(path[len(pypy_root):]).split(os.sep) - if newname != '__init__': - modpaths.append(newname) - modpath = '.'.join(modpaths) - if modpath not in sys.modules: - munged[modpath] = mod - - for name, mod in munged.iteritems(): - if name not in sys.modules: - sys.modules[name] = mod - if '.' in name: - prename = name[:name.rfind('.')] - postname = name[len(prename)+1:] - if prename not in sys.modules: - __import__(prename) - if not hasattr(sys.modules[prename], postname): - setattr(sys.modules[prename], postname, mod) - - return partdir, this_dir - -def __clone(): - """ clone master version of autopath.py into all subdirs """ - from os.path import join, walk - if not this_dir.endswith(join('pypy','tool')): - raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) - - - def sync_walker(arg, dirname, fnames): - if _myname in fnames: - fn = join(dirname, _myname) - f = open(fn, 'rwb+') - try: - if f.read() == arg: - print "checkok", fn - else: - print "syncing", fn - f = open(fn, 'w') - f.write(arg) - finally: - f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) - -_myname = 'autopath.py' - -# set guaranteed attributes - -pypydir, this_dir = __dirinfo('pypy') - -if __name__ == '__main__': - __clone() diff --git a/lib_pypy/ctypes_config_cache/dumpcache.py b/lib_pypy/ctypes_config_cache/dumpcache.py --- a/lib_pypy/ctypes_config_cache/dumpcache.py +++ b/lib_pypy/ctypes_config_cache/dumpcache.py @@ -1,6 +1,6 @@ import os from ctypes_configure import dumpcache -from pypy.jit.backend import detect_cpu +from rpython.jit.backend import detect_cpu def dumpcache2(basename, config): model = detect_cpu.autodetect_main_model_and_size() @@ -14,7 +14,7 @@ try: from __pypy__ import cpumodel except ImportError: - from pypy.jit.backend import detect_cpu + from rpython.jit.backend import detect_cpu cpumodel = detect_cpu.autodetect_main_model_and_size() # XXX relative import, should be removed together with # XXX the relative imports done e.g. by lib_pypy/pypy_test/test_hashlib diff --git a/lib_pypy/ctypes_config_cache/rebuild.py b/lib_pypy/ctypes_config_cache/rebuild.py --- a/lib_pypy/ctypes_config_cache/rebuild.py +++ b/lib_pypy/ctypes_config_cache/rebuild.py @@ -1,21 +1,15 @@ #! /usr/bin/env python # Run this script to rebuild all caches from the *.ctc.py files. -# hack: we cannot directly import autopath, as we are outside the pypy -# package. However, we pretend to be inside pypy/tool and manually run it, to -# get the correct path -import os.path -this_dir = os.path.dirname(__file__) -autopath_py = os.path.join(this_dir, '../../pypy/tool/autopath.py') -autopath_py = os.path.abspath(autopath_py) -execfile(autopath_py, dict(__name__='autopath', __file__=autopath_py)) +import os, sys -import os, sys +sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..'))) + import py _dirpath = os.path.dirname(__file__) or os.curdir -from pypy.tool.ansi_print import ansi_log +from rpython.tool.ansi_print import ansi_log log = py.log.Producer("ctypes_config_cache") py.log.setconsumer("ctypes_config_cache", ansi_log) @@ -31,7 +25,7 @@ sys.path[:] = path def try_rebuild(): - from pypy.jit.backend import detect_cpu + from rpython.jit.backend import detect_cpu model = detect_cpu.autodetect_main_model_and_size() # remove the files '_*_model_.py' left = {} diff --git a/lib_pypy/ctypes_config_cache/test/test_cache.py b/lib_pypy/ctypes_config_cache/test/test_cache.py --- a/lib_pypy/ctypes_config_cache/test/test_cache.py +++ b/lib_pypy/ctypes_config_cache/test/test_cache.py @@ -1,6 +1,6 @@ import py import sys, os -from pypy.tool.udir import udir +from rpython.tool.udir import udir dirpath = py.path.local(__file__).dirpath().dirpath() diff --git a/lib_pypy/pypy_test/test_dbm_extra.py b/lib_pypy/pypy_test/test_dbm_extra.py --- a/lib_pypy/pypy_test/test_dbm_extra.py +++ b/lib_pypy/pypy_test/test_dbm_extra.py @@ -1,6 +1,6 @@ from __future__ import absolute_import import py -from pypy.tool.udir import udir +from rpython.tool.udir import udir try: from lib_pypy import dbm except ImportError, e: diff --git a/lib_pypy/pypy_test/test_marshal_extra.py b/lib_pypy/pypy_test/test_marshal_extra.py --- a/lib_pypy/pypy_test/test_marshal_extra.py +++ b/lib_pypy/pypy_test/test_marshal_extra.py @@ -4,7 +4,7 @@ import marshal as cpy_marshal from lib_pypy import _marshal as marshal -from pypy.tool.udir import udir +from rpython.tool.udir import udir hello = "he" hello += "llo" diff --git a/pypy/annotation/annrpython.py b/pypy/annotation/annrpython.py deleted file mode 100644 --- a/pypy/annotation/annrpython.py +++ /dev/null @@ -1,676 +0,0 @@ -from __future__ import absolute_import - -import types -from pypy.tool.ansi_print import ansi_log -from pypy.tool.pairtype import pair -from pypy.tool.error import (format_blocked_annotation_error, - AnnotatorError, gather_error, ErrorWrapper) -from pypy.objspace.flow.model import (Variable, Constant, FunctionGraph, - c_last_exception, checkgraph) -from pypy.translator import simplify, transform -from pypy.annotation import model as annmodel, signature, unaryop, binaryop -from pypy.annotation.bookkeeper import Bookkeeper -import py -log = py.log.Producer("annrpython") -py.log.setconsumer("annrpython", ansi_log) - - -FAIL = object() - -class RPythonAnnotator(object): - """Block annotator for RPython. - See description in doc/translation.txt.""" - - def __init__(self, translator=None, policy=None, bookkeeper=None): - import pypy.rpython.ootypesystem.ooregistry # has side effects - import pypy.rpython.extfuncregistry # has side effects - import pypy.rlib.nonconst # has side effects - - if translator is None: - # interface for tests - from pypy.translator.translator import TranslationContext - translator = TranslationContext() - translator.annotator = self - self.translator = translator - self.pendingblocks = {} # map {block: graph-containing-it} - self.bindings = {} # map Variables to SomeValues - self.annotated = {} # set of blocks already seen - self.added_blocks = None # see processblock() below - self.links_followed = {} # set of links that have ever been followed - self.notify = {} # {block: {positions-to-reflow-from-when-done}} - self.fixed_graphs = {} # set of graphs not to annotate again - self.blocked_blocks = {} # set of {blocked_block: (graph, index)} - # --- the following information is recorded for debugging --- - self.blocked_graphs = {} # set of graphs that have blocked blocks - # --- end of debugging information --- - self.frozen = False - if policy is None: - from pypy.annotation.policy import AnnotatorPolicy - self.policy = AnnotatorPolicy() - else: - self.policy = policy - if bookkeeper is None: - bookkeeper = Bookkeeper(self) - self.bookkeeper = bookkeeper - - def __getstate__(self): - attrs = """translator pendingblocks bindings annotated links_followed - notify bookkeeper frozen policy added_blocks""".split() - ret = self.__dict__.copy() - for key, value in ret.items(): - if key not in attrs: - assert type(value) is dict, ( - "%r is not dict. please update %s.__getstate__" % - (key, self.__class__.__name__)) - ret[key] = {} - return ret - - #___ convenience high-level interface __________________ - - def build_types(self, function, input_arg_types, complete_now=True, - main_entry_point=False): - """Recursively build annotations about the specific entry point.""" - assert isinstance(function, types.FunctionType), "fix that!" - - from pypy.annotation.policy import AnnotatorPolicy - policy = AnnotatorPolicy() - # make input arguments and set their type - args_s = [self.typeannotation(t) for t in input_arg_types] - - # XXX hack - annmodel.TLS.check_str_without_nul = ( - self.translator.config.translation.check_str_without_nul) - - flowgraph, inputcells = self.get_call_parameters(function, args_s, policy) - if not isinstance(flowgraph, FunctionGraph): - assert isinstance(flowgraph, annmodel.SomeObject) - return flowgraph - - if main_entry_point: - self.translator.entry_point_graph = flowgraph - return self.build_graph_types(flowgraph, inputcells, complete_now=complete_now) - - def get_call_parameters(self, function, args_s, policy): - desc = self.bookkeeper.getdesc(function) - args = self.bookkeeper.build_args("simple_call", args_s[:]) - result = [] - def schedule(graph, inputcells): - result.append((graph, inputcells)) - return annmodel.s_ImpossibleValue - - prevpolicy = self.policy - self.policy = policy - self.bookkeeper.enter(None) - try: - desc.pycall(schedule, args, annmodel.s_ImpossibleValue) - finally: - self.bookkeeper.leave() - self.policy = prevpolicy - [(graph, inputcells)] = result - return graph, inputcells - - def annotate_helper(self, function, args_s, policy=None): - if policy is None: - from pypy.annotation.policy import AnnotatorPolicy - policy = AnnotatorPolicy() - graph, inputcells = self.get_call_parameters(function, args_s, policy) - self.build_graph_types(graph, inputcells, complete_now=False) - self.complete_helpers(policy) - return graph - - def complete_helpers(self, policy): - saved = self.policy, self.added_blocks - self.policy = policy - try: - self.added_blocks = {} - self.complete() - # invoke annotation simplifications for the new blocks - self.simplify(block_subset=self.added_blocks) - finally: - self.policy, self.added_blocks = saved - - def build_graph_types(self, flowgraph, inputcells, complete_now=True): - checkgraph(flowgraph) - - nbarg = len(flowgraph.getargs()) - if len(inputcells) != nbarg: - raise TypeError("%s expects %d args, got %d" %( - flowgraph, nbarg, len(inputcells))) - - # register the entry point - self.addpendinggraph(flowgraph, inputcells) - # recursively proceed until no more pending block is left - if complete_now: - self.complete() - return self.binding(flowgraph.getreturnvar(), None) - - def gettype(self, variable): - """Return the known type of a control flow graph variable, - defaulting to 'object'.""" - if isinstance(variable, Constant): - return type(variable.value) - elif isinstance(variable, Variable): - cell = self.bindings.get(variable) - if cell: - return cell.knowntype - else: - return object - else: - raise TypeError, ("Variable or Constant instance expected, " - "got %r" % (variable,)) - - def getuserclassdefinitions(self): - """Return a list of ClassDefs.""" - return self.bookkeeper.classdefs - - #___ medium-level interface ____________________________ - - def addpendinggraph(self, flowgraph, inputcells): - self.addpendingblock(flowgraph, flowgraph.startblock, inputcells) - - def addpendingblock(self, graph, block, cells): - """Register an entry point into block with the given input cells.""" - if graph in self.fixed_graphs: - # special case for annotating/rtyping in several phases: calling - # a graph that has already been rtyped. Safety-check the new - # annotations that are passed in, and don't annotate the old - # graph -- it's already low-level operations! - for a, s_newarg in zip(graph.getargs(), cells): - s_oldarg = self.binding(a) - assert annmodel.unionof(s_oldarg, s_newarg) == s_oldarg - else: - assert not self.frozen - for a in cells: - assert isinstance(a, annmodel.SomeObject) - if block not in self.annotated: - self.bindinputargs(graph, block, cells) - else: - self.mergeinputargs(graph, block, cells) - if not self.annotated[block]: - self.pendingblocks[block] = graph - - def complete(self): - """Process pending blocks until none is left.""" - while True: - while self.pendingblocks: - block, graph = self.pendingblocks.popitem() - self.processblock(graph, block) - self.policy.no_more_blocks_to_annotate(self) - if not self.pendingblocks: - break # finished - # make sure that the return variables of all graphs is annotated - if self.added_blocks is not None: - newgraphs = [self.annotated[block] for block in self.added_blocks] - newgraphs = dict.fromkeys(newgraphs) - got_blocked_blocks = False in newgraphs - else: - newgraphs = self.translator.graphs #all of them - got_blocked_blocks = False in self.annotated.values() - if got_blocked_blocks: - for graph in self.blocked_graphs.values(): - self.blocked_graphs[graph] = True - - blocked_blocks = [block for block, done in self.annotated.items() - if done is False] - assert len(blocked_blocks) == len(self.blocked_blocks) - - text = format_blocked_annotation_error(self, self.blocked_blocks) - #raise SystemExit() - raise AnnotatorError(text) - for graph in newgraphs: - v = graph.getreturnvar() - if v not in self.bindings: - self.setbinding(v, annmodel.s_ImpossibleValue) - # policy-dependent computation - self.bookkeeper.compute_at_fixpoint() - - def binding(self, arg, default=FAIL): - "Gives the SomeValue corresponding to the given Variable or Constant." - if isinstance(arg, Variable): - try: - return self.bindings[arg] - except KeyError: - if default is not FAIL: - return default - else: - raise - elif isinstance(arg, Constant): - #if arg.value is undefined_value: # undefined local variables - # return annmodel.s_ImpossibleValue - return self.bookkeeper.immutableconstant(arg) - else: - raise TypeError, 'Variable or Constant expected, got %r' % (arg,) - - def typeannotation(self, t): - return signature.annotation(t, self.bookkeeper) - - def setbinding(self, arg, s_value): - if arg in self.bindings: - assert s_value.contains(self.bindings[arg]) - self.bindings[arg] = s_value - - def transfer_binding(self, v_target, v_source): - assert v_source in self.bindings - self.bindings[v_target] = self.bindings[v_source] - - def warning(self, msg, pos=None): - if pos is None: - try: - pos = self.bookkeeper.position_key - except AttributeError: - pos = '?' - if pos != '?': - pos = self.whereami(pos) - - log.WARNING("%s/ %s" % (pos, msg)) - - - #___ interface for annotator.bookkeeper _______ - - def recursivecall(self, graph, whence, inputcells): - if isinstance(whence, tuple): - parent_graph, parent_block, parent_index = whence - tag = parent_block, parent_index - self.translator.update_call_graph(parent_graph, graph, tag) - # self.notify[graph.returnblock] is a dictionary of call - # points to this func which triggers a reflow whenever the - # return block of this graph has been analysed. - callpositions = self.notify.setdefault(graph.returnblock, {}) - if whence is not None: - if callable(whence): - def callback(): - whence(self, graph) - else: - callback = whence - callpositions[callback] = True - - # generalize the function's input arguments - self.addpendingblock(graph, graph.startblock, inputcells) - - # get the (current) return value - v = graph.getreturnvar() - try: - return self.bindings[v] - except KeyError: - # the function didn't reach any return statement so far. - # (some functions actually never do, they always raise exceptions) - return annmodel.s_ImpossibleValue - - def reflowfromposition(self, position_key): - graph, block, index = position_key - self.reflowpendingblock(graph, block) - - - #___ simplification (should be moved elsewhere?) _______ - - def simplify(self, block_subset=None, extra_passes=None): - # Generic simplifications - transform.transform_graph(self, block_subset=block_subset, - extra_passes=extra_passes) - if block_subset is None: - graphs = self.translator.graphs - else: - graphs = {} - for block in block_subset: - graph = self.annotated.get(block) - if graph: - graphs[graph] = True - for graph in graphs: - simplify.eliminate_empty_blocks(graph) - - - #___ flowing annotations in blocks _____________________ - - def processblock(self, graph, block): - # Important: this is not called recursively. - # self.flowin() can only issue calls to self.addpendingblock(). - # The analysis of a block can be in three states: - # * block not in self.annotated: - # never seen the block. - # * self.annotated[block] == False: - # the input variables of the block are in self.bindings but we - # still have to consider all the operations in the block. - # * self.annotated[block] == graph-containing-block: - # analysis done (at least until we find we must generalize the - # input variables). - - #print '* processblock', block, cells - self.annotated[block] = graph - if block in self.blocked_blocks: - del self.blocked_blocks[block] - try: - self.flowin(graph, block) - except BlockedInference, e: - self.annotated[block] = False # failed, hopefully temporarily - self.blocked_blocks[block] = (graph, e.opindex) - except Exception, e: - # hack for debug tools only - if not hasattr(e, '__annotator_block'): - setattr(e, '__annotator_block', block) - raise - - # The dict 'added_blocks' is used by rpython.annlowlevel to - # detect which are the new blocks that annotating an additional - # small helper creates. - if self.added_blocks is not None: - self.added_blocks[block] = True - - def reflowpendingblock(self, graph, block): - assert not self.frozen - assert graph not in self.fixed_graphs - self.pendingblocks[block] = graph - assert block in self.annotated - self.annotated[block] = False # must re-flow - self.blocked_blocks[block] = (graph, None) - - def bindinputargs(self, graph, block, inputcells): - # Create the initial bindings for the input args of a block. - assert len(block.inputargs) == len(inputcells) - for a, cell in zip(block.inputargs, inputcells): - self.setbinding(a, cell) - self.annotated[block] = False # must flowin. - self.blocked_blocks[block] = (graph, None) - - def mergeinputargs(self, graph, block, inputcells): - # Merge the new 'cells' with each of the block's existing input - # variables. - oldcells = [self.binding(a) for a in block.inputargs] - try: - unions = [annmodel.unionof(c1,c2) for c1, c2 in zip(oldcells,inputcells)] - except annmodel.UnionError, e: - e.args = e.args + ( - ErrorWrapper(gather_error(self, graph, block, None)),) - raise - # if the merged cells changed, we must redo the analysis - if unions != oldcells: - self.bindinputargs(graph, block, unions) - - def whereami(self, position_key): - graph, block, i = position_key - blk = "" - if block: - at = block.at() - if at: - blk = " block"+at - opid="" - if i is not None: - opid = " op=%d" % i - return repr(graph) + blk + opid - - def flowin(self, graph, block): - #print 'Flowing', block, [self.binding(a) for a in block.inputargs] - try: - for i in range(len(block.operations)): - try: - self.bookkeeper.enter((graph, block, i)) - self.consider_op(block, i) - finally: - self.bookkeeper.leave() - - except BlockedInference, e: - if (e.op is block.operations[-1] and - block.exitswitch == c_last_exception): - # this is the case where the last operation of the block will - # always raise an exception which is immediately caught by - # an exception handler. We then only follow the exceptional - # branches. - exits = [link for link in block.exits - if link.exitcase is not None] - - elif e.op.opname in ('simple_call', 'call_args', 'next'): - # XXX warning, keep the name of the call operations in sync - # with the flow object space. These are the operations for - # which it is fine to always raise an exception. We then - # swallow the BlockedInference and that's it. - # About 'next': see test_annotate_iter_empty_container(). - return - - else: - # other cases are problematic (but will hopefully be solved - # later by reflowing). Throw the BlockedInference up to - # processblock(). - raise - - except annmodel.HarmlesslyBlocked: - return - - else: - # dead code removal: don't follow all exits if the exitswitch - # is known - exits = block.exits - if isinstance(block.exitswitch, Variable): - s_exitswitch = self.bindings[block.exitswitch] - if s_exitswitch.is_constant(): - exits = [link for link in exits - if link.exitcase == s_exitswitch.const] - - # mapping (exitcase, variable) -> s_annotation - # that can be attached to booleans, exitswitches - knowntypedata = getattr(self.bindings.get(block.exitswitch), - "knowntypedata", {}) - - # filter out those exceptions which cannot - # occour for this specific, typed operation. - if block.exitswitch == c_last_exception: - op = block.operations[-1] - if op.opname in binaryop.BINARY_OPERATIONS: - arg1 = self.binding(op.args[0]) - arg2 = self.binding(op.args[1]) - binop = getattr(pair(arg1, arg2), op.opname, None) - can_only_throw = annmodel.read_can_only_throw(binop, arg1, arg2) - elif op.opname in unaryop.UNARY_OPERATIONS: - arg1 = self.binding(op.args[0]) - opname = op.opname - if opname == 'contains': opname = 'op_contains' - unop = getattr(arg1, opname, None) - can_only_throw = annmodel.read_can_only_throw(unop, arg1) - else: - can_only_throw = None - - if can_only_throw is not None: - candidates = can_only_throw - candidate_exits = exits - exits = [] - for link in candidate_exits: - case = link.exitcase - if case is None: - exits.append(link) - continue - covered = [c for c in candidates if issubclass(c, case)] - if covered: - exits.append(link) - candidates = [c for c in candidates if c not in covered] - - for link in exits: - in_except_block = False - - last_exception_var = link.last_exception # may be None for non-exception link - last_exc_value_var = link.last_exc_value # may be None for non-exception link - - if isinstance(link.exitcase, (types.ClassType, type)) \ - and issubclass(link.exitcase, py.builtin.BaseException): - assert last_exception_var and last_exc_value_var - last_exc_value_object = self.bookkeeper.valueoftype(link.exitcase) - last_exception_object = annmodel.SomeType() - if isinstance(last_exception_var, Constant): - last_exception_object.const = last_exception_var.value - last_exception_object.is_type_of = [last_exc_value_var] - - if isinstance(last_exception_var, Variable): - self.setbinding(last_exception_var, last_exception_object) - if isinstance(last_exc_value_var, Variable): - self.setbinding(last_exc_value_var, last_exc_value_object) - - last_exception_object = annmodel.SomeType() - if isinstance(last_exception_var, Constant): - last_exception_object.const = last_exception_var.value - #if link.exitcase is Exception: - # last_exc_value_object = annmodel.SomeObject() - #else: - last_exc_value_vars = [] - in_except_block = True - - ignore_link = False - cells = [] - renaming = {} - for a,v in zip(link.args,link.target.inputargs): - renaming.setdefault(a, []).append(v) - for a,v in zip(link.args,link.target.inputargs): - if a == last_exception_var: - assert in_except_block - cells.append(last_exception_object) - elif a == last_exc_value_var: - assert in_except_block - cells.append(last_exc_value_object) - last_exc_value_vars.append(v) - else: - cell = self.binding(a) - if (link.exitcase, a) in knowntypedata: - knownvarvalue = knowntypedata[(link.exitcase, a)] - cell = pair(cell, knownvarvalue).improve() - # ignore links that try to pass impossible values - if cell == annmodel.s_ImpossibleValue: - ignore_link = True - - if hasattr(cell,'is_type_of'): - renamed_is_type_of = [] - for v in cell.is_type_of: - new_vs = renaming.get(v,[]) - renamed_is_type_of += new_vs - assert cell.knowntype is type - newcell = annmodel.SomeType() - if cell.is_constant(): - newcell.const = cell.const - cell = newcell - cell.is_type_of = renamed_is_type_of - - if hasattr(cell, 'knowntypedata'): - renamed_knowntypedata = {} - for (value, v), s in cell.knowntypedata.items(): - new_vs = renaming.get(v, []) - for new_v in new_vs: - renamed_knowntypedata[value, new_v] = s - assert isinstance(cell, annmodel.SomeBool) - newcell = annmodel.SomeBool() - if cell.is_constant(): - newcell.const = cell.const - cell = newcell - cell.set_knowntypedata(renamed_knowntypedata) - - cells.append(cell) - - if ignore_link: - continue - - if in_except_block: - last_exception_object.is_type_of = last_exc_value_vars - - self.links_followed[link] = True - self.addpendingblock(graph, link.target, cells) - - if block in self.notify: - # reflow from certain positions when this block is done - for callback in self.notify[block]: - if isinstance(callback, tuple): - self.reflowfromposition(callback) # callback is a position - else: - callback() - - - #___ creating the annotations based on operations ______ - - def consider_op(self, block, opindex): - op = block.operations[opindex] - argcells = [self.binding(a) for a in op.args] - consider_meth = getattr(self,'consider_op_'+op.opname, - None) - if not consider_meth: - raise Exception,"unknown op: %r" % op - - # let's be careful about avoiding propagated SomeImpossibleValues - # to enter an op; the latter can result in violations of the - # more general results invariant: e.g. if SomeImpossibleValue enters is_ - # is_(SomeImpossibleValue, None) -> SomeBool - # is_(SomeInstance(not None), None) -> SomeBool(const=False) ... - # boom -- in the assert of setbinding() - for arg in argcells: - if isinstance(arg, annmodel.SomeImpossibleValue): - raise BlockedInference(self, op, opindex) - try: - resultcell = consider_meth(*argcells) - except Exception, e: - graph = self.bookkeeper.position_key[0] - e.args = e.args + ( - ErrorWrapper(gather_error(self, graph, block, opindex)),) - raise - if resultcell is None: - resultcell = self.noreturnvalue(op) - elif resultcell == annmodel.s_ImpossibleValue: - raise BlockedInference(self, op, opindex) # the operation cannot succeed - assert isinstance(resultcell, annmodel.SomeObject) - assert isinstance(op.result, Variable) - self.setbinding(op.result, resultcell) # bind resultcell to op.result - - def noreturnvalue(self, op): - return annmodel.s_ImpossibleValue # no return value (hook method) - - # XXX "contains" clash with SomeObject method - def consider_op_contains(self, seq, elem): - self.bookkeeper.count("contains", seq) - return seq.op_contains(elem) - - def consider_op_newtuple(self, *args): - return annmodel.SomeTuple(items = args) - - def consider_op_newlist(self, *args): - return self.bookkeeper.newlist(*args) - - def consider_op_newdict(self): - return self.bookkeeper.newdict() - - - def _registeroperations(cls, unary_ops, binary_ops): - # All unary operations - d = {} - for opname in unary_ops: - fnname = 'consider_op_' + opname - exec py.code.Source(""" -def consider_op_%s(self, arg, *args): - return arg.%s(*args) -""" % (opname, opname)).compile() in globals(), d - setattr(cls, fnname, d[fnname]) - # All binary operations - for opname in binary_ops: - fnname = 'consider_op_' + opname - exec py.code.Source(""" -def consider_op_%s(self, arg1, arg2, *args): - return pair(arg1,arg2).%s(*args) -""" % (opname, opname)).compile() in globals(), d - setattr(cls, fnname, d[fnname]) - _registeroperations = classmethod(_registeroperations) - -# register simple operations handling -RPythonAnnotator._registeroperations(unaryop.UNARY_OPERATIONS, binaryop.BINARY_OPERATIONS) - - -class BlockedInference(Exception): - """This exception signals the type inference engine that the situation - is currently blocked, and that it should try to progress elsewhere.""" - - def __init__(self, annotator, op, opindex): - self.annotator = annotator - try: - self.break_at = annotator.bookkeeper.position_key - except AttributeError: - self.break_at = None - self.op = op - self.opindex = opindex - - def __repr__(self): - if not self.break_at: - break_at = "?" - else: - break_at = self.annotator.whereami(self.break_at) - return "" %(break_at, self.op) - - __str__ = __repr__ diff --git a/pypy/annotation/binaryop.py b/pypy/annotation/binaryop.py deleted file mode 100644 --- a/pypy/annotation/binaryop.py +++ /dev/null @@ -1,1052 +0,0 @@ -""" -Binary operations between SomeValues. -""" - -import py -import operator -from pypy.tool.pairtype import pair, pairtype -from pypy.annotation.model import SomeObject, SomeInteger, SomeBool, s_Bool -from pypy.annotation.model import SomeString, SomeChar, SomeList, SomeDict -from pypy.annotation.model import SomeUnicodeCodePoint, SomeUnicodeString -from pypy.annotation.model import SomeTuple, SomeImpossibleValue, s_ImpossibleValue -from pypy.annotation.model import SomeInstance, SomeBuiltin, SomeIterator -from pypy.annotation.model import SomePBC, SomeFloat, s_None, SomeByteArray -from pypy.annotation.model import SomeExternalObject, SomeWeakRef -from pypy.annotation.model import SomeAddress, SomeTypedAddressAccess -from pypy.annotation.model import SomeSingleFloat, SomeLongFloat, SomeType -from pypy.annotation.model import unionof, UnionError, missing_operation -from pypy.annotation.model import TLS -from pypy.annotation.model import read_can_only_throw -from pypy.annotation.model import add_knowntypedata, merge_knowntypedata -from pypy.annotation.model import SomeGenericCallable -from pypy.annotation.bookkeeper import getbookkeeper -from pypy.objspace.flow.model import Variable, Constant -from pypy.rlib import rarithmetic -from pypy.tool.error import AnnotatorError - -# convenience only! -def immutablevalue(x): - return getbookkeeper().immutablevalue(x) - -# XXX unify this with ObjSpace.MethodTable -BINARY_OPERATIONS = set(['add', 'sub', 'mul', 'div', 'mod', - 'truediv', 'floordiv', 'divmod', - 'and_', 'or_', 'xor', - 'lshift', 'rshift', - 'getitem', 'setitem', 'delitem', - 'getitem_idx', 'getitem_key', 'getitem_idx_key', - 'inplace_add', 'inplace_sub', 'inplace_mul', - 'inplace_truediv', 'inplace_floordiv', 'inplace_div', - 'inplace_mod', - 'inplace_lshift', 'inplace_rshift', - 'inplace_and', 'inplace_or', 'inplace_xor', - 'lt', 'le', 'eq', 'ne', 'gt', 'ge', 'is_', 'cmp', - 'coerce', - ] - +[opname+'_ovf' for opname in - """add sub mul floordiv div mod lshift - """.split() - ]) - -for opname in BINARY_OPERATIONS: - missing_operation(pairtype(SomeObject, SomeObject), opname) - -class __extend__(pairtype(SomeObject, SomeObject)): - - def union((obj1, obj2)): - raise UnionError(obj1, obj2) - - # inplace_xxx ---> xxx by default - def inplace_add((obj1, obj2)): return pair(obj1, obj2).add() - def inplace_sub((obj1, obj2)): return pair(obj1, obj2).sub() - def inplace_mul((obj1, obj2)): return pair(obj1, obj2).mul() - def inplace_truediv((obj1, obj2)): return pair(obj1, obj2).truediv() - def inplace_floordiv((obj1, obj2)): return pair(obj1, obj2).floordiv() - def inplace_div((obj1, obj2)): return pair(obj1, obj2).div() - def inplace_mod((obj1, obj2)): return pair(obj1, obj2).mod() - def inplace_lshift((obj1, obj2)): return pair(obj1, obj2).lshift() - def inplace_rshift((obj1, obj2)): return pair(obj1, obj2).rshift() - def inplace_and((obj1, obj2)): return pair(obj1, obj2).and_() - def inplace_or((obj1, obj2)): return pair(obj1, obj2).or_() - def inplace_xor((obj1, obj2)): return pair(obj1, obj2).xor() - - for name, func in locals().items(): - if name.startswith('inplace_'): - func.can_only_throw = [] - - inplace_div.can_only_throw = [ZeroDivisionError] - inplace_truediv.can_only_throw = [ZeroDivisionError] - inplace_floordiv.can_only_throw = [ZeroDivisionError] - inplace_mod.can_only_throw = [ZeroDivisionError] - - def lt((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const < obj2.const) - else: - getbookkeeper().count("non_int_comp", obj1, obj2) - return s_Bool - - def le((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const <= obj2.const) - else: - getbookkeeper().count("non_int_comp", obj1, obj2) - return s_Bool - - def eq((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const == obj2.const) - else: - getbookkeeper().count("non_int_eq", obj1, obj2) - return s_Bool - - def ne((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const != obj2.const) - else: - getbookkeeper().count("non_int_eq", obj1, obj2) - return s_Bool - - def gt((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const > obj2.const) - else: - getbookkeeper().count("non_int_comp", obj1, obj2) - return s_Bool - - def ge((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const >= obj2.const) - else: - getbookkeeper().count("non_int_comp", obj1, obj2) - return s_Bool - - def cmp((obj1, obj2)): - getbookkeeper().count("cmp", obj1, obj2) - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(cmp(obj1.const, obj2.const)) - else: - return SomeInteger() - - def is_((obj1, obj2)): - r = SomeBool() - if obj2.is_constant(): - if obj1.is_constant(): - r.const = obj1.const is obj2.const - if obj2.const is None and not obj1.can_be_none(): - r.const = False - elif obj1.is_constant(): - if obj1.const is None and not obj2.can_be_none(): - r.const = False - # XXX HACK HACK HACK - # XXX HACK HACK HACK - # XXX HACK HACK HACK - bk = getbookkeeper() - if bk is not None: # for testing - knowntypedata = {} - fn, block, i = bk.position_key - - annotator = bk.annotator - op = block.operations[i] - assert op.opname == "is_" - assert len(op.args) == 2 - - def bind(src_obj, tgt_obj, tgt_arg): - if hasattr(tgt_obj, 'is_type_of') and src_obj.is_constant(): - add_knowntypedata(knowntypedata, True, tgt_obj.is_type_of, - bk.valueoftype(src_obj.const)) - - assert annotator.binding(op.args[tgt_arg]) == tgt_obj - add_knowntypedata(knowntypedata, True, [op.args[tgt_arg]], src_obj) - - nonnone_obj = tgt_obj - if src_obj.is_constant() and src_obj.const is None and tgt_obj.can_be_none(): - nonnone_obj = tgt_obj.nonnoneify() - - add_knowntypedata(knowntypedata, False, [op.args[tgt_arg]], nonnone_obj) - - bind(obj2, obj1, 0) - bind(obj1, obj2, 1) - r.set_knowntypedata(knowntypedata) - - return r - - def divmod((obj1, obj2)): - getbookkeeper().count("divmod", obj1, obj2) - return SomeTuple([pair(obj1, obj2).div(), pair(obj1, obj2).mod()]) - - def coerce((obj1, obj2)): - getbookkeeper().count("coerce", obj1, obj2) - return pair(obj1, obj2).union() # reasonable enough - - # approximation of an annotation intersection, the result should be the annotation obj or - # the intersection of obj and improvement - def improve((obj, improvement)): - if not improvement.contains(obj) and obj.contains(improvement): - return improvement - else: - return obj - - # checked getitems - - def _getitem_can_only_throw(s_c1, s_o2): - impl = pair(s_c1, s_o2).getitem - return read_can_only_throw(impl, s_c1, s_o2) - - def getitem_idx_key((s_c1, s_o2)): - impl = pair(s_c1, s_o2).getitem - return impl() - getitem_idx_key.can_only_throw = _getitem_can_only_throw - - getitem_idx = getitem_idx_key - getitem_key = getitem_idx_key - - -class __extend__(pairtype(SomeType, SomeType)): - - def union((obj1, obj2)): - result = SomeType() - is_type_of1 = getattr(obj1, 'is_type_of', None) - is_type_of2 = getattr(obj2, 'is_type_of', None) - if obj1.is_immutable_constant() and obj2.is_immutable_constant() and obj1.const == obj2.const: - result.const = obj1.const - is_type_of = {} - if is_type_of1: - for v in is_type_of1: - is_type_of[v] = True - if is_type_of2: - for v in is_type_of2: - is_type_of[v] = True - if is_type_of: - result.is_type_of = is_type_of.keys() - else: - if is_type_of1 and is_type_of1 == is_type_of2: - result.is_type_of = is_type_of1 - return result - - -# cloning a function with identical code, for the can_only_throw attribute -def _clone(f, can_only_throw = None): - newfunc = type(f)(f.func_code, f.func_globals, f.func_name, - f.func_defaults, f.func_closure) - if can_only_throw is not None: - newfunc.can_only_throw = can_only_throw - return newfunc - -class __extend__(pairtype(SomeInteger, SomeInteger)): - # unsignedness is considered a rare and contagious disease - - def union((int1, int2)): - if int1.unsigned == int2.unsigned: - knowntype = rarithmetic.compute_restype(int1.knowntype, int2.knowntype) - else: - t1 = int1.knowntype - if t1 is bool: - t1 = int - t2 = int2.knowntype - if t2 is bool: - t2 = int - - if t2 is int: - if int2.nonneg == False: - raise UnionError, "Merging %s and a possibly negative int is not allowed" % t1 - knowntype = t1 - elif t1 is int: - if int1.nonneg == False: - raise UnionError, "Merging %s and a possibly negative int is not allowed" % t2 - knowntype = t2 - else: - raise UnionError, "Merging these types (%s, %s) is not supported" % (t1, t2) - return SomeInteger(nonneg=int1.nonneg and int2.nonneg, - knowntype=knowntype) - - or_ = xor = add = mul = _clone(union, []) - add_ovf = mul_ovf = _clone(union, [OverflowError]) - div = floordiv = mod = _clone(union, [ZeroDivisionError]) - div_ovf= floordiv_ovf = mod_ovf = _clone(union, [ZeroDivisionError, OverflowError]) - - def truediv((int1, int2)): - return SomeFloat() - truediv.can_only_throw = [ZeroDivisionError] - truediv_ovf = _clone(truediv, [ZeroDivisionError, OverflowError]) - - inplace_div = div - inplace_truediv = truediv - - def sub((int1, int2)): - knowntype = rarithmetic.compute_restype(int1.knowntype, int2.knowntype) - return SomeInteger(knowntype=knowntype) - sub.can_only_throw = [] - sub_ovf = _clone(sub, [OverflowError]) - - def and_((int1, int2)): - knowntype = rarithmetic.compute_restype(int1.knowntype, int2.knowntype) - return SomeInteger(nonneg=int1.nonneg or int2.nonneg, - knowntype=knowntype) - and_.can_only_throw = [] - - def lshift((int1, int2)): - if isinstance(int1, SomeBool): - return SomeInteger() - else: - return SomeInteger(knowntype=int1.knowntype) - lshift.can_only_throw = [] - lshift_ovf = _clone(lshift, [OverflowError]) - - def rshift((int1, int2)): - if isinstance(int1, SomeBool): - return SomeInteger(nonneg=True) - else: - return SomeInteger(nonneg=int1.nonneg, knowntype=int1.knowntype) - rshift.can_only_throw = [] - - def _compare_helper((int1, int2), opname, operation): - r = SomeBool() - if int1.is_immutable_constant() and int2.is_immutable_constant(): - r.const = operation(int1.const, int2.const) - # - # The rest of the code propagates nonneg information between - # the two arguments. - # - # Doing the right thing when int1 or int2 change from signed - # to unsigned (r_uint) is almost impossible. See test_intcmp_bug. - # Instead, we only deduce constrains on the operands in the - # case where they are both signed. In other words, if y is - # nonneg then "assert x>=y" will let the annotator know that - # x is nonneg too, but it will not work if y is unsigned. - # - if not (rarithmetic.signedtype(int1.knowntype) and - rarithmetic.signedtype(int2.knowntype)): - return r - knowntypedata = {} - # XXX HACK HACK HACK - fn, block, i = getbookkeeper().position_key - op = block.operations[i] - assert op.opname == opname - assert len(op.args) == 2 - def tointtype(int0): - if int0.knowntype is bool: - return int - return int0.knowntype - if int1.nonneg and isinstance(op.args[1], Variable): - case = opname in ('lt', 'le', 'eq') - - add_knowntypedata(knowntypedata, case, [op.args[1]], - SomeInteger(nonneg=True, knowntype=tointtype(int2))) - if int2.nonneg and isinstance(op.args[0], Variable): - case = opname in ('gt', 'ge', 'eq') - add_knowntypedata(knowntypedata, case, [op.args[0]], - SomeInteger(nonneg=True, knowntype=tointtype(int1))) - r.set_knowntypedata(knowntypedata) - # a special case for 'x < 0' or 'x >= 0', - # where 0 is a flow graph Constant - # (in this case we are sure that it cannot become a r_uint later) - if (isinstance(op.args[1], Constant) and - type(op.args[1].value) is int and # filter out Symbolics - op.args[1].value == 0): - if int1.nonneg: - if opname == 'lt': - r.const = False - if opname == 'ge': - r.const = True - return r - - def lt(intint): return intint._compare_helper('lt', operator.lt) - def le(intint): return intint._compare_helper('le', operator.le) - def eq(intint): return intint._compare_helper('eq', operator.eq) - def ne(intint): return intint._compare_helper('ne', operator.ne) - def gt(intint): return intint._compare_helper('gt', operator.gt) - def ge(intint): return intint._compare_helper('ge', operator.ge) - - -class __extend__(pairtype(SomeBool, SomeBool)): - - def union((boo1, boo2)): - s = SomeBool() - if getattr(boo1, 'const', -1) == getattr(boo2, 'const', -2): - s.const = boo1.const - if hasattr(boo1, 'knowntypedata') and \ - hasattr(boo2, 'knowntypedata'): - ktd = merge_knowntypedata(boo1.knowntypedata, boo2.knowntypedata) - s.set_knowntypedata(ktd) - return s - - def and_((boo1, boo2)): - s = SomeBool() - if boo1.is_constant(): - if not boo1.const: - s.const = False - else: - return boo2 - if boo2.is_constant(): - if not boo2.const: - s.const = False - return s - - def or_((boo1, boo2)): - s = SomeBool() - if boo1.is_constant(): - if boo1.const: - s.const = True - else: - return boo2 - if boo2.is_constant(): - if boo2.const: - s.const = True - return s - - def xor((boo1, boo2)): - s = SomeBool() - if boo1.is_constant() and boo2.is_constant(): - s.const = boo1.const ^ boo2.const - return s - -class __extend__(pairtype(SomeString, SomeString)): - - def union((str1, str2)): - can_be_None = str1.can_be_None or str2.can_be_None - no_nul = str1.no_nul and str2.no_nul - return SomeString(can_be_None=can_be_None, no_nul=no_nul) - - def add((str1, str2)): - # propagate const-ness to help getattr(obj, 'prefix' + const_name) - result = SomeString(no_nul=str1.no_nul and str2.no_nul) - if str1.is_immutable_constant() and str2.is_immutable_constant(): - result.const = str1.const + str2.const - return result - -class __extend__(pairtype(SomeByteArray, SomeByteArray)): - def union((b1, b2)): - can_be_None = b1.can_be_None or b2.can_be_None - return SomeByteArray(can_be_None=can_be_None) - - def add((b1, b2)): - result = SomeByteArray() - if b1.is_immutable_constant() and b2.is_immutable_constant(): - result.const = b1.const + b2.const - return result - -class __extend__(pairtype(SomeByteArray, SomeInteger)): - def getitem((s_b, s_i)): - return SomeInteger() - - def setitem((s_b, s_i), s_i2): - assert isinstance(s_i2, SomeInteger) - -class __extend__(pairtype(SomeString, SomeByteArray), - pairtype(SomeByteArray, SomeString), - pairtype(SomeChar, SomeByteArray), - pairtype(SomeByteArray, SomeChar)): - def add((b1, b2)): - result = SomeByteArray() - if b1.is_immutable_constant() and b2.is_immutable_constant(): - result.const = b1.const + b2.const - return result - -class __extend__(pairtype(SomeChar, SomeChar)): - - def union((chr1, chr2)): - return SomeChar() - - -class __extend__(pairtype(SomeChar, SomeUnicodeCodePoint), - pairtype(SomeUnicodeCodePoint, SomeChar)): - def union((uchr1, uchr2)): - return SomeUnicodeCodePoint() - -class __extend__(pairtype(SomeUnicodeCodePoint, SomeUnicodeCodePoint)): - def union((uchr1, uchr2)): - return SomeUnicodeCodePoint() - - def add((chr1, chr2)): - return SomeUnicodeString() - -class __extend__(pairtype(SomeString, SomeUnicodeString), - pairtype(SomeUnicodeString, SomeString)): - def mod((str, unistring)): - raise NotImplementedError( - "string formatting mixing strings and unicode not supported") - - -class __extend__(pairtype(SomeString, SomeTuple), - pairtype(SomeUnicodeString, SomeTuple)): - def mod((s_string, s_tuple)): - is_string = isinstance(s_string, SomeString) - is_unicode = isinstance(s_string, SomeUnicodeString) - assert is_string or is_unicode - for s_item in s_tuple.items: - if (is_unicode and isinstance(s_item, (SomeChar, SomeString)) or - is_string and isinstance(s_item, (SomeUnicodeCodePoint, - SomeUnicodeString))): - raise NotImplementedError( - "string formatting mixing strings and unicode not supported") - getbookkeeper().count('strformat', s_string, s_tuple) - no_nul = s_string.no_nul - for s_item in s_tuple.items: - if isinstance(s_item, SomeFloat): - pass # or s_item is a subclass, like SomeInteger - elif (isinstance(s_item, SomeString) or - isinstance(s_item, SomeUnicodeString)) and s_item.no_nul: - pass - else: - no_nul = False - break - return s_string.__class__(no_nul=no_nul) - - -class __extend__(pairtype(SomeString, SomeObject), - pairtype(SomeUnicodeString, SomeObject)): - - def mod((s_string, args)): - getbookkeeper().count('strformat', s_string, args) - return s_string.__class__() - -class __extend__(pairtype(SomeFloat, SomeFloat)): - - def union((flt1, flt2)): - return SomeFloat() - - add = sub = mul = union - - def div((flt1, flt2)): - return SomeFloat() - div.can_only_throw = [] - truediv = div - - # repeat these in order to copy the 'can_only_throw' attribute - inplace_div = div - inplace_truediv = truediv - - -class __extend__(pairtype(SomeSingleFloat, SomeSingleFloat)): - - def union((flt1, flt2)): - return SomeSingleFloat() - - -class __extend__(pairtype(SomeLongFloat, SomeLongFloat)): - - def union((flt1, flt2)): - return SomeLongFloat() - - -class __extend__(pairtype(SomeList, SomeList)): - - def union((lst1, lst2)): - return SomeList(lst1.listdef.union(lst2.listdef)) - - def add((lst1, lst2)): - return lst1.listdef.offspring(lst2.listdef) - - def eq((lst1, lst2)): - lst1.listdef.agree(lst2.listdef) - return s_Bool - ne = eq - - -class __extend__(pairtype(SomeList, SomeObject)): - - def inplace_add((lst1, obj2)): - lst1.method_extend(obj2) - return lst1 - inplace_add.can_only_throw = [] - - def inplace_mul((lst1, obj2)): - lst1.listdef.resize() - return lst1 - inplace_mul.can_only_throw = [] - -class __extend__(pairtype(SomeTuple, SomeTuple)): - - def union((tup1, tup2)): - if len(tup1.items) != len(tup2.items): - raise UnionError("cannot take the union of a tuple of length %d " - "and a tuple of length %d" % (len(tup1.items), - len(tup2.items))) - else: - unions = [unionof(x,y) for x,y in zip(tup1.items, tup2.items)] - return SomeTuple(items = unions) - - def add((tup1, tup2)): - return SomeTuple(items = tup1.items + tup2.items) - - def eq(tup1tup2): - tup1tup2.union() - return s_Bool - ne = eq - - def lt((tup1, tup2)): - raise Exception("unsupported: (...) < (...)") - def le((tup1, tup2)): - raise Exception("unsupported: (...) <= (...)") - def gt((tup1, tup2)): - raise Exception("unsupported: (...) > (...)") - def ge((tup1, tup2)): - raise Exception("unsupported: (...) >= (...)") - - -class __extend__(pairtype(SomeDict, SomeDict)): - - def union((dic1, dic2)): - return SomeDict(dic1.dictdef.union(dic2.dictdef)) - - -class __extend__(pairtype(SomeDict, SomeObject)): - - def _can_only_throw(dic1, *ignore): - if dic1.dictdef.dictkey.custom_eq_hash: - return None - return [KeyError] - - def getitem((dic1, obj2)): - getbookkeeper().count("dict_getitem", dic1) - dic1.dictdef.generalize_key(obj2) - return dic1.dictdef.read_value() - getitem.can_only_throw = _can_only_throw - - def setitem((dic1, obj2), s_value): - getbookkeeper().count("dict_setitem", dic1) - dic1.dictdef.generalize_key(obj2) - dic1.dictdef.generalize_value(s_value) - setitem.can_only_throw = _can_only_throw - - def delitem((dic1, obj2)): - getbookkeeper().count("dict_delitem", dic1) - dic1.dictdef.generalize_key(obj2) - delitem.can_only_throw = _can_only_throw - - -class __extend__(pairtype(SomeTuple, SomeInteger)): - - def getitem((tup1, int2)): - if int2.is_immutable_constant(): - try: - return tup1.items[int2.const] - except IndexError: - return s_ImpossibleValue - else: - getbookkeeper().count("tuple_random_getitem", tup1) - return unionof(*tup1.items) - getitem.can_only_throw = [IndexError] - - -class __extend__(pairtype(SomeList, SomeInteger)): - - def mul((lst1, int2)): - return lst1.listdef.offspring() - - def getitem((lst1, int2)): - getbookkeeper().count("list_getitem", int2) - return lst1.listdef.read_item() - getitem.can_only_throw = [] - - getitem_key = getitem - - def getitem_idx((lst1, int2)): - getbookkeeper().count("list_getitem", int2) - return lst1.listdef.read_item() - getitem_idx.can_only_throw = [IndexError] - - getitem_idx_key = getitem_idx - - def setitem((lst1, int2), s_value): - getbookkeeper().count("list_setitem", int2) - lst1.listdef.mutate() - lst1.listdef.generalize(s_value) - setitem.can_only_throw = [IndexError] - - def delitem((lst1, int2)): - getbookkeeper().count("list_delitem", int2) - lst1.listdef.resize() - delitem.can_only_throw = [IndexError] - -class __extend__(pairtype(SomeString, SomeInteger)): - - def getitem((str1, int2)): - getbookkeeper().count("str_getitem", int2) - return SomeChar() - getitem.can_only_throw = [] - - getitem_key = getitem - - def getitem_idx((str1, int2)): - getbookkeeper().count("str_getitem", int2) - return SomeChar() - getitem_idx.can_only_throw = [IndexError] - - getitem_idx_key = getitem_idx - - def mul((str1, int2)): # xxx do we want to support this - getbookkeeper().count("str_mul", str1, int2) - return SomeString(no_nul=str1.no_nul) - -class __extend__(pairtype(SomeUnicodeString, SomeInteger)): - def getitem((str1, int2)): - getbookkeeper().count("str_getitem", int2) - return SomeUnicodeCodePoint() - getitem.can_only_throw = [] - - getitem_key = getitem - - def getitem_idx((str1, int2)): - getbookkeeper().count("str_getitem", int2) - return SomeUnicodeCodePoint() - getitem_idx.can_only_throw = [IndexError] - - getitem_idx_key = getitem_idx - - def mul((str1, int2)): # xxx do we want to support this - getbookkeeper().count("str_mul", str1, int2) - return SomeUnicodeString() - -class __extend__(pairtype(SomeInteger, SomeString), - pairtype(SomeInteger, SomeUnicodeString)): - - def mul((int1, str2)): # xxx do we want to support this - getbookkeeper().count("str_mul", str2, int1) - return str2.basestringclass() - -class __extend__(pairtype(SomeUnicodeCodePoint, SomeUnicodeString), - pairtype(SomeUnicodeString, SomeUnicodeCodePoint), - pairtype(SomeUnicodeString, SomeUnicodeString)): - def union((str1, str2)): - return SomeUnicodeString(can_be_None=str1.can_be_none() or - str2.can_be_none()) - - def add((str1, str2)): - # propagate const-ness to help getattr(obj, 'prefix' + const_name) - result = SomeUnicodeString() - if str1.is_immutable_constant() and str2.is_immutable_constant(): - result.const = str1.const + str2.const - return result - -class __extend__(pairtype(SomeInteger, SomeList)): - - def mul((int1, lst2)): - return lst2.listdef.offspring() - - -class __extend__(pairtype(SomeInstance, SomeInstance)): - - def union((ins1, ins2)): - if ins1.classdef is None or ins2.classdef is None: - # special case only - basedef = None - else: - basedef = ins1.classdef.commonbase(ins2.classdef) - if basedef is None: - raise UnionError(ins1, ins2) - flags = ins1.flags - if flags: - flags = flags.copy() - for key, value in flags.items(): - if key not in ins2.flags or ins2.flags[key] != value: - del flags[key] - return SomeInstance(basedef, - can_be_None=ins1.can_be_None or ins2.can_be_None, - flags=flags) - - def improve((ins1, ins2)): - if ins1.classdef is None: - resdef = ins2.classdef - elif ins2.classdef is None: - resdef = ins1.classdef - else: - basedef = ins1.classdef.commonbase(ins2.classdef) - if basedef is ins1.classdef: - resdef = ins2.classdef - elif basedef is ins2.classdef: - resdef = ins1.classdef - else: - if ins1.can_be_None and ins2.can_be_None: - return s_None - else: - return s_ImpossibleValue - res = SomeInstance(resdef, can_be_None=ins1.can_be_None and ins2.can_be_None) - if ins1.contains(res) and ins2.contains(res): - return res # fine - else: - # this case can occur in the presence of 'const' attributes, - # which we should try to preserve. Fall-back... - thistype = pairtype(SomeInstance, SomeInstance) - return super(thistype, pair(ins1, ins2)).improve() - - -class __extend__(pairtype(SomeIterator, SomeIterator)): - - def union((iter1, iter2)): - s_cont = unionof(iter1.s_container, iter2.s_container) - if iter1.variant != iter2.variant: - raise UnionError("merging incompatible iterators variants") - return SomeIterator(s_cont, *iter1.variant) - - -class __extend__(pairtype(SomeBuiltin, SomeBuiltin)): - - def union((bltn1, bltn2)): - if (bltn1.analyser != bltn2.analyser or - bltn1.methodname != bltn2.methodname or - bltn1.s_self is None or bltn2.s_self is None): - raise UnionError("cannot merge two different builtin functions " - "or methods:\n %r\n %r" % (bltn1, bltn2)) - s_self = unionof(bltn1.s_self, bltn2.s_self) - return SomeBuiltin(bltn1.analyser, s_self, methodname=bltn1.methodname) - -class __extend__(pairtype(SomePBC, SomePBC)): - - def union((pbc1, pbc2)): - d = pbc1.descriptions.copy() - d.update(pbc2.descriptions) - return SomePBC(d, can_be_None = pbc1.can_be_None or pbc2.can_be_None) - - def is_((pbc1, pbc2)): - thistype = pairtype(SomePBC, SomePBC) - s = super(thistype, pair(pbc1, pbc2)).is_() - if not s.is_constant(): - if not pbc1.can_be_None or not pbc2.can_be_None: - for desc in pbc1.descriptions: - if desc in pbc2.descriptions: - break - else: - s.const = False # no common desc in the two sets - return s - -class __extend__(pairtype(SomeGenericCallable, SomePBC)): - def union((gencall, pbc)): - for desc in pbc.descriptions: - unique_key = desc - bk = desc.bookkeeper - s_result = bk.emulate_pbc_call(unique_key, pbc, gencall.args_s) - s_result = unionof(s_result, gencall.s_result) - assert gencall.s_result.contains(s_result) - return gencall - -class __extend__(pairtype(SomePBC, SomeGenericCallable)): - def union((pbc, gencall)): - return pair(gencall, pbc).union() - -class __extend__(pairtype(SomeImpossibleValue, SomeObject)): - def union((imp1, obj2)): - return obj2 - -class __extend__(pairtype(SomeObject, SomeImpossibleValue)): - def union((obj1, imp2)): - return obj1 - -# mixing Nones with other objects - -def _make_none_union(classname, constructor_args='', glob=None): - if glob is None: - glob = globals() - loc = locals() - source = py.code.Source(""" - class __extend__(pairtype(%(classname)s, SomePBC)): - def union((obj, pbc)): - if pbc.isNone(): - return %(classname)s(%(constructor_args)s) - else: - return SomeObject() - - class __extend__(pairtype(SomePBC, %(classname)s)): - def union((pbc, obj)): - if pbc.isNone(): - return %(classname)s(%(constructor_args)s) - else: - return SomeObject() - """ % loc) - exec source.compile() in glob - -_make_none_union('SomeInstance', 'classdef=obj.classdef, can_be_None=True') -_make_none_union('SomeString', 'no_nul=obj.no_nul, can_be_None=True') -_make_none_union('SomeUnicodeString', 'can_be_None=True') -_make_none_union('SomeList', 'obj.listdef') -_make_none_union('SomeDict', 'obj.dictdef') -_make_none_union('SomeExternalObject', 'obj.knowntype') -_make_none_union('SomeWeakRef', 'obj.classdef') - -# getitem on SomePBCs, in particular None fails - -class __extend__(pairtype(SomePBC, SomeObject)): - def getitem((pbc, o)): - if not pbc.isNone(): - raise AnnotatorError("getitem on %r" % pbc) - return s_ImpossibleValue - - def setitem((pbc, o), s_value): - if not pbc.isNone(): - raise AnnotatorError("setitem on %r" % pbc) - -class __extend__(pairtype(SomePBC, SomeString)): - def add((pbc, o)): - if not pbc.isNone(): - raise AnnotatorError('add on %r' % pbc) - return s_ImpossibleValue - -class __extend__(pairtype(SomeString, SomePBC)): - def add((o, pbc)): - if not pbc.isNone(): - raise AnnotatorError('add on %r' % pbc) - return s_ImpossibleValue - -class __extend__(pairtype(SomeExternalObject, SomeExternalObject)): - def union((ext1, ext2)): - if ext1.knowntype == ext2.knowntype: - return SomeExternalObject(ext1.knowntype) - return SomeObject() - -# ____________________________________________________________ -# annotation of low-level types -from pypy.annotation.model import SomePtr, SomeOOInstance, SomeOOClass -from pypy.annotation.model import SomeOOObject -from pypy.annotation.model import ll_to_annotation, annotation_to_lltype -from pypy.rpython.ootypesystem import ootype - -_make_none_union('SomeOOInstance', 'ootype=obj.ootype, can_be_None=True') - -class __extend__(pairtype(SomePtr, SomePtr)): - def union((p1, p2)): - assert p1.ll_ptrtype == p2.ll_ptrtype,("mixing of incompatible pointer types: %r, %r" % - (p1.ll_ptrtype, p2.ll_ptrtype)) - return SomePtr(p1.ll_ptrtype) - -class __extend__(pairtype(SomePtr, SomeInteger)): - - def getitem((p, int1)): - example = p.ll_ptrtype._example() - try: - v = example[0] - except IndexError: - return None # impossible value, e.g. FixedSizeArray(0) - return ll_to_annotation(v) - getitem.can_only_throw = [] - - def setitem((p, int1), s_value): # just doing checking - example = p.ll_ptrtype._example() - if example[0] is not None: # ignore Void s_value - v_lltype = annotation_to_lltype(s_value) - example[0] = v_lltype._defl() - setitem.can_only_throw = [] - -class __extend__(pairtype(SomePtr, SomeObject)): - def union((p, obj)): - assert False, ("mixing pointer type %r with something else %r" % (p.ll_ptrtype, obj)) - - def getitem((p, obj)): - assert False,"ptr %r getitem index not an int: %r" % (p.ll_ptrtype, obj) - - def setitem((p, obj), s_value): - assert False,"ptr %r setitem index not an int: %r" % (p.ll_ptrtype, obj) - -class __extend__(pairtype(SomeObject, SomePtr)): - def union((obj, p2)): - return pair(p2, obj).union() - - -class __extend__(pairtype(SomeOOInstance, SomeOOInstance)): - def union((r1, r2)): - common = ootype.commonBaseclass(r1.ootype, r2.ootype) - assert common is not None, 'Mixing of incompatible instances %r, %r' %(r1.ootype, r2.ootype) - return SomeOOInstance(common, can_be_None=r1.can_be_None or r2.can_be_None) - -class __extend__(pairtype(SomeOOClass, SomeOOClass)): - def union((r1, r2)): - if r1.ootype is None: - common = r2.ootype - elif r2.ootype is None: - common = r1.ootype - elif r1.ootype == r2.ootype: - common = r1.ootype - elif isinstance(r1.ootype, ootype.Instance) and isinstance(r2.ootype, ootype.Instance): - common = ootype.commonBaseclass(r1.ootype, r2.ootype) - assert common is not None, ('Mixing of incompatible classes %r, %r' - % (r1.ootype, r2.ootype)) - else: - common = ootype.Object - return SomeOOClass(common) - -class __extend__(pairtype(SomeOOInstance, SomeObject)): - def union((r, obj)): - assert False, ("mixing reference type %r with something else %r" % (r.ootype, obj)) - -class __extend__(pairtype(SomeObject, SomeOOInstance)): - def union((obj, r2)): - return pair(r2, obj).union() - -class __extend__(pairtype(SomeOOObject, SomeOOObject)): - def union((r1, r2)): - assert r1.ootype is ootype.Object and r2.ootype is ootype.Object - return SomeOOObject() - -#_________________________________________ -# weakrefs - -class __extend__(pairtype(SomeWeakRef, SomeWeakRef)): - def union((s_wrf1, s_wrf2)): - if s_wrf1.classdef is None: - basedef = s_wrf2.classdef # s_wrf1 is known to be dead - elif s_wrf2.classdef is None: - basedef = s_wrf1.classdef # s_wrf2 is known to be dead - else: - basedef = s_wrf1.classdef.commonbase(s_wrf2.classdef) - if basedef is None: # no common base class! complain... - return SomeObject() - return SomeWeakRef(basedef) - -#_________________________________________ -# memory addresses - -class __extend__(pairtype(SomeAddress, SomeAddress)): - def union((s_addr1, s_addr2)): - return SomeAddress() - - def sub((s_addr1, s_addr2)): - if s_addr1.is_null_address() and s_addr2.is_null_address(): - return getbookkeeper().immutablevalue(0) - return SomeInteger() - - def is_((s_addr1, s_addr2)): - assert False, "comparisons with is not supported by addresses" - -class __extend__(pairtype(SomeTypedAddressAccess, SomeTypedAddressAccess)): - def union((s_taa1, s_taa2)): - assert s_taa1.type == s_taa2.type - return s_taa1 - -class __extend__(pairtype(SomeTypedAddressAccess, SomeInteger)): - def getitem((s_taa, s_int)): - from pypy.annotation.model import lltype_to_annotation - return lltype_to_annotation(s_taa.type) - getitem.can_only_throw = [] - - def setitem((s_taa, s_int), s_value): - from pypy.annotation.model import annotation_to_lltype - assert annotation_to_lltype(s_value) is s_taa.type - setitem.can_only_throw = [] - - -class __extend__(pairtype(SomeAddress, SomeInteger)): - def add((s_addr, s_int)): - return SomeAddress() - - def sub((s_addr, s_int)): - return SomeAddress() - -class __extend__(pairtype(SomeAddress, SomeImpossibleValue)): - # need to override this specifically to hide the 'raise UnionError' - # of pairtype(SomeAddress, SomeObject). - def union((s_addr, s_imp)): - return s_addr - -class __extend__(pairtype(SomeImpossibleValue, SomeAddress)): - # need to override this specifically to hide the 'raise UnionError' - # of pairtype(SomeObject, SomeAddress). - def union((s_imp, s_addr)): - return s_addr - -class __extend__(pairtype(SomeAddress, SomeObject)): - def union((s_addr, s_obj)): - raise UnionError, "union of address and anything else makes no sense" - -class __extend__(pairtype(SomeObject, SomeAddress)): - def union((s_obj, s_addr)): - raise UnionError, "union of address and anything else makes no sense" diff --git a/pypy/annotation/bookkeeper.py b/pypy/annotation/bookkeeper.py deleted file mode 100644 --- a/pypy/annotation/bookkeeper.py +++ /dev/null @@ -1,797 +0,0 @@ -""" -The Bookkeeper class. -""" - -from __future__ import absolute_import - -import sys, types, inspect, weakref - -from pypy.objspace.flow.model import Constant From noreply at buildbot.pypy.org Mon Jan 21 02:19:50 2013 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Mon, 21 Jan 2013 02:19:50 +0100 (CET) Subject: [pypy-commit] pypy pytest: fix up viewer options and kill them from rpython conftest Message-ID: <20130121011950.97FCC1C1306@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: pytest Changeset: r60263:fb7a8dc86ae4 Date: 2013-01-21 01:54 +0100 http://bitbucket.org/pypy/pypy/changeset/fb7a8dc86ae4/ Log: fix up viewer options and kill them from rpython conftest diff --git a/rpython/conftest.py b/rpython/conftest.py --- a/rpython/conftest.py +++ b/rpython/conftest.py @@ -34,15 +34,9 @@ def pytest_addoption(parser): group = parser.getgroup("rpython options") - group.addoption('--view', action="store_true", dest="view", default=False, - help="view translation tests' flow graphs with Pygame") group.addoption('-P', '--platform', action="callback", type="string", default="host", callback=_set_platform, help="set up tests to use specified platform as compile/run target") - group = parser.getgroup("JIT options") - group.addoption('--viewloops', action="store_true", - default=False, dest="viewloops", - help="show only the compiled loops") def pytest_pycollect_makeitem(__multicall__,collector, name, obj): diff --git a/rpython/tool/pytest/viewerplugin.py b/rpython/tool/pytest/viewerplugin.py --- a/rpython/tool/pytest/viewerplugin.py +++ b/rpython/tool/pytest/viewerplugin.py @@ -17,7 +17,7 @@ def pytest_addoption(parser): - group = parser.getgroup("pypy options") + group = parser.getgroup("rpython options") group.addoption('--view', action="store_true", dest="view", default=False, help="view translation tests' flow graphs with Pygame") From noreply at buildbot.pypy.org Mon Jan 21 02:19:51 2013 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Mon, 21 Jan 2013 02:19:51 +0100 (CET) Subject: [pypy-commit] pypy pytest: move some common code into the pytest plugins in rpython Message-ID: <20130121011951.D4A711C1309@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: pytest Changeset: r60264:5852b6a995fa Date: 2013-01-21 02:19 +0100 http://bitbucket.org/pypy/pypy/changeset/5852b6a995fa/ Log: move some common code into the pytest plugins in rpython diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -8,34 +8,23 @@ # PyPy's command line extra options (these are added # to py.test's standard options) # + +pytest_plugins = [ + 'rpython.tool.pytest.leakfinder', + 'rpython.tool.pytest.viewerplugin', + 'rpython.tool.pytest.platform', +] + option = None pypydir = os.path.realpath(os.path.dirname(__file__)) -def braindead_deindent(self): - """monkeypatch that wont end up doing stupid in the python tokenizer""" - text = '\n'.join(self.lines) - short = py.std.textwrap.dedent(text) - newsource = py.code.Source() - newsource.lines[:] = short.splitlines() - return newsource - -py.code.Source.deindent = braindead_deindent - -def pytest_report_header(): - return "pytest-%s from %s" %(pytest.__version__, pytest.__file__) - -def pytest_addhooks(pluginmanager): - from rpython.conftest import LeakFinder - pluginmanager.register(LeakFinder()) def pytest_configure(config): global option option = config.option def pytest_addoption(parser): - from rpython.conftest import pytest_addoption - pytest_addoption(parser) group = parser.getgroup("pypy options") group.addoption('-A', '--runappdirect', action="store_true", diff --git a/rpython/conftest.py b/rpython/conftest.py --- a/rpython/conftest.py +++ b/rpython/conftest.py @@ -1,43 +1,21 @@ from os.path import * -import py, pytest -from rpython.tool import leakfinder +import py -pytest_plugins = 'rpython.tool.pytest.expecttest' +pytest_plugins = [ + 'rpython.tool.pytest.expecttest', + 'rpython.tool.pytest.leakfinder', + 'rpython.tool.pytest.platform', + 'rpython.tool.pytest.viewerplugin', +] cdir = realpath(join(dirname(__file__), 'translator', 'c')) cache_dir = realpath(join(dirname(__file__), '_cache')) option = None -def braindead_deindent(self): - """monkeypatch that wont end up doing stupid in the python tokenizer""" - text = '\n'.join(self.lines) - short = py.std.textwrap.dedent(text) - newsource = py.code.Source() - newsource.lines[:] = short.splitlines() - return newsource - -py.code.Source.deindent = braindead_deindent - -def pytest_report_header(): - return "pytest-%s from %s" %(pytest.__version__, pytest.__file__) - def pytest_configure(config): global option option = config.option -def _set_platform(opt, opt_str, value, parser): - from rpython.config.translationoption import PLATFORMS - from rpython.translator.platform import set_platform - if value not in PLATFORMS: - raise ValueError("%s not in %s" % (value, PLATFORMS)) - set_platform(value, None) - -def pytest_addoption(parser): - group = parser.getgroup("rpython options") - group.addoption('-P', '--platform', action="callback", type="string", - default="host", callback=_set_platform, - help="set up tests to use specified platform as compile/run target") - def pytest_pycollect_makeitem(__multicall__,collector, name, obj): res = __multicall__.execute() @@ -49,37 +27,3 @@ return res -def pytest_addhooks(pluginmanager): - pluginmanager.register(LeakFinder()) - -class LeakFinder: - """Track memory allocations during test execution. - - So far, only used by the function lltype.malloc(flavor='raw'). - """ - def pytest_runtest_setup(self, __multicall__, item): - __multicall__.execute() - if not isinstance(item, py.test.collect.Function): - return - if not getattr(item.obj, 'dont_track_allocations', False): - leakfinder.start_tracking_allocations() - - def pytest_runtest_call(self, __multicall__, item): - __multicall__.execute() - if not isinstance(item, py.test.collect.Function): - return - item._success = True - - def pytest_runtest_teardown(self, __multicall__, item): - __multicall__.execute() - if not isinstance(item, py.test.collect.Function): - return - if (not getattr(item.obj, 'dont_track_allocations', False) - and leakfinder.TRACK_ALLOCATIONS): - item._pypytest_leaks = leakfinder.stop_tracking_allocations(False) - else: # stop_tracking_allocations() already called - item._pypytest_leaks = None - - # check for leaks, but only if the test passed so far - if getattr(item, '_success', False) and item._pypytest_leaks: - raise leakfinder.MallocMismatch(item._pypytest_leaks) diff --git a/rpython/tool/pytest/leakfinder.py b/rpython/tool/pytest/leakfinder.py new file mode 100644 --- /dev/null +++ b/rpython/tool/pytest/leakfinder.py @@ -0,0 +1,31 @@ +"""Track memory allocations during test execution. + +So far, only used by the function lltype.malloc(flavor='raw'). +""" + +def pytest_runtest_setup(__multicall__, item): + __multicall__.execute() + if not isinstance(item, py.test.collect.Function): + return + if not getattr(item.obj, 'dont_track_allocations', False): + leakfinder.start_tracking_allocations() + +def pytest_runtest_call(__multicall__, item): + __multicall__.execute() + if not isinstance(item, py.test.collect.Function): + return + item._success = True + +def pytest_runtest_teardown(__multicall__, item): + __multicall__.execute() + if not isinstance(item, py.test.collect.Function): + return + if (not getattr(item.obj, 'dont_track_allocations', False) + and leakfinder.TRACK_ALLOCATIONS): + item._pypytest_leaks = leakfinder.stop_tracking_allocations(False) + else: # stop_tracking_allocations() already called + item._pypytest_leaks = None + + # check for leaks, but only if the test passed so far + if getattr(item, '_success', False) and item._pypytest_leaks: + raise leakfinder.MallocMismatch(item._pypytest_leaks) diff --git a/rpython/tool/pytest/platform.py b/rpython/tool/pytest/platform.py new file mode 100644 --- /dev/null +++ b/rpython/tool/pytest/platform.py @@ -0,0 +1,30 @@ +import py +import pytest + +def braindead_deindent(self): + """monkeypatch that wont end up doing stupid in the python tokenizer""" + text = '\n'.join(self.lines) + short = py.std.textwrap.dedent(text) + newsource = py.code.Source() + newsource.lines[:] = short.splitlines() + return newsource + +py.code.Source.deindent = braindead_deindent + +def pytest_report_header(): + return "pytest-%s from %s" %(pytest.__version__, pytest.__file__) + + +def _set_platform(opt, opt_str, value, parser): + from rpython.config.translationoption import PLATFORMS + from rpython.translator.platform import set_platform + if value not in PLATFORMS: + raise ValueError("%s not in %s" % (value, PLATFORMS)) + set_platform(value, None) + +def pytest_addoption(parser): + group = parser.getgroup("rpython options") + group.addoption('-P', '--platform', action="callback", type="string", + default="host", callback=_set_platform, + help="set up tests to use specified platform as compile/run target") + From noreply at buildbot.pypy.org Mon Jan 21 02:23:51 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Mon, 21 Jan 2013 02:23:51 +0100 (CET) Subject: [pypy-commit] pypy inline-virtualref: Mreged default in. Message-ID: <20130121012351.F40771C1306@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: inline-virtualref Changeset: r60265:08123444e619 Date: 2013-01-20 17:23 -0800 http://bitbucket.org/pypy/pypy/changeset/08123444e619/ Log: Mreged default in. diff too long, truncating to 2000 out of 723009 lines diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -85,3 +85,4 @@ ^compiled ^.git/ ^release/ +^rpython/_cache$ diff --git a/dotviewer/test/test_translator.py b/dotviewer/test/test_translator.py --- a/dotviewer/test/test_translator.py +++ b/dotviewer/test/test_translator.py @@ -21,7 +21,7 @@ def test_annotated(): - from pypy.translator.interactive import Translation + from rpython.translator.interactive import Translation t = Translation(is_prime) t.annotate([int]) t.viewcg() diff --git a/lib_pypy/ctypes_config_cache/autopath.py b/lib_pypy/ctypes_config_cache/autopath.py deleted file mode 100644 --- a/lib_pypy/ctypes_config_cache/autopath.py +++ /dev/null @@ -1,131 +0,0 @@ -""" -self cloning, automatic path configuration - -copy this into any subdirectory of pypy from which scripts need -to be run, typically all of the test subdirs. -The idea is that any such script simply issues - - import autopath - -and this will make sure that the parent directory containing "pypy" -is in sys.path. - -If you modify the master "autopath.py" version (in pypy/tool/autopath.py) -you can directly run it which will copy itself on all autopath.py files -it finds under the pypy root directory. - -This module always provides these attributes: - - pypydir pypy root directory path - this_dir directory where this autopath.py resides - -""" - -def __dirinfo(part): - """ return (partdir, this_dir) and insert parent of partdir - into sys.path. If the parent directories don't have the part - an EnvironmentError is raised.""" - - import sys, os - try: - head = this_dir = os.path.realpath(os.path.dirname(__file__)) - except NameError: - head = this_dir = os.path.realpath(os.path.dirname(sys.argv[0])) - - error = None - while head: - partdir = head - head, tail = os.path.split(head) - if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') - if not os.path.exists(checkfile): - error = "Cannot find %r" % (os.path.normpath(checkfile),) - break - else: - error = "Cannot find the parent directory %r of the path %r" % ( - partdir, this_dir) - if not error: - # check for bogus end-of-line style (e.g. files checked out on - # Windows and moved to Unix) - f = open(__file__.replace('.pyc', '.py'), 'r') - data = f.read() - f.close() - if data.endswith('\r\n') or data.endswith('\r'): - error = ("Bad end-of-line style in the .py files. Typically " - "caused by a zip file or a checkout done on Windows and " - "moved to Unix or vice-versa.") - if error: - raise EnvironmentError("Invalid source tree - bogus checkout! " + - error) - - pypy_root = os.path.join(head, '') - try: - sys.path.remove(head) - except ValueError: - pass - sys.path.insert(0, head) - - munged = {} - for name, mod in sys.modules.items(): - if '.' in name: - continue - fn = getattr(mod, '__file__', None) - if not isinstance(fn, str): - continue - newname = os.path.splitext(os.path.basename(fn))[0] - if not newname.startswith(part + '.'): - continue - path = os.path.join(os.path.dirname(os.path.realpath(fn)), '') - if path.startswith(pypy_root) and newname != part: - modpaths = os.path.normpath(path[len(pypy_root):]).split(os.sep) - if newname != '__init__': - modpaths.append(newname) - modpath = '.'.join(modpaths) - if modpath not in sys.modules: - munged[modpath] = mod - - for name, mod in munged.iteritems(): - if name not in sys.modules: - sys.modules[name] = mod - if '.' in name: - prename = name[:name.rfind('.')] - postname = name[len(prename)+1:] - if prename not in sys.modules: - __import__(prename) - if not hasattr(sys.modules[prename], postname): - setattr(sys.modules[prename], postname, mod) - - return partdir, this_dir - -def __clone(): - """ clone master version of autopath.py into all subdirs """ - from os.path import join, walk - if not this_dir.endswith(join('pypy','tool')): - raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) - - - def sync_walker(arg, dirname, fnames): - if _myname in fnames: - fn = join(dirname, _myname) - f = open(fn, 'rwb+') - try: - if f.read() == arg: - print "checkok", fn - else: - print "syncing", fn - f = open(fn, 'w') - f.write(arg) - finally: - f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) - -_myname = 'autopath.py' - -# set guaranteed attributes - -pypydir, this_dir = __dirinfo('pypy') - -if __name__ == '__main__': - __clone() diff --git a/lib_pypy/ctypes_config_cache/dumpcache.py b/lib_pypy/ctypes_config_cache/dumpcache.py --- a/lib_pypy/ctypes_config_cache/dumpcache.py +++ b/lib_pypy/ctypes_config_cache/dumpcache.py @@ -1,6 +1,6 @@ import os from ctypes_configure import dumpcache -from pypy.jit.backend import detect_cpu +from rpython.jit.backend import detect_cpu def dumpcache2(basename, config): model = detect_cpu.autodetect_main_model_and_size() @@ -14,7 +14,7 @@ try: from __pypy__ import cpumodel except ImportError: - from pypy.jit.backend import detect_cpu + from rpython.jit.backend import detect_cpu cpumodel = detect_cpu.autodetect_main_model_and_size() # XXX relative import, should be removed together with # XXX the relative imports done e.g. by lib_pypy/pypy_test/test_hashlib diff --git a/lib_pypy/ctypes_config_cache/rebuild.py b/lib_pypy/ctypes_config_cache/rebuild.py --- a/lib_pypy/ctypes_config_cache/rebuild.py +++ b/lib_pypy/ctypes_config_cache/rebuild.py @@ -1,21 +1,15 @@ #! /usr/bin/env python # Run this script to rebuild all caches from the *.ctc.py files. -# hack: we cannot directly import autopath, as we are outside the pypy -# package. However, we pretend to be inside pypy/tool and manually run it, to -# get the correct path -import os.path -this_dir = os.path.dirname(__file__) -autopath_py = os.path.join(this_dir, '../../pypy/tool/autopath.py') -autopath_py = os.path.abspath(autopath_py) -execfile(autopath_py, dict(__name__='autopath', __file__=autopath_py)) +import os, sys -import os, sys +sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..'))) + import py _dirpath = os.path.dirname(__file__) or os.curdir -from pypy.tool.ansi_print import ansi_log +from rpython.tool.ansi_print import ansi_log log = py.log.Producer("ctypes_config_cache") py.log.setconsumer("ctypes_config_cache", ansi_log) @@ -31,7 +25,7 @@ sys.path[:] = path def try_rebuild(): - from pypy.jit.backend import detect_cpu + from rpython.jit.backend import detect_cpu model = detect_cpu.autodetect_main_model_and_size() # remove the files '_*_model_.py' left = {} diff --git a/lib_pypy/ctypes_config_cache/test/test_cache.py b/lib_pypy/ctypes_config_cache/test/test_cache.py --- a/lib_pypy/ctypes_config_cache/test/test_cache.py +++ b/lib_pypy/ctypes_config_cache/test/test_cache.py @@ -1,6 +1,6 @@ import py import sys, os -from pypy.tool.udir import udir +from rpython.tool.udir import udir dirpath = py.path.local(__file__).dirpath().dirpath() diff --git a/lib_pypy/pypy_test/test_dbm_extra.py b/lib_pypy/pypy_test/test_dbm_extra.py --- a/lib_pypy/pypy_test/test_dbm_extra.py +++ b/lib_pypy/pypy_test/test_dbm_extra.py @@ -1,6 +1,6 @@ from __future__ import absolute_import import py -from pypy.tool.udir import udir +from rpython.tool.udir import udir try: from lib_pypy import dbm except ImportError, e: diff --git a/lib_pypy/pypy_test/test_marshal_extra.py b/lib_pypy/pypy_test/test_marshal_extra.py --- a/lib_pypy/pypy_test/test_marshal_extra.py +++ b/lib_pypy/pypy_test/test_marshal_extra.py @@ -4,7 +4,7 @@ import marshal as cpy_marshal from lib_pypy import _marshal as marshal -from pypy.tool.udir import udir +from rpython.tool.udir import udir hello = "he" hello += "llo" diff --git a/pypy/annotation/annrpython.py b/pypy/annotation/annrpython.py deleted file mode 100644 --- a/pypy/annotation/annrpython.py +++ /dev/null @@ -1,676 +0,0 @@ -from __future__ import absolute_import - -import types -from pypy.tool.ansi_print import ansi_log -from pypy.tool.pairtype import pair -from pypy.tool.error import (format_blocked_annotation_error, - AnnotatorError, gather_error, ErrorWrapper) -from pypy.objspace.flow.model import (Variable, Constant, FunctionGraph, - c_last_exception, checkgraph) -from pypy.translator import simplify, transform -from pypy.annotation import model as annmodel, signature, unaryop, binaryop -from pypy.annotation.bookkeeper import Bookkeeper -import py -log = py.log.Producer("annrpython") -py.log.setconsumer("annrpython", ansi_log) - - -FAIL = object() - -class RPythonAnnotator(object): - """Block annotator for RPython. - See description in doc/translation.txt.""" - - def __init__(self, translator=None, policy=None, bookkeeper=None): - import pypy.rpython.ootypesystem.ooregistry # has side effects - import pypy.rpython.extfuncregistry # has side effects - import pypy.rlib.nonconst # has side effects - - if translator is None: - # interface for tests - from pypy.translator.translator import TranslationContext - translator = TranslationContext() - translator.annotator = self - self.translator = translator - self.pendingblocks = {} # map {block: graph-containing-it} - self.bindings = {} # map Variables to SomeValues - self.annotated = {} # set of blocks already seen - self.added_blocks = None # see processblock() below - self.links_followed = {} # set of links that have ever been followed - self.notify = {} # {block: {positions-to-reflow-from-when-done}} - self.fixed_graphs = {} # set of graphs not to annotate again - self.blocked_blocks = {} # set of {blocked_block: (graph, index)} - # --- the following information is recorded for debugging --- - self.blocked_graphs = {} # set of graphs that have blocked blocks - # --- end of debugging information --- - self.frozen = False - if policy is None: - from pypy.annotation.policy import AnnotatorPolicy - self.policy = AnnotatorPolicy() - else: - self.policy = policy - if bookkeeper is None: - bookkeeper = Bookkeeper(self) - self.bookkeeper = bookkeeper - - def __getstate__(self): - attrs = """translator pendingblocks bindings annotated links_followed - notify bookkeeper frozen policy added_blocks""".split() - ret = self.__dict__.copy() - for key, value in ret.items(): - if key not in attrs: - assert type(value) is dict, ( - "%r is not dict. please update %s.__getstate__" % - (key, self.__class__.__name__)) - ret[key] = {} - return ret - - #___ convenience high-level interface __________________ - - def build_types(self, function, input_arg_types, complete_now=True, - main_entry_point=False): - """Recursively build annotations about the specific entry point.""" - assert isinstance(function, types.FunctionType), "fix that!" - - from pypy.annotation.policy import AnnotatorPolicy - policy = AnnotatorPolicy() - # make input arguments and set their type - args_s = [self.typeannotation(t) for t in input_arg_types] - - # XXX hack - annmodel.TLS.check_str_without_nul = ( - self.translator.config.translation.check_str_without_nul) - - flowgraph, inputcells = self.get_call_parameters(function, args_s, policy) - if not isinstance(flowgraph, FunctionGraph): - assert isinstance(flowgraph, annmodel.SomeObject) - return flowgraph - - if main_entry_point: - self.translator.entry_point_graph = flowgraph - return self.build_graph_types(flowgraph, inputcells, complete_now=complete_now) - - def get_call_parameters(self, function, args_s, policy): - desc = self.bookkeeper.getdesc(function) - args = self.bookkeeper.build_args("simple_call", args_s[:]) - result = [] - def schedule(graph, inputcells): - result.append((graph, inputcells)) - return annmodel.s_ImpossibleValue - - prevpolicy = self.policy - self.policy = policy - self.bookkeeper.enter(None) - try: - desc.pycall(schedule, args, annmodel.s_ImpossibleValue) - finally: - self.bookkeeper.leave() - self.policy = prevpolicy - [(graph, inputcells)] = result - return graph, inputcells - - def annotate_helper(self, function, args_s, policy=None): - if policy is None: - from pypy.annotation.policy import AnnotatorPolicy - policy = AnnotatorPolicy() - graph, inputcells = self.get_call_parameters(function, args_s, policy) - self.build_graph_types(graph, inputcells, complete_now=False) - self.complete_helpers(policy) - return graph - - def complete_helpers(self, policy): - saved = self.policy, self.added_blocks - self.policy = policy - try: - self.added_blocks = {} - self.complete() - # invoke annotation simplifications for the new blocks - self.simplify(block_subset=self.added_blocks) - finally: - self.policy, self.added_blocks = saved - - def build_graph_types(self, flowgraph, inputcells, complete_now=True): - checkgraph(flowgraph) - - nbarg = len(flowgraph.getargs()) - if len(inputcells) != nbarg: - raise TypeError("%s expects %d args, got %d" %( - flowgraph, nbarg, len(inputcells))) - - # register the entry point - self.addpendinggraph(flowgraph, inputcells) - # recursively proceed until no more pending block is left - if complete_now: - self.complete() - return self.binding(flowgraph.getreturnvar(), None) - - def gettype(self, variable): - """Return the known type of a control flow graph variable, - defaulting to 'object'.""" - if isinstance(variable, Constant): - return type(variable.value) - elif isinstance(variable, Variable): - cell = self.bindings.get(variable) - if cell: - return cell.knowntype - else: - return object - else: - raise TypeError, ("Variable or Constant instance expected, " - "got %r" % (variable,)) - - def getuserclassdefinitions(self): - """Return a list of ClassDefs.""" - return self.bookkeeper.classdefs - - #___ medium-level interface ____________________________ - - def addpendinggraph(self, flowgraph, inputcells): - self.addpendingblock(flowgraph, flowgraph.startblock, inputcells) - - def addpendingblock(self, graph, block, cells): - """Register an entry point into block with the given input cells.""" - if graph in self.fixed_graphs: - # special case for annotating/rtyping in several phases: calling - # a graph that has already been rtyped. Safety-check the new - # annotations that are passed in, and don't annotate the old - # graph -- it's already low-level operations! - for a, s_newarg in zip(graph.getargs(), cells): - s_oldarg = self.binding(a) - assert annmodel.unionof(s_oldarg, s_newarg) == s_oldarg - else: - assert not self.frozen - for a in cells: - assert isinstance(a, annmodel.SomeObject) - if block not in self.annotated: - self.bindinputargs(graph, block, cells) - else: - self.mergeinputargs(graph, block, cells) - if not self.annotated[block]: - self.pendingblocks[block] = graph - - def complete(self): - """Process pending blocks until none is left.""" - while True: - while self.pendingblocks: - block, graph = self.pendingblocks.popitem() - self.processblock(graph, block) - self.policy.no_more_blocks_to_annotate(self) - if not self.pendingblocks: - break # finished - # make sure that the return variables of all graphs is annotated - if self.added_blocks is not None: - newgraphs = [self.annotated[block] for block in self.added_blocks] - newgraphs = dict.fromkeys(newgraphs) - got_blocked_blocks = False in newgraphs - else: - newgraphs = self.translator.graphs #all of them - got_blocked_blocks = False in self.annotated.values() - if got_blocked_blocks: - for graph in self.blocked_graphs.values(): - self.blocked_graphs[graph] = True - - blocked_blocks = [block for block, done in self.annotated.items() - if done is False] - assert len(blocked_blocks) == len(self.blocked_blocks) - - text = format_blocked_annotation_error(self, self.blocked_blocks) - #raise SystemExit() - raise AnnotatorError(text) - for graph in newgraphs: - v = graph.getreturnvar() - if v not in self.bindings: - self.setbinding(v, annmodel.s_ImpossibleValue) - # policy-dependent computation - self.bookkeeper.compute_at_fixpoint() - - def binding(self, arg, default=FAIL): - "Gives the SomeValue corresponding to the given Variable or Constant." - if isinstance(arg, Variable): - try: - return self.bindings[arg] - except KeyError: - if default is not FAIL: - return default - else: - raise - elif isinstance(arg, Constant): - #if arg.value is undefined_value: # undefined local variables - # return annmodel.s_ImpossibleValue - return self.bookkeeper.immutableconstant(arg) - else: - raise TypeError, 'Variable or Constant expected, got %r' % (arg,) - - def typeannotation(self, t): - return signature.annotation(t, self.bookkeeper) - - def setbinding(self, arg, s_value): - if arg in self.bindings: - assert s_value.contains(self.bindings[arg]) - self.bindings[arg] = s_value - - def transfer_binding(self, v_target, v_source): - assert v_source in self.bindings - self.bindings[v_target] = self.bindings[v_source] - - def warning(self, msg, pos=None): - if pos is None: - try: - pos = self.bookkeeper.position_key - except AttributeError: - pos = '?' - if pos != '?': - pos = self.whereami(pos) - - log.WARNING("%s/ %s" % (pos, msg)) - - - #___ interface for annotator.bookkeeper _______ - - def recursivecall(self, graph, whence, inputcells): - if isinstance(whence, tuple): - parent_graph, parent_block, parent_index = whence - tag = parent_block, parent_index - self.translator.update_call_graph(parent_graph, graph, tag) - # self.notify[graph.returnblock] is a dictionary of call - # points to this func which triggers a reflow whenever the - # return block of this graph has been analysed. - callpositions = self.notify.setdefault(graph.returnblock, {}) - if whence is not None: - if callable(whence): - def callback(): - whence(self, graph) - else: - callback = whence - callpositions[callback] = True - - # generalize the function's input arguments - self.addpendingblock(graph, graph.startblock, inputcells) - - # get the (current) return value - v = graph.getreturnvar() - try: - return self.bindings[v] - except KeyError: - # the function didn't reach any return statement so far. - # (some functions actually never do, they always raise exceptions) - return annmodel.s_ImpossibleValue - - def reflowfromposition(self, position_key): - graph, block, index = position_key - self.reflowpendingblock(graph, block) - - - #___ simplification (should be moved elsewhere?) _______ - - def simplify(self, block_subset=None, extra_passes=None): - # Generic simplifications - transform.transform_graph(self, block_subset=block_subset, - extra_passes=extra_passes) - if block_subset is None: - graphs = self.translator.graphs - else: - graphs = {} - for block in block_subset: - graph = self.annotated.get(block) - if graph: - graphs[graph] = True - for graph in graphs: - simplify.eliminate_empty_blocks(graph) - - - #___ flowing annotations in blocks _____________________ - - def processblock(self, graph, block): - # Important: this is not called recursively. - # self.flowin() can only issue calls to self.addpendingblock(). - # The analysis of a block can be in three states: - # * block not in self.annotated: - # never seen the block. - # * self.annotated[block] == False: - # the input variables of the block are in self.bindings but we - # still have to consider all the operations in the block. - # * self.annotated[block] == graph-containing-block: - # analysis done (at least until we find we must generalize the - # input variables). - - #print '* processblock', block, cells - self.annotated[block] = graph - if block in self.blocked_blocks: - del self.blocked_blocks[block] - try: - self.flowin(graph, block) - except BlockedInference, e: - self.annotated[block] = False # failed, hopefully temporarily - self.blocked_blocks[block] = (graph, e.opindex) - except Exception, e: - # hack for debug tools only - if not hasattr(e, '__annotator_block'): - setattr(e, '__annotator_block', block) - raise - - # The dict 'added_blocks' is used by rpython.annlowlevel to - # detect which are the new blocks that annotating an additional - # small helper creates. - if self.added_blocks is not None: - self.added_blocks[block] = True - - def reflowpendingblock(self, graph, block): - assert not self.frozen - assert graph not in self.fixed_graphs - self.pendingblocks[block] = graph - assert block in self.annotated - self.annotated[block] = False # must re-flow - self.blocked_blocks[block] = (graph, None) - - def bindinputargs(self, graph, block, inputcells): - # Create the initial bindings for the input args of a block. - assert len(block.inputargs) == len(inputcells) - for a, cell in zip(block.inputargs, inputcells): - self.setbinding(a, cell) - self.annotated[block] = False # must flowin. - self.blocked_blocks[block] = (graph, None) - - def mergeinputargs(self, graph, block, inputcells): - # Merge the new 'cells' with each of the block's existing input - # variables. - oldcells = [self.binding(a) for a in block.inputargs] - try: - unions = [annmodel.unionof(c1,c2) for c1, c2 in zip(oldcells,inputcells)] - except annmodel.UnionError, e: - e.args = e.args + ( - ErrorWrapper(gather_error(self, graph, block, None)),) - raise - # if the merged cells changed, we must redo the analysis - if unions != oldcells: - self.bindinputargs(graph, block, unions) - - def whereami(self, position_key): - graph, block, i = position_key - blk = "" - if block: - at = block.at() - if at: - blk = " block"+at - opid="" - if i is not None: - opid = " op=%d" % i - return repr(graph) + blk + opid - - def flowin(self, graph, block): - #print 'Flowing', block, [self.binding(a) for a in block.inputargs] - try: - for i in range(len(block.operations)): - try: - self.bookkeeper.enter((graph, block, i)) - self.consider_op(block, i) - finally: - self.bookkeeper.leave() - - except BlockedInference, e: - if (e.op is block.operations[-1] and - block.exitswitch == c_last_exception): - # this is the case where the last operation of the block will - # always raise an exception which is immediately caught by - # an exception handler. We then only follow the exceptional - # branches. - exits = [link for link in block.exits - if link.exitcase is not None] - - elif e.op.opname in ('simple_call', 'call_args', 'next'): - # XXX warning, keep the name of the call operations in sync - # with the flow object space. These are the operations for - # which it is fine to always raise an exception. We then - # swallow the BlockedInference and that's it. - # About 'next': see test_annotate_iter_empty_container(). - return - - else: - # other cases are problematic (but will hopefully be solved - # later by reflowing). Throw the BlockedInference up to - # processblock(). - raise - - except annmodel.HarmlesslyBlocked: - return - - else: - # dead code removal: don't follow all exits if the exitswitch - # is known - exits = block.exits - if isinstance(block.exitswitch, Variable): - s_exitswitch = self.bindings[block.exitswitch] - if s_exitswitch.is_constant(): - exits = [link for link in exits - if link.exitcase == s_exitswitch.const] - - # mapping (exitcase, variable) -> s_annotation - # that can be attached to booleans, exitswitches - knowntypedata = getattr(self.bindings.get(block.exitswitch), - "knowntypedata", {}) - - # filter out those exceptions which cannot - # occour for this specific, typed operation. - if block.exitswitch == c_last_exception: - op = block.operations[-1] - if op.opname in binaryop.BINARY_OPERATIONS: - arg1 = self.binding(op.args[0]) - arg2 = self.binding(op.args[1]) - binop = getattr(pair(arg1, arg2), op.opname, None) - can_only_throw = annmodel.read_can_only_throw(binop, arg1, arg2) - elif op.opname in unaryop.UNARY_OPERATIONS: - arg1 = self.binding(op.args[0]) - opname = op.opname - if opname == 'contains': opname = 'op_contains' - unop = getattr(arg1, opname, None) - can_only_throw = annmodel.read_can_only_throw(unop, arg1) - else: - can_only_throw = None - - if can_only_throw is not None: - candidates = can_only_throw - candidate_exits = exits - exits = [] - for link in candidate_exits: - case = link.exitcase - if case is None: - exits.append(link) - continue - covered = [c for c in candidates if issubclass(c, case)] - if covered: - exits.append(link) - candidates = [c for c in candidates if c not in covered] - - for link in exits: - in_except_block = False - - last_exception_var = link.last_exception # may be None for non-exception link - last_exc_value_var = link.last_exc_value # may be None for non-exception link - - if isinstance(link.exitcase, (types.ClassType, type)) \ - and issubclass(link.exitcase, py.builtin.BaseException): - assert last_exception_var and last_exc_value_var - last_exc_value_object = self.bookkeeper.valueoftype(link.exitcase) - last_exception_object = annmodel.SomeType() - if isinstance(last_exception_var, Constant): - last_exception_object.const = last_exception_var.value - last_exception_object.is_type_of = [last_exc_value_var] - - if isinstance(last_exception_var, Variable): - self.setbinding(last_exception_var, last_exception_object) - if isinstance(last_exc_value_var, Variable): - self.setbinding(last_exc_value_var, last_exc_value_object) - - last_exception_object = annmodel.SomeType() - if isinstance(last_exception_var, Constant): - last_exception_object.const = last_exception_var.value - #if link.exitcase is Exception: - # last_exc_value_object = annmodel.SomeObject() - #else: - last_exc_value_vars = [] - in_except_block = True - - ignore_link = False - cells = [] - renaming = {} - for a,v in zip(link.args,link.target.inputargs): - renaming.setdefault(a, []).append(v) - for a,v in zip(link.args,link.target.inputargs): - if a == last_exception_var: - assert in_except_block - cells.append(last_exception_object) - elif a == last_exc_value_var: - assert in_except_block - cells.append(last_exc_value_object) - last_exc_value_vars.append(v) - else: - cell = self.binding(a) - if (link.exitcase, a) in knowntypedata: - knownvarvalue = knowntypedata[(link.exitcase, a)] - cell = pair(cell, knownvarvalue).improve() - # ignore links that try to pass impossible values - if cell == annmodel.s_ImpossibleValue: - ignore_link = True - - if hasattr(cell,'is_type_of'): - renamed_is_type_of = [] - for v in cell.is_type_of: - new_vs = renaming.get(v,[]) - renamed_is_type_of += new_vs - assert cell.knowntype is type - newcell = annmodel.SomeType() - if cell.is_constant(): - newcell.const = cell.const - cell = newcell - cell.is_type_of = renamed_is_type_of - - if hasattr(cell, 'knowntypedata'): - renamed_knowntypedata = {} - for (value, v), s in cell.knowntypedata.items(): - new_vs = renaming.get(v, []) - for new_v in new_vs: - renamed_knowntypedata[value, new_v] = s - assert isinstance(cell, annmodel.SomeBool) - newcell = annmodel.SomeBool() - if cell.is_constant(): - newcell.const = cell.const - cell = newcell - cell.set_knowntypedata(renamed_knowntypedata) - - cells.append(cell) - - if ignore_link: - continue - - if in_except_block: - last_exception_object.is_type_of = last_exc_value_vars - - self.links_followed[link] = True - self.addpendingblock(graph, link.target, cells) - - if block in self.notify: - # reflow from certain positions when this block is done - for callback in self.notify[block]: - if isinstance(callback, tuple): - self.reflowfromposition(callback) # callback is a position - else: - callback() - - - #___ creating the annotations based on operations ______ - - def consider_op(self, block, opindex): - op = block.operations[opindex] - argcells = [self.binding(a) for a in op.args] - consider_meth = getattr(self,'consider_op_'+op.opname, - None) - if not consider_meth: - raise Exception,"unknown op: %r" % op - - # let's be careful about avoiding propagated SomeImpossibleValues - # to enter an op; the latter can result in violations of the - # more general results invariant: e.g. if SomeImpossibleValue enters is_ - # is_(SomeImpossibleValue, None) -> SomeBool - # is_(SomeInstance(not None), None) -> SomeBool(const=False) ... - # boom -- in the assert of setbinding() - for arg in argcells: - if isinstance(arg, annmodel.SomeImpossibleValue): - raise BlockedInference(self, op, opindex) - try: - resultcell = consider_meth(*argcells) - except Exception, e: - graph = self.bookkeeper.position_key[0] - e.args = e.args + ( - ErrorWrapper(gather_error(self, graph, block, opindex)),) - raise - if resultcell is None: - resultcell = self.noreturnvalue(op) - elif resultcell == annmodel.s_ImpossibleValue: - raise BlockedInference(self, op, opindex) # the operation cannot succeed - assert isinstance(resultcell, annmodel.SomeObject) - assert isinstance(op.result, Variable) - self.setbinding(op.result, resultcell) # bind resultcell to op.result - - def noreturnvalue(self, op): - return annmodel.s_ImpossibleValue # no return value (hook method) - - # XXX "contains" clash with SomeObject method - def consider_op_contains(self, seq, elem): - self.bookkeeper.count("contains", seq) - return seq.op_contains(elem) - - def consider_op_newtuple(self, *args): - return annmodel.SomeTuple(items = args) - - def consider_op_newlist(self, *args): - return self.bookkeeper.newlist(*args) - - def consider_op_newdict(self): - return self.bookkeeper.newdict() - - - def _registeroperations(cls, unary_ops, binary_ops): - # All unary operations - d = {} - for opname in unary_ops: - fnname = 'consider_op_' + opname - exec py.code.Source(""" -def consider_op_%s(self, arg, *args): - return arg.%s(*args) -""" % (opname, opname)).compile() in globals(), d - setattr(cls, fnname, d[fnname]) - # All binary operations - for opname in binary_ops: - fnname = 'consider_op_' + opname - exec py.code.Source(""" -def consider_op_%s(self, arg1, arg2, *args): - return pair(arg1,arg2).%s(*args) -""" % (opname, opname)).compile() in globals(), d - setattr(cls, fnname, d[fnname]) - _registeroperations = classmethod(_registeroperations) - -# register simple operations handling -RPythonAnnotator._registeroperations(unaryop.UNARY_OPERATIONS, binaryop.BINARY_OPERATIONS) - - -class BlockedInference(Exception): - """This exception signals the type inference engine that the situation - is currently blocked, and that it should try to progress elsewhere.""" - - def __init__(self, annotator, op, opindex): - self.annotator = annotator - try: - self.break_at = annotator.bookkeeper.position_key - except AttributeError: - self.break_at = None - self.op = op - self.opindex = opindex - - def __repr__(self): - if not self.break_at: - break_at = "?" - else: - break_at = self.annotator.whereami(self.break_at) - return "" %(break_at, self.op) - - __str__ = __repr__ diff --git a/pypy/annotation/binaryop.py b/pypy/annotation/binaryop.py deleted file mode 100644 --- a/pypy/annotation/binaryop.py +++ /dev/null @@ -1,1052 +0,0 @@ -""" -Binary operations between SomeValues. -""" - -import py -import operator -from pypy.tool.pairtype import pair, pairtype -from pypy.annotation.model import SomeObject, SomeInteger, SomeBool, s_Bool -from pypy.annotation.model import SomeString, SomeChar, SomeList, SomeDict -from pypy.annotation.model import SomeUnicodeCodePoint, SomeUnicodeString -from pypy.annotation.model import SomeTuple, SomeImpossibleValue, s_ImpossibleValue -from pypy.annotation.model import SomeInstance, SomeBuiltin, SomeIterator -from pypy.annotation.model import SomePBC, SomeFloat, s_None, SomeByteArray -from pypy.annotation.model import SomeExternalObject, SomeWeakRef -from pypy.annotation.model import SomeAddress, SomeTypedAddressAccess -from pypy.annotation.model import SomeSingleFloat, SomeLongFloat, SomeType -from pypy.annotation.model import unionof, UnionError, missing_operation -from pypy.annotation.model import TLS -from pypy.annotation.model import read_can_only_throw -from pypy.annotation.model import add_knowntypedata, merge_knowntypedata -from pypy.annotation.model import SomeGenericCallable -from pypy.annotation.bookkeeper import getbookkeeper -from pypy.objspace.flow.model import Variable, Constant -from pypy.rlib import rarithmetic -from pypy.tool.error import AnnotatorError - -# convenience only! -def immutablevalue(x): - return getbookkeeper().immutablevalue(x) - -# XXX unify this with ObjSpace.MethodTable -BINARY_OPERATIONS = set(['add', 'sub', 'mul', 'div', 'mod', - 'truediv', 'floordiv', 'divmod', - 'and_', 'or_', 'xor', - 'lshift', 'rshift', - 'getitem', 'setitem', 'delitem', - 'getitem_idx', 'getitem_key', 'getitem_idx_key', - 'inplace_add', 'inplace_sub', 'inplace_mul', - 'inplace_truediv', 'inplace_floordiv', 'inplace_div', - 'inplace_mod', - 'inplace_lshift', 'inplace_rshift', - 'inplace_and', 'inplace_or', 'inplace_xor', - 'lt', 'le', 'eq', 'ne', 'gt', 'ge', 'is_', 'cmp', - 'coerce', - ] - +[opname+'_ovf' for opname in - """add sub mul floordiv div mod lshift - """.split() - ]) - -for opname in BINARY_OPERATIONS: - missing_operation(pairtype(SomeObject, SomeObject), opname) - -class __extend__(pairtype(SomeObject, SomeObject)): - - def union((obj1, obj2)): - raise UnionError(obj1, obj2) - - # inplace_xxx ---> xxx by default - def inplace_add((obj1, obj2)): return pair(obj1, obj2).add() - def inplace_sub((obj1, obj2)): return pair(obj1, obj2).sub() - def inplace_mul((obj1, obj2)): return pair(obj1, obj2).mul() - def inplace_truediv((obj1, obj2)): return pair(obj1, obj2).truediv() - def inplace_floordiv((obj1, obj2)): return pair(obj1, obj2).floordiv() - def inplace_div((obj1, obj2)): return pair(obj1, obj2).div() - def inplace_mod((obj1, obj2)): return pair(obj1, obj2).mod() - def inplace_lshift((obj1, obj2)): return pair(obj1, obj2).lshift() - def inplace_rshift((obj1, obj2)): return pair(obj1, obj2).rshift() - def inplace_and((obj1, obj2)): return pair(obj1, obj2).and_() - def inplace_or((obj1, obj2)): return pair(obj1, obj2).or_() - def inplace_xor((obj1, obj2)): return pair(obj1, obj2).xor() - - for name, func in locals().items(): - if name.startswith('inplace_'): - func.can_only_throw = [] - - inplace_div.can_only_throw = [ZeroDivisionError] - inplace_truediv.can_only_throw = [ZeroDivisionError] - inplace_floordiv.can_only_throw = [ZeroDivisionError] - inplace_mod.can_only_throw = [ZeroDivisionError] - - def lt((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const < obj2.const) - else: - getbookkeeper().count("non_int_comp", obj1, obj2) - return s_Bool - - def le((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const <= obj2.const) - else: - getbookkeeper().count("non_int_comp", obj1, obj2) - return s_Bool - - def eq((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const == obj2.const) - else: - getbookkeeper().count("non_int_eq", obj1, obj2) - return s_Bool - - def ne((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const != obj2.const) - else: - getbookkeeper().count("non_int_eq", obj1, obj2) - return s_Bool - - def gt((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const > obj2.const) - else: - getbookkeeper().count("non_int_comp", obj1, obj2) - return s_Bool - - def ge((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const >= obj2.const) - else: - getbookkeeper().count("non_int_comp", obj1, obj2) - return s_Bool - - def cmp((obj1, obj2)): - getbookkeeper().count("cmp", obj1, obj2) - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(cmp(obj1.const, obj2.const)) - else: - return SomeInteger() - - def is_((obj1, obj2)): - r = SomeBool() - if obj2.is_constant(): - if obj1.is_constant(): - r.const = obj1.const is obj2.const - if obj2.const is None and not obj1.can_be_none(): - r.const = False - elif obj1.is_constant(): - if obj1.const is None and not obj2.can_be_none(): - r.const = False - # XXX HACK HACK HACK - # XXX HACK HACK HACK - # XXX HACK HACK HACK - bk = getbookkeeper() - if bk is not None: # for testing - knowntypedata = {} - fn, block, i = bk.position_key - - annotator = bk.annotator - op = block.operations[i] - assert op.opname == "is_" - assert len(op.args) == 2 - - def bind(src_obj, tgt_obj, tgt_arg): - if hasattr(tgt_obj, 'is_type_of') and src_obj.is_constant(): - add_knowntypedata(knowntypedata, True, tgt_obj.is_type_of, - bk.valueoftype(src_obj.const)) - - assert annotator.binding(op.args[tgt_arg]) == tgt_obj - add_knowntypedata(knowntypedata, True, [op.args[tgt_arg]], src_obj) - - nonnone_obj = tgt_obj - if src_obj.is_constant() and src_obj.const is None and tgt_obj.can_be_none(): - nonnone_obj = tgt_obj.nonnoneify() - - add_knowntypedata(knowntypedata, False, [op.args[tgt_arg]], nonnone_obj) - - bind(obj2, obj1, 0) - bind(obj1, obj2, 1) - r.set_knowntypedata(knowntypedata) - - return r - - def divmod((obj1, obj2)): - getbookkeeper().count("divmod", obj1, obj2) - return SomeTuple([pair(obj1, obj2).div(), pair(obj1, obj2).mod()]) - - def coerce((obj1, obj2)): - getbookkeeper().count("coerce", obj1, obj2) - return pair(obj1, obj2).union() # reasonable enough - - # approximation of an annotation intersection, the result should be the annotation obj or - # the intersection of obj and improvement - def improve((obj, improvement)): - if not improvement.contains(obj) and obj.contains(improvement): - return improvement - else: - return obj - - # checked getitems - - def _getitem_can_only_throw(s_c1, s_o2): - impl = pair(s_c1, s_o2).getitem - return read_can_only_throw(impl, s_c1, s_o2) - - def getitem_idx_key((s_c1, s_o2)): - impl = pair(s_c1, s_o2).getitem - return impl() - getitem_idx_key.can_only_throw = _getitem_can_only_throw - - getitem_idx = getitem_idx_key - getitem_key = getitem_idx_key - - -class __extend__(pairtype(SomeType, SomeType)): - - def union((obj1, obj2)): - result = SomeType() - is_type_of1 = getattr(obj1, 'is_type_of', None) - is_type_of2 = getattr(obj2, 'is_type_of', None) - if obj1.is_immutable_constant() and obj2.is_immutable_constant() and obj1.const == obj2.const: - result.const = obj1.const - is_type_of = {} - if is_type_of1: - for v in is_type_of1: - is_type_of[v] = True - if is_type_of2: - for v in is_type_of2: - is_type_of[v] = True - if is_type_of: - result.is_type_of = is_type_of.keys() - else: - if is_type_of1 and is_type_of1 == is_type_of2: - result.is_type_of = is_type_of1 - return result - - -# cloning a function with identical code, for the can_only_throw attribute -def _clone(f, can_only_throw = None): - newfunc = type(f)(f.func_code, f.func_globals, f.func_name, - f.func_defaults, f.func_closure) - if can_only_throw is not None: - newfunc.can_only_throw = can_only_throw - return newfunc - -class __extend__(pairtype(SomeInteger, SomeInteger)): - # unsignedness is considered a rare and contagious disease - - def union((int1, int2)): - if int1.unsigned == int2.unsigned: - knowntype = rarithmetic.compute_restype(int1.knowntype, int2.knowntype) - else: - t1 = int1.knowntype - if t1 is bool: - t1 = int - t2 = int2.knowntype - if t2 is bool: - t2 = int - - if t2 is int: - if int2.nonneg == False: - raise UnionError, "Merging %s and a possibly negative int is not allowed" % t1 - knowntype = t1 - elif t1 is int: - if int1.nonneg == False: - raise UnionError, "Merging %s and a possibly negative int is not allowed" % t2 - knowntype = t2 - else: - raise UnionError, "Merging these types (%s, %s) is not supported" % (t1, t2) - return SomeInteger(nonneg=int1.nonneg and int2.nonneg, - knowntype=knowntype) - - or_ = xor = add = mul = _clone(union, []) - add_ovf = mul_ovf = _clone(union, [OverflowError]) - div = floordiv = mod = _clone(union, [ZeroDivisionError]) - div_ovf= floordiv_ovf = mod_ovf = _clone(union, [ZeroDivisionError, OverflowError]) - - def truediv((int1, int2)): - return SomeFloat() - truediv.can_only_throw = [ZeroDivisionError] - truediv_ovf = _clone(truediv, [ZeroDivisionError, OverflowError]) - - inplace_div = div - inplace_truediv = truediv - - def sub((int1, int2)): - knowntype = rarithmetic.compute_restype(int1.knowntype, int2.knowntype) - return SomeInteger(knowntype=knowntype) - sub.can_only_throw = [] - sub_ovf = _clone(sub, [OverflowError]) - - def and_((int1, int2)): - knowntype = rarithmetic.compute_restype(int1.knowntype, int2.knowntype) - return SomeInteger(nonneg=int1.nonneg or int2.nonneg, - knowntype=knowntype) - and_.can_only_throw = [] - - def lshift((int1, int2)): - if isinstance(int1, SomeBool): - return SomeInteger() - else: - return SomeInteger(knowntype=int1.knowntype) - lshift.can_only_throw = [] - lshift_ovf = _clone(lshift, [OverflowError]) - - def rshift((int1, int2)): - if isinstance(int1, SomeBool): - return SomeInteger(nonneg=True) - else: - return SomeInteger(nonneg=int1.nonneg, knowntype=int1.knowntype) - rshift.can_only_throw = [] - - def _compare_helper((int1, int2), opname, operation): - r = SomeBool() - if int1.is_immutable_constant() and int2.is_immutable_constant(): - r.const = operation(int1.const, int2.const) - # - # The rest of the code propagates nonneg information between - # the two arguments. - # - # Doing the right thing when int1 or int2 change from signed - # to unsigned (r_uint) is almost impossible. See test_intcmp_bug. - # Instead, we only deduce constrains on the operands in the - # case where they are both signed. In other words, if y is - # nonneg then "assert x>=y" will let the annotator know that - # x is nonneg too, but it will not work if y is unsigned. - # - if not (rarithmetic.signedtype(int1.knowntype) and - rarithmetic.signedtype(int2.knowntype)): - return r - knowntypedata = {} - # XXX HACK HACK HACK - fn, block, i = getbookkeeper().position_key - op = block.operations[i] - assert op.opname == opname - assert len(op.args) == 2 - def tointtype(int0): - if int0.knowntype is bool: - return int - return int0.knowntype - if int1.nonneg and isinstance(op.args[1], Variable): - case = opname in ('lt', 'le', 'eq') - - add_knowntypedata(knowntypedata, case, [op.args[1]], - SomeInteger(nonneg=True, knowntype=tointtype(int2))) - if int2.nonneg and isinstance(op.args[0], Variable): - case = opname in ('gt', 'ge', 'eq') - add_knowntypedata(knowntypedata, case, [op.args[0]], - SomeInteger(nonneg=True, knowntype=tointtype(int1))) - r.set_knowntypedata(knowntypedata) - # a special case for 'x < 0' or 'x >= 0', - # where 0 is a flow graph Constant - # (in this case we are sure that it cannot become a r_uint later) - if (isinstance(op.args[1], Constant) and - type(op.args[1].value) is int and # filter out Symbolics - op.args[1].value == 0): - if int1.nonneg: - if opname == 'lt': - r.const = False - if opname == 'ge': - r.const = True - return r - - def lt(intint): return intint._compare_helper('lt', operator.lt) - def le(intint): return intint._compare_helper('le', operator.le) - def eq(intint): return intint._compare_helper('eq', operator.eq) - def ne(intint): return intint._compare_helper('ne', operator.ne) - def gt(intint): return intint._compare_helper('gt', operator.gt) - def ge(intint): return intint._compare_helper('ge', operator.ge) - - -class __extend__(pairtype(SomeBool, SomeBool)): - - def union((boo1, boo2)): - s = SomeBool() - if getattr(boo1, 'const', -1) == getattr(boo2, 'const', -2): - s.const = boo1.const - if hasattr(boo1, 'knowntypedata') and \ - hasattr(boo2, 'knowntypedata'): - ktd = merge_knowntypedata(boo1.knowntypedata, boo2.knowntypedata) - s.set_knowntypedata(ktd) - return s - - def and_((boo1, boo2)): - s = SomeBool() - if boo1.is_constant(): - if not boo1.const: - s.const = False - else: - return boo2 - if boo2.is_constant(): - if not boo2.const: - s.const = False - return s - - def or_((boo1, boo2)): - s = SomeBool() - if boo1.is_constant(): - if boo1.const: - s.const = True - else: - return boo2 - if boo2.is_constant(): - if boo2.const: - s.const = True - return s - - def xor((boo1, boo2)): - s = SomeBool() - if boo1.is_constant() and boo2.is_constant(): - s.const = boo1.const ^ boo2.const - return s - -class __extend__(pairtype(SomeString, SomeString)): - - def union((str1, str2)): - can_be_None = str1.can_be_None or str2.can_be_None - no_nul = str1.no_nul and str2.no_nul - return SomeString(can_be_None=can_be_None, no_nul=no_nul) - - def add((str1, str2)): - # propagate const-ness to help getattr(obj, 'prefix' + const_name) - result = SomeString(no_nul=str1.no_nul and str2.no_nul) - if str1.is_immutable_constant() and str2.is_immutable_constant(): - result.const = str1.const + str2.const - return result - -class __extend__(pairtype(SomeByteArray, SomeByteArray)): - def union((b1, b2)): - can_be_None = b1.can_be_None or b2.can_be_None - return SomeByteArray(can_be_None=can_be_None) - - def add((b1, b2)): - result = SomeByteArray() - if b1.is_immutable_constant() and b2.is_immutable_constant(): - result.const = b1.const + b2.const - return result - -class __extend__(pairtype(SomeByteArray, SomeInteger)): - def getitem((s_b, s_i)): - return SomeInteger() - - def setitem((s_b, s_i), s_i2): - assert isinstance(s_i2, SomeInteger) - -class __extend__(pairtype(SomeString, SomeByteArray), - pairtype(SomeByteArray, SomeString), - pairtype(SomeChar, SomeByteArray), - pairtype(SomeByteArray, SomeChar)): - def add((b1, b2)): - result = SomeByteArray() - if b1.is_immutable_constant() and b2.is_immutable_constant(): - result.const = b1.const + b2.const - return result - -class __extend__(pairtype(SomeChar, SomeChar)): - - def union((chr1, chr2)): - return SomeChar() - - -class __extend__(pairtype(SomeChar, SomeUnicodeCodePoint), - pairtype(SomeUnicodeCodePoint, SomeChar)): - def union((uchr1, uchr2)): - return SomeUnicodeCodePoint() - -class __extend__(pairtype(SomeUnicodeCodePoint, SomeUnicodeCodePoint)): - def union((uchr1, uchr2)): - return SomeUnicodeCodePoint() - - def add((chr1, chr2)): - return SomeUnicodeString() - -class __extend__(pairtype(SomeString, SomeUnicodeString), - pairtype(SomeUnicodeString, SomeString)): - def mod((str, unistring)): - raise NotImplementedError( - "string formatting mixing strings and unicode not supported") - - -class __extend__(pairtype(SomeString, SomeTuple), - pairtype(SomeUnicodeString, SomeTuple)): - def mod((s_string, s_tuple)): - is_string = isinstance(s_string, SomeString) - is_unicode = isinstance(s_string, SomeUnicodeString) - assert is_string or is_unicode - for s_item in s_tuple.items: - if (is_unicode and isinstance(s_item, (SomeChar, SomeString)) or - is_string and isinstance(s_item, (SomeUnicodeCodePoint, - SomeUnicodeString))): - raise NotImplementedError( - "string formatting mixing strings and unicode not supported") - getbookkeeper().count('strformat', s_string, s_tuple) - no_nul = s_string.no_nul - for s_item in s_tuple.items: - if isinstance(s_item, SomeFloat): - pass # or s_item is a subclass, like SomeInteger - elif (isinstance(s_item, SomeString) or - isinstance(s_item, SomeUnicodeString)) and s_item.no_nul: - pass - else: - no_nul = False - break - return s_string.__class__(no_nul=no_nul) - - -class __extend__(pairtype(SomeString, SomeObject), - pairtype(SomeUnicodeString, SomeObject)): - - def mod((s_string, args)): - getbookkeeper().count('strformat', s_string, args) - return s_string.__class__() - -class __extend__(pairtype(SomeFloat, SomeFloat)): - - def union((flt1, flt2)): - return SomeFloat() - - add = sub = mul = union - - def div((flt1, flt2)): - return SomeFloat() - div.can_only_throw = [] - truediv = div - - # repeat these in order to copy the 'can_only_throw' attribute - inplace_div = div - inplace_truediv = truediv - - -class __extend__(pairtype(SomeSingleFloat, SomeSingleFloat)): - - def union((flt1, flt2)): - return SomeSingleFloat() - - -class __extend__(pairtype(SomeLongFloat, SomeLongFloat)): - - def union((flt1, flt2)): - return SomeLongFloat() - - -class __extend__(pairtype(SomeList, SomeList)): - - def union((lst1, lst2)): - return SomeList(lst1.listdef.union(lst2.listdef)) - - def add((lst1, lst2)): - return lst1.listdef.offspring(lst2.listdef) - - def eq((lst1, lst2)): - lst1.listdef.agree(lst2.listdef) - return s_Bool - ne = eq - - -class __extend__(pairtype(SomeList, SomeObject)): - - def inplace_add((lst1, obj2)): - lst1.method_extend(obj2) - return lst1 - inplace_add.can_only_throw = [] - - def inplace_mul((lst1, obj2)): - lst1.listdef.resize() - return lst1 - inplace_mul.can_only_throw = [] - -class __extend__(pairtype(SomeTuple, SomeTuple)): - - def union((tup1, tup2)): - if len(tup1.items) != len(tup2.items): - raise UnionError("cannot take the union of a tuple of length %d " - "and a tuple of length %d" % (len(tup1.items), - len(tup2.items))) - else: - unions = [unionof(x,y) for x,y in zip(tup1.items, tup2.items)] - return SomeTuple(items = unions) - - def add((tup1, tup2)): - return SomeTuple(items = tup1.items + tup2.items) - - def eq(tup1tup2): - tup1tup2.union() - return s_Bool - ne = eq - - def lt((tup1, tup2)): - raise Exception("unsupported: (...) < (...)") - def le((tup1, tup2)): - raise Exception("unsupported: (...) <= (...)") - def gt((tup1, tup2)): - raise Exception("unsupported: (...) > (...)") - def ge((tup1, tup2)): - raise Exception("unsupported: (...) >= (...)") - - -class __extend__(pairtype(SomeDict, SomeDict)): - - def union((dic1, dic2)): - return SomeDict(dic1.dictdef.union(dic2.dictdef)) - - -class __extend__(pairtype(SomeDict, SomeObject)): - - def _can_only_throw(dic1, *ignore): - if dic1.dictdef.dictkey.custom_eq_hash: - return None - return [KeyError] - - def getitem((dic1, obj2)): - getbookkeeper().count("dict_getitem", dic1) - dic1.dictdef.generalize_key(obj2) - return dic1.dictdef.read_value() - getitem.can_only_throw = _can_only_throw - - def setitem((dic1, obj2), s_value): - getbookkeeper().count("dict_setitem", dic1) - dic1.dictdef.generalize_key(obj2) - dic1.dictdef.generalize_value(s_value) - setitem.can_only_throw = _can_only_throw - - def delitem((dic1, obj2)): - getbookkeeper().count("dict_delitem", dic1) - dic1.dictdef.generalize_key(obj2) - delitem.can_only_throw = _can_only_throw - - -class __extend__(pairtype(SomeTuple, SomeInteger)): - - def getitem((tup1, int2)): - if int2.is_immutable_constant(): - try: - return tup1.items[int2.const] - except IndexError: - return s_ImpossibleValue - else: - getbookkeeper().count("tuple_random_getitem", tup1) - return unionof(*tup1.items) - getitem.can_only_throw = [IndexError] - - -class __extend__(pairtype(SomeList, SomeInteger)): - - def mul((lst1, int2)): - return lst1.listdef.offspring() - - def getitem((lst1, int2)): - getbookkeeper().count("list_getitem", int2) - return lst1.listdef.read_item() - getitem.can_only_throw = [] - - getitem_key = getitem - - def getitem_idx((lst1, int2)): - getbookkeeper().count("list_getitem", int2) - return lst1.listdef.read_item() - getitem_idx.can_only_throw = [IndexError] - - getitem_idx_key = getitem_idx - - def setitem((lst1, int2), s_value): - getbookkeeper().count("list_setitem", int2) - lst1.listdef.mutate() - lst1.listdef.generalize(s_value) - setitem.can_only_throw = [IndexError] - - def delitem((lst1, int2)): - getbookkeeper().count("list_delitem", int2) - lst1.listdef.resize() - delitem.can_only_throw = [IndexError] - -class __extend__(pairtype(SomeString, SomeInteger)): - - def getitem((str1, int2)): - getbookkeeper().count("str_getitem", int2) - return SomeChar() - getitem.can_only_throw = [] - - getitem_key = getitem - - def getitem_idx((str1, int2)): - getbookkeeper().count("str_getitem", int2) - return SomeChar() - getitem_idx.can_only_throw = [IndexError] - - getitem_idx_key = getitem_idx - - def mul((str1, int2)): # xxx do we want to support this - getbookkeeper().count("str_mul", str1, int2) - return SomeString(no_nul=str1.no_nul) - -class __extend__(pairtype(SomeUnicodeString, SomeInteger)): - def getitem((str1, int2)): - getbookkeeper().count("str_getitem", int2) - return SomeUnicodeCodePoint() - getitem.can_only_throw = [] - - getitem_key = getitem - - def getitem_idx((str1, int2)): - getbookkeeper().count("str_getitem", int2) - return SomeUnicodeCodePoint() - getitem_idx.can_only_throw = [IndexError] - - getitem_idx_key = getitem_idx - - def mul((str1, int2)): # xxx do we want to support this - getbookkeeper().count("str_mul", str1, int2) - return SomeUnicodeString() - -class __extend__(pairtype(SomeInteger, SomeString), - pairtype(SomeInteger, SomeUnicodeString)): - - def mul((int1, str2)): # xxx do we want to support this - getbookkeeper().count("str_mul", str2, int1) - return str2.basestringclass() - -class __extend__(pairtype(SomeUnicodeCodePoint, SomeUnicodeString), - pairtype(SomeUnicodeString, SomeUnicodeCodePoint), - pairtype(SomeUnicodeString, SomeUnicodeString)): - def union((str1, str2)): - return SomeUnicodeString(can_be_None=str1.can_be_none() or - str2.can_be_none()) - - def add((str1, str2)): - # propagate const-ness to help getattr(obj, 'prefix' + const_name) - result = SomeUnicodeString() - if str1.is_immutable_constant() and str2.is_immutable_constant(): - result.const = str1.const + str2.const - return result - -class __extend__(pairtype(SomeInteger, SomeList)): - - def mul((int1, lst2)): - return lst2.listdef.offspring() - - -class __extend__(pairtype(SomeInstance, SomeInstance)): - - def union((ins1, ins2)): - if ins1.classdef is None or ins2.classdef is None: - # special case only - basedef = None - else: - basedef = ins1.classdef.commonbase(ins2.classdef) - if basedef is None: - raise UnionError(ins1, ins2) - flags = ins1.flags - if flags: - flags = flags.copy() - for key, value in flags.items(): - if key not in ins2.flags or ins2.flags[key] != value: - del flags[key] - return SomeInstance(basedef, - can_be_None=ins1.can_be_None or ins2.can_be_None, - flags=flags) - - def improve((ins1, ins2)): - if ins1.classdef is None: - resdef = ins2.classdef - elif ins2.classdef is None: - resdef = ins1.classdef - else: - basedef = ins1.classdef.commonbase(ins2.classdef) - if basedef is ins1.classdef: - resdef = ins2.classdef - elif basedef is ins2.classdef: - resdef = ins1.classdef - else: - if ins1.can_be_None and ins2.can_be_None: - return s_None - else: - return s_ImpossibleValue - res = SomeInstance(resdef, can_be_None=ins1.can_be_None and ins2.can_be_None) - if ins1.contains(res) and ins2.contains(res): - return res # fine - else: - # this case can occur in the presence of 'const' attributes, - # which we should try to preserve. Fall-back... - thistype = pairtype(SomeInstance, SomeInstance) - return super(thistype, pair(ins1, ins2)).improve() - - -class __extend__(pairtype(SomeIterator, SomeIterator)): - - def union((iter1, iter2)): - s_cont = unionof(iter1.s_container, iter2.s_container) - if iter1.variant != iter2.variant: - raise UnionError("merging incompatible iterators variants") - return SomeIterator(s_cont, *iter1.variant) - - -class __extend__(pairtype(SomeBuiltin, SomeBuiltin)): - - def union((bltn1, bltn2)): - if (bltn1.analyser != bltn2.analyser or - bltn1.methodname != bltn2.methodname or - bltn1.s_self is None or bltn2.s_self is None): - raise UnionError("cannot merge two different builtin functions " - "or methods:\n %r\n %r" % (bltn1, bltn2)) - s_self = unionof(bltn1.s_self, bltn2.s_self) - return SomeBuiltin(bltn1.analyser, s_self, methodname=bltn1.methodname) - -class __extend__(pairtype(SomePBC, SomePBC)): - - def union((pbc1, pbc2)): - d = pbc1.descriptions.copy() - d.update(pbc2.descriptions) - return SomePBC(d, can_be_None = pbc1.can_be_None or pbc2.can_be_None) - - def is_((pbc1, pbc2)): - thistype = pairtype(SomePBC, SomePBC) - s = super(thistype, pair(pbc1, pbc2)).is_() - if not s.is_constant(): - if not pbc1.can_be_None or not pbc2.can_be_None: - for desc in pbc1.descriptions: - if desc in pbc2.descriptions: - break - else: - s.const = False # no common desc in the two sets - return s - -class __extend__(pairtype(SomeGenericCallable, SomePBC)): - def union((gencall, pbc)): - for desc in pbc.descriptions: - unique_key = desc - bk = desc.bookkeeper - s_result = bk.emulate_pbc_call(unique_key, pbc, gencall.args_s) - s_result = unionof(s_result, gencall.s_result) - assert gencall.s_result.contains(s_result) - return gencall - -class __extend__(pairtype(SomePBC, SomeGenericCallable)): - def union((pbc, gencall)): - return pair(gencall, pbc).union() - -class __extend__(pairtype(SomeImpossibleValue, SomeObject)): - def union((imp1, obj2)): - return obj2 - -class __extend__(pairtype(SomeObject, SomeImpossibleValue)): - def union((obj1, imp2)): - return obj1 - -# mixing Nones with other objects - -def _make_none_union(classname, constructor_args='', glob=None): - if glob is None: - glob = globals() - loc = locals() - source = py.code.Source(""" - class __extend__(pairtype(%(classname)s, SomePBC)): - def union((obj, pbc)): - if pbc.isNone(): - return %(classname)s(%(constructor_args)s) - else: - return SomeObject() - - class __extend__(pairtype(SomePBC, %(classname)s)): - def union((pbc, obj)): - if pbc.isNone(): - return %(classname)s(%(constructor_args)s) - else: - return SomeObject() - """ % loc) - exec source.compile() in glob - -_make_none_union('SomeInstance', 'classdef=obj.classdef, can_be_None=True') -_make_none_union('SomeString', 'no_nul=obj.no_nul, can_be_None=True') -_make_none_union('SomeUnicodeString', 'can_be_None=True') -_make_none_union('SomeList', 'obj.listdef') -_make_none_union('SomeDict', 'obj.dictdef') -_make_none_union('SomeExternalObject', 'obj.knowntype') -_make_none_union('SomeWeakRef', 'obj.classdef') - -# getitem on SomePBCs, in particular None fails - -class __extend__(pairtype(SomePBC, SomeObject)): - def getitem((pbc, o)): - if not pbc.isNone(): - raise AnnotatorError("getitem on %r" % pbc) - return s_ImpossibleValue - - def setitem((pbc, o), s_value): - if not pbc.isNone(): - raise AnnotatorError("setitem on %r" % pbc) - -class __extend__(pairtype(SomePBC, SomeString)): - def add((pbc, o)): - if not pbc.isNone(): - raise AnnotatorError('add on %r' % pbc) - return s_ImpossibleValue - -class __extend__(pairtype(SomeString, SomePBC)): - def add((o, pbc)): - if not pbc.isNone(): - raise AnnotatorError('add on %r' % pbc) - return s_ImpossibleValue - -class __extend__(pairtype(SomeExternalObject, SomeExternalObject)): - def union((ext1, ext2)): - if ext1.knowntype == ext2.knowntype: - return SomeExternalObject(ext1.knowntype) - return SomeObject() - -# ____________________________________________________________ -# annotation of low-level types -from pypy.annotation.model import SomePtr, SomeOOInstance, SomeOOClass -from pypy.annotation.model import SomeOOObject -from pypy.annotation.model import ll_to_annotation, annotation_to_lltype -from pypy.rpython.ootypesystem import ootype - -_make_none_union('SomeOOInstance', 'ootype=obj.ootype, can_be_None=True') - -class __extend__(pairtype(SomePtr, SomePtr)): - def union((p1, p2)): - assert p1.ll_ptrtype == p2.ll_ptrtype,("mixing of incompatible pointer types: %r, %r" % - (p1.ll_ptrtype, p2.ll_ptrtype)) - return SomePtr(p1.ll_ptrtype) - -class __extend__(pairtype(SomePtr, SomeInteger)): - - def getitem((p, int1)): - example = p.ll_ptrtype._example() - try: - v = example[0] - except IndexError: - return None # impossible value, e.g. FixedSizeArray(0) - return ll_to_annotation(v) - getitem.can_only_throw = [] - - def setitem((p, int1), s_value): # just doing checking - example = p.ll_ptrtype._example() - if example[0] is not None: # ignore Void s_value - v_lltype = annotation_to_lltype(s_value) - example[0] = v_lltype._defl() - setitem.can_only_throw = [] - -class __extend__(pairtype(SomePtr, SomeObject)): - def union((p, obj)): - assert False, ("mixing pointer type %r with something else %r" % (p.ll_ptrtype, obj)) - - def getitem((p, obj)): - assert False,"ptr %r getitem index not an int: %r" % (p.ll_ptrtype, obj) - - def setitem((p, obj), s_value): - assert False,"ptr %r setitem index not an int: %r" % (p.ll_ptrtype, obj) - -class __extend__(pairtype(SomeObject, SomePtr)): - def union((obj, p2)): - return pair(p2, obj).union() - - -class __extend__(pairtype(SomeOOInstance, SomeOOInstance)): - def union((r1, r2)): - common = ootype.commonBaseclass(r1.ootype, r2.ootype) - assert common is not None, 'Mixing of incompatible instances %r, %r' %(r1.ootype, r2.ootype) - return SomeOOInstance(common, can_be_None=r1.can_be_None or r2.can_be_None) - -class __extend__(pairtype(SomeOOClass, SomeOOClass)): - def union((r1, r2)): - if r1.ootype is None: - common = r2.ootype - elif r2.ootype is None: - common = r1.ootype - elif r1.ootype == r2.ootype: - common = r1.ootype - elif isinstance(r1.ootype, ootype.Instance) and isinstance(r2.ootype, ootype.Instance): - common = ootype.commonBaseclass(r1.ootype, r2.ootype) - assert common is not None, ('Mixing of incompatible classes %r, %r' - % (r1.ootype, r2.ootype)) - else: - common = ootype.Object - return SomeOOClass(common) - -class __extend__(pairtype(SomeOOInstance, SomeObject)): - def union((r, obj)): - assert False, ("mixing reference type %r with something else %r" % (r.ootype, obj)) - -class __extend__(pairtype(SomeObject, SomeOOInstance)): - def union((obj, r2)): - return pair(r2, obj).union() - -class __extend__(pairtype(SomeOOObject, SomeOOObject)): - def union((r1, r2)): - assert r1.ootype is ootype.Object and r2.ootype is ootype.Object - return SomeOOObject() - -#_________________________________________ -# weakrefs - -class __extend__(pairtype(SomeWeakRef, SomeWeakRef)): - def union((s_wrf1, s_wrf2)): - if s_wrf1.classdef is None: - basedef = s_wrf2.classdef # s_wrf1 is known to be dead - elif s_wrf2.classdef is None: - basedef = s_wrf1.classdef # s_wrf2 is known to be dead - else: - basedef = s_wrf1.classdef.commonbase(s_wrf2.classdef) - if basedef is None: # no common base class! complain... - return SomeObject() - return SomeWeakRef(basedef) - -#_________________________________________ -# memory addresses - -class __extend__(pairtype(SomeAddress, SomeAddress)): - def union((s_addr1, s_addr2)): - return SomeAddress() - - def sub((s_addr1, s_addr2)): - if s_addr1.is_null_address() and s_addr2.is_null_address(): - return getbookkeeper().immutablevalue(0) - return SomeInteger() - - def is_((s_addr1, s_addr2)): - assert False, "comparisons with is not supported by addresses" - -class __extend__(pairtype(SomeTypedAddressAccess, SomeTypedAddressAccess)): - def union((s_taa1, s_taa2)): - assert s_taa1.type == s_taa2.type - return s_taa1 - -class __extend__(pairtype(SomeTypedAddressAccess, SomeInteger)): - def getitem((s_taa, s_int)): - from pypy.annotation.model import lltype_to_annotation - return lltype_to_annotation(s_taa.type) - getitem.can_only_throw = [] - - def setitem((s_taa, s_int), s_value): - from pypy.annotation.model import annotation_to_lltype - assert annotation_to_lltype(s_value) is s_taa.type - setitem.can_only_throw = [] - - -class __extend__(pairtype(SomeAddress, SomeInteger)): - def add((s_addr, s_int)): - return SomeAddress() - - def sub((s_addr, s_int)): - return SomeAddress() - -class __extend__(pairtype(SomeAddress, SomeImpossibleValue)): - # need to override this specifically to hide the 'raise UnionError' - # of pairtype(SomeAddress, SomeObject). - def union((s_addr, s_imp)): - return s_addr - -class __extend__(pairtype(SomeImpossibleValue, SomeAddress)): - # need to override this specifically to hide the 'raise UnionError' - # of pairtype(SomeObject, SomeAddress). - def union((s_imp, s_addr)): - return s_addr - -class __extend__(pairtype(SomeAddress, SomeObject)): - def union((s_addr, s_obj)): - raise UnionError, "union of address and anything else makes no sense" - -class __extend__(pairtype(SomeObject, SomeAddress)): - def union((s_obj, s_addr)): - raise UnionError, "union of address and anything else makes no sense" diff --git a/pypy/annotation/bookkeeper.py b/pypy/annotation/bookkeeper.py deleted file mode 100644 --- a/pypy/annotation/bookkeeper.py +++ /dev/null @@ -1,797 +0,0 @@ -""" -The Bookkeeper class. -""" - -from __future__ import absolute_import - -import sys, types, inspect, weakref - -from pypy.objspace.flow.model import Constant From noreply at buildbot.pypy.org Mon Jan 21 02:26:01 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Mon, 21 Jan 2013 02:26:01 +0100 (CET) Subject: [pypy-commit] pypy inline-virtualref: fixed typo Message-ID: <20130121012601.73D621C1309@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: inline-virtualref Changeset: r60266:9f224d4bcfde Date: 2013-01-20 17:25 -0800 http://bitbucket.org/pypy/pypy/changeset/9f224d4bcfde/ Log: fixed typo diff --git a/rpython/jit/metainterp/virtualref.py b/rpython/jit/metainterp/virtualref.py --- a/rpython/jit/metainterp/virtualref.py +++ b/rpython/jit/metainterp/virtualref.py @@ -1,4 +1,4 @@ -from rpython.annotation import model as annmodel +from rpython.annotator import model as annmodel from rpython.jit.codewriter import heaptracker from rpython.jit.metainterp import history from rpython.rlib.jit import InvalidVirtualRef From noreply at buildbot.pypy.org Mon Jan 21 06:46:39 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Mon, 21 Jan 2013 06:46:39 +0100 (CET) Subject: [pypy-commit] pypy inline-virtualref: try to pass stuff that's really the right type, except now it fails for another (more obscure reasons) Message-ID: <20130121054639.6355A1C0A33@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: inline-virtualref Changeset: r60267:81cea2b96fd7 Date: 2013-01-20 21:46 -0800 http://bitbucket.org/pypy/pypy/changeset/81cea2b96fd7/ Log: try to pass stuff that's really the right type, except now it fails for another (more obscure reasons) diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py --- a/rpython/jit/backend/llgraph/runner.py +++ b/rpython/jit/backend/llgraph/runner.py @@ -1,19 +1,21 @@ -import py, weakref +import weakref + +import py + from rpython.jit.backend import model from rpython.jit.backend.llgraph import support -from rpython.jit.metainterp.history import AbstractDescr -from rpython.jit.metainterp.history import Const, getkind -from rpython.jit.metainterp.history import INT, REF, FLOAT, VOID -from rpython.jit.metainterp.resoperation import rop from rpython.jit.codewriter import longlong, heaptracker from rpython.jit.codewriter.effectinfo import EffectInfo - +from rpython.jit.metainterp.history import (AbstractDescr, Const, getkind, INT, + REF, FLOAT, VOID) +from rpython.jit.metainterp.resoperation import rop from rpython.rtyper.llinterp import LLInterpreter, LLException from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rclass, rstr - +from rpython.rlib.objectmodel import compute_unique_id from rpython.rlib.rarithmetic import ovfcheck, r_uint, r_ulonglong from rpython.rlib.rtimer import read_timestamp + class LLTrace(object): has_been_freed = False invalid = False @@ -171,6 +173,7 @@ self.rtyper = rtyper self.llinterp = LLInterpreter(rtyper) self.descrs = {} + self.token_to_frame = {} class MiniStats: pass self.stats = stats or MiniStats() @@ -262,7 +265,7 @@ return gcref def force(self, force_token): - frame = force_token + frame = self.token_to_frame[force_token] assert isinstance(frame, LLFrame) assert frame.forced_deadframe is None values = [] @@ -905,7 +908,8 @@ return self.cpu.bh_new_with_vtable(vtable, descr) def execute_force_token(self, _): - return self + self.cpu.token_to_frame[compute_unique_id(self)] = self + return compute_unique_id(self) def execute_cond_call_gc_wb(self, descr, a, b): py.test.skip("cond_call_gc_wb not supported") diff --git a/rpython/jit/metainterp/virtualref.py b/rpython/jit/metainterp/virtualref.py --- a/rpython/jit/metainterp/virtualref.py +++ b/rpython/jit/metainterp/virtualref.py @@ -1,7 +1,7 @@ from rpython.annotator import model as annmodel from rpython.jit.codewriter import heaptracker from rpython.jit.metainterp import history -from rpython.rlib.jit import InvalidVirtualRef +from rpython.rlib.jit import InvalidVirtualRef, look_inside_iff from rpython.rtyper.annlowlevel import MixLevelHelperAnnotator from rpython.rtyper.lltypesystem import lltype, llmemory, rclass from rpython.rtyper.rmodel import inputconst, log diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py --- a/rpython/rtyper/lltypesystem/rffi.py +++ b/rpython/rtyper/lltypesystem/rffi.py @@ -1,26 +1,26 @@ +import os +import sys + import py + from rpython.annotator import model as annmodel -from rpython.rtyper.lltypesystem import lltype, rstr -from rpython.rtyper.lltypesystem import ll2ctypes -from rpython.rtyper.lltypesystem.llmemory import cast_ptr_to_adr -from rpython.rtyper.lltypesystem.llmemory import itemoffsetof, raw_memcopy from rpython.annotator.model import lltype_to_annotation +from rpython.rlib import rarithmetic, rgc, jit +from rpython.rlib.objectmodel import (Symbolic, keepalive_until_here, + enforceargs, we_are_translated) +from rpython.rlib.rarithmetic import maxint, LONG_BIT +from rpython.rlib.rstring import StringBuilder, UnicodeBuilder, assert_str0 +from rpython.rlib.unroll import unrolling_iterable +from rpython.rtyper.annlowlevel import llhelper +from rpython.rtyper.extregistry import ExtRegistryEntry +from rpython.rtyper.lltypesystem import lltype, ll2ctypes, llmemory +from rpython.rtyper.lltypesystem.llmemory import (cast_ptr_to_adr, itemoffsetof, + raw_memcopy) +from rpython.rtyper.tool.rfficache import platform, sizeof_c_type from rpython.tool.sourcetools import func_with_new_name -from rpython.rlib.objectmodel import Symbolic -from rpython.rlib.objectmodel import keepalive_until_here, enforceargs -from rpython.rlib import rarithmetic, rgc -from rpython.rtyper.extregistry import ExtRegistryEntry -from rpython.rlib.unroll import unrolling_iterable -from rpython.rtyper.tool.rfficache import platform, sizeof_c_type +from rpython.translator.platform import CompilationError from rpython.translator.tool.cbuild import ExternalCompilationInfo -from rpython.rtyper.annlowlevel import llhelper -from rpython.rlib.objectmodel import we_are_translated -from rpython.rlib.rstring import StringBuilder, UnicodeBuilder, assert_str0 -from rpython.rlib import jit -from rpython.rtyper.lltypesystem import llmemory -from rpython.rlib.rarithmetic import maxint, LONG_BIT -from rpython.translator.platform import CompilationError -import os, sys + class CConstant(Symbolic): """ A C-level constant, maybe #define, rendered directly. From noreply at buildbot.pypy.org Mon Jan 21 07:02:52 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Mon, 21 Jan 2013 07:02:52 +0100 (CET) Subject: [pypy-commit] pypy inline-virtualref: stuff begins to work! Message-ID: <20130121060252.686371C025C@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: inline-virtualref Changeset: r60268:483efa5ba3d5 Date: 2013-01-20 22:02 -0800 http://bitbucket.org/pypy/pypy/changeset/483efa5ba3d5/ Log: stuff begins to work! diff --git a/rpython/jit/metainterp/virtualref.py b/rpython/jit/metainterp/virtualref.py --- a/rpython/jit/metainterp/virtualref.py +++ b/rpython/jit/metainterp/virtualref.py @@ -132,6 +132,7 @@ return inst # common, fast case return self.force_virtual(inst) + self.set_force_now_ptr() FUNC = lltype.FuncType([rclass.OBJECTPTR], rclass.OBJECTPTR) args_s = [annmodel.lltype_to_annotation(v) for v in FUNC.ARGS] s_result = annmodel.lltype_to_annotation(FUNC.RESULT) @@ -140,6 +141,17 @@ mixlevelann.finish() return c_func + def set_force_now_ptr(self): + if hasattr(self, "force_now_ptr"): + return + + def force_now(token): + from rpython.jit.metainterp.compile import ResumeGuardForcedDescr + ResumeGuardForcedDescr.force_now(self.cpu, token) + FUNC = lltype.FuncType([lltype.Signed], lltype.Void) + funcptr = self.warmrunnerdesc.helper_func(lltype.Ptr(FUNC), force_now) + self.force_now_ptr = funcptr + def get_is_virtual_fnptr(self): def is_virtual(inst): if not inst: @@ -151,8 +163,6 @@ return inputconst(lltype.typeOf(funcptr), funcptr) def force_virtual(self, inst): - from rpython.jit.metainterp.compile import ResumeGuardForcedDescr - vref = lltype.cast_pointer(lltype.Ptr(self.JIT_VIRTUAL_REF), inst) token = vref.virtual_token if token != self.TOKEN_NONE: @@ -165,7 +175,7 @@ vref.virtual_token = self.TOKEN_NONE else: assert not vref.forced - ResumeGuardForcedDescr.force_now(self.cpu, token) + self.force_now_ptr(token) assert vref.virtual_token == self.TOKEN_NONE assert vref.forced elif not vref.forced: From noreply at buildbot.pypy.org Mon Jan 21 08:43:17 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 08:43:17 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: uh, fix the bug Message-ID: <20130121074317.DC8FC1C025C@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60269:ef194d2932d8 Date: 2013-01-21 09:42 +0200 http://bitbucket.org/pypy/pypy/changeset/ef194d2932d8/ Log: uh, fix the bug diff --git a/pypy/TODO b/pypy/TODO --- a/pypy/TODO +++ b/pypy/TODO @@ -1,6 +1,8 @@ * 32bit x86 * ARM -* shadowstack + asmgcc +* shadowstack + asmgcc (shadowstack needs reloading of ebp after frame + got potentially moved, which is after each potentially collecting call + or slowpath malloc) * kill jit2gc on translator * fix test_singlefloats in test_calling_conventions \ No newline at end of file diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -1,4 +1,5 @@ from rpython.rlib.rarithmetic import ovfcheck +from rpython.rlib import rgc from rpython.rtyper.lltypesystem import lltype, llmemory from rpython.jit.metainterp import history from rpython.jit.metainterp.history import ConstInt, BoxPtr, ConstPtr @@ -142,6 +143,7 @@ lgt_box = history.BoxInt() frame = history.BoxPtr() jfi = loop_token.compiled_loop_token.frame_info + rgc._make_sure_does_not_move(jfi) llref = lltype.cast_opaque_ptr(llmemory.GCREF, jfi) op0 = ResOperation(rop.GETFIELD_GC, [history.ConstPtr(llref)], lgt_box, descr=descrs.jfi_frame_depth) @@ -154,6 +156,10 @@ op2 = ResOperation(rop.SETFIELD_GC, [frame, history.ConstPtr(llref)], None, descr=descrs.jf_frame_info) self.newops.append(op2) + llref = lltype.cast_opaque_ptr(llmemory.GCREF, jfi.jfi_gcmap) + op3 = ResOperation(rop.SETFIELD_GC, [frame, history.ConstPtr(llref)], + None, descr=descrs.jf_gcmap) + self.newops.append(op3) for i, arg in enumerate(op.getarglist()): index, descr = self.cpu.getarraydescr_for_frame(arg.type, i) self.newops.append(ResOperation(rop.SETARRAYITEM_GC, diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -556,7 +556,7 @@ operations, self.current_clt.allgcrefs, self.current_clt.frame_info) - stack_check_patch_ofs = self._check_frame_depth() + stack_check_patch_ofs = self._check_frame_depth(self.mc) frame_depth = self._assemble(regalloc, inputargs, operations) codeendpos = self.mc.get_relative_pos() self.write_pending_failure_recoveries() @@ -638,7 +638,7 @@ mc.writeimm32(self.error_trampoline_64 - pos_after_jz) mc.copy_to_raw_memory(rawstart + pos_after_jz - 4) - def _check_frame_depth(self): + def _check_frame_depth(self, mc): """ check if the frame is of enough depth to follow this bridge. Otherwise reallocate the frame in a helper. There are other potential solutions @@ -647,16 +647,16 @@ descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) ofs = self.cpu.unpack_fielddescr(descrs.arraydescr.lendescr) base_ofs = self.cpu.get_baseofs_of_frame_field() - self.mc.CMP_bi(ofs - base_ofs, 0xffffff) - stack_check_cmp_ofs = self.mc.get_relative_pos() - 4 + mc.CMP_bi(ofs - base_ofs, 0xffffff) + stack_check_cmp_ofs = mc.get_relative_pos() - 4 assert not IS_X86_32 - self.mc.J_il8(rx86.Conditions['GE'], 0) - jg_location = self.mc.get_relative_pos() - self.mc.CALL(imm(self._stack_check_failure)) + mc.J_il8(rx86.Conditions['GE'], 0) + jg_location = mc.get_relative_pos() + mc.CALL(imm(self._stack_check_failure)) # patch the JG above - offset = self.mc.get_relative_pos() - jg_location + offset = mc.get_relative_pos() - jg_location assert 0 < offset <= 127 - self.mc.overwrite(jg_location-1, chr(offset)) + mc.overwrite(jg_location-1, chr(offset)) return stack_check_cmp_ofs def _insert_frame_adjustment(self, frame_info): @@ -882,6 +882,11 @@ # Ideally we should rather patch all existing CALLs, but well. oldadr = oldlooptoken._x86_function_addr target = newlooptoken._x86_function_addr + # copy frame-info data + old_fi = oldlooptoken.compiled_loop_token.frame_info + new_fi = newlooptoken.compiled_loop_token.frame_info + old_fi.jfi_frame_depth = new_fi.jfi_frame_depth + old_fi.jfi_gcmap = new_fi.jfi_gcmap mc = codebuf.MachineCodeBlockWrapper() mc.JMP(imm(target)) if WORD == 4: # keep in sync with prepare_loop() @@ -1857,12 +1862,12 @@ if WORD == 4: mc.PUSH(imm(fail_descr)) mc.PUSH(imm(gcpattern)) - mc.JMP(imm(target)) + mc.CALL(imm(target)) else: mc.MOV_ri64(X86_64_SCRATCH_REG.value, target) mc.PUSH(imm(fail_descr)) mc.PUSH(imm(gcpattern)) - mc.JMP_r(X86_64_SCRATCH_REG.value) + self.mc.JMP_r(X86_64_SCRATCH_REG.value) return startpos def rebuild_faillocs_from_descr(self, descr, inputargs): From noreply at buildbot.pypy.org Mon Jan 21 09:04:18 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Mon, 21 Jan 2013 09:04:18 +0100 (CET) Subject: [pypy-commit] pypy default: Fixed wrong import path Message-ID: <20130121080418.8AE2E1C1138@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: Changeset: r60270:41fc75f90447 Date: 2013-01-21 09:03 +0100 http://bitbucket.org/pypy/pypy/changeset/41fc75f90447/ Log: Fixed wrong import path diff --git a/pypy/interpreter/test2/test_app_main.py b/pypy/interpreter/test2/test_app_main.py --- a/pypy/interpreter/test2/test_app_main.py +++ b/pypy/interpreter/test2/test_app_main.py @@ -84,7 +84,7 @@ def check(self, argv, **expected): import StringIO - from rpython.translator.goal import app_main + from pypy.interpreter import app_main saved_sys_argv = sys.argv[:] saved_sys_stdout = sys.stdout saved_sys_stderr = sys.stdout @@ -825,7 +825,7 @@ class TestAppMain: def test_print_info(self): - from rpython.translator.goal import app_main + from pypy.interpreter import app_main import sys, cStringIO prev_so = sys.stdout prev_ti = getattr(sys, 'pypy_translation_info', 'missing') From noreply at buildbot.pypy.org Mon Jan 21 09:04:19 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Mon, 21 Jan 2013 09:04:19 +0100 (CET) Subject: [pypy-commit] pypy default: Fixed test_targetpypy Message-ID: <20130121080419.B92C71C1138@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: Changeset: r60271:e436205050f4 Date: 2013-01-21 09:03 +0100 http://bitbucket.org/pypy/pypy/changeset/e436205050f4/ Log: Fixed test_targetpypy diff --git a/pypy/interpreter/test2/test_targetpypy.py b/pypy/interpreter/test2/test_targetpypy.py --- a/pypy/interpreter/test2/test_targetpypy.py +++ b/pypy/interpreter/test2/test_targetpypy.py @@ -1,6 +1,6 @@ import py -from goal.targetpypystandalone import get_entry_point +from pypy.goal.targetpypystandalone import get_entry_point from pypy.config.pypyoption import get_pypy_config class TestTargetPyPy(object): From noreply at buildbot.pypy.org Mon Jan 21 09:37:05 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 09:37:05 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: a bit of cargo culting Message-ID: <20130121083705.680511C1102@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60272:3e0a0a706592 Date: 2013-01-21 10:36 +0200 http://bitbucket.org/pypy/pypy/changeset/3e0a0a706592/ Log: a bit of cargo culting diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -1844,7 +1844,7 @@ positions = [0] * len(guardtok.fail_locs) gcpattern = 0 for i, loc in enumerate(guardtok.fail_locs): - if loc is None: + if loc is None or loc is ebp: # frame positions[i] = -1 elif isinstance(loc, StackLoc): positions[i] = loc.value From noreply at buildbot.pypy.org Mon Jan 21 09:39:55 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Mon, 21 Jan 2013 09:39:55 +0100 (CET) Subject: [pypy-commit] pypy default: Added __init__.py to pypy/goal Message-ID: <20130121083955.B95411C1241@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: Changeset: r60273:c6b47aa5ced3 Date: 2013-01-21 09:39 +0100 http://bitbucket.org/pypy/pypy/changeset/c6b47aa5ced3/ Log: Added __init__.py to pypy/goal diff --git a/pypy/goal/__init__.py b/pypy/goal/__init__.py new file mode 100644 --- /dev/null +++ b/pypy/goal/__init__.py @@ -0,0 +1,1 @@ +#empty From noreply at buildbot.pypy.org Mon Jan 21 09:53:03 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 09:53:03 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: add _freeze_ to the CPU and move movable state somewhere else. this is needed for hacking at hacks Message-ID: <20130121085303.C8F461C1243@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60274:291fb306aafa Date: 2013-01-21 10:52 +0200 http://bitbucket.org/pypy/pypy/changeset/291fb306aafa/ Log: add _freeze_ to the CPU and move movable state somewhere else. this is needed for hacking at hacks diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py --- a/rpython/jit/backend/model.py +++ b/rpython/jit/backend/model.py @@ -2,6 +2,11 @@ from rpython.jit.metainterp import compile from rpython.rtyper.lltypesystem import lltype +class CPUTotalTracker(object): + total_compiled_loops = 0 + total_compiled_bridges = 0 + total_freed_loops = 0 + total_freed_bridges = 0 class AbstractCPU(object): supports_floats = False @@ -11,20 +16,18 @@ # Boxes and Consts are BoxFloats and ConstFloats. supports_singlefloats = False - total_compiled_loops = 0 - total_compiled_bridges = 0 - total_freed_loops = 0 - total_freed_bridges = 0 - propagate_exception_descr = None # for heaptracker # _all_size_descrs_with_vtable = None _vtable_to_descr_dict = None - def __init__(self): self.__dict__.update(compile.make_done_loop_tokens()) + self.tracker = CPUTotalTracker() + + def _freeze_(self): + return True def setup_once(self): """Called once by the front-end when the program starts.""" @@ -290,7 +293,7 @@ frame_depth = 0 def __init__(self, cpu, number): - cpu.total_compiled_loops += 1 + cpu.tracker.total_compiled_loops += 1 self.cpu = cpu self.number = number self.bridges_count = 0 @@ -314,7 +317,7 @@ return n def compiling_a_bridge(self): - self.cpu.total_compiled_bridges += 1 + self.cpu.tracker.total_compiled_bridges += 1 self.bridges_count += 1 debug_start("jit-mem-looptoken-alloc") debug_print("allocating Bridge #", self.bridges_count, "of Loop #", self.number) @@ -325,6 +328,6 @@ debug_print("freeing Loop #", self.number, 'with', self.bridges_count, 'attached bridges') self.cpu.free_loop_and_bridges(self) - self.cpu.total_freed_loops += 1 - self.cpu.total_freed_bridges += self.bridges_count + self.cpu.tracker.total_freed_loops += 1 + self.cpu.tracker.total_freed_bridges += self.bridges_count debug_stop("jit-mem-looptoken-free") diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py --- a/rpython/jit/backend/test/runner_test.py +++ b/rpython/jit/backend/test/runner_test.py @@ -223,8 +223,8 @@ assert not wr_i1() and not wr_guard() def test_compile_bridge(self): - self.cpu.total_compiled_loops = 0 - self.cpu.total_compiled_bridges = 0 + self.cpu.tracker.total_compiled_loops = 0 + self.cpu.tracker.total_compiled_bridges = 0 i0 = BoxInt() i1 = BoxInt() i2 = BoxInt() @@ -260,8 +260,8 @@ res = self.cpu.get_int_value(deadframe, 0) assert res == 20 - assert self.cpu.total_compiled_loops == 1 - assert self.cpu.total_compiled_bridges == 1 + assert self.cpu.tracker.total_compiled_loops == 1 + assert self.cpu.tracker.total_compiled_bridges == 1 return looptoken def test_compile_bridge_with_holes(self): diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -302,7 +302,7 @@ i = 0 for loc in locs: if loc is None: # xxx bit kludgy - continue + loc = ebp arg = inputargs[i] i += 1 if isinstance(loc, RegLoc): diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py --- a/rpython/jit/backend/x86/runner.py +++ b/rpython/jit/backend/x86/runner.py @@ -159,11 +159,11 @@ immortal=True) def force(self, addr_of_force_token): - descr = self.gc_ll_descr.getframedescrs(self).arraydescr + descr = self.signedarraydescr ofs = self.unpack_arraydescr(descr) frame = rffi.cast(jitframe.JITFRAMEPTR, addr_of_force_token - ofs) frame.jf_descr = frame.jf_force_descr - return frame + return lltype.cast_opaque_ptr(llmemory.GCREF, frame) def redirect_call_assembler(self, oldlooptoken, newlooptoken): self.assembler.redirect_call_assembler(oldlooptoken, newlooptoken) @@ -249,6 +249,7 @@ descrs = self.gc_ll_descr.getframedescrs(self) ad = descrs.arraydescr # the same as normal JITFRAME, however with an array of pointers + self.signedarraydescr = ad self.refarraydescr = ArrayDescr(ad.basesize, ad.itemsize, ad.lendescr, FLAG_POINTER) self.floatarraydescr = ArrayDescr(ad.basesize, ad.itemsize, ad.lendescr, @@ -260,8 +261,7 @@ elif type == history.REF: descr = self.refarraydescr else: - descrs = self.gc_ll_descr.getframedescrs(self) - descr = descrs.arraydescr + descr = self.signedarraydescr return JITFRAME_FIXED_SIZE + index, descr CPU = CPU386 diff --git a/rpython/jit/backend/x86/test/test_gc_integration.py b/rpython/jit/backend/x86/test/test_gc_integration.py --- a/rpython/jit/backend/x86/test/test_gc_integration.py +++ b/rpython/jit/backend/x86/test/test_gc_integration.py @@ -2,8 +2,9 @@ """ Tests for register allocation for common constructs """ -from rpython.jit.metainterp.history import TargetToken, AbstractDescr -from rpython.jit.backend.llsupport.gc import GcLLDescription, GcLLDescr_boehm +from rpython.jit.metainterp.history import TargetToken, BasicFinalDescr +from rpython.jit.backend.llsupport.gc import GcLLDescription, GcLLDescr_boehm,\ + GcLLDescr_framework from rpython.jit.backend.detect_cpu import getcpuclass from rpython.jit.backend.x86.arch import WORD from rpython.jit.backend.llsupport import jitframe @@ -319,3 +320,15 @@ s1ref = self.cpu.get_ref_value(self.deadframe, i) s1 = lltype.cast_opaque_ptr(lltype.Ptr(S1), s1ref) assert s1 == getattr(s2, 's%d' % i) + +class TestGcShadowstackDirect(object): + + cpu = CPU(None, None) + cpu.gc_ll_descr = GcLLDescr_framework(None, None, None) + cpu.setup_once() + + def test_shadowstack_call(self): + ops = parse(""" + [] + finish(i0, descr=finaldescr) + """, namespace={'finaldescr': BasicFinalDescr()}) diff --git a/rpython/jit/metainterp/jitprof.py b/rpython/jit/metainterp/jitprof.py --- a/rpython/jit/metainterp/jitprof.py +++ b/rpython/jit/metainterp/jitprof.py @@ -100,13 +100,13 @@ def get_counter(self, num): if num == Counters.TOTAL_COMPILED_LOOPS: - return self.cpu.total_compiled_loops + return self.cpu.tracker.total_compiled_loops elif num == Counters.TOTAL_COMPILED_BRIDGES: - return self.cpu.total_compiled_bridges + return self.cpu.tracker.total_compiled_bridges elif num == Counters.TOTAL_FREED_LOOPS: - return self.cpu.total_freed_loops + return self.cpu.tracker.total_freed_loops elif num == Counters.TOTAL_FREED_BRIDGES: - return self.cpu.total_freed_bridges + return self.cpu.tracker.total_freed_bridges return self.counters[num] def count_ops(self, opnum, kind=Counters.OPS): @@ -151,13 +151,13 @@ cpu = self.cpu if cpu is not None: # for some tests self._print_intline("Total # of loops", - cpu.total_compiled_loops) + cpu.tracker.total_compiled_loops) self._print_intline("Total # of bridges", - cpu.total_compiled_bridges) + cpu.tracker.total_compiled_bridges) self._print_intline("Freed # of loops", - cpu.total_freed_loops) + cpu.tracker.total_freed_loops) self._print_intline("Freed # of bridges", - cpu.total_freed_bridges) + cpu.tracker.total_freed_bridges) def _print_line_time(self, string, i, tim): final = "%s:%s\t%d\t%f" % (string, " " * max(0, 13-len(string)), i, tim) From noreply at buildbot.pypy.org Mon Jan 21 09:56:10 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 09:56:10 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fix Message-ID: <20130121085610.D8D8F1C1243@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60275:9e4122b124f4 Date: 2013-01-21 10:55 +0200 http://bitbucket.org/pypy/pypy/changeset/9e4122b124f4/ Log: fix diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -1,7 +1,6 @@ import weakref from rpython.rtyper.lltypesystem import lltype -from rpython.rtyper.ootypesystem import ootype -from rpython.flowspace.model import Constant, Variable +from rpython.rtyper.annlowlevel import cast_instance_to_gcref from rpython.rlib.objectmodel import we_are_translated from rpython.rlib.debug import debug_start, debug_stop, debug_print from rpython.rlib import rstack @@ -867,7 +866,7 @@ cpu = metainterp_sd.cpu exception = cpu.grab_exc_value(deadframe) if not exception: - exception = memory_error + exception = cast_instance_to_gcref(memory_error) assert exception, "PropagateExceptionDescr: no exception??" raise metainterp_sd.ExitFrameWithExceptionRef(cpu, exception) From noreply at buildbot.pypy.org Mon Jan 21 09:58:22 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 09:58:22 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: move more mutable state Message-ID: <20130121085822.89F5C1C1243@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60276:9972d2e1d6d7 Date: 2013-01-21 10:57 +0200 http://bitbucket.org/pypy/pypy/changeset/9972d2e1d6d7/ Log: move more mutable state diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py --- a/rpython/jit/backend/model.py +++ b/rpython/jit/backend/model.py @@ -8,6 +8,10 @@ total_freed_loops = 0 total_freed_bridges = 0 + # for heaptracker + # _all_size_descrs_with_vtable = None + _vtable_to_descr_dict = None + class AbstractCPU(object): supports_floats = False supports_longlong = False @@ -18,10 +22,6 @@ propagate_exception_descr = None - # for heaptracker - # _all_size_descrs_with_vtable = None - _vtable_to_descr_dict = None - def __init__(self): self.__dict__.update(compile.make_done_loop_tokens()) self.tracker = CPUTotalTracker() diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py --- a/rpython/jit/codewriter/heaptracker.py +++ b/rpython/jit/codewriter/heaptracker.py @@ -89,9 +89,9 @@ except AttributeError: pass assert lltype.typeOf(vtable) == VTABLETYPE - if not hasattr(cpu, '_all_size_descrs_with_vtable'): - cpu._all_size_descrs_with_vtable = [] - cpu._vtable_to_descr_dict = None + if not hasattr(cpu.tracker, '_all_size_descrs_with_vtable'): + cpu.tracker._all_size_descrs_with_vtable = [] + cpu.tracker._vtable_to_descr_dict = None cpu._all_size_descrs_with_vtable.append(sizedescr) sizedescr._corresponding_vtable = vtable @@ -108,9 +108,9 @@ # Build the dict {vtable: sizedescr} at runtime. # This is necessary because the 'vtables' are just pointers to # static data, so they can't be used as keys in prebuilt dicts. - d = cpu._vtable_to_descr_dict + d = cpu.tracker._vtable_to_descr_dict if d is None: - d = cpu._vtable_to_descr_dict = {} + d = cpu.tracker._vtable_to_descr_dict = {} for descr in cpu._all_size_descrs_with_vtable: key = descr._corresponding_vtable key = llmemory.cast_ptr_to_adr(key) From noreply at buildbot.pypy.org Mon Jan 21 09:59:43 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 09:59:43 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: be more persistent in moving Message-ID: <20130121085943.BE43B1C1243@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60277:79cc9323b465 Date: 2013-01-21 10:59 +0200 http://bitbucket.org/pypy/pypy/changeset/79cc9323b465/ Log: be more persistent in moving diff --git a/rpython/jit/codewriter/heaptracker.py b/rpython/jit/codewriter/heaptracker.py --- a/rpython/jit/codewriter/heaptracker.py +++ b/rpython/jit/codewriter/heaptracker.py @@ -92,12 +92,12 @@ if not hasattr(cpu.tracker, '_all_size_descrs_with_vtable'): cpu.tracker._all_size_descrs_with_vtable = [] cpu.tracker._vtable_to_descr_dict = None - cpu._all_size_descrs_with_vtable.append(sizedescr) + cpu.tracker._all_size_descrs_with_vtable.append(sizedescr) sizedescr._corresponding_vtable = vtable def finish_registering(cpu): # annotation hack for small examples which have no vtable at all - if not hasattr(cpu, '_all_size_descrs_with_vtable'): + if not hasattr(cpu.tracker, '_all_size_descrs_with_vtable'): vtable = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True) register_known_gctype(cpu, vtable, rclass.OBJECT) @@ -111,14 +111,14 @@ d = cpu.tracker._vtable_to_descr_dict if d is None: d = cpu.tracker._vtable_to_descr_dict = {} - for descr in cpu._all_size_descrs_with_vtable: + for descr in cpu.tracker._all_size_descrs_with_vtable: key = descr._corresponding_vtable key = llmemory.cast_ptr_to_adr(key) d[key] = descr return d[vtable] else: vtable = llmemory.cast_adr_to_ptr(vtable, VTABLETYPE) - for descr in cpu._all_size_descrs_with_vtable: + for descr in cpu.tracker._all_size_descrs_with_vtable: if descr._corresponding_vtable == vtable: return descr raise KeyError(vtable) From noreply at buildbot.pypy.org Mon Jan 21 10:02:48 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 10:02:48 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fix types Message-ID: <20130121090248.50F991C1243@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60278:eff89c98c5e9 Date: 2013-01-21 11:02 +0200 http://bitbucket.org/pypy/pypy/changeset/eff89c98c5e9/ Log: fix types diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py --- a/rpython/jit/backend/llsupport/gc.py +++ b/rpython/jit/backend/llsupport/gc.py @@ -5,7 +5,7 @@ from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rclass, rstr from rpython.rtyper.lltypesystem import llgroup from rpython.rtyper.lltypesystem.lloperation import llop -from rpython.rtyper.annlowlevel import llhelper +from rpython.rtyper.annlowlevel import llhelper, cast_instance_to_gcref from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.jit.codewriter import heaptracker from rpython.jit.metainterp.history import ConstPtr, AbstractDescr @@ -102,7 +102,7 @@ rgc._make_sure_does_not_move(p) gcrefs_output_list.append(p) if op.is_guard() or op.getopnum() == rop.FINISH: - rgc._make_sure_does_not_move(op.getdescr()) + rgc._make_sure_does_not_move(cast_instance_to_gcref(op.getdescr())) gcrefs_output_list.append(op.getdescr()) def rewrite_assembler(self, cpu, operations, gcrefs_output_list): diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -143,8 +143,8 @@ lgt_box = history.BoxInt() frame = history.BoxPtr() jfi = loop_token.compiled_loop_token.frame_info - rgc._make_sure_does_not_move(jfi) llref = lltype.cast_opaque_ptr(llmemory.GCREF, jfi) + rgc._make_sure_does_not_move(llref) op0 = ResOperation(rop.GETFIELD_GC, [history.ConstPtr(llref)], lgt_box, descr=descrs.jfi_frame_depth) self.newops.append(op0) diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -498,7 +498,8 @@ self._call_header_with_stack_check() operations = regalloc.prepare_loop(inputargs, operations, looptoken, clt.allgcrefs) - rgc._make_sure_does_not_move(clt.frame_info) + rgc._make_sure_does_not_move(lltype.cast_opaque_ptr(llmemory.GCREF, + clt.frame_info)) looppos = self.mc.get_relative_pos() looptoken._x86_loop_code = looppos frame_depth = self._assemble(regalloc, inputargs, operations) From noreply at buildbot.pypy.org Mon Jan 21 10:06:07 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 10:06:07 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: opops Message-ID: <20130121090607.042B51C1243@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60279:8429de3e7fa8 Date: 2013-01-21 11:05 +0200 http://bitbucket.org/pypy/pypy/changeset/8429de3e7fa8/ Log: opops diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py --- a/rpython/jit/backend/llsupport/gc.py +++ b/rpython/jit/backend/llsupport/gc.py @@ -102,8 +102,9 @@ rgc._make_sure_does_not_move(p) gcrefs_output_list.append(p) if op.is_guard() or op.getopnum() == rop.FINISH: - rgc._make_sure_does_not_move(cast_instance_to_gcref(op.getdescr())) - gcrefs_output_list.append(op.getdescr()) + llref = cast_instance_to_gcref(op.getdescr()) + rgc._make_sure_does_not_move(llref) + gcrefs_output_list.append(llref) def rewrite_assembler(self, cpu, operations, gcrefs_output_list): rewriter = GcRewriterAssembler(self, cpu) From noreply at buildbot.pypy.org Mon Jan 21 10:08:00 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 10:08:00 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: frame size accepts size Message-ID: <20130121090800.53AD01C1243@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60280:f3b606c20a0f Date: 2013-01-21 11:07 +0200 http://bitbucket.org/pypy/pypy/changeset/f3b606c20a0f/ Log: frame size accepts size diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py --- a/rpython/jit/backend/llsupport/regalloc.py +++ b/rpython/jit/backend/llsupport/regalloc.py @@ -218,7 +218,7 @@ if box.type == REF: self.freelist_gcrefs.append(1, loc) else: - size = self.frame_size(box) + size = self.frame_size(box.type) self.freelist_others.append(size, loc) def try_to_reuse_location(self, box, loc): From noreply at buildbot.pypy.org Mon Jan 21 10:09:40 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 10:09:40 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: more casts Message-ID: <20130121090940.CF4421C00EC@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60281:899e3db54931 Date: 2013-01-21 11:09 +0200 http://bitbucket.org/pypy/pypy/changeset/899e3db54931/ Log: more casts diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -455,6 +455,9 @@ # the potential argument to FINISH descr = op.getdescr() fail_descr = cast_instance_to_gcref(descr) + # we know it does not move, but well + rgc._make_sure_does_not_move(fail_descr) + fail_descr = rffi.cast(lltype.Signed, fail_descr) if op.numargs() == 1: loc = self.make_sure_var_in_reg(op.getarg(0)) locs = [loc, imm(fail_descr)] From noreply at buildbot.pypy.org Mon Jan 21 10:11:24 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 10:11:24 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: mark those as unimplemented Message-ID: <20130121091124.C292C1C00EC@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60282:c8e6b57929df Date: 2013-01-21 11:11 +0200 http://bitbucket.org/pypy/pypy/changeset/c8e6b57929df/ Log: mark those as unimplemented diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -2060,6 +2060,7 @@ # first, close the stack in the sense of the asmgcc GC root tracker gcrootmap = self.cpu.gc_ll_descr.gcrootmap if gcrootmap: + raise NotImplementedError # we need tests xxx self.call_release_gil(gcrootmap, arglocs) # do the call @@ -2067,6 +2068,7 @@ self.genop_call(op, arglocs, result_loc) # then reopen the stack if gcrootmap: + raise NotImplementedError # we need tests xxx self.call_reacquire_gil(gcrootmap, result_loc) # finally, the guard_not_forced From noreply at buildbot.pypy.org Mon Jan 21 10:13:26 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 10:13:26 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: None has no append, right Message-ID: <20130121091326.1255F1C00EC@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60283:1d41cab0b4e6 Date: 2013-01-21 11:13 +0200 http://bitbucket.org/pypy/pypy/changeset/1d41cab0b4e6/ Log: None has no append, right diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -1187,9 +1187,9 @@ on_stack += 1 elif argtypes is not None and argtypes[i-start] == 'S': # Singlefloat argument + if singlefloats is None: + singlefloats = [] if len(unused_xmm) > 0: - if singlefloats is None: - singlefloats = [] singlefloats.append((loc, unused_xmm.pop())) else: singlefloats.append((loc, RawEspLoc(on_stack * WORD, INT))) From noreply at buildbot.pypy.org Mon Jan 21 10:18:33 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 10:18:33 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: make sure we don't store stuff on AbstractDescr Message-ID: <20130121091833.3DE9D1C00EC@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60284:2d0e75c45f56 Date: 2013-01-21 11:18 +0200 http://bitbucket.org/pypy/pypy/changeset/2d0e75c45f56/ Log: make sure we don't store stuff on AbstractDescr diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -44,6 +44,7 @@ class GuardToken(object): def __init__(self, faildescr, failargs, fail_locs, exc, is_guard_not_invalidated, is_guard_not_forced): + assert isinstance(faildescr, AbstractFailDescr) self.faildescr = faildescr self.failargs = failargs self.fail_locs = fail_locs From noreply at buildbot.pypy.org Mon Jan 21 10:20:58 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 10:20:58 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: hack differently Message-ID: <20130121092058.9AFBC1C00EC@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60285:22b28490113e Date: 2013-01-21 11:20 +0200 http://bitbucket.org/pypy/pypy/changeset/22b28490113e/ Log: hack differently diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -44,7 +44,6 @@ class GuardToken(object): def __init__(self, faildescr, failargs, fail_locs, exc, is_guard_not_invalidated, is_guard_not_forced): - assert isinstance(faildescr, AbstractFailDescr) self.faildescr = faildescr self.failargs = failargs self.fail_locs = fail_locs diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -309,6 +309,7 @@ def do_compile_bridge(metainterp_sd, faildescr, inputargs, operations, original_loop_token, log=True): metainterp_sd.logger_ops.log_bridge(inputargs, operations, "compiling") + assert isinstance(faildescr, AbstractFailDescr) return metainterp_sd.cpu.compile_bridge(faildescr, inputargs, operations, original_loop_token, log=log) From noreply at buildbot.pypy.org Mon Jan 21 10:23:29 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 10:23:29 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: uh and in here Message-ID: <20130121092329.3160A1C00EC@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60286:1cdcf8c757d7 Date: 2013-01-21 11:23 +0200 http://bitbucket.org/pypy/pypy/changeset/1cdcf8c757d7/ Log: uh and in here diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py --- a/rpython/jit/backend/llsupport/llmodel.py +++ b/rpython/jit/backend/llsupport/llmodel.py @@ -226,7 +226,9 @@ def get_latest_descr(self, deadframe): deadframe = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, deadframe) descr = deadframe.jf_descr - return history.AbstractDescr.show(self, descr) + res = history.AbstractDescr.show(self, descr) + assert isinstance(res, history.AbstractFailDescr) + return res def _decode_pos(self, deadframe, index): descr = self.get_latest_descr(deadframe) From noreply at buildbot.pypy.org Mon Jan 21 10:25:30 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 10:25:30 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: pass the correct type Message-ID: <20130121092530.DA3251C00EC@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60287:d19a633a353a Date: 2013-01-21 11:25 +0200 http://bitbucket.org/pypy/pypy/changeset/d19a633a353a/ Log: pass the correct type diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py --- a/rpython/jit/backend/x86/runner.py +++ b/rpython/jit/backend/x86/runner.py @@ -129,14 +129,14 @@ for i, kind in kinds: arg = args[i] if kind == history.INT: - self.set_int_value(frame, num, arg) + self.set_int_value(ll_frame, num, arg) elif kind == history.FLOAT: - self.set_float_value(frame, num, arg) + self.set_float_value(ll_frame, num, arg) if IS_X86_32: num += WORD else: assert kind == history.REF - self.set_ref_value(frame, num, arg) + self.set_ref_value(ll_frame, num, arg) num += WORD ll_frame = func(ll_frame) finally: From noreply at buildbot.pypy.org Mon Jan 21 10:32:06 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 10:32:06 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: rpythonify realloc frame Message-ID: <20130121093206.42C821C1102@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60288:4bbbefa03c00 Date: 2013-01-21 11:31 +0200 http://bitbucket.org/pypy/pypy/changeset/4bbbefa03c00/ Log: rpythonify realloc frame diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py --- a/rpython/jit/backend/llsupport/llmodel.py +++ b/rpython/jit/backend/llsupport/llmodel.py @@ -1,7 +1,7 @@ from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rclass, rstr from rpython.rtyper.lltypesystem.lloperation import llop from rpython.rtyper.llinterp import LLInterpreter -from rpython.rtyper.annlowlevel import llhelper +from rpython.rtyper.annlowlevel import llhelper, MixLevelHelperAnnotator from rpython.rlib.objectmodel import we_are_translated, specialize from rpython.jit.metainterp import history from rpython.jit.codewriter import heaptracker, longlong @@ -13,6 +13,7 @@ get_call_descr, get_interiorfield_descr, FieldDescr, ArrayDescr, CallDescr, InteriorFieldDescr) from rpython.jit.backend.llsupport.asmmemmgr import AsmMemoryManager +from rpython.annotator import model as annmodel class AbstractLLCPU(AbstractCPU): @@ -44,13 +45,13 @@ else: self._setup_exception_handling_untranslated() self.asmmemmgr = AsmMemoryManager() - self._setup_frame_realloc() + self._setup_frame_realloc(translate_support_code) self.setup() def setup(self): pass - def _setup_frame_realloc(self): + def _setup_frame_realloc(self, translate_support_code): FUNC_TP = lltype.Ptr(lltype.FuncType([llmemory.GCREF], llmemory.GCREF)) @@ -69,9 +70,18 @@ new_frame.jf_savedata = frame.jf_savedata # all other fields are empty return lltype.cast_opaque_ptr(llmemory.GCREF, new_frame) - - f = llhelper(FUNC_TP, realloc_frame) - self.realloc_frame = heaptracker.adr2int(llmemory.cast_ptr_to_adr(f)) + + if not translate_support_code: + fptr = llhelper(FUNC_TP, realloc_frame) + else: + FUNC = FUNC_TP.TO + args_s = [annmodel.lltype_to_annotation(ARG) for ARG in FUNC.ARGS] + s_result = annmodel.lltype_to_annotation(FUNC.RESULT) + mixlevelann = MixLevelHelperAnnotator(self.rtyper) + graph = mixlevelann.getgraph(realloc_frame, args_s, s_result) + fptr = mixlevelann.graph2delayed(graph, FUNC) + mixlevelann.finish() + self.realloc_frame = heaptracker.adr2int(llmemory.cast_ptr_to_adr(fptr)) def _setup_exception_handling_untranslated(self): # for running un-translated only, all exceptions occurring in the From noreply at buildbot.pypy.org Mon Jan 21 10:42:30 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 10:42:30 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fix Message-ID: <20130121094230.212551C1243@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60289:bc3bbf5d8db6 Date: 2013-01-21 11:42 +0200 http://bitbucket.org/pypy/pypy/changeset/bc3bbf5d8db6/ Log: fix diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -198,7 +198,7 @@ mc.SUB_rr(edi.value, eax.value) # compute the size we want assert not IS_X86_32 # the arg is already in edi - if getattr(self.cpu.gc_ll_descr, 'passes_frame'): + if hasattr(self.cpu.gc_ll_descr, 'passes_frame'): base_ofs = self.cpu.get_baseofs_of_frame_field() mc.LEA_rb(esi.value, -base_ofs) mc.SUB_ri(esp.value, 16 - WORD) From noreply at buildbot.pypy.org Mon Jan 21 10:44:15 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 10:44:15 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: oops Message-ID: <20130121094415.6C84F1C1243@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60290:a41de64f9293 Date: 2013-01-21 11:43 +0200 http://bitbucket.org/pypy/pypy/changeset/a41de64f9293/ Log: oops diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -214,7 +214,7 @@ nursery_free_adr = self.cpu.gc_ll_descr.get_nursery_free_addr() mc.MOV(edi, heap(nursery_free_adr)) # load this in EDX # clear the gc pattern - mc.MOV_bi(ofs, imm(0)) + mc.MOV_bi(ofs, 0) mc.RET() # # If the slowpath malloc failed, we raise a MemoryError that From noreply at buildbot.pypy.org Mon Jan 21 11:39:27 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 11:39:27 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: basic integration, reloading of frame Message-ID: <20130121103927.48B121C106A@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60291:c3e92471a8d8 Date: 2013-01-21 12:36 +0200 http://bitbucket.org/pypy/pypy/changeset/c3e92471a8d8/ Log: basic integration, reloading of frame diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -1228,6 +1228,13 @@ self.mc.CALL(x) if align: self.mc.ADD_ri(esp.value, align * WORD) + gcrootmap = self.cpu.gc_ll_descr.gcrootmap + if gcrootmap and gcrootmap.is_shadow_stack: + rst = gcrootmap.get_root_stack_top_addr() + self.mc.MOV(edx, heap(rst)) + self.mc.MOV(ebp, mem(edx, -WORD)) + base_ofs = self.cpu.get_baseofs_of_frame_field() + self.mc.ADD_ri(ebp.value, base_ofs) def call(self, addr, args, res): self._emit_call(imm(addr), args) diff --git a/rpython/jit/backend/x86/test/test_gc_integration.py b/rpython/jit/backend/x86/test/test_gc_integration.py --- a/rpython/jit/backend/x86/test/test_gc_integration.py +++ b/rpython/jit/backend/x86/test/test_gc_integration.py @@ -2,16 +2,19 @@ """ Tests for register allocation for common constructs """ -from rpython.jit.metainterp.history import TargetToken, BasicFinalDescr +from rpython.jit.metainterp.history import TargetToken, BasicFinalDescr,\ + JitCellToken from rpython.jit.backend.llsupport.gc import GcLLDescription, GcLLDescr_boehm,\ - GcLLDescr_framework + GcLLDescr_framework, GcCache from rpython.jit.backend.detect_cpu import getcpuclass -from rpython.jit.backend.x86.arch import WORD +from rpython.jit.backend.x86.arch import WORD, JITFRAME_FIXED_SIZE from rpython.jit.backend.llsupport import jitframe from rpython.rtyper.lltypesystem import lltype, llmemory, rffi +from rpython.rtyper.annlowlevel import llhelper from rpython.jit.backend.x86.test.test_regalloc import BaseTestRegalloc from rpython.jit.backend.x86.regalloc import gpr_reg_mgr_cls +from rpython.jit.codewriter.effectinfo import EffectInfo CPU = getcpuclass() @@ -321,14 +324,81 @@ s1 = lltype.cast_opaque_ptr(lltype.Ptr(S1), s1ref) assert s1 == getattr(s2, 's%d' % i) -class TestGcShadowstackDirect(object): +class MockShadowStackRootMap(object): + is_shadow_stack = True + + def __init__(self): + TP = rffi.CArray(lltype.Signed) + self.stack = lltype.malloc(TP, 10, flavor='raw') + self.stack_addr = lltype.malloc(TP, 1, + flavor='raw') + self.stack_addr[0] = rffi.cast(lltype.Signed, self.stack) + + def __del__(self): + lltype.free(self.stack_addr, flavor='raw') + lltype.free(self.stack, flavor='raw') + + def register_asm_addr(self, start, mark): + pass + + def get_root_stack_top_addr(self): + return rffi.cast(lltype.Signed, self.stack_addr) + +class GCDescrShadowstackDirect(GcLLDescr_framework): + layoutbuilder = None + write_barrier_descr = None + + def get_malloc_slowpath_addr(self): + return 0 + + def get_nursery_free_addr(self): + return 0 + + def __init__(self): + GcCache.__init__(self, False, None) + self.gcrootmap = MockShadowStackRootMap() + +class TestGcShadowstackDirect(BaseTestRegalloc): cpu = CPU(None, None) - cpu.gc_ll_descr = GcLLDescr_framework(None, None, None) + cpu.gc_ll_descr = GCDescrShadowstackDirect() cpu.setup_once() def test_shadowstack_call(self): - ops = parse(""" + ofs = self.cpu.get_baseofs_of_frame_field() + frames = [] + + def check(i): + assert self.cpu.gc_ll_descr.gcrootmap.stack[0] == i - ofs + frame = rffi.cast(jitframe.JITFRAMEPTR, i - ofs) + frames.append(frame) + assert len(frame.jf_frame) == JITFRAME_FIXED_SIZE + # we "collect" + new_frame = frame.copy() + self.cpu.gc_ll_descr.gcrootmap.stack[0] = rffi.cast(lltype.Signed, new_frame) + frames.append(new_frame) + + def check2(i): + assert self.cpu.gc_ll_descr.gcrootmap.stack[0] == i - ofs + frame = rffi.cast(jitframe.JITFRAMEPTR, i - ofs) + assert frame == frames[1] + assert frame != frames[0] + + CHECK = lltype.FuncType([lltype.Signed], lltype.Void) + checkptr = llhelper(lltype.Ptr(CHECK), check) + check2ptr = llhelper(lltype.Ptr(CHECK), check2) + checkdescr = self.cpu.calldescrof(CHECK, CHECK.ARGS, CHECK.RESULT, + EffectInfo.MOST_GENERAL) + + loop = self.parse(""" [] + i0 = force_token() # this is a bit below the frame + call(ConstClass(check_adr), i0, descr=checkdescr) # this can collect + call(ConstClass(check2_adr), i0, descr=checkdescr) finish(i0, descr=finaldescr) - """, namespace={'finaldescr': BasicFinalDescr()}) + """, namespace={'finaldescr': BasicFinalDescr(), + 'check_adr': checkptr, 'check2_adr': check2ptr, + 'checkdescr': checkdescr}) + token = JitCellToken() + self.cpu.compile_loop(loop.inputargs, loop.operations, token) + frame = self.cpu.execute_token(token) From noreply at buildbot.pypy.org Mon Jan 21 11:39:28 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 11:39:28 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: use this also in malloc slowpath Message-ID: <20130121103928.8E5DC1C106A@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60292:3f75b755c718 Date: 2013-01-21 12:39 +0200 http://bitbucket.org/pypy/pypy/changeset/3f75b755c718/ Log: use this also in malloc slowpath diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -215,6 +215,7 @@ mc.MOV(edi, heap(nursery_free_adr)) # load this in EDX # clear the gc pattern mc.MOV_bi(ofs, 0) + self._reload_frame_if_necessary(mc) mc.RET() # # If the slowpath malloc failed, we raise a MemoryError that @@ -1228,13 +1229,16 @@ self.mc.CALL(x) if align: self.mc.ADD_ri(esp.value, align * WORD) + self._reload_frame_if_necessary(self.mc) + + def _reload_frame_if_necessary(self, mc): gcrootmap = self.cpu.gc_ll_descr.gcrootmap if gcrootmap and gcrootmap.is_shadow_stack: rst = gcrootmap.get_root_stack_top_addr() - self.mc.MOV(edx, heap(rst)) - self.mc.MOV(ebp, mem(edx, -WORD)) + mc.MOV(edx, heap(rst)) + mc.MOV(ebp, mem(edx, -WORD)) base_ofs = self.cpu.get_baseofs_of_frame_field() - self.mc.ADD_ri(ebp.value, base_ofs) + mc.ADD_ri(ebp.value, base_ofs) def call(self, addr, args, res): self._emit_call(imm(addr), args) From noreply at buildbot.pypy.org Mon Jan 21 12:11:42 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 12:11:42 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: I'm such a moron Message-ID: <20130121111142.369AA1C00EC@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60293:7e03419e42df Date: 2013-01-21 13:11 +0200 http://bitbucket.org/pypy/pypy/changeset/7e03419e42df/ Log: I'm such a moron diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -1871,15 +1871,14 @@ positions[i] = v * WORD # write down the positions of locs guardtok.faildescr.rd_locs = positions - if WORD == 4: - mc.PUSH(imm(fail_descr)) - mc.PUSH(imm(gcpattern)) - mc.CALL(imm(target)) - else: - mc.MOV_ri64(X86_64_SCRATCH_REG.value, target) - mc.PUSH(imm(fail_descr)) - mc.PUSH(imm(gcpattern)) - self.mc.JMP_r(X86_64_SCRATCH_REG.value) + #if WORD == 4: + # mc.PUSH(imm(fail_descr)) + # mc.PUSH(imm(gcpattern)) + # mc.JMP(imm(target)) + #else: + mc.PUSH(imm(fail_descr)) + mc.PUSH(imm(gcpattern)) + mc.JMP(imm(target)) return startpos def rebuild_faillocs_from_descr(self, descr, inputargs): From noreply at buildbot.pypy.org Mon Jan 21 14:17:20 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 14:17:20 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: minor fixes Message-ID: <20130121131720.028DE1C00EC@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60294:d5b4848e8385 Date: 2013-01-21 15:16 +0200 http://bitbucket.org/pypy/pypy/changeset/d5b4848e8385/ Log: minor fixes diff --git a/rpython/jit/backend/llsupport/jitframe.py b/rpython/jit/backend/llsupport/jitframe.py --- a/rpython/jit/backend/llsupport/jitframe.py +++ b/rpython/jit/backend/llsupport/jitframe.py @@ -38,6 +38,7 @@ new_frame = lltype.malloc(JITFRAME, frame_info.jfi_frame_depth, zero=True) new_frame.jf_gcmap = frame_info.jfi_gcmap new_frame.jf_frame_info = frame_info + new_frame.jf_force_descr = frame.jf_force_descr return new_frame JITFRAME = lltype.GcStruct( diff --git a/rpython/jit/backend/llsupport/test/test_gc.py b/rpython/jit/backend/llsupport/test/test_gc.py --- a/rpython/jit/backend/llsupport/test/test_gc.py +++ b/rpython/jit/backend/llsupport/test/test_gc.py @@ -273,8 +273,8 @@ PREV_STATICSIZE = jitframe.STATICSIZE try: jitframe.STATICSIZE = 3 - frame_info = lltype.malloc(jitframe.JITFRAMEINFO) - frame = lltype.malloc(jitframe.JITFRAME, 15) + frame_info = lltype.malloc(jitframe.JITFRAMEINFO, zero=True) + frame = lltype.malloc(jitframe.JITFRAME, 15, zero=True) frame.jf_frame_info = frame_info frame.jf_gcmap = lltype.malloc(jitframe.GCMAP, 4) frame.jf_gcmap[0] = 5 From noreply at buildbot.pypy.org Mon Jan 21 15:31:32 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 15:31:32 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: a bit of cowboy style here, but I might change the strategy, so let's see if it works. Fix gcmap passing across loop boundaries Message-ID: <20130121143132.EE0741C106A@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60295:e33a94b89fba Date: 2013-01-21 16:31 +0200 http://bitbucket.org/pypy/pypy/changeset/e33a94b89fba/ Log: a bit of cowboy style here, but I might change the strategy, so let's see if it works. Fix gcmap passing across loop boundaries diff --git a/rpython/jit/backend/llsupport/jitframe.py b/rpython/jit/backend/llsupport/jitframe.py --- a/rpython/jit/backend/llsupport/jitframe.py +++ b/rpython/jit/backend/llsupport/jitframe.py @@ -38,7 +38,6 @@ new_frame = lltype.malloc(JITFRAME, frame_info.jfi_frame_depth, zero=True) new_frame.jf_gcmap = frame_info.jfi_gcmap new_frame.jf_frame_info = frame_info - new_frame.jf_force_descr = frame.jf_force_descr return new_frame JITFRAME = lltype.GcStruct( diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -695,7 +695,7 @@ len(gcmap)) for i in range(len(gcmap)): frame_info.jfi_gcmap[i] = gcmap[i] - + def get_asmmemmgr_blocks(self, looptoken): clt = looptoken.compiled_loop_token if clt.asmmemmgr_blocks is None: @@ -1235,6 +1235,7 @@ gcrootmap = self.cpu.gc_ll_descr.gcrootmap if gcrootmap and gcrootmap.is_shadow_stack: rst = gcrootmap.get_root_stack_top_addr() + mc.MOV(ecx, ebp) # debugging mc.MOV(edx, heap(rst)) mc.MOV(ebp, mem(edx, -WORD)) base_ofs = self.cpu.get_baseofs_of_frame_field() @@ -2396,12 +2397,25 @@ not_implemented("not implemented operation (guard): %s" % op.getopname()) - def closing_jump(self, target_token): + def closing_jump(self, target_token, gcmap): target = target_token._x86_loop_code if target_token in self.target_tokens_currently_compiling: curpos = self.mc.get_relative_pos() + 5 self.mc.JMP_l(target - curpos) else: + # bleh, we need to clean up all the references that are not + # in our gcmap, but are in the target gcmap + if target_token.original_jitcell_token: + tgt_clt = target_token.original_jitcell_token.compiled_loop_token + tgt_gcmap = tgt_clt.frame_info.jfi_gcmap + rev = {} + for i in tgt_gcmap: + rev[i] = None + # for now clean them all, we might change the strategy + for k in gcmap: + if k not in rev: + # all ours that are not known to the target + self.mc.MOV_bi((k + JITFRAME_FIXED_SIZE) * WORD, 0) self.mc.JMP(imm(target)) def malloc_cond(self, nursery_free_adr, nursery_top_adr, size, gcpattern): diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -1284,7 +1284,7 @@ src_locations1, dst_locations1, tmpreg, src_locations2, dst_locations2, xmmtmp) self.possibly_free_vars_for_op(op) - assembler.closing_jump(self.jump_target_descr) + assembler.closing_jump(self.jump_target_descr, self.get_gc_map()) def consider_debug_merge_point(self, op): pass From noreply at buildbot.pypy.org Mon Jan 21 15:33:50 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 15:33:50 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: seems you can't iterate over those, too bade Message-ID: <20130121143350.C014B1C1241@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60296:a90b88ed15c4 Date: 2013-01-21 16:33 +0200 http://bitbucket.org/pypy/pypy/changeset/a90b88ed15c4/ Log: seems you can't iterate over those, too bade diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -2409,8 +2409,10 @@ tgt_clt = target_token.original_jitcell_token.compiled_loop_token tgt_gcmap = tgt_clt.frame_info.jfi_gcmap rev = {} - for i in tgt_gcmap: - rev[i] = None + i = 0 + while i < len(tgt_gcmap): + rev[tgt_gcmap[i]] = None + i += 1 # for now clean them all, we might change the strategy for k in gcmap: if k not in rev: From noreply at buildbot.pypy.org Mon Jan 21 17:04:56 2013 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 21 Jan 2013 17:04:56 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: add failing tests Message-ID: <20130121160456.DEB151C106A@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: missing-ndarray-attributes Changeset: r60297:f7cc69fa2ef6 Date: 2013-01-21 17:55 +0200 http://bitbucket.org/pypy/pypy/changeset/f7cc69fa2ef6/ Log: add failing tests diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -1634,6 +1634,13 @@ b = array([1, 2]).astype(float) assert (b == [1, 2]).all() assert b.dtype == 'float' + b = array([1, 2], dtype=complex).astype(int) + assert (b == [1, 2]).all() + assert b.dtype == 'int' + b = array([0, 1, 2], dtype=complex).astype(bool) + assert (b == [False, True, True]).all() + assert b.dtype == 'bool' + def test_base(self): from _numpypy import array @@ -2120,11 +2127,14 @@ assert (a[:,::2].take([3, 2, 1]) == [6, 4, 2]).all() def test_compress(self): - from _numpypy import arange + from _numpypy import arange, array a = arange(10) assert (a.compress([True, False, True]) == [0, 2]).all() assert (a.compress([1, 0, 13]) == [0, 2]).all() assert (a.compress([1, 0, 13.5]) == [0, 2]).all() + assert (a.compress(array([1, 0, 13.5], dtype='>f4')) == [0, 2]).all() + assert (a.compress(array([1, 0, 13.5], dtype=' Author: mattip Branch: missing-ndarray-attributes Changeset: r60298:317b0a0672b2 Date: 2013-01-21 18:03 +0200 http://bitbucket.org/pypy/pypy/changeset/317b0a0672b2/ Log: implement complex astype, complex compress diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -134,6 +134,11 @@ def box(self, value): return self.BoxType(rffi.cast(self.T, value)) + @specialize.argtype(1, 2) + def box_complex(self, real, imag): + #XXX this is the place to display a warning + return self.BoxType(rffi.cast(self.T, real)) + def unbox(self, box): assert isinstance(box, self.BoxType) return box.value @@ -319,6 +324,16 @@ else: return self.False + @specialize.argtype(1, 2) + def box_complex(self, real, imag): + box = Primitive.box(self, real) + if box.value: + return self.True + box = Primitive.box(self, imag) + if box.value: + return self.True + return self.False + def coerce_subtype(self, space, w_subtype, w_item): # Doesn't return subclasses so it can return the constants. return self._coerce(space, w_item) @@ -1050,6 +1065,10 @@ def for_computation(self, v): return float(v[0]), float(v[1]) + def read_bool(self, arr, i, offset): + v = self.for_computation(self._read(arr.storage, i, offset)) + return bool(v[0]) or bool(v[1]) + def get_element_size(self): return 2 * rffi.sizeof(self._COMPONENTS_T) From noreply at buildbot.pypy.org Mon Jan 21 18:04:45 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 18:04:45 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: make gcmap work per-call and per-malloc-slowpath Message-ID: <20130121170445.A1A571C134E@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60299:8f0f2d2b73bf Date: 2013-01-21 19:04 +0200 http://bitbucket.org/pypy/pypy/changeset/8f0f2d2b73bf/ Log: make gcmap work per-call and per-malloc-slowpath diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py --- a/rpython/jit/backend/llsupport/gc.py +++ b/rpython/jit/backend/llsupport/gc.py @@ -120,12 +120,10 @@ descrs = JitFrameDescrs() descrs.arraydescr = cpu.arraydescrof(jitframe.JITFRAME) for name in ['jf_descr', 'jf_guard_exc', 'jf_force_descr', - 'jf_frame_info', 'jf_gcpattern', 'jf_gcmap']: + 'jf_frame_info', 'jf_gcmap']: setattr(descrs, name, cpu.fielddescrof(jitframe.JITFRAME, name)) descrs.jfi_frame_depth = cpu.fielddescrof(jitframe.JITFRAMEINFO, 'jfi_frame_depth') - descrs.jfi_gcmap = cpu.fielddescrof(jitframe.JITFRAMEINFO, - 'jfi_gcmap') return descrs def getarraydescr_for_frame(self, type, index): diff --git a/rpython/jit/backend/llsupport/jitframe.py b/rpython/jit/backend/llsupport/jitframe.py --- a/rpython/jit/backend/llsupport/jitframe.py +++ b/rpython/jit/backend/llsupport/jitframe.py @@ -1,25 +1,23 @@ -from rpython.rtyper.lltypesystem import lltype, llmemory +from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.rtyper.annlowlevel import llhelper from rpython.rlib.objectmodel import specialize from rpython.rlib.debug import ll_assert STATICSIZE = 0 # patch from the assembler backend +SIZEOFSIGNED = rffi.sizeof(lltype.Signed) +IS_32BIT = (SIZEOFSIGNED == 2 ** 31 - 1) # this is an info that only depends on the assembler executed, copied from # compiled loop token (in fact we could use this as a compiled loop token # XXX do this -GCMAP = lltype.GcArray(lltype.Signed) +GCMAP = lltype.GcArray(lltype.Unsigned) NULLGCMAP = lltype.nullptr(GCMAP) -# XXX make it SHORT not Signed JITFRAMEINFO = lltype.GcStruct( 'JITFRAMEINFO', # the depth of frame ('jfi_frame_depth', lltype.Signed), - # gcindexlist is a list of indexes of GC ptrs - # in the actual array jf_frame of JITFRAME - ('jfi_gcmap', lltype.Ptr(GCMAP)), ) NULLFRAMEINFO = lltype.nullptr(JITFRAMEINFO) @@ -29,14 +27,13 @@ def jitframe_allocate(frame_info): frame = lltype.malloc(JITFRAME, frame_info.jfi_frame_depth, zero=True) - frame.jf_gcmap = frame_info.jfi_gcmap frame.jf_frame_info = frame_info return frame def jitframe_copy(frame): frame_info = frame.jf_frame_info new_frame = lltype.malloc(JITFRAME, frame_info.jfi_frame_depth, zero=True) - new_frame.jf_gcmap = frame_info.jfi_gcmap + ll_assert(frame.jf_gcmap == NULLGCMAP, "non empty gc map when copying") new_frame.jf_frame_info = frame_info return new_frame @@ -50,10 +47,7 @@ ('jf_descr', llmemory.GCREF), # guard_not_forced descr ('jf_force_descr', llmemory.GCREF), - # a bitmask of where are GCREFS in the top of the frame (saved registers) - # used for calls and failures - ('jf_gcpattern', lltype.Signed), - # a copy of gcmap from frameinfo + # a map of GC pointers ('jf_gcmap', lltype.Ptr(GCMAP)), # For the front-end: a GCREF for the savedata ('jf_savedata', llmemory.GCREF), @@ -83,14 +77,14 @@ GCMAPBASEOFS = llmemory.itemoffsetof(GCMAP, 0) BASEITEMOFS = llmemory.itemoffsetof(JITFRAME.jf_frame, 0) SIGN_SIZE = llmemory.sizeof(lltype.Signed) +UNSIGN_SIZE = llmemory.sizeof(lltype.Unsigned) def jitframe_trace(obj_addr, prev): if prev == llmemory.NULL: (obj_addr + getofs('jf_gc_trace_state')).signed[0] = 0 return obj_addr + getofs('jf_frame_info') fld = (obj_addr + getofs('jf_gc_trace_state')).signed[0] - state = fld & 0xff - no = fld >> 8 + state = fld & 0x7 # 3bits of possible states if state == 0: (obj_addr + getofs('jf_gc_trace_state')).signed[0] = 1 return obj_addr + getofs('jf_descr') @@ -106,26 +100,38 @@ elif state == 4: (obj_addr + getofs('jf_gc_trace_state')).signed[0] = 5 return obj_addr + getofs('jf_guard_exc') - elif state == 5: - # bit pattern - gcpat = (obj_addr + getofs('jf_gcpattern')).signed[0] - while no < STATICSIZE and gcpat & (1 << no) == 0: - no += 1 - if no != STATICSIZE: - newstate = 5 | ((no + 1) << 8) - (obj_addr + getofs('jf_gc_trace_state')).signed[0] = newstate - return obj_addr + getofs('jf_frame') + BASEITEMOFS + SIGN_SIZE * no - state = 6 - no = 0 - ll_assert(state == 6, "invalid tracer state") + ll_assert(state == 5, "invalid state") + # bit pattern + # decode the pattern + if IS_32BIT: + # 32 possible bits + state = (fld >> 3) & 0x1f + no = fld >> (3 + 5) + MAX = 31 + else: + # 64 possible bits + state = (fld >> 3) & 0x3f + no = fld >> (3 + 6) + MAX = 63 gcmap = (obj_addr + getofs('jf_gcmap')).address[0] - gcmaplen = (gcmap + GCMAPLENGTHOFS).signed[0] - if no >= gcmaplen: - return llmemory.NULL - index = (gcmap + GCMAPBASEOFS + SIGN_SIZE * no).signed[0] + STATICSIZE - newstate = 6 | ((no + 1) << 8) - (obj_addr + getofs('jf_gc_trace_state')).signed[0] = newstate - return obj_addr + getofs('jf_frame') + BASEITEMOFS + SIGN_SIZE * index + gcmap_lgt = (gcmap + GCMAPLENGTHOFS).signed[0] + while no < gcmap_lgt: + cur = (gcmap + GCMAPBASEOFS + UNSIGN_SIZE * no).unsigned[0] + while state < MAX and not (cur & (1 << state)): + state += 1 + if state < MAX: + # found it + # save new state + if IS_32BIT: + new_state = 5 | ((state + 1) << 3) | (no << 8) + else: + new_state = 5 | ((state + 1) << 3) | (no << 9) + (obj_addr + getofs('jf_gc_trace_state')).signed[0] = new_state + return (obj_addr + getofs('jf_frame') + BASEITEMOFS + SIGN_SIZE * + (no * SIZEOFSIGNED * 8 + state)) + no += 1 + state = 0 + return llmemory.NULL CUSTOMTRACEFUNC = lltype.FuncType([llmemory.Address, llmemory.Address], llmemory.Address) diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py --- a/rpython/jit/backend/llsupport/regalloc.py +++ b/rpython/jit/backend/llsupport/regalloc.py @@ -1,5 +1,5 @@ import os -from rpython.jit.metainterp.history import Const, Box, REF +from rpython.jit.metainterp.history import Const, Box, REF, INT from rpython.rlib.objectmodel import we_are_translated, specialize from rpython.jit.metainterp.resoperation import rop @@ -112,56 +112,18 @@ node = node.next return 'LinkedList(%s)' % '->'.join(l) -def frame_manager_from_gcmap(FmClass, gcmap, depth, frame_bindings): - if not gcmap: - return FmClass() - rev_bindings = [False] * depth - for arg, loc in frame_bindings.iteritems(): - size = FmClass.frame_size(arg.type) - if size == 2: - rev_bindings[FmClass.get_loc_index(loc) + 1] = True - assert size == 1 - rev_bindings[FmClass.get_loc_index(loc)] = True - gcrefs = [] - others = [] - c = 0 - for i in range(len(gcmap)): - item = gcmap[i] - while c < item: - if not rev_bindings[c]: - others.append(c) - c += 1 - if not rev_bindings[item]: - gcrefs.append(item) - c += 1 - for i in range(c, depth): - if not rev_bindings[i]: - others.append(i) - fm = FmClass(depth, gcrefs, others) - for arg, loc in frame_bindings.iteritems(): - fm.bindings[arg] = loc - return fm - class FrameManager(object): """ Manage frame positions start_free_depth is the start where we can allocate in whatever order we like. - - freelist_gcrefs and freelist_others are free lists of locations that - can be used for gcrefs and others. below stack_free_depth. Note - that if floats are occupying more than one spot, in order to allocate - the correct size, we need to use more than one from the freelist in - the consecutive order. """ - def __init__(self, start_free_depth=0, freelist_gcrefs=None, - freelist_others=None): + def __init__(self, start_free_depth=0, freelist=None): self.bindings = {} self.current_frame_depth = start_free_depth # we disable hints for now #self.hint_frame_locations = {} - self.freelist_gcrefs = LinkedList(self, freelist_gcrefs) - self.freelist_others = LinkedList(self, freelist_others) + self.freelist = LinkedList(self, freelist) def get_frame_depth(self): return self.current_frame_depth @@ -191,10 +153,7 @@ # that 'size' is a power of two. The reason for doing so is to # avoid obscure issues in jump.py with stack locations that try # to move from position (6,7) to position (7,8). - if box.type == REF: - newloc = self.freelist_gcrefs.pop(1, box.type) - else: - newloc = self.freelist_others.pop(size, box.type) + newloc = self.freelist.pop(size, box.type) if newloc is None: # index = self.get_frame_depth() @@ -209,17 +168,23 @@ self.bindings[box] = newloc return newloc + def bind(self, box, loc): + pos = self.get_loc_index(loc) + size = self.frame_size(box.type) + if self.current_frame_depth < pos: + for i in range(self.current_frame_depth, pos): + self.freelist.append(1, self.frame_pos(i, INT)) + self.current_frame_depth = pos + size + self.bindings[box] = loc + def mark_as_free(self, box): try: loc = self.bindings[box] except KeyError: return # already gone del self.bindings[box] - if box.type == REF: - self.freelist_gcrefs.append(1, loc) - else: - size = self.frame_size(box.type) - self.freelist_others.append(size, loc) + size = self.frame_size(box.type) + self.freelist.append(size, loc) def try_to_reuse_location(self, box, loc): xxx @@ -242,16 +207,6 @@ def _gather_gcroots(lst, var): lst.append(var) - def get_gc_map(self): - """ returns a list of locations where GC pointers are - """ - assert not self.bindings - # XXX unsure, maybe what we want is to - # free everything instead - lst = [] - self.freelist_gcrefs.foreach(self._gather_gcroots, lst) - return lst - # abstract methods that need to be overwritten for specific assemblers @staticmethod def frame_pos(loc, type): diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -156,10 +156,6 @@ op2 = ResOperation(rop.SETFIELD_GC, [frame, history.ConstPtr(llref)], None, descr=descrs.jf_frame_info) self.newops.append(op2) - llref = lltype.cast_opaque_ptr(llmemory.GCREF, jfi.jfi_gcmap) - op3 = ResOperation(rop.SETFIELD_GC, [frame, history.ConstPtr(llref)], - None, descr=descrs.jf_gcmap) - self.newops.append(op3) for i, arg in enumerate(op.getarglist()): index, descr = self.cpu.getarraydescr_for_frame(arg.type, i) self.newops.append(ResOperation(rop.SETARRAYITEM_GC, diff --git a/rpython/jit/backend/llsupport/test/test_gc.py b/rpython/jit/backend/llsupport/test/test_gc.py --- a/rpython/jit/backend/llsupport/test/test_gc.py +++ b/rpython/jit/backend/llsupport/test/test_gc.py @@ -7,7 +7,7 @@ from rpython.jit.metainterp.history import BoxPtr, BoxInt, ConstPtr from rpython.jit.metainterp.resoperation import get_deep_immutable_oplist, rop,\ ResOperation -from rpython.rlib.rarithmetic import is_valid_int +from rpython.rlib.rarithmetic import is_valid_int, r_uint def test_boehm(): gc_ll_descr = gc.GcLLDescr_boehm(None, None, None) @@ -276,12 +276,9 @@ frame_info = lltype.malloc(jitframe.JITFRAMEINFO, zero=True) frame = lltype.malloc(jitframe.JITFRAME, 15, zero=True) frame.jf_frame_info = frame_info - frame.jf_gcmap = lltype.malloc(jitframe.GCMAP, 4) - frame.jf_gcmap[0] = 5 - frame.jf_gcmap[1] = 7 - frame.jf_gcmap[2] = 8 - frame.jf_gcmap[3] = 10 - frame.jf_gcpattern = 1 | 4 + frame.jf_gcmap = lltype.malloc(jitframe.GCMAP, 2) + frame.jf_gcmap[0] = r_uint(1 | 2 | 8 | 32 | 128) + frame.jf_gcmap[1] = r_uint(2 | 16 | 32 | 128) frame_adr = llmemory.cast_ptr_to_adr(frame) all_addrs = [] next = jitframe.jitframe_trace(frame_adr, llmemory.NULL) @@ -296,12 +293,14 @@ counter += 1 # gcpattern assert all_addrs[6] == indexof(0) - assert all_addrs[7] == indexof(2) - assert all_addrs[8] == indexof(3 + 5) - assert all_addrs[9] == indexof(3 + 7) - assert all_addrs[10] == indexof(3 + 8) - assert all_addrs[11] == indexof(3 + 10) - assert len(all_addrs) == 6 + 4 + 2 + assert all_addrs[7] == indexof(1) + assert all_addrs[8] == indexof(3) + assert all_addrs[9] == indexof(5) + assert all_addrs[10] == indexof(7) + # XXX 32bit + assert all_addrs[11] == indexof(65) + + assert len(all_addrs) == 6 + 5 + 4 # 6 static fields, 4 addresses from gcmap, 2 from gcpattern finally: jitframe.STATICSIZE = PREV_STATICSIZE diff --git a/rpython/jit/backend/llsupport/test/test_regalloc.py b/rpython/jit/backend/llsupport/test/test_regalloc.py --- a/rpython/jit/backend/llsupport/test/test_regalloc.py +++ b/rpython/jit/backend/llsupport/test/test_regalloc.py @@ -1,8 +1,7 @@ import py from rpython.jit.metainterp.history import BoxInt, ConstInt, BoxFloat, INT, FLOAT,\ BoxPtr -from rpython.jit.backend.llsupport.regalloc import FrameManager, LinkedList,\ - frame_manager_from_gcmap +from rpython.jit.backend.llsupport.regalloc import FrameManager, LinkedList from rpython.jit.backend.llsupport.regalloc import RegisterManager as BaseRegMan def newboxes(*values): @@ -510,7 +509,7 @@ assert fm.get_loc_index(locf1) == 4 assert fm.get_frame_depth() == 5 fm.mark_as_free(b1) - assert fm.freelist_others + assert fm.freelist b2 = BoxInt() fm.loc(b2) # should be in the same spot as b1 before assert fm.get(b1) is None @@ -518,22 +517,22 @@ fm.mark_as_free(b0) p0 = BoxPtr() ploc = fm.loc(p0) - assert fm.get_loc_index(ploc) == 5 - assert fm.get_frame_depth() == 6 + assert fm.get_loc_index(ploc) == 0 + assert fm.get_frame_depth() == 5 assert ploc != loc1 p1 = BoxPtr() p1loc = fm.loc(p1) - assert fm.get_loc_index(p1loc) == 6 - assert fm.get_frame_depth() == 7 + assert fm.get_loc_index(p1loc) == 5 + assert fm.get_frame_depth() == 6 fm.mark_as_free(p0) p2 = BoxPtr() p2loc = fm.loc(p2) assert p2loc == ploc - assert not fm.freelist_gcrefs - assert len(fm.freelist_others) == 1 + assert len(fm.freelist) == 0 for box in fm.bindings.keys(): fm.mark_as_free(box) - assert fm.get_gc_map() == [5, 6] + fm.bind(BoxPtr(), FakeFramePos(3, 'r')) + assert len(fm.freelist) == 6 def test_frame_manager_basic(self): b0, b1 = newboxes(0, 1) @@ -562,7 +561,7 @@ assert fm.get_loc_index(locf1) == 5 assert fm.get_frame_depth() == 7 fm.mark_as_free(b1) - assert fm.freelist_others + assert fm.freelist b2 = BoxInt() fm.loc(b2) # should be in the same spot as b1 before assert fm.get(b1) is None @@ -570,45 +569,22 @@ fm.mark_as_free(b0) p0 = BoxPtr() ploc = fm.loc(p0) - assert fm.get_loc_index(ploc) == 7 - assert fm.get_frame_depth() == 8 + assert fm.get_loc_index(ploc) == 0 + assert fm.get_frame_depth() == 7 assert ploc != loc1 p1 = BoxPtr() p1loc = fm.loc(p1) - assert fm.get_loc_index(p1loc) == 8 - assert fm.get_frame_depth() == 9 + assert fm.get_loc_index(p1loc) == 7 + assert fm.get_frame_depth() == 8 fm.mark_as_free(p0) p2 = BoxPtr() p2loc = fm.loc(p2) assert p2loc == ploc - assert not fm.freelist_gcrefs - assert len(fm.freelist_others) == 1 + assert len(fm.freelist) == 0 fm.mark_as_free(b2) f3 = BoxFloat() + fm.mark_as_free(p2) floc = fm.loc(f3) assert fm.get_loc_index(floc) == 0 for box in fm.bindings.keys(): fm.mark_as_free(box) - assert fm.get_gc_map() == [7, 8] - - def test_fm_from_gcmap(self): - class Loc(object): - def __init__(self, l): - self.l = l - - class Fm(FrameManager): - @staticmethod - def get_loc_index(l): - return l.l - - b0 = BoxInt() - b1 = BoxInt() - l0 = Loc(5) - l1 = Loc(2) - bindings = {b0: l0, b1: l1} - fm = frame_manager_from_gcmap(Fm, [1, 5, 6, 8], 13, - bindings) - assert repr(fm.freelist_gcrefs) == "LinkedList(1->6->8)" - assert repr(fm.freelist_others) == "LinkedList(0->3->4->7->9->10->11->12)" - assert fm.current_frame_depth == 13 - assert fm.bindings == bindings diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py --- a/rpython/jit/backend/llsupport/test/test_rewrite.py +++ b/rpython/jit/backend/llsupport/test/test_rewrite.py @@ -98,8 +98,12 @@ []) equaloplists(operations, expected.operations) +class FakeTracker(object): + pass + class BaseFakeCPU(object): def __init__(self): + self.tracker = FakeTracker() self._cache = {} self.signedframedescr = ArrayDescr(3, 8, FieldDescr('len', 0, 0, 0), 0) self.floatframedescr = ArrayDescr(5, 8, FieldDescr('len', 0, 0, 0), 0) diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -30,7 +30,7 @@ from rpython.jit.backend.x86.jump import remap_frame_layout from rpython.jit.codewriter.effectinfo import EffectInfo from rpython.jit.codewriter import longlong -from rpython.rlib.rarithmetic import intmask +from rpython.rlib.rarithmetic import intmask, r_uint from rpython.rlib.objectmodel import compute_unique_id # darwin requires the stack to be 16 bytes aligned on calls. Same for gcc 4.5.0, @@ -42,15 +42,39 @@ class GuardToken(object): - def __init__(self, faildescr, failargs, fail_locs, exc, + def __init__(self, faildescr, failargs, fail_locs, exc, frame_depth, is_guard_not_invalidated, is_guard_not_forced): self.faildescr = faildescr self.failargs = failargs self.fail_locs = fail_locs + self.gcmap = self.compute_gcmap(failargs, fail_locs, frame_depth) self.exc = exc self.is_guard_not_invalidated = is_guard_not_invalidated self.is_guard_not_forced = is_guard_not_forced + def compute_gcmap(self, failargs, fail_locs, frame_depth): + # note that regalloc has a very similar compute, but + # one that does iteration over all bindings, so slightly different, + # eh + size = frame_depth + JITFRAME_FIXED_SIZE + gcmap = lltype.malloc(jitframe.GCMAP, size // WORD // 8 + 1, + zero=True) + input_i = 0 + for i in range(len(failargs)): + arg = failargs[i] + if arg is None: + continue + loc = fail_locs[input_i] + input_i += 1 + if arg.type == REF: + loc = fail_locs[i] + if isinstance(loc, RegLoc): + val = gpr_reg_mgr_cls.all_reg_indexes[loc.value] + else: + val = loc.value // WORD + gcmap[val // WORD // 8] |= r_uint(1) << (val % (WORD * 8)) + return gcmap + DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', ('i', lltype.Signed), ('type', lltype.Char), # 'b'ridge, 'l'abel or # 'e'ntry point @@ -188,7 +212,7 @@ """ mc = codebuf.MachineCodeBlockWrapper() self._push_all_regs_to_frame(mc, [eax, edi], self.cpu.supports_floats) - ofs = self.cpu.get_ofs_of_frame_field('jf_gcpattern') + ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') # store the gc pattern mc.MOV_rs(ecx.value, WORD) mc.MOV_br(ofs, ecx.value) @@ -505,7 +529,6 @@ looptoken._x86_loop_code = looppos frame_depth = self._assemble(regalloc, inputargs, operations) clt.frame_info.jfi_frame_depth = frame_depth + JITFRAME_FIXED_SIZE - self._update_gcmap(clt.frame_info, regalloc) # size_excluding_failure_stuff = self.mc.get_relative_pos() self.write_pending_failure_recoveries() @@ -581,7 +604,6 @@ self._patch_stackadjust(stack_check_patch_ofs + rawstart, frame_depth) self.fixup_target_tokens(rawstart) self.current_clt.frame_info.jfi_frame_depth = frame_depth - self._update_gcmap(self.current_clt.frame_info, regalloc) self.teardown() # oprofile support if self.cpu.profile_agent is not None: @@ -659,42 +681,12 @@ offset = mc.get_relative_pos() - jg_location assert 0 < offset <= 127 mc.overwrite(jg_location-1, chr(offset)) - return stack_check_cmp_ofs - - def _insert_frame_adjustment(self, frame_info): - """ Our frame might end up being different than what we expect. - Note that depth is fine (since bridges check that), but we need - to update gcmap - """ - # XXX note that this can be easily shifted to JUMP - # instead of LABEL, would be slightly faster - gcmap_ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') - frame_info_addr = rffi.cast(lltype.Signed, frame_info) - frame_info_ofs = self.cpu.get_ofs_of_frame_field('jf_frame_info') - jfi_gc_map_ofs = self.cpu.get_ofs_of_frame_field('jfi_gcmap') - if IS_X86_32: - self.mc.MOV_bi(frame_info_ofs, frame_info_addr) - XXX - else: - self.mc.MOV_ri(X86_64_SCRATCH_REG.value, frame_info_addr) - self.mc.MOV_br(frame_info_ofs, X86_64_SCRATCH_REG.value) - self.mc.MOV_rm(X86_64_SCRATCH_REG.value, - (X86_64_SCRATCH_REG.value, - jfi_gc_map_ofs)) - self.mc.MOV_br(gcmap_ofs, X86_64_SCRATCH_REG.value) - + return stack_check_cmp_ofs def _patch_stackadjust(self, adr, allocated_depth): mc = codebuf.MachineCodeBlockWrapper() mc.writeimm32(allocated_depth) mc.copy_to_raw_memory(adr) - - def _update_gcmap(self, frame_info, regalloc): - gcmap = regalloc.get_gc_map() - frame_info.jfi_gcmap = lltype.malloc(jitframe.GCMAP, - len(gcmap)) - for i in range(len(gcmap)): - frame_info.jfi_gcmap[i] = gcmap[i] def get_asmmemmgr_blocks(self, looptoken): clt = looptoken.compiled_loop_token @@ -888,7 +880,6 @@ old_fi = oldlooptoken.compiled_loop_token.frame_info new_fi = newlooptoken.compiled_loop_token.frame_info old_fi.jfi_frame_depth = new_fi.jfi_frame_depth - old_fi.jfi_gcmap = new_fi.jfi_gcmap mc = codebuf.MachineCodeBlockWrapper() mc.JMP(imm(target)) if WORD == 4: # keep in sync with prepare_loop() @@ -974,14 +965,14 @@ genop_math_list[oopspecindex](self, op, arglocs, resloc) def regalloc_perform_with_guard(self, op, guard_op, faillocs, - arglocs, resloc): + arglocs, resloc, frame_depth): faildescr = guard_op.getdescr() assert isinstance(faildescr, AbstractFailDescr) failargs = guard_op.getfailargs() guard_opnum = guard_op.getopnum() guard_token = self.implement_guard_recovery(guard_opnum, faildescr, failargs, - faillocs) + faillocs, frame_depth) if op is None: dispatch_opnum = guard_opnum else: @@ -992,9 +983,10 @@ # must be added by the genop_guard_list[]() assert guard_token is self.pending_guard_tokens[-1] - def regalloc_perform_guard(self, guard_op, faillocs, arglocs, resloc): + def regalloc_perform_guard(self, guard_op, faillocs, arglocs, resloc, + frame_depth): self.regalloc_perform_with_guard(None, guard_op, faillocs, arglocs, - resloc) + resloc, frame_depth) def load_effective_addr(self, sizereg, baseofs, scale, result, frm=imm0): self.mc.LEA(result, addr_add(frm, sizereg, baseofs, scale)) @@ -1824,13 +1816,13 @@ self.implement_guard(guard_token, 'NE') def implement_guard_recovery(self, guard_opnum, faildescr, failargs, - fail_locs): + fail_locs, frame_depth): exc = (guard_opnum == rop.GUARD_EXCEPTION or guard_opnum == rop.GUARD_NO_EXCEPTION or guard_opnum == rop.GUARD_NOT_FORCED) is_guard_not_invalidated = guard_opnum == rop.GUARD_NOT_INVALIDATED is_guard_not_forced = guard_opnum == rop.GUARD_NOT_FORCED - return GuardToken(faildescr, failargs, fail_locs, exc, + return GuardToken(faildescr, failargs, fail_locs, exc, frame_depth, is_guard_not_invalidated, is_guard_not_forced) def generate_propagate_error_64(self): @@ -1855,7 +1847,6 @@ fail_descr = cast_instance_to_gcref(guardtok.faildescr) fail_descr = rffi.cast(lltype.Signed, fail_descr) positions = [0] * len(guardtok.fail_locs) - gcpattern = 0 for i, loc in enumerate(guardtok.fail_locs): if loc is None or loc is ebp: # frame positions[i] = -1 @@ -1867,8 +1858,6 @@ v = len(gpr_reg_mgr_cls.all_regs) + loc.value else: v = gpr_reg_mgr_cls.all_reg_indexes[loc.value] - if guardtok.failargs[i].type == REF: - gcpattern |= (1 << v) positions[i] = v * WORD # write down the positions of locs guardtok.faildescr.rd_locs = positions @@ -1878,7 +1867,11 @@ # mc.JMP(imm(target)) #else: mc.PUSH(imm(fail_descr)) - mc.PUSH(imm(gcpattern)) + gcmapref = lltype.cast_opaque_ptr(llmemory.GCREF, guardtok.gcmap) + # keep the ref alive + self.current_clt.allgcrefs.append(gcmapref) + rgc._make_sure_does_not_move(gcmapref) + mc.PUSH(imm(rffi.cast(lltype.Signed, gcmapref))) mc.JMP(imm(target)) return startpos @@ -1949,7 +1942,7 @@ # throws away most of the frame, including all the PUSHes that we # did just above. ofs = self.cpu.get_ofs_of_frame_field('jf_descr') - ofs2 = self.cpu.get_ofs_of_frame_field('jf_gcpattern') + ofs2 = self.cpu.get_ofs_of_frame_field('jf_gcmap') base_ofs = self.cpu.get_baseofs_of_frame_field() mc.POP(eax) mc.MOV_br(ofs2, eax.value) @@ -1980,9 +1973,6 @@ # exit function self._call_footer() - def genop_discard_label(self, op, arglocs): - self._insert_frame_adjustment(self.current_clt.frame_info) - def implement_guard(self, guard_token, condition=None): # These jumps are patched later. if condition: @@ -2397,30 +2387,15 @@ not_implemented("not implemented operation (guard): %s" % op.getopname()) - def closing_jump(self, target_token, gcmap): + def closing_jump(self, target_token): target = target_token._x86_loop_code if target_token in self.target_tokens_currently_compiling: curpos = self.mc.get_relative_pos() + 5 self.mc.JMP_l(target - curpos) else: - # bleh, we need to clean up all the references that are not - # in our gcmap, but are in the target gcmap - if target_token.original_jitcell_token: - tgt_clt = target_token.original_jitcell_token.compiled_loop_token - tgt_gcmap = tgt_clt.frame_info.jfi_gcmap - rev = {} - i = 0 - while i < len(tgt_gcmap): - rev[tgt_gcmap[i]] = None - i += 1 - # for now clean them all, we might change the strategy - for k in gcmap: - if k not in rev: - # all ours that are not known to the target - self.mc.MOV_bi((k + JITFRAME_FIXED_SIZE) * WORD, 0) self.mc.JMP(imm(target)) - def malloc_cond(self, nursery_free_adr, nursery_top_adr, size, gcpattern): + def malloc_cond(self, nursery_free_adr, nursery_top_adr, size, gcmap): assert size & (WORD-1) == 0 # must be correctly aligned self.mc.MOV(eax, heap(nursery_free_adr)) self.mc.LEA_rm(edi.value, (eax.value, size)) @@ -2430,7 +2405,9 @@ # this is a 32bit gcpattern that describes where are GC roots # in registers (which will be saved while doing slowpath) # store it on top of the stack (we might not have a register free) - self.mc.MOV_si(0, gcpattern) + gcmapref = lltype.cast_opaque_ptr(llmemory.GCREF, gcmap) + rgc._make_sure_does_not_move(gcmapref) + self.mc.MOV(RawEspLoc(0, REF), imm(rffi.cast(lltype.Signed, gcmapref))) self.mc.CALL(imm(self.malloc_slowpath)) offset = self.mc.get_relative_pos() - jmp_adr assert 0 < offset <= 127 diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -17,19 +17,18 @@ from rpython.jit.codewriter import longlong from rpython.jit.codewriter.effectinfo import EffectInfo from rpython.jit.metainterp.resoperation import rop -from rpython.jit.backend.llsupport.jitframe import NULLGCMAP +from rpython.jit.backend.llsupport.jitframe import NULLGCMAP, GCMAP from rpython.jit.backend.llsupport.descr import ArrayDescr from rpython.jit.backend.llsupport.descr import CallDescr from rpython.jit.backend.llsupport.descr import unpack_arraydescr from rpython.jit.backend.llsupport.descr import unpack_fielddescr from rpython.jit.backend.llsupport.descr import unpack_interiorfielddescr -from rpython.jit.backend.llsupport.regalloc import FrameManager, RegisterManager,\ - TempBox, compute_vars_longevity, is_comparison_or_ovf_op,\ - frame_manager_from_gcmap +from rpython.jit.backend.llsupport.regalloc import FrameManager,\ + RegisterManager, TempBox, compute_vars_longevity, is_comparison_or_ovf_op from rpython.jit.backend.x86.arch import WORD, JITFRAME_FIXED_SIZE from rpython.jit.backend.x86.arch import IS_X86_32, IS_X86_64 from rpython.jit.backend.x86 import rx86 -from rpython.rlib.rarithmetic import r_longlong +from rpython.rlib.rarithmetic import r_longlong, r_uint class X86RegisterManager(RegisterManager): @@ -173,11 +172,8 @@ self.close_stack_struct = 0 self.final_jump_op = None - def _prepare(self, inputargs, operations, allgcrefs, gcmap=NULLGCMAP, - parent_frame_depth=0, frame_bindings=None): - self.fm = frame_manager_from_gcmap(X86FrameManager, gcmap, - parent_frame_depth, - frame_bindings) + def _prepare(self, inputargs, operations, allgcrefs): + self.fm = X86FrameManager() cpu = self.assembler.cpu operations = cpu.gc_ll_descr.rewrite_assembler(cpu, operations, allgcrefs) @@ -205,16 +201,9 @@ self.min_bytes_before_label = 13 return operations - def get_gc_map(self): - return self.fm.get_gc_map() - def prepare_bridge(self, inputargs, arglocs, operations, allgcrefs, frame_info): - frame_bindings = self._frame_bindings(arglocs, inputargs) - operations = self._prepare(inputargs, operations, allgcrefs, - frame_info.jfi_gcmap, - frame_info.jfi_frame_depth, - frame_bindings) + operations = self._prepare(inputargs, operations, allgcrefs) self._update_bindings(arglocs, inputargs) self.min_bytes_before_label = 0 return operations @@ -315,6 +304,8 @@ else: self.rm.reg_bindings[arg] = loc used[loc] = None + else: + self.fm.bind(arg, loc) self.rm.free_regs = [] for reg in self.rm.all_regs: if reg not in used: @@ -350,7 +341,8 @@ self.rm.position += 1 self.xrm.position += 1 self.assembler.regalloc_perform_with_guard(op, guard_op, faillocs, - arglocs, result_loc) + arglocs, result_loc, + self.fm.get_frame_depth()) self.possibly_free_vars(guard_op.getfailargs()) def perform_guard(self, guard_op, arglocs, result_loc): @@ -362,7 +354,8 @@ else: self.assembler.dump('%s(%s)' % (guard_op, arglocs)) self.assembler.regalloc_perform_guard(guard_op, faillocs, arglocs, - result_loc) + result_loc, + self.fm.get_frame_depth()) self.possibly_free_vars(guard_op.getfailargs()) def PerformDiscard(self, op, arglocs): @@ -903,15 +896,30 @@ self.rm.force_allocate_reg(tmp_box, selected_reg=edi) self.rm.possibly_free_var(tmp_box) # + gcmap = self._compute_gcmap() gc_ll_descr = self.assembler.cpu.gc_ll_descr - gcpattern = 0 - for box, reg in self.rm.reg_bindings.iteritems(): - if box.type == REF: - gcpattern |= (1 << self.rm.all_reg_indexes[reg.value]) self.assembler.malloc_cond( gc_ll_descr.get_nursery_free_addr(), gc_ll_descr.get_nursery_top_addr(), - size, gcpattern) + size, gcmap) + + def _compute_gcmap(self): + frame_depth = self.fm.get_frame_depth() + size = frame_depth + JITFRAME_FIXED_SIZE + gcmap = lltype.malloc(GCMAP, size // WORD // 8 + 1, + zero=True) + for box, loc in self.rm.reg_bindings.iteritems(): + if box.type == REF: + assert isinstance(loc, RegLoc) + val = gpr_reg_mgr_cls.all_reg_indexes[loc.value] + gcmap[val // WORD // 8] |= r_uint(1) << (val % (WORD * 8)) + for box, loc in self.fm.bindings.iteritems(): + if box.type == REF: + assert isinstance(loc, StackLoc) + val = loc.value // WORD + gcmap[val // WORD // 8] |= r_uint(1) << (val % (WORD * 8)) + return gcmap + def consider_setfield_gc(self, op): ofs, size, _ = unpack_fielddescr(op.getdescr()) @@ -1284,7 +1292,7 @@ src_locations1, dst_locations1, tmpreg, src_locations2, dst_locations2, xmmtmp) self.possibly_free_vars_for_op(op) - assembler.closing_jump(self.jump_target_descr, self.get_gc_map()) + assembler.closing_jump(self.jump_target_descr) def consider_debug_merge_point(self, op): pass @@ -1347,7 +1355,6 @@ # loop that ends up jumping to this LABEL, then we can now provide # the hints about the expected position of the spilled variables. - self.PerformDiscard(op, []) # XXX we never compile code like that? #jump_op = self.final_jump_op #if jump_op is not None and jump_op.getdescr() is descr: diff --git a/rpython/jit/backend/x86/test/test_gc_integration.py b/rpython/jit/backend/x86/test/test_gc_integration.py --- a/rpython/jit/backend/x86/test/test_gc_integration.py +++ b/rpython/jit/backend/x86/test/test_gc_integration.py @@ -67,38 +67,14 @@ # p0 and p3 should be in registers, p1 not so much assert self.getptr(0, lltype.Ptr(self.S)) == s1 # this is a fairly CPU specific check - all = len(gpr_reg_mgr_cls.all_regs) - assert frame.jf_gcpattern == (1 << (all - 1)) | (1 << (all - 2)) - # the gcmap should contain three things, p0, p1 and p3, but - # p3 stays in a register and is only represented in gcpattern, - # while p0 is in both - assert len(frame.jf_gcmap) == 2 - for i in range(2): - assert frame.jf_gcmap[i] == frame.jf_frame_info.jfi_gcmap[i] - assert frame.jf_gcmap[0] == 1 - assert frame.jf_gcmap[1] == 3 - - def test_label(self): - ops = ''' - [i0, p0, i1, p1] - label(i0, p0, i1, p1, descr=targettoken2) - p3 = getfield_gc(p0, descr=fielddescr) - force_spill(p3) - guard_true(i0) [p0, i1, p1, p3] - finish() - ''' - s1 = lltype.malloc(self.S) - s2 = lltype.malloc(self.S) - s1.field = s2 - self.interpret(ops, [0, s1, 1, s2]) - ops2 = ''' - [p0] - jump(1, p0, 1, p0, descr=targettoken2) - ''' - self.interpret(ops2, [s1]) - frame = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, self.deadframe) - assert len(frame.jf_gcmap) == 3 - assert [frame.jf_gcmap[i] for i in range(3)] == [1, 3, 4] + assert len(frame.jf_gcmap) == 1 + # the gcmap should contain three things, p0, p1 and p3 + # p3 stays in a register + # while p0 and p1 are on the frame + assert frame.jf_gcmap[0] == (1 << 11) | (1 << 12) | (1 << 31) + assert frame.jf_frame[11] + assert frame.jf_frame[12] + assert frame.jf_frame[31] def test_rewrite_constptr(self): ops = ''' @@ -228,10 +204,8 @@ def test_malloc_slowpath(self): def check(frame): - assert len(frame.jf_frame_info.jfi_gcmap) == 2 # 2 pointers - assert frame.jf_frame_info.jfi_gcmap[0] == 1 - assert frame.jf_frame_info.jfi_gcmap[1] == 2 - assert frame.jf_gcpattern == 0x2 + assert len(frame.jf_gcmap) == 1 + assert frame.jf_gcmap[0] == (2 | (1<<29) | (1 << 30)) self.cpu = self.getcpu(check) ops = ''' @@ -257,8 +231,9 @@ def test_save_regs_around_malloc(self): def check(frame): - x = frame.jf_gcpattern - assert bin(x) == '0b1111111011111' + x = frame.jf_gcmap + assert len(x) == 1 + assert bin(x[0]) == '0b1111100000000000000001111111011111' # all but two self.cpu = self.getcpu(check) diff --git a/rpython/jit/backend/x86/test/test_runner.py b/rpython/jit/backend/x86/test/test_runner.py --- a/rpython/jit/backend/x86/test/test_runner.py +++ b/rpython/jit/backend/x86/test/test_runner.py @@ -30,8 +30,7 @@ # for the individual tests see # ====> ../../test/runner_test.py - add_loop_instructions = ['mov', 'mov', 'mov', 'mov', - 'mov', 'add', 'test', 'je', 'jmp'] + add_loop_instructions = ['mov', 'add', 'test', 'je', 'jmp'] bridge_loop_instructions = ['cmp', 'jge', 'mov', 'call', 'mov', 'jmp'] diff --git a/rpython/rtyper/lltypesystem/llmemory.py b/rpython/rtyper/lltypesystem/llmemory.py --- a/rpython/rtyper/lltypesystem/llmemory.py +++ b/rpython/rtyper/lltypesystem/llmemory.py @@ -618,6 +618,9 @@ class _signed_fakeaccessor(_fakeaccessor): TYPE = lltype.Signed +class _unsigned_fakeaccessor(_fakeaccessor): + TYPE = lltype.Unsigned + class _float_fakeaccessor(_fakeaccessor): TYPE = lltype.Float @@ -654,6 +657,7 @@ } fakeaddress.signed = property(_signed_fakeaccessor) +fakeaddress.unsigned = property(_unsigned_fakeaccessor) fakeaddress.float = property(_float_fakeaccessor) fakeaddress.char = property(_char_fakeaccessor) fakeaddress.address = property(_address_fakeaccessor) From noreply at buildbot.pypy.org Mon Jan 21 18:26:40 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 18:26:40 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: solid refactoring - gcmap is now per call Message-ID: <20130121172640.A9DC51C106A@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60300:9071ec11ee34 Date: 2013-01-21 19:26 +0200 http://bitbucket.org/pypy/pypy/changeset/9071ec11ee34/ Log: solid refactoring - gcmap is now per call diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -197,11 +197,13 @@ self._push_all_regs_to_frame(mc, [], self.cpu.supports_floats) assert not IS_X86_32 # push first arg + mc.POP_r(ecx.value) + gcmap_ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') + mc.MOV_br(gcmap_ofs, ecx.value) mc.LEA_rb(edi.value, -base_ofs) - mc.SUB_ri(esp.value, WORD) # we need that cause we're inside a call mc.CALL(imm(self.cpu.realloc_frame)) - mc.ADD_ri(esp.value, WORD) mc.LEA_rm(ebp.value, (eax.value, base_ofs)) + mc.MOV_bi(gcmap_ofs, 0) self._pop_all_regs_from_frame(mc, [], self.cpu.supports_floats) mc.RET() self._stack_check_failure = mc.materialize(self.cpu.asmmemmgr, []) @@ -581,7 +583,8 @@ operations, self.current_clt.allgcrefs, self.current_clt.frame_info) - stack_check_patch_ofs = self._check_frame_depth(self.mc) + stack_check_patch_ofs = self._check_frame_depth(self.mc, + regalloc.get_gcmap()) frame_depth = self._assemble(regalloc, inputargs, operations) codeendpos = self.mc.get_relative_pos() self.write_pending_failure_recoveries() @@ -662,7 +665,7 @@ mc.writeimm32(self.error_trampoline_64 - pos_after_jz) mc.copy_to_raw_memory(rawstart + pos_after_jz - 4) - def _check_frame_depth(self, mc): + def _check_frame_depth(self, mc, gcmap): """ check if the frame is of enough depth to follow this bridge. Otherwise reallocate the frame in a helper. There are other potential solutions @@ -676,6 +679,7 @@ assert not IS_X86_32 mc.J_il8(rx86.Conditions['GE'], 0) jg_location = mc.get_relative_pos() + self.push_gcmap(mc, gcmap, push=True) mc.CALL(imm(self._stack_check_failure)) # patch the JG above offset = mc.get_relative_pos() - jg_location @@ -1218,7 +1222,9 @@ dst_locs.append(r10) x = r10 remap_frame_layout(self, src_locs, dst_locs, X86_64_SCRATCH_REG) + self.push_gcmap(self.mc, self._regalloc.get_gcmap(), store=True) self.mc.CALL(x) + self.pop_gcmap(self.mc) if align: self.mc.ADD_ri(esp.value, align * WORD) self._reload_frame_if_necessary(self.mc) @@ -1867,13 +1873,28 @@ # mc.JMP(imm(target)) #else: mc.PUSH(imm(fail_descr)) - gcmapref = lltype.cast_opaque_ptr(llmemory.GCREF, guardtok.gcmap) + self.push_gcmap(mc, guardtok.gcmap, push=True) + mc.JMP(imm(target)) + return startpos + + def push_gcmap(self, mc, gcmap, push=False, mov=False, store=False): + gcmapref = lltype.cast_opaque_ptr(llmemory.GCREF, gcmap) # keep the ref alive self.current_clt.allgcrefs.append(gcmapref) rgc._make_sure_does_not_move(gcmapref) - mc.PUSH(imm(rffi.cast(lltype.Signed, gcmapref))) - mc.JMP(imm(target)) - return startpos + if push: + mc.PUSH(imm(rffi.cast(lltype.Signed, gcmapref))) + elif mov: + mc.MOV(RawEspLoc(0, REF), + imm(rffi.cast(lltype.Signed, gcmapref))) + else: + assert store + ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') + mc.MOV(raw_stack(ofs), imm(rffi.cast(lltype.Signed, gcmapref))) + + def pop_gcmap(self, mc): + ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') + mc.MOV_bi(ofs, 0) def rebuild_faillocs_from_descr(self, descr, inputargs): locs = [] @@ -2402,12 +2423,8 @@ self.mc.CMP(edi, heap(nursery_top_adr)) self.mc.J_il8(rx86.Conditions['NA'], 0) # patched later jmp_adr = self.mc.get_relative_pos() - # this is a 32bit gcpattern that describes where are GC roots - # in registers (which will be saved while doing slowpath) - # store it on top of the stack (we might not have a register free) - gcmapref = lltype.cast_opaque_ptr(llmemory.GCREF, gcmap) - rgc._make_sure_does_not_move(gcmapref) - self.mc.MOV(RawEspLoc(0, REF), imm(rffi.cast(lltype.Signed, gcmapref))) + # save the gcmap + self.push_gcmap(self.mc, gcmap, mov=True) self.mc.CALL(imm(self.malloc_slowpath)) offset = self.mc.get_relative_pos() - jmp_adr assert 0 < offset <= 127 diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -896,14 +896,14 @@ self.rm.force_allocate_reg(tmp_box, selected_reg=edi) self.rm.possibly_free_var(tmp_box) # - gcmap = self._compute_gcmap() + gcmap = self.get_gcmap() gc_ll_descr = self.assembler.cpu.gc_ll_descr self.assembler.malloc_cond( gc_ll_descr.get_nursery_free_addr(), gc_ll_descr.get_nursery_top_addr(), size, gcmap) - def _compute_gcmap(self): + def get_gcmap(self): frame_depth = self.fm.get_frame_depth() size = frame_depth + JITFRAME_FIXED_SIZE gcmap = lltype.malloc(GCMAP, size // WORD // 8 + 1, From noreply at buildbot.pypy.org Mon Jan 21 18:27:43 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 18:27:43 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fix and skip for now Message-ID: <20130121172743.11C861C106A@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60301:fab1725937a5 Date: 2013-01-21 19:27 +0200 http://bitbucket.org/pypy/pypy/changeset/fab1725937a5/ Log: fix and skip for now diff --git a/rpython/jit/backend/test/calling_convention_test.py b/rpython/jit/backend/test/calling_convention_test.py --- a/rpython/jit/backend/test/calling_convention_test.py +++ b/rpython/jit/backend/test/calling_convention_test.py @@ -315,6 +315,7 @@ def test_call_with_singlefloats(self): + py.test.skip("skip for now") cpu = self.cpu if not cpu.supports_floats or not cpu.supports_singlefloats: py.test.skip('requires floats and singlefloats') diff --git a/rpython/jit/backend/x86/test/test_runner.py b/rpython/jit/backend/x86/test/test_runner.py --- a/rpython/jit/backend/x86/test/test_runner.py +++ b/rpython/jit/backend/x86/test/test_runner.py @@ -31,7 +31,7 @@ # ====> ../../test/runner_test.py add_loop_instructions = ['mov', 'add', 'test', 'je', 'jmp'] - bridge_loop_instructions = ['cmp', 'jge', 'mov', 'call', + bridge_loop_instructions = ['cmp', 'jge', 'mov', 'push', 'mov', 'call', 'mov', 'jmp'] def setup_method(self, meth): From noreply at buildbot.pypy.org Mon Jan 21 18:53:47 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 18:53:47 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: oops Message-ID: <20130121175347.83E9A1C00EC@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60302:bd2e2eb96f1f Date: 2013-01-21 19:53 +0200 http://bitbucket.org/pypy/pypy/changeset/bd2e2eb96f1f/ Log: oops diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py --- a/rpython/jit/backend/llsupport/regalloc.py +++ b/rpython/jit/backend/llsupport/regalloc.py @@ -18,6 +18,9 @@ self.val = val self.next = next + def __repr__(self): + return '' % (self.val, next) + class LinkedList(object): def __init__(self, fm, lst=None): # assume the list is sorted @@ -49,6 +52,8 @@ while node and node.val < key: prev_node = node node = node.next + import pdb + pdb.set_trace() prev_node.next = Node(key, node) @specialize.arg(1) @@ -171,12 +176,20 @@ def bind(self, box, loc): pos = self.get_loc_index(loc) size = self.frame_size(box.type) - if self.current_frame_depth < pos: - for i in range(self.current_frame_depth, pos): - self.freelist.append(1, self.frame_pos(i, INT)) - self.current_frame_depth = pos + size + self.current_frame_depth = max(pos + size, self.current_frame_depth) self.bindings[box] = loc + def finish_binding(self): + all = [0] * self.get_frame_depth() + for b, loc in self.bindings.iteritems(): + size = self.frame_size(b) + pos = self.get_loc_index(loc) + for i in range(pos, pos + size): + all[i] = 1 + for elem in range(len(all)): + if not all[elem]: + self.freelist._append(elem) + def mark_as_free(self, box): try: loc = self.bindings[box] diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -504,7 +504,6 @@ # duplication with assemble_bridge(). Also, we should think # about not storing on 'self' attributes that will live only # for the duration of compiling one loop or a one bridge. - clt = CompiledLoopToken(self.cpu, looptoken.number) clt.frame_info = lltype.malloc(jitframe.JITFRAMEINFO) clt.allgcrefs = [] diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -315,6 +315,7 @@ if reg not in used: self.xrm.free_regs.append(reg) self.possibly_free_vars(list(inputargs)) + self.fm.finish_binding() self.rm._check_invariants() self.xrm._check_invariants() From noreply at buildbot.pypy.org Mon Jan 21 18:54:58 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 18:54:58 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: forgot Message-ID: <20130121175458.626AF1C00EC@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60303:ace0631181b0 Date: 2013-01-21 19:54 +0200 http://bitbucket.org/pypy/pypy/changeset/ace0631181b0/ Log: forgot diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py --- a/rpython/jit/backend/llsupport/regalloc.py +++ b/rpython/jit/backend/llsupport/regalloc.py @@ -52,8 +52,6 @@ while node and node.val < key: prev_node = node node = node.next - import pdb - pdb.set_trace() prev_node.next = Node(key, node) @specialize.arg(1) From noreply at buildbot.pypy.org Mon Jan 21 19:05:58 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 19:05:58 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: not a great idea, clean up frame manager Message-ID: <20130121180558.3F3441C106A@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60304:43a50c020ed3 Date: 2013-01-21 20:05 +0200 http://bitbucket.org/pypy/pypy/changeset/43a50c020ed3/ Log: not a great idea, clean up frame manager diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py --- a/rpython/jit/backend/llsupport/regalloc.py +++ b/rpython/jit/backend/llsupport/regalloc.py @@ -184,8 +184,10 @@ pos = self.get_loc_index(loc) for i in range(pos, pos + size): all[i] = 1 + self.freelist = LinkedList(self) # we don't care for elem in range(len(all)): if not all[elem]: + print self.freelist, elem self.freelist._append(elem) def mark_as_free(self, box): From noreply at buildbot.pypy.org Mon Jan 21 19:06:59 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 19:06:59 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: kill one more print Message-ID: <20130121180659.27FBD1C106A@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60305:19b339a29db6 Date: 2013-01-21 20:06 +0200 http://bitbucket.org/pypy/pypy/changeset/19b339a29db6/ Log: kill one more print diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py --- a/rpython/jit/backend/llsupport/regalloc.py +++ b/rpython/jit/backend/llsupport/regalloc.py @@ -187,7 +187,6 @@ self.freelist = LinkedList(self) # we don't care for elem in range(len(all)): if not all[elem]: - print self.freelist, elem self.freelist._append(elem) def mark_as_free(self, box): From noreply at buildbot.pypy.org Mon Jan 21 19:41:27 2013 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 21 Jan 2013 19:41:27 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: hack to make ztranslation pass Message-ID: <20130121184127.65F231C00EC@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: missing-ndarray-attributes Changeset: r60306:b345f036991d Date: 2013-01-21 20:16 +0200 http://bitbucket.org/pypy/pypy/changeset/b345f036991d/ Log: hack to make ztranslation pass diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -1015,6 +1015,14 @@ BoxType = interp_boxes.W_Float32Box format_code = "f" + def read_bool(self, arr, i, offset): + # it's not clear to me why this is needed + # but a hint might be that calling for_computation(v) + # causes translation to fail, and the assert is necessary + v = self._read(arr.storage, i, offset) + assert isinstance(v, float) + return bool(v) + class Float64(BaseType, Float): _attrs_ = () From noreply at buildbot.pypy.org Mon Jan 21 19:41:28 2013 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 21 Jan 2013 19:41:28 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: test_zjit thinks cumltative is constant, hack around it Message-ID: <20130121184128.B5BE91C00EC@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: missing-ndarray-attributes Changeset: r60307:92d3270bf0ab Date: 2013-01-21 20:40 +0200 http://bitbucket.org/pypy/pypy/changeset/92d3270bf0ab/ Log: test_zjit thinks cumltative is constant, hack around it diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py --- a/pypy/module/micronumpy/loop.py +++ b/pypy/module/micronumpy/loop.py @@ -173,32 +173,44 @@ y_iter.next() return out -axis_reduce__driver = jit.JitDriver(name='numpy_axis_reduce', - greens=['shapelen', 'cumultative', +axis_reduce1__driver = jit.JitDriver(name='numpy_axis_reduce', + greens=['shapelen', 'func', 'dtype', 'identity'], reds=['axis', 'arr', 'out', 'shape', 'out_iter', 'arr_iter', 'temp_iter']) +axis_reduce2__driver = jit.JitDriver(name='numpy_axis_reduce', + greens=['shapelen', + 'func', 'dtype', + 'identity'], + reds=['axis', 'arr', 'out', 'shape', + 'out_iter', 'arr_iter', + ]) def do_axis_reduce(shape, func, arr, dtype, axis, out, identity, cumultative, temp): - out_iter = out.create_axis_iter(arr.get_shape(), axis, cumultative) if cumultative: - temp_iter = temp.create_axis_iter(arr.get_shape(), axis, False) + return do_cum_axis_reduce(shape, func, arr, dtype, axis, out, + identity, temp) else: - temp_iter = out_iter # hack + return do_nocum_axis_reduce(shape, func, arr, dtype, axis, out, + identity, temp) + +def do_cum_axis_reduce(shape, func, arr, dtype, axis, out, identity, + temp): + out_iter = out.create_axis_iter(arr.get_shape(), axis, True) + temp_iter = temp.create_axis_iter(arr.get_shape(), axis, False) arr_iter = arr.create_iter() if identity is not None: identity = identity.convert_to(dtype) shapelen = len(shape) while not out_iter.done(): - axis_reduce__driver.jit_merge_point(shapelen=shapelen, func=func, + axis_reduce1__driver.jit_merge_point(shapelen=shapelen, func=func, dtype=dtype, identity=identity, axis=axis, arr=arr, out=out, shape=shape, out_iter=out_iter, arr_iter=arr_iter, - cumultative=cumultative, temp_iter=temp_iter) w_val = arr_iter.getitem().convert_to(dtype) if out_iter.first_line: @@ -208,9 +220,34 @@ cur = temp_iter.getitem() w_val = func(dtype, cur, w_val) out_iter.setitem(w_val) - if cumultative: - temp_iter.setitem(w_val) - temp_iter.next() + temp_iter.setitem(w_val) + temp_iter.next() + arr_iter.next() + out_iter.next() + return out + +def do_nocum_axis_reduce(shape, func, arr, dtype, axis, out, identity, + temp): + out_iter = out.create_axis_iter(arr.get_shape(), axis, False) + arr_iter = arr.create_iter() + if identity is not None: + identity = identity.convert_to(dtype) + shapelen = len(shape) + while not out_iter.done(): + axis_reduce2__driver.jit_merge_point(shapelen=shapelen, func=func, + dtype=dtype, identity=identity, + axis=axis, arr=arr, out=out, + shape=shape, out_iter=out_iter, + arr_iter=arr_iter, + ) + w_val = arr_iter.getitem().convert_to(dtype) + if out_iter.first_line: + if identity is not None: + w_val = func(dtype, identity, w_val) + else: + cur = out_iter.getitem() + w_val = func(dtype, cur, w_val) + out_iter.setitem(w_val) arr_iter.next() out_iter.next() return out From noreply at buildbot.pypy.org Mon Jan 21 20:07:34 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 20:07:34 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: try to kill some mess of violently reusing registers. we can no longer do this due to reusing of frame values Message-ID: <20130121190734.DC1D31C1102@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60308:0b5a92341348 Date: 2013-01-21 21:07 +0200 http://bitbucket.org/pypy/pypy/changeset/0b5a92341348/ Log: try to kill some mess of violently reusing registers. we can no longer do this due to reusing of frame values diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py --- a/rpython/jit/backend/llsupport/regalloc.py +++ b/rpython/jit/backend/llsupport/regalloc.py @@ -169,6 +169,8 @@ # self.bindings[box] = newloc + if not we_are_translated(): + self._check_invariants() return newloc def bind(self, box, loc): @@ -188,6 +190,8 @@ for elem in range(len(all)): if not all[elem]: self.freelist._append(elem) + if not we_are_translated(): + self._check_invariants() def mark_as_free(self, box): try: @@ -197,6 +201,23 @@ del self.bindings[box] size = self.frame_size(box.type) self.freelist.append(size, loc) + if not we_are_translated(): + self._check_invariants() + + def _check_invariants(self): + all = [0] * self.get_frame_depth() + for b, loc in self.bindings.iteritems(): + size = self.frame_size(b) + pos = self.get_loc_index(loc) + for i in range(pos, pos + size): + assert not all[i] + all[i] = 1 + node = self.freelist.master_node + while node is not None: + assert not all[node.val] + all[node.val] = 1 + node = node.next + assert all == [1] * self.get_frame_depth() def try_to_reuse_location(self, box, loc): xxx diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -1865,6 +1865,7 @@ v = gpr_reg_mgr_cls.all_reg_indexes[loc.value] positions[i] = v * WORD # write down the positions of locs + print guardtok.fail_locs guardtok.faildescr.rd_locs = positions #if WORD == 4: # mc.PUSH(imm(fail_descr)) diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -160,6 +160,9 @@ for _i, _reg in enumerate(gpr_reg_mgr_cls.all_regs): gpr_reg_mgr_cls.all_reg_indexes[_reg.value] = _i +if __name__ == '__main__': + print gpr_reg_mgr_cls.all_reg_indexes + class RegAlloc(object): def __init__(self, assembler, translate_support_code=False): @@ -483,7 +486,6 @@ else: resloc = None self.perform_guard(op, [loc, loc1], resloc) - self.rm.possibly_free_var(box) consider_guard_no_overflow = consider_guard_no_exception consider_guard_overflow = consider_guard_no_exception @@ -506,7 +508,6 @@ argloc = self.loc(op.getarg(1)) args = op.getarglist() loc = self.rm.force_result_in_reg(op.result, x, args) - self.rm.possibly_free_var(op.getarg(1)) return loc, argloc def _consider_binop(self, op): @@ -515,7 +516,6 @@ def _consider_lea(self, op, loc): argloc = self.loc(op.getarg(1)) - self.rm.possibly_free_var(op.getarg(0)) resloc = self.force_allocate_reg(op.result) self.Perform(op, [loc, argloc], resloc) @@ -595,13 +595,12 @@ vx = op.getarg(0) vy = op.getarg(1) arglocs = [self.loc(vx), self.loc(vy)] + args = op.getarglist() if (vx in self.rm.reg_bindings or vy in self.rm.reg_bindings or isinstance(vx, Const) or isinstance(vy, Const)): pass else: arglocs[0] = self.rm.make_sure_var_in_reg(vx) - args = op.getarglist() - self.rm.possibly_free_vars(args) if guard_op is None: loc = self.rm.force_allocate_reg(op.result, args, need_lower_byte=True) @@ -643,7 +642,6 @@ arglocs[1] = self.xrm.make_sure_var_in_reg(vy) else: arglocs[0] = self.xrm.make_sure_var_in_reg(vx) - self.xrm.possibly_free_vars_for_op(op) if guard_op is None: res = self.rm.force_allocate_reg(op.result, need_lower_byte=True) self.Perform(op, arglocs, res) @@ -677,7 +675,6 @@ def consider_cast_float_to_singlefloat(self, op): loc0 = self.xrm.make_sure_var_in_reg(op.getarg(0)) loc1 = self.rm.force_allocate_reg(op.result) - self.xrm.possibly_free_var(op.getarg(0)) tmpxvar = TempBox() loctmp = self.xrm.force_allocate_reg(tmpxvar) # may be equal to loc0 self.xrm.possibly_free_var(tmpxvar) @@ -742,7 +739,6 @@ loc1 = self.xrm.make_sure_var_in_reg(box) loc0 = self.rm.force_allocate_reg(op.result) self.PerformLLong(op, [loc1], loc0) - self.xrm.possibly_free_var(box) return True def _consider_llong_to_int(self, op): @@ -958,8 +954,6 @@ if not isinstance(index_loc, ImmedLoc): # ...that is, except in a corner case where 'index_loc' would be # in the same register as 'value_loc'... - if index_loc is not value_loc: - self.rm.possibly_free_var(box_index) tempvar = TempBox() temp_loc = self.rm.force_allocate_reg(tempvar, [box_base, box_value]) @@ -1007,7 +1001,6 @@ size_loc = imm(size) args = op.getarglist() base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args) - self.rm.possibly_free_vars(args) result_loc = self.force_allocate_reg(op.result) if sign: sign_loc = imm1 @@ -1024,7 +1017,6 @@ args = op.getarglist() base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args) ofs_loc = self.rm.make_sure_var_in_reg(op.getarg(1), args) - self.rm.possibly_free_vars_for_op(op) result_loc = self.force_allocate_reg(op.result) if sign: sign_loc = imm1 @@ -1051,7 +1043,6 @@ # 'base' and 'index' are put in two registers (or one if 'index' # is an immediate). 'result' can be in the same register as # 'index' but must be in a different register than 'base'. - self.rm.possibly_free_var(op.getarg(1)) result_loc = self.force_allocate_reg(op.result, [op.getarg(0)]) assert isinstance(result_loc, RegLoc) # two cases: 1) if result_loc is a normal register, use it as temp_loc @@ -1064,14 +1055,12 @@ tempvar = TempBox() temp_loc = self.rm.force_allocate_reg(tempvar, [op.getarg(0)]) self.rm.possibly_free_var(tempvar) - self.rm.possibly_free_var(op.getarg(0)) self.Perform(op, [base_loc, ofs, itemsize, fieldsize, index_loc, temp_loc, sign_loc], result_loc) def consider_int_is_true(self, op, guard_op): # doesn't need arg to be in a register argloc = self.loc(op.getarg(0)) - self.rm.possibly_free_var(op.getarg(0)) if guard_op is not None: self.perform_with_guard(op, guard_op, [argloc], None) else: @@ -1082,7 +1071,6 @@ def consider_same_as(self, op): argloc = self.loc(op.getarg(0)) - self.possibly_free_var(op.getarg(0)) resloc = self.force_allocate_reg(op.result) self.Perform(op, [argloc], resloc) consider_cast_ptr_to_int = consider_same_as @@ -1091,13 +1079,11 @@ def consider_int_force_ge_zero(self, op): argloc = self.make_sure_var_in_reg(op.getarg(0)) resloc = self.force_allocate_reg(op.result, [op.getarg(0)]) - self.possibly_free_var(op.getarg(0)) self.Perform(op, [argloc], resloc) def consider_strlen(self, op): args = op.getarglist() base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args) - self.rm.possibly_free_vars_for_op(op) result_loc = self.rm.force_allocate_reg(op.result) self.Perform(op, [base_loc], result_loc) @@ -1109,7 +1095,6 @@ ofs = arraydescr.lendescr.offset args = op.getarglist() base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args) - self.rm.possibly_free_vars_for_op(op) result_loc = self.rm.force_allocate_reg(op.result) self.Perform(op, [base_loc, imm(ofs)], result_loc) @@ -1117,7 +1102,6 @@ args = op.getarglist() base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args) ofs_loc = self.rm.make_sure_var_in_reg(op.getarg(1), args) - self.rm.possibly_free_vars_for_op(op) result_loc = self.rm.force_allocate_reg(op.result) self.Perform(op, [base_loc, ofs_loc], result_loc) @@ -1135,9 +1119,6 @@ base_loc = self.rm.make_sure_var_in_reg(args[0], args) ofs_loc = self.rm.make_sure_var_in_reg(args[2], args) assert args[0] is not args[1] # forbidden case of aliasing - self.rm.possibly_free_var(args[0]) - if args[3] is not args[2] is not args[4]: # MESS MESS MESS: don't free - self.rm.possibly_free_var(args[2]) # it if ==args[3] or args[4] srcaddr_box = TempBox() forbidden_vars = [args[1], args[3], args[4], srcaddr_box] srcaddr_loc = self.rm.force_allocate_reg(srcaddr_box, forbidden_vars) @@ -1146,9 +1127,6 @@ # compute the destination address base_loc = self.rm.make_sure_var_in_reg(args[1], forbidden_vars) ofs_loc = self.rm.make_sure_var_in_reg(args[3], forbidden_vars) - self.rm.possibly_free_var(args[1]) - if args[3] is not args[4]: # more of the MESS described above - self.rm.possibly_free_var(args[3]) forbidden_vars = [args[4], srcaddr_box] dstaddr_box = TempBox() dstaddr_loc = self.rm.force_allocate_reg(dstaddr_box, forbidden_vars) @@ -1158,7 +1136,6 @@ length_box = args[4] length_loc = self.loc(length_box) if is_unicode: - self.rm.possibly_free_var(length_box) forbidden_vars = [srcaddr_box, dstaddr_box] bytes_box = TempBox() bytes_loc = self.rm.force_allocate_reg(bytes_box, forbidden_vars) diff --git a/rpython/jit/backend/x86/test/test_gc_integration.py b/rpython/jit/backend/x86/test/test_gc_integration.py --- a/rpython/jit/backend/x86/test/test_gc_integration.py +++ b/rpython/jit/backend/x86/test/test_gc_integration.py @@ -344,9 +344,9 @@ frames = [] def check(i): + import pdb + pdb.set_trace() assert self.cpu.gc_ll_descr.gcrootmap.stack[0] == i - ofs - frame = rffi.cast(jitframe.JITFRAMEPTR, i - ofs) - frames.append(frame) assert len(frame.jf_frame) == JITFRAME_FIXED_SIZE # we "collect" new_frame = frame.copy() From noreply at buildbot.pypy.org Mon Jan 21 20:09:10 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 20:09:10 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: missed one Message-ID: <20130121190910.CBAEE1C1102@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60309:133bca1191f7 Date: 2013-01-21 21:08 +0200 http://bitbucket.org/pypy/pypy/changeset/133bca1191f7/ Log: missed one diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -486,6 +486,7 @@ else: resloc = None self.perform_guard(op, [loc, loc1], resloc) + self.possibly_free_var(box) consider_guard_no_overflow = consider_guard_no_exception consider_guard_overflow = consider_guard_no_exception From noreply at buildbot.pypy.org Mon Jan 21 20:09:40 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 20:09:40 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: grumble Message-ID: <20130121190940.CE9411C1102@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60310:6527de6db1dc Date: 2013-01-21 21:09 +0200 http://bitbucket.org/pypy/pypy/changeset/6527de6db1dc/ Log: grumble diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -486,7 +486,7 @@ else: resloc = None self.perform_guard(op, [loc, loc1], resloc) - self.possibly_free_var(box) + self.rm.possibly_free_var(box) consider_guard_no_overflow = consider_guard_no_exception consider_guard_overflow = consider_guard_no_exception From noreply at buildbot.pypy.org Mon Jan 21 20:12:24 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 20:12:24 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: too strong Message-ID: <20130121191224.F1CDE1C1102@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60311:ae35a5b97af4 Date: 2013-01-21 21:12 +0200 http://bitbucket.org/pypy/pypy/changeset/ae35a5b97af4/ Log: too strong diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py --- a/rpython/jit/backend/llsupport/regalloc.py +++ b/rpython/jit/backend/llsupport/regalloc.py @@ -217,7 +217,6 @@ assert not all[node.val] all[node.val] = 1 node = node.next - assert all == [1] * self.get_frame_depth() def try_to_reuse_location(self, box, loc): xxx From noreply at buildbot.pypy.org Mon Jan 21 20:17:05 2013 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 21 Jan 2013 20:17:05 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: merge default into branch Message-ID: <20130121191705.1F6AB1C1102@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: missing-ndarray-attributes Changeset: r60312:418caa1027eb Date: 2013-01-21 21:15 +0200 http://bitbucket.org/pypy/pypy/changeset/418caa1027eb/ Log: merge default into branch diff too long, truncating to 2000 out of 728411 lines diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -85,3 +85,4 @@ ^compiled ^.git/ ^release/ +^rpython/_cache$ diff --git a/dotviewer/test/test_translator.py b/dotviewer/test/test_translator.py --- a/dotviewer/test/test_translator.py +++ b/dotviewer/test/test_translator.py @@ -21,7 +21,7 @@ def test_annotated(): - from pypy.translator.interactive import Translation + from rpython.translator.interactive import Translation t = Translation(is_prime) t.annotate([int]) t.viewcg() diff --git a/lib_pypy/ctypes_config_cache/autopath.py b/lib_pypy/ctypes_config_cache/autopath.py deleted file mode 100644 --- a/lib_pypy/ctypes_config_cache/autopath.py +++ /dev/null @@ -1,131 +0,0 @@ -""" -self cloning, automatic path configuration - -copy this into any subdirectory of pypy from which scripts need -to be run, typically all of the test subdirs. -The idea is that any such script simply issues - - import autopath - -and this will make sure that the parent directory containing "pypy" -is in sys.path. - -If you modify the master "autopath.py" version (in pypy/tool/autopath.py) -you can directly run it which will copy itself on all autopath.py files -it finds under the pypy root directory. - -This module always provides these attributes: - - pypydir pypy root directory path - this_dir directory where this autopath.py resides - -""" - -def __dirinfo(part): - """ return (partdir, this_dir) and insert parent of partdir - into sys.path. If the parent directories don't have the part - an EnvironmentError is raised.""" - - import sys, os - try: - head = this_dir = os.path.realpath(os.path.dirname(__file__)) - except NameError: - head = this_dir = os.path.realpath(os.path.dirname(sys.argv[0])) - - error = None - while head: - partdir = head - head, tail = os.path.split(head) - if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') - if not os.path.exists(checkfile): - error = "Cannot find %r" % (os.path.normpath(checkfile),) - break - else: - error = "Cannot find the parent directory %r of the path %r" % ( - partdir, this_dir) - if not error: - # check for bogus end-of-line style (e.g. files checked out on - # Windows and moved to Unix) - f = open(__file__.replace('.pyc', '.py'), 'r') - data = f.read() - f.close() - if data.endswith('\r\n') or data.endswith('\r'): - error = ("Bad end-of-line style in the .py files. Typically " - "caused by a zip file or a checkout done on Windows and " - "moved to Unix or vice-versa.") - if error: - raise EnvironmentError("Invalid source tree - bogus checkout! " + - error) - - pypy_root = os.path.join(head, '') - try: - sys.path.remove(head) - except ValueError: - pass - sys.path.insert(0, head) - - munged = {} - for name, mod in sys.modules.items(): - if '.' in name: - continue - fn = getattr(mod, '__file__', None) - if not isinstance(fn, str): - continue - newname = os.path.splitext(os.path.basename(fn))[0] - if not newname.startswith(part + '.'): - continue - path = os.path.join(os.path.dirname(os.path.realpath(fn)), '') - if path.startswith(pypy_root) and newname != part: - modpaths = os.path.normpath(path[len(pypy_root):]).split(os.sep) - if newname != '__init__': - modpaths.append(newname) - modpath = '.'.join(modpaths) - if modpath not in sys.modules: - munged[modpath] = mod - - for name, mod in munged.iteritems(): - if name not in sys.modules: - sys.modules[name] = mod - if '.' in name: - prename = name[:name.rfind('.')] - postname = name[len(prename)+1:] - if prename not in sys.modules: - __import__(prename) - if not hasattr(sys.modules[prename], postname): - setattr(sys.modules[prename], postname, mod) - - return partdir, this_dir - -def __clone(): - """ clone master version of autopath.py into all subdirs """ - from os.path import join, walk - if not this_dir.endswith(join('pypy','tool')): - raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) - - - def sync_walker(arg, dirname, fnames): - if _myname in fnames: - fn = join(dirname, _myname) - f = open(fn, 'rwb+') - try: - if f.read() == arg: - print "checkok", fn - else: - print "syncing", fn - f = open(fn, 'w') - f.write(arg) - finally: - f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) - -_myname = 'autopath.py' - -# set guaranteed attributes - -pypydir, this_dir = __dirinfo('pypy') - -if __name__ == '__main__': - __clone() diff --git a/lib_pypy/ctypes_config_cache/dumpcache.py b/lib_pypy/ctypes_config_cache/dumpcache.py --- a/lib_pypy/ctypes_config_cache/dumpcache.py +++ b/lib_pypy/ctypes_config_cache/dumpcache.py @@ -1,6 +1,6 @@ import os from ctypes_configure import dumpcache -from pypy.jit.backend import detect_cpu +from rpython.jit.backend import detect_cpu def dumpcache2(basename, config): model = detect_cpu.autodetect_main_model_and_size() @@ -14,7 +14,7 @@ try: from __pypy__ import cpumodel except ImportError: - from pypy.jit.backend import detect_cpu + from rpython.jit.backend import detect_cpu cpumodel = detect_cpu.autodetect_main_model_and_size() # XXX relative import, should be removed together with # XXX the relative imports done e.g. by lib_pypy/pypy_test/test_hashlib diff --git a/lib_pypy/ctypes_config_cache/rebuild.py b/lib_pypy/ctypes_config_cache/rebuild.py --- a/lib_pypy/ctypes_config_cache/rebuild.py +++ b/lib_pypy/ctypes_config_cache/rebuild.py @@ -1,21 +1,15 @@ #! /usr/bin/env python # Run this script to rebuild all caches from the *.ctc.py files. -# hack: we cannot directly import autopath, as we are outside the pypy -# package. However, we pretend to be inside pypy/tool and manually run it, to -# get the correct path -import os.path -this_dir = os.path.dirname(__file__) -autopath_py = os.path.join(this_dir, '../../pypy/tool/autopath.py') -autopath_py = os.path.abspath(autopath_py) -execfile(autopath_py, dict(__name__='autopath', __file__=autopath_py)) +import os, sys -import os, sys +sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..'))) + import py _dirpath = os.path.dirname(__file__) or os.curdir -from pypy.tool.ansi_print import ansi_log +from rpython.tool.ansi_print import ansi_log log = py.log.Producer("ctypes_config_cache") py.log.setconsumer("ctypes_config_cache", ansi_log) @@ -31,7 +25,7 @@ sys.path[:] = path def try_rebuild(): - from pypy.jit.backend import detect_cpu + from rpython.jit.backend import detect_cpu model = detect_cpu.autodetect_main_model_and_size() # remove the files '_*_model_.py' left = {} diff --git a/lib_pypy/ctypes_config_cache/test/test_cache.py b/lib_pypy/ctypes_config_cache/test/test_cache.py --- a/lib_pypy/ctypes_config_cache/test/test_cache.py +++ b/lib_pypy/ctypes_config_cache/test/test_cache.py @@ -1,6 +1,6 @@ import py import sys, os -from pypy.tool.udir import udir +from rpython.tool.udir import udir dirpath = py.path.local(__file__).dirpath().dirpath() diff --git a/lib_pypy/pypy_test/test_dbm_extra.py b/lib_pypy/pypy_test/test_dbm_extra.py --- a/lib_pypy/pypy_test/test_dbm_extra.py +++ b/lib_pypy/pypy_test/test_dbm_extra.py @@ -1,6 +1,6 @@ from __future__ import absolute_import import py -from pypy.tool.udir import udir +from rpython.tool.udir import udir try: from lib_pypy import dbm except ImportError, e: diff --git a/lib_pypy/pypy_test/test_marshal_extra.py b/lib_pypy/pypy_test/test_marshal_extra.py --- a/lib_pypy/pypy_test/test_marshal_extra.py +++ b/lib_pypy/pypy_test/test_marshal_extra.py @@ -4,7 +4,7 @@ import marshal as cpy_marshal from lib_pypy import _marshal as marshal -from pypy.tool.udir import udir +from rpython.tool.udir import udir hello = "he" hello += "llo" diff --git a/pypy/annotation/annrpython.py b/pypy/annotation/annrpython.py deleted file mode 100644 --- a/pypy/annotation/annrpython.py +++ /dev/null @@ -1,676 +0,0 @@ -from __future__ import absolute_import - -import types -from pypy.tool.ansi_print import ansi_log -from pypy.tool.pairtype import pair -from pypy.tool.error import (format_blocked_annotation_error, - AnnotatorError, gather_error, ErrorWrapper) -from pypy.objspace.flow.model import (Variable, Constant, FunctionGraph, - c_last_exception, checkgraph) -from pypy.translator import simplify, transform -from pypy.annotation import model as annmodel, signature, unaryop, binaryop -from pypy.annotation.bookkeeper import Bookkeeper -import py -log = py.log.Producer("annrpython") -py.log.setconsumer("annrpython", ansi_log) - - -FAIL = object() - -class RPythonAnnotator(object): - """Block annotator for RPython. - See description in doc/translation.txt.""" - - def __init__(self, translator=None, policy=None, bookkeeper=None): - import pypy.rpython.ootypesystem.ooregistry # has side effects - import pypy.rpython.extfuncregistry # has side effects - import pypy.rlib.nonconst # has side effects - - if translator is None: - # interface for tests - from pypy.translator.translator import TranslationContext - translator = TranslationContext() - translator.annotator = self - self.translator = translator - self.pendingblocks = {} # map {block: graph-containing-it} - self.bindings = {} # map Variables to SomeValues - self.annotated = {} # set of blocks already seen - self.added_blocks = None # see processblock() below - self.links_followed = {} # set of links that have ever been followed - self.notify = {} # {block: {positions-to-reflow-from-when-done}} - self.fixed_graphs = {} # set of graphs not to annotate again - self.blocked_blocks = {} # set of {blocked_block: (graph, index)} - # --- the following information is recorded for debugging --- - self.blocked_graphs = {} # set of graphs that have blocked blocks - # --- end of debugging information --- - self.frozen = False - if policy is None: - from pypy.annotation.policy import AnnotatorPolicy - self.policy = AnnotatorPolicy() - else: - self.policy = policy - if bookkeeper is None: - bookkeeper = Bookkeeper(self) - self.bookkeeper = bookkeeper - - def __getstate__(self): - attrs = """translator pendingblocks bindings annotated links_followed - notify bookkeeper frozen policy added_blocks""".split() - ret = self.__dict__.copy() - for key, value in ret.items(): - if key not in attrs: - assert type(value) is dict, ( - "%r is not dict. please update %s.__getstate__" % - (key, self.__class__.__name__)) - ret[key] = {} - return ret - - #___ convenience high-level interface __________________ - - def build_types(self, function, input_arg_types, complete_now=True, - main_entry_point=False): - """Recursively build annotations about the specific entry point.""" - assert isinstance(function, types.FunctionType), "fix that!" - - from pypy.annotation.policy import AnnotatorPolicy - policy = AnnotatorPolicy() - # make input arguments and set their type - args_s = [self.typeannotation(t) for t in input_arg_types] - - # XXX hack - annmodel.TLS.check_str_without_nul = ( - self.translator.config.translation.check_str_without_nul) - - flowgraph, inputcells = self.get_call_parameters(function, args_s, policy) - if not isinstance(flowgraph, FunctionGraph): - assert isinstance(flowgraph, annmodel.SomeObject) - return flowgraph - - if main_entry_point: - self.translator.entry_point_graph = flowgraph - return self.build_graph_types(flowgraph, inputcells, complete_now=complete_now) - - def get_call_parameters(self, function, args_s, policy): - desc = self.bookkeeper.getdesc(function) - args = self.bookkeeper.build_args("simple_call", args_s[:]) - result = [] - def schedule(graph, inputcells): - result.append((graph, inputcells)) - return annmodel.s_ImpossibleValue - - prevpolicy = self.policy - self.policy = policy - self.bookkeeper.enter(None) - try: - desc.pycall(schedule, args, annmodel.s_ImpossibleValue) - finally: - self.bookkeeper.leave() - self.policy = prevpolicy - [(graph, inputcells)] = result - return graph, inputcells - - def annotate_helper(self, function, args_s, policy=None): - if policy is None: - from pypy.annotation.policy import AnnotatorPolicy - policy = AnnotatorPolicy() - graph, inputcells = self.get_call_parameters(function, args_s, policy) - self.build_graph_types(graph, inputcells, complete_now=False) - self.complete_helpers(policy) - return graph - - def complete_helpers(self, policy): - saved = self.policy, self.added_blocks - self.policy = policy - try: - self.added_blocks = {} - self.complete() - # invoke annotation simplifications for the new blocks - self.simplify(block_subset=self.added_blocks) - finally: - self.policy, self.added_blocks = saved - - def build_graph_types(self, flowgraph, inputcells, complete_now=True): - checkgraph(flowgraph) - - nbarg = len(flowgraph.getargs()) - if len(inputcells) != nbarg: - raise TypeError("%s expects %d args, got %d" %( - flowgraph, nbarg, len(inputcells))) - - # register the entry point - self.addpendinggraph(flowgraph, inputcells) - # recursively proceed until no more pending block is left - if complete_now: - self.complete() - return self.binding(flowgraph.getreturnvar(), None) - - def gettype(self, variable): - """Return the known type of a control flow graph variable, - defaulting to 'object'.""" - if isinstance(variable, Constant): - return type(variable.value) - elif isinstance(variable, Variable): - cell = self.bindings.get(variable) - if cell: - return cell.knowntype - else: - return object - else: - raise TypeError, ("Variable or Constant instance expected, " - "got %r" % (variable,)) - - def getuserclassdefinitions(self): - """Return a list of ClassDefs.""" - return self.bookkeeper.classdefs - - #___ medium-level interface ____________________________ - - def addpendinggraph(self, flowgraph, inputcells): - self.addpendingblock(flowgraph, flowgraph.startblock, inputcells) - - def addpendingblock(self, graph, block, cells): - """Register an entry point into block with the given input cells.""" - if graph in self.fixed_graphs: - # special case for annotating/rtyping in several phases: calling - # a graph that has already been rtyped. Safety-check the new - # annotations that are passed in, and don't annotate the old - # graph -- it's already low-level operations! - for a, s_newarg in zip(graph.getargs(), cells): - s_oldarg = self.binding(a) - assert annmodel.unionof(s_oldarg, s_newarg) == s_oldarg - else: - assert not self.frozen - for a in cells: - assert isinstance(a, annmodel.SomeObject) - if block not in self.annotated: - self.bindinputargs(graph, block, cells) - else: - self.mergeinputargs(graph, block, cells) - if not self.annotated[block]: - self.pendingblocks[block] = graph - - def complete(self): - """Process pending blocks until none is left.""" - while True: - while self.pendingblocks: - block, graph = self.pendingblocks.popitem() - self.processblock(graph, block) - self.policy.no_more_blocks_to_annotate(self) - if not self.pendingblocks: - break # finished - # make sure that the return variables of all graphs is annotated - if self.added_blocks is not None: - newgraphs = [self.annotated[block] for block in self.added_blocks] - newgraphs = dict.fromkeys(newgraphs) - got_blocked_blocks = False in newgraphs - else: - newgraphs = self.translator.graphs #all of them - got_blocked_blocks = False in self.annotated.values() - if got_blocked_blocks: - for graph in self.blocked_graphs.values(): - self.blocked_graphs[graph] = True - - blocked_blocks = [block for block, done in self.annotated.items() - if done is False] - assert len(blocked_blocks) == len(self.blocked_blocks) - - text = format_blocked_annotation_error(self, self.blocked_blocks) - #raise SystemExit() - raise AnnotatorError(text) - for graph in newgraphs: - v = graph.getreturnvar() - if v not in self.bindings: - self.setbinding(v, annmodel.s_ImpossibleValue) - # policy-dependent computation - self.bookkeeper.compute_at_fixpoint() - - def binding(self, arg, default=FAIL): - "Gives the SomeValue corresponding to the given Variable or Constant." - if isinstance(arg, Variable): - try: - return self.bindings[arg] - except KeyError: - if default is not FAIL: - return default - else: - raise - elif isinstance(arg, Constant): - #if arg.value is undefined_value: # undefined local variables - # return annmodel.s_ImpossibleValue - return self.bookkeeper.immutableconstant(arg) - else: - raise TypeError, 'Variable or Constant expected, got %r' % (arg,) - - def typeannotation(self, t): - return signature.annotation(t, self.bookkeeper) - - def setbinding(self, arg, s_value): - if arg in self.bindings: - assert s_value.contains(self.bindings[arg]) - self.bindings[arg] = s_value - - def transfer_binding(self, v_target, v_source): - assert v_source in self.bindings - self.bindings[v_target] = self.bindings[v_source] - - def warning(self, msg, pos=None): - if pos is None: - try: - pos = self.bookkeeper.position_key - except AttributeError: - pos = '?' - if pos != '?': - pos = self.whereami(pos) - - log.WARNING("%s/ %s" % (pos, msg)) - - - #___ interface for annotator.bookkeeper _______ - - def recursivecall(self, graph, whence, inputcells): - if isinstance(whence, tuple): - parent_graph, parent_block, parent_index = whence - tag = parent_block, parent_index - self.translator.update_call_graph(parent_graph, graph, tag) - # self.notify[graph.returnblock] is a dictionary of call - # points to this func which triggers a reflow whenever the - # return block of this graph has been analysed. - callpositions = self.notify.setdefault(graph.returnblock, {}) - if whence is not None: - if callable(whence): - def callback(): - whence(self, graph) - else: - callback = whence - callpositions[callback] = True - - # generalize the function's input arguments - self.addpendingblock(graph, graph.startblock, inputcells) - - # get the (current) return value - v = graph.getreturnvar() - try: - return self.bindings[v] - except KeyError: - # the function didn't reach any return statement so far. - # (some functions actually never do, they always raise exceptions) - return annmodel.s_ImpossibleValue - - def reflowfromposition(self, position_key): - graph, block, index = position_key - self.reflowpendingblock(graph, block) - - - #___ simplification (should be moved elsewhere?) _______ - - def simplify(self, block_subset=None, extra_passes=None): - # Generic simplifications - transform.transform_graph(self, block_subset=block_subset, - extra_passes=extra_passes) - if block_subset is None: - graphs = self.translator.graphs - else: - graphs = {} - for block in block_subset: - graph = self.annotated.get(block) - if graph: - graphs[graph] = True - for graph in graphs: - simplify.eliminate_empty_blocks(graph) - - - #___ flowing annotations in blocks _____________________ - - def processblock(self, graph, block): - # Important: this is not called recursively. - # self.flowin() can only issue calls to self.addpendingblock(). - # The analysis of a block can be in three states: - # * block not in self.annotated: - # never seen the block. - # * self.annotated[block] == False: - # the input variables of the block are in self.bindings but we - # still have to consider all the operations in the block. - # * self.annotated[block] == graph-containing-block: - # analysis done (at least until we find we must generalize the - # input variables). - - #print '* processblock', block, cells - self.annotated[block] = graph - if block in self.blocked_blocks: - del self.blocked_blocks[block] - try: - self.flowin(graph, block) - except BlockedInference, e: - self.annotated[block] = False # failed, hopefully temporarily - self.blocked_blocks[block] = (graph, e.opindex) - except Exception, e: - # hack for debug tools only - if not hasattr(e, '__annotator_block'): - setattr(e, '__annotator_block', block) - raise - - # The dict 'added_blocks' is used by rpython.annlowlevel to - # detect which are the new blocks that annotating an additional - # small helper creates. - if self.added_blocks is not None: - self.added_blocks[block] = True - - def reflowpendingblock(self, graph, block): - assert not self.frozen - assert graph not in self.fixed_graphs - self.pendingblocks[block] = graph - assert block in self.annotated - self.annotated[block] = False # must re-flow - self.blocked_blocks[block] = (graph, None) - - def bindinputargs(self, graph, block, inputcells): - # Create the initial bindings for the input args of a block. - assert len(block.inputargs) == len(inputcells) - for a, cell in zip(block.inputargs, inputcells): - self.setbinding(a, cell) - self.annotated[block] = False # must flowin. - self.blocked_blocks[block] = (graph, None) - - def mergeinputargs(self, graph, block, inputcells): - # Merge the new 'cells' with each of the block's existing input - # variables. - oldcells = [self.binding(a) for a in block.inputargs] - try: - unions = [annmodel.unionof(c1,c2) for c1, c2 in zip(oldcells,inputcells)] - except annmodel.UnionError, e: - e.args = e.args + ( - ErrorWrapper(gather_error(self, graph, block, None)),) - raise - # if the merged cells changed, we must redo the analysis - if unions != oldcells: - self.bindinputargs(graph, block, unions) - - def whereami(self, position_key): - graph, block, i = position_key - blk = "" - if block: - at = block.at() - if at: - blk = " block"+at - opid="" - if i is not None: - opid = " op=%d" % i - return repr(graph) + blk + opid - - def flowin(self, graph, block): - #print 'Flowing', block, [self.binding(a) for a in block.inputargs] - try: - for i in range(len(block.operations)): - try: - self.bookkeeper.enter((graph, block, i)) - self.consider_op(block, i) - finally: - self.bookkeeper.leave() - - except BlockedInference, e: - if (e.op is block.operations[-1] and - block.exitswitch == c_last_exception): - # this is the case where the last operation of the block will - # always raise an exception which is immediately caught by - # an exception handler. We then only follow the exceptional - # branches. - exits = [link for link in block.exits - if link.exitcase is not None] - - elif e.op.opname in ('simple_call', 'call_args', 'next'): - # XXX warning, keep the name of the call operations in sync - # with the flow object space. These are the operations for - # which it is fine to always raise an exception. We then - # swallow the BlockedInference and that's it. - # About 'next': see test_annotate_iter_empty_container(). - return - - else: - # other cases are problematic (but will hopefully be solved - # later by reflowing). Throw the BlockedInference up to - # processblock(). - raise - - except annmodel.HarmlesslyBlocked: - return - - else: - # dead code removal: don't follow all exits if the exitswitch - # is known - exits = block.exits - if isinstance(block.exitswitch, Variable): - s_exitswitch = self.bindings[block.exitswitch] - if s_exitswitch.is_constant(): - exits = [link for link in exits - if link.exitcase == s_exitswitch.const] - - # mapping (exitcase, variable) -> s_annotation - # that can be attached to booleans, exitswitches - knowntypedata = getattr(self.bindings.get(block.exitswitch), - "knowntypedata", {}) - - # filter out those exceptions which cannot - # occour for this specific, typed operation. - if block.exitswitch == c_last_exception: - op = block.operations[-1] - if op.opname in binaryop.BINARY_OPERATIONS: - arg1 = self.binding(op.args[0]) - arg2 = self.binding(op.args[1]) - binop = getattr(pair(arg1, arg2), op.opname, None) - can_only_throw = annmodel.read_can_only_throw(binop, arg1, arg2) - elif op.opname in unaryop.UNARY_OPERATIONS: - arg1 = self.binding(op.args[0]) - opname = op.opname - if opname == 'contains': opname = 'op_contains' - unop = getattr(arg1, opname, None) - can_only_throw = annmodel.read_can_only_throw(unop, arg1) - else: - can_only_throw = None - - if can_only_throw is not None: - candidates = can_only_throw - candidate_exits = exits - exits = [] - for link in candidate_exits: - case = link.exitcase - if case is None: - exits.append(link) - continue - covered = [c for c in candidates if issubclass(c, case)] - if covered: - exits.append(link) - candidates = [c for c in candidates if c not in covered] - - for link in exits: - in_except_block = False - - last_exception_var = link.last_exception # may be None for non-exception link - last_exc_value_var = link.last_exc_value # may be None for non-exception link - - if isinstance(link.exitcase, (types.ClassType, type)) \ - and issubclass(link.exitcase, py.builtin.BaseException): - assert last_exception_var and last_exc_value_var - last_exc_value_object = self.bookkeeper.valueoftype(link.exitcase) - last_exception_object = annmodel.SomeType() - if isinstance(last_exception_var, Constant): - last_exception_object.const = last_exception_var.value - last_exception_object.is_type_of = [last_exc_value_var] - - if isinstance(last_exception_var, Variable): - self.setbinding(last_exception_var, last_exception_object) - if isinstance(last_exc_value_var, Variable): - self.setbinding(last_exc_value_var, last_exc_value_object) - - last_exception_object = annmodel.SomeType() - if isinstance(last_exception_var, Constant): - last_exception_object.const = last_exception_var.value - #if link.exitcase is Exception: - # last_exc_value_object = annmodel.SomeObject() - #else: - last_exc_value_vars = [] - in_except_block = True - - ignore_link = False - cells = [] - renaming = {} - for a,v in zip(link.args,link.target.inputargs): - renaming.setdefault(a, []).append(v) - for a,v in zip(link.args,link.target.inputargs): - if a == last_exception_var: - assert in_except_block - cells.append(last_exception_object) - elif a == last_exc_value_var: - assert in_except_block - cells.append(last_exc_value_object) - last_exc_value_vars.append(v) - else: - cell = self.binding(a) - if (link.exitcase, a) in knowntypedata: - knownvarvalue = knowntypedata[(link.exitcase, a)] - cell = pair(cell, knownvarvalue).improve() - # ignore links that try to pass impossible values - if cell == annmodel.s_ImpossibleValue: - ignore_link = True - - if hasattr(cell,'is_type_of'): - renamed_is_type_of = [] - for v in cell.is_type_of: - new_vs = renaming.get(v,[]) - renamed_is_type_of += new_vs - assert cell.knowntype is type - newcell = annmodel.SomeType() - if cell.is_constant(): - newcell.const = cell.const - cell = newcell - cell.is_type_of = renamed_is_type_of - - if hasattr(cell, 'knowntypedata'): - renamed_knowntypedata = {} - for (value, v), s in cell.knowntypedata.items(): - new_vs = renaming.get(v, []) - for new_v in new_vs: - renamed_knowntypedata[value, new_v] = s - assert isinstance(cell, annmodel.SomeBool) - newcell = annmodel.SomeBool() - if cell.is_constant(): - newcell.const = cell.const - cell = newcell - cell.set_knowntypedata(renamed_knowntypedata) - - cells.append(cell) - - if ignore_link: - continue - - if in_except_block: - last_exception_object.is_type_of = last_exc_value_vars - - self.links_followed[link] = True - self.addpendingblock(graph, link.target, cells) - - if block in self.notify: - # reflow from certain positions when this block is done - for callback in self.notify[block]: - if isinstance(callback, tuple): - self.reflowfromposition(callback) # callback is a position - else: - callback() - - - #___ creating the annotations based on operations ______ - - def consider_op(self, block, opindex): - op = block.operations[opindex] - argcells = [self.binding(a) for a in op.args] - consider_meth = getattr(self,'consider_op_'+op.opname, - None) - if not consider_meth: - raise Exception,"unknown op: %r" % op - - # let's be careful about avoiding propagated SomeImpossibleValues - # to enter an op; the latter can result in violations of the - # more general results invariant: e.g. if SomeImpossibleValue enters is_ - # is_(SomeImpossibleValue, None) -> SomeBool - # is_(SomeInstance(not None), None) -> SomeBool(const=False) ... - # boom -- in the assert of setbinding() - for arg in argcells: - if isinstance(arg, annmodel.SomeImpossibleValue): - raise BlockedInference(self, op, opindex) - try: - resultcell = consider_meth(*argcells) - except Exception, e: - graph = self.bookkeeper.position_key[0] - e.args = e.args + ( - ErrorWrapper(gather_error(self, graph, block, opindex)),) - raise - if resultcell is None: - resultcell = self.noreturnvalue(op) - elif resultcell == annmodel.s_ImpossibleValue: - raise BlockedInference(self, op, opindex) # the operation cannot succeed - assert isinstance(resultcell, annmodel.SomeObject) - assert isinstance(op.result, Variable) - self.setbinding(op.result, resultcell) # bind resultcell to op.result - - def noreturnvalue(self, op): - return annmodel.s_ImpossibleValue # no return value (hook method) - - # XXX "contains" clash with SomeObject method - def consider_op_contains(self, seq, elem): - self.bookkeeper.count("contains", seq) - return seq.op_contains(elem) - - def consider_op_newtuple(self, *args): - return annmodel.SomeTuple(items = args) - - def consider_op_newlist(self, *args): - return self.bookkeeper.newlist(*args) - - def consider_op_newdict(self): - return self.bookkeeper.newdict() - - - def _registeroperations(cls, unary_ops, binary_ops): - # All unary operations - d = {} - for opname in unary_ops: - fnname = 'consider_op_' + opname - exec py.code.Source(""" -def consider_op_%s(self, arg, *args): - return arg.%s(*args) -""" % (opname, opname)).compile() in globals(), d - setattr(cls, fnname, d[fnname]) - # All binary operations - for opname in binary_ops: - fnname = 'consider_op_' + opname - exec py.code.Source(""" -def consider_op_%s(self, arg1, arg2, *args): - return pair(arg1,arg2).%s(*args) -""" % (opname, opname)).compile() in globals(), d - setattr(cls, fnname, d[fnname]) - _registeroperations = classmethod(_registeroperations) - -# register simple operations handling -RPythonAnnotator._registeroperations(unaryop.UNARY_OPERATIONS, binaryop.BINARY_OPERATIONS) - - -class BlockedInference(Exception): - """This exception signals the type inference engine that the situation - is currently blocked, and that it should try to progress elsewhere.""" - - def __init__(self, annotator, op, opindex): - self.annotator = annotator - try: - self.break_at = annotator.bookkeeper.position_key - except AttributeError: - self.break_at = None - self.op = op - self.opindex = opindex - - def __repr__(self): - if not self.break_at: - break_at = "?" - else: - break_at = self.annotator.whereami(self.break_at) - return "" %(break_at, self.op) - - __str__ = __repr__ diff --git a/pypy/annotation/binaryop.py b/pypy/annotation/binaryop.py deleted file mode 100644 --- a/pypy/annotation/binaryop.py +++ /dev/null @@ -1,1052 +0,0 @@ -""" -Binary operations between SomeValues. -""" - -import py -import operator -from pypy.tool.pairtype import pair, pairtype -from pypy.annotation.model import SomeObject, SomeInteger, SomeBool, s_Bool -from pypy.annotation.model import SomeString, SomeChar, SomeList, SomeDict -from pypy.annotation.model import SomeUnicodeCodePoint, SomeUnicodeString -from pypy.annotation.model import SomeTuple, SomeImpossibleValue, s_ImpossibleValue -from pypy.annotation.model import SomeInstance, SomeBuiltin, SomeIterator -from pypy.annotation.model import SomePBC, SomeFloat, s_None, SomeByteArray -from pypy.annotation.model import SomeExternalObject, SomeWeakRef -from pypy.annotation.model import SomeAddress, SomeTypedAddressAccess -from pypy.annotation.model import SomeSingleFloat, SomeLongFloat, SomeType -from pypy.annotation.model import unionof, UnionError, missing_operation -from pypy.annotation.model import TLS -from pypy.annotation.model import read_can_only_throw -from pypy.annotation.model import add_knowntypedata, merge_knowntypedata -from pypy.annotation.model import SomeGenericCallable -from pypy.annotation.bookkeeper import getbookkeeper -from pypy.objspace.flow.model import Variable, Constant -from pypy.rlib import rarithmetic -from pypy.tool.error import AnnotatorError - -# convenience only! -def immutablevalue(x): - return getbookkeeper().immutablevalue(x) - -# XXX unify this with ObjSpace.MethodTable -BINARY_OPERATIONS = set(['add', 'sub', 'mul', 'div', 'mod', - 'truediv', 'floordiv', 'divmod', - 'and_', 'or_', 'xor', - 'lshift', 'rshift', - 'getitem', 'setitem', 'delitem', - 'getitem_idx', 'getitem_key', 'getitem_idx_key', - 'inplace_add', 'inplace_sub', 'inplace_mul', - 'inplace_truediv', 'inplace_floordiv', 'inplace_div', - 'inplace_mod', - 'inplace_lshift', 'inplace_rshift', - 'inplace_and', 'inplace_or', 'inplace_xor', - 'lt', 'le', 'eq', 'ne', 'gt', 'ge', 'is_', 'cmp', - 'coerce', - ] - +[opname+'_ovf' for opname in - """add sub mul floordiv div mod lshift - """.split() - ]) - -for opname in BINARY_OPERATIONS: - missing_operation(pairtype(SomeObject, SomeObject), opname) - -class __extend__(pairtype(SomeObject, SomeObject)): - - def union((obj1, obj2)): - raise UnionError(obj1, obj2) - - # inplace_xxx ---> xxx by default - def inplace_add((obj1, obj2)): return pair(obj1, obj2).add() - def inplace_sub((obj1, obj2)): return pair(obj1, obj2).sub() - def inplace_mul((obj1, obj2)): return pair(obj1, obj2).mul() - def inplace_truediv((obj1, obj2)): return pair(obj1, obj2).truediv() - def inplace_floordiv((obj1, obj2)): return pair(obj1, obj2).floordiv() - def inplace_div((obj1, obj2)): return pair(obj1, obj2).div() - def inplace_mod((obj1, obj2)): return pair(obj1, obj2).mod() - def inplace_lshift((obj1, obj2)): return pair(obj1, obj2).lshift() - def inplace_rshift((obj1, obj2)): return pair(obj1, obj2).rshift() - def inplace_and((obj1, obj2)): return pair(obj1, obj2).and_() - def inplace_or((obj1, obj2)): return pair(obj1, obj2).or_() - def inplace_xor((obj1, obj2)): return pair(obj1, obj2).xor() - - for name, func in locals().items(): - if name.startswith('inplace_'): - func.can_only_throw = [] - - inplace_div.can_only_throw = [ZeroDivisionError] - inplace_truediv.can_only_throw = [ZeroDivisionError] - inplace_floordiv.can_only_throw = [ZeroDivisionError] - inplace_mod.can_only_throw = [ZeroDivisionError] - - def lt((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const < obj2.const) - else: - getbookkeeper().count("non_int_comp", obj1, obj2) - return s_Bool - - def le((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const <= obj2.const) - else: - getbookkeeper().count("non_int_comp", obj1, obj2) - return s_Bool - - def eq((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const == obj2.const) - else: - getbookkeeper().count("non_int_eq", obj1, obj2) - return s_Bool - - def ne((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const != obj2.const) - else: - getbookkeeper().count("non_int_eq", obj1, obj2) - return s_Bool - - def gt((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const > obj2.const) - else: - getbookkeeper().count("non_int_comp", obj1, obj2) - return s_Bool - - def ge((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const >= obj2.const) - else: - getbookkeeper().count("non_int_comp", obj1, obj2) - return s_Bool - - def cmp((obj1, obj2)): - getbookkeeper().count("cmp", obj1, obj2) - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(cmp(obj1.const, obj2.const)) - else: - return SomeInteger() - - def is_((obj1, obj2)): - r = SomeBool() - if obj2.is_constant(): - if obj1.is_constant(): - r.const = obj1.const is obj2.const - if obj2.const is None and not obj1.can_be_none(): - r.const = False - elif obj1.is_constant(): - if obj1.const is None and not obj2.can_be_none(): - r.const = False - # XXX HACK HACK HACK - # XXX HACK HACK HACK - # XXX HACK HACK HACK - bk = getbookkeeper() - if bk is not None: # for testing - knowntypedata = {} - fn, block, i = bk.position_key - - annotator = bk.annotator - op = block.operations[i] - assert op.opname == "is_" - assert len(op.args) == 2 - - def bind(src_obj, tgt_obj, tgt_arg): - if hasattr(tgt_obj, 'is_type_of') and src_obj.is_constant(): - add_knowntypedata(knowntypedata, True, tgt_obj.is_type_of, - bk.valueoftype(src_obj.const)) - - assert annotator.binding(op.args[tgt_arg]) == tgt_obj - add_knowntypedata(knowntypedata, True, [op.args[tgt_arg]], src_obj) - - nonnone_obj = tgt_obj - if src_obj.is_constant() and src_obj.const is None and tgt_obj.can_be_none(): - nonnone_obj = tgt_obj.nonnoneify() - - add_knowntypedata(knowntypedata, False, [op.args[tgt_arg]], nonnone_obj) - - bind(obj2, obj1, 0) - bind(obj1, obj2, 1) - r.set_knowntypedata(knowntypedata) - - return r - - def divmod((obj1, obj2)): - getbookkeeper().count("divmod", obj1, obj2) - return SomeTuple([pair(obj1, obj2).div(), pair(obj1, obj2).mod()]) - - def coerce((obj1, obj2)): - getbookkeeper().count("coerce", obj1, obj2) - return pair(obj1, obj2).union() # reasonable enough - - # approximation of an annotation intersection, the result should be the annotation obj or - # the intersection of obj and improvement - def improve((obj, improvement)): - if not improvement.contains(obj) and obj.contains(improvement): - return improvement - else: - return obj - - # checked getitems - - def _getitem_can_only_throw(s_c1, s_o2): - impl = pair(s_c1, s_o2).getitem - return read_can_only_throw(impl, s_c1, s_o2) - - def getitem_idx_key((s_c1, s_o2)): - impl = pair(s_c1, s_o2).getitem - return impl() - getitem_idx_key.can_only_throw = _getitem_can_only_throw - - getitem_idx = getitem_idx_key - getitem_key = getitem_idx_key - - -class __extend__(pairtype(SomeType, SomeType)): - - def union((obj1, obj2)): - result = SomeType() - is_type_of1 = getattr(obj1, 'is_type_of', None) - is_type_of2 = getattr(obj2, 'is_type_of', None) - if obj1.is_immutable_constant() and obj2.is_immutable_constant() and obj1.const == obj2.const: - result.const = obj1.const - is_type_of = {} - if is_type_of1: - for v in is_type_of1: - is_type_of[v] = True - if is_type_of2: - for v in is_type_of2: - is_type_of[v] = True - if is_type_of: - result.is_type_of = is_type_of.keys() - else: - if is_type_of1 and is_type_of1 == is_type_of2: - result.is_type_of = is_type_of1 - return result - - -# cloning a function with identical code, for the can_only_throw attribute -def _clone(f, can_only_throw = None): - newfunc = type(f)(f.func_code, f.func_globals, f.func_name, - f.func_defaults, f.func_closure) - if can_only_throw is not None: - newfunc.can_only_throw = can_only_throw - return newfunc - -class __extend__(pairtype(SomeInteger, SomeInteger)): - # unsignedness is considered a rare and contagious disease - - def union((int1, int2)): - if int1.unsigned == int2.unsigned: - knowntype = rarithmetic.compute_restype(int1.knowntype, int2.knowntype) - else: - t1 = int1.knowntype - if t1 is bool: - t1 = int - t2 = int2.knowntype - if t2 is bool: - t2 = int - - if t2 is int: - if int2.nonneg == False: - raise UnionError, "Merging %s and a possibly negative int is not allowed" % t1 - knowntype = t1 - elif t1 is int: - if int1.nonneg == False: - raise UnionError, "Merging %s and a possibly negative int is not allowed" % t2 - knowntype = t2 - else: - raise UnionError, "Merging these types (%s, %s) is not supported" % (t1, t2) - return SomeInteger(nonneg=int1.nonneg and int2.nonneg, - knowntype=knowntype) - - or_ = xor = add = mul = _clone(union, []) - add_ovf = mul_ovf = _clone(union, [OverflowError]) - div = floordiv = mod = _clone(union, [ZeroDivisionError]) - div_ovf= floordiv_ovf = mod_ovf = _clone(union, [ZeroDivisionError, OverflowError]) - - def truediv((int1, int2)): - return SomeFloat() - truediv.can_only_throw = [ZeroDivisionError] - truediv_ovf = _clone(truediv, [ZeroDivisionError, OverflowError]) - - inplace_div = div - inplace_truediv = truediv - - def sub((int1, int2)): - knowntype = rarithmetic.compute_restype(int1.knowntype, int2.knowntype) - return SomeInteger(knowntype=knowntype) - sub.can_only_throw = [] - sub_ovf = _clone(sub, [OverflowError]) - - def and_((int1, int2)): - knowntype = rarithmetic.compute_restype(int1.knowntype, int2.knowntype) - return SomeInteger(nonneg=int1.nonneg or int2.nonneg, - knowntype=knowntype) - and_.can_only_throw = [] - - def lshift((int1, int2)): - if isinstance(int1, SomeBool): - return SomeInteger() - else: - return SomeInteger(knowntype=int1.knowntype) - lshift.can_only_throw = [] - lshift_ovf = _clone(lshift, [OverflowError]) - - def rshift((int1, int2)): - if isinstance(int1, SomeBool): - return SomeInteger(nonneg=True) - else: - return SomeInteger(nonneg=int1.nonneg, knowntype=int1.knowntype) - rshift.can_only_throw = [] - - def _compare_helper((int1, int2), opname, operation): - r = SomeBool() - if int1.is_immutable_constant() and int2.is_immutable_constant(): - r.const = operation(int1.const, int2.const) - # - # The rest of the code propagates nonneg information between - # the two arguments. - # - # Doing the right thing when int1 or int2 change from signed - # to unsigned (r_uint) is almost impossible. See test_intcmp_bug. - # Instead, we only deduce constrains on the operands in the - # case where they are both signed. In other words, if y is - # nonneg then "assert x>=y" will let the annotator know that - # x is nonneg too, but it will not work if y is unsigned. - # - if not (rarithmetic.signedtype(int1.knowntype) and - rarithmetic.signedtype(int2.knowntype)): - return r - knowntypedata = {} - # XXX HACK HACK HACK - fn, block, i = getbookkeeper().position_key - op = block.operations[i] - assert op.opname == opname - assert len(op.args) == 2 - def tointtype(int0): - if int0.knowntype is bool: - return int - return int0.knowntype - if int1.nonneg and isinstance(op.args[1], Variable): - case = opname in ('lt', 'le', 'eq') - - add_knowntypedata(knowntypedata, case, [op.args[1]], - SomeInteger(nonneg=True, knowntype=tointtype(int2))) - if int2.nonneg and isinstance(op.args[0], Variable): - case = opname in ('gt', 'ge', 'eq') - add_knowntypedata(knowntypedata, case, [op.args[0]], - SomeInteger(nonneg=True, knowntype=tointtype(int1))) - r.set_knowntypedata(knowntypedata) - # a special case for 'x < 0' or 'x >= 0', - # where 0 is a flow graph Constant - # (in this case we are sure that it cannot become a r_uint later) - if (isinstance(op.args[1], Constant) and - type(op.args[1].value) is int and # filter out Symbolics - op.args[1].value == 0): - if int1.nonneg: - if opname == 'lt': - r.const = False - if opname == 'ge': - r.const = True - return r - - def lt(intint): return intint._compare_helper('lt', operator.lt) - def le(intint): return intint._compare_helper('le', operator.le) - def eq(intint): return intint._compare_helper('eq', operator.eq) - def ne(intint): return intint._compare_helper('ne', operator.ne) - def gt(intint): return intint._compare_helper('gt', operator.gt) - def ge(intint): return intint._compare_helper('ge', operator.ge) - - -class __extend__(pairtype(SomeBool, SomeBool)): - - def union((boo1, boo2)): - s = SomeBool() - if getattr(boo1, 'const', -1) == getattr(boo2, 'const', -2): - s.const = boo1.const - if hasattr(boo1, 'knowntypedata') and \ - hasattr(boo2, 'knowntypedata'): - ktd = merge_knowntypedata(boo1.knowntypedata, boo2.knowntypedata) - s.set_knowntypedata(ktd) - return s - - def and_((boo1, boo2)): - s = SomeBool() - if boo1.is_constant(): - if not boo1.const: - s.const = False - else: - return boo2 - if boo2.is_constant(): - if not boo2.const: - s.const = False - return s - - def or_((boo1, boo2)): - s = SomeBool() - if boo1.is_constant(): - if boo1.const: - s.const = True - else: - return boo2 - if boo2.is_constant(): - if boo2.const: - s.const = True - return s - - def xor((boo1, boo2)): - s = SomeBool() - if boo1.is_constant() and boo2.is_constant(): - s.const = boo1.const ^ boo2.const - return s - -class __extend__(pairtype(SomeString, SomeString)): - - def union((str1, str2)): - can_be_None = str1.can_be_None or str2.can_be_None - no_nul = str1.no_nul and str2.no_nul - return SomeString(can_be_None=can_be_None, no_nul=no_nul) - - def add((str1, str2)): - # propagate const-ness to help getattr(obj, 'prefix' + const_name) - result = SomeString(no_nul=str1.no_nul and str2.no_nul) - if str1.is_immutable_constant() and str2.is_immutable_constant(): - result.const = str1.const + str2.const - return result - -class __extend__(pairtype(SomeByteArray, SomeByteArray)): - def union((b1, b2)): - can_be_None = b1.can_be_None or b2.can_be_None - return SomeByteArray(can_be_None=can_be_None) - - def add((b1, b2)): - result = SomeByteArray() - if b1.is_immutable_constant() and b2.is_immutable_constant(): - result.const = b1.const + b2.const - return result - -class __extend__(pairtype(SomeByteArray, SomeInteger)): - def getitem((s_b, s_i)): - return SomeInteger() - - def setitem((s_b, s_i), s_i2): - assert isinstance(s_i2, SomeInteger) - -class __extend__(pairtype(SomeString, SomeByteArray), - pairtype(SomeByteArray, SomeString), - pairtype(SomeChar, SomeByteArray), - pairtype(SomeByteArray, SomeChar)): - def add((b1, b2)): - result = SomeByteArray() - if b1.is_immutable_constant() and b2.is_immutable_constant(): - result.const = b1.const + b2.const - return result - -class __extend__(pairtype(SomeChar, SomeChar)): - - def union((chr1, chr2)): - return SomeChar() - - -class __extend__(pairtype(SomeChar, SomeUnicodeCodePoint), - pairtype(SomeUnicodeCodePoint, SomeChar)): - def union((uchr1, uchr2)): - return SomeUnicodeCodePoint() - -class __extend__(pairtype(SomeUnicodeCodePoint, SomeUnicodeCodePoint)): - def union((uchr1, uchr2)): - return SomeUnicodeCodePoint() - - def add((chr1, chr2)): - return SomeUnicodeString() - -class __extend__(pairtype(SomeString, SomeUnicodeString), - pairtype(SomeUnicodeString, SomeString)): - def mod((str, unistring)): - raise NotImplementedError( - "string formatting mixing strings and unicode not supported") - - -class __extend__(pairtype(SomeString, SomeTuple), - pairtype(SomeUnicodeString, SomeTuple)): - def mod((s_string, s_tuple)): - is_string = isinstance(s_string, SomeString) - is_unicode = isinstance(s_string, SomeUnicodeString) - assert is_string or is_unicode - for s_item in s_tuple.items: - if (is_unicode and isinstance(s_item, (SomeChar, SomeString)) or - is_string and isinstance(s_item, (SomeUnicodeCodePoint, - SomeUnicodeString))): - raise NotImplementedError( - "string formatting mixing strings and unicode not supported") - getbookkeeper().count('strformat', s_string, s_tuple) - no_nul = s_string.no_nul - for s_item in s_tuple.items: - if isinstance(s_item, SomeFloat): - pass # or s_item is a subclass, like SomeInteger - elif (isinstance(s_item, SomeString) or - isinstance(s_item, SomeUnicodeString)) and s_item.no_nul: - pass - else: - no_nul = False - break - return s_string.__class__(no_nul=no_nul) - - -class __extend__(pairtype(SomeString, SomeObject), - pairtype(SomeUnicodeString, SomeObject)): - - def mod((s_string, args)): - getbookkeeper().count('strformat', s_string, args) - return s_string.__class__() - -class __extend__(pairtype(SomeFloat, SomeFloat)): - - def union((flt1, flt2)): - return SomeFloat() - - add = sub = mul = union - - def div((flt1, flt2)): - return SomeFloat() - div.can_only_throw = [] - truediv = div - - # repeat these in order to copy the 'can_only_throw' attribute - inplace_div = div - inplace_truediv = truediv - - -class __extend__(pairtype(SomeSingleFloat, SomeSingleFloat)): - - def union((flt1, flt2)): - return SomeSingleFloat() - - -class __extend__(pairtype(SomeLongFloat, SomeLongFloat)): - - def union((flt1, flt2)): - return SomeLongFloat() - - -class __extend__(pairtype(SomeList, SomeList)): - - def union((lst1, lst2)): - return SomeList(lst1.listdef.union(lst2.listdef)) - - def add((lst1, lst2)): - return lst1.listdef.offspring(lst2.listdef) - - def eq((lst1, lst2)): - lst1.listdef.agree(lst2.listdef) - return s_Bool - ne = eq - - -class __extend__(pairtype(SomeList, SomeObject)): - - def inplace_add((lst1, obj2)): - lst1.method_extend(obj2) - return lst1 - inplace_add.can_only_throw = [] - - def inplace_mul((lst1, obj2)): - lst1.listdef.resize() - return lst1 - inplace_mul.can_only_throw = [] - -class __extend__(pairtype(SomeTuple, SomeTuple)): - - def union((tup1, tup2)): - if len(tup1.items) != len(tup2.items): - raise UnionError("cannot take the union of a tuple of length %d " - "and a tuple of length %d" % (len(tup1.items), - len(tup2.items))) - else: - unions = [unionof(x,y) for x,y in zip(tup1.items, tup2.items)] - return SomeTuple(items = unions) - - def add((tup1, tup2)): - return SomeTuple(items = tup1.items + tup2.items) - - def eq(tup1tup2): - tup1tup2.union() - return s_Bool - ne = eq - - def lt((tup1, tup2)): - raise Exception("unsupported: (...) < (...)") - def le((tup1, tup2)): - raise Exception("unsupported: (...) <= (...)") - def gt((tup1, tup2)): - raise Exception("unsupported: (...) > (...)") - def ge((tup1, tup2)): - raise Exception("unsupported: (...) >= (...)") - - -class __extend__(pairtype(SomeDict, SomeDict)): - - def union((dic1, dic2)): - return SomeDict(dic1.dictdef.union(dic2.dictdef)) - - -class __extend__(pairtype(SomeDict, SomeObject)): - - def _can_only_throw(dic1, *ignore): - if dic1.dictdef.dictkey.custom_eq_hash: - return None - return [KeyError] - - def getitem((dic1, obj2)): - getbookkeeper().count("dict_getitem", dic1) - dic1.dictdef.generalize_key(obj2) - return dic1.dictdef.read_value() - getitem.can_only_throw = _can_only_throw - - def setitem((dic1, obj2), s_value): - getbookkeeper().count("dict_setitem", dic1) - dic1.dictdef.generalize_key(obj2) - dic1.dictdef.generalize_value(s_value) - setitem.can_only_throw = _can_only_throw - - def delitem((dic1, obj2)): - getbookkeeper().count("dict_delitem", dic1) - dic1.dictdef.generalize_key(obj2) - delitem.can_only_throw = _can_only_throw - - -class __extend__(pairtype(SomeTuple, SomeInteger)): - - def getitem((tup1, int2)): - if int2.is_immutable_constant(): - try: - return tup1.items[int2.const] - except IndexError: - return s_ImpossibleValue - else: - getbookkeeper().count("tuple_random_getitem", tup1) - return unionof(*tup1.items) - getitem.can_only_throw = [IndexError] - - -class __extend__(pairtype(SomeList, SomeInteger)): - - def mul((lst1, int2)): - return lst1.listdef.offspring() - - def getitem((lst1, int2)): - getbookkeeper().count("list_getitem", int2) - return lst1.listdef.read_item() - getitem.can_only_throw = [] - - getitem_key = getitem - - def getitem_idx((lst1, int2)): - getbookkeeper().count("list_getitem", int2) - return lst1.listdef.read_item() - getitem_idx.can_only_throw = [IndexError] - - getitem_idx_key = getitem_idx - - def setitem((lst1, int2), s_value): - getbookkeeper().count("list_setitem", int2) - lst1.listdef.mutate() - lst1.listdef.generalize(s_value) - setitem.can_only_throw = [IndexError] - - def delitem((lst1, int2)): - getbookkeeper().count("list_delitem", int2) - lst1.listdef.resize() - delitem.can_only_throw = [IndexError] - -class __extend__(pairtype(SomeString, SomeInteger)): - - def getitem((str1, int2)): - getbookkeeper().count("str_getitem", int2) - return SomeChar() - getitem.can_only_throw = [] - - getitem_key = getitem - - def getitem_idx((str1, int2)): - getbookkeeper().count("str_getitem", int2) - return SomeChar() - getitem_idx.can_only_throw = [IndexError] - - getitem_idx_key = getitem_idx - - def mul((str1, int2)): # xxx do we want to support this - getbookkeeper().count("str_mul", str1, int2) - return SomeString(no_nul=str1.no_nul) - -class __extend__(pairtype(SomeUnicodeString, SomeInteger)): - def getitem((str1, int2)): - getbookkeeper().count("str_getitem", int2) - return SomeUnicodeCodePoint() - getitem.can_only_throw = [] - - getitem_key = getitem - - def getitem_idx((str1, int2)): - getbookkeeper().count("str_getitem", int2) - return SomeUnicodeCodePoint() - getitem_idx.can_only_throw = [IndexError] - - getitem_idx_key = getitem_idx - - def mul((str1, int2)): # xxx do we want to support this - getbookkeeper().count("str_mul", str1, int2) - return SomeUnicodeString() - -class __extend__(pairtype(SomeInteger, SomeString), - pairtype(SomeInteger, SomeUnicodeString)): - - def mul((int1, str2)): # xxx do we want to support this - getbookkeeper().count("str_mul", str2, int1) - return str2.basestringclass() - -class __extend__(pairtype(SomeUnicodeCodePoint, SomeUnicodeString), - pairtype(SomeUnicodeString, SomeUnicodeCodePoint), - pairtype(SomeUnicodeString, SomeUnicodeString)): - def union((str1, str2)): - return SomeUnicodeString(can_be_None=str1.can_be_none() or - str2.can_be_none()) - - def add((str1, str2)): - # propagate const-ness to help getattr(obj, 'prefix' + const_name) - result = SomeUnicodeString() - if str1.is_immutable_constant() and str2.is_immutable_constant(): - result.const = str1.const + str2.const - return result - -class __extend__(pairtype(SomeInteger, SomeList)): - - def mul((int1, lst2)): - return lst2.listdef.offspring() - - -class __extend__(pairtype(SomeInstance, SomeInstance)): - - def union((ins1, ins2)): - if ins1.classdef is None or ins2.classdef is None: - # special case only - basedef = None - else: - basedef = ins1.classdef.commonbase(ins2.classdef) - if basedef is None: - raise UnionError(ins1, ins2) - flags = ins1.flags - if flags: - flags = flags.copy() - for key, value in flags.items(): - if key not in ins2.flags or ins2.flags[key] != value: - del flags[key] - return SomeInstance(basedef, - can_be_None=ins1.can_be_None or ins2.can_be_None, - flags=flags) - - def improve((ins1, ins2)): - if ins1.classdef is None: - resdef = ins2.classdef - elif ins2.classdef is None: - resdef = ins1.classdef - else: - basedef = ins1.classdef.commonbase(ins2.classdef) - if basedef is ins1.classdef: - resdef = ins2.classdef - elif basedef is ins2.classdef: - resdef = ins1.classdef - else: - if ins1.can_be_None and ins2.can_be_None: - return s_None - else: - return s_ImpossibleValue - res = SomeInstance(resdef, can_be_None=ins1.can_be_None and ins2.can_be_None) - if ins1.contains(res) and ins2.contains(res): - return res # fine - else: - # this case can occur in the presence of 'const' attributes, - # which we should try to preserve. Fall-back... - thistype = pairtype(SomeInstance, SomeInstance) - return super(thistype, pair(ins1, ins2)).improve() - - -class __extend__(pairtype(SomeIterator, SomeIterator)): - - def union((iter1, iter2)): - s_cont = unionof(iter1.s_container, iter2.s_container) - if iter1.variant != iter2.variant: - raise UnionError("merging incompatible iterators variants") - return SomeIterator(s_cont, *iter1.variant) - - -class __extend__(pairtype(SomeBuiltin, SomeBuiltin)): - - def union((bltn1, bltn2)): - if (bltn1.analyser != bltn2.analyser or - bltn1.methodname != bltn2.methodname or - bltn1.s_self is None or bltn2.s_self is None): - raise UnionError("cannot merge two different builtin functions " - "or methods:\n %r\n %r" % (bltn1, bltn2)) - s_self = unionof(bltn1.s_self, bltn2.s_self) - return SomeBuiltin(bltn1.analyser, s_self, methodname=bltn1.methodname) - -class __extend__(pairtype(SomePBC, SomePBC)): - - def union((pbc1, pbc2)): - d = pbc1.descriptions.copy() - d.update(pbc2.descriptions) - return SomePBC(d, can_be_None = pbc1.can_be_None or pbc2.can_be_None) - - def is_((pbc1, pbc2)): - thistype = pairtype(SomePBC, SomePBC) - s = super(thistype, pair(pbc1, pbc2)).is_() - if not s.is_constant(): - if not pbc1.can_be_None or not pbc2.can_be_None: - for desc in pbc1.descriptions: - if desc in pbc2.descriptions: - break - else: - s.const = False # no common desc in the two sets - return s - -class __extend__(pairtype(SomeGenericCallable, SomePBC)): - def union((gencall, pbc)): - for desc in pbc.descriptions: - unique_key = desc - bk = desc.bookkeeper - s_result = bk.emulate_pbc_call(unique_key, pbc, gencall.args_s) - s_result = unionof(s_result, gencall.s_result) - assert gencall.s_result.contains(s_result) - return gencall - -class __extend__(pairtype(SomePBC, SomeGenericCallable)): - def union((pbc, gencall)): - return pair(gencall, pbc).union() - -class __extend__(pairtype(SomeImpossibleValue, SomeObject)): - def union((imp1, obj2)): - return obj2 - -class __extend__(pairtype(SomeObject, SomeImpossibleValue)): - def union((obj1, imp2)): - return obj1 - -# mixing Nones with other objects - -def _make_none_union(classname, constructor_args='', glob=None): - if glob is None: - glob = globals() - loc = locals() - source = py.code.Source(""" - class __extend__(pairtype(%(classname)s, SomePBC)): - def union((obj, pbc)): - if pbc.isNone(): - return %(classname)s(%(constructor_args)s) - else: - return SomeObject() - - class __extend__(pairtype(SomePBC, %(classname)s)): - def union((pbc, obj)): - if pbc.isNone(): - return %(classname)s(%(constructor_args)s) - else: - return SomeObject() - """ % loc) - exec source.compile() in glob - -_make_none_union('SomeInstance', 'classdef=obj.classdef, can_be_None=True') -_make_none_union('SomeString', 'no_nul=obj.no_nul, can_be_None=True') -_make_none_union('SomeUnicodeString', 'can_be_None=True') -_make_none_union('SomeList', 'obj.listdef') -_make_none_union('SomeDict', 'obj.dictdef') -_make_none_union('SomeExternalObject', 'obj.knowntype') -_make_none_union('SomeWeakRef', 'obj.classdef') - -# getitem on SomePBCs, in particular None fails - -class __extend__(pairtype(SomePBC, SomeObject)): - def getitem((pbc, o)): - if not pbc.isNone(): - raise AnnotatorError("getitem on %r" % pbc) - return s_ImpossibleValue - - def setitem((pbc, o), s_value): - if not pbc.isNone(): - raise AnnotatorError("setitem on %r" % pbc) - -class __extend__(pairtype(SomePBC, SomeString)): - def add((pbc, o)): - if not pbc.isNone(): - raise AnnotatorError('add on %r' % pbc) - return s_ImpossibleValue - -class __extend__(pairtype(SomeString, SomePBC)): - def add((o, pbc)): - if not pbc.isNone(): - raise AnnotatorError('add on %r' % pbc) - return s_ImpossibleValue - -class __extend__(pairtype(SomeExternalObject, SomeExternalObject)): - def union((ext1, ext2)): - if ext1.knowntype == ext2.knowntype: - return SomeExternalObject(ext1.knowntype) - return SomeObject() - -# ____________________________________________________________ -# annotation of low-level types -from pypy.annotation.model import SomePtr, SomeOOInstance, SomeOOClass -from pypy.annotation.model import SomeOOObject -from pypy.annotation.model import ll_to_annotation, annotation_to_lltype -from pypy.rpython.ootypesystem import ootype - -_make_none_union('SomeOOInstance', 'ootype=obj.ootype, can_be_None=True') - -class __extend__(pairtype(SomePtr, SomePtr)): - def union((p1, p2)): - assert p1.ll_ptrtype == p2.ll_ptrtype,("mixing of incompatible pointer types: %r, %r" % - (p1.ll_ptrtype, p2.ll_ptrtype)) - return SomePtr(p1.ll_ptrtype) - -class __extend__(pairtype(SomePtr, SomeInteger)): - - def getitem((p, int1)): - example = p.ll_ptrtype._example() - try: - v = example[0] - except IndexError: - return None # impossible value, e.g. FixedSizeArray(0) - return ll_to_annotation(v) - getitem.can_only_throw = [] - - def setitem((p, int1), s_value): # just doing checking - example = p.ll_ptrtype._example() - if example[0] is not None: # ignore Void s_value - v_lltype = annotation_to_lltype(s_value) - example[0] = v_lltype._defl() - setitem.can_only_throw = [] - -class __extend__(pairtype(SomePtr, SomeObject)): - def union((p, obj)): - assert False, ("mixing pointer type %r with something else %r" % (p.ll_ptrtype, obj)) - - def getitem((p, obj)): - assert False,"ptr %r getitem index not an int: %r" % (p.ll_ptrtype, obj) - - def setitem((p, obj), s_value): - assert False,"ptr %r setitem index not an int: %r" % (p.ll_ptrtype, obj) - -class __extend__(pairtype(SomeObject, SomePtr)): - def union((obj, p2)): - return pair(p2, obj).union() - - -class __extend__(pairtype(SomeOOInstance, SomeOOInstance)): - def union((r1, r2)): - common = ootype.commonBaseclass(r1.ootype, r2.ootype) - assert common is not None, 'Mixing of incompatible instances %r, %r' %(r1.ootype, r2.ootype) - return SomeOOInstance(common, can_be_None=r1.can_be_None or r2.can_be_None) - -class __extend__(pairtype(SomeOOClass, SomeOOClass)): - def union((r1, r2)): - if r1.ootype is None: - common = r2.ootype - elif r2.ootype is None: - common = r1.ootype - elif r1.ootype == r2.ootype: - common = r1.ootype - elif isinstance(r1.ootype, ootype.Instance) and isinstance(r2.ootype, ootype.Instance): - common = ootype.commonBaseclass(r1.ootype, r2.ootype) - assert common is not None, ('Mixing of incompatible classes %r, %r' - % (r1.ootype, r2.ootype)) - else: - common = ootype.Object - return SomeOOClass(common) - -class __extend__(pairtype(SomeOOInstance, SomeObject)): - def union((r, obj)): - assert False, ("mixing reference type %r with something else %r" % (r.ootype, obj)) - -class __extend__(pairtype(SomeObject, SomeOOInstance)): - def union((obj, r2)): - return pair(r2, obj).union() - -class __extend__(pairtype(SomeOOObject, SomeOOObject)): - def union((r1, r2)): - assert r1.ootype is ootype.Object and r2.ootype is ootype.Object - return SomeOOObject() - -#_________________________________________ -# weakrefs - -class __extend__(pairtype(SomeWeakRef, SomeWeakRef)): - def union((s_wrf1, s_wrf2)): - if s_wrf1.classdef is None: - basedef = s_wrf2.classdef # s_wrf1 is known to be dead - elif s_wrf2.classdef is None: - basedef = s_wrf1.classdef # s_wrf2 is known to be dead - else: - basedef = s_wrf1.classdef.commonbase(s_wrf2.classdef) - if basedef is None: # no common base class! complain... - return SomeObject() - return SomeWeakRef(basedef) - -#_________________________________________ -# memory addresses - -class __extend__(pairtype(SomeAddress, SomeAddress)): - def union((s_addr1, s_addr2)): - return SomeAddress() - - def sub((s_addr1, s_addr2)): - if s_addr1.is_null_address() and s_addr2.is_null_address(): - return getbookkeeper().immutablevalue(0) - return SomeInteger() - - def is_((s_addr1, s_addr2)): - assert False, "comparisons with is not supported by addresses" - -class __extend__(pairtype(SomeTypedAddressAccess, SomeTypedAddressAccess)): - def union((s_taa1, s_taa2)): - assert s_taa1.type == s_taa2.type - return s_taa1 - -class __extend__(pairtype(SomeTypedAddressAccess, SomeInteger)): - def getitem((s_taa, s_int)): - from pypy.annotation.model import lltype_to_annotation - return lltype_to_annotation(s_taa.type) - getitem.can_only_throw = [] - - def setitem((s_taa, s_int), s_value): - from pypy.annotation.model import annotation_to_lltype - assert annotation_to_lltype(s_value) is s_taa.type - setitem.can_only_throw = [] - - -class __extend__(pairtype(SomeAddress, SomeInteger)): - def add((s_addr, s_int)): - return SomeAddress() - - def sub((s_addr, s_int)): - return SomeAddress() - -class __extend__(pairtype(SomeAddress, SomeImpossibleValue)): - # need to override this specifically to hide the 'raise UnionError' - # of pairtype(SomeAddress, SomeObject). - def union((s_addr, s_imp)): - return s_addr - -class __extend__(pairtype(SomeImpossibleValue, SomeAddress)): - # need to override this specifically to hide the 'raise UnionError' - # of pairtype(SomeObject, SomeAddress). - def union((s_imp, s_addr)): - return s_addr - -class __extend__(pairtype(SomeAddress, SomeObject)): - def union((s_addr, s_obj)): - raise UnionError, "union of address and anything else makes no sense" - -class __extend__(pairtype(SomeObject, SomeAddress)): - def union((s_obj, s_addr)): - raise UnionError, "union of address and anything else makes no sense" diff --git a/pypy/annotation/bookkeeper.py b/pypy/annotation/bookkeeper.py deleted file mode 100644 --- a/pypy/annotation/bookkeeper.py +++ /dev/null @@ -1,797 +0,0 @@ -""" -The Bookkeeper class. -""" - -from __future__ import absolute_import - -import sys, types, inspect, weakref - -from pypy.objspace.flow.model import Constant From noreply at buildbot.pypy.org Mon Jan 21 20:46:31 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 20:46:31 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fix fix fix Message-ID: <20130121194631.EF1951C106A@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60313:417a88dc4c57 Date: 2013-01-21 21:46 +0200 http://bitbucket.org/pypy/pypy/changeset/417a88dc4c57/ Log: fix fix fix diff --git a/rpython/jit/backend/llsupport/jitframe.py b/rpython/jit/backend/llsupport/jitframe.py --- a/rpython/jit/backend/llsupport/jitframe.py +++ b/rpython/jit/backend/llsupport/jitframe.py @@ -33,7 +33,6 @@ def jitframe_copy(frame): frame_info = frame.jf_frame_info new_frame = lltype.malloc(JITFRAME, frame_info.jfi_frame_depth, zero=True) - ll_assert(frame.jf_gcmap == NULLGCMAP, "non empty gc map when copying") new_frame.jf_frame_info = frame_info return new_frame diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -196,11 +196,12 @@ base_ofs = self.cpu.get_baseofs_of_frame_field() self._push_all_regs_to_frame(mc, [], self.cpu.supports_floats) assert not IS_X86_32 - # push first arg - mc.POP_r(ecx.value) + mc.MOV_rs(ecx.value, WORD) gcmap_ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') mc.MOV_br(gcmap_ofs, ecx.value) + # push first arg mc.LEA_rb(edi.value, -base_ofs) + # aligned already mc.CALL(imm(self.cpu.realloc_frame)) mc.LEA_rm(ebp.value, (eax.value, base_ofs)) mc.MOV_bi(gcmap_ofs, 0) @@ -678,7 +679,7 @@ assert not IS_X86_32 mc.J_il8(rx86.Conditions['GE'], 0) jg_location = mc.get_relative_pos() - self.push_gcmap(mc, gcmap, push=True) + self.push_gcmap(mc, gcmap, mov=True) mc.CALL(imm(self._stack_check_failure)) # patch the JG above offset = mc.get_relative_pos() - jg_location @@ -1865,7 +1866,6 @@ v = gpr_reg_mgr_cls.all_reg_indexes[loc.value] positions[i] = v * WORD # write down the positions of locs - print guardtok.fail_locs guardtok.faildescr.rd_locs = positions #if WORD == 4: # mc.PUSH(imm(fail_descr)) From noreply at buildbot.pypy.org Mon Jan 21 21:19:21 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 21:19:21 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fix Message-ID: <20130121201921.CAFD11C1241@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60314:64b3d731b073 Date: 2013-01-21 22:18 +0200 http://bitbucket.org/pypy/pypy/changeset/64b3d731b073/ Log: fix diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py --- a/rpython/jit/backend/llsupport/llmodel.py +++ b/rpython/jit/backend/llsupport/llmodel.py @@ -55,9 +55,9 @@ FUNC_TP = lltype.Ptr(lltype.FuncType([llmemory.GCREF], llmemory.GCREF)) - def realloc_frame(frame): + def realloc_frame(frame, size): frame = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, frame) - new_frame = frame.copy() + new_frame = jitframe.JITFRAME.allocate(frame.frame_info) # XXX now we know, rewrite this # we need to do this, because we're not sure what things # are GC pointers and which ones are not diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -685,7 +685,7 @@ offset = mc.get_relative_pos() - jg_location assert 0 < offset <= 127 mc.overwrite(jg_location-1, chr(offset)) - return stack_check_cmp_ofs + return stack_check_cmp_ofs def _patch_stackadjust(self, adr, allocated_depth): mc = codebuf.MachineCodeBlockWrapper() From noreply at buildbot.pypy.org Mon Jan 21 21:24:12 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 21:24:12 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: typo Message-ID: <20130121202412.63FE91C1241@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60315:b63e94b8a3a4 Date: 2013-01-21 22:23 +0200 http://bitbucket.org/pypy/pypy/changeset/b63e94b8a3a4/ Log: typo diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py --- a/rpython/jit/backend/llsupport/llmodel.py +++ b/rpython/jit/backend/llsupport/llmodel.py @@ -55,9 +55,9 @@ FUNC_TP = lltype.Ptr(lltype.FuncType([llmemory.GCREF], llmemory.GCREF)) - def realloc_frame(frame, size): + def realloc_frame(frame): frame = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, frame) - new_frame = jitframe.JITFRAME.allocate(frame.frame_info) + new_frame = jitframe.JITFRAME.allocate(frame.jf_frame_info) # XXX now we know, rewrite this # we need to do this, because we're not sure what things # are GC pointers and which ones are not From noreply at buildbot.pypy.org Mon Jan 21 21:27:00 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 21:27:00 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: eh, this time? Message-ID: <20130121202700.9E50B1C1241@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60316:7d7a9610746a Date: 2013-01-21 22:26 +0200 http://bitbucket.org/pypy/pypy/changeset/7d7a9610746a/ Log: eh, this time? diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -201,8 +201,10 @@ mc.MOV_br(gcmap_ofs, ecx.value) # push first arg mc.LEA_rb(edi.value, -base_ofs) - # aligned already + # align + mc.SUB_ri(esp.value, WORD) mc.CALL(imm(self.cpu.realloc_frame)) + mc.ADD_ri(esp.value, WORD) mc.LEA_rm(ebp.value, (eax.value, base_ofs)) mc.MOV_bi(gcmap_ofs, 0) self._pop_all_regs_from_frame(mc, [], self.cpu.supports_floats) From noreply at buildbot.pypy.org Mon Jan 21 21:33:58 2013 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 21 Jan 2013 21:33:58 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: function naming clash Message-ID: <20130121203358.316291C134E@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: missing-ndarray-attributes Changeset: r60317:6d2a659cc6dd Date: 2012-12-31 10:22 +0200 http://bitbucket.org/pypy/pypy/changeset/6d2a659cc6dd/ Log: function naming clash diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -700,7 +700,8 @@ return getattr(interp_ufuncs.get(space), ufunc_name).reduce( space, self, True, promote_to_largest, w_axis, False, out, w_dtype, cumultative=cumultative) - return func_with_new_name(impl, "reduce_%s_impl" % ufunc_name) + return func_with_new_name(impl, "reduce_%s_impl_%d_%d" % (ufunc_name, + promote_to_largest, cumultative)) descr_sum = _reduce_ufunc_impl("add") descr_sum_promote = _reduce_ufunc_impl("add", True) From noreply at buildbot.pypy.org Mon Jan 21 22:36:52 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Mon, 21 Jan 2013 22:36:52 +0100 (CET) Subject: [pypy-commit] pypy py3k: fix imports Message-ID: <20130121213652.80BEE1C106A@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60318:bb983e5cb781 Date: 2013-01-21 13:35 -0800 http://bitbucket.org/pypy/pypy/changeset/bb983e5cb781/ Log: fix imports diff --git a/pypy/interpreter/test2/test_app_main.py b/pypy/interpreter/test2/test_app_main.py --- a/pypy/interpreter/test2/test_app_main.py +++ b/pypy/interpreter/test2/test_app_main.py @@ -86,7 +86,7 @@ @contextmanager def setpythonpath(): old_pythonpath = os.getenv('PYTHONPATH') - rootdir = os.path.dirname(autopath.pypydir) + rootdir = os.path.dirname(pypydir) os.putenv('PYTHONPATH', rootdir) try: yield @@ -857,7 +857,7 @@ class TestAppMain: def test_print_info(self): - from rpython.translator.goal import app_main + from pypy.interpreter import app_main import sys, cStringIO prev_so = sys.stdout prev_ti = getattr(sys, 'pypy_translation_info', 'missing') diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py --- a/pypy/module/_ssl/test/test_ssl.py +++ b/pypy/module/_ssl/test/test_ssl.py @@ -1,4 +1,4 @@ -from pypy.tool.udir import udir +from rpython.tool.udir import udir class AppTestSSL: spaceconfig = dict(usemodules=('_ssl', '_socket', 'binascii')) diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py --- a/pypy/module/cpyext/test/test_bytesobject.py +++ b/pypy/module/cpyext/test/test_bytesobject.py @@ -1,4 +1,4 @@ -from pypy.rpython.lltypesystem import rffi, lltype +from rpython.rtyper.lltypesystem import rffi, lltype from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.bytesobject import new_empty_str, PyBytesObject diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py --- a/pypy/module/imp/test/test_app.py +++ b/pypy/module/imp/test/test_app.py @@ -1,5 +1,5 @@ from __future__ import with_statement -from pypy.tool.udir import udir +from rpython.tool.udir import udir class AppTestImpModule: diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py --- a/pypy/module/imp/test/test_import.py +++ b/pypy/module/imp/test/test_import.py @@ -1040,8 +1040,8 @@ stream.close() def test_annotation(self): - from pypy.annotation.annrpython import RPythonAnnotator - from pypy.annotation import model as annmodel + from rpython.annotator.annrpython import RPythonAnnotator + from rpython.annotator import model as annmodel def f(): return importing.make_compiled_pathname('abc/foo.py') a = RPythonAnnotator() From noreply at buildbot.pypy.org Mon Jan 21 22:36:53 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Mon, 21 Jan 2013 22:36:53 +0100 (CET) Subject: [pypy-commit] pypy py3k: func_code -> __code__ Message-ID: <20130121213653.D7B8E1C1102@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60319:0eaadba95e28 Date: 2013-01-21 13:36 -0800 http://bitbucket.org/pypy/pypy/changeset/0eaadba95e28/ Log: func_code -> __code__ diff --git a/lib-python/3.2/inspect.py b/lib-python/3.2/inspect.py --- a/lib-python/3.2/inspect.py +++ b/lib-python/3.2/inspect.py @@ -767,7 +767,7 @@ and 'varkw' are the names of the * and ** arguments or None.""" if not iscode(co): - if hasattr(len, 'func_code') and type(co) is type(len.func_code): + if hasattr(len, '__code__') and type(co) is type(len.func_code): # PyPy extension: built-in function objects have a func_code too. # There is no co_code on it, but co_argcount and co_varnames and # co_flags are present. From noreply at buildbot.pypy.org Mon Jan 21 22:52:47 2013 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 21 Jan 2013 22:52:47 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: fix translation Message-ID: <20130121215247.1A1C51C106A@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: missing-ndarray-attributes Changeset: r60320:0f922676e9c6 Date: 2012-12-31 11:24 +0200 http://bitbucket.org/pypy/pypy/changeset/0f922676e9c6/ Log: fix translation diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -14,7 +14,7 @@ from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rlib.rstruct.runpack import runpack from rpython.rlib.rstruct.nativefmttable import native_is_bigendian -from rpython.rlib.rstruct.ieee import (float_pack, float_unpack, pack_float, +from rpython.rlib.rstruct.ieee import (float_pack, float_unpack, pack_float80, unpack_float, unpack_float128) from rpython.tool.sourcetools import func_with_new_name from rpython.rlib import jit @@ -1569,7 +1569,7 @@ def byteswap(self, w_v): value = self.unbox(w_v) result = StringBuilder(12) - pack_float(result, value, 12, not native_is_bigendian) + pack_float80(result, value, 12, not native_is_bigendian) return self.box(unpack_float128(result.build(), native_is_bigendian)) class NonNativeFloat96(Float96): @@ -1602,7 +1602,7 @@ def byteswap(self, w_v): value = self.unbox(w_v) result = StringBuilder(16) - pack_float(result, value, 16, not native_is_bigendian) + pack_float80(result, value, 16, not native_is_bigendian) return self.box(unpack_float128(result.build(), native_is_bigendian)) class NonNativeFloat128(Float128): diff --git a/rpython/rlib/rstruct/ieee.py b/rpython/rlib/rstruct/ieee.py --- a/rpython/rlib/rstruct/ieee.py +++ b/rpython/rlib/rstruct/ieee.py @@ -239,18 +239,23 @@ @jit.unroll_safe +def pack_float80(result, x, size, be): + l = [] + unsigned = float_pack80(x) + for i in range(8): + l.append(chr((unsigned[0] >> (i * 8)) & 0xFF)) + for i in range(size - 8): + l.append(chr((unsigned[1] >> (i * 8)) & 0xFF)) + if be: + l.reverse() + result.append("".join(l)) + + at jit.unroll_safe def pack_float(result, x, size, be): l = [] - if size == 12 or size == 16: - unsigned = float_pack80(x) - for i in range(8): - l.append(chr((unsigned[0] >> (i * 8)) & 0xFF)) - for i in range(size - 8): - l.append(chr((unsigned[1] >> (i * 8)) & 0xFF)) - else: - unsigned = float_pack(x, size) - for i in range(size): - l.append(chr((unsigned >> (i * 8)) & 0xFF)) + unsigned = float_pack(x, size) + for i in range(size): + l.append(chr((unsigned >> (i * 8)) & 0xFF)) if be: l.reverse() result.append("".join(l)) From noreply at buildbot.pypy.org Mon Jan 21 23:00:48 2013 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 21 Jan 2013 23:00:48 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: back out changeset: 92d3270bf0ab Message-ID: <20130121220048.225171C106A@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: missing-ndarray-attributes Changeset: r60321:fd0b8b563e81 Date: 2013-01-21 23:54 +0200 http://bitbucket.org/pypy/pypy/changeset/fd0b8b563e81/ Log: back out changeset: 92d3270bf0ab diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py --- a/pypy/module/micronumpy/loop.py +++ b/pypy/module/micronumpy/loop.py @@ -173,44 +173,32 @@ y_iter.next() return out -axis_reduce1__driver = jit.JitDriver(name='numpy_axis_reduce', - greens=['shapelen', +axis_reduce__driver = jit.JitDriver(name='numpy_axis_reduce', + greens=['shapelen', 'cumultative', 'func', 'dtype', 'identity'], reds=['axis', 'arr', 'out', 'shape', 'out_iter', 'arr_iter', 'temp_iter']) -axis_reduce2__driver = jit.JitDriver(name='numpy_axis_reduce', - greens=['shapelen', - 'func', 'dtype', - 'identity'], - reds=['axis', 'arr', 'out', 'shape', - 'out_iter', 'arr_iter', - ]) def do_axis_reduce(shape, func, arr, dtype, axis, out, identity, cumultative, temp): + out_iter = out.create_axis_iter(arr.get_shape(), axis, cumultative) if cumultative: - return do_cum_axis_reduce(shape, func, arr, dtype, axis, out, - identity, temp) + temp_iter = temp.create_axis_iter(arr.get_shape(), axis, False) else: - return do_nocum_axis_reduce(shape, func, arr, dtype, axis, out, - identity, temp) - -def do_cum_axis_reduce(shape, func, arr, dtype, axis, out, identity, - temp): - out_iter = out.create_axis_iter(arr.get_shape(), axis, True) - temp_iter = temp.create_axis_iter(arr.get_shape(), axis, False) + temp_iter = out_iter # hack arr_iter = arr.create_iter() if identity is not None: identity = identity.convert_to(dtype) shapelen = len(shape) while not out_iter.done(): - axis_reduce1__driver.jit_merge_point(shapelen=shapelen, func=func, + axis_reduce__driver.jit_merge_point(shapelen=shapelen, func=func, dtype=dtype, identity=identity, axis=axis, arr=arr, out=out, shape=shape, out_iter=out_iter, arr_iter=arr_iter, + cumultative=cumultative, temp_iter=temp_iter) w_val = arr_iter.getitem().convert_to(dtype) if out_iter.first_line: @@ -220,34 +208,9 @@ cur = temp_iter.getitem() w_val = func(dtype, cur, w_val) out_iter.setitem(w_val) - temp_iter.setitem(w_val) - temp_iter.next() - arr_iter.next() - out_iter.next() - return out - -def do_nocum_axis_reduce(shape, func, arr, dtype, axis, out, identity, - temp): - out_iter = out.create_axis_iter(arr.get_shape(), axis, False) - arr_iter = arr.create_iter() - if identity is not None: - identity = identity.convert_to(dtype) - shapelen = len(shape) - while not out_iter.done(): - axis_reduce2__driver.jit_merge_point(shapelen=shapelen, func=func, - dtype=dtype, identity=identity, - axis=axis, arr=arr, out=out, - shape=shape, out_iter=out_iter, - arr_iter=arr_iter, - ) - w_val = arr_iter.getitem().convert_to(dtype) - if out_iter.first_line: - if identity is not None: - w_val = func(dtype, identity, w_val) - else: - cur = out_iter.getitem() - w_val = func(dtype, cur, w_val) - out_iter.setitem(w_val) + if cumultative: + temp_iter.setitem(w_val) + temp_iter.next() arr_iter.next() out_iter.next() return out From noreply at buildbot.pypy.org Mon Jan 21 23:00:49 2013 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 21 Jan 2013 23:00:49 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: use reds=auto to avoid hack Message-ID: <20130121220049.686B51C106A@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: missing-ndarray-attributes Changeset: r60322:d2e7ae35c850 Date: 2013-01-21 23:58 +0200 http://bitbucket.org/pypy/pypy/changeset/d2e7ae35c850/ Log: use reds=auto to avoid hack diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py --- a/pypy/module/micronumpy/loop.py +++ b/pypy/module/micronumpy/loop.py @@ -174,12 +174,10 @@ return out axis_reduce__driver = jit.JitDriver(name='numpy_axis_reduce', - greens=['shapelen', 'cumultative', + greens=['shapelen', 'func', 'dtype', 'identity'], - reds=['axis', 'arr', 'out', 'shape', - 'out_iter', 'arr_iter', - 'temp_iter']) + reds='auto') def do_axis_reduce(shape, func, arr, dtype, axis, out, identity, cumultative, temp): @@ -195,11 +193,7 @@ while not out_iter.done(): axis_reduce__driver.jit_merge_point(shapelen=shapelen, func=func, dtype=dtype, identity=identity, - axis=axis, arr=arr, out=out, - shape=shape, out_iter=out_iter, - arr_iter=arr_iter, - cumultative=cumultative, - temp_iter=temp_iter) + ) w_val = arr_iter.getitem().convert_to(dtype) if out_iter.first_line: if identity is not None: From noreply at buildbot.pypy.org Mon Jan 21 23:00:50 2013 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 21 Jan 2013 23:00:50 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: merge heads Message-ID: <20130121220050.A58321C106A@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: missing-ndarray-attributes Changeset: r60323:8045286b0e44 Date: 2013-01-22 00:00 +0200 http://bitbucket.org/pypy/pypy/changeset/8045286b0e44/ Log: merge heads diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py --- a/pypy/module/micronumpy/types.py +++ b/pypy/module/micronumpy/types.py @@ -14,7 +14,7 @@ from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rlib.rstruct.runpack import runpack from rpython.rlib.rstruct.nativefmttable import native_is_bigendian -from rpython.rlib.rstruct.ieee import (float_pack, float_unpack, pack_float, +from rpython.rlib.rstruct.ieee import (float_pack, float_unpack, pack_float80, unpack_float, unpack_float128) from rpython.tool.sourcetools import func_with_new_name from rpython.rlib import jit @@ -1569,7 +1569,7 @@ def byteswap(self, w_v): value = self.unbox(w_v) result = StringBuilder(12) - pack_float(result, value, 12, not native_is_bigendian) + pack_float80(result, value, 12, not native_is_bigendian) return self.box(unpack_float128(result.build(), native_is_bigendian)) class NonNativeFloat96(Float96): @@ -1602,7 +1602,7 @@ def byteswap(self, w_v): value = self.unbox(w_v) result = StringBuilder(16) - pack_float(result, value, 16, not native_is_bigendian) + pack_float80(result, value, 16, not native_is_bigendian) return self.box(unpack_float128(result.build(), native_is_bigendian)) class NonNativeFloat128(Float128): diff --git a/rpython/rlib/rstruct/ieee.py b/rpython/rlib/rstruct/ieee.py --- a/rpython/rlib/rstruct/ieee.py +++ b/rpython/rlib/rstruct/ieee.py @@ -239,18 +239,23 @@ @jit.unroll_safe +def pack_float80(result, x, size, be): + l = [] + unsigned = float_pack80(x) + for i in range(8): + l.append(chr((unsigned[0] >> (i * 8)) & 0xFF)) + for i in range(size - 8): + l.append(chr((unsigned[1] >> (i * 8)) & 0xFF)) + if be: + l.reverse() + result.append("".join(l)) + + at jit.unroll_safe def pack_float(result, x, size, be): l = [] - if size == 12 or size == 16: - unsigned = float_pack80(x) - for i in range(8): - l.append(chr((unsigned[0] >> (i * 8)) & 0xFF)) - for i in range(size - 8): - l.append(chr((unsigned[1] >> (i * 8)) & 0xFF)) - else: - unsigned = float_pack(x, size) - for i in range(size): - l.append(chr((unsigned >> (i * 8)) & 0xFF)) + unsigned = float_pack(x, size) + for i in range(size): + l.append(chr((unsigned >> (i * 8)) & 0xFF)) if be: l.reverse() result.append("".join(l)) From noreply at buildbot.pypy.org Mon Jan 21 23:02:30 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 23:02:30 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fix a few bugs, took a while Message-ID: <20130121220230.9EA231C106A@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60324:ad3465ae1d6c Date: 2013-01-22 00:02 +0200 http://bitbucket.org/pypy/pypy/changeset/ad3465ae1d6c/ Log: fix a few bugs, took a while diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -885,7 +885,8 @@ # copy frame-info data old_fi = oldlooptoken.compiled_loop_token.frame_info new_fi = newlooptoken.compiled_loop_token.frame_info - old_fi.jfi_frame_depth = new_fi.jfi_frame_depth + old_fi.jfi_frame_depth = max(old_fi.jfi_frame_depth, + new_fi.jfi_frame_depth) mc = codebuf.MachineCodeBlockWrapper() mc.JMP(imm(target)) if WORD == 4: # keep in sync with prepare_loop() @@ -2211,10 +2212,7 @@ # we need to allocate the frame, keep in sync with runner's # execute_token jd = descr.outermost_jitdriver_sd - if jd.index_of_virtualizable >= 0: - vloc = stack(jd.index_of_virtualizable, REF) - else: - vloc = imm(0) + base_ofs = self.cpu.get_baseofs_of_frame_field() self._emit_call(imm(descr._x86_function_addr), [argloc], 0, tmp=eax) if op.result is None: @@ -2244,6 +2242,13 @@ # Path A: use assembler_helper_adr assert jd is not None asm_helper_adr = self.cpu.cast_adr_to_int(jd.assembler_helper_adr) + if jd.index_of_virtualizable >= 0: + idx = jd.index_of_virtualizable + JITFRAME_FIXED_SIZE + self.mc.MOV(esi, mem(eax, base_ofs + idx * WORD)) + vloc = esi + else: + vloc = imm(0) + self._emit_call(imm(asm_helper_adr), [eax, vloc], 0, tmp=ecx) if IS_X86_32 and isinstance(result_loc, StackLoc) and result_loc.type == FLOAT: @@ -2264,7 +2269,7 @@ assert isinstance(fielddescr, FieldDescr) vtoken_ofs = fielddescr.offset vable_ofs = (jd.index_of_virtualizable + JITFRAME_FIXED_SIZE) * WORD - self.mc.MOV_rb(edx.value, vable_ofs) + self.mc.MOV_rm(edx.value, (eax.value, vable_ofs)) self.mc.MOV_mi((edx.value, vtoken_ofs), 0) # in the line above, TOKEN_NONE = 0 # diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -1316,8 +1316,9 @@ loc = self.loc(arg) assert loc is not ebp arglocs[i] = loc - if isinstance(loc, RegLoc): - self.fm.mark_as_free(arg) + # ARGH + #if isinstance(loc, RegLoc): + # self.fm.mark_as_free(arg) # # if we are too close to the start of the loop, the label's target may # get overridden by redirect_call_assembler(). (rare case) diff --git a/rpython/jit/metainterp/warmspot.py b/rpython/jit/metainterp/warmspot.py --- a/rpython/jit/metainterp/warmspot.py +++ b/rpython/jit/metainterp/warmspot.py @@ -1,6 +1,6 @@ import sys, py from rpython.tool.sourcetools import func_with_new_name -from rpython.rtyper.lltypesystem import lltype, llmemory +from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.rtyper.annlowlevel import llhelper, MixLevelHelperAnnotator,\ cast_base_ptr_to_instance, hlstr from rpython.annotator import model as annmodel From noreply at buildbot.pypy.org Mon Jan 21 23:04:59 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 21 Jan 2013 23:04:59 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fix Message-ID: <20130121220459.8249B1C106A@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60325:79796536c3ff Date: 2013-01-22 00:04 +0200 http://bitbucket.org/pypy/pypy/changeset/79796536c3ff/ Log: fix diff --git a/rpython/jit/backend/x86/test/test_runner.py b/rpython/jit/backend/x86/test/test_runner.py --- a/rpython/jit/backend/x86/test/test_runner.py +++ b/rpython/jit/backend/x86/test/test_runner.py @@ -31,7 +31,7 @@ # ====> ../../test/runner_test.py add_loop_instructions = ['mov', 'add', 'test', 'je', 'jmp'] - bridge_loop_instructions = ['cmp', 'jge', 'mov', 'push', 'mov', 'call', + bridge_loop_instructions = ['cmp', 'jge', 'mov', 'mov', 'mov', 'call', 'mov', 'jmp'] def setup_method(self, meth): From noreply at buildbot.pypy.org Mon Jan 21 23:41:47 2013 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 21 Jan 2013 23:41:47 +0100 (CET) Subject: [pypy-commit] pypy default: Finally found a workaround for test_datetime failures on linux64. Message-ID: <20130121224147.2F9811C00EC@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: Changeset: r60326:a57b37e858ae Date: 2013-01-21 22:40 +0100 http://bitbucket.org/pypy/pypy/changeset/a57b37e858ae/ Log: Finally found a workaround for test_datetime failures on linux64. I suspect a ll2ctypes caching issue. diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py --- a/pypy/module/cpyext/test/conftest.py +++ b/pypy/module/cpyext/test/conftest.py @@ -1,6 +1,15 @@ import py import pytest +def pytest_configure(config): + from pypy.tool.pytest.objspace import gettestobjspace + # For some reason (probably a ll2ctypes cache issue on linux64) + # it's necessary to run "import time" at least once before any + # other cpyext test, otherwise the same statement will fail in + # test_datetime.py. + space = gettestobjspace(usemodules=['rctime']) + space.getbuiltinmodule("time") + def pytest_ignore_collect(path, config): if config.option.runappdirect: return True # "cannot be run by py.test -A" From noreply at buildbot.pypy.org Mon Jan 21 23:41:48 2013 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 21 Jan 2013 23:41:48 +0100 (CET) Subject: [pypy-commit] pypy default: Fix a compiler crash when an ast tree contains the empty set literal Message-ID: <20130121224148.5FCBE1C106A@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: Changeset: r60327:0a0a78963ee3 Date: 2013-01-21 23:12 +0100 http://bitbucket.org/pypy/pypy/changeset/0a0a78963ee3/ Log: Fix a compiler crash when an ast tree contains the empty set literal (which cannot appear in Python source: {} is a dictionary) diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py --- a/pypy/interpreter/astcompiler/codegen.py +++ b/pypy/interpreter/astcompiler/codegen.py @@ -904,8 +904,9 @@ def visit_Set(self, s): self.update_position(s.lineno) + elt_count = len(s.elts) if s.elts is not None else 0 self.visit_sequence(s.elts) - self.emit_op_arg(ops.BUILD_SET, len(s.elts)) + self.emit_op_arg(ops.BUILD_SET, elt_count) def visit_Name(self, name): self.update_position(name.lineno) diff --git a/pypy/module/_ast/test/test_ast.py b/pypy/module/_ast/test/test_ast.py --- a/pypy/module/_ast/test/test_ast.py +++ b/pypy/module/_ast/test/test_ast.py @@ -290,6 +290,12 @@ ]) exec compile(body, '', 'exec') + def test_empty_set(self): + import ast + m = ast.Module(body=[ast.Expr(value=ast.Set(elts=[]))]) + ast.fix_missing_locations(m) + compile(m, "", "exec") + def test_invalid_sum(self): import _ast as ast pos = dict(lineno=2, col_offset=3) From noreply at buildbot.pypy.org Mon Jan 21 23:41:49 2013 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 21 Jan 2013 23:41:49 +0100 (CET) Subject: [pypy-commit] pypy default: cpyext: implement PyInstance_New (for oldstyle classes) Message-ID: <20130121224149.A096E1C00EC@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: Changeset: r60328:2dd5a00fac80 Date: 2013-01-21 23:39 +0100 http://bitbucket.org/pypy/pypy/changeset/2dd5a00fac80/ Log: cpyext: implement PyInstance_New (for oldstyle classes) diff --git a/pypy/module/cpyext/classobject.py b/pypy/module/cpyext/classobject.py --- a/pypy/module/cpyext/classobject.py +++ b/pypy/module/cpyext/classobject.py @@ -22,6 +22,12 @@ w_result.setdict(space, w_dict) return w_result + at cpython_api([PyObject, PyObject, PyObject], PyObject) +def PyInstance_New(space, w_cls, w_arg, w_kw): + """Create a new instance of a specific class. The parameters arg and kw are + used as the positional and keyword parameters to the object's constructor.""" + return space.call(w_cls, w_arg, w_kw) + @cpython_api([PyObject, PyObject], PyObject, error=CANNOT_FAIL) def _PyInstance_Lookup(space, w_instance, w_name): name = space.str_w(w_name) diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py --- a/pypy/module/cpyext/stubs.py +++ b/pypy/module/cpyext/stubs.py @@ -176,12 +176,6 @@ """Return true if klass is a subclass of base. Return false in all other cases.""" raise NotImplementedError - at cpython_api([PyObject, PyObject, PyObject], PyObject) -def PyInstance_New(space, cls, arg, kw): - """Create a new instance of a specific class. The parameters arg and kw are - used as the positional and keyword parameters to the object's constructor.""" - raise NotImplementedError - @cpython_api([PyObject], rffi.INT_real, error=-1) def PyCodec_Register(space, search_function): """Register a new codec search function. diff --git a/pypy/module/cpyext/test/test_classobject.py b/pypy/module/cpyext/test/test_classobject.py --- a/pypy/module/cpyext/test/test_classobject.py +++ b/pypy/module/cpyext/test/test_classobject.py @@ -7,8 +7,10 @@ w_class = space.appexec([], """(): class C: x = None - def __init__(self): + def __init__(self, *args, **kwargs): self.x = 1 + self.args = args + self.__dict__.update(kwargs) return C """) @@ -22,6 +24,13 @@ assert space.getattr(w_instance, space.wrap('x')) is space.w_None assert space.unwrap(space.getattr(w_instance, space.wrap('a'))) == 3 + w_instance = api.PyInstance_New(w_class, + space.wrap((3,)), space.wrap(dict(y=2))) + assert space.unwrap(space.getattr(w_instance, space.wrap('x'))) == 1 + assert space.unwrap(space.getattr(w_instance, space.wrap('y'))) == 2 + assert space.unwrap(space.getattr(w_instance, space.wrap('args'))) == (3,) + + def test_lookup(self, space, api): w_instance = space.appexec([], """(): class C: From noreply at buildbot.pypy.org Mon Jan 21 23:41:50 2013 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 21 Jan 2013 23:41:50 +0100 (CET) Subject: [pypy-commit] pypy default: Add an empty ceval.h, for the poor guys who don't know that CPython Message-ID: <20130121224150.C5E911C00EC@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: Changeset: r60329:b901441252e9 Date: 2013-01-21 23:41 +0100 http://bitbucket.org/pypy/pypy/changeset/b901441252e9/ Log: Add an empty ceval.h, for the poor guys who don't know that CPython already includes it in Python.h. diff --git a/pypy/module/cpyext/include/ceval.h b/pypy/module/cpyext/include/ceval.h new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/include/ceval.h @@ -0,0 +1,1 @@ +/* empty */ From noreply at buildbot.pypy.org Mon Jan 21 23:44:37 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Mon, 21 Jan 2013 23:44:37 +0100 (CET) Subject: [pypy-commit] pypy py3k: oops, more func_code -> __code__ (thanks amaury) Message-ID: <20130121224437.9AE7A1C00EC@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60330:dfc05d1615c3 Date: 2013-01-21 14:33 -0800 http://bitbucket.org/pypy/pypy/changeset/dfc05d1615c3/ Log: oops, more func_code -> __code__ (thanks amaury) diff --git a/lib-python/3.2/inspect.py b/lib-python/3.2/inspect.py --- a/lib-python/3.2/inspect.py +++ b/lib-python/3.2/inspect.py @@ -767,10 +767,10 @@ and 'varkw' are the names of the * and ** arguments or None.""" if not iscode(co): - if hasattr(len, '__code__') and type(co) is type(len.func_code): - # PyPy extension: built-in function objects have a func_code too. - # There is no co_code on it, but co_argcount and co_varnames and - # co_flags are present. + if hasattr(len, '__code__') and type(co) is type(len.__code__): + # PyPy extension: built-in function objects have a __code__ + # too. There is no co_code on it, but co_argcount and + # co_varnames and co_flags are present. pass else: raise TypeError('{!r} is not a code object'.format(co)) From noreply at buildbot.pypy.org Tue Jan 22 16:31:01 2013 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 22 Jan 2013 16:31:01 +0100 (CET) Subject: [pypy-commit] pypy default: update docs (bwesterb) Message-ID: <20130122153101.90C9A1C139E@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r60343:6018156230ed Date: 2013-01-22 17:10 +0200 http://bitbucket.org/pypy/pypy/changeset/6018156230ed/ Log: update docs (bwesterb) diff --git a/pypy/doc/arm.rst b/pypy/doc/arm.rst --- a/pypy/doc/arm.rst +++ b/pypy/doc/arm.rst @@ -130,15 +130,7 @@ export SB2=/srv/chroot/precise_arm export SB2OPT='-t ARM' -Once this is set, you can call the translator - -:: - - pypy ~/path_to_pypy_checkout/pypy/translator/goal/translate.py -O1 --platform=arm target.py - -If everything worked correctly this should yield an ARM binary. Running this binary in the ARM chroot or on an ARM device should produce the output ``"Hello World"``. - -.. _`this`: +Once this is set, you can call the translator. For example save this file :: @@ -148,3 +140,22 @@ def target(*args): return main, None + +and call the translator + +:: + + pypy ~/path_to_pypy_checkout/pypy/translator/goal/translate.py -O1 --platform=arm target.py + +If everything worked correctly this should yield an ARM binary. Running this binary in the ARM chroot or on an ARM device should produce the output ``"Hello World"``. + +To translate the full python pypy interpreter, you can cd into pypy/goal and call + +:: + + pypy /rpython/translator/goal/translate.py -O1 --platform=arm --gcrootfinder=shadowstack --jit-backend=arm targetpypystandalone.py + +The gcrootfinder option is needed to work around `issue 1377`_ and the jit-backend works around `issue 1376`_ + +.. _`issue 1377`: https://bugs.pypy.org/issue1377 +.. _`issue 1376`: https://bugs.pypy.org/issue1376 From noreply at buildbot.pypy.org Tue Jan 22 16:31:02 2013 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 22 Jan 2013 16:31:02 +0100 (CET) Subject: [pypy-commit] pypy default: update location of pypy-c binary Message-ID: <20130122153102.F393C1C139E@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r60344:b8ab553e4729 Date: 2013-01-22 17:13 +0200 http://bitbucket.org/pypy/pypy/changeset/b8ab553e4729/ Log: update location of pypy-c binary diff --git a/pypy/pytest-A.cfg b/pypy/pytest-A.cfg --- a/pypy/pytest-A.cfg +++ b/pypy/pytest-A.cfg @@ -1,5 +1,5 @@ cherrypick = ['interpreter', 'objspace/test', 'objspace/std', 'module'] -interp = ['translator/goal/pypy-c'] +interp = ['goal/pypy-c'] test_driver = ['test_all.py', '-A'] From noreply at buildbot.pypy.org Tue Jan 22 16:34:57 2013 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 22 Jan 2013 16:34:57 +0100 (CET) Subject: [pypy-commit] pypy default: fix (bwesterb) Message-ID: <20130122153457.F17831C13A0@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r60345:bcf6320e7a59 Date: 2013-01-22 17:34 +0200 http://bitbucket.org/pypy/pypy/changeset/bcf6320e7a59/ Log: fix (bwesterb) diff --git a/pypy/doc/arm.rst b/pypy/doc/arm.rst --- a/pypy/doc/arm.rst +++ b/pypy/doc/arm.rst @@ -149,11 +149,11 @@ If everything worked correctly this should yield an ARM binary. Running this binary in the ARM chroot or on an ARM device should produce the output ``"Hello World"``. -To translate the full python pypy interpreter, you can cd into pypy/goal and call +To translate the full python pypy interpreter with a jit, you can cd into pypy/goal and call :: - pypy /rpython/translator/goal/translate.py -O1 --platform=arm --gcrootfinder=shadowstack --jit-backend=arm targetpypystandalone.py + pypy /rpython/translator/goal/translate.py -Ojit --platform=arm --gcrootfinder=shadowstack --jit-backend=arm targetpypystandalone.py The gcrootfinder option is needed to work around `issue 1377`_ and the jit-backend works around `issue 1376`_ From noreply at buildbot.pypy.org Tue Jan 22 17:20:11 2013 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 22 Jan 2013 17:20:11 +0100 (CET) Subject: [pypy-commit] pypy default: add top level path to callable files Message-ID: <20130122162011.0E80C1C1368@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r60346:955fb7e3402f Date: 2013-01-22 18:19 +0200 http://bitbucket.org/pypy/pypy/changeset/955fb7e3402f/ Log: add top level path to callable files diff --git a/pypy/test_all.py b/pypy/test_all.py --- a/pypy/test_all.py +++ b/pypy/test_all.py @@ -23,7 +23,8 @@ if len(sys.argv) == 1 and os.path.dirname(sys.argv[0]) in '.': print >> sys.stderr, __doc__ sys.exit(2) - + #Add toplevel repository dir to sys.path + sys.path.insert(0,os.path.dirname(os.path.dirname(__file__))) import pytest import pytest_cov sys.exit(pytest.main(plugins=[pytest_cov])) diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -11,8 +11,10 @@ import shutil import sys +import os +#Add toplevel repository dir to sys.path +sys.path.insert(0,os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))) import py -import os import fnmatch from rpython.tool.udir import udir From noreply at buildbot.pypy.org Tue Jan 22 17:45:59 2013 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 22 Jan 2013 17:45:59 +0100 (CET) Subject: [pypy-commit] pypy default: skip arm tests on windows Message-ID: <20130122164559.89FC31C087A@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r60348:9526fc1a5a73 Date: 2013-01-22 18:44 +0200 http://bitbucket.org/pypy/pypy/changeset/9526fc1a5a73/ Log: skip arm tests on windows diff --git a/rpython/jit/backend/arm/test/support.py b/rpython/jit/backend/arm/test/support.py --- a/rpython/jit/backend/arm/test/support.py +++ b/rpython/jit/backend/arm/test/support.py @@ -14,6 +14,9 @@ def check_jumps(self, maxcount): pass +if not getattr(os, 'uname', None): + pytest.skip('cannot run arm tests on non-posix platform') + if os.uname()[1] == 'llaima.local': AS = '~/Code/arm-jit/android/android-ndk-r4b//build/prebuilt/darwin-x86/arm-eabi-4.4.0/arm-eabi/bin/as' else: From noreply at buildbot.pypy.org Tue Jan 22 18:12:42 2013 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 22 Jan 2013 18:12:42 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: use reds=auto, try to fix zjit tests Message-ID: <20130122171242.705AE1C134E@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: missing-ndarray-attributes Changeset: r60350:b859ed621be9 Date: 2013-01-01 04:48 +0200 http://bitbucket.org/pypy/pypy/changeset/b859ed621be9/ Log: use reds=auto, try to fix zjit tests diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -61,6 +61,7 @@ self.fields = fields self.fieldnames = fieldnames self.native = native + self.float_type = None @specialize.argtype(1) def box(self, value): @@ -172,7 +173,6 @@ return self.itemtype.get_element_size() class W_ComplexDtype(W_Dtype): - def __init__(self, itemtype, num, kind, name, char, w_box_type, alternate_constructors=[], aliases=[], fields=None, fieldnames=None, native=True, float_type=None): diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py --- a/pypy/module/micronumpy/loop.py +++ b/pypy/module/micronumpy/loop.py @@ -9,7 +9,7 @@ from rpython.rlib import jit from rpython.rtyper.lltypesystem import lltype, rffi from pypy.module.micronumpy.base import W_NDimArray -from pypy.module.micronumpy.iter import PureShapeIterator, ConcreteArrayIterator +from pypy.module.micronumpy.iter import PureShapeIterator from pypy.module.micronumpy import constants from pypy.module.micronumpy.support import int_w @@ -90,7 +90,7 @@ reduce_driver = jit.JitDriver(name='numpy_reduce', greens = ['shapelen', 'func', 'done_func', 'calc_dtype', 'identity'], - reds = ['obj', 'obj_iter', 'cur_value']) + reds = 'auto') def compute_reduce(obj, calc_dtype, func, done_func, identity): obj_iter = obj.create_iter() @@ -102,9 +102,9 @@ shapelen = len(obj.get_shape()) while not obj_iter.done(): reduce_driver.jit_merge_point(shapelen=shapelen, func=func, + done_func=done_func, calc_dtype=calc_dtype, identity=identity, - done_func=done_func, obj=obj, - obj_iter=obj_iter, cur_value=cur_value) + ) rval = obj_iter.getitem().convert_to(calc_dtype) if done_func is not None and done_func(calc_dtype, rval): return rval @@ -113,7 +113,7 @@ return cur_value reduce_cum_driver = jit.JitDriver(greens = ['shapelen', 'func', 'dtype'], - reds = ['obj_iter', 'out_iter']) + reds = 'auto') def compute_reduce_cumultative(obj, out, calc_dtype, func, identity): obj_iter = obj.create_iter() @@ -122,8 +122,8 @@ shapelen = len(obj.get_shape()) while not obj_iter.done(): reduce_cum_driver.jit_merge_point(shapelen=shapelen, func=func, - dtype=calc_dtype, obj_iter=obj_iter, - out_iter=out_iter) + dtype=calc_dtype, + ) rval = obj_iter.getitem().convert_to(calc_dtype) cur_value = func(calc_dtype, cur_value, rval) out_iter.setitem(cur_value) @@ -138,8 +138,7 @@ where_driver = jit.JitDriver(name='numpy_where', greens = ['shapelen', 'dtype', 'arr_dtype'], - reds = ['shape', 'arr', 'x', 'y','arr_iter', 'out', - 'x_iter', 'y_iter', 'iter', 'out_iter']) + reds = 'auto') def where(out, shape, arr, x, y, dtype): out_iter = out.create_iter(shape) @@ -157,10 +156,8 @@ shapelen = len(shape) while not iter.done(): where_driver.jit_merge_point(shapelen=shapelen, shape=shape, - dtype=dtype, iter=iter, x_iter=x_iter, - y_iter=y_iter, arr_iter=arr_iter, - arr=arr, x=x, y=y, arr_dtype=arr_dtype, - out_iter=out_iter, out=out) + dtype=dtype, arr_dtype=arr_dtype, + ) w_cond = arr_iter.getitem() if arr_dtype.itemtype.bool(w_cond): w_val = x_iter.getitem().convert_to(dtype) @@ -213,8 +210,7 @@ def _new_argmin_argmax(op_name): arg_driver = jit.JitDriver(name='numpy_' + op_name, greens = ['shapelen', 'dtype'], - reds = ['result', 'idx', 'cur_best', 'arr', - 'iter']) + reds = 'auto') def argmin_argmax(arr): result = 0 @@ -226,8 +222,7 @@ shapelen = len(arr.get_shape()) while not iter.done(): arg_driver.jit_merge_point(shapelen=shapelen, dtype=dtype, - result=result, idx=idx, - cur_best=cur_best, arr=arr, iter=iter) + ) w_val = iter.getitem() new_best = getattr(dtype.itemtype, op_name)(cur_best, w_val) if dtype.itemtype.ne(new_best, cur_best): @@ -243,7 +238,7 @@ # note that shapelen == 2 always dot_driver = jit.JitDriver(name = 'numpy_dot', greens = ['dtype'], - reds = ['outi', 'lefti', 'righti', 'result']) + reds = 'auto') def multidim_dot(space, left, right, result, dtype, right_critical_dim): ''' assumes left, right are concrete arrays @@ -270,8 +265,7 @@ lefti = left.create_dot_iter(broadcast_shape, left_skip) righti = right.create_dot_iter(broadcast_shape, right_skip) while not outi.done(): - dot_driver.jit_merge_point(dtype=dtype, outi=outi, lefti=lefti, - righti=righti, result=result) + dot_driver.jit_merge_point(dtype=dtype) lval = lefti.getitem().convert_to(dtype) rval = righti.getitem().convert_to(dtype) outval = outi.getitem().convert_to(dtype) @@ -285,7 +279,7 @@ count_all_true_driver = jit.JitDriver(name = 'numpy_count', greens = ['shapelen', 'dtype'], - reds = ['s', 'iter']) + reds = 'auto') def count_all_true(arr): s = 0 @@ -295,8 +289,7 @@ shapelen = len(arr.get_shape()) dtype = arr.get_dtype() while not iter.done(): - count_all_true_driver.jit_merge_point(shapelen=shapelen, iter=iter, - s=s, dtype=dtype) + count_all_true_driver.jit_merge_point(shapelen=shapelen, dtype=dtype) s += iter.getitem_bool() iter.next() return s @@ -304,8 +297,7 @@ getitem_filter_driver = jit.JitDriver(name = 'numpy_getitem_bool', greens = ['shapelen', 'arr_dtype', 'index_dtype'], - reds = ['res', 'index_iter', 'res_iter', - 'arr_iter']) + reds = 'auto') def getitem_filter(res, arr, index): res_iter = res.create_iter() @@ -319,9 +311,7 @@ getitem_filter_driver.jit_merge_point(shapelen=shapelen, index_dtype=index_dtype, arr_dtype=arr_dtype, - res=res, index_iter=index_iter, - res_iter=res_iter, - arr_iter=arr_iter) + ) if index_iter.getitem_bool(): res_iter.setitem(arr_iter.getitem()) res_iter.next() @@ -332,8 +322,7 @@ setitem_filter_driver = jit.JitDriver(name = 'numpy_setitem_bool', greens = ['shapelen', 'arr_dtype', 'index_dtype'], - reds = ['index_iter', 'value_iter', - 'arr_iter']) + reds = 'auto') def setitem_filter(arr, index, value): arr_iter = arr.create_iter() @@ -346,9 +335,7 @@ setitem_filter_driver.jit_merge_point(shapelen=shapelen, index_dtype=index_dtype, arr_dtype=arr_dtype, - index_iter=index_iter, - value_iter=value_iter, - arr_iter=arr_iter) + ) if index_iter.getitem_bool(): arr_iter.setitem(value_iter.getitem()) value_iter.next() @@ -357,16 +344,13 @@ flatiter_getitem_driver = jit.JitDriver(name = 'numpy_flatiter_getitem', greens = ['dtype'], - reds = ['step', 'ri', 'res', - 'base_iter']) + reds = 'auto') def flatiter_getitem(res, base_iter, step): ri = res.create_iter() dtype = res.get_dtype() while not ri.done(): - flatiter_getitem_driver.jit_merge_point(dtype=dtype, - base_iter=base_iter, - ri=ri, res=res, step=step) + flatiter_getitem_driver.jit_merge_point(dtype=dtype) ri.setitem(base_iter.getitem()) base_iter.next_skip_x(step) ri.next() @@ -374,8 +358,7 @@ flatiter_setitem_driver = jit.JitDriver(name = 'numpy_flatiter_setitem', greens = ['dtype'], - reds = ['length', 'step', 'arr_iter', - 'val_iter']) + reds = 'auto') def flatiter_setitem(arr, val, start, step, length): dtype = arr.get_dtype() @@ -383,9 +366,7 @@ val_iter = val.create_iter() arr_iter.next_skip_x(start) while length > 0: - flatiter_setitem_driver.jit_merge_point(dtype=dtype, length=length, - step=step, arr_iter=arr_iter, - val_iter=val_iter) + flatiter_setitem_driver.jit_merge_point(dtype=dtype) arr_iter.setitem(val_iter.getitem().convert_to(dtype)) # need to repeat i_nput values until all assignments are done arr_iter.next_skip_x(step) @@ -396,14 +377,13 @@ fromstring_driver = jit.JitDriver(name = 'numpy_fromstring', greens = ['itemsize', 'dtype'], - reds = ['i', 's', 'ai']) + reds = 'auto') def fromstring_loop(a, dtype, itemsize, s): i = 0 ai = a.create_iter() while not ai.done(): - fromstring_driver.jit_merge_point(dtype=dtype, s=s, ai=ai, i=i, - itemsize=itemsize) + fromstring_driver.jit_merge_point(dtype=dtype, itemsize=itemsize) val = dtype.itemtype.runpack_str(s[i*itemsize:i*itemsize + itemsize]) ai.setitem(val) ai.next() @@ -426,8 +406,7 @@ getitem_int_driver = jit.JitDriver(name = 'numpy_getitem_int', greens = ['shapelen', 'indexlen', 'prefixlen', 'dtype'], - reds = ['arr', 'res', 'iter', 'indexes_w', - 'prefix_w']) + reds = 'auto') def getitem_array_int(space, arr, res, iter_shape, indexes_w, prefix_w): shapelen = len(iter_shape) @@ -438,10 +417,7 @@ indexlen = len(indexes_w) while not iter.done(): getitem_int_driver.jit_merge_point(shapelen=shapelen, indexlen=indexlen, - dtype=dtype, arr=arr, res=res, - iter=iter, indexes_w=indexes_w, - prefix_w=prefix_w, - prefixlen=prefixlen) + dtype=dtype, prefixlen=prefixlen) # prepare the index index_w = [None] * indexlen for i in range(indexlen): @@ -458,8 +434,7 @@ setitem_int_driver = jit.JitDriver(name = 'numpy_setitem_int', greens = ['shapelen', 'indexlen', 'prefixlen', 'dtype'], - reds = ['arr', 'iter', 'indexes_w', - 'prefix_w', 'val_arr']) + reds = 'auto') def setitem_array_int(space, arr, iter_shape, indexes_w, val_arr, prefix_w): @@ -470,10 +445,7 @@ iter = PureShapeIterator(iter_shape, indexes_w) while not iter.done(): setitem_int_driver.jit_merge_point(shapelen=shapelen, indexlen=indexlen, - dtype=dtype, arr=arr, - iter=iter, indexes_w=indexes_w, - prefix_w=prefix_w, val_arr=val_arr, - prefixlen=prefixlen) + dtype=dtype, prefixlen=prefixlen) # prepare the index index_w = [None] * indexlen for i in range(indexlen): @@ -488,35 +460,32 @@ iter.next() copy_from_to_driver = jit.JitDriver(greens = ['dtype'], - reds = ['from_iter', 'to_iter']) + reds = 'auto') def copy_from_to(from_, to, dtype): from_iter = from_.create_iter() to_iter = to.create_iter() while not from_iter.done(): - copy_from_to_driver.jit_merge_point(dtype=dtype, from_iter=from_iter, - to_iter=to_iter) + copy_from_to_driver.jit_merge_point(dtype=dtype) to_iter.setitem(from_iter.getitem().convert_to(dtype)) to_iter.next() from_iter.next() byteswap_driver = jit.JitDriver(greens = ['dtype'], - reds = ['from_iter', 'to_iter']) + reds = 'auto') def byteswap(from_, to): dtype = from_.dtype from_iter = from_.create_iter() to_iter = to.create_iter() while not from_iter.done(): - byteswap_driver.jit_merge_point(dtype=dtype, from_iter=from_iter, - to_iter=to_iter) + byteswap_driver.jit_merge_point(dtype=dtype) to_iter.setitem(dtype.itemtype.byteswap(from_iter.getitem())) to_iter.next() from_iter.next() choose_driver = jit.JitDriver(greens = ['shapelen', 'mode', 'dtype'], - reds = ['shape', 'iterators', 'arr_iter', - 'out_iter']) + reds = 'auto') def choose(space, arr, choices, shape, dtype, out, mode): shapelen = len(shape) @@ -525,9 +494,7 @@ out_iter = out.create_iter(shape) while not arr_iter.done(): choose_driver.jit_merge_point(shapelen=shapelen, dtype=dtype, - mode=mode, shape=shape, - iterators=iterators, arr_iter=arr_iter, - out_iter=out_iter) + mode=mode) index = int_w(space, arr_iter.getitem()) if index < 0 or index >= len(iterators): if mode == constants.MODE_RAISE: @@ -548,8 +515,7 @@ arr_iter.next() clip_driver = jit.JitDriver(greens = ['shapelen', 'dtype'], - reds = ['min_iter', 'max_iter', 'arr_iter', - 'out_iter']) + reds = 'auto') def clip(space, arr, shape, min, max, out): arr_iter = arr.create_iter(shape) @@ -559,9 +525,7 @@ max_iter = max.create_iter(shape) out_iter = out.create_iter(shape) while not arr_iter.done(): - clip_driver.jit_merge_point(shapelen=shapelen, dtype=dtype, - min_iter=min_iter, max_iter=max_iter, - arr_iter=arr_iter, out_iter=out_iter) + clip_driver.jit_merge_point(shapelen=shapelen, dtype=dtype) w_v = arr_iter.getitem().convert_to(dtype) w_min = min_iter.getitem().convert_to(dtype) w_max = max_iter.getitem().convert_to(dtype) @@ -576,17 +540,14 @@ min_iter.next() diagonal_simple_driver = jit.JitDriver(greens = ['axis1', 'axis2'], - reds = ['i', 'offset', 'out_iter', - 'arr']) + reds = 'auto') def diagonal_simple(space, arr, out, offset, axis1, axis2, size): out_iter = out.create_iter() i = 0 index = [0] * 2 while i < size: - diagonal_simple_driver.jit_merge_point(axis1=axis1, axis2=axis2, - out_iter=out_iter, offset=offset, - i=i, arr=arr) + diagonal_simple_driver.jit_merge_point(axis1=axis1, axis2=axis2) index[axis1] = i index[axis2] = i + offset out_iter.setitem(arr.getitem_index(space, index)) diff --git a/pypy/module/micronumpy/test/test_zjit.py b/pypy/module/micronumpy/test/test_zjit.py --- a/pypy/module/micronumpy/test/test_zjit.py +++ b/pypy/module/micronumpy/test/test_zjit.py @@ -65,8 +65,9 @@ self.__class__.graph = graph reset_stats() pyjitpl._warmrunnerdesc.memory_manager.alive_loops.clear() + retval = self.interp.eval_graph(self.graph, [i]) py.test.skip("don't run for now") - return self.interp.eval_graph(self.graph, [i]) + return retval def define_add(): return """ @@ -290,6 +291,7 @@ """ def test_take(self): + skip('"take" not implmenented yet') result = self.run("take") assert result == 3 self.check_simple_loop({'raw_load': 2, From noreply at buildbot.pypy.org Tue Jan 22 22:12:45 2013 From: noreply at buildbot.pypy.org (rlamy) Date: Tue, 22 Jan 2013 22:12:45 +0100 (CET) Subject: [pypy-commit] pypy default: move translatorshell.py to rpython/ Message-ID: <20130122211245.624531C05CA@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: Changeset: r60351:9b998e123968 Date: 2013-01-22 21:12 +0000 http://bitbucket.org/pypy/pypy/changeset/9b998e123968/ Log: move translatorshell.py to rpython/ diff --git a/pypy/bin/translatorshell.py b/rpython/bin/translatorshell.py rename from pypy/bin/translatorshell.py rename to rpython/bin/translatorshell.py From noreply at buildbot.pypy.org Tue Jan 22 22:53:13 2013 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 22 Jan 2013 22:53:13 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: oops Message-ID: <20130122215313.2D71B1C05CA@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: missing-ndarray-attributes Changeset: r60352:ecab849d6362 Date: 2013-01-01 09:29 +0200 http://bitbucket.org/pypy/pypy/changeset/ecab849d6362/ Log: oops diff --git a/pypy/module/micronumpy/loop.py b/pypy/module/micronumpy/loop.py --- a/pypy/module/micronumpy/loop.py +++ b/pypy/module/micronumpy/loop.py @@ -155,9 +155,8 @@ iter = x_iter shapelen = len(shape) while not iter.done(): - where_driver.jit_merge_point(shapelen=shapelen, shape=shape, - dtype=dtype, arr_dtype=arr_dtype, - ) + where_driver.jit_merge_point(shapelen=shapelen, dtype=dtype, + arr_dtype=arr_dtype) w_cond = arr_iter.getitem() if arr_dtype.itemtype.bool(w_cond): w_val = x_iter.getitem().convert_to(dtype) From noreply at buildbot.pypy.org Tue Jan 22 22:58:53 2013 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 22 Jan 2013 22:58:53 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: merge default into branch Message-ID: <20130122215853.77F1A1C05CA@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: missing-ndarray-attributes Changeset: r60353:e22c2ae3da06 Date: 2013-01-22 23:58 +0200 http://bitbucket.org/pypy/pypy/changeset/e22c2ae3da06/ Log: merge default into branch diff --git a/pypy/doc/arm.rst b/pypy/doc/arm.rst --- a/pypy/doc/arm.rst +++ b/pypy/doc/arm.rst @@ -130,15 +130,7 @@ export SB2=/srv/chroot/precise_arm export SB2OPT='-t ARM' -Once this is set, you can call the translator - -:: - - pypy ~/path_to_pypy_checkout/pypy/translator/goal/translate.py -O1 --platform=arm target.py - -If everything worked correctly this should yield an ARM binary. Running this binary in the ARM chroot or on an ARM device should produce the output ``"Hello World"``. - -.. _`this`: +Once this is set, you can call the translator. For example save this file :: @@ -148,3 +140,22 @@ def target(*args): return main, None + +and call the translator + +:: + + pypy ~/path_to_pypy_checkout/pypy/translator/goal/translate.py -O1 --platform=arm target.py + +If everything worked correctly this should yield an ARM binary. Running this binary in the ARM chroot or on an ARM device should produce the output ``"Hello World"``. + +To translate the full python pypy interpreter with a jit, you can cd into pypy/goal and call + +:: + + pypy /rpython/translator/goal/translate.py -Ojit --platform=arm --gcrootfinder=shadowstack --jit-backend=arm targetpypystandalone.py + +The gcrootfinder option is needed to work around `issue 1377`_ and the jit-backend works around `issue 1376`_ + +.. _`issue 1377`: https://bugs.pypy.org/issue1377 +.. _`issue 1376`: https://bugs.pypy.org/issue1376 diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py --- a/pypy/interpreter/astcompiler/codegen.py +++ b/pypy/interpreter/astcompiler/codegen.py @@ -904,8 +904,9 @@ def visit_Set(self, s): self.update_position(s.lineno) + elt_count = len(s.elts) if s.elts is not None else 0 self.visit_sequence(s.elts) - self.emit_op_arg(ops.BUILD_SET, len(s.elts)) + self.emit_op_arg(ops.BUILD_SET, elt_count) def visit_Name(self, name): self.update_position(name.lineno) diff --git a/pypy/module/_ast/test/test_ast.py b/pypy/module/_ast/test/test_ast.py --- a/pypy/module/_ast/test/test_ast.py +++ b/pypy/module/_ast/test/test_ast.py @@ -290,6 +290,12 @@ ]) exec compile(body, '', 'exec') + def test_empty_set(self): + import ast + m = ast.Module(body=[ast.Expr(value=ast.Set(elts=[]))]) + ast.fix_missing_locations(m) + compile(m, "", "exec") + def test_invalid_sum(self): import _ast as ast pos = dict(lineno=2, col_offset=3) diff --git a/pypy/module/cpyext/classobject.py b/pypy/module/cpyext/classobject.py --- a/pypy/module/cpyext/classobject.py +++ b/pypy/module/cpyext/classobject.py @@ -22,6 +22,12 @@ w_result.setdict(space, w_dict) return w_result + at cpython_api([PyObject, PyObject, PyObject], PyObject) +def PyInstance_New(space, w_cls, w_arg, w_kw): + """Create a new instance of a specific class. The parameters arg and kw are + used as the positional and keyword parameters to the object's constructor.""" + return space.call(w_cls, w_arg, w_kw) + @cpython_api([PyObject, PyObject], PyObject, error=CANNOT_FAIL) def _PyInstance_Lookup(space, w_instance, w_name): name = space.str_w(w_name) diff --git a/pypy/module/cpyext/include/ceval.h b/pypy/module/cpyext/include/ceval.h new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/include/ceval.h @@ -0,0 +1,1 @@ +/* empty */ diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py --- a/pypy/module/cpyext/stubs.py +++ b/pypy/module/cpyext/stubs.py @@ -176,12 +176,6 @@ """Return true if klass is a subclass of base. Return false in all other cases.""" raise NotImplementedError - at cpython_api([PyObject, PyObject, PyObject], PyObject) -def PyInstance_New(space, cls, arg, kw): - """Create a new instance of a specific class. The parameters arg and kw are - used as the positional and keyword parameters to the object's constructor.""" - raise NotImplementedError - @cpython_api([PyObject], rffi.INT_real, error=-1) def PyCodec_Register(space, search_function): """Register a new codec search function. diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py --- a/pypy/module/cpyext/test/conftest.py +++ b/pypy/module/cpyext/test/conftest.py @@ -1,6 +1,15 @@ import py import pytest +def pytest_configure(config): + from pypy.tool.pytest.objspace import gettestobjspace + # For some reason (probably a ll2ctypes cache issue on linux64) + # it's necessary to run "import time" at least once before any + # other cpyext test, otherwise the same statement will fail in + # test_datetime.py. + space = gettestobjspace(usemodules=['rctime']) + space.getbuiltinmodule("time") + def pytest_ignore_collect(path, config): if config.option.runappdirect: return True # "cannot be run by py.test -A" diff --git a/pypy/module/cpyext/test/test_classobject.py b/pypy/module/cpyext/test/test_classobject.py --- a/pypy/module/cpyext/test/test_classobject.py +++ b/pypy/module/cpyext/test/test_classobject.py @@ -7,8 +7,10 @@ w_class = space.appexec([], """(): class C: x = None - def __init__(self): + def __init__(self, *args, **kwargs): self.x = 1 + self.args = args + self.__dict__.update(kwargs) return C """) @@ -22,6 +24,13 @@ assert space.getattr(w_instance, space.wrap('x')) is space.w_None assert space.unwrap(space.getattr(w_instance, space.wrap('a'))) == 3 + w_instance = api.PyInstance_New(w_class, + space.wrap((3,)), space.wrap(dict(y=2))) + assert space.unwrap(space.getattr(w_instance, space.wrap('x'))) == 1 + assert space.unwrap(space.getattr(w_instance, space.wrap('y'))) == 2 + assert space.unwrap(space.getattr(w_instance, space.wrap('args'))) == (3,) + + def test_lookup(self, space, api): w_instance = space.appexec([], """(): class C: diff --git a/pypy/pytest-A.cfg b/pypy/pytest-A.cfg --- a/pypy/pytest-A.cfg +++ b/pypy/pytest-A.cfg @@ -1,5 +1,5 @@ cherrypick = ['interpreter', 'objspace/test', 'objspace/std', 'module'] -interp = ['translator/goal/pypy-c'] +interp = ['goal/pypy-c'] test_driver = ['test_all.py', '-A'] diff --git a/pypy/test_all.py b/pypy/test_all.py --- a/pypy/test_all.py +++ b/pypy/test_all.py @@ -23,7 +23,8 @@ if len(sys.argv) == 1 and os.path.dirname(sys.argv[0]) in '.': print >> sys.stderr, __doc__ sys.exit(2) - + #Add toplevel repository dir to sys.path + sys.path.insert(0,os.path.dirname(os.path.dirname(__file__))) import pytest import pytest_cov sys.exit(pytest.main(plugins=[pytest_cov])) diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -11,8 +11,10 @@ import shutil import sys +import os +#Add toplevel repository dir to sys.path +sys.path.insert(0,os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))) import py -import os import fnmatch from rpython.tool.udir import udir diff --git a/pypy/bin/translatorshell.py b/rpython/bin/translatorshell.py rename from pypy/bin/translatorshell.py rename to rpython/bin/translatorshell.py diff --git a/rpython/jit/backend/arm/test/support.py b/rpython/jit/backend/arm/test/support.py --- a/rpython/jit/backend/arm/test/support.py +++ b/rpython/jit/backend/arm/test/support.py @@ -14,6 +14,9 @@ def check_jumps(self, maxcount): pass +if not getattr(os, 'uname', None): + pytest.skip('cannot run arm tests on non-posix platform') + if os.uname()[1] == 'llaima.local': AS = '~/Code/arm-jit/android/android-ndk-r4b//build/prebuilt/darwin-x86/arm-eabi-4.4.0/arm-eabi/bin/as' else: From noreply at buildbot.pypy.org Wed Jan 23 03:01:30 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Wed, 23 Jan 2013 03:01:30 +0100 (CET) Subject: [pypy-commit] pypy default: be less strict about tm_wday to match cpython Message-ID: <20130123020130.7845C1C134E@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: Changeset: r60354:e4fa0b2b9d64 Date: 2013-01-22 17:30 -0800 http://bitbucket.org/pypy/pypy/changeset/e4fa0b2b9d64/ Log: be less strict about tm_wday to match cpython diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py --- a/pypy/module/rctime/interp_time.py +++ b/pypy/module/rctime/interp_time.py @@ -448,10 +448,6 @@ raise OperationError(space.w_ValueError, space.wrap("year out of range")) - if rffi.getintfield(glob_buf, 'c_tm_wday') < 0: - raise OperationError(space.w_ValueError, - space.wrap("day of week out of range")) - rffi.setintfield(glob_buf, 'c_tm_year', y - 1900) rffi.setintfield(glob_buf, 'c_tm_mon', rffi.getintfield(glob_buf, 'c_tm_mon') - 1) @@ -460,6 +456,12 @@ rffi.setintfield(glob_buf, 'c_tm_yday', rffi.getintfield(glob_buf, 'c_tm_yday') - 1) + # tm_wday does not need checking of its upper-bound since taking "% + # 7" in gettmarg() automatically restricts the range. + if rffi.getintfield(glob_buf, 'c_tm_wday') < 0: + raise OperationError(space.w_ValueError, + space.wrap("day of week out of range")) + return glob_buf def time(space): diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py --- a/pypy/module/rctime/test/test_rctime.py +++ b/pypy/module/rctime/test/test_rctime.py @@ -113,6 +113,9 @@ if os.name != 'nt': assert rctime.mktime(rctime.localtime(-1)) == -1 + res = rctime.mktime((2000, 1, 1, 0, 0, 0, -1, -1, -1)) + assert rctime.ctime(res) == 'Sat Jan 1 00:00:00 2000' + def test_asctime(self): import time as rctime rctime.asctime() From noreply at buildbot.pypy.org Wed Jan 23 03:11:15 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Wed, 23 Jan 2013 03:11:15 +0100 (CET) Subject: [pypy-commit] pypy py3k: merge default Message-ID: <20130123021115.8E12A1C05CA@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60355:99491555f303 Date: 2013-01-22 18:10 -0800 http://bitbucket.org/pypy/pypy/changeset/99491555f303/ Log: merge default diff --git a/pypy/doc/arm.rst b/pypy/doc/arm.rst --- a/pypy/doc/arm.rst +++ b/pypy/doc/arm.rst @@ -130,15 +130,7 @@ export SB2=/srv/chroot/precise_arm export SB2OPT='-t ARM' -Once this is set, you can call the translator - -:: - - pypy ~/path_to_pypy_checkout/pypy/translator/goal/translate.py -O1 --platform=arm target.py - -If everything worked correctly this should yield an ARM binary. Running this binary in the ARM chroot or on an ARM device should produce the output ``"Hello World"``. - -.. _`this`: +Once this is set, you can call the translator. For example save this file :: @@ -148,3 +140,22 @@ def target(*args): return main, None + +and call the translator + +:: + + pypy ~/path_to_pypy_checkout/pypy/translator/goal/translate.py -O1 --platform=arm target.py + +If everything worked correctly this should yield an ARM binary. Running this binary in the ARM chroot or on an ARM device should produce the output ``"Hello World"``. + +To translate the full python pypy interpreter with a jit, you can cd into pypy/goal and call + +:: + + pypy /rpython/translator/goal/translate.py -Ojit --platform=arm --gcrootfinder=shadowstack --jit-backend=arm targetpypystandalone.py + +The gcrootfinder option is needed to work around `issue 1377`_ and the jit-backend works around `issue 1376`_ + +.. _`issue 1377`: https://bugs.pypy.org/issue1377 +.. _`issue 1376`: https://bugs.pypy.org/issue1376 diff --git a/pypy/goal/__init__.py b/pypy/goal/__init__.py new file mode 100644 --- /dev/null +++ b/pypy/goal/__init__.py @@ -0,0 +1,1 @@ +#empty diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py --- a/pypy/interpreter/astcompiler/codegen.py +++ b/pypy/interpreter/astcompiler/codegen.py @@ -985,8 +985,9 @@ def visit_Set(self, s): self.update_position(s.lineno) + elt_count = len(s.elts) if s.elts is not None else 0 self.visit_sequence(s.elts) - self.emit_op_arg(ops.BUILD_SET, len(s.elts)) + self.emit_op_arg(ops.BUILD_SET, elt_count) def visit_Name(self, name): self.update_position(name.lineno) diff --git a/pypy/interpreter/test2/test_targetpypy.py b/pypy/interpreter/test2/test_targetpypy.py --- a/pypy/interpreter/test2/test_targetpypy.py +++ b/pypy/interpreter/test2/test_targetpypy.py @@ -1,6 +1,6 @@ import py -from goal.targetpypystandalone import get_entry_point +from pypy.goal.targetpypystandalone import get_entry_point from pypy.config.pypyoption import get_pypy_config class TestTargetPyPy(object): diff --git a/pypy/module/_ast/test/test_ast.py b/pypy/module/_ast/test/test_ast.py --- a/pypy/module/_ast/test/test_ast.py +++ b/pypy/module/_ast/test/test_ast.py @@ -282,6 +282,12 @@ ]) exec(compile(body, '', 'exec')) + def test_empty_set(self): + import ast + m = ast.Module(body=[ast.Expr(value=ast.Set(elts=[]))]) + ast.fix_missing_locations(m) + compile(m, "", "exec") + def test_invalid_sum(self): import _ast as ast pos = dict(lineno=2, col_offset=3) diff --git a/pypy/module/cpyext/include/ceval.h b/pypy/module/cpyext/include/ceval.h new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/include/ceval.h @@ -0,0 +1,1 @@ +/* empty */ diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py --- a/pypy/module/cpyext/test/conftest.py +++ b/pypy/module/cpyext/test/conftest.py @@ -1,6 +1,15 @@ import py import pytest +def pytest_configure(config): + from pypy.tool.pytest.objspace import gettestobjspace + # For some reason (probably a ll2ctypes cache issue on linux64) + # it's necessary to run "import time" at least once before any + # other cpyext test, otherwise the same statement will fail in + # test_datetime.py. + space = gettestobjspace(usemodules=['rctime']) + space.getbuiltinmodule("time") + def pytest_ignore_collect(path, config): if config.option.runappdirect: return True # "cannot be run by py.test -A" diff --git a/pypy/module/micronumpy/arrayimpl/scalar.py b/pypy/module/micronumpy/arrayimpl/scalar.py --- a/pypy/module/micronumpy/arrayimpl/scalar.py +++ b/pypy/module/micronumpy/arrayimpl/scalar.py @@ -34,6 +34,9 @@ def get_shape(self): return [] + def get_strides(self): + return [] + def create_iter(self, shape=None): return ScalarIterator(self.value) diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -40,6 +40,10 @@ self.implementation = self.implementation.set_shape(space, get_shape_from_iterable(space, self.get_size(), w_new_shape)) + def descr_get_strides(self, space): + strides = self.implementation.get_strides() + return space.newtuple([space.wrap(i) for i in strides]) + def get_dtype(self): return self.implementation.dtype @@ -645,6 +649,7 @@ dtype = GetSetProperty(W_NDimArray.descr_get_dtype), shape = GetSetProperty(W_NDimArray.descr_get_shape, W_NDimArray.descr_set_shape), + strides = GetSetProperty(W_NDimArray.descr_get_strides), ndim = GetSetProperty(W_NDimArray.descr_get_ndim), size = GetSetProperty(W_NDimArray.descr_get_size), itemsize = GetSetProperty(W_NDimArray.descr_get_itemsize), diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -684,6 +684,18 @@ assert a.reshape([1]).shape == (1,) raises(ValueError, "a.reshape(3)") + def test_strides(self): + from _numpypy import array + a = array([[1.0, 2.0], + [3.0, 4.0]]) + assert a.strides == (16, 8) + assert a[1:].strides == (16, 8) + + def test_strides_scalar(self): + from _numpypy import array + a = array(42) + assert a.strides == () + def test_add(self): from _numpypy import array a = array(range(5)) diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py --- a/pypy/module/rctime/interp_time.py +++ b/pypy/module/rctime/interp_time.py @@ -447,10 +447,6 @@ space.warn("Century info guessed for a 2-digit year.", space.w_DeprecationWarning) - if rffi.getintfield(glob_buf, 'c_tm_wday') < 0: - raise OperationError(space.w_ValueError, - space.wrap("day of week out of range")) - rffi.setintfield(glob_buf, 'c_tm_year', y - 1900) rffi.setintfield(glob_buf, 'c_tm_mon', rffi.getintfield(glob_buf, 'c_tm_mon') - 1) @@ -459,6 +455,12 @@ rffi.setintfield(glob_buf, 'c_tm_yday', rffi.getintfield(glob_buf, 'c_tm_yday') - 1) + # tm_wday does not need checking of its upper-bound since taking "% + # 7" in gettmarg() automatically restricts the range. + if rffi.getintfield(glob_buf, 'c_tm_wday') < 0: + raise OperationError(space.w_ValueError, + space.wrap("day of week out of range")) + return glob_buf def time(space): diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py --- a/pypy/module/rctime/test/test_rctime.py +++ b/pypy/module/rctime/test/test_rctime.py @@ -113,6 +113,9 @@ if os.name != 'nt': assert rctime.mktime(rctime.localtime(-1)) == -1 + res = rctime.mktime((2000, 1, 1, 0, 0, 0, -1, -1, -1)) + assert rctime.ctime(res) == 'Sat Jan 1 00:00:00 2000' + def test_asctime(self): import time as rctime rctime.asctime() diff --git a/pypy/module/unicodedata/test/test_unicodedata.py b/pypy/module/unicodedata/test/test_unicodedata.py new file mode 100644 --- /dev/null +++ b/pypy/module/unicodedata/test/test_unicodedata.py @@ -0,0 +1,105 @@ +import py +import sys + +class AppTestUnicodeData: + spaceconfig = dict(usemodules=('unicodedata',)) + + def test_hangul_syllables(self): + import unicodedata + # Test all leading, vowel and trailing jamo + # but not every combination of them. + for code, name in ((0xAC00, 'HANGUL SYLLABLE GA'), + (0xAE69, 'HANGUL SYLLABLE GGAEG'), + (0xB0D2, 'HANGUL SYLLABLE NYAGG'), + (0xB33B, 'HANGUL SYLLABLE DYAEGS'), + (0xB5A4, 'HANGUL SYLLABLE DDEON'), + (0xB80D, 'HANGUL SYLLABLE RENJ'), + (0xBA76, 'HANGUL SYLLABLE MYEONH'), + (0xBCDF, 'HANGUL SYLLABLE BYED'), + (0xBF48, 'HANGUL SYLLABLE BBOL'), + (0xC1B1, 'HANGUL SYLLABLE SWALG'), + (0xC41A, 'HANGUL SYLLABLE SSWAELM'), + (0xC683, 'HANGUL SYLLABLE OELB'), + (0xC8EC, 'HANGUL SYLLABLE JYOLS'), + (0xCB55, 'HANGUL SYLLABLE JJULT'), + (0xCDBE, 'HANGUL SYLLABLE CWEOLP'), + (0xD027, 'HANGUL SYLLABLE KWELH'), + (0xD290, 'HANGUL SYLLABLE TWIM'), + (0xD4F9, 'HANGUL SYLLABLE PYUB'), + (0xD762, 'HANGUL SYLLABLE HEUBS'), + (0xAE27, 'HANGUL SYLLABLE GYIS'), + (0xB090, 'HANGUL SYLLABLE GGISS'), + (0xB0AD, 'HANGUL SYLLABLE NANG'), + (0xB316, 'HANGUL SYLLABLE DAEJ'), + (0xB57F, 'HANGUL SYLLABLE DDYAC'), + (0xB7E8, 'HANGUL SYLLABLE RYAEK'), + (0xBA51, 'HANGUL SYLLABLE MEOT'), + (0xBCBA, 'HANGUL SYLLABLE BEP'), + (0xBF23, 'HANGUL SYLLABLE BBYEOH'), + (0xD7A3, 'HANGUL SYLLABLE HIH')): + assert unicodedata.name(chr(code)) == name + assert unicodedata.lookup(name) == chr(code) + # Test outside the range + raises(ValueError, unicodedata.name, chr(0xAC00 - 1)) + raises(ValueError, unicodedata.name, chr(0xD7A3 + 1)) + + def test_cjk(self): + import sys + import unicodedata + cases = ((0x3400, 0x4DB5), + (0x4E00, 0x9FA5)) + if unicodedata.unidata_version >= "5": # don't know the exact limit + cases = ((0x3400, 0x4DB5), + (0x4E00, 0x9FCB), + (0x20000, 0x2A6D6), + (0x2A700, 0x2B734)) + elif unicodedata.unidata_version >= "4.1": + cases = ((0x3400, 0x4DB5), + (0x4E00, 0x9FBB), + (0x20000, 0x2A6D6)) + for first, last in cases: + # Test at and inside the boundary + for i in (first, first + 1, last - 1, last): + charname = 'CJK UNIFIED IDEOGRAPH-%X'%i + char = chr(i) + assert unicodedata.name(char) == charname + assert unicodedata.lookup(charname) == char + # Test outside the boundary + for i in first - 1, last + 1: + charname = 'CJK UNIFIED IDEOGRAPH-%X'%i + char = chr(i) + try: + unicodedata.name(char) + except ValueError as e: + assert e.message == 'no such name' + raises(KeyError, unicodedata.lookup, charname) + + def test_bug_1704793(self): # from CPython + import unicodedata + assert unicodedata.lookup("GOTHIC LETTER FAIHU") == '\U00010346' + + def test_normalize(self): + import unicodedata + raises(TypeError, unicodedata.normalize, 'x') + + @py.test.mark.skipif("sys.maxunicode < 0x10ffff", + reason="requires a 'wide' python build.") + def test_normalize_wide(self): + import unicodedata + assert unicodedata.normalize('NFC', '\U000110a5\U000110ba') == u'\U000110ab' + + def test_linebreaks(self): + linebreaks = (0x0a, 0x0b, 0x0c, 0x0d, 0x85, + 0x1c, 0x1d, 0x1e, 0x2028, 0x2029) + for i in linebreaks: + for j in range(-2, 3): + lines = (chr(i + j) + 'A').splitlines() + if i + j in linebreaks: + assert len(lines) == 2 + else: + assert len(lines) == 1 + + def test_mirrored(self): + import unicodedata + # For no reason, unicodedata.mirrored() returns an int, not a bool + assert repr(unicodedata.mirrored(' ')) == '0' diff --git a/pypy/module/unicodedata/test_unicodedata.py b/pypy/module/unicodedata/test_unicodedata.py deleted file mode 100644 --- a/pypy/module/unicodedata/test_unicodedata.py +++ /dev/null @@ -1,103 +0,0 @@ - -class AppTestUnicodeData: - spaceconfig = dict(usemodules=('unicodedata',)) - - def test_hangul_syllables(self): - import unicodedata - # Test all leading, vowel and trailing jamo - # but not every combination of them. - for code, name in ((0xAC00, 'HANGUL SYLLABLE GA'), - (0xAE69, 'HANGUL SYLLABLE GGAEG'), - (0xB0D2, 'HANGUL SYLLABLE NYAGG'), - (0xB33B, 'HANGUL SYLLABLE DYAEGS'), - (0xB5A4, 'HANGUL SYLLABLE DDEON'), - (0xB80D, 'HANGUL SYLLABLE RENJ'), - (0xBA76, 'HANGUL SYLLABLE MYEONH'), - (0xBCDF, 'HANGUL SYLLABLE BYED'), - (0xBF48, 'HANGUL SYLLABLE BBOL'), - (0xC1B1, 'HANGUL SYLLABLE SWALG'), - (0xC41A, 'HANGUL SYLLABLE SSWAELM'), - (0xC683, 'HANGUL SYLLABLE OELB'), - (0xC8EC, 'HANGUL SYLLABLE JYOLS'), - (0xCB55, 'HANGUL SYLLABLE JJULT'), - (0xCDBE, 'HANGUL SYLLABLE CWEOLP'), - (0xD027, 'HANGUL SYLLABLE KWELH'), - (0xD290, 'HANGUL SYLLABLE TWIM'), - (0xD4F9, 'HANGUL SYLLABLE PYUB'), - (0xD762, 'HANGUL SYLLABLE HEUBS'), - (0xAE27, 'HANGUL SYLLABLE GYIS'), - (0xB090, 'HANGUL SYLLABLE GGISS'), - (0xB0AD, 'HANGUL SYLLABLE NANG'), - (0xB316, 'HANGUL SYLLABLE DAEJ'), - (0xB57F, 'HANGUL SYLLABLE DDYAC'), - (0xB7E8, 'HANGUL SYLLABLE RYAEK'), - (0xBA51, 'HANGUL SYLLABLE MEOT'), - (0xBCBA, 'HANGUL SYLLABLE BEP'), - (0xBF23, 'HANGUL SYLLABLE BBYEOH'), - (0xD7A3, 'HANGUL SYLLABLE HIH')): - assert unicodedata.name(chr(code)) == name - assert unicodedata.lookup(name) == chr(code) - # Test outside the range - raises(ValueError, unicodedata.name, chr(0xAC00 - 1)) - raises(ValueError, unicodedata.name, chr(0xD7A3 + 1)) - - def test_cjk(self): - import sys - import unicodedata - cases = ((0x3400, 0x4DB5), - (0x4E00, 0x9FA5)) - if unicodedata.unidata_version >= "5": # don't know the exact limit - cases = ((0x3400, 0x4DB5), - (0x4E00, 0x9FCB), - (0x20000, 0x2A6D6), - (0x2A700, 0x2B734)) - elif unicodedata.unidata_version >= "4.1": - cases = ((0x3400, 0x4DB5), - (0x4E00, 0x9FBB), - (0x20000, 0x2A6D6)) - for first, last in cases: - # Test at and inside the boundary - for i in (first, first + 1, last - 1, last): - charname = 'CJK UNIFIED IDEOGRAPH-%X'%i - char = chr(i) - assert unicodedata.name(char) == charname - assert unicodedata.lookup(charname) == char - # Test outside the boundary - for i in first - 1, last + 1: - charname = 'CJK UNIFIED IDEOGRAPH-%X'%i - char = chr(i) - try: - unicodedata.name(char) - except ValueError as e: - assert e.message == 'no such name' - raises(KeyError, unicodedata.lookup, charname) - - def test_bug_1704793(self): # from CPython - import unicodedata - assert unicodedata.lookup("GOTHIC LETTER FAIHU") == '\U00010346' - - def test_normalize(self): - import unicodedata - raises(TypeError, unicodedata.normalize, 'x') - - def test_normalize_wide(self): - import sys, unicodedata - if sys.maxunicode < 0x10ffff: - skip("requires a 'wide' python build.") - assert unicodedata.normalize('NFC', '\U000110a5\U000110ba') == '\U000110ab' - - def test_linebreaks(self): - linebreaks = (0x0a, 0x0b, 0x0c, 0x0d, 0x85, - 0x1c, 0x1d, 0x1e, 0x2028, 0x2029) - for i in linebreaks: - for j in range(-2, 3): - lines = (chr(i + j) + 'A').splitlines() - if i + j in linebreaks: - assert len(lines) == 2 - else: - assert len(lines) == 1 - - def test_mirrored(self): - import unicodedata - # For no reason, unicodedata.mirrored() returns an int, not a bool - assert repr(unicodedata.mirrored(' ')) == '0' diff --git a/pypy/pytest-A.cfg b/pypy/pytest-A.cfg --- a/pypy/pytest-A.cfg +++ b/pypy/pytest-A.cfg @@ -1,5 +1,5 @@ cherrypick = ['interpreter', 'objspace/test', 'objspace/std', 'module'] -interp = ['translator/goal/pypy-c'] +interp = ['goal/pypy-c'] test_driver = ['test_all.py', '-A'] diff --git a/pypy/test_all.py b/pypy/test_all.py --- a/pypy/test_all.py +++ b/pypy/test_all.py @@ -23,7 +23,8 @@ if len(sys.argv) == 1 and os.path.dirname(sys.argv[0]) in '.': print >> sys.stderr, __doc__ sys.exit(2) - + #Add toplevel repository dir to sys.path + sys.path.insert(0,os.path.dirname(os.path.dirname(__file__))) import pytest import pytest_cov sys.exit(pytest.main(plugins=[pytest_cov])) diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -11,8 +11,10 @@ import shutil import sys +import os +#Add toplevel repository dir to sys.path +sys.path.insert(0,os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))) import py -import os import fnmatch from rpython.tool.udir import udir diff --git a/pypy/bin/translatorshell.py b/rpython/bin/translatorshell.py rename from pypy/bin/translatorshell.py rename to rpython/bin/translatorshell.py diff --git a/rpython/jit/backend/arm/test/support.py b/rpython/jit/backend/arm/test/support.py --- a/rpython/jit/backend/arm/test/support.py +++ b/rpython/jit/backend/arm/test/support.py @@ -14,6 +14,9 @@ def check_jumps(self, maxcount): pass +if not getattr(os, 'uname', None): + pytest.skip('cannot run arm tests on non-posix platform') + if os.uname()[1] == 'llaima.local': AS = '~/Code/arm-jit/android/android-ndk-r4b//build/prebuilt/darwin-x86/arm-eabi-4.4.0/arm-eabi/bin/as' else: diff --git a/rpython/rlib/parsing/test/test_pcre_regtest.py b/rpython/rlib/parsing/test/test_pcre_regtest.py --- a/rpython/rlib/parsing/test/test_pcre_regtest.py +++ b/rpython/rlib/parsing/test/test_pcre_regtest.py @@ -84,8 +84,7 @@ from rpython.rlib.parsing.regexparse import make_runner, unescape import string import re -import os -this_dir = py.path.local(os.path.realpath(os.path.dirname(__file__))) +this_dir = py.path.local(__file__).join('..') #py.test.skip("Still in progress") diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py --- a/rpython/translator/c/genc.py +++ b/rpython/translator/c/genc.py @@ -131,7 +131,7 @@ self.secondary_entrypoints = secondary_entrypoints def get_eci(self): - pypy_include_dir = py.path.local(os.path.realpath(os.path.dirname(__file__))) + pypy_include_dir = py.path.local(__file__).join('..') include_dirs = [pypy_include_dir] return ExternalCompilationInfo(include_dirs=include_dirs) @@ -753,7 +753,7 @@ defines['PYPY_LONGLONG_BIT'] = LONGLONG_BIT def add_extra_files(eci): - srcdir = py.path.local(os.path.realpath(os.path.dirname(__file__))).join('src') + srcdir = py.path.local(__file__).join('..', 'src') files = [ srcdir / 'entrypoint.c', # ifdef PYPY_STANDALONE srcdir / 'allocator.c', # ifdef PYPY_STANDALONE diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py --- a/rpython/translator/platform/posix.py +++ b/rpython/translator/platform/posix.py @@ -5,7 +5,7 @@ from rpython.translator.platform import Platform, log, _run_subprocess import rpython -rpydir = os.path.dirname(rpython.__file__) +rpydir = str(py.path.local(rpython.__file__).join('..')) class BasePosix(Platform): exe_ext = '' diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -7,7 +7,7 @@ from rpython.translator.platform import Platform, posix import rpython -rpydir = os.path.dirname(rpython.__file__) +rpydir = str(py.path.local(rpython.__file__).join('..')) def _get_compiler_type(cc, x64_flag): import subprocess From noreply at buildbot.pypy.org Wed Jan 23 03:34:07 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Wed, 23 Jan 2013 03:34:07 +0100 (CET) Subject: [pypy-commit] pypy py3k: o __builtin__ -> builtins Message-ID: <20130123023407.DC1771C05CA@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60356:be09cec779a0 Date: 2013-01-22 18:28 -0800 http://bitbucket.org/pypy/pypy/changeset/be09cec779a0/ Log: o __builtin__ -> builtins o quick hack to _lsprof objects' __module__ so the cprofile tests format more similarly to cpython diff --git a/pypy/module/_lsprof/interp_lsprof.py b/pypy/module/_lsprof/interp_lsprof.py --- a/pypy/module/_lsprof/interp_lsprof.py +++ b/pypy/module/_lsprof/interp_lsprof.py @@ -62,7 +62,7 @@ return self.frame W_StatsEntry.typedef = TypeDef( - 'StatsEntry', + '_lsprof.StatsEntry', code = GetSetProperty(W_StatsEntry.get_code), callcount = interp_attrproperty('callcount', W_StatsEntry), reccallcount = interp_attrproperty('reccallcount', W_StatsEntry), @@ -89,7 +89,7 @@ return self.frame W_StatsSubEntry.typedef = TypeDef( - 'SubStatsEntry', + '_lsprof.SubStatsEntry', code = GetSetProperty(W_StatsSubEntry.get_code), callcount = interp_attrproperty('callcount', W_StatsSubEntry), reccallcount = interp_attrproperty('reccallcount', W_StatsSubEntry), @@ -210,7 +210,7 @@ module = '' else: module = space.str_w(w_func.w_module) - if module == '__builtin__': + if module == 'builtins': module = '' else: module += '.' @@ -401,8 +401,7 @@ return space.wrap(p) W_Profiler.typedef = TypeDef( - 'Profiler', - __module__ = '_lsprof', + '_lsprof.Profiler', __new__ = interp2app(descr_new_profile), enable = interp2app(W_Profiler.enable), disable = interp2app(W_Profiler.disable), diff --git a/pypy/module/_lsprof/test/test_cprofile.py b/pypy/module/_lsprof/test/test_cprofile.py --- a/pypy/module/_lsprof/test/test_cprofile.py +++ b/pypy/module/_lsprof/test/test_cprofile.py @@ -198,7 +198,7 @@ expected_output = {} expected_output['print_stats'] = """\ - 126 function calls (106 primitive calls) in 1.000 seconds + 119 function calls (99 primitive calls) in 1.000 seconds Ordered by: standard name @@ -213,10 +213,10 @@ 2 0.000 0.000 0.140 0.070 profilee.py:84(helper2_indirect) 8 0.312 0.039 0.400 0.050 profilee.py:88(helper2) 8 0.064 0.008 0.080 0.010 profilee.py:98(subhelper) + 1 0.000 0.000 1.000 1.000 {exec} + 12 0.000 0.000 0.012 0.001 {hasattr} 4 0.000 0.000 0.000 0.000 {method 'append' of 'list' objects} 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects} - 12 0.000 0.000 0.012 0.001 {hasattr} - 8 0.000 0.000 0.000 0.000 {range} 4 0.000 0.000 0.000 0.000 {sys.exc_info} @@ -225,60 +225,59 @@ expected_output['print_callers'] = """\ Ordered by: standard name -Function * was called by... - * ncalls tottime cumtime -:1() * <- -profilee.py:110(__getattr__) * <- 16 0.016 0.016 profilee.py:98(subhelper) - * 12 0.012 0.012 {hasattr.*} -profilee.py:25(testfunc) * <- 1 0.270 1.000 :1() -profilee.py:35(factorial) * <- 1 0.014 0.130 profilee.py:25(testfunc) - * 20/3 0.130 0.147 profilee.py:35(factorial) - * 2 0.006 0.040 profilee.py:84(helper2_indirect) -profilee.py:48(mul) * <- 20 0.020 0.020 profilee.py:35(factorial) -profilee.py:55(helper) * <- 2 0.040 0.600 profilee.py:25(testfunc) -profilee.py:73(helper1) * <- 4 0.116 0.120 profilee.py:55(helper) -profilee.py:84(helper2_indirect) *<- 2 0.000 0.140 profilee.py:55(helper) -profilee.py:88(helper2) * <- 6 0.234 0.300 profilee.py:55(helper) - * 2 0.078 0.100 profilee.py:84(helper2_indirect) -profilee.py:98(subhelper) * <- 8 0.064 0.080 profilee.py:88(helper2) -{.*append.*} * <- 4 0.000 0.000 profilee.py:73(helper1) -{.*disable.*} * <- -{hasattr.*} * <- 4 0.000 0.004 profilee.py:73(helper1) - * 8 0.000 0.008 profilee.py:88(helper2) -{range.*} * <- 8 0.000 0.000 profilee.py:98(subhelper) -{sys.exc_info.*} * <- 4 0.000 0.000 profilee.py:73(helper1) +Function was called by... + ncalls tottime cumtime +:1() <- 1 0.000 1.000 {exec} +profilee.py:110(__getattr__) <- 16 0.016 0.016 profilee.py:98(subhelper) + 12 0.012 0.012 {hasattr} +profilee.py:25(testfunc) <- 1 0.270 1.000 :1() +profilee.py:35(factorial) <- 1 0.014 0.130 profilee.py:25(testfunc) + 20/3 0.130 0.147 profilee.py:35(factorial) + 2 0.006 0.040 profilee.py:84(helper2_indirect) +profilee.py:48(mul) <- 20 0.020 0.020 profilee.py:35(factorial) +profilee.py:55(helper) <- 2 0.040 0.600 profilee.py:25(testfunc) +profilee.py:73(helper1) <- 4 0.116 0.120 profilee.py:55(helper) +profilee.py:84(helper2_indirect) <- 2 0.000 0.140 profilee.py:55(helper) +profilee.py:88(helper2) <- 6 0.234 0.300 profilee.py:55(helper) + 2 0.078 0.100 profilee.py:84(helper2_indirect) +profilee.py:98(subhelper) <- 8 0.064 0.080 profilee.py:88(helper2) +{exec} <- +{hasattr} <- 4 0.000 0.004 profilee.py:73(helper1) + 8 0.000 0.008 profilee.py:88(helper2) +{method 'append' of 'list' objects} <- 4 0.000 0.000 profilee.py:73(helper1) +{method 'disable' of '_lsprof.Profiler' objects} <- +{sys.exc_info} <- 4 0.000 0.000 profilee.py:73(helper1) """ expected_output['print_callees'] = """\ Ordered by: standard name -Function * called... - * ncalls tottime cumtime -:1() * -> 1 0.270 1.000 profilee.py:25(testfunc) -profilee.py:110(__getattr__) * -> -profilee.py:25(testfunc) * -> 1 0.014 0.130 profilee.py:35(factorial) - * 2 0.040 0.600 profilee.py:55(helper) -profilee.py:35(factorial) * -> 20/3 0.130 0.147 profilee.py:35(factorial) - * 20 0.020 0.020 profilee.py:48(mul) -profilee.py:48(mul) * -> -profilee.py:55(helper) * -> 4 0.116 0.120 profilee.py:73(helper1) - * 2 0.000 0.140 profilee.py:84(helper2_indirect) - * 6 0.234 0.300 profilee.py:88(helper2) -\\(profilee.py:73(helper1)\\)\\? * .. 4 0.000 0.000 {.*append.*} -\\(profilee.py:73(helper1)\\)\\? * .. 4 0.000 0.004 {.*hasattr.*} - * 4 0.000 0.000 {sys.exc_info.*} -profilee.py:84(helper2_indirect) * -> 2 0.006 0.040 profilee.py:35(factorial) - * 2 0.078 0.100 profilee.py:88(helper2) -profilee.py:88(helper2) * -> 8 0.064 0.080 profilee.py:98(subhelper) - * 8 0.000 0.008 {hasattr.*} -profilee.py:98(subhelper) * -> 16 0.016 0.016 profilee.py:110(__getattr__) - * 8 0.000 0.000 {range.*} -{.*append.*} * -> -{.*disable.*} * -> -{hasattr.*} * -> 12 0.012 0.012 profilee.py:110(__getattr__) -{range.*} * -> -{sys.exc_info.*} * -> +Function called... + ncalls tottime cumtime +:1() -> 1 0.270 1.000 profilee.py:25(testfunc) +profilee.py:110(__getattr__) -> +profilee.py:25(testfunc) -> 1 0.014 0.130 profilee.py:35(factorial) + 2 0.040 0.600 profilee.py:55(helper) +profilee.py:35(factorial) -> 20/3 0.130 0.147 profilee.py:35(factorial) + 20 0.020 0.020 profilee.py:48(mul) +profilee.py:48(mul) -> +profilee.py:55(helper) -> 4 0.116 0.120 profilee.py:73(helper1) + 2 0.000 0.140 profilee.py:84(helper2_indirect) + 6 0.234 0.300 profilee.py:88(helper2) +profilee.py:73(helper1) -> 4 0.000 0.004 {hasattr} + 4 0.000 0.000 {method 'append' of 'list' objects} + 4 0.000 0.000 {sys.exc_info} +profilee.py:84(helper2_indirect) -> 2 0.006 0.040 profilee.py:35(factorial) + 2 0.078 0.100 profilee.py:88(helper2) +profilee.py:88(helper2) -> 8 0.064 0.080 profilee.py:98(subhelper) + 8 0.000 0.008 {hasattr} +profilee.py:98(subhelper) -> 16 0.016 0.016 profilee.py:110(__getattr__) +{exec} -> 1 0.000 1.000 :1() +{hasattr} -> 12 0.012 0.012 profilee.py:110(__getattr__) +{method 'append' of 'list' objects} -> +{method 'disable' of '_lsprof.Profiler' objects} -> +{sys.exc_info} -> """ From noreply at buildbot.pypy.org Wed Jan 23 03:34:09 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Wed, 23 Jan 2013 03:34:09 +0100 (CET) Subject: [pypy-commit] pypy py3k: kill execfile tests from default Message-ID: <20130123023409.45A581C05CA@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60357:598293f5ca3e Date: 2013-01-22 18:29 -0800 http://bitbucket.org/pypy/pypy/changeset/598293f5ca3e/ Log: kill execfile tests from default diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py --- a/pypy/module/__builtin__/test/test_builtin.py +++ b/pypy/module/__builtin__/test/test_builtin.py @@ -776,47 +776,3 @@ class AppTestGetattrWithGetAttributeShortcut(AppTestGetattr): spaceconfig = {"objspace.std.getattributeshortcut": True} - - -class TestInternal: - def test_execfile(self, space): - from rpython.tool.udir import udir - fn = str(udir.join('test_execfile')) - f = open(fn, 'w') - print >>f, "i=42" - f.close() - - w_execfile = space.builtin.get("execfile") - w_dict = space.newdict() - space.call_function(w_execfile, - space.wrap(fn), w_dict, space.w_None) - w_value = space.getitem(w_dict, space.wrap('i')) - assert space.eq_w(w_value, space.wrap(42)) - - def test_execfile_different_lineendings(self, space): - from rpython.tool.udir import udir - d = udir.ensure('lineending', dir=1) - dos = d.join('dos.py') - f = dos.open('wb') - f.write("x=3\r\n\r\ny=4\r\n") - f.close() - space.appexec([space.wrap(str(dos))], """ - (filename): - d = {} - execfile(filename, d) - assert d['x'] == 3 - assert d['y'] == 4 - """) - - unix = d.join('unix.py') - f = unix.open('wb') - f.write("x=5\n\ny=6\n") - f.close() - - space.appexec([space.wrap(str(unix))], """ - (filename): - d = {} - execfile(filename, d) - assert d['x'] == 5 - assert d['y'] == 6 - """) From noreply at buildbot.pypy.org Wed Jan 23 03:34:10 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Wed, 23 Jan 2013 03:34:10 +0100 (CET) Subject: [pypy-commit] pypy py3k: accept strs in fcntl.ioctl, 2to3 Message-ID: <20130123023410.85E9E1C05CA@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60358:3e9bb91ae0d4 Date: 2013-01-22 18:29 -0800 http://bitbucket.org/pypy/pypy/changeset/3e9bb91ae0d4/ Log: accept strs in fcntl.ioctl, 2to3 diff --git a/pypy/module/fcntl/interp_fcntl.py b/pypy/module/fcntl/interp_fcntl.py --- a/pypy/module/fcntl/interp_fcntl.py +++ b/pypy/module/fcntl/interp_fcntl.py @@ -248,14 +248,19 @@ except OperationError, e: if not e.match(space, space.w_TypeError): raise - else: - ll_arg = rffi.str2charp(arg) - rv = ioctl_str(fd, op, ll_arg) - arg = rffi.charpsize2str(ll_arg, len(arg)) - lltype.free(ll_arg, flavor='raw') - if rv < 0: - raise _get_error(space, "ioctl") - return space.wrapbytes(arg) + try: + arg = space.str_w(w_arg) + except OperationError, e: + if not e.match(space, space.w_TypeError): + raise + raise OperationError( + space.w_TypeError, + space.wrap("int or string or buffer required")) - raise OperationError(space.w_TypeError, - space.wrap("int or string or buffer required")) + ll_arg = rffi.str2charp(arg) + rv = ioctl_str(fd, op, ll_arg) + arg = rffi.charpsize2str(ll_arg, len(arg)) + lltype.free(ll_arg, flavor='raw') + if rv < 0: + raise _get_error(space, "ioctl") + return space.wrapbytes(arg) diff --git a/pypy/module/fcntl/test/test_fcntl.py b/pypy/module/fcntl/test/test_fcntl.py --- a/pypy/module/fcntl/test/test_fcntl.py +++ b/pypy/module/fcntl/test/test_fcntl.py @@ -115,7 +115,7 @@ rval = 2 try: fcntl.flock(open(f.name, f.mode), fcntl.LOCK_EX | fcntl.LOCK_NB) - except IOError, e: + except IOError as e: if e.errno not in (errno.EACCES, errno.EAGAIN): raise rval = 0 @@ -152,7 +152,7 @@ rval = 2 try: fcntl.lockf(open(f.name, f.mode), fcntl.LOCK_EX | fcntl.LOCK_NB) - except IOError, e: + except IOError as e: if e.errno not in (errno.EACCES, errno.EAGAIN): raise rval = 0 From noreply at buildbot.pypy.org Wed Jan 23 03:34:11 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Wed, 23 Jan 2013 03:34:11 +0100 (CET) Subject: [pypy-commit] pypy py3k: workaround impl details in doctest Message-ID: <20130123023411.B33E31C05CA@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60359:590b300b77af Date: 2013-01-22 18:31 -0800 http://bitbucket.org/pypy/pypy/changeset/590b300b77af/ Log: workaround impl details in doctest diff --git a/lib-python/3.2/test/test_descrtut.py b/lib-python/3.2/test/test_descrtut.py --- a/lib-python/3.2/test/test_descrtut.py +++ b/lib-python/3.2/test/test_descrtut.py @@ -188,6 +188,7 @@ '__mul__', '__ne__', '__new__', + '__radd__', '__reduce__', '__reduce_ex__', '__repr__', @@ -195,7 +196,6 @@ '__rmul__', '__setattr__', '__setitem__', - '__sizeof__', '__str__', '__subclasshook__', 'append', From noreply at buildbot.pypy.org Wed Jan 23 03:34:12 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Wed, 23 Jan 2013 03:34:12 +0100 (CET) Subject: [pypy-commit] pypy py3k: pypy lacks sys.getrefcount Message-ID: <20130123023412.E6EAF1C05CA@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60360:0b275da37609 Date: 2013-01-22 18:31 -0800 http://bitbucket.org/pypy/pypy/changeset/0b275da37609/ Log: pypy lacks sys.getrefcount diff --git a/lib-python/3.2/test/test_optparse.py b/lib-python/3.2/test/test_optparse.py --- a/lib-python/3.2/test/test_optparse.py +++ b/lib-python/3.2/test/test_optparse.py @@ -379,6 +379,8 @@ self.assertRaises(self.parser.remove_option, ('foo',), None, ValueError, "no such option 'foo'") + @unittest.skipUnless(hasattr(sys, 'getrefcount'), + "requires sys.getrefcount") def test_refleak(self): # If an OptionParser is carrying around a reference to a large # object, various cycles can prevent it from being GC'd in From noreply at buildbot.pypy.org Wed Jan 23 03:34:14 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Wed, 23 Jan 2013 03:34:14 +0100 (CET) Subject: [pypy-commit] pypy py3k: fix print with file=None Message-ID: <20130123023414.2D9EF1C05CA@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60361:ebff654ad84e Date: 2013-01-22 18:31 -0800 http://bitbucket.org/pypy/pypy/changeset/ebff654ad84e/ Log: fix print with file=None diff --git a/pypy/module/__builtin__/app_io.py b/pypy/module/__builtin__/app_io.py --- a/pypy/module/__builtin__/app_io.py +++ b/pypy/module/__builtin__/app_io.py @@ -53,9 +53,11 @@ def print_(*args, **kwargs): """The new-style print function from py3k.""" - fp = kwargs.pop("file", sys.stdout) + fp = kwargs.pop("file", None) if fp is None: - return + fp = sys.stdout + if fp is None: + return def write(data): fp.write(str(data)) sep = kwargs.pop("sep", None) diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py --- a/pypy/module/__builtin__/test/test_builtin.py +++ b/pypy/module/__builtin__/test/test_builtin.py @@ -623,6 +623,12 @@ out = io.StringIO() pr(None, file=out) assert out.getvalue() == "None\n" + out = sys.stdout = io.StringIO() + try: + pr("amaury", file=None) + finally: + sys.stdout = save + assert out.getvalue() == "amaury\n" def test_print_function2(self): import builtins From noreply at buildbot.pypy.org Wed Jan 23 04:23:59 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Wed, 23 Jan 2013 04:23:59 +0100 (CET) Subject: [pypy-commit] pypy py3k: input must eat potential exceptions from fileno() Message-ID: <20130123032359.7AFBF1C05CA@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60362:f413409ca215 Date: 2013-01-22 19:22 -0800 http://bitbucket.org/pypy/pypy/changeset/f413409ca215/ Log: input must eat potential exceptions from fileno() diff --git a/pypy/module/__builtin__/app_io.py b/pypy/module/__builtin__/app_io.py --- a/pypy/module/__builtin__/app_io.py +++ b/pypy/module/__builtin__/app_io.py @@ -14,6 +14,13 @@ else: flush() +def _is_std_tty(stdin, stdout): + try: + infileno, outfileno = stdin.fileno(), stdout.fileno() + except: + return False + return infileno == 0 and stdin.isatty() and outfileno == 1 + def input(prompt=''): """input([prompt]) -> string @@ -37,9 +44,7 @@ stderr.flush() # hook for the readline module - if (hasattr(sys, '__raw_input__') and - stdin.fileno() == 0 and stdin.isatty() and - stdout.fileno() == 1): + if hasattr(sys, '__raw_input__') and _is_std_tty(stdin, stdout): _write_prompt(stdout, '') return sys.__raw_input__(str(prompt)) diff --git a/pypy/module/__builtin__/test/test_rawinput.py b/pypy/module/__builtin__/test/test_rawinput.py --- a/pypy/module/__builtin__/test/test_rawinput.py +++ b/pypy/module/__builtin__/test/test_rawinput.py @@ -39,6 +39,19 @@ assert flushed[0] assert got == gottext + def test_bad_fileno(self): + import io + import sys + class BadFileno(io.StringIO): + def fileno(self): + 1 / 0 + stdin, sys.stdin = sys.stdin, BadFileno('foo') + try: + result = input() + finally: + sys.stdin = stdin + assert result == 'foo' + def test_softspace(self): import sys import io From noreply at buildbot.pypy.org Wed Jan 23 09:24:05 2013 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 23 Jan 2013 09:24:05 +0100 (CET) Subject: [pypy-commit] pypy default: handle case where running from current dir Message-ID: <20130123082405.F00C11C0634@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r60363:fa8a924ed735 Date: 2013-01-23 10:24 +0200 http://bitbucket.org/pypy/pypy/changeset/fa8a924ed735/ Log: handle case where running from current dir diff --git a/pypy/test_all.py b/pypy/test_all.py --- a/pypy/test_all.py +++ b/pypy/test_all.py @@ -24,7 +24,7 @@ print >> sys.stderr, __doc__ sys.exit(2) #Add toplevel repository dir to sys.path - sys.path.insert(0,os.path.dirname(os.path.dirname(__file__))) + sys.path.insert(0,os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) import pytest import pytest_cov sys.exit(pytest.main(plugins=[pytest_cov])) From noreply at buildbot.pypy.org Wed Jan 23 09:30:36 2013 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 23 Jan 2013 09:30:36 +0100 (CET) Subject: [pypy-commit] pypy default: file moved Message-ID: <20130123083036.E3B8B1C0634@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r60364:950e5a445280 Date: 2013-01-23 10:31 +0200 http://bitbucket.org/pypy/pypy/changeset/950e5a445280/ Log: file moved diff --git a/lib-python/conftest.py b/lib-python/conftest.py --- a/lib-python/conftest.py +++ b/lib-python/conftest.py @@ -18,7 +18,7 @@ from pypy.tool.pytest import appsupport from pypy.tool.pytest.confpath import pypydir, testdir, testresultdir -from pypy.config.parse import parse_info +from rpython.config.parse import parse_info pytest_plugins = "resultlog", rsyncdirs = ['.', '../pypy/'] From noreply at buildbot.pypy.org Wed Jan 23 09:37:46 2013 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 23 Jan 2013 09:37:46 +0100 (CET) Subject: [pypy-commit] pypy default: fix path of pypy-c Message-ID: <20130123083747.012731C1027@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r60365:5d24ffacd54e Date: 2013-01-23 10:38 +0200 http://bitbucket.org/pypy/pypy/changeset/5d24ffacd54e/ Log: fix path of pypy-c diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -45,7 +45,7 @@ def package(basedir, name='pypy-nightly', rename_pypy_c='pypy', copy_to_dir = None, override_pypy_c = None): - basedir = py.path.local(basedir) + basedir = py.path.local('goal', basenamebasedir) if override_pypy_c is None: basename = 'pypy-c' if sys.platform == 'win32': From noreply at buildbot.pypy.org Wed Jan 23 11:39:14 2013 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 23 Jan 2013 11:39:14 +0100 (CET) Subject: [pypy-commit] pypy default: whoops Message-ID: <20130123103914.D94C31C05CA@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r60366:4a2e49d52691 Date: 2013-01-23 12:36 +0200 http://bitbucket.org/pypy/pypy/changeset/4a2e49d52691/ Log: whoops diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -45,7 +45,7 @@ def package(basedir, name='pypy-nightly', rename_pypy_c='pypy', copy_to_dir = None, override_pypy_c = None): - basedir = py.path.local('goal', basenamebasedir) + basedir = py.path.local('goal', basedir) if override_pypy_c is None: basename = 'pypy-c' if sys.platform == 'win32': From noreply at buildbot.pypy.org Wed Jan 23 13:09:17 2013 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 23 Jan 2013 13:09:17 +0100 (CET) Subject: [pypy-commit] pypy default: adapt to moved files Message-ID: <20130123120917.A0EDE1C0634@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r60367:de67d376b3df Date: 2013-01-23 14:09 +0200 http://bitbucket.org/pypy/pypy/changeset/de67d376b3df/ Log: adapt to moved files diff --git a/lib-python/conftest.py b/lib-python/conftest.py --- a/lib-python/conftest.py +++ b/lib-python/conftest.py @@ -17,7 +17,7 @@ from pypy.conftest import option as pypy_option from pypy.tool.pytest import appsupport -from pypy.tool.pytest.confpath import pypydir, testdir, testresultdir +from pypy.tool.pytest.confpath import pypydir, rpythondir, testdir, testresultdir from rpython.config.parse import parse_info pytest_plugins = "resultlog", @@ -40,7 +40,7 @@ dest="unittest_filter", help="Similar to -k, XXX") def gettimeout(timeout): - from test import pystone + from rpython.translator.test import rpystone if timeout.endswith('mp'): megapystone = float(timeout[:-2]) t, stone = pystone.Proc0(10000) @@ -584,7 +584,7 @@ watchdog_name = 'watchdog_nt.py' else: watchdog_name = 'watchdog.py' - watchdog_script = pypydir.join('tool', watchdog_name) + watchdog_script = rpythondir.join('tool', watchdog_name) regr_script = pypydir.join('tool', 'pytest', 'run-script', 'regrverbose.py') diff --git a/pypy/tool/pytest/confpath.py b/pypy/tool/pytest/confpath.py --- a/pypy/tool/pytest/confpath.py +++ b/pypy/tool/pytest/confpath.py @@ -1,8 +1,10 @@ import py import pypy +import rpython from pypy.tool import lib_pypy pypydir = py.path.local(pypy.__file__).dirpath() +rpythondir = py.path.local(rpython.__file__).dirpath() distdir = pypydir.dirpath() testresultdir = distdir.join('testresult') assert pypydir.check(dir=1) From noreply at buildbot.pypy.org Wed Jan 23 14:14:14 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Wed, 23 Jan 2013 14:14:14 +0100 (CET) Subject: [pypy-commit] pypy inline-virtualref: merged default in Message-ID: <20130123131414.AFF0C1C0634@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: inline-virtualref Changeset: r60368:36fae7dd9821 Date: 2013-01-23 05:12 -0800 http://bitbucket.org/pypy/pypy/changeset/36fae7dd9821/ Log: merged default in diff --git a/lib-python/conftest.py b/lib-python/conftest.py --- a/lib-python/conftest.py +++ b/lib-python/conftest.py @@ -17,8 +17,8 @@ from pypy.conftest import option as pypy_option from pypy.tool.pytest import appsupport -from pypy.tool.pytest.confpath import pypydir, testdir, testresultdir -from pypy.config.parse import parse_info +from pypy.tool.pytest.confpath import pypydir, rpythondir, testdir, testresultdir +from rpython.config.parse import parse_info pytest_plugins = "resultlog", rsyncdirs = ['.', '../pypy/'] @@ -40,7 +40,7 @@ dest="unittest_filter", help="Similar to -k, XXX") def gettimeout(timeout): - from test import pystone + from rpython.translator.test import rpystone if timeout.endswith('mp'): megapystone = float(timeout[:-2]) t, stone = pystone.Proc0(10000) @@ -584,7 +584,7 @@ watchdog_name = 'watchdog_nt.py' else: watchdog_name = 'watchdog.py' - watchdog_script = pypydir.join('tool', watchdog_name) + watchdog_script = rpythondir.join('tool', watchdog_name) regr_script = pypydir.join('tool', 'pytest', 'run-script', 'regrverbose.py') diff --git a/pypy/doc/arm.rst b/pypy/doc/arm.rst --- a/pypy/doc/arm.rst +++ b/pypy/doc/arm.rst @@ -130,15 +130,7 @@ export SB2=/srv/chroot/precise_arm export SB2OPT='-t ARM' -Once this is set, you can call the translator - -:: - - pypy ~/path_to_pypy_checkout/pypy/translator/goal/translate.py -O1 --platform=arm target.py - -If everything worked correctly this should yield an ARM binary. Running this binary in the ARM chroot or on an ARM device should produce the output ``"Hello World"``. - -.. _`this`: +Once this is set, you can call the translator. For example save this file :: @@ -148,3 +140,22 @@ def target(*args): return main, None + +and call the translator + +:: + + pypy ~/path_to_pypy_checkout/pypy/translator/goal/translate.py -O1 --platform=arm target.py + +If everything worked correctly this should yield an ARM binary. Running this binary in the ARM chroot or on an ARM device should produce the output ``"Hello World"``. + +To translate the full python pypy interpreter with a jit, you can cd into pypy/goal and call + +:: + + pypy /rpython/translator/goal/translate.py -Ojit --platform=arm --gcrootfinder=shadowstack --jit-backend=arm targetpypystandalone.py + +The gcrootfinder option is needed to work around `issue 1377`_ and the jit-backend works around `issue 1376`_ + +.. _`issue 1377`: https://bugs.pypy.org/issue1377 +.. _`issue 1376`: https://bugs.pypy.org/issue1376 diff --git a/pypy/goal/__init__.py b/pypy/goal/__init__.py new file mode 100644 --- /dev/null +++ b/pypy/goal/__init__.py @@ -0,0 +1,1 @@ +#empty diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py --- a/pypy/interpreter/astcompiler/codegen.py +++ b/pypy/interpreter/astcompiler/codegen.py @@ -904,8 +904,9 @@ def visit_Set(self, s): self.update_position(s.lineno) + elt_count = len(s.elts) if s.elts is not None else 0 self.visit_sequence(s.elts) - self.emit_op_arg(ops.BUILD_SET, len(s.elts)) + self.emit_op_arg(ops.BUILD_SET, elt_count) def visit_Name(self, name): self.update_position(name.lineno) diff --git a/pypy/interpreter/test2/test_app_main.py b/pypy/interpreter/test2/test_app_main.py --- a/pypy/interpreter/test2/test_app_main.py +++ b/pypy/interpreter/test2/test_app_main.py @@ -84,7 +84,7 @@ def check(self, argv, **expected): import StringIO - from rpython.translator.goal import app_main + from pypy.interpreter import app_main saved_sys_argv = sys.argv[:] saved_sys_stdout = sys.stdout saved_sys_stderr = sys.stdout @@ -825,7 +825,7 @@ class TestAppMain: def test_print_info(self): - from rpython.translator.goal import app_main + from pypy.interpreter import app_main import sys, cStringIO prev_so = sys.stdout prev_ti = getattr(sys, 'pypy_translation_info', 'missing') diff --git a/pypy/interpreter/test2/test_targetpypy.py b/pypy/interpreter/test2/test_targetpypy.py --- a/pypy/interpreter/test2/test_targetpypy.py +++ b/pypy/interpreter/test2/test_targetpypy.py @@ -1,6 +1,6 @@ import py -from goal.targetpypystandalone import get_entry_point +from pypy.goal.targetpypystandalone import get_entry_point from pypy.config.pypyoption import get_pypy_config class TestTargetPyPy(object): diff --git a/pypy/module/_ast/test/test_ast.py b/pypy/module/_ast/test/test_ast.py --- a/pypy/module/_ast/test/test_ast.py +++ b/pypy/module/_ast/test/test_ast.py @@ -290,6 +290,12 @@ ]) exec compile(body, '', 'exec') + def test_empty_set(self): + import ast + m = ast.Module(body=[ast.Expr(value=ast.Set(elts=[]))]) + ast.fix_missing_locations(m) + compile(m, "", "exec") + def test_invalid_sum(self): import _ast as ast pos = dict(lineno=2, col_offset=3) diff --git a/pypy/module/cpyext/classobject.py b/pypy/module/cpyext/classobject.py --- a/pypy/module/cpyext/classobject.py +++ b/pypy/module/cpyext/classobject.py @@ -22,6 +22,12 @@ w_result.setdict(space, w_dict) return w_result + at cpython_api([PyObject, PyObject, PyObject], PyObject) +def PyInstance_New(space, w_cls, w_arg, w_kw): + """Create a new instance of a specific class. The parameters arg and kw are + used as the positional and keyword parameters to the object's constructor.""" + return space.call(w_cls, w_arg, w_kw) + @cpython_api([PyObject, PyObject], PyObject, error=CANNOT_FAIL) def _PyInstance_Lookup(space, w_instance, w_name): name = space.str_w(w_name) diff --git a/pypy/module/cpyext/include/ceval.h b/pypy/module/cpyext/include/ceval.h new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/include/ceval.h @@ -0,0 +1,1 @@ +/* empty */ diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py --- a/pypy/module/cpyext/stubs.py +++ b/pypy/module/cpyext/stubs.py @@ -176,12 +176,6 @@ """Return true if klass is a subclass of base. Return false in all other cases.""" raise NotImplementedError - at cpython_api([PyObject, PyObject, PyObject], PyObject) -def PyInstance_New(space, cls, arg, kw): - """Create a new instance of a specific class. The parameters arg and kw are - used as the positional and keyword parameters to the object's constructor.""" - raise NotImplementedError - @cpython_api([PyObject], rffi.INT_real, error=-1) def PyCodec_Register(space, search_function): """Register a new codec search function. diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py --- a/pypy/module/cpyext/test/conftest.py +++ b/pypy/module/cpyext/test/conftest.py @@ -1,6 +1,15 @@ import py import pytest +def pytest_configure(config): + from pypy.tool.pytest.objspace import gettestobjspace + # For some reason (probably a ll2ctypes cache issue on linux64) + # it's necessary to run "import time" at least once before any + # other cpyext test, otherwise the same statement will fail in + # test_datetime.py. + space = gettestobjspace(usemodules=['rctime']) + space.getbuiltinmodule("time") + def pytest_ignore_collect(path, config): if config.option.runappdirect: return True # "cannot be run by py.test -A" diff --git a/pypy/module/cpyext/test/test_classobject.py b/pypy/module/cpyext/test/test_classobject.py --- a/pypy/module/cpyext/test/test_classobject.py +++ b/pypy/module/cpyext/test/test_classobject.py @@ -7,8 +7,10 @@ w_class = space.appexec([], """(): class C: x = None - def __init__(self): + def __init__(self, *args, **kwargs): self.x = 1 + self.args = args + self.__dict__.update(kwargs) return C """) @@ -22,6 +24,13 @@ assert space.getattr(w_instance, space.wrap('x')) is space.w_None assert space.unwrap(space.getattr(w_instance, space.wrap('a'))) == 3 + w_instance = api.PyInstance_New(w_class, + space.wrap((3,)), space.wrap(dict(y=2))) + assert space.unwrap(space.getattr(w_instance, space.wrap('x'))) == 1 + assert space.unwrap(space.getattr(w_instance, space.wrap('y'))) == 2 + assert space.unwrap(space.getattr(w_instance, space.wrap('args'))) == (3,) + + def test_lookup(self, space, api): w_instance = space.appexec([], """(): class C: diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py --- a/pypy/module/rctime/interp_time.py +++ b/pypy/module/rctime/interp_time.py @@ -448,10 +448,6 @@ raise OperationError(space.w_ValueError, space.wrap("year out of range")) - if rffi.getintfield(glob_buf, 'c_tm_wday') < 0: - raise OperationError(space.w_ValueError, - space.wrap("day of week out of range")) - rffi.setintfield(glob_buf, 'c_tm_year', y - 1900) rffi.setintfield(glob_buf, 'c_tm_mon', rffi.getintfield(glob_buf, 'c_tm_mon') - 1) @@ -460,6 +456,12 @@ rffi.setintfield(glob_buf, 'c_tm_yday', rffi.getintfield(glob_buf, 'c_tm_yday') - 1) + # tm_wday does not need checking of its upper-bound since taking "% + # 7" in gettmarg() automatically restricts the range. + if rffi.getintfield(glob_buf, 'c_tm_wday') < 0: + raise OperationError(space.w_ValueError, + space.wrap("day of week out of range")) + return glob_buf def time(space): diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py --- a/pypy/module/rctime/test/test_rctime.py +++ b/pypy/module/rctime/test/test_rctime.py @@ -113,6 +113,9 @@ if os.name != 'nt': assert rctime.mktime(rctime.localtime(-1)) == -1 + res = rctime.mktime((2000, 1, 1, 0, 0, 0, -1, -1, -1)) + assert rctime.ctime(res) == 'Sat Jan 1 00:00:00 2000' + def test_asctime(self): import time as rctime rctime.asctime() diff --git a/pypy/pytest-A.cfg b/pypy/pytest-A.cfg --- a/pypy/pytest-A.cfg +++ b/pypy/pytest-A.cfg @@ -1,5 +1,5 @@ cherrypick = ['interpreter', 'objspace/test', 'objspace/std', 'module'] -interp = ['translator/goal/pypy-c'] +interp = ['goal/pypy-c'] test_driver = ['test_all.py', '-A'] diff --git a/pypy/test_all.py b/pypy/test_all.py --- a/pypy/test_all.py +++ b/pypy/test_all.py @@ -23,7 +23,8 @@ if len(sys.argv) == 1 and os.path.dirname(sys.argv[0]) in '.': print >> sys.stderr, __doc__ sys.exit(2) - + #Add toplevel repository dir to sys.path + sys.path.insert(0,os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) import pytest import pytest_cov sys.exit(pytest.main(plugins=[pytest_cov])) diff --git a/pypy/tool/pytest/confpath.py b/pypy/tool/pytest/confpath.py --- a/pypy/tool/pytest/confpath.py +++ b/pypy/tool/pytest/confpath.py @@ -1,8 +1,10 @@ import py import pypy +import rpython from pypy.tool import lib_pypy pypydir = py.path.local(pypy.__file__).dirpath() +rpythondir = py.path.local(rpython.__file__).dirpath() distdir = pypydir.dirpath() testresultdir = distdir.join('testresult') assert pypydir.check(dir=1) diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -11,8 +11,10 @@ import shutil import sys +import os +#Add toplevel repository dir to sys.path +sys.path.insert(0,os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))) import py -import os import fnmatch from rpython.tool.udir import udir @@ -43,7 +45,7 @@ def package(basedir, name='pypy-nightly', rename_pypy_c='pypy', copy_to_dir = None, override_pypy_c = None): - basedir = py.path.local(basedir) + basedir = py.path.local('goal', basedir) if override_pypy_c is None: basename = 'pypy-c' if sys.platform == 'win32': diff --git a/pypy/bin/translatorshell.py b/rpython/bin/translatorshell.py rename from pypy/bin/translatorshell.py rename to rpython/bin/translatorshell.py diff --git a/rpython/jit/backend/arm/test/support.py b/rpython/jit/backend/arm/test/support.py --- a/rpython/jit/backend/arm/test/support.py +++ b/rpython/jit/backend/arm/test/support.py @@ -14,6 +14,9 @@ def check_jumps(self, maxcount): pass +if not getattr(os, 'uname', None): + pytest.skip('cannot run arm tests on non-posix platform') + if os.uname()[1] == 'llaima.local': AS = '~/Code/arm-jit/android/android-ndk-r4b//build/prebuilt/darwin-x86/arm-eabi-4.4.0/arm-eabi/bin/as' else: From noreply at buildbot.pypy.org Wed Jan 23 14:14:15 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Wed, 23 Jan 2013 14:14:15 +0100 (CET) Subject: [pypy-commit] pypy inline-virtualref: Closed branch, approach wasn't going anywhere. Message-ID: <20130123131415.F289E1C0634@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: inline-virtualref Changeset: r60369:243ed364aa3c Date: 2013-01-23 05:13 -0800 http://bitbucket.org/pypy/pypy/changeset/243ed364aa3c/ Log: Closed branch, approach wasn't going anywhere. From noreply at buildbot.pypy.org Wed Jan 23 14:14:17 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Wed, 23 Jan 2013 14:14:17 +0100 (CET) Subject: [pypy-commit] pypy inline-virtualref-2: A branch to try the approach of inlining virtual virtualrefs in optimizeopt. Message-ID: <20130123131417.2082D1C0634@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: inline-virtualref-2 Changeset: r60370:89f8a8352050 Date: 2013-01-23 05:13 -0800 http://bitbucket.org/pypy/pypy/changeset/89f8a8352050/ Log: A branch to try the approach of inlining virtual virtualrefs in optimizeopt. From noreply at buildbot.pypy.org Wed Jan 23 14:41:44 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Wed, 23 Jan 2013 14:41:44 +0100 (CET) Subject: [pypy-commit] pypy inline-virtualref-2: bring over the test Message-ID: <20130123134144.8C3931C00A8@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: inline-virtualref-2 Changeset: r60371:029788868de3 Date: 2013-01-23 05:15 -0800 http://bitbucket.org/pypy/pypy/changeset/029788868de3/ Log: bring over the test diff --git a/rpython/jit/metainterp/test/test_virtualref.py b/rpython/jit/metainterp/test/test_virtualref.py --- a/rpython/jit/metainterp/test/test_virtualref.py +++ b/rpython/jit/metainterp/test/test_virtualref.py @@ -1,19 +1,20 @@ import py -from rpython.rtyper.lltypesystem import lltype, llmemory, lloperation + +from rpython.rtyper.lltypesystem import lltype, lloperation from rpython.rtyper.exceptiondata import UnknownException from rpython.rlib.jit import JitDriver, dont_look_inside, vref_None from rpython.rlib.jit import virtual_ref, virtual_ref_finish, InvalidVirtualRef from rpython.rlib.jit import non_virtual_ref from rpython.rlib.objectmodel import compute_unique_id -from rpython.jit.metainterp.test.support import LLJitMixin, OOJitMixin, _get_jitcodes +from rpython.jit.metainterp.test.support import LLJitMixin, _get_jitcodes from rpython.jit.metainterp.resoperation import rop from rpython.jit.metainterp.virtualref import VirtualRefInfo + debug_print = lloperation.llop.debug_print -class VRefTests: - +class VRefTests(object): def finish_setup_for_interp_operations(self): self.vrefinfo = VirtualRefInfo(self.warmrunnerstate) self.cw.setup_vrefinfo(self.vrefinfo) @@ -115,8 +116,8 @@ from rpython.jit.metainterp.resume import ResumeDataDirectReader cpu = self.metainterp.cpu cpu.get_latest_value_count = lambda df: len(guard_op.getfailargs()) - cpu.get_latest_value_int = lambda df,i:guard_op.getfailargs()[i].getint() - cpu.get_latest_value_ref = lambda df,i:guard_op.getfailargs()[i].getref_base() + cpu.get_latest_value_int = lambda df, i: guard_op.getfailargs()[i].getint() + cpu.get_latest_value_ref = lambda df, i: guard_op.getfailargs()[i].getref_base() cpu.clear_latest_values = lambda count: None class FakeMetaInterpSd: callinfocollection = None @@ -656,6 +657,31 @@ res = self.meta_interp(f, [10]) assert res == 0 + def test_force_virtual_vref(self): + myjitdriver = JitDriver(greens=[], reds=['n', 'ec']) + + class ExecutionContext(object): + pass + + class Frame(object): + def __init__(self, x): + self.x = x + + def f(n): + ec = ExecutionContext() + while n > 0: + myjitdriver.jit_merge_point(n=n, ec=ec) + frame = Frame(1) + ec.topframeref = virtual_ref(frame) + n -= ec.topframeref().x + frame_vref = ec.topframeref + ec.topframeref = vref_None + virtual_ref_finish(frame_vref, frame) + return n + res = self.meta_interp(f, [10]) + assert res == 0 + self.check_resops({'int_sub': 1, 'int_gt': 1, 'jump': 1}) + class TestLLtype(VRefTests, LLJitMixin): pass From noreply at buildbot.pypy.org Wed Jan 23 14:41:45 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Wed, 23 Jan 2013 14:41:45 +0100 (CET) Subject: [pypy-commit] pypy inline-virtualref-2: start implementing, mostly works, two remaining issues: Message-ID: <20130123134145.EBFFB1C00A8@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: inline-virtualref-2 Changeset: r60372:b6915c54d1d2 Date: 2013-01-23 05:41 -0800 http://bitbucket.org/pypy/pypy/changeset/b6915c54d1d2/ Log: start implementing, mostly works, two remaining issues: 1) Tests fail because check_resops looks at entry + loop, want only loop. 2) Trace still contains: force_token, guard_no_exception, and guard_not_forced diff --git a/rpython/jit/codewriter/effectinfo.py b/rpython/jit/codewriter/effectinfo.py --- a/rpython/jit/codewriter/effectinfo.py +++ b/rpython/jit/codewriter/effectinfo.py @@ -79,9 +79,11 @@ OS_RAW_MALLOC_VARSIZE = 110 OS_RAW_FREE = 111 + OS_JIT_FORCE_VIRTUAL = 120 + # for debugging: _OS_CANRAISE = set([OS_NONE, OS_STR2UNICODE, OS_LIBFFI_CALL, - OS_RAW_MALLOC_VARSIZE]) + OS_RAW_MALLOC_VARSIZE, OS_JIT_FORCE_VIRTUAL]) def __new__(cls, readonly_descrs_fields, readonly_descrs_arrays, write_descrs_fields, write_descrs_arrays, diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -1393,6 +1393,8 @@ elif oopspec_name == 'jit.isvirtual': kind = getkind(args[0].concretetype) return SpaceOperation('%s_isvirtual' % kind, args, op.result) + elif oopspec_name == 'jit.force_virtual': + return self._handle_oopspec_call(op, args, EffectInfo.OS_JIT_FORCE_VIRTUAL, EffectInfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE) else: raise AssertionError("missing support for %r" % oopspec_name) diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py --- a/rpython/jit/codewriter/support.py +++ b/rpython/jit/codewriter/support.py @@ -1,24 +1,24 @@ import sys -from rpython.rtyper.lltypesystem import lltype, rclass, rffi, llmemory -from rpython.rtyper.ootypesystem import ootype -from rpython.rtyper import rlist -from rpython.rtyper.lltypesystem import rstr as ll_rstr, rdict as ll_rdict -from rpython.rtyper.lltypesystem.module import ll_math -from rpython.rtyper.lltypesystem.lloperation import llop -from rpython.rtyper.ootypesystem import rdict as oo_rdict -from rpython.rtyper.llinterp import LLInterpreter -from rpython.rtyper.extregistry import ExtRegistryEntry -from rpython.translator.simplify import get_funcobj -from rpython.translator.unsimplify import split_block + +from rpython.annotator import model as annmodel +from rpython.annotator.policy import AnnotatorPolicy from rpython.flowspace.model import Variable, Constant -from rpython.translator.translator import TranslationContext -from rpython.annotator.policy import AnnotatorPolicy -from rpython.annotator import model as annmodel -from rpython.rtyper.annlowlevel import MixLevelHelperAnnotator from rpython.jit.metainterp.typesystem import deref from rpython.rlib import rgc -from rpython.rlib.jit import elidable +from rpython.rlib.jit import elidable, oopspec from rpython.rlib.rarithmetic import r_longlong, r_ulonglong, r_uint, intmask +from rpython.rtyper import rlist +from rpython.rtyper.annlowlevel import MixLevelHelperAnnotator +from rpython.rtyper.extregistry import ExtRegistryEntry +from rpython.rtyper.llinterp import LLInterpreter +from rpython.rtyper.lltypesystem import lltype, rclass, rffi, llmemory, rstr as ll_rstr, rdict as ll_rdict +from rpython.rtyper.lltypesystem.lloperation import llop +from rpython.rtyper.lltypesystem.module import ll_math +from rpython.rtyper.ootypesystem import ootype, rdict as oo_rdict +from rpython.translator.simplify import get_funcobj +from rpython.translator.translator import TranslationContext +from rpython.translator.unsimplify import split_block + def getargtypes(annotator, values): if values is None: # for backend tests producing stand-alone exe's @@ -213,10 +213,12 @@ _ll_5_list_ll_arraycopy = rgc.ll_arraycopy + @elidable def _ll_1_gc_identityhash(x): return lltype.identityhash(x) + # the following function should not be "@elidable": I can think of # a corner case in which id(const) is constant-folded, and then 'const' # disappears and is collected too early (possibly causing another object @@ -224,6 +226,8 @@ def _ll_1_gc_id(ptr): return llop.gc_id(lltype.Signed, ptr) + + at oopspec("jit.force_virtual(inst)") def _ll_1_jit_force_virtual(inst): return llop.jit_force_virtual(lltype.typeOf(inst), inst) diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py --- a/rpython/jit/metainterp/history.py +++ b/rpython/jit/metainterp/history.py @@ -978,11 +978,9 @@ def get_all_jitcell_tokens(self): tokens = [t() for t in self.jitcell_token_wrefs] if None in tokens: - assert False, "get_all_jitcell_tokens will not work as "+\ - "loops have been freed" + assert False, ("get_all_jitcell_tokens will not work as " + "loops have been freed") return tokens - - def check_history(self, expected=None, **check): insns = {} diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -1,12 +1,13 @@ +from rpython.jit.codewriter.effectinfo import EffectInfo +from rpython.jit.metainterp.executor import execute from rpython.jit.codewriter.heaptracker import vtable2descr -from rpython.jit.metainterp.executor import execute from rpython.jit.metainterp.history import Const, ConstInt, BoxInt from rpython.jit.metainterp.optimizeopt import optimizer +from rpython.jit.metainterp.optimizeopt.optimizer import OptValue from rpython.jit.metainterp.optimizeopt.util import (make_dispatcher_method, descrlist_dict, sort_descrs) from rpython.jit.metainterp.resoperation import rop, ResOperation from rpython.rlib.objectmodel import we_are_translated -from rpython.jit.metainterp.optimizeopt.optimizer import OptValue class AbstractVirtualValue(optimizer.OptValue): @@ -386,6 +387,14 @@ self.make_equal_to(box, vvalue) return vvalue + def optimize_CALL_MAY_FORCE(self, op): + effectinfo = op.getdescr().get_extra_info() + oopspecindex = effectinfo.oopspecindex + if oopspecindex == EffectInfo.OS_JIT_FORCE_VIRTUAL: + if self._optimize_JIT_FORCE_VIRTUAL(op): + return + self.emit_operation(op) + def optimize_VIRTUAL_REF(self, op): indexbox = op.getarg(1) # @@ -429,7 +438,7 @@ # - set 'virtual_token' to TOKEN_NONE args = [op.getarg(0), ConstInt(vrefinfo.TOKEN_NONE)] seo(ResOperation(rop.SETFIELD_GC, args, None, - descr = vrefinfo.descr_virtual_token)) + descr=vrefinfo.descr_virtual_token)) # Note that in some cases the virtual in op.getarg(1) has been forced # already. This is fine. In that case, and *if* a residual # CALL_MAY_FORCE suddenly turns out to access it, then it will @@ -437,6 +446,19 @@ # will work too (but just be a little pointless, as the structure # was already forced). + def _optimize_JIT_FORCE_VIRTUAL(self, op): + vref = self.getvalue(op.getarg(1)) + vrefinfo = self.optimizer.metainterp_sd.virtualref_info + if vref.is_virtual(): + tokenvalue = vref.getfield(vrefinfo.descr_virtual_token, None) + if (tokenvalue is not None and tokenvalue.is_constant() and + tokenvalue.box.getint() == vrefinfo.TOKEN_NONE): + forcedvalue = vref.getfield(vrefinfo.descr_forced, None) + if forcedvalue is not None: + self.make_equal_to(op.result, forcedvalue) + return True + return False + def optimize_GETFIELD_GC(self, op): value = self.getvalue(op.getarg(0)) # If this is an immutable field (as indicated by op.is_always_pure()) diff --git a/rpython/jit/metainterp/test/support.py b/rpython/jit/metainterp/test/support.py --- a/rpython/jit/metainterp/test/support.py +++ b/rpython/jit/metainterp/test/support.py @@ -160,16 +160,17 @@ class JitMixin: basic = True + def check_resops(self, expected=None, **check): get_stats().check_resops(expected=expected, **check) + def check_simple_loop(self, expected=None, **check): get_stats().check_simple_loop(expected=expected, **check) - - def check_trace_count(self, count): # was check_loop_count # The number of traces compiled assert get_stats().compiled_count == count + def check_trace_count_at_most(self, count): assert get_stats().compiled_count <= count @@ -178,11 +179,12 @@ def check_target_token_count(self, count): tokens = get_stats().get_all_jitcell_tokens() - n = sum ([len(t.target_tokens) for t in tokens]) + n = sum([len(t.target_tokens) for t in tokens]) assert n == count def check_enter_count(self, count): assert get_stats().enter_count == count + def check_enter_count_at_most(self, count): assert get_stats().enter_count <= count @@ -192,6 +194,7 @@ def check_aborted_count(self, count): assert get_stats().aborted_count == count + def check_aborted_count_at_least(self, count): assert get_stats().aborted_count >= count diff --git a/rpython/jit/metainterp/test/test_virtualref.py b/rpython/jit/metainterp/test/test_virtualref.py --- a/rpython/jit/metainterp/test/test_virtualref.py +++ b/rpython/jit/metainterp/test/test_virtualref.py @@ -680,7 +680,7 @@ return n res = self.meta_interp(f, [10]) assert res == 0 - self.check_resops({'int_sub': 1, 'int_gt': 1, 'jump': 1}) + self.check_resops({'int_sub': 1, 'int_gt': 1, 'jump': 1, 'guard_true': 1}) class TestLLtype(VRefTests, LLJitMixin): From noreply at buildbot.pypy.org Wed Jan 23 14:48:20 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 23 Jan 2013 14:48:20 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: work some on test_random Message-ID: <20130123134820.99FB31C0634@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60373:68a674e72126 Date: 2013-01-23 15:47 +0200 http://bitbucket.org/pypy/pypy/changeset/68a674e72126/ Log: work some on test_random diff --git a/rpython/jit/backend/test/test_random.py b/rpython/jit/backend/test/test_random.py --- a/rpython/jit/backend/test/test_random.py +++ b/rpython/jit/backend/test/test_random.py @@ -2,7 +2,7 @@ import pytest from rpython.rlib.rarithmetic import intmask, LONG_BIT from rpython.jit.metainterp.history import BasicFailDescr, TreeLoop, BasicFinalDescr -from rpython.jit.metainterp.history import BoxInt, ConstInt, JitCellToken +from rpython.jit.metainterp.history import BoxInt, ConstInt, JitCellToken, Box from rpython.jit.metainterp.history import BoxPtr, ConstPtr, TargetToken from rpython.jit.metainterp.history import BoxFloat, ConstFloat, Const from rpython.jit.metainterp.history import INT, FLOAT @@ -41,10 +41,12 @@ self.should_fail_by = None self.counter = 0 assert len(self.intvars) == len(dict.fromkeys(self.intvars)) + self.descr_counters = {} def fork(self, cpu, loop, vars): fork = self.__class__(cpu, loop, vars) fork.prebuilt_ptr_consts = self.prebuilt_ptr_consts + fork.descr_counters = self.descr_counters return fork def do(self, opnum, argboxes, descr=None): @@ -87,21 +89,13 @@ seen[v] = True return subset - def process_operation(self, s, op, names, subops): + def process_operation(self, s, op, names, namespace): args = [] for v in op.getarglist(): if v in names: args.append(names[v]) -## elif isinstance(v, ConstAddr): -## try: -## name = ''.join([v.value.ptr.name[i] -## for i in range(len(v.value.ptr.name)-1)]) -## except AttributeError: -## args.append('ConstAddr(...)') -## else: -## args.append( -## 'ConstAddr(llmemory.cast_ptr_to_adr(%s_vtable), cpu)' -## % name) + elif isinstance(v, ConstPtr): + args.append('ConstPtr(...') elif isinstance(v, ConstFloat): args.append('ConstFloat(longlong.getfloatstorage(%r))' % v.getfloat()) @@ -115,21 +109,27 @@ try: descrstr = ', ' + op.getdescr()._random_info except AttributeError: - descrstr = ', descr=...' + descrstr = ', descr=' + self.descr_counters.get(op.getdescr(), '...') print >>s, ' ResOperation(rop.%s, [%s], %s%s),' % ( opname[op.getopnum()], ', '.join(args), names[op.result], descrstr) - #if getattr(op, 'suboperations', None) is not None: - # subops.append(op) - def print_loop(self): - #raise PleaseRewriteMe() + def print_loop(self, output, fail_descr=None): def update_names(ops): for op in ops: v = op.result if v not in names: writevar(v, 'tmp') - #if getattr(op, 'suboperations', None) is not None: - # update_names(op.suboperations) + if op.is_guard() or op.opnum == rop.FINISH: + descr = op.getdescr() + no = len(self.descr_counters) + if op.is_guard(): + name = 'faildescr%d' % no + clsname = 'BasicFailDescr' + else: + name = 'finishdescr%d' % no + clsname = 'BasicFinalDescr' + self.descr_counters[descr] = name + print >>s, " %s = %s()" % (name, clsname) def print_loop_prebuilt(ops): for op in ops: @@ -137,13 +137,8 @@ if isinstance(arg, ConstPtr): if arg not in names: writevar(arg, 'const_ptr') - #if getattr(op, 'suboperations', None) is not None: - # print_loop_prebuilt(op.suboperations) - if pytest.config.option.output: - s = open(pytest.config.option.output, "w") - else: - s = sys.stdout + s = output names = {None: 'None'} subops = [] # @@ -161,7 +156,8 @@ update_names(self.loop.operations) print_loop_prebuilt(self.loop.operations) # - print >>s, ' cpu = CPU(None, None)' + if fail_descr is None: + print >>s, ' cpu = CPU(None, None)' if hasattr(self.loop, 'inputargs'): print >>s, ' inputargs = [%s]' % ( ', '.join([names[v] for v in self.loop.inputargs])) @@ -169,29 +165,21 @@ for op in self.loop.operations: self.process_operation(s, op, names, subops) print >>s, ' ]' - while subops: - next = subops.pop(0) - #for op in next.suboperations: - # self.process_operation(s, op, names, subops) - # XXX think what to do about the one below - #if len(op.suboperations) > 1: - # continue # XXX - #[op] = op.suboperations - #assert op.opnum == rop.FAIL - #print >>s, ' operations[%d].suboperations = [' % i - #print >>s, ' ResOperation(rop.FAIL, [%s], None)]' % ( - # ', '.join([names[v] for v in op.args])) - print >>s, ' looptoken = JitCellToken()' - print >>s, ' cpu.compile_loop(inputargs, operations, looptoken)' + if fail_descr is None: + print >>s, ' looptoken = JitCellToken()' + print >>s, ' cpu.compile_loop(inputargs, operations, looptoken)' + else: + print >>s, ' cpu.compile_bridge(%s, inputargs, operations, looptoken)' % self.descr_counters[fail_descr] if hasattr(self.loop, 'inputargs'): + vals = [] for i, v in enumerate(self.loop.inputargs): - if isinstance(v, (BoxFloat, ConstFloat)): - print >>s, (' cpu.set_future_value_float(%d,' - 'longlong.getfloatstorage(%r))' % (i, v.getfloat())) + assert isinstance(v, Box) + if isinstance(v, BoxFloat): + vals.append("longlong.getfloatstorage(%r)" % v.getfloat()) else: - print >>s, ' cpu.set_future_value_int(%d, %d)' % (i, - v.value) - print >>s, ' op = cpu.execute_token(looptoken)' + vals.append("%r" % v.getint()) + print >>s, ' loop_args = [%s]' % ", ".join(vals) + print >>s, ' op = cpu.execute_token(looptoken, *loop_args)' if self.should_fail_by is None: fail_args = self.loop.operations[-1].getarglist() else: @@ -204,8 +192,7 @@ print >>s, (' assert cpu.get_int_value(%d) == %d' % (i, v.value)) self.names = names - if pytest.config.option.output: - s.close() + s.flush() def getfaildescr(self, is_finish=False): if is_finish: @@ -537,8 +524,9 @@ class RandomLoop(object): dont_generate_more = False - def __init__(self, cpu, builder_factory, r, startvars=None): + def __init__(self, cpu, builder_factory, r, startvars=None, output=None): self.cpu = cpu + self.output = output if startvars is None: startvars = [] if cpu.supports_floats: @@ -566,7 +554,8 @@ self.subloops = [] self.build_random_loop(cpu, builder_factory, r, startvars, allow_delay) - def build_random_loop(self, cpu, builder_factory, r, startvars, allow_delay): + def build_random_loop(self, cpu, builder_factory, r, startvars, + allow_delay): loop = TreeLoop('test_random_function') loop.inputargs = startvars[:] @@ -583,6 +572,8 @@ self.loop = loop dump(loop) cpu.compile_loop(loop.inputargs, loop.operations, loop._jitcelltoken) + if self.output: + builder.print_loop(self.output) def insert_label(self, loop, position, r): assert not hasattr(loop, '_targettoken') @@ -641,8 +632,6 @@ self.expected = {} for v in endvars: self.expected[v] = v.value - if pytest.config.option.output: - builder.print_loop() def runjitcelltoken(self): if self.startvars == self.loop.inputargs: @@ -793,6 +782,9 @@ self.builder.cpu.compile_bridge(fail_descr, fail_args, subloop.operations, self.loop._jitcelltoken) + + if self.output: + bridge_builder.print_loop(self.output, fail_descr) return True def dump(loop): @@ -800,21 +792,31 @@ if hasattr(loop, 'inputargs'): print >> sys.stderr, '\t', loop.inputargs for op in loop.operations: - print >> sys.stderr, '\t', op + if op.is_guard(): + print >> sys.stderr, '\t', op, op.getfailargs() + else: + print >> sys.stderr, '\t', op def check_random_function(cpu, BuilderClass, r, num=None, max=None): - loop = RandomLoop(cpu, BuilderClass, r) + if pytest.config.option.output: + output = open(pytest.config.option.output, "w") + else: + output = None + loop = RandomLoop(cpu, BuilderClass, r, output=output) while True: loop.run_loop() if loop.guard_op is not None: if not loop.build_bridge(): break + import pdb + pdb.set_trace() else: break if num is not None: print ' # passed (%d/%d).' % (num + 1, max) else: print ' # passed.' + output.close() print def test_random_function(BuilderClass=OperationBuilder): diff --git a/rpython/jit/backend/x86/test/test_regalloc2.py b/rpython/jit/backend/x86/test/test_regalloc2.py --- a/rpython/jit/backend/x86/test/test_regalloc2.py +++ b/rpython/jit/backend/x86/test/test_regalloc2.py @@ -1,8 +1,13 @@ from rpython.jit.metainterp.history import ResOperation, BoxInt, ConstInt,\ - BasicFailDescr, JitCellToken, BasicFinalDescr + BasicFailDescr, JitCellToken, BasicFinalDescr, TargetToken from rpython.jit.metainterp.resoperation import rop from rpython.jit.backend.detect_cpu import getcpuclass from rpython.jit.backend.x86.arch import WORD +from rpython.jit.tool.oparser import parse +from rpython.rtyper.lltypesystem import lltype, rffi, rclass, llmemory +from rpython.rtyper.llinterp import LLException +from rpython.rtyper.annlowlevel import llhelper +from rpython.jit.codewriter.effectinfo import EffectInfo CPU = getcpuclass() @@ -291,3 +296,19 @@ elif WORD == 8: assert cpu.get_int_value(deadframe, 19) == 19327352832 assert cpu.get_int_value(deadframe, 20) == -49 + +def getllhelper(cpu, f, ARGS, RES): + FPTR = lltype.Ptr(lltype.FuncType([lltype.Signed], lltype.Void)) + fptr = llhelper(FPTR, f) + calldescr = cpu.calldescrof(FPTR.TO, FPTR.TO.ARGS, FPTR.TO.RESULT, + EffectInfo.MOST_GENERAL) + return fptr, calldescr + +def getexception(): + xtp = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True) + xtp.subclassrange_min = 1 + xtp.subclassrange_max = 3 + X = lltype.GcStruct('X', ('parent', rclass.OBJECT), + hints={'vtable': xtp._obj}) + xptr = lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(X)) + return xptr, xtp From noreply at buildbot.pypy.org Wed Jan 23 14:51:48 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 23 Jan 2013 14:51:48 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: I dont need this pdb Message-ID: <20130123135148.BEB521C0634@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60374:a19665b99655 Date: 2013-01-23 15:51 +0200 http://bitbucket.org/pypy/pypy/changeset/a19665b99655/ Log: I dont need this pdb diff --git a/rpython/jit/backend/test/test_random.py b/rpython/jit/backend/test/test_random.py --- a/rpython/jit/backend/test/test_random.py +++ b/rpython/jit/backend/test/test_random.py @@ -808,8 +808,6 @@ if loop.guard_op is not None: if not loop.build_bridge(): break - import pdb - pdb.set_trace() else: break if num is not None: From noreply at buildbot.pypy.org Wed Jan 23 14:53:03 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Wed, 23 Jan 2013 14:53:03 +0100 (CET) Subject: [pypy-commit] pypy inline-virtualref-2: Remove the guard_not_forced and guard_no_exceptions, fix test. Message-ID: <20130123135303.325821C0634@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: inline-virtualref-2 Changeset: r60375:e52b7d1bba2a Date: 2013-01-23 05:52 -0800 http://bitbucket.org/pypy/pypy/changeset/e52b7d1bba2a/ Log: Remove the guard_not_forced and guard_no_exceptions, fix test. diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -3,7 +3,7 @@ from rpython.jit.codewriter.heaptracker import vtable2descr from rpython.jit.metainterp.history import Const, ConstInt, BoxInt from rpython.jit.metainterp.optimizeopt import optimizer -from rpython.jit.metainterp.optimizeopt.optimizer import OptValue +from rpython.jit.metainterp.optimizeopt.optimizer import OptValue, REMOVED from rpython.jit.metainterp.optimizeopt.util import (make_dispatcher_method, descrlist_dict, sort_descrs) from rpython.jit.metainterp.resoperation import rop, ResOperation @@ -387,6 +387,16 @@ self.make_equal_to(box, vvalue) return vvalue + def optimize_GUARD_NO_EXCEPTION(self, op): + if self.last_emitted_operation is REMOVED: + return + self.emit_operation(op) + + def optimize_GUARD_NOT_FORCED(self, op): + if self.last_emitted_operation is REMOVED: + return + self.emit_operation(op) + def optimize_CALL_MAY_FORCE(self, op): effectinfo = op.getdescr().get_extra_info() oopspecindex = effectinfo.oopspecindex @@ -456,6 +466,7 @@ forcedvalue = vref.getfield(vrefinfo.descr_forced, None) if forcedvalue is not None: self.make_equal_to(op.result, forcedvalue) + self.last_emitted_operation = REMOVED return True return False diff --git a/rpython/jit/metainterp/test/test_virtualref.py b/rpython/jit/metainterp/test/test_virtualref.py --- a/rpython/jit/metainterp/test/test_virtualref.py +++ b/rpython/jit/metainterp/test/test_virtualref.py @@ -680,7 +680,10 @@ return n res = self.meta_interp(f, [10]) assert res == 0 - self.check_resops({'int_sub': 1, 'int_gt': 1, 'jump': 1, 'guard_true': 1}) + self.check_resops({ + 'int_sub': 2, 'int_gt': 2, 'jump': 1, 'guard_true': 2, + 'force_token': 2, 'setfield_gc': 1 + }) class TestLLtype(VRefTests, LLJitMixin): From noreply at buildbot.pypy.org Wed Jan 23 14:57:50 2013 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 23 Jan 2013 14:57:50 +0100 (CET) Subject: [pypy-commit] pypy default: fix again Message-ID: <20130123135750.5BA541C0634@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r60376:d5a8cd3f7fd3 Date: 2013-01-23 15:58 +0200 http://bitbucket.org/pypy/pypy/changeset/d5a8cd3f7fd3/ Log: fix again diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -45,12 +45,12 @@ def package(basedir, name='pypy-nightly', rename_pypy_c='pypy', copy_to_dir = None, override_pypy_c = None): - basedir = py.path.local('goal', basedir) + basedir = py.path.local(basedir) if override_pypy_c is None: basename = 'pypy-c' if sys.platform == 'win32': basename += '.exe' - pypy_c = basedir.join(basename) + pypy_c = basedir.join('pypy', 'goal', basename) else: pypy_c = py.path.local(override_pypy_c) if not pypy_c.check(): From noreply at buildbot.pypy.org Wed Jan 23 15:56:19 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 23 Jan 2013 15:56:19 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: work on test_zll_random Message-ID: <20130123145619.0A2DA1C00A8@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60377:19aef0a0fee3 Date: 2013-01-23 16:55 +0200 http://bitbucket.org/pypy/pypy/changeset/19aef0a0fee3/ Log: work on test_zll_random diff --git a/rpython/jit/backend/test/test_ll_random.py b/rpython/jit/backend/test/test_ll_random.py --- a/rpython/jit/backend/test/test_ll_random.py +++ b/rpython/jit/backend/test/test_ll_random.py @@ -258,7 +258,8 @@ names = names[1:] name = r.choice(names) descr = builder.cpu.fielddescrof(S, name) - descr._random_info = 'cpu.fielddescrof(%s, %r)' % (S._name, name) + descr._random_info = 'cpu.fielddescrof(..., %r)' % (name,) + descr._random_type = S TYPE = getattr(S, name) return v, descr, TYPE @@ -279,8 +280,8 @@ v_index = builder.get_index(len(array), r) name = r.choice(A.OF._names) descr = builder.cpu.interiorfielddescrof(A, name) - descr._random_info = 'cpu.interiorfielddescrof(%s, %r)' % (A.OF._name, - name) + descr._random_info = 'cpu.interiorfielddescrof(..., %r)' % (name,) + descr._random_type = A TYPE = getattr(A.OF, name) return v, v_index, descr, TYPE @@ -320,7 +321,8 @@ class NewOperation(test_random.AbstractOperation): def size_descr(self, builder, S): descr = builder.cpu.sizeof(S) - descr._random_info = 'cpu.sizeof(%s)' % (S._name,) + descr._random_info = 'cpu.sizeof(...)' + descr._random_type = S return descr def produce_into(self, builder, r): @@ -339,6 +341,7 @@ def array_descr(self, builder, A): descr = builder.cpu.arraydescrof(A) descr._random_info = 'cpu.arraydescrof(...)' + descr._random_type = A return descr class GetArrayItemOperation(ArrayOperation): diff --git a/rpython/jit/backend/test/test_random.py b/rpython/jit/backend/test/test_random.py --- a/rpython/jit/backend/test/test_random.py +++ b/rpython/jit/backend/test/test_random.py @@ -10,6 +10,7 @@ from rpython.jit.metainterp.executor import execute_nonspec from rpython.jit.metainterp.resoperation import opname from rpython.jit.codewriter import longlong +from rpython.rtyper.lltypesystem import lltype, rstr, rclass class PleaseRewriteMe(Exception): pass @@ -89,13 +90,14 @@ seen[v] = True return subset - def process_operation(self, s, op, names, namespace): + def process_operation(self, s, op, names): args = [] for v in op.getarglist(): if v in names: args.append(names[v]) elif isinstance(v, ConstPtr): - args.append('ConstPtr(...') + assert not v.value # otherwise should be in the names + args.append('ConstPtr(lltype.nullptr(llmemory.GCREF.TO))') elif isinstance(v, ConstFloat): args.append('ConstFloat(longlong.getfloatstorage(%r))' % v.getfloat()) @@ -107,9 +109,12 @@ descrstr = '' else: try: - descrstr = ', ' + op.getdescr()._random_info + descrstr = getattr(op.getdescr(), '_random_info') except AttributeError: - descrstr = ', descr=' + self.descr_counters.get(op.getdescr(), '...') + if op.opnum == rop.LABEL: + descrstr = 'TargetToken()' + else: + descrstr = ', descr=' + self.descr_counters.get(op.getdescr(), '...') print >>s, ' ResOperation(rop.%s, [%s], %s%s),' % ( opname[op.getopnum()], ', '.join(args), names[op.result], descrstr) @@ -138,11 +143,61 @@ if arg not in names: writevar(arg, 'const_ptr') + def type_descr(TP, num): + if TP in TYPE_NAMES: + return TYPE_NAMES[TP] + if isinstance(TP, lltype.Ptr): + return "lltype.Ptr(%s)" % type_descr(TP.TO, num) + if isinstance(TP, lltype.Struct): + if TP._gckind == 'gc': + pref = 'Gc' + else: + pref = '' + fields = [] + for k in TP._names: + v = getattr(TP, k) + fields.append('("%s", %s)' % (k, type_descr(v, num))) + return "lltype.%sStruct('S%d', %s)" % (pref, num, + ", ".join(fields)) + elif isinstance(TP, lltype.GcArray): + return "lltype.GcArray(%s)" % (type_descr(TP.OF, num),) + if TP._name.upper() == TP._name: + return 'rffi.%s' % TP._name + return 'lltype.%s' % TP._name + s = output names = {None: 'None'} - subops = [] + TYPE_NAMES = { + rstr.STR: 'rstr.STR', + rstr.UNICODE: 'rstr.UNICODE', + rclass.OBJECT: 'rclass.OBJECT', + rclass.OBJECT_VTABLE: 'rclass.OBJECT_VTABLE', + } + for op in self.loop.operations: + descr = op.getdescr() + if hasattr(descr, '_random_info'): + num = len(TYPE_NAMES) + tp_name = 'S' + str(num) + descr._random_info = descr._random_info.replace('...', tp_name) + print >>s, " %s = %s" % (tp_name, + type_descr(descr._random_type, num)) + TYPE_NAMES[descr._random_type] = tp_name # def writevar(v, nameprefix, init=''): + if nameprefix == 'const_ptr': + if not v.value: + return 'lltype.nullptr(llmemory.GCREF.TO)' + TYPE = v.value._obj.ORIGTYPE + cont = lltype.cast_opaque_ptr(TYPE, v.value) + if TYPE.TO._is_varsize(): + if isinstance(TYPE.TO, lltype.GcStruct): + lgt = len(cont.chars) + else: + lgt = len(cont) + init = 'lltype.malloc(%s, %d)' % (TYPE_NAMES[TYPE.TO], + lgt) + else: + init = 'lltype.malloc(%s)' % TYPE_NAMES[TYPE.TO] names[v] = '%s%d' % (nameprefix, len(names)) print >>s, ' %s = %s(%s)' % (names[v], v.__class__.__name__, init) @@ -163,7 +218,11 @@ ', '.join([names[v] for v in self.loop.inputargs])) print >>s, ' operations = [' for op in self.loop.operations: - self.process_operation(s, op, names, subops) + self.process_operation(s, op, names) + for i, op in enumerate(self.loop.operations): + if op.is_guard(): + fa = ", ".join([names[v] for v in op.getfailargs()]) + print >>s, ' operations[%d].setfailargs([%s])' % (i, fa) print >>s, ' ]' if fail_descr is None: print >>s, ' looptoken = JitCellToken()' @@ -179,7 +238,7 @@ else: vals.append("%r" % v.getint()) print >>s, ' loop_args = [%s]' % ", ".join(vals) - print >>s, ' op = cpu.execute_token(looptoken, *loop_args)' + print >>s, ' frame = cpu.execute_token(looptoken, *loop_args)' if self.should_fail_by is None: fail_args = self.loop.operations[-1].getarglist() else: @@ -187,9 +246,9 @@ for i, v in enumerate(fail_args): if isinstance(v, (BoxFloat, ConstFloat)): print >>s, (' assert longlong.getrealfloat(' - 'cpu.get_float_value(%d)) == %r' % (i, v.value)) + 'cpu.get_float_value(frame, %d)) == %r' % (i, v.value)) else: - print >>s, (' assert cpu.get_int_value(%d) == %d' + print >>s, (' assert cpu.get_int_value(frame, %d) == %d' % (i, v.value)) self.names = names s.flush() diff --git a/rpython/jit/backend/x86/test/test_regalloc2.py b/rpython/jit/backend/x86/test/test_regalloc2.py --- a/rpython/jit/backend/x86/test/test_regalloc2.py +++ b/rpython/jit/backend/x86/test/test_regalloc2.py @@ -1,5 +1,6 @@ from rpython.jit.metainterp.history import ResOperation, BoxInt, ConstInt,\ - BasicFailDescr, JitCellToken, BasicFinalDescr, TargetToken + BasicFailDescr, JitCellToken, BasicFinalDescr, TargetToken, ConstPtr,\ + BoxPtr from rpython.jit.metainterp.resoperation import rop from rpython.jit.backend.detect_cpu import getcpuclass from rpython.jit.backend.x86.arch import WORD @@ -302,7 +303,7 @@ fptr = llhelper(FPTR, f) calldescr = cpu.calldescrof(FPTR.TO, FPTR.TO.ARGS, FPTR.TO.RESULT, EffectInfo.MOST_GENERAL) - return fptr, calldescr + return rffi.cast(lltype.Signed, fptr), calldescr def getexception(): xtp = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True) @@ -312,3 +313,4 @@ hints={'vtable': xtp._obj}) xptr = lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(X)) return xptr, xtp + From noreply at buildbot.pypy.org Wed Jan 23 16:15:45 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 23 Jan 2013 16:15:45 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: some more work on test_random Message-ID: <20130123151545.269B51C00A8@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60378:6cda1d4693bf Date: 2013-01-23 17:15 +0200 http://bitbucket.org/pypy/pypy/changeset/6cda1d4693bf/ Log: some more work on test_random diff --git a/rpython/jit/backend/test/test_random.py b/rpython/jit/backend/test/test_random.py --- a/rpython/jit/backend/test/test_random.py +++ b/rpython/jit/backend/test/test_random.py @@ -109,10 +109,10 @@ descrstr = '' else: try: - descrstr = getattr(op.getdescr(), '_random_info') + descrstr = ', ' + getattr(op.getdescr(), '_random_info') except AttributeError: if op.opnum == rop.LABEL: - descrstr = 'TargetToken()' + descrstr = ', TargetToken()' else: descrstr = ', descr=' + self.descr_counters.get(op.getdescr(), '...') print >>s, ' ResOperation(rop.%s, [%s], %s%s),' % ( @@ -176,12 +176,15 @@ for op in self.loop.operations: descr = op.getdescr() if hasattr(descr, '_random_info'): - num = len(TYPE_NAMES) - tp_name = 'S' + str(num) - descr._random_info = descr._random_info.replace('...', tp_name) + if descr._random_type in TYPE_NAMES: + tp_name = TYPE_NAMES[descr._random_type] + else: + num = len(TYPE_NAMES) + tp_name = 'S' + str(num) + descr._random_info = descr._random_info.replace('...', tp_name) + TYPE_NAMES[descr._random_type] = tp_name print >>s, " %s = %s" % (tp_name, type_descr(descr._random_type, num)) - TYPE_NAMES[descr._random_type] = tp_name # def writevar(v, nameprefix, init=''): if nameprefix == 'const_ptr': @@ -198,6 +201,7 @@ lgt) else: init = 'lltype.malloc(%s)' % TYPE_NAMES[TYPE.TO] + init = 'lltype.cast_opaque_ptr(llmemory.GCREF, %s)' % init names[v] = '%s%d' % (nameprefix, len(names)) print >>s, ' %s = %s(%s)' % (names[v], v.__class__.__name__, init) diff --git a/rpython/jit/backend/x86/test/test_regalloc2.py b/rpython/jit/backend/x86/test/test_regalloc2.py --- a/rpython/jit/backend/x86/test/test_regalloc2.py +++ b/rpython/jit/backend/x86/test/test_regalloc2.py @@ -1,14 +1,15 @@ from rpython.jit.metainterp.history import ResOperation, BoxInt, ConstInt,\ BasicFailDescr, JitCellToken, BasicFinalDescr, TargetToken, ConstPtr,\ - BoxPtr + BoxPtr, BoxFloat from rpython.jit.metainterp.resoperation import rop from rpython.jit.backend.detect_cpu import getcpuclass from rpython.jit.backend.x86.arch import WORD from rpython.jit.tool.oparser import parse -from rpython.rtyper.lltypesystem import lltype, rffi, rclass, llmemory +from rpython.rtyper.lltypesystem import lltype, rffi, rclass, llmemory, rstr from rpython.rtyper.llinterp import LLException from rpython.rtyper.annlowlevel import llhelper from rpython.jit.codewriter.effectinfo import EffectInfo +from rpython.jit.codewriter import longlong CPU = getcpuclass() @@ -299,7 +300,7 @@ assert cpu.get_int_value(deadframe, 20) == -49 def getllhelper(cpu, f, ARGS, RES): - FPTR = lltype.Ptr(lltype.FuncType([lltype.Signed], lltype.Void)) + FPTR = lltype.Ptr(lltype.FuncType(ARGS, RES)) fptr = llhelper(FPTR, f) calldescr = cpu.calldescrof(FPTR.TO, FPTR.TO.ARGS, FPTR.TO.RESULT, EffectInfo.MOST_GENERAL) @@ -314,3 +315,5 @@ xptr = lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(X)) return xptr, xtp +def test_bug2(): + pass From noreply at buildbot.pypy.org Wed Jan 23 16:29:02 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 23 Jan 2013 16:29:02 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: test_random is almost ideal by now\ Message-ID: <20130123152902.7A41B1C0627@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60379:93d898dde547 Date: 2013-01-23 17:28 +0200 http://bitbucket.org/pypy/pypy/changeset/93d898dde547/ Log: test_random is almost ideal by now\ diff --git a/rpython/jit/backend/test/test_random.py b/rpython/jit/backend/test/test_random.py --- a/rpython/jit/backend/test/test_random.py +++ b/rpython/jit/backend/test/test_random.py @@ -118,7 +118,7 @@ print >>s, ' ResOperation(rop.%s, [%s], %s%s),' % ( opname[op.getopnum()], ', '.join(args), names[op.result], descrstr) - def print_loop(self, output, fail_descr=None): + def print_loop(self, output, fail_descr=None, fail_args=None): def update_names(ops): for op in ops: v = op.result @@ -143,11 +143,22 @@ if arg not in names: writevar(arg, 'const_ptr') - def type_descr(TP, num): + def type_descr(TP): if TP in TYPE_NAMES: return TYPE_NAMES[TP] + elif isinstance(TP, lltype.Primitive): + return _type_descr(TP) # don't cache + else: + descr = _type_descr(TP) + no = len(TYPE_NAMES) + tp_name = 'S' + str(no) + TYPE_NAMES[TP] = tp_name + print >>s, ' %s = %s' % (tp_name, descr) + return tp_name + + def _type_descr(TP): if isinstance(TP, lltype.Ptr): - return "lltype.Ptr(%s)" % type_descr(TP.TO, num) + return "lltype.Ptr(%s)" % type_descr(TP.TO) if isinstance(TP, lltype.Struct): if TP._gckind == 'gc': pref = 'Gc' @@ -156,11 +167,11 @@ fields = [] for k in TP._names: v = getattr(TP, k) - fields.append('("%s", %s)' % (k, type_descr(v, num))) - return "lltype.%sStruct('S%d', %s)" % (pref, num, + fields.append('("%s", %s)' % (k, type_descr(v))) + return "lltype.%sStruct('Sx', %s)" % (pref, ", ".join(fields)) elif isinstance(TP, lltype.GcArray): - return "lltype.GcArray(%s)" % (type_descr(TP.OF, num),) + return "lltype.GcArray(%s)" % (type_descr(TP.OF),) if TP._name.upper() == TP._name: return 'rffi.%s' % TP._name return 'lltype.%s' % TP._name @@ -176,15 +187,9 @@ for op in self.loop.operations: descr = op.getdescr() if hasattr(descr, '_random_info'): - if descr._random_type in TYPE_NAMES: - tp_name = TYPE_NAMES[descr._random_type] - else: - num = len(TYPE_NAMES) - tp_name = 'S' + str(num) - descr._random_info = descr._random_info.replace('...', tp_name) - TYPE_NAMES[descr._random_type] = tp_name - print >>s, " %s = %s" % (tp_name, - type_descr(descr._random_type, num)) + tp_name = type_descr(descr._random_type) + descr._random_info = descr._random_info.replace('...', tp_name) + # def writevar(v, nameprefix, init=''): if nameprefix == 'const_ptr': @@ -217,17 +222,21 @@ # if fail_descr is None: print >>s, ' cpu = CPU(None, None)' + print >>s, ' cpu.setup_once()' if hasattr(self.loop, 'inputargs'): print >>s, ' inputargs = [%s]' % ( ', '.join([names[v] for v in self.loop.inputargs])) + else: + print >>s, ' inputargs = [%s]' % ( + ', '.join([names[v] for v in fail_args])) print >>s, ' operations = [' for op in self.loop.operations: - self.process_operation(s, op, names) + self.process_operation(s, op, names) + print >>s, ' ]' for i, op in enumerate(self.loop.operations): if op.is_guard(): fa = ", ".join([names[v] for v in op.getfailargs()]) print >>s, ' operations[%d].setfailargs([%s])' % (i, fa) - print >>s, ' ]' if fail_descr is None: print >>s, ' looptoken = JitCellToken()' print >>s, ' cpu.compile_loop(inputargs, operations, looptoken)' @@ -847,7 +856,7 @@ self.loop._jitcelltoken) if self.output: - bridge_builder.print_loop(self.output, fail_descr) + bridge_builder.print_loop(self.output, fail_descr, fail_args) return True def dump(loop): diff --git a/rpython/jit/backend/x86/test/test_regalloc2.py b/rpython/jit/backend/x86/test/test_regalloc2.py --- a/rpython/jit/backend/x86/test/test_regalloc2.py +++ b/rpython/jit/backend/x86/test/test_regalloc2.py @@ -316,4 +316,4 @@ return xptr, xtp def test_bug2(): - pass + xxx From noreply at buildbot.pypy.org Wed Jan 23 16:46:10 2013 From: noreply at buildbot.pypy.org (kkris) Date: Wed, 23 Jan 2013 16:46:10 +0100 (CET) Subject: [pypy-commit] pypy default: Add split-rpython branch to whatsnew-head.rst Message-ID: <20130123154610.3A3091C0627@cobra.cs.uni-duesseldorf.de> Author: Kristoffer Kleine Branch: Changeset: r60380:e7099b42d5a6 Date: 2013-01-23 16:31 +0100 http://bitbucket.org/pypy/pypy/changeset/e7099b42d5a6/ Log: Add split-rpython branch to whatsnew-head.rst diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -5,6 +5,9 @@ .. this is a revision shortly after release-2.0-beta1 .. startrev: 0e6161a009c6 +.. branch: split-rpython +Split rpython and pypy into seperate directories + .. branch: callback-jit Callbacks from C are now better JITted From noreply at buildbot.pypy.org Wed Jan 23 17:40:06 2013 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 23 Jan 2013 17:40:06 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: merge default into branch Message-ID: <20130123164006.B82C21C0627@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: missing-ndarray-attributes Changeset: r60381:37442491cfde Date: 2013-01-23 18:09 +0200 http://bitbucket.org/pypy/pypy/changeset/37442491cfde/ Log: merge default into branch diff --git a/lib-python/conftest.py b/lib-python/conftest.py --- a/lib-python/conftest.py +++ b/lib-python/conftest.py @@ -17,8 +17,8 @@ from pypy.conftest import option as pypy_option from pypy.tool.pytest import appsupport -from pypy.tool.pytest.confpath import pypydir, testdir, testresultdir -from pypy.config.parse import parse_info +from pypy.tool.pytest.confpath import pypydir, rpythondir, testdir, testresultdir +from rpython.config.parse import parse_info pytest_plugins = "resultlog", rsyncdirs = ['.', '../pypy/'] @@ -40,7 +40,7 @@ dest="unittest_filter", help="Similar to -k, XXX") def gettimeout(timeout): - from test import pystone + from rpython.translator.test import rpystone if timeout.endswith('mp'): megapystone = float(timeout[:-2]) t, stone = pystone.Proc0(10000) @@ -584,7 +584,7 @@ watchdog_name = 'watchdog_nt.py' else: watchdog_name = 'watchdog.py' - watchdog_script = pypydir.join('tool', watchdog_name) + watchdog_script = rpythondir.join('tool', watchdog_name) regr_script = pypydir.join('tool', 'pytest', 'run-script', 'regrverbose.py') diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -5,6 +5,9 @@ .. this is a revision shortly after release-2.0-beta1 .. startrev: 0e6161a009c6 +.. branch: split-rpython +Split rpython and pypy into seperate directories + .. branch: callback-jit Callbacks from C are now better JITted diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py --- a/pypy/module/rctime/interp_time.py +++ b/pypy/module/rctime/interp_time.py @@ -448,10 +448,6 @@ raise OperationError(space.w_ValueError, space.wrap("year out of range")) - if rffi.getintfield(glob_buf, 'c_tm_wday') < 0: - raise OperationError(space.w_ValueError, - space.wrap("day of week out of range")) - rffi.setintfield(glob_buf, 'c_tm_year', y - 1900) rffi.setintfield(glob_buf, 'c_tm_mon', rffi.getintfield(glob_buf, 'c_tm_mon') - 1) @@ -460,6 +456,12 @@ rffi.setintfield(glob_buf, 'c_tm_yday', rffi.getintfield(glob_buf, 'c_tm_yday') - 1) + # tm_wday does not need checking of its upper-bound since taking "% + # 7" in gettmarg() automatically restricts the range. + if rffi.getintfield(glob_buf, 'c_tm_wday') < 0: + raise OperationError(space.w_ValueError, + space.wrap("day of week out of range")) + return glob_buf def time(space): diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py --- a/pypy/module/rctime/test/test_rctime.py +++ b/pypy/module/rctime/test/test_rctime.py @@ -113,6 +113,9 @@ if os.name != 'nt': assert rctime.mktime(rctime.localtime(-1)) == -1 + res = rctime.mktime((2000, 1, 1, 0, 0, 0, -1, -1, -1)) + assert rctime.ctime(res) == 'Sat Jan 1 00:00:00 2000' + def test_asctime(self): import time as rctime rctime.asctime() diff --git a/pypy/test_all.py b/pypy/test_all.py --- a/pypy/test_all.py +++ b/pypy/test_all.py @@ -24,7 +24,7 @@ print >> sys.stderr, __doc__ sys.exit(2) #Add toplevel repository dir to sys.path - sys.path.insert(0,os.path.dirname(os.path.dirname(__file__))) + sys.path.insert(0,os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) import pytest import pytest_cov sys.exit(pytest.main(plugins=[pytest_cov])) diff --git a/pypy/tool/pytest/confpath.py b/pypy/tool/pytest/confpath.py --- a/pypy/tool/pytest/confpath.py +++ b/pypy/tool/pytest/confpath.py @@ -1,8 +1,10 @@ import py import pypy +import rpython from pypy.tool import lib_pypy pypydir = py.path.local(pypy.__file__).dirpath() +rpythondir = py.path.local(rpython.__file__).dirpath() distdir = pypydir.dirpath() testresultdir = distdir.join('testresult') assert pypydir.check(dir=1) diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -50,7 +50,7 @@ basename = 'pypy-c' if sys.platform == 'win32': basename += '.exe' - pypy_c = basedir.join(basename) + pypy_c = basedir.join('pypy', 'goal', basename) else: pypy_c = py.path.local(override_pypy_c) if not pypy_c.check(): From noreply at buildbot.pypy.org Wed Jan 23 19:39:24 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Wed, 23 Jan 2013 19:39:24 +0100 (CET) Subject: [pypy-commit] pypy py3k: merge default Message-ID: <20130123183924.B50891C00A8@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60382:816887f2e726 Date: 2013-01-23 10:39 -0800 http://bitbucket.org/pypy/pypy/changeset/816887f2e726/ Log: merge default diff --git a/lib-python/conftest.py b/lib-python/conftest.py --- a/lib-python/conftest.py +++ b/lib-python/conftest.py @@ -17,8 +17,8 @@ from pypy.conftest import option as pypy_option from pypy.tool.pytest import appsupport -from pypy.tool.pytest.confpath import pypydir, testdir, testresultdir -from pypy.config.parse import parse_info +from pypy.tool.pytest.confpath import pypydir, rpythondir, testdir, testresultdir +from rpython.config.parse import parse_info pytest_plugins = "resultlog", rsyncdirs = ['.', '../pypy/'] @@ -40,7 +40,7 @@ dest="unittest_filter", help="Similar to -k, XXX") def gettimeout(timeout): - from test import pystone + from rpython.translator.test import rpystone if timeout.endswith('mp'): megapystone = float(timeout[:-2]) t, stone = pystone.Proc0(10000) @@ -522,7 +522,7 @@ watchdog_name = 'watchdog_nt.py' else: watchdog_name = 'watchdog.py' - watchdog_script = pypydir.join('tool', watchdog_name) + watchdog_script = rpythondir.join('tool', watchdog_name) regr_script = pypydir.join('tool', 'pytest', 'run-script', 'regrverbose.py') diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -5,6 +5,9 @@ .. this is a revision shortly after release-2.0-beta1 .. startrev: 0e6161a009c6 +.. branch: split-rpython +Split rpython and pypy into seperate directories + .. branch: callback-jit Callbacks from C are now better JITted diff --git a/pypy/test_all.py b/pypy/test_all.py --- a/pypy/test_all.py +++ b/pypy/test_all.py @@ -24,7 +24,7 @@ print >> sys.stderr, __doc__ sys.exit(2) #Add toplevel repository dir to sys.path - sys.path.insert(0,os.path.dirname(os.path.dirname(__file__))) + sys.path.insert(0,os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) import pytest import pytest_cov sys.exit(pytest.main(plugins=[pytest_cov])) diff --git a/pypy/tool/pytest/confpath.py b/pypy/tool/pytest/confpath.py --- a/pypy/tool/pytest/confpath.py +++ b/pypy/tool/pytest/confpath.py @@ -1,8 +1,10 @@ import py import pypy +import rpython from pypy.tool import lib_pypy pypydir = py.path.local(pypy.__file__).dirpath() +rpythondir = py.path.local(rpython.__file__).dirpath() distdir = pypydir.dirpath() testresultdir = distdir.join('testresult') assert pypydir.check(dir=1) diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -50,7 +50,7 @@ basename = 'pypy-c' if sys.platform == 'win32': basename += '.exe' - pypy_c = basedir.join(basename) + pypy_c = basedir.join('pypy', 'goal', basename) else: pypy_c = py.path.local(override_pypy_c) if not pypy_c.check(): From noreply at buildbot.pypy.org Wed Jan 23 19:52:32 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 23 Jan 2013 19:52:32 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: try not using boehm, when we dno't have to Message-ID: <20130123185232.C06161C00A8@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60383:348409f40c57 Date: 2013-01-23 20:52 +0200 http://bitbucket.org/pypy/pypy/changeset/348409f40c57/ Log: try not using boehm, when we dno't have to diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py --- a/rpython/jit/backend/llsupport/gc.py +++ b/rpython/jit/backend/llsupport/gc.py @@ -154,6 +154,7 @@ def configure_boehm_once(cls): """ Configure boehm only once, since we don't cache failures """ + return if hasattr(cls, 'malloc_fn_ptr'): return cls.malloc_fn_ptr from rpython.rtyper.tool import rffi_platform @@ -193,9 +194,33 @@ def __init__(self, gcdescr, translator, rtyper): GcLLDescription.__init__(self, gcdescr, translator, rtyper) # grab a pointer to the Boehm 'malloc' function - self.malloc_fn_ptr = self.configure_boehm_once() + #self.malloc_fn_ptr = self.configure_boehm_once() self._setup_str() self._make_functions() + self.memory = 0 + self.reset_mem() + + def reset_mem(self): + malloc = rffi.llexternal('malloc', [lltype.Signed], lltype.Signed) + free = rffi.llexternal('free', [lltype.Signed], lltype.Void) + if self.memory != 0: + free(self.memory) + self.memory = 0 + A_LOT = 100*1024*1024 + self.memory = malloc(A_LOT) + self.mem_top = self.memory + A_LOT + self.mem_ptr = self.memory + + def malloc_fn_ptr(self, size): + # aligned + if size & 7: + size += 8 + size &= ~7 + from rpython.rtyper.lltypesystem.ll2ctypes import _llgcopaque + assert self.mem_ptr + size < self.mem_top + r = self.mem_ptr + self.mem_ptr += size + return lltype._ptr(llmemory.GCREF, _llgcopaque(r)) def _make_functions(self): diff --git a/rpython/jit/backend/test/test_random.py b/rpython/jit/backend/test/test_random.py --- a/rpython/jit/backend/test/test_random.py +++ b/rpython/jit/backend/test/test_random.py @@ -886,7 +886,12 @@ print ' # passed (%d/%d).' % (num + 1, max) else: print ' # passed.' - output.close() + if pytest.config.option.output: + output.close() + if hasattr(cpu, 'gc_ll_descr'): + func = getattr(cpu.gc_ll_descr, 'reset_mem', None) + if func: + func() print def test_random_function(BuilderClass=OperationBuilder): diff --git a/rpython/jit/backend/x86/test/test_regalloc2.py b/rpython/jit/backend/x86/test/test_regalloc2.py --- a/rpython/jit/backend/x86/test/test_regalloc2.py +++ b/rpython/jit/backend/x86/test/test_regalloc2.py @@ -1,6 +1,6 @@ from rpython.jit.metainterp.history import ResOperation, BoxInt, ConstInt,\ BasicFailDescr, JitCellToken, BasicFinalDescr, TargetToken, ConstPtr,\ - BoxPtr, BoxFloat + BoxPtr, BoxFloat, ConstFloat from rpython.jit.metainterp.resoperation import rop from rpython.jit.backend.detect_cpu import getcpuclass from rpython.jit.backend.x86.arch import WORD @@ -9,7 +9,7 @@ from rpython.rtyper.llinterp import LLException from rpython.rtyper.annlowlevel import llhelper from rpython.jit.codewriter.effectinfo import EffectInfo -from rpython.jit.codewriter import longlong +from rpython.jit.codewriter import longlong, heaptracker CPU = getcpuclass() @@ -306,14 +306,30 @@ EffectInfo.MOST_GENERAL) return rffi.cast(lltype.Signed, fptr), calldescr -def getexception(): +def getexception(cpu, ARGS): xtp = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True) xtp.subclassrange_min = 1 xtp.subclassrange_max = 3 X = lltype.GcStruct('X', ('parent', rclass.OBJECT), hints={'vtable': xtp._obj}) - xptr = lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(X)) - return xptr, xtp + xptr = lltype.malloc(X) + vtableptr = X._hints['vtable']._as_ptr() -def test_bug2(): - xxx + def f(*args): + raise LLException(vtableptr, xptr) + + fptr, funcdescr = getllhelper(cpu, f, ARGS, lltype.Void) + + return heaptracker.adr2int(llmemory.cast_ptr_to_adr(vtableptr)), fptr, funcdescr + +def getvtable(cpu, S=None): + cls1 = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True) + cls1.subclassrange_min = 1 + cls1.subclassrange_max = 3 + if S is not None: + descr = cpu.sizeof(S) + if not hasattr(cpu.tracker, '_all_size_descrs_with_vtable'): + cpu.tracker._all_size_descrs_with_vtable = [] + cpu.tracker._all_size_descrs_with_vtable.append(descr) + descr._corresponding_vtable = cls1 + return llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(cls1), "symbolic") From noreply at buildbot.pypy.org Wed Jan 23 19:55:58 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 23 Jan 2013 19:55:58 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: try harder to use valgrind Message-ID: <20130123185558.35D6D1C00A8@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60384:9de45c866547 Date: 2013-01-23 20:55 +0200 http://bitbucket.org/pypy/pypy/changeset/9de45c866547/ Log: try harder to use valgrind diff --git a/rpython/jit/backend/x86/valgrind.py b/rpython/jit/backend/x86/valgrind.py --- a/rpython/jit/backend/x86/valgrind.py +++ b/rpython/jit/backend/x86/valgrind.py @@ -26,5 +26,5 @@ # ____________________________________________________________ def discard_translations(data, size): - if we_are_translated() and VALGRIND_DISCARD_TRANSLATIONS is not None: - VALGRIND_DISCARD_TRANSLATIONS(llmemory.cast_int_to_adr(data), size) + #if we_are_translated() and VALGRIND_DISCARD_TRANSLATIONS is not None: + VALGRIND_DISCARD_TRANSLATIONS(llmemory.cast_int_to_adr(data), size) From noreply at buildbot.pypy.org Wed Jan 23 20:12:08 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 23 Jan 2013 20:12:08 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: give up on running valgrind Message-ID: <20130123191208.67A4F1C00A8@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60385:85c727af38e8 Date: 2013-01-23 21:11 +0200 http://bitbucket.org/pypy/pypy/changeset/85c727af38e8/ Log: give up on running valgrind diff --git a/rpython/jit/backend/x86/valgrind.py b/rpython/jit/backend/x86/valgrind.py --- a/rpython/jit/backend/x86/valgrind.py +++ b/rpython/jit/backend/x86/valgrind.py @@ -26,5 +26,5 @@ # ____________________________________________________________ def discard_translations(data, size): - #if we_are_translated() and VALGRIND_DISCARD_TRANSLATIONS is not None: - VALGRIND_DISCARD_TRANSLATIONS(llmemory.cast_int_to_adr(data), size) + if we_are_translated() and VALGRIND_DISCARD_TRANSLATIONS is not None: + VALGRIND_DISCARD_TRANSLATIONS(llmemory.cast_int_to_adr(data), size) From noreply at buildbot.pypy.org Wed Jan 23 20:12:09 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 23 Jan 2013 20:12:09 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: another stress test Message-ID: <20130123191209.B2C051C00A8@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60386:8d265133ef42 Date: 2013-01-23 21:11 +0200 http://bitbucket.org/pypy/pypy/changeset/8d265133ef42/ Log: another stress test diff --git a/rpython/jit/backend/x86/test/test_regalloc2.py b/rpython/jit/backend/x86/test/test_regalloc2.py --- a/rpython/jit/backend/x86/test/test_regalloc2.py +++ b/rpython/jit/backend/x86/test/test_regalloc2.py @@ -306,7 +306,7 @@ EffectInfo.MOST_GENERAL) return rffi.cast(lltype.Signed, fptr), calldescr -def getexception(cpu, ARGS): +def getexception(cpu, count): xtp = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True) xtp.subclassrange_min = 1 xtp.subclassrange_max = 3 @@ -318,10 +318,16 @@ def f(*args): raise LLException(vtableptr, xptr) - fptr, funcdescr = getllhelper(cpu, f, ARGS, lltype.Void) + fptr, funcdescr = getllhelper(cpu, f, [lltype.Signed] * count, lltype.Void) return heaptracker.adr2int(llmemory.cast_ptr_to_adr(vtableptr)), fptr, funcdescr +def getnoexception(cpu, count): + def f(*args): + return sum(args) + + return getllhelper(cpu, f, [lltype.Signed] * count, lltype.Signed) + def getvtable(cpu, S=None): cls1 = lltype.malloc(rclass.OBJECT_VTABLE, immortal=True) cls1.subclassrange_min = 1 @@ -333,3 +339,282 @@ cpu.tracker._all_size_descrs_with_vtable.append(descr) descr._corresponding_vtable = cls1 return llmemory.cast_adr_to_int(llmemory.cast_ptr_to_adr(cls1), "symbolic") + +def test_bug_2(): + cpu = CPU(None, None) + cpu.setup_once() + + for i in range(1000): + S4 = lltype.Struct('Sx', ("f0", lltype.Char), ("f1", lltype.Signed), ("f2", lltype.Signed), ("f3", lltype.Signed)) + S5 = lltype.GcArray(S4) + v1 = BoxInt() + v2 = BoxInt() + v3 = BoxInt() + v4 = BoxInt() + v5 = BoxInt() + v6 = BoxInt() + v7 = BoxInt() + v8 = BoxInt() + v9 = BoxInt() + v10 = BoxInt() + tmp11 = BoxInt() + tmp12 = BoxPtr() + faildescr0 = BasicFailDescr() + tmp13 = BoxPtr() + faildescr1 = BasicFailDescr() + finishdescr2 = BasicFinalDescr() + const_ptr14 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(rstr.STR, 1))) + const_ptr15 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(rstr.UNICODE, 489))) + const_ptr16 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(rstr.UNICODE, 16))) + const_ptr17 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S5, 299))) + inputargs = [v1, v2, v3, v4, v5, v6, v7, v8, v9, v10] + + xtp, func, funcdescr = getexception(cpu, 3) + xtp2, func2, func2descr = getexception(cpu, 2) + + operations = [ + ResOperation(rop.STRGETITEM, [const_ptr14, ConstInt(0)], tmp11), + ResOperation(rop.LABEL, [v1, v2, tmp11, v3, v4, v5, v6, v7, v8, v9, v10], None, TargetToken()), + ResOperation(rop.UNICODESETITEM, [const_ptr15, v4, ConstInt(22)], None), + ResOperation(rop.CALL, [ConstInt(func), v2, v1, v9], None, descr=funcdescr), + ResOperation(rop.GUARD_EXCEPTION, [ConstInt(xtp)], tmp12, descr=faildescr0), + ResOperation(rop.UNICODESETITEM, [const_ptr16, ConstInt(13), ConstInt(9)], None), + ResOperation(rop.SETINTERIORFIELD_GC, [const_ptr17, v3, v7], None, cpu.interiorfielddescrof(S5, 'f3')), + ResOperation(rop.CALL, [ConstInt(func2), v7, v10], None, descr=func2descr), + ResOperation(rop.GUARD_NO_EXCEPTION, [], tmp13, descr=faildescr1), + ResOperation(rop.FINISH, [], None, descr=finishdescr2), + ] + operations[4].setfailargs([v4, v8, v10, v2, v9, v7, v6, v1]) + operations[8].setfailargs([v3, v9, v2, v6, v4]) + looptoken = JitCellToken() + cpu.compile_loop(inputargs, operations, looptoken) + loop_args = [1, -39, 46, 21, 16, 6, -4611686018427387905, 12, 14, 2] + frame = cpu.execute_token(looptoken, *loop_args) + assert cpu.get_int_value(frame, 0) == 46 + assert cpu.get_int_value(frame, 1) == 14 + assert cpu.get_int_value(frame, 2) == -39 + assert cpu.get_int_value(frame, 3) == 6 + assert cpu.get_int_value(frame, 4) == 21 + S4 = lltype.GcStruct('Sx', ("parent", rclass.OBJECT), ("f0", lltype.Signed)) + S5 = lltype.GcStruct('Sx', ("f0", lltype.Signed)) + S6 = lltype.GcArray(lltype.Signed) + S7 = lltype.GcStruct('Sx', ("parent", rclass.OBJECT), ("f0", lltype.Char)) + S8 = lltype.Struct('Sx', ("f0", lltype.Char), ("f1", lltype.Signed), ("f2", lltype.Signed), ("f3", lltype.Signed)) + S9 = lltype.GcArray(S8) + v1 = BoxInt() + v2 = BoxInt() + v3 = BoxInt() + v4 = BoxInt() + v5 = BoxInt() + v6 = BoxInt() + v7 = BoxInt() + v8 = BoxInt() + v9 = BoxInt() + v10 = BoxInt() + v11 = BoxInt() + v12 = BoxInt() + v13 = BoxInt() + v14 = BoxInt() + v15 = BoxInt() + v16 = BoxInt() + v17 = BoxInt() + v18 = BoxInt() + v19 = BoxInt() + p20 = BoxPtr() + tmp21 = BoxPtr() + faildescr3 = BasicFailDescr() + tmp22 = BoxPtr() + faildescr4 = BasicFailDescr() + tmp23 = BoxInt() + tmp24 = BoxInt() + tmp25 = BoxInt() + tmp26 = BoxInt() + tmp27 = BoxInt() + tmp28 = BoxInt() + tmp29 = BoxInt() + faildescr5 = BasicFailDescr() + tmp30 = BoxPtr() + faildescr6 = BasicFailDescr() + finishdescr7 = BasicFinalDescr() + const_ptr31 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S4))) + const_ptr32 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(rstr.STR, 46))) + const_ptr33 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S5))) + const_ptr34 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(rstr.UNICODE, 26))) + const_ptr35 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(rstr.UNICODE, 15))) + const_ptr36 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S7))) + const_ptr37 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(rstr.STR, 484))) + const_ptr38 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S9, 299))) + inputargs = [v1, v2, v3, v4, v5] + + func3, func3descr = getnoexception(cpu, 5) + xtp3, func4, func4descr = getexception(cpu, 10) + + operations = [ + ResOperation(rop.GUARD_EXCEPTION, [ConstInt(xtp2)], tmp21, descr=faildescr3), + ResOperation(rop.INT_IS_ZERO, [v4], v6), + ResOperation(rop.INT_NE, [v6, ConstInt(13)], v7), + ResOperation(rop.GETFIELD_GC, [const_ptr31], v8, cpu.fielddescrof(S4, 'f0')), + ResOperation(rop.STRSETITEM, [const_ptr32, v6, ConstInt(0)], None), + ResOperation(rop.NEWSTR, [ConstInt(5)], tmp22), + ResOperation(rop.STRSETITEM, [tmp22, ConstInt(0), ConstInt(42)], None), + ResOperation(rop.STRSETITEM, [tmp22, ConstInt(1), ConstInt(42)], None), + ResOperation(rop.STRSETITEM, [tmp22, ConstInt(2), ConstInt(20)], None), + ResOperation(rop.STRSETITEM, [tmp22, ConstInt(3), ConstInt(48)], None), + ResOperation(rop.STRSETITEM, [tmp22, ConstInt(4), ConstInt(6)], None), + ResOperation(rop.GETFIELD_GC, [const_ptr33], v9, cpu.fielddescrof(S5, 'f0')), + ResOperation(rop.UNICODESETITEM, [const_ptr34, ConstInt(24), ConstInt(65533)], None), + ResOperation(rop.GETFIELD_GC, [const_ptr31], v10, cpu.fielddescrof(S4, 'f0')), + ResOperation(rop.INT_NE, [v10, ConstInt(25)], v11), + ResOperation(rop.CALL, [ConstInt(func3), v5, v1, v8, v3, v2], v12, descr=func3descr), + ResOperation(rop.GUARD_NO_EXCEPTION, [], None, descr=faildescr4), + ResOperation(rop.UNICODELEN, [const_ptr35], tmp23), + ResOperation(rop.NEW_ARRAY, [v2], p20, cpu.arraydescrof(S6)), + ResOperation(rop.GETFIELD_GC, [const_ptr36], v13, cpu.fielddescrof(S7, 'f0')), + ResOperation(rop.INT_OR, [v8, ConstInt(2)], tmp24), + ResOperation(rop.INT_FLOORDIV, [ConstInt(8), tmp24], v14), + ResOperation(rop.GETARRAYITEM_GC, [p20, ConstInt(3)], v15, cpu.arraydescrof(S6)), + ResOperation(rop.COPYSTRCONTENT, [tmp22, const_ptr37, ConstInt(1), ConstInt(163), ConstInt(0)], None), + ResOperation(rop.COPYUNICODECONTENT, [const_ptr35, const_ptr34, ConstInt(13), ConstInt(0), v6], None), + ResOperation(rop.STRGETITEM, [tmp22, v6], tmp25), + ResOperation(rop.STRGETITEM, [tmp22, ConstInt(0)], tmp26), + ResOperation(rop.GETINTERIORFIELD_GC, [const_ptr38, v13], v16, cpu.interiorfielddescrof(S9, 'f0')), + ResOperation(rop.INT_GE, [v4, v5], v17), + ResOperation(rop.INT_OR, [v13, ConstInt(2)], tmp27), + ResOperation(rop.INT_FLOORDIV, [ConstInt(12), tmp27], v18), + ResOperation(rop.INT_AND, [v1, ConstInt(-4)], tmp28), + ResOperation(rop.INT_OR, [tmp28, ConstInt(2)], tmp29), + ResOperation(rop.INT_FLOORDIV, [v15, tmp29], v19), + ResOperation(rop.GUARD_FALSE, [v17], None, descr=faildescr5), + ResOperation(rop.UNICODESETITEM, [const_ptr34, ConstInt(20), ConstInt(65522)], None), + ResOperation(rop.CALL, [ConstInt(func4), v3, v9, v10, v8, v11, v5, v13, v14, v15, v6], None, descr=func4descr), + ResOperation(rop.GUARD_NO_EXCEPTION, [], tmp30, descr=faildescr6), + ResOperation(rop.FINISH, [], None, descr=finishdescr7), + ] + operations[0].setfailargs([]) + operations[16].setfailargs([v5, v9]) + operations[34].setfailargs([]) + operations[37].setfailargs([v12, v19, v10, v7, v4, v8, v18, v15, v9]) + cpu.compile_bridge(faildescr1, inputargs, operations, looptoken) + frame = cpu.execute_token(looptoken, *loop_args) + #assert cpu.get_int_value(frame, 0) == -9223372036854775766 + assert cpu.get_int_value(frame, 1) == 0 + #assert cpu.get_int_value(frame, 2) == -9223372036854775808 + assert cpu.get_int_value(frame, 3) == 1 + assert cpu.get_int_value(frame, 4) == 6 + #assert cpu.get_int_value(frame, 5) == -9223372036854775808 + assert cpu.get_int_value(frame, 6) == 0 + assert cpu.get_int_value(frame, 7) == 0 + #assert cpu.get_int_value(frame, 8) == 26 + S4 = lltype.GcStruct('Sx', ("parent", rclass.OBJECT), ("f0", lltype.Signed), ("f1", lltype.Signed)) + S5 = lltype.GcStruct('Sx', ("parent", rclass.OBJECT), ("f0", lltype.Signed)) + S6 = lltype.GcStruct('Sx', ("f0", lltype.Signed), ("f1", rffi.UCHAR)) + v1 = BoxInt() + v2 = BoxInt() + v3 = BoxInt() + v4 = BoxInt() + v5 = BoxInt() + v6 = BoxInt() + v7 = BoxInt() + v8 = BoxInt() + v9 = BoxInt() + v10 = BoxInt() + v11 = BoxInt() + v12 = BoxInt() + v13 = BoxInt() + v14 = BoxInt() + v15 = BoxInt() + v16 = BoxInt() + v17 = BoxInt() + v18 = BoxInt() + tmp19 = BoxPtr() + faildescr8 = BasicFailDescr() + tmp20 = BoxInt() + tmp21 = BoxInt() + tmp22 = BoxInt() + tmp23 = BoxInt() + faildescr9 = BasicFailDescr() + tmp24 = BoxInt() + tmp25 = BoxInt() + tmp26 = BoxInt() + tmp27 = BoxPtr() + tmp28 = BoxPtr() + faildescr10 = BasicFailDescr() + finishdescr11 = BasicFinalDescr() + const_ptr29 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S4))) + const_ptr30 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(rstr.UNICODE, 26))) + const_ptr31 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(rstr.UNICODE, 1))) + const_ptr32 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S5))) + const_ptr33 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S6))) + const_ptr34 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(rstr.STR, 26))) + inputargs = [v1, v2, v3, v4, v5, v6, v7, v8, v9] + operations = [ + ResOperation(rop.GUARD_EXCEPTION, [ConstInt(xtp3)], tmp19, descr=faildescr8), + ResOperation(rop.SETFIELD_GC, [const_ptr29, v7], None, cpu.fielddescrof(S4, 'f0')), + ResOperation(rop.UNICODEGETITEM, [const_ptr30, ConstInt(21)], tmp20), + ResOperation(rop.UNICODEGETITEM, [const_ptr30, ConstInt(10)], tmp21), + ResOperation(rop.UINT_RSHIFT, [v9, ConstInt(40)], v10), + ResOperation(rop.UNICODEGETITEM, [const_ptr30, ConstInt(25)], tmp22), + ResOperation(rop.INT_NE, [ConstInt(-8), v9], v11), + ResOperation(rop.INT_MUL_OVF, [v3, ConstInt(-4)], tmp23), + ResOperation(rop.GUARD_OVERFLOW, [], None, descr=faildescr9), + ResOperation(rop.UNICODESETITEM, [const_ptr31, ConstInt(0), ConstInt(50175)], None), + ResOperation(rop.UINT_GT, [v8, ConstInt(-6)], v12), + ResOperation(rop.GETFIELD_GC, [const_ptr32], v13, cpu.fielddescrof(S5, 'f0')), + ResOperation(rop.INT_AND, [ConstInt(8), v8], v14), + ResOperation(rop.INT_INVERT, [v1], v15), + ResOperation(rop.SETFIELD_GC, [const_ptr33, ConstInt(3)], None, cpu.fielddescrof(S6, 'f1')), + ResOperation(rop.INT_GE, [v14, v6], v16), + ResOperation(rop.INT_AND, [v5, ConstInt(-4)], tmp24), + ResOperation(rop.INT_OR, [tmp24, ConstInt(2)], tmp25), + ResOperation(rop.INT_FLOORDIV, [v9, tmp25], v17), + ResOperation(rop.STRLEN, [const_ptr34], tmp26), + ResOperation(rop.NEWSTR, [ConstInt(7)], tmp27), + ResOperation(rop.STRSETITEM, [tmp27, ConstInt(0), ConstInt(21)], None), + ResOperation(rop.STRSETITEM, [tmp27, ConstInt(1), ConstInt(79)], None), + ResOperation(rop.STRSETITEM, [tmp27, ConstInt(2), ConstInt(7)], None), + ResOperation(rop.STRSETITEM, [tmp27, ConstInt(3), ConstInt(2)], None), + ResOperation(rop.STRSETITEM, [tmp27, ConstInt(4), ConstInt(229)], None), + ResOperation(rop.STRSETITEM, [tmp27, ConstInt(5), ConstInt(233)], None), + ResOperation(rop.STRSETITEM, [tmp27, ConstInt(6), ConstInt(208)], None), + ResOperation(rop.INT_LT, [ConstInt(-31), v10], v18), + ResOperation(rop.SAME_AS, [ConstPtr(lltype.nullptr(llmemory.GCREF.TO))], tmp28), + ResOperation(rop.GUARD_NONNULL_CLASS, [tmp28, ConstInt(xtp2)], None, descr=faildescr10), + ResOperation(rop.FINISH, [v4], None, descr=finishdescr11), + ] + operations[0].setfailargs([]) + operations[8].setfailargs([tmp23, v5, v3, v11, v6]) + operations[30].setfailargs([v6]) + cpu.compile_bridge(faildescr6, inputargs, operations, looptoken) + frame = cpu.execute_token(looptoken, *loop_args) + #assert cpu.get_int_value(frame, 0) == -9223372036854775808 + v1 = BoxInt() + v2 = BoxInt() + p3 = BoxPtr() + tmp4 = BoxInt() + tmp5 = BoxPtr() + faildescr12 = BasicFailDescr() + finishdescr13 = BasicFinalDescr() + inputargs = [v1] + + _, func5, func5descr = getexception(cpu, 0) + vt = getvtable(cpu, S4) + + operations = [ + ResOperation(rop.INT_AND, [v1, ConstInt(63)], tmp4), + ResOperation(rop.INT_LSHIFT, [ConstInt(10), tmp4], v2), + ResOperation(rop.NEW_WITH_VTABLE, [ConstInt(vt)], p3), + ResOperation(rop.CALL, [ConstInt(func5)], None, descr=func5descr), + ResOperation(rop.GUARD_EXCEPTION, [ConstInt(xtp2)], tmp5, descr=faildescr12), + ResOperation(rop.FINISH, [], None, descr=finishdescr13), + ] + operations[4].setfailargs([v2]) + cpu.compile_bridge(faildescr10, inputargs, operations, looptoken) + frame = cpu.execute_token(looptoken, *loop_args) + #assert cpu.get_int_value(frame, 0) == 10 + print + print + print + print "Passed %d/1000" % (i + 1) + print + print + print From noreply at buildbot.pypy.org Wed Jan 23 20:31:04 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 23 Jan 2013 20:31:04 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: some extra sanity Message-ID: <20130123193104.AFCBB1C00A8@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60387:4f5e4988ffef Date: 2013-01-23 21:30 +0200 http://bitbucket.org/pypy/pypy/changeset/4f5e4988ffef/ Log: some extra sanity diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py --- a/rpython/jit/backend/llsupport/llmodel.py +++ b/rpython/jit/backend/llsupport/llmodel.py @@ -52,11 +52,15 @@ pass def _setup_frame_realloc(self, translate_support_code): - FUNC_TP = lltype.Ptr(lltype.FuncType([llmemory.GCREF], + FUNC_TP = lltype.Ptr(lltype.FuncType([llmemory.GCREF, lltype.Signed, + lltype.Signed], llmemory.GCREF)) - def realloc_frame(frame): + def realloc_frame(frame, size, asm): frame = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, frame) + assert frame.jf_frame_info.jfi_frame_depth >= size + print "realloc frame (%d) %d->%d" % (size, len(frame.jf_frame), frame.jf_frame_info.jfi_frame_depth) + print "from %x" % asm new_frame = jitframe.JITFRAME.allocate(frame.jf_frame_info) # XXX now we know, rewrite this # we need to do this, because we're not sure what things diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -24,7 +24,7 @@ from rpython.jit.metainterp.resoperation import rop, ResOperation from rpython.jit.backend.x86 import support from rpython.rlib.debug import (debug_print, debug_start, debug_stop, - have_debug_prints) + have_debug_prints) from rpython.rlib import rgc from rpython.rlib.clibffi import FFI_DEFAULT_ABI from rpython.jit.backend.x86.jump import remap_frame_layout @@ -199,6 +199,8 @@ mc.MOV_rs(ecx.value, WORD) gcmap_ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') mc.MOV_br(gcmap_ofs, ecx.value) + mc.MOV_rs(esi.value, WORD*2) + mc.MOV_rs(edx.value, WORD*3) # push first arg mc.LEA_rb(edi.value, -base_ofs) # align @@ -585,8 +587,8 @@ operations, self.current_clt.allgcrefs, self.current_clt.frame_info) - stack_check_patch_ofs = self._check_frame_depth(self.mc, - regalloc.get_gcmap()) + stack_check_patch_ofs, ofs2 = self._check_frame_depth(self.mc, + regalloc.get_gcmap()) frame_depth = self._assemble(regalloc, inputargs, operations) codeendpos = self.mc.get_relative_pos() self.write_pending_failure_recoveries() @@ -607,6 +609,7 @@ frame_depth = max(self.current_clt.frame_info.jfi_frame_depth, frame_depth + JITFRAME_FIXED_SIZE) self._patch_stackadjust(stack_check_patch_ofs + rawstart, frame_depth) + self._patch_stackadjust(ofs2 + rawstart, frame_depth) self.fixup_target_tokens(rawstart) self.current_clt.frame_info.jfi_frame_depth = frame_depth self.teardown() @@ -682,12 +685,15 @@ mc.J_il8(rx86.Conditions['GE'], 0) jg_location = mc.get_relative_pos() self.push_gcmap(mc, gcmap, mov=True) + mc.MOV_si(WORD, 0xffffff) + ofs2 = mc.get_relative_pos() - 4 + mc.MOV_si(WORD*2, mc.get_relative_pos()) mc.CALL(imm(self._stack_check_failure)) # patch the JG above offset = mc.get_relative_pos() - jg_location assert 0 < offset <= 127 mc.overwrite(jg_location-1, chr(offset)) - return stack_check_cmp_ofs + return stack_check_cmp_ofs, ofs2 def _patch_stackadjust(self, adr, allocated_depth): mc = codebuf.MachineCodeBlockWrapper() From noreply at buildbot.pypy.org Wed Jan 23 20:41:39 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Wed, 23 Jan 2013 20:41:39 +0100 (CET) Subject: [pypy-commit] pypy inline-virtualref-2: make this test pass nwo that it's better optimized Message-ID: <20130123194139.D01EC1C00A8@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: inline-virtualref-2 Changeset: r60388:d59d143b055a Date: 2013-01-23 13:23 -0600 http://bitbucket.org/pypy/pypy/changeset/d59d143b055a/ Log: make this test pass nwo that it's better optimized diff --git a/rpython/jit/metainterp/test/test_virtualref.py b/rpython/jit/metainterp/test/test_virtualref.py --- a/rpython/jit/metainterp/test/test_virtualref.py +++ b/rpython/jit/metainterp/test/test_virtualref.py @@ -419,15 +419,18 @@ self.check_aborted_count(0) def test_jit_force_virtual_seen(self): - myjitdriver = JitDriver(greens = [], reds = ['n']) - # + myjitdriver = JitDriver(greens=[], reds=['n']) + A = lltype.GcArray(lltype.Signed) - class XY: + + class XY(object): pass - class ExCtx: + + class ExCtx(object): pass exctx = ExCtx() - # + escapes = [] + def f(n): while n > 0: myjitdriver.can_enter_jit(n=n) @@ -435,16 +438,16 @@ xy = XY() xy.n = n exctx.topframeref = vref = virtual_ref(xy) + escapes.append(xy) xy.next1 = lltype.malloc(A, 0) n = exctx.topframeref().n - 1 - xy.next1 = lltype.nullptr(A) exctx.topframeref = vref_None virtual_ref_finish(vref, xy) return 1 # res = self.meta_interp(f, [15]) assert res == 1 - self.check_resops(new_with_vtable=4, # vref, xy + self.check_resops(new_with_vtable=2, # xy new_array=2) # next1 self.check_aborted_count(0) From noreply at buildbot.pypy.org Wed Jan 23 20:41:41 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Wed, 23 Jan 2013 20:41:41 +0100 (CET) Subject: [pypy-commit] pypy inline-virtualref-2: One more sanity check. Message-ID: <20130123194141.340C61C00A8@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: inline-virtualref-2 Changeset: r60389:49b84471ada8 Date: 2013-01-23 13:40 -0600 http://bitbucket.org/pypy/pypy/changeset/49b84471ada8/ Log: One more sanity check. diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -464,7 +464,7 @@ if (tokenvalue is not None and tokenvalue.is_constant() and tokenvalue.box.getint() == vrefinfo.TOKEN_NONE): forcedvalue = vref.getfield(vrefinfo.descr_forced, None) - if forcedvalue is not None: + if forcedvalue is not None and not forcedvalue.is_null(): self.make_equal_to(op.result, forcedvalue) self.last_emitted_operation = REMOVED return True From noreply at buildbot.pypy.org Wed Jan 23 20:41:42 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Wed, 23 Jan 2013 20:41:42 +0100 (CET) Subject: [pypy-commit] pypy inline-virtualref-2: merged default in Message-ID: <20130123194142.784961C00A8@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: inline-virtualref-2 Changeset: r60390:28889438d586 Date: 2013-01-23 13:41 -0600 http://bitbucket.org/pypy/pypy/changeset/28889438d586/ Log: merged default in diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -5,6 +5,9 @@ .. this is a revision shortly after release-2.0-beta1 .. startrev: 0e6161a009c6 +.. branch: split-rpython +Split rpython and pypy into seperate directories + .. branch: callback-jit Callbacks from C are now better JITted diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -45,12 +45,12 @@ def package(basedir, name='pypy-nightly', rename_pypy_c='pypy', copy_to_dir = None, override_pypy_c = None): - basedir = py.path.local('goal', basedir) + basedir = py.path.local(basedir) if override_pypy_c is None: basename = 'pypy-c' if sys.platform == 'win32': basename += '.exe' - pypy_c = basedir.join(basename) + pypy_c = basedir.join('pypy', 'goal', basename) else: pypy_c = py.path.local(override_pypy_c) if not pypy_c.check(): From noreply at buildbot.pypy.org Wed Jan 23 20:49:04 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Wed, 23 Jan 2013 20:49:04 +0100 (CET) Subject: [pypy-commit] pypy inline-virtualref-2: Close to be merged branch. Message-ID: <20130123194904.EC6DC1C00A8@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: inline-virtualref-2 Changeset: r60391:78aa957cdde4 Date: 2013-01-23 13:48 -0600 http://bitbucket.org/pypy/pypy/changeset/78aa957cdde4/ Log: Close to be merged branch. From noreply at buildbot.pypy.org Wed Jan 23 20:49:06 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Wed, 23 Jan 2013 20:49:06 +0100 (CET) Subject: [pypy-commit] pypy default: Merged inline-virtualref-2: this means that if you force a virtualref (as is done in Python when an exception leaves a frame), if the frame doesn't escape, it still isn't forced to be allocated. Message-ID: <20130123194906.4A0AB1C00A8@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: Changeset: r60392:bcd6eb991231 Date: 2013-01-23 13:48 -0600 http://bitbucket.org/pypy/pypy/changeset/bcd6eb991231/ Log: Merged inline-virtualref-2: this means that if you force a virtualref (as is done in Python when an exception leaves a frame), if the frame doesn't escape, it still isn't forced to be allocated. diff --git a/rpython/jit/codewriter/effectinfo.py b/rpython/jit/codewriter/effectinfo.py --- a/rpython/jit/codewriter/effectinfo.py +++ b/rpython/jit/codewriter/effectinfo.py @@ -79,9 +79,11 @@ OS_RAW_MALLOC_VARSIZE = 110 OS_RAW_FREE = 111 + OS_JIT_FORCE_VIRTUAL = 120 + # for debugging: _OS_CANRAISE = set([OS_NONE, OS_STR2UNICODE, OS_LIBFFI_CALL, - OS_RAW_MALLOC_VARSIZE]) + OS_RAW_MALLOC_VARSIZE, OS_JIT_FORCE_VIRTUAL]) def __new__(cls, readonly_descrs_fields, readonly_descrs_arrays, write_descrs_fields, write_descrs_arrays, diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -1393,6 +1393,8 @@ elif oopspec_name == 'jit.isvirtual': kind = getkind(args[0].concretetype) return SpaceOperation('%s_isvirtual' % kind, args, op.result) + elif oopspec_name == 'jit.force_virtual': + return self._handle_oopspec_call(op, args, EffectInfo.OS_JIT_FORCE_VIRTUAL, EffectInfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE) else: raise AssertionError("missing support for %r" % oopspec_name) diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py --- a/rpython/jit/codewriter/support.py +++ b/rpython/jit/codewriter/support.py @@ -1,24 +1,24 @@ import sys -from rpython.rtyper.lltypesystem import lltype, rclass, rffi, llmemory -from rpython.rtyper.ootypesystem import ootype -from rpython.rtyper import rlist -from rpython.rtyper.lltypesystem import rstr as ll_rstr, rdict as ll_rdict -from rpython.rtyper.lltypesystem.module import ll_math -from rpython.rtyper.lltypesystem.lloperation import llop -from rpython.rtyper.ootypesystem import rdict as oo_rdict -from rpython.rtyper.llinterp import LLInterpreter -from rpython.rtyper.extregistry import ExtRegistryEntry -from rpython.translator.simplify import get_funcobj -from rpython.translator.unsimplify import split_block + +from rpython.annotator import model as annmodel +from rpython.annotator.policy import AnnotatorPolicy from rpython.flowspace.model import Variable, Constant -from rpython.translator.translator import TranslationContext -from rpython.annotator.policy import AnnotatorPolicy -from rpython.annotator import model as annmodel -from rpython.rtyper.annlowlevel import MixLevelHelperAnnotator from rpython.jit.metainterp.typesystem import deref from rpython.rlib import rgc -from rpython.rlib.jit import elidable +from rpython.rlib.jit import elidable, oopspec from rpython.rlib.rarithmetic import r_longlong, r_ulonglong, r_uint, intmask +from rpython.rtyper import rlist +from rpython.rtyper.annlowlevel import MixLevelHelperAnnotator +from rpython.rtyper.extregistry import ExtRegistryEntry +from rpython.rtyper.llinterp import LLInterpreter +from rpython.rtyper.lltypesystem import lltype, rclass, rffi, llmemory, rstr as ll_rstr, rdict as ll_rdict +from rpython.rtyper.lltypesystem.lloperation import llop +from rpython.rtyper.lltypesystem.module import ll_math +from rpython.rtyper.ootypesystem import ootype, rdict as oo_rdict +from rpython.translator.simplify import get_funcobj +from rpython.translator.translator import TranslationContext +from rpython.translator.unsimplify import split_block + def getargtypes(annotator, values): if values is None: # for backend tests producing stand-alone exe's @@ -213,10 +213,12 @@ _ll_5_list_ll_arraycopy = rgc.ll_arraycopy + @elidable def _ll_1_gc_identityhash(x): return lltype.identityhash(x) + # the following function should not be "@elidable": I can think of # a corner case in which id(const) is constant-folded, and then 'const' # disappears and is collected too early (possibly causing another object @@ -224,6 +226,8 @@ def _ll_1_gc_id(ptr): return llop.gc_id(lltype.Signed, ptr) + + at oopspec("jit.force_virtual(inst)") def _ll_1_jit_force_virtual(inst): return llop.jit_force_virtual(lltype.typeOf(inst), inst) diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py --- a/rpython/jit/metainterp/history.py +++ b/rpython/jit/metainterp/history.py @@ -978,11 +978,9 @@ def get_all_jitcell_tokens(self): tokens = [t() for t in self.jitcell_token_wrefs] if None in tokens: - assert False, "get_all_jitcell_tokens will not work as "+\ - "loops have been freed" + assert False, ("get_all_jitcell_tokens will not work as " + "loops have been freed") return tokens - - def check_history(self, expected=None, **check): insns = {} diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -1,12 +1,13 @@ +from rpython.jit.codewriter.effectinfo import EffectInfo +from rpython.jit.metainterp.executor import execute from rpython.jit.codewriter.heaptracker import vtable2descr -from rpython.jit.metainterp.executor import execute from rpython.jit.metainterp.history import Const, ConstInt, BoxInt from rpython.jit.metainterp.optimizeopt import optimizer +from rpython.jit.metainterp.optimizeopt.optimizer import OptValue, REMOVED from rpython.jit.metainterp.optimizeopt.util import (make_dispatcher_method, descrlist_dict, sort_descrs) from rpython.jit.metainterp.resoperation import rop, ResOperation from rpython.rlib.objectmodel import we_are_translated -from rpython.jit.metainterp.optimizeopt.optimizer import OptValue class AbstractVirtualValue(optimizer.OptValue): @@ -386,6 +387,24 @@ self.make_equal_to(box, vvalue) return vvalue + def optimize_GUARD_NO_EXCEPTION(self, op): + if self.last_emitted_operation is REMOVED: + return + self.emit_operation(op) + + def optimize_GUARD_NOT_FORCED(self, op): + if self.last_emitted_operation is REMOVED: + return + self.emit_operation(op) + + def optimize_CALL_MAY_FORCE(self, op): + effectinfo = op.getdescr().get_extra_info() + oopspecindex = effectinfo.oopspecindex + if oopspecindex == EffectInfo.OS_JIT_FORCE_VIRTUAL: + if self._optimize_JIT_FORCE_VIRTUAL(op): + return + self.emit_operation(op) + def optimize_VIRTUAL_REF(self, op): indexbox = op.getarg(1) # @@ -429,7 +448,7 @@ # - set 'virtual_token' to TOKEN_NONE args = [op.getarg(0), ConstInt(vrefinfo.TOKEN_NONE)] seo(ResOperation(rop.SETFIELD_GC, args, None, - descr = vrefinfo.descr_virtual_token)) + descr=vrefinfo.descr_virtual_token)) # Note that in some cases the virtual in op.getarg(1) has been forced # already. This is fine. In that case, and *if* a residual # CALL_MAY_FORCE suddenly turns out to access it, then it will @@ -437,6 +456,20 @@ # will work too (but just be a little pointless, as the structure # was already forced). + def _optimize_JIT_FORCE_VIRTUAL(self, op): + vref = self.getvalue(op.getarg(1)) + vrefinfo = self.optimizer.metainterp_sd.virtualref_info + if vref.is_virtual(): + tokenvalue = vref.getfield(vrefinfo.descr_virtual_token, None) + if (tokenvalue is not None and tokenvalue.is_constant() and + tokenvalue.box.getint() == vrefinfo.TOKEN_NONE): + forcedvalue = vref.getfield(vrefinfo.descr_forced, None) + if forcedvalue is not None and not forcedvalue.is_null(): + self.make_equal_to(op.result, forcedvalue) + self.last_emitted_operation = REMOVED + return True + return False + def optimize_GETFIELD_GC(self, op): value = self.getvalue(op.getarg(0)) # If this is an immutable field (as indicated by op.is_always_pure()) diff --git a/rpython/jit/metainterp/test/support.py b/rpython/jit/metainterp/test/support.py --- a/rpython/jit/metainterp/test/support.py +++ b/rpython/jit/metainterp/test/support.py @@ -160,16 +160,17 @@ class JitMixin: basic = True + def check_resops(self, expected=None, **check): get_stats().check_resops(expected=expected, **check) + def check_simple_loop(self, expected=None, **check): get_stats().check_simple_loop(expected=expected, **check) - - def check_trace_count(self, count): # was check_loop_count # The number of traces compiled assert get_stats().compiled_count == count + def check_trace_count_at_most(self, count): assert get_stats().compiled_count <= count @@ -178,11 +179,12 @@ def check_target_token_count(self, count): tokens = get_stats().get_all_jitcell_tokens() - n = sum ([len(t.target_tokens) for t in tokens]) + n = sum([len(t.target_tokens) for t in tokens]) assert n == count def check_enter_count(self, count): assert get_stats().enter_count == count + def check_enter_count_at_most(self, count): assert get_stats().enter_count <= count @@ -192,6 +194,7 @@ def check_aborted_count(self, count): assert get_stats().aborted_count == count + def check_aborted_count_at_least(self, count): assert get_stats().aborted_count >= count diff --git a/rpython/jit/metainterp/test/test_virtualref.py b/rpython/jit/metainterp/test/test_virtualref.py --- a/rpython/jit/metainterp/test/test_virtualref.py +++ b/rpython/jit/metainterp/test/test_virtualref.py @@ -1,19 +1,20 @@ import py -from rpython.rtyper.lltypesystem import lltype, llmemory, lloperation + +from rpython.rtyper.lltypesystem import lltype, lloperation from rpython.rtyper.exceptiondata import UnknownException from rpython.rlib.jit import JitDriver, dont_look_inside, vref_None from rpython.rlib.jit import virtual_ref, virtual_ref_finish, InvalidVirtualRef from rpython.rlib.jit import non_virtual_ref from rpython.rlib.objectmodel import compute_unique_id -from rpython.jit.metainterp.test.support import LLJitMixin, OOJitMixin, _get_jitcodes +from rpython.jit.metainterp.test.support import LLJitMixin, _get_jitcodes from rpython.jit.metainterp.resoperation import rop from rpython.jit.metainterp.virtualref import VirtualRefInfo + debug_print = lloperation.llop.debug_print -class VRefTests: - +class VRefTests(object): def finish_setup_for_interp_operations(self): self.vrefinfo = VirtualRefInfo(self.warmrunnerstate) self.cw.setup_vrefinfo(self.vrefinfo) @@ -115,8 +116,8 @@ from rpython.jit.metainterp.resume import ResumeDataDirectReader cpu = self.metainterp.cpu cpu.get_latest_value_count = lambda df: len(guard_op.getfailargs()) - cpu.get_latest_value_int = lambda df,i:guard_op.getfailargs()[i].getint() - cpu.get_latest_value_ref = lambda df,i:guard_op.getfailargs()[i].getref_base() + cpu.get_latest_value_int = lambda df, i: guard_op.getfailargs()[i].getint() + cpu.get_latest_value_ref = lambda df, i: guard_op.getfailargs()[i].getref_base() cpu.clear_latest_values = lambda count: None class FakeMetaInterpSd: callinfocollection = None @@ -418,15 +419,18 @@ self.check_aborted_count(0) def test_jit_force_virtual_seen(self): - myjitdriver = JitDriver(greens = [], reds = ['n']) - # + myjitdriver = JitDriver(greens=[], reds=['n']) + A = lltype.GcArray(lltype.Signed) - class XY: + + class XY(object): pass - class ExCtx: + + class ExCtx(object): pass exctx = ExCtx() - # + escapes = [] + def f(n): while n > 0: myjitdriver.can_enter_jit(n=n) @@ -434,16 +438,16 @@ xy = XY() xy.n = n exctx.topframeref = vref = virtual_ref(xy) + escapes.append(xy) xy.next1 = lltype.malloc(A, 0) n = exctx.topframeref().n - 1 - xy.next1 = lltype.nullptr(A) exctx.topframeref = vref_None virtual_ref_finish(vref, xy) return 1 # res = self.meta_interp(f, [15]) assert res == 1 - self.check_resops(new_with_vtable=4, # vref, xy + self.check_resops(new_with_vtable=2, # xy new_array=2) # next1 self.check_aborted_count(0) @@ -656,6 +660,34 @@ res = self.meta_interp(f, [10]) assert res == 0 + def test_force_virtual_vref(self): + myjitdriver = JitDriver(greens=[], reds=['n', 'ec']) + + class ExecutionContext(object): + pass + + class Frame(object): + def __init__(self, x): + self.x = x + + def f(n): + ec = ExecutionContext() + while n > 0: + myjitdriver.jit_merge_point(n=n, ec=ec) + frame = Frame(1) + ec.topframeref = virtual_ref(frame) + n -= ec.topframeref().x + frame_vref = ec.topframeref + ec.topframeref = vref_None + virtual_ref_finish(frame_vref, frame) + return n + res = self.meta_interp(f, [10]) + assert res == 0 + self.check_resops({ + 'int_sub': 2, 'int_gt': 2, 'jump': 1, 'guard_true': 2, + 'force_token': 2, 'setfield_gc': 1 + }) + class TestLLtype(VRefTests, LLJitMixin): pass From noreply at buildbot.pypy.org Wed Jan 23 20:55:40 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Wed, 23 Jan 2013 20:55:40 +0100 (CET) Subject: [pypy-commit] pypy default: fix these tests Message-ID: <20130123195540.BAF4D1C00A8@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: Changeset: r60393:90e9534b0602 Date: 2013-01-23 13:55 -0600 http://bitbucket.org/pypy/pypy/changeset/90e9534b0602/ Log: fix these tests diff --git a/rpython/rlib/unicodedata/test/test_unicodedata.py b/rpython/rlib/unicodedata/test/test_unicodedata.py --- a/rpython/rlib/unicodedata/test/test_unicodedata.py +++ b/rpython/rlib/unicodedata/test/test_unicodedata.py @@ -1,11 +1,15 @@ +import random +import unicodedata + import py + from rpython.rlib.unicodedata import unicodedb_3_2_0, unicodedb_5_2_0 + class TestUnicodeData(object): def setup_class(cls): - import random, unicodedata if unicodedata.unidata_version != '5.2.0': - skip('Needs python with unicode 5.2.0 database.') + py.test.skip('Needs python with unicode 5.2.0 database.') seed = random.getrandbits(32) print "random seed: ", seed @@ -29,14 +33,12 @@ py.test.raises(KeyError, unicodedb_5_2_0.name, ord(chr)) def test_compare_functions(self): - import unicodedata # CPython implementation - def getX(fun, code): try: return getattr(unicodedb_5_2_0, fun)(code) except KeyError: return -1 - + for code in range(0x10000): char = unichr(code) assert unicodedata.digit(char, -1) == getX('digit', code) @@ -73,5 +75,3 @@ assert unicodedb_5_2_0.lookup('BENZENE RING WITH CIRCLE') == 9187 py.test.raises(KeyError, unicodedb_3_2_0.lookup, 'BENZENE RING WITH CIRCLE') py.test.raises(KeyError, unicodedb_3_2_0.name, 9187) - - From noreply at buildbot.pypy.org Wed Jan 23 20:56:44 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Wed, 23 Jan 2013 20:56:44 +0100 (CET) Subject: [pypy-commit] pypy default: import from the right place Message-ID: <20130123195644.DECFB1C00A8@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: Changeset: r60394:8641bde33d47 Date: 2013-01-23 13:56 -0600 http://bitbucket.org/pypy/pypy/changeset/8641bde33d47/ Log: import from the right place diff --git a/rpython/rlib/unicodedata/test/test_ucd.py b/rpython/rlib/unicodedata/test/test_ucd.py --- a/rpython/rlib/unicodedata/test/test_ucd.py +++ b/rpython/rlib/unicodedata/test/test_ucd.py @@ -1,9 +1,9 @@ +from rpython.rlib.runicode import code_to_unichr +from rpython.rlib.unicodedata import unicodedb_5_2_0 from rpython.rtyper.test.tool import BaseRtypingTest, LLRtypeMixin -from rpython.rlib.unicodedata import unicodedb_5_2_0 -from rpython.rlib.unicodedata.ucd import code_to_unichr + class TestTranslated(BaseRtypingTest, LLRtypeMixin): - def test_translated(self): def f(n): if n == 0: From noreply at buildbot.pypy.org Wed Jan 23 21:17:19 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 23 Jan 2013 21:17:19 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: add some info Message-ID: <20130123201719.95CE11C00A8@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60395:08749a4312fb Date: 2013-01-23 22:16 +0200 http://bitbucket.org/pypy/pypy/changeset/08749a4312fb/ Log: add some info diff --git a/rpython/jit/backend/llsupport/jitframe.py b/rpython/jit/backend/llsupport/jitframe.py --- a/rpython/jit/backend/llsupport/jitframe.py +++ b/rpython/jit/backend/llsupport/jitframe.py @@ -47,6 +47,7 @@ # guard_not_forced descr ('jf_force_descr', llmemory.GCREF), # a map of GC pointers + ('jf_comingfrom', llmemory.GCREF), ('jf_gcmap', lltype.Ptr(GCMAP)), # For the front-end: a GCREF for the savedata ('jf_savedata', llmemory.GCREF), diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py --- a/rpython/jit/backend/llsupport/llmodel.py +++ b/rpython/jit/backend/llsupport/llmodel.py @@ -58,7 +58,9 @@ def realloc_frame(frame, size, asm): frame = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, frame) - assert frame.jf_frame_info.jfi_frame_depth >= size + if not frame.jf_frame_info.jfi_frame_depth >= size: + import pdb + pdb.set_trace() print "realloc frame (%d) %d->%d" % (size, len(frame.jf_frame), frame.jf_frame_info.jfi_frame_depth) print "from %x" % asm new_frame = jitframe.JITFRAME.allocate(frame.jf_frame_info) diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py --- a/rpython/jit/backend/x86/runner.py +++ b/rpython/jit/backend/x86/runner.py @@ -119,6 +119,10 @@ #llop.debug_print(lltype.Void, ">>>> Entering", addr) frame_info = clt.frame_info frame = jitframe.JITFRAME.allocate(frame_info) + + from rpython.rtyper.annlowlevel import cast_instance_to_gcref + + frame.jf_comingfrom = cast_instance_to_gcref(executable_token) ll_frame = lltype.cast_opaque_ptr(llmemory.GCREF, frame) prev_interpreter = None # help flow space if not self.translate_support_code: From noreply at buildbot.pypy.org Wed Jan 23 21:17:50 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 23 Jan 2013 21:17:50 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: don't run this test 1000 times Message-ID: <20130123201750.DB01F1C00A8@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60396:d838e94b3a28 Date: 2013-01-23 22:17 +0200 http://bitbucket.org/pypy/pypy/changeset/d838e94b3a28/ Log: don't run this test 1000 times diff --git a/rpython/jit/backend/x86/test/test_regalloc2.py b/rpython/jit/backend/x86/test/test_regalloc2.py --- a/rpython/jit/backend/x86/test/test_regalloc2.py +++ b/rpython/jit/backend/x86/test/test_regalloc2.py @@ -344,277 +344,269 @@ cpu = CPU(None, None) cpu.setup_once() - for i in range(1000): - S4 = lltype.Struct('Sx', ("f0", lltype.Char), ("f1", lltype.Signed), ("f2", lltype.Signed), ("f3", lltype.Signed)) - S5 = lltype.GcArray(S4) - v1 = BoxInt() - v2 = BoxInt() - v3 = BoxInt() - v4 = BoxInt() - v5 = BoxInt() - v6 = BoxInt() - v7 = BoxInt() - v8 = BoxInt() - v9 = BoxInt() - v10 = BoxInt() - tmp11 = BoxInt() - tmp12 = BoxPtr() - faildescr0 = BasicFailDescr() - tmp13 = BoxPtr() - faildescr1 = BasicFailDescr() - finishdescr2 = BasicFinalDescr() - const_ptr14 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(rstr.STR, 1))) - const_ptr15 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(rstr.UNICODE, 489))) - const_ptr16 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(rstr.UNICODE, 16))) - const_ptr17 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S5, 299))) - inputargs = [v1, v2, v3, v4, v5, v6, v7, v8, v9, v10] + S4 = lltype.Struct('Sx', ("f0", lltype.Char), ("f1", lltype.Signed), ("f2", lltype.Signed), ("f3", lltype.Signed)) + S5 = lltype.GcArray(S4) + v1 = BoxInt() + v2 = BoxInt() + v3 = BoxInt() + v4 = BoxInt() + v5 = BoxInt() + v6 = BoxInt() + v7 = BoxInt() + v8 = BoxInt() + v9 = BoxInt() + v10 = BoxInt() + tmp11 = BoxInt() + tmp12 = BoxPtr() + faildescr0 = BasicFailDescr() + tmp13 = BoxPtr() + faildescr1 = BasicFailDescr() + finishdescr2 = BasicFinalDescr() + const_ptr14 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(rstr.STR, 1))) + const_ptr15 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(rstr.UNICODE, 489))) + const_ptr16 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(rstr.UNICODE, 16))) + const_ptr17 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S5, 299))) + inputargs = [v1, v2, v3, v4, v5, v6, v7, v8, v9, v10] - xtp, func, funcdescr = getexception(cpu, 3) - xtp2, func2, func2descr = getexception(cpu, 2) + xtp, func, funcdescr = getexception(cpu, 3) + xtp2, func2, func2descr = getexception(cpu, 2) - operations = [ - ResOperation(rop.STRGETITEM, [const_ptr14, ConstInt(0)], tmp11), - ResOperation(rop.LABEL, [v1, v2, tmp11, v3, v4, v5, v6, v7, v8, v9, v10], None, TargetToken()), - ResOperation(rop.UNICODESETITEM, [const_ptr15, v4, ConstInt(22)], None), - ResOperation(rop.CALL, [ConstInt(func), v2, v1, v9], None, descr=funcdescr), - ResOperation(rop.GUARD_EXCEPTION, [ConstInt(xtp)], tmp12, descr=faildescr0), - ResOperation(rop.UNICODESETITEM, [const_ptr16, ConstInt(13), ConstInt(9)], None), - ResOperation(rop.SETINTERIORFIELD_GC, [const_ptr17, v3, v7], None, cpu.interiorfielddescrof(S5, 'f3')), - ResOperation(rop.CALL, [ConstInt(func2), v7, v10], None, descr=func2descr), - ResOperation(rop.GUARD_NO_EXCEPTION, [], tmp13, descr=faildescr1), - ResOperation(rop.FINISH, [], None, descr=finishdescr2), - ] - operations[4].setfailargs([v4, v8, v10, v2, v9, v7, v6, v1]) - operations[8].setfailargs([v3, v9, v2, v6, v4]) - looptoken = JitCellToken() - cpu.compile_loop(inputargs, operations, looptoken) - loop_args = [1, -39, 46, 21, 16, 6, -4611686018427387905, 12, 14, 2] - frame = cpu.execute_token(looptoken, *loop_args) - assert cpu.get_int_value(frame, 0) == 46 - assert cpu.get_int_value(frame, 1) == 14 - assert cpu.get_int_value(frame, 2) == -39 - assert cpu.get_int_value(frame, 3) == 6 - assert cpu.get_int_value(frame, 4) == 21 - S4 = lltype.GcStruct('Sx', ("parent", rclass.OBJECT), ("f0", lltype.Signed)) - S5 = lltype.GcStruct('Sx', ("f0", lltype.Signed)) - S6 = lltype.GcArray(lltype.Signed) - S7 = lltype.GcStruct('Sx', ("parent", rclass.OBJECT), ("f0", lltype.Char)) - S8 = lltype.Struct('Sx', ("f0", lltype.Char), ("f1", lltype.Signed), ("f2", lltype.Signed), ("f3", lltype.Signed)) - S9 = lltype.GcArray(S8) - v1 = BoxInt() - v2 = BoxInt() - v3 = BoxInt() - v4 = BoxInt() - v5 = BoxInt() - v6 = BoxInt() - v7 = BoxInt() - v8 = BoxInt() - v9 = BoxInt() - v10 = BoxInt() - v11 = BoxInt() - v12 = BoxInt() - v13 = BoxInt() - v14 = BoxInt() - v15 = BoxInt() - v16 = BoxInt() - v17 = BoxInt() - v18 = BoxInt() - v19 = BoxInt() - p20 = BoxPtr() - tmp21 = BoxPtr() - faildescr3 = BasicFailDescr() - tmp22 = BoxPtr() - faildescr4 = BasicFailDescr() - tmp23 = BoxInt() - tmp24 = BoxInt() - tmp25 = BoxInt() - tmp26 = BoxInt() - tmp27 = BoxInt() - tmp28 = BoxInt() - tmp29 = BoxInt() - faildescr5 = BasicFailDescr() - tmp30 = BoxPtr() - faildescr6 = BasicFailDescr() - finishdescr7 = BasicFinalDescr() - const_ptr31 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S4))) - const_ptr32 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(rstr.STR, 46))) - const_ptr33 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S5))) - const_ptr34 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(rstr.UNICODE, 26))) - const_ptr35 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(rstr.UNICODE, 15))) - const_ptr36 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S7))) - const_ptr37 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(rstr.STR, 484))) - const_ptr38 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S9, 299))) - inputargs = [v1, v2, v3, v4, v5] + operations = [ + ResOperation(rop.STRGETITEM, [const_ptr14, ConstInt(0)], tmp11), + ResOperation(rop.LABEL, [v1, v2, tmp11, v3, v4, v5, v6, v7, v8, v9, v10], None, TargetToken()), + ResOperation(rop.UNICODESETITEM, [const_ptr15, v4, ConstInt(22)], None), + ResOperation(rop.CALL, [ConstInt(func), v2, v1, v9], None, descr=funcdescr), + ResOperation(rop.GUARD_EXCEPTION, [ConstInt(xtp)], tmp12, descr=faildescr0), + ResOperation(rop.UNICODESETITEM, [const_ptr16, ConstInt(13), ConstInt(9)], None), + ResOperation(rop.SETINTERIORFIELD_GC, [const_ptr17, v3, v7], None, cpu.interiorfielddescrof(S5, 'f3')), + ResOperation(rop.CALL, [ConstInt(func2), v7, v10], None, descr=func2descr), + ResOperation(rop.GUARD_NO_EXCEPTION, [], tmp13, descr=faildescr1), + ResOperation(rop.FINISH, [], None, descr=finishdescr2), + ] + operations[4].setfailargs([v4, v8, v10, v2, v9, v7, v6, v1]) + operations[8].setfailargs([v3, v9, v2, v6, v4]) + looptoken = JitCellToken() + cpu.compile_loop(inputargs, operations, looptoken) + loop_args = [1, -39, 46, 21, 16, 6, -4611686018427387905, 12, 14, 2] + frame = cpu.execute_token(looptoken, *loop_args) + assert cpu.get_int_value(frame, 0) == 46 + assert cpu.get_int_value(frame, 1) == 14 + assert cpu.get_int_value(frame, 2) == -39 + assert cpu.get_int_value(frame, 3) == 6 + assert cpu.get_int_value(frame, 4) == 21 + S4 = lltype.GcStruct('Sx', ("parent", rclass.OBJECT), ("f0", lltype.Signed)) + S5 = lltype.GcStruct('Sx', ("f0", lltype.Signed)) + S6 = lltype.GcArray(lltype.Signed) + S7 = lltype.GcStruct('Sx', ("parent", rclass.OBJECT), ("f0", lltype.Char)) + S8 = lltype.Struct('Sx', ("f0", lltype.Char), ("f1", lltype.Signed), ("f2", lltype.Signed), ("f3", lltype.Signed)) + S9 = lltype.GcArray(S8) + v1 = BoxInt() + v2 = BoxInt() + v3 = BoxInt() + v4 = BoxInt() + v5 = BoxInt() + v6 = BoxInt() + v7 = BoxInt() + v8 = BoxInt() + v9 = BoxInt() + v10 = BoxInt() + v11 = BoxInt() + v12 = BoxInt() + v13 = BoxInt() + v14 = BoxInt() + v15 = BoxInt() + v16 = BoxInt() + v17 = BoxInt() + v18 = BoxInt() + v19 = BoxInt() + p20 = BoxPtr() + tmp21 = BoxPtr() + faildescr3 = BasicFailDescr() + tmp22 = BoxPtr() + faildescr4 = BasicFailDescr() + tmp23 = BoxInt() + tmp24 = BoxInt() + tmp25 = BoxInt() + tmp26 = BoxInt() + tmp27 = BoxInt() + tmp28 = BoxInt() + tmp29 = BoxInt() + faildescr5 = BasicFailDescr() + tmp30 = BoxPtr() + faildescr6 = BasicFailDescr() + finishdescr7 = BasicFinalDescr() + const_ptr31 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S4))) + const_ptr32 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(rstr.STR, 46))) + const_ptr33 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S5))) + const_ptr34 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(rstr.UNICODE, 26))) + const_ptr35 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(rstr.UNICODE, 15))) + const_ptr36 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S7))) + const_ptr37 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(rstr.STR, 484))) + const_ptr38 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S9, 299))) + inputargs = [v1, v2, v3, v4, v5] - func3, func3descr = getnoexception(cpu, 5) - xtp3, func4, func4descr = getexception(cpu, 10) + func3, func3descr = getnoexception(cpu, 5) + xtp3, func4, func4descr = getexception(cpu, 10) - operations = [ - ResOperation(rop.GUARD_EXCEPTION, [ConstInt(xtp2)], tmp21, descr=faildescr3), - ResOperation(rop.INT_IS_ZERO, [v4], v6), - ResOperation(rop.INT_NE, [v6, ConstInt(13)], v7), - ResOperation(rop.GETFIELD_GC, [const_ptr31], v8, cpu.fielddescrof(S4, 'f0')), - ResOperation(rop.STRSETITEM, [const_ptr32, v6, ConstInt(0)], None), - ResOperation(rop.NEWSTR, [ConstInt(5)], tmp22), - ResOperation(rop.STRSETITEM, [tmp22, ConstInt(0), ConstInt(42)], None), - ResOperation(rop.STRSETITEM, [tmp22, ConstInt(1), ConstInt(42)], None), - ResOperation(rop.STRSETITEM, [tmp22, ConstInt(2), ConstInt(20)], None), - ResOperation(rop.STRSETITEM, [tmp22, ConstInt(3), ConstInt(48)], None), - ResOperation(rop.STRSETITEM, [tmp22, ConstInt(4), ConstInt(6)], None), - ResOperation(rop.GETFIELD_GC, [const_ptr33], v9, cpu.fielddescrof(S5, 'f0')), - ResOperation(rop.UNICODESETITEM, [const_ptr34, ConstInt(24), ConstInt(65533)], None), - ResOperation(rop.GETFIELD_GC, [const_ptr31], v10, cpu.fielddescrof(S4, 'f0')), - ResOperation(rop.INT_NE, [v10, ConstInt(25)], v11), - ResOperation(rop.CALL, [ConstInt(func3), v5, v1, v8, v3, v2], v12, descr=func3descr), - ResOperation(rop.GUARD_NO_EXCEPTION, [], None, descr=faildescr4), - ResOperation(rop.UNICODELEN, [const_ptr35], tmp23), - ResOperation(rop.NEW_ARRAY, [v2], p20, cpu.arraydescrof(S6)), - ResOperation(rop.GETFIELD_GC, [const_ptr36], v13, cpu.fielddescrof(S7, 'f0')), - ResOperation(rop.INT_OR, [v8, ConstInt(2)], tmp24), - ResOperation(rop.INT_FLOORDIV, [ConstInt(8), tmp24], v14), - ResOperation(rop.GETARRAYITEM_GC, [p20, ConstInt(3)], v15, cpu.arraydescrof(S6)), - ResOperation(rop.COPYSTRCONTENT, [tmp22, const_ptr37, ConstInt(1), ConstInt(163), ConstInt(0)], None), - ResOperation(rop.COPYUNICODECONTENT, [const_ptr35, const_ptr34, ConstInt(13), ConstInt(0), v6], None), - ResOperation(rop.STRGETITEM, [tmp22, v6], tmp25), - ResOperation(rop.STRGETITEM, [tmp22, ConstInt(0)], tmp26), - ResOperation(rop.GETINTERIORFIELD_GC, [const_ptr38, v13], v16, cpu.interiorfielddescrof(S9, 'f0')), - ResOperation(rop.INT_GE, [v4, v5], v17), - ResOperation(rop.INT_OR, [v13, ConstInt(2)], tmp27), - ResOperation(rop.INT_FLOORDIV, [ConstInt(12), tmp27], v18), - ResOperation(rop.INT_AND, [v1, ConstInt(-4)], tmp28), - ResOperation(rop.INT_OR, [tmp28, ConstInt(2)], tmp29), - ResOperation(rop.INT_FLOORDIV, [v15, tmp29], v19), - ResOperation(rop.GUARD_FALSE, [v17], None, descr=faildescr5), - ResOperation(rop.UNICODESETITEM, [const_ptr34, ConstInt(20), ConstInt(65522)], None), - ResOperation(rop.CALL, [ConstInt(func4), v3, v9, v10, v8, v11, v5, v13, v14, v15, v6], None, descr=func4descr), - ResOperation(rop.GUARD_NO_EXCEPTION, [], tmp30, descr=faildescr6), - ResOperation(rop.FINISH, [], None, descr=finishdescr7), - ] - operations[0].setfailargs([]) - operations[16].setfailargs([v5, v9]) - operations[34].setfailargs([]) - operations[37].setfailargs([v12, v19, v10, v7, v4, v8, v18, v15, v9]) - cpu.compile_bridge(faildescr1, inputargs, operations, looptoken) - frame = cpu.execute_token(looptoken, *loop_args) - #assert cpu.get_int_value(frame, 0) == -9223372036854775766 - assert cpu.get_int_value(frame, 1) == 0 - #assert cpu.get_int_value(frame, 2) == -9223372036854775808 - assert cpu.get_int_value(frame, 3) == 1 - assert cpu.get_int_value(frame, 4) == 6 - #assert cpu.get_int_value(frame, 5) == -9223372036854775808 - assert cpu.get_int_value(frame, 6) == 0 - assert cpu.get_int_value(frame, 7) == 0 - #assert cpu.get_int_value(frame, 8) == 26 - S4 = lltype.GcStruct('Sx', ("parent", rclass.OBJECT), ("f0", lltype.Signed), ("f1", lltype.Signed)) - S5 = lltype.GcStruct('Sx', ("parent", rclass.OBJECT), ("f0", lltype.Signed)) - S6 = lltype.GcStruct('Sx', ("f0", lltype.Signed), ("f1", rffi.UCHAR)) - v1 = BoxInt() - v2 = BoxInt() - v3 = BoxInt() - v4 = BoxInt() - v5 = BoxInt() - v6 = BoxInt() - v7 = BoxInt() - v8 = BoxInt() - v9 = BoxInt() - v10 = BoxInt() - v11 = BoxInt() - v12 = BoxInt() - v13 = BoxInt() - v14 = BoxInt() - v15 = BoxInt() - v16 = BoxInt() - v17 = BoxInt() - v18 = BoxInt() - tmp19 = BoxPtr() - faildescr8 = BasicFailDescr() - tmp20 = BoxInt() - tmp21 = BoxInt() - tmp22 = BoxInt() - tmp23 = BoxInt() - faildescr9 = BasicFailDescr() - tmp24 = BoxInt() - tmp25 = BoxInt() - tmp26 = BoxInt() - tmp27 = BoxPtr() - tmp28 = BoxPtr() - faildescr10 = BasicFailDescr() - finishdescr11 = BasicFinalDescr() - const_ptr29 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S4))) - const_ptr30 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(rstr.UNICODE, 26))) - const_ptr31 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(rstr.UNICODE, 1))) - const_ptr32 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S5))) - const_ptr33 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S6))) - const_ptr34 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(rstr.STR, 26))) - inputargs = [v1, v2, v3, v4, v5, v6, v7, v8, v9] - operations = [ - ResOperation(rop.GUARD_EXCEPTION, [ConstInt(xtp3)], tmp19, descr=faildescr8), - ResOperation(rop.SETFIELD_GC, [const_ptr29, v7], None, cpu.fielddescrof(S4, 'f0')), - ResOperation(rop.UNICODEGETITEM, [const_ptr30, ConstInt(21)], tmp20), - ResOperation(rop.UNICODEGETITEM, [const_ptr30, ConstInt(10)], tmp21), - ResOperation(rop.UINT_RSHIFT, [v9, ConstInt(40)], v10), - ResOperation(rop.UNICODEGETITEM, [const_ptr30, ConstInt(25)], tmp22), - ResOperation(rop.INT_NE, [ConstInt(-8), v9], v11), - ResOperation(rop.INT_MUL_OVF, [v3, ConstInt(-4)], tmp23), - ResOperation(rop.GUARD_OVERFLOW, [], None, descr=faildescr9), - ResOperation(rop.UNICODESETITEM, [const_ptr31, ConstInt(0), ConstInt(50175)], None), - ResOperation(rop.UINT_GT, [v8, ConstInt(-6)], v12), - ResOperation(rop.GETFIELD_GC, [const_ptr32], v13, cpu.fielddescrof(S5, 'f0')), - ResOperation(rop.INT_AND, [ConstInt(8), v8], v14), - ResOperation(rop.INT_INVERT, [v1], v15), - ResOperation(rop.SETFIELD_GC, [const_ptr33, ConstInt(3)], None, cpu.fielddescrof(S6, 'f1')), - ResOperation(rop.INT_GE, [v14, v6], v16), - ResOperation(rop.INT_AND, [v5, ConstInt(-4)], tmp24), - ResOperation(rop.INT_OR, [tmp24, ConstInt(2)], tmp25), - ResOperation(rop.INT_FLOORDIV, [v9, tmp25], v17), - ResOperation(rop.STRLEN, [const_ptr34], tmp26), - ResOperation(rop.NEWSTR, [ConstInt(7)], tmp27), - ResOperation(rop.STRSETITEM, [tmp27, ConstInt(0), ConstInt(21)], None), - ResOperation(rop.STRSETITEM, [tmp27, ConstInt(1), ConstInt(79)], None), - ResOperation(rop.STRSETITEM, [tmp27, ConstInt(2), ConstInt(7)], None), - ResOperation(rop.STRSETITEM, [tmp27, ConstInt(3), ConstInt(2)], None), - ResOperation(rop.STRSETITEM, [tmp27, ConstInt(4), ConstInt(229)], None), - ResOperation(rop.STRSETITEM, [tmp27, ConstInt(5), ConstInt(233)], None), - ResOperation(rop.STRSETITEM, [tmp27, ConstInt(6), ConstInt(208)], None), - ResOperation(rop.INT_LT, [ConstInt(-31), v10], v18), - ResOperation(rop.SAME_AS, [ConstPtr(lltype.nullptr(llmemory.GCREF.TO))], tmp28), - ResOperation(rop.GUARD_NONNULL_CLASS, [tmp28, ConstInt(xtp2)], None, descr=faildescr10), - ResOperation(rop.FINISH, [v4], None, descr=finishdescr11), - ] - operations[0].setfailargs([]) - operations[8].setfailargs([tmp23, v5, v3, v11, v6]) - operations[30].setfailargs([v6]) - cpu.compile_bridge(faildescr6, inputargs, operations, looptoken) - frame = cpu.execute_token(looptoken, *loop_args) - #assert cpu.get_int_value(frame, 0) == -9223372036854775808 - v1 = BoxInt() - v2 = BoxInt() - p3 = BoxPtr() - tmp4 = BoxInt() - tmp5 = BoxPtr() - faildescr12 = BasicFailDescr() - finishdescr13 = BasicFinalDescr() - inputargs = [v1] + operations = [ + ResOperation(rop.GUARD_EXCEPTION, [ConstInt(xtp2)], tmp21, descr=faildescr3), + ResOperation(rop.INT_IS_ZERO, [v4], v6), + ResOperation(rop.INT_NE, [v6, ConstInt(13)], v7), + ResOperation(rop.GETFIELD_GC, [const_ptr31], v8, cpu.fielddescrof(S4, 'f0')), + ResOperation(rop.STRSETITEM, [const_ptr32, v6, ConstInt(0)], None), + ResOperation(rop.NEWSTR, [ConstInt(5)], tmp22), + ResOperation(rop.STRSETITEM, [tmp22, ConstInt(0), ConstInt(42)], None), + ResOperation(rop.STRSETITEM, [tmp22, ConstInt(1), ConstInt(42)], None), + ResOperation(rop.STRSETITEM, [tmp22, ConstInt(2), ConstInt(20)], None), + ResOperation(rop.STRSETITEM, [tmp22, ConstInt(3), ConstInt(48)], None), + ResOperation(rop.STRSETITEM, [tmp22, ConstInt(4), ConstInt(6)], None), + ResOperation(rop.GETFIELD_GC, [const_ptr33], v9, cpu.fielddescrof(S5, 'f0')), + ResOperation(rop.UNICODESETITEM, [const_ptr34, ConstInt(24), ConstInt(65533)], None), + ResOperation(rop.GETFIELD_GC, [const_ptr31], v10, cpu.fielddescrof(S4, 'f0')), + ResOperation(rop.INT_NE, [v10, ConstInt(25)], v11), + ResOperation(rop.CALL, [ConstInt(func3), v5, v1, v8, v3, v2], v12, descr=func3descr), + ResOperation(rop.GUARD_NO_EXCEPTION, [], None, descr=faildescr4), + ResOperation(rop.UNICODELEN, [const_ptr35], tmp23), + ResOperation(rop.NEW_ARRAY, [v2], p20, cpu.arraydescrof(S6)), + ResOperation(rop.GETFIELD_GC, [const_ptr36], v13, cpu.fielddescrof(S7, 'f0')), + ResOperation(rop.INT_OR, [v8, ConstInt(2)], tmp24), + ResOperation(rop.INT_FLOORDIV, [ConstInt(8), tmp24], v14), + ResOperation(rop.GETARRAYITEM_GC, [p20, ConstInt(3)], v15, cpu.arraydescrof(S6)), + ResOperation(rop.COPYSTRCONTENT, [tmp22, const_ptr37, ConstInt(1), ConstInt(163), ConstInt(0)], None), + ResOperation(rop.COPYUNICODECONTENT, [const_ptr35, const_ptr34, ConstInt(13), ConstInt(0), v6], None), + ResOperation(rop.STRGETITEM, [tmp22, v6], tmp25), + ResOperation(rop.STRGETITEM, [tmp22, ConstInt(0)], tmp26), + ResOperation(rop.GETINTERIORFIELD_GC, [const_ptr38, v13], v16, cpu.interiorfielddescrof(S9, 'f0')), + ResOperation(rop.INT_GE, [v4, v5], v17), + ResOperation(rop.INT_OR, [v13, ConstInt(2)], tmp27), + ResOperation(rop.INT_FLOORDIV, [ConstInt(12), tmp27], v18), + ResOperation(rop.INT_AND, [v1, ConstInt(-4)], tmp28), + ResOperation(rop.INT_OR, [tmp28, ConstInt(2)], tmp29), + ResOperation(rop.INT_FLOORDIV, [v15, tmp29], v19), + ResOperation(rop.GUARD_FALSE, [v17], None, descr=faildescr5), + ResOperation(rop.UNICODESETITEM, [const_ptr34, ConstInt(20), ConstInt(65522)], None), + ResOperation(rop.CALL, [ConstInt(func4), v3, v9, v10, v8, v11, v5, v13, v14, v15, v6], None, descr=func4descr), + ResOperation(rop.GUARD_NO_EXCEPTION, [], tmp30, descr=faildescr6), + ResOperation(rop.FINISH, [], None, descr=finishdescr7), + ] + operations[0].setfailargs([]) + operations[16].setfailargs([v5, v9]) + operations[34].setfailargs([]) + operations[37].setfailargs([v12, v19, v10, v7, v4, v8, v18, v15, v9]) + cpu.compile_bridge(faildescr1, inputargs, operations, looptoken) + frame = cpu.execute_token(looptoken, *loop_args) + #assert cpu.get_int_value(frame, 0) == -9223372036854775766 + assert cpu.get_int_value(frame, 1) == 0 + #assert cpu.get_int_value(frame, 2) == -9223372036854775808 + assert cpu.get_int_value(frame, 3) == 1 + assert cpu.get_int_value(frame, 4) == 6 + #assert cpu.get_int_value(frame, 5) == -9223372036854775808 + assert cpu.get_int_value(frame, 6) == 0 + assert cpu.get_int_value(frame, 7) == 0 + #assert cpu.get_int_value(frame, 8) == 26 + S4 = lltype.GcStruct('Sx', ("parent", rclass.OBJECT), ("f0", lltype.Signed), ("f1", lltype.Signed)) + S5 = lltype.GcStruct('Sx', ("parent", rclass.OBJECT), ("f0", lltype.Signed)) + S6 = lltype.GcStruct('Sx', ("f0", lltype.Signed), ("f1", rffi.UCHAR)) + v1 = BoxInt() + v2 = BoxInt() + v3 = BoxInt() + v4 = BoxInt() + v5 = BoxInt() + v6 = BoxInt() + v7 = BoxInt() + v8 = BoxInt() + v9 = BoxInt() + v10 = BoxInt() + v11 = BoxInt() + v12 = BoxInt() + v13 = BoxInt() + v14 = BoxInt() + v15 = BoxInt() + v16 = BoxInt() + v17 = BoxInt() + v18 = BoxInt() + tmp19 = BoxPtr() + faildescr8 = BasicFailDescr() + tmp20 = BoxInt() + tmp21 = BoxInt() + tmp22 = BoxInt() + tmp23 = BoxInt() + faildescr9 = BasicFailDescr() + tmp24 = BoxInt() + tmp25 = BoxInt() + tmp26 = BoxInt() + tmp27 = BoxPtr() + tmp28 = BoxPtr() + faildescr10 = BasicFailDescr() + finishdescr11 = BasicFinalDescr() + const_ptr29 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S4))) + const_ptr30 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(rstr.UNICODE, 26))) + const_ptr31 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(rstr.UNICODE, 1))) + const_ptr32 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S5))) + const_ptr33 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(S6))) + const_ptr34 = ConstPtr(lltype.cast_opaque_ptr(llmemory.GCREF, lltype.malloc(rstr.STR, 26))) + inputargs = [v1, v2, v3, v4, v5, v6, v7, v8, v9] + operations = [ + ResOperation(rop.GUARD_EXCEPTION, [ConstInt(xtp3)], tmp19, descr=faildescr8), + ResOperation(rop.SETFIELD_GC, [const_ptr29, v7], None, cpu.fielddescrof(S4, 'f0')), + ResOperation(rop.UNICODEGETITEM, [const_ptr30, ConstInt(21)], tmp20), + ResOperation(rop.UNICODEGETITEM, [const_ptr30, ConstInt(10)], tmp21), + ResOperation(rop.UINT_RSHIFT, [v9, ConstInt(40)], v10), + ResOperation(rop.UNICODEGETITEM, [const_ptr30, ConstInt(25)], tmp22), + ResOperation(rop.INT_NE, [ConstInt(-8), v9], v11), + ResOperation(rop.INT_MUL_OVF, [v3, ConstInt(-4)], tmp23), + ResOperation(rop.GUARD_OVERFLOW, [], None, descr=faildescr9), + ResOperation(rop.UNICODESETITEM, [const_ptr31, ConstInt(0), ConstInt(50175)], None), + ResOperation(rop.UINT_GT, [v8, ConstInt(-6)], v12), + ResOperation(rop.GETFIELD_GC, [const_ptr32], v13, cpu.fielddescrof(S5, 'f0')), + ResOperation(rop.INT_AND, [ConstInt(8), v8], v14), + ResOperation(rop.INT_INVERT, [v1], v15), + ResOperation(rop.SETFIELD_GC, [const_ptr33, ConstInt(3)], None, cpu.fielddescrof(S6, 'f1')), + ResOperation(rop.INT_GE, [v14, v6], v16), + ResOperation(rop.INT_AND, [v5, ConstInt(-4)], tmp24), + ResOperation(rop.INT_OR, [tmp24, ConstInt(2)], tmp25), + ResOperation(rop.INT_FLOORDIV, [v9, tmp25], v17), + ResOperation(rop.STRLEN, [const_ptr34], tmp26), + ResOperation(rop.NEWSTR, [ConstInt(7)], tmp27), + ResOperation(rop.STRSETITEM, [tmp27, ConstInt(0), ConstInt(21)], None), + ResOperation(rop.STRSETITEM, [tmp27, ConstInt(1), ConstInt(79)], None), + ResOperation(rop.STRSETITEM, [tmp27, ConstInt(2), ConstInt(7)], None), + ResOperation(rop.STRSETITEM, [tmp27, ConstInt(3), ConstInt(2)], None), + ResOperation(rop.STRSETITEM, [tmp27, ConstInt(4), ConstInt(229)], None), + ResOperation(rop.STRSETITEM, [tmp27, ConstInt(5), ConstInt(233)], None), + ResOperation(rop.STRSETITEM, [tmp27, ConstInt(6), ConstInt(208)], None), + ResOperation(rop.INT_LT, [ConstInt(-31), v10], v18), + ResOperation(rop.SAME_AS, [ConstPtr(lltype.nullptr(llmemory.GCREF.TO))], tmp28), + ResOperation(rop.GUARD_NONNULL_CLASS, [tmp28, ConstInt(xtp2)], None, descr=faildescr10), + ResOperation(rop.FINISH, [v4], None, descr=finishdescr11), + ] + operations[0].setfailargs([]) + operations[8].setfailargs([tmp23, v5, v3, v11, v6]) + operations[30].setfailargs([v6]) + cpu.compile_bridge(faildescr6, inputargs, operations, looptoken) + frame = cpu.execute_token(looptoken, *loop_args) + #assert cpu.get_int_value(frame, 0) == -9223372036854775808 + v1 = BoxInt() + v2 = BoxInt() + p3 = BoxPtr() + tmp4 = BoxInt() + tmp5 = BoxPtr() + faildescr12 = BasicFailDescr() + finishdescr13 = BasicFinalDescr() + inputargs = [v1] - _, func5, func5descr = getexception(cpu, 0) - vt = getvtable(cpu, S4) + _, func5, func5descr = getexception(cpu, 0) + vt = getvtable(cpu, S4) - operations = [ - ResOperation(rop.INT_AND, [v1, ConstInt(63)], tmp4), - ResOperation(rop.INT_LSHIFT, [ConstInt(10), tmp4], v2), - ResOperation(rop.NEW_WITH_VTABLE, [ConstInt(vt)], p3), - ResOperation(rop.CALL, [ConstInt(func5)], None, descr=func5descr), - ResOperation(rop.GUARD_EXCEPTION, [ConstInt(xtp2)], tmp5, descr=faildescr12), - ResOperation(rop.FINISH, [], None, descr=finishdescr13), - ] - operations[4].setfailargs([v2]) - cpu.compile_bridge(faildescr10, inputargs, operations, looptoken) - frame = cpu.execute_token(looptoken, *loop_args) - #assert cpu.get_int_value(frame, 0) == 10 - print - print - print - print "Passed %d/1000" % (i + 1) - print - print - print + operations = [ + ResOperation(rop.INT_AND, [v1, ConstInt(63)], tmp4), + ResOperation(rop.INT_LSHIFT, [ConstInt(10), tmp4], v2), + ResOperation(rop.NEW_WITH_VTABLE, [ConstInt(vt)], p3), + ResOperation(rop.CALL, [ConstInt(func5)], None, descr=func5descr), + ResOperation(rop.GUARD_EXCEPTION, [ConstInt(xtp2)], tmp5, descr=faildescr12), + ResOperation(rop.FINISH, [], None, descr=finishdescr13), + ] + operations[4].setfailargs([v2]) + cpu.compile_bridge(faildescr10, inputargs, operations, looptoken) + frame = cpu.execute_token(looptoken, *loop_args) + #assert cpu.get_int_value(frame, 0) == 10 From noreply at buildbot.pypy.org Wed Jan 23 21:23:21 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 23 Jan 2013 21:23:21 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: more checks Message-ID: <20130123202321.70AA31C00A8@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60397:952e13de787c Date: 2013-01-23 22:22 +0200 http://bitbucket.org/pypy/pypy/changeset/952e13de787c/ Log: more checks diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -200,7 +200,8 @@ gcmap_ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') mc.MOV_br(gcmap_ofs, ecx.value) mc.MOV_rs(esi.value, WORD*2) - mc.MOV_rs(edx.value, WORD*3) + gcref = cast_instance_to_gcref(self.current_clt) + mc.MOV_rs(edx.value, rffi.cast(lltype.Signed, gcref)) # push first arg mc.LEA_rb(edi.value, -base_ofs) # align @@ -687,7 +688,6 @@ self.push_gcmap(mc, gcmap, mov=True) mc.MOV_si(WORD, 0xffffff) ofs2 = mc.get_relative_pos() - 4 - mc.MOV_si(WORD*2, mc.get_relative_pos()) mc.CALL(imm(self._stack_check_failure)) # patch the JG above offset = mc.get_relative_pos() - jg_location From noreply at buildbot.pypy.org Wed Jan 23 21:28:50 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 23 Jan 2013 21:28:50 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fix Message-ID: <20130123202850.B147A1C00A8@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60398:adee01f700cd Date: 2013-01-23 22:28 +0200 http://bitbucket.org/pypy/pypy/changeset/adee01f700cd/ Log: fix diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -200,8 +200,7 @@ gcmap_ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') mc.MOV_br(gcmap_ofs, ecx.value) mc.MOV_rs(esi.value, WORD*2) - gcref = cast_instance_to_gcref(self.current_clt) - mc.MOV_rs(edx.value, rffi.cast(lltype.Signed, gcref)) + mc.MOV_rs(edx.value, WORD*3) # push first arg mc.LEA_rb(edi.value, -base_ofs) # align @@ -687,6 +686,8 @@ jg_location = mc.get_relative_pos() self.push_gcmap(mc, gcmap, mov=True) mc.MOV_si(WORD, 0xffffff) + gcref = cast_instance_to_gcref(self.current_clt) + mc.MOV_si(2*WORD, rffi.cast(lltype.Signed, gcref)) ofs2 = mc.get_relative_pos() - 4 mc.CALL(imm(self._stack_check_failure)) # patch the JG above From noreply at buildbot.pypy.org Wed Jan 23 21:33:10 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 23 Jan 2013 21:33:10 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: moar debugging Message-ID: <20130123203310.6E5AC1C00A8@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60399:64b36a3ea02a Date: 2013-01-23 22:32 +0200 http://bitbucket.org/pypy/pypy/changeset/64b36a3ea02a/ Log: moar debugging diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py --- a/rpython/jit/backend/llsupport/llmodel.py +++ b/rpython/jit/backend/llsupport/llmodel.py @@ -57,7 +57,11 @@ llmemory.GCREF)) def realloc_frame(frame, size, asm): + from rpython.rtyper.lltypesystem.ll2ctypes import _opaque_objs + from rpython.jit.backend.x86.assembler import all_clts frame = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, frame) + called_from_clt = all_clts[asm] + coming_from_loop = _opaque_objs[frame.jf_comingfrom._obj.intval // 2] if not frame.jf_frame_info.jfi_frame_depth >= size: import pdb pdb.set_trace() diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -33,6 +33,8 @@ from rpython.rlib.rarithmetic import intmask, r_uint from rpython.rlib.objectmodel import compute_unique_id +all_clts = [] + # darwin requires the stack to be 16 bytes aligned on calls. Same for gcc 4.5.0, # better safe than sorry CALL_ALIGN = 16 // WORD @@ -686,9 +688,9 @@ jg_location = mc.get_relative_pos() self.push_gcmap(mc, gcmap, mov=True) mc.MOV_si(WORD, 0xffffff) - gcref = cast_instance_to_gcref(self.current_clt) - mc.MOV_si(2*WORD, rffi.cast(lltype.Signed, gcref)) ofs2 = mc.get_relative_pos() - 4 + all_clts.append(self.current_clt) + mc.MOV_si(2*WORD, len(all_clts) - 1) mc.CALL(imm(self._stack_check_failure)) # patch the JG above offset = mc.get_relative_pos() - jg_location From noreply at buildbot.pypy.org Wed Jan 23 21:56:24 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 23 Jan 2013 21:56:24 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fixes. Also stupid cpython interpreter exit Message-ID: <20130123205624.BE46A1C0627@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60400:dad42ccc5996 Date: 2013-01-23 22:55 +0200 http://bitbucket.org/pypy/pypy/changeset/dad42ccc5996/ Log: fixes. Also stupid cpython interpreter exit diff --git a/rpython/jit/backend/llsupport/asmmemmgr.py b/rpython/jit/backend/llsupport/asmmemmgr.py --- a/rpython/jit/backend/llsupport/asmmemmgr.py +++ b/rpython/jit/backend/llsupport/asmmemmgr.py @@ -42,7 +42,8 @@ def free(self, start, stop): """Free a block (start, stop) returned by a previous malloc().""" - self.total_mallocs -= r_uint(stop - start) + if r_uint is not None: + self.total_mallocs -= r_uint(stop - start) self._add_free_block(start, stop) def open_malloc(self, minsize): diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py --- a/rpython/jit/backend/model.py +++ b/rpython/jit/backend/model.py @@ -297,6 +297,7 @@ self.cpu = cpu self.number = number self.bridges_count = 0 + self.jumping_to = [] # a list of weakrefs who jump here # This growing list gives the 'descr_number' of all fail descrs # that belong to this loop or to a bridge attached to it. # Filled by the frontend calling record_faildescr_index(). diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -1,3 +1,5 @@ + +import weakref import sys, os from rpython.jit.backend.llsupport import symbolic, jitframe from rpython.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper @@ -536,7 +538,7 @@ looppos = self.mc.get_relative_pos() looptoken._x86_loop_code = looppos frame_depth = self._assemble(regalloc, inputargs, operations) - clt.frame_info.jfi_frame_depth = frame_depth + JITFRAME_FIXED_SIZE + self.update_frame_depth(frame_depth + JITFRAME_FIXED_SIZE) # size_excluding_failure_stuff = self.mc.get_relative_pos() self.write_pending_failure_recoveries() @@ -613,7 +615,7 @@ self._patch_stackadjust(stack_check_patch_ofs + rawstart, frame_depth) self._patch_stackadjust(ofs2 + rawstart, frame_depth) self.fixup_target_tokens(rawstart) - self.current_clt.frame_info.jfi_frame_depth = frame_depth + self.update_frame_depth(frame_depth) self.teardown() # oprofile support if self.cpu.profile_agent is not None: @@ -672,6 +674,17 @@ mc.writeimm32(self.error_trampoline_64 - pos_after_jz) mc.copy_to_raw_memory(rawstart + pos_after_jz - 4) + def update_frame_depth(self, frame_depth): + self.current_clt.frame_info.jfi_frame_depth = frame_depth + new_jumping_to = [] + for wref in self.current_clt.jumping_to: + clt = wref() + if clt: + clt.frame_info.jfi_frame_depth = max(frame_depth, + clt.frame_info.jfi_frame_depth) + new_jumping_to.append(weakref.ref(clt)) + self.current_clt.jumping_to = new_jumping_to + def _check_frame_depth(self, mc, gcmap): """ check if the frame is of enough depth to follow this bridge. Otherwise reallocate the frame in a helper. @@ -2428,6 +2441,8 @@ curpos = self.mc.get_relative_pos() + 5 self.mc.JMP_l(target - curpos) else: + target_token._x86_clt.jumping_to.append( + weakref.ref(self.current_clt)) self.mc.JMP(imm(target)) def malloc_cond(self, nursery_free_adr, nursery_top_adr, size, gcmap): From noreply at buildbot.pypy.org Wed Jan 23 22:06:09 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 23 Jan 2013 22:06:09 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: cleanup my hacks Message-ID: <20130123210609.08EC31C0627@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60401:6b5fd1995177 Date: 2013-01-23 23:05 +0200 http://bitbucket.org/pypy/pypy/changeset/6b5fd1995177/ Log: cleanup my hacks diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py --- a/rpython/jit/backend/llsupport/gc.py +++ b/rpython/jit/backend/llsupport/gc.py @@ -9,7 +9,7 @@ from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.jit.codewriter import heaptracker from rpython.jit.metainterp.history import ConstPtr, AbstractDescr -from rpython.jit.metainterp.resoperation import ResOperation, rop +from rpython.jit.metainterp.resoperation import rop from rpython.jit.backend.llsupport import symbolic, jitframe from rpython.jit.backend.llsupport.symbolic import WORD from rpython.jit.backend.llsupport.descr import SizeDescr, ArrayDescr @@ -154,7 +154,6 @@ def configure_boehm_once(cls): """ Configure boehm only once, since we don't cache failures """ - return if hasattr(cls, 'malloc_fn_ptr'): return cls.malloc_fn_ptr from rpython.rtyper.tool import rffi_platform @@ -194,33 +193,10 @@ def __init__(self, gcdescr, translator, rtyper): GcLLDescription.__init__(self, gcdescr, translator, rtyper) # grab a pointer to the Boehm 'malloc' function - #self.malloc_fn_ptr = self.configure_boehm_once() + self.malloc_fn_ptr = self.configure_boehm_once() self._setup_str() self._make_functions() self.memory = 0 - self.reset_mem() - - def reset_mem(self): - malloc = rffi.llexternal('malloc', [lltype.Signed], lltype.Signed) - free = rffi.llexternal('free', [lltype.Signed], lltype.Void) - if self.memory != 0: - free(self.memory) - self.memory = 0 - A_LOT = 100*1024*1024 - self.memory = malloc(A_LOT) - self.mem_top = self.memory + A_LOT - self.mem_ptr = self.memory - - def malloc_fn_ptr(self, size): - # aligned - if size & 7: - size += 8 - size &= ~7 - from rpython.rtyper.lltypesystem.ll2ctypes import _llgcopaque - assert self.mem_ptr + size < self.mem_top - r = self.mem_ptr - self.mem_ptr += size - return lltype._ptr(llmemory.GCREF, _llgcopaque(r)) def _make_functions(self): diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py --- a/rpython/jit/backend/llsupport/llmodel.py +++ b/rpython/jit/backend/llsupport/llmodel.py @@ -52,21 +52,11 @@ pass def _setup_frame_realloc(self, translate_support_code): - FUNC_TP = lltype.Ptr(lltype.FuncType([llmemory.GCREF, lltype.Signed, - lltype.Signed], + FUNC_TP = lltype.Ptr(lltype.FuncType([llmemory.GCREF], llmemory.GCREF)) def realloc_frame(frame, size, asm): - from rpython.rtyper.lltypesystem.ll2ctypes import _opaque_objs - from rpython.jit.backend.x86.assembler import all_clts frame = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, frame) - called_from_clt = all_clts[asm] - coming_from_loop = _opaque_objs[frame.jf_comingfrom._obj.intval // 2] - if not frame.jf_frame_info.jfi_frame_depth >= size: - import pdb - pdb.set_trace() - print "realloc frame (%d) %d->%d" % (size, len(frame.jf_frame), frame.jf_frame_info.jfi_frame_depth) - print "from %x" % asm new_frame = jitframe.JITFRAME.allocate(frame.jf_frame_info) # XXX now we know, rewrite this # we need to do this, because we're not sure what things diff --git a/rpython/jit/backend/test/test_random.py b/rpython/jit/backend/test/test_random.py --- a/rpython/jit/backend/test/test_random.py +++ b/rpython/jit/backend/test/test_random.py @@ -888,10 +888,6 @@ print ' # passed.' if pytest.config.option.output: output.close() - if hasattr(cpu, 'gc_ll_descr'): - func = getattr(cpu.gc_ll_descr, 'reset_mem', None) - if func: - func() print def test_random_function(BuilderClass=OperationBuilder): diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -35,8 +35,6 @@ from rpython.rlib.rarithmetic import intmask, r_uint from rpython.rlib.objectmodel import compute_unique_id -all_clts = [] - # darwin requires the stack to be 16 bytes aligned on calls. Same for gcc 4.5.0, # better safe than sorry CALL_ALIGN = 16 // WORD @@ -591,7 +589,7 @@ operations, self.current_clt.allgcrefs, self.current_clt.frame_info) - stack_check_patch_ofs, ofs2 = self._check_frame_depth(self.mc, + stack_check_patch_ofs = self._check_frame_depth(self.mc, regalloc.get_gcmap()) frame_depth = self._assemble(regalloc, inputargs, operations) codeendpos = self.mc.get_relative_pos() @@ -613,7 +611,6 @@ frame_depth = max(self.current_clt.frame_info.jfi_frame_depth, frame_depth + JITFRAME_FIXED_SIZE) self._patch_stackadjust(stack_check_patch_ofs + rawstart, frame_depth) - self._patch_stackadjust(ofs2 + rawstart, frame_depth) self.fixup_target_tokens(rawstart) self.update_frame_depth(frame_depth) self.teardown() @@ -700,16 +697,12 @@ mc.J_il8(rx86.Conditions['GE'], 0) jg_location = mc.get_relative_pos() self.push_gcmap(mc, gcmap, mov=True) - mc.MOV_si(WORD, 0xffffff) - ofs2 = mc.get_relative_pos() - 4 - all_clts.append(self.current_clt) - mc.MOV_si(2*WORD, len(all_clts) - 1) mc.CALL(imm(self._stack_check_failure)) # patch the JG above offset = mc.get_relative_pos() - jg_location assert 0 < offset <= 127 mc.overwrite(jg_location-1, chr(offset)) - return stack_check_cmp_ofs, ofs2 + return stack_check_cmp_ofs def _patch_stackadjust(self, adr, allocated_depth): mc = codebuf.MachineCodeBlockWrapper() From noreply at buildbot.pypy.org Wed Jan 23 22:09:06 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 23 Jan 2013 22:09:06 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: this is the correct fix for recursion Message-ID: <20130123210906.F330B1C0627@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60402:317214eeafaf Date: 2013-01-23 23:08 +0200 http://bitbucket.org/pypy/pypy/changeset/317214eeafaf/ Log: this is the correct fix for recursion diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -898,10 +898,8 @@ oldadr = oldlooptoken._x86_function_addr target = newlooptoken._x86_function_addr # copy frame-info data - old_fi = oldlooptoken.compiled_loop_token.frame_info new_fi = newlooptoken.compiled_loop_token.frame_info - old_fi.jfi_frame_depth = max(old_fi.jfi_frame_depth, - new_fi.jfi_frame_depth) + oldlooptoken.compiled_loop_token.frame_info = new_fi mc = codebuf.MachineCodeBlockWrapper() mc.JMP(imm(target)) if WORD == 4: # keep in sync with prepare_loop() From noreply at buildbot.pypy.org Wed Jan 23 22:27:01 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 23 Jan 2013 22:27:01 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: leave this check in Message-ID: <20130123212701.D79981C0627@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60403:4e6a23ba8977 Date: 2013-01-23 23:26 +0200 http://bitbucket.org/pypy/pypy/changeset/4e6a23ba8977/ Log: leave this check in diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py --- a/rpython/jit/backend/llsupport/llmodel.py +++ b/rpython/jit/backend/llsupport/llmodel.py @@ -52,11 +52,12 @@ pass def _setup_frame_realloc(self, translate_support_code): - FUNC_TP = lltype.Ptr(lltype.FuncType([llmemory.GCREF], + FUNC_TP = lltype.Ptr(lltype.FuncType([llmemory.GCREF, lltype.Signed], llmemory.GCREF)) - def realloc_frame(frame, size, asm): + def realloc_frame(frame, size): frame = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, frame) + assert size <= frame.jf_frame_info.jfi_frame_depth new_frame = jitframe.JITFRAME.allocate(frame.jf_frame_info) # XXX now we know, rewrite this # we need to do this, because we're not sure what things diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -202,7 +202,6 @@ gcmap_ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') mc.MOV_br(gcmap_ofs, ecx.value) mc.MOV_rs(esi.value, WORD*2) - mc.MOV_rs(edx.value, WORD*3) # push first arg mc.LEA_rb(edi.value, -base_ofs) # align @@ -589,8 +588,8 @@ operations, self.current_clt.allgcrefs, self.current_clt.frame_info) - stack_check_patch_ofs = self._check_frame_depth(self.mc, - regalloc.get_gcmap()) + stack_check_patch_ofs, ofs2 = self._check_frame_depth(self.mc, + regalloc.get_gcmap()) frame_depth = self._assemble(regalloc, inputargs, operations) codeendpos = self.mc.get_relative_pos() self.write_pending_failure_recoveries() @@ -611,6 +610,7 @@ frame_depth = max(self.current_clt.frame_info.jfi_frame_depth, frame_depth + JITFRAME_FIXED_SIZE) self._patch_stackadjust(stack_check_patch_ofs + rawstart, frame_depth) + self._patch_stackadjust(ofs2 + rawstart, frame_depth) self.fixup_target_tokens(rawstart) self.update_frame_depth(frame_depth) self.teardown() @@ -696,13 +696,15 @@ assert not IS_X86_32 mc.J_il8(rx86.Conditions['GE'], 0) jg_location = mc.get_relative_pos() + mc.MOV_si(0, 0xffffff) + ofs2 = mc.get_relative_pos() - 4 self.push_gcmap(mc, gcmap, mov=True) mc.CALL(imm(self._stack_check_failure)) # patch the JG above offset = mc.get_relative_pos() - jg_location assert 0 < offset <= 127 mc.overwrite(jg_location-1, chr(offset)) - return stack_check_cmp_ofs + return stack_check_cmp_ofs, ofs2 def _patch_stackadjust(self, adr, allocated_depth): mc = codebuf.MachineCodeBlockWrapper() From noreply at buildbot.pypy.org Wed Jan 23 22:32:46 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 23 Jan 2013 22:32:46 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: I think I convinced myself this is enough Message-ID: <20130123213246.3F91E1C0627@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60404:111a1351fe47 Date: 2013-01-23 23:32 +0200 http://bitbucket.org/pypy/pypy/changeset/111a1351fe47/ Log: I think I convinced myself this is enough diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -900,8 +900,9 @@ oldadr = oldlooptoken._x86_function_addr target = newlooptoken._x86_function_addr # copy frame-info data + old_fi = oldlooptoken.compiled_loop_token.frame_info new_fi = newlooptoken.compiled_loop_token.frame_info - oldlooptoken.compiled_loop_token.frame_info = new_fi + old_fi.jfi_frame_depth = new_fi.jfi_frame_depth mc = codebuf.MachineCodeBlockWrapper() mc.JMP(imm(target)) if WORD == 4: # keep in sync with prepare_loop() From noreply at buildbot.pypy.org Wed Jan 23 22:40:12 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 23 Jan 2013 22:40:12 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: oops Message-ID: <20130123214012.9E9671C00A8@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60405:a200f8221461 Date: 2013-01-23 23:39 +0200 http://bitbucket.org/pypy/pypy/changeset/a200f8221461/ Log: oops diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py --- a/rpython/jit/backend/llsupport/regalloc.py +++ b/rpython/jit/backend/llsupport/regalloc.py @@ -2,6 +2,7 @@ from rpython.jit.metainterp.history import Const, Box, REF, INT from rpython.rlib.objectmodel import we_are_translated, specialize from rpython.jit.metainterp.resoperation import rop +from collections import OrderedDict class TempBox(Box): def __init__(self): @@ -268,7 +269,10 @@ def __init__(self, longevity, frame_manager=None, assembler=None): self.free_regs = self.all_regs[:] self.longevity = longevity - self.reg_bindings = {} + if not we_are_translated(): + self.reg_bindings = OrderedDict() + else: + self.reg_bindings = {} self.bindings_to_frame_reg = {} self.position = -1 self.frame_manager = frame_manager diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -696,7 +696,7 @@ assert not IS_X86_32 mc.J_il8(rx86.Conditions['GE'], 0) jg_location = mc.get_relative_pos() - mc.MOV_si(0, 0xffffff) + mc.MOV_si(WORD, 0xffffff) ofs2 = mc.get_relative_pos() - 4 self.push_gcmap(mc, gcmap, mov=True) mc.CALL(imm(self._stack_check_failure)) diff --git a/rpython/jit/backend/x86/test/test_gc_integration.py b/rpython/jit/backend/x86/test/test_gc_integration.py --- a/rpython/jit/backend/x86/test/test_gc_integration.py +++ b/rpython/jit/backend/x86/test/test_gc_integration.py @@ -334,6 +334,11 @@ GcCache.__init__(self, False, None) self.gcrootmap = MockShadowStackRootMap() +def unpack_gcmap(frame): + pass + #for i in range(len(frame.jf_gcmap)): + # item = frame.jf_gcmap[item] + class TestGcShadowstackDirect(BaseTestRegalloc): cpu = CPU(None, None) @@ -353,6 +358,8 @@ new_frame = frame.copy() self.cpu.gc_ll_descr.gcrootmap.stack[0] = rffi.cast(lltype.Signed, new_frame) frames.append(new_frame) + import pdb + pdb.set_trace() def check2(i): assert self.cpu.gc_ll_descr.gcrootmap.stack[0] == i - ofs From noreply at buildbot.pypy.org Wed Jan 23 23:16:47 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 23 Jan 2013 23:16:47 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: disable those debug prints Message-ID: <20130123221647.564E91C00A8@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60406:23c6dc20e6d7 Date: 2013-01-24 00:16 +0200 http://bitbucket.org/pypy/pypy/changeset/23c6dc20e6d7/ Log: disable those debug prints diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py --- a/rpython/jit/backend/model.py +++ b/rpython/jit/backend/model.py @@ -325,10 +325,10 @@ debug_stop("jit-mem-looptoken-alloc") def __del__(self): - debug_start("jit-mem-looptoken-free") - debug_print("freeing Loop #", self.number, 'with', - self.bridges_count, 'attached bridges') + #debug_start("jit-mem-looptoken-free") + #debug_print("freeing Loop #", self.number, 'with', + # self.bridges_count, 'attached bridges') self.cpu.free_loop_and_bridges(self) self.cpu.tracker.total_freed_loops += 1 self.cpu.tracker.total_freed_bridges += self.bridges_count - debug_stop("jit-mem-looptoken-free") + #debug_stop("jit-mem-looptoken-free") From noreply at buildbot.pypy.org Wed Jan 23 23:29:10 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 23 Jan 2013 23:29:10 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: one extra mov for a bit Message-ID: <20130123222910.7D60A1C0627@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60407:d29ceae110c7 Date: 2013-01-24 00:28 +0200 http://bitbucket.org/pypy/pypy/changeset/d29ceae110c7/ Log: one extra mov for a bit diff --git a/rpython/jit/backend/x86/test/test_runner.py b/rpython/jit/backend/x86/test/test_runner.py --- a/rpython/jit/backend/x86/test/test_runner.py +++ b/rpython/jit/backend/x86/test/test_runner.py @@ -31,8 +31,8 @@ # ====> ../../test/runner_test.py add_loop_instructions = ['mov', 'add', 'test', 'je', 'jmp'] - bridge_loop_instructions = ['cmp', 'jge', 'mov', 'mov', 'mov', 'call', - 'mov', 'jmp'] + bridge_loop_instructions = ['cmp', 'jge', 'mov', 'mov', 'mov', 'mov', + 'call', 'mov', 'jmp'] def setup_method(self, meth): self.cpu = CPU(rtyper=None, stats=FakeStats()) From noreply at buildbot.pypy.org Wed Jan 23 23:41:52 2013 From: noreply at buildbot.pypy.org (amauryfa) Date: Wed, 23 Jan 2013 23:41:52 +0100 (CET) Subject: [pypy-commit] pypy py3k: Undo 5364600: dict.pop() was implemented independently in trunk. Message-ID: <20130123224152.E8FFE1C0627@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r60408:05813142f683 Date: 2013-01-23 23:28 +0100 http://bitbucket.org/pypy/pypy/changeset/05813142f683/ Log: Undo 5364600: dict.pop() was implemented independently in trunk. diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py --- a/rpython/annotator/unaryop.py +++ b/rpython/annotator/unaryop.py @@ -398,7 +398,6 @@ return dct.dictdef.read_value() method_setdefault = method_get - method_pop = method_get def method_copy(dct): return SomeDict(dct.dictdef) diff --git a/rpython/rtyper/lltypesystem/rdict.py b/rpython/rtyper/lltypesystem/rdict.py --- a/rpython/rtyper/lltypesystem/rdict.py +++ b/rpython/rtyper/lltypesystem/rdict.py @@ -274,13 +274,6 @@ v_res = hop.gendirectcall(ll_setdefault, v_dict, v_key, v_default) return self.recast_value(hop.llops, v_res) - def rtype_method_pop(self, hop): - v_dict, v_key, v_default = hop.inputargs(self, self.key_repr, - self.value_repr) - hop.exception_cannot_occur() - v_res = hop.gendirectcall(ll_dict_pop, v_dict, v_key, v_default) - return self.recast_value(hop.llops, v_res) - def rtype_method_copy(self, hop): v_dict, = hop.inputargs(self) hop.exception_cannot_occur() @@ -518,15 +511,6 @@ raise KeyError _ll_dict_del(d, i) -def ll_dict_pop(d, key, default): - i = ll_dict_lookup(d, key, d.keyhash(key)) - if not i & HIGHEST_BIT: - value = ll_get_value(d, i) - _ll_dict_del(d, i) - return value - else: - return default - @jit.look_inside_iff(lambda d, i: jit.isvirtual(d) and jit.isconstant(i)) def _ll_dict_del(d, i): d.entries.mark_deleted(i) diff --git a/rpython/rtyper/ootypesystem/rdict.py b/rpython/rtyper/ootypesystem/rdict.py --- a/rpython/rtyper/ootypesystem/rdict.py +++ b/rpython/rtyper/ootypesystem/rdict.py @@ -100,13 +100,6 @@ v_res = hop.gendirectcall(ll_dict_setdefault, v_dict, v_key, v_default) return self.recast_value(hop.llops, v_res) - def rtype_method_pop(self, hop): - v_dict, v_key, v_default = hop.inputargs(self, self.key_repr, - self.value_repr) - hop.exception_cannot_occur() - v_res = hop.gendirectcall(ll_dict_pop, v_dict, v_key, v_default) - return self.recast_value(hop.llops, v_res) - def rtype_method_copy(self, hop): v_dict, = hop.inputargs(self) cDICT = hop.inputconst(ootype.Void, self.lowleveltype) @@ -346,14 +339,6 @@ d.ll_set(key, default) return default -def ll_dict_pop(d, key, default): - if d.ll_contains(key): - value = d.ll_get(key) - d.ll_remove(key) - return value - else: - return default - def _make_ll_keys_values_items(kind): def ll_dict_kvi(LIST, d): length = d.ll_length() diff --git a/rpython/rtyper/test/test_rdict.py b/rpython/rtyper/test/test_rdict.py --- a/rpython/rtyper/test/test_rdict.py +++ b/rpython/rtyper/test/test_rdict.py @@ -181,16 +181,6 @@ res = self.interpret(func, ()) assert res == 422 - def test_dict_pop(self): - def func(): - dic = {} - x1 = dic.pop('hi', 42) - dic['blah'] = 1 - x2 = dic.pop('blah', 2) - return len(dic) * 100 + x1 * 10 + x2 - res = self.interpret(func, ()) - assert res == 421 - def test_dict_setdefault(self): def f(): d = {} From noreply at buildbot.pypy.org Wed Jan 23 23:49:29 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 23 Jan 2013 23:49:29 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: RPythonization Message-ID: <20130123224929.0E0C21C0634@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60409:2a8f21ff4463 Date: 2013-01-24 00:49 +0200 http://bitbucket.org/pypy/pypy/changeset/2a8f21ff4463/ Log: RPythonization diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -676,7 +676,7 @@ new_jumping_to = [] for wref in self.current_clt.jumping_to: clt = wref() - if clt: + if clt is not None: clt.frame_info.jfi_frame_depth = max(frame_depth, clt.frame_info.jfi_frame_depth) new_jumping_to.append(weakref.ref(clt)) @@ -2435,8 +2435,10 @@ curpos = self.mc.get_relative_pos() + 5 self.mc.JMP_l(target - curpos) else: + clt = self.current_clt + assert clt is not None target_token._x86_clt.jumping_to.append( - weakref.ref(self.current_clt)) + weakref.ref(clt)) self.mc.JMP(imm(target)) def malloc_cond(self, nursery_free_adr, nursery_top_adr, size, gcmap): From noreply at buildbot.pypy.org Thu Jan 24 00:05:50 2013 From: noreply at buildbot.pypy.org (amauryfa) Date: Thu, 24 Jan 2013 00:05:50 +0100 (CET) Subject: [pypy-commit] pypy py3k: hg merge default Message-ID: <20130123230550.0A6181C1055@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r60410:792caad9d5ca Date: 2013-01-23 23:58 +0100 http://bitbucket.org/pypy/pypy/changeset/792caad9d5ca/ Log: hg merge default diff --git a/rpython/jit/codewriter/effectinfo.py b/rpython/jit/codewriter/effectinfo.py --- a/rpython/jit/codewriter/effectinfo.py +++ b/rpython/jit/codewriter/effectinfo.py @@ -79,9 +79,11 @@ OS_RAW_MALLOC_VARSIZE = 110 OS_RAW_FREE = 111 + OS_JIT_FORCE_VIRTUAL = 120 + # for debugging: _OS_CANRAISE = set([OS_NONE, OS_STR2UNICODE, OS_LIBFFI_CALL, - OS_RAW_MALLOC_VARSIZE]) + OS_RAW_MALLOC_VARSIZE, OS_JIT_FORCE_VIRTUAL]) def __new__(cls, readonly_descrs_fields, readonly_descrs_arrays, write_descrs_fields, write_descrs_arrays, diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -1393,6 +1393,8 @@ elif oopspec_name == 'jit.isvirtual': kind = getkind(args[0].concretetype) return SpaceOperation('%s_isvirtual' % kind, args, op.result) + elif oopspec_name == 'jit.force_virtual': + return self._handle_oopspec_call(op, args, EffectInfo.OS_JIT_FORCE_VIRTUAL, EffectInfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE) else: raise AssertionError("missing support for %r" % oopspec_name) diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py --- a/rpython/jit/codewriter/support.py +++ b/rpython/jit/codewriter/support.py @@ -1,24 +1,24 @@ import sys -from rpython.rtyper.lltypesystem import lltype, rclass, rffi, llmemory -from rpython.rtyper.ootypesystem import ootype -from rpython.rtyper import rlist -from rpython.rtyper.lltypesystem import rstr as ll_rstr, rdict as ll_rdict -from rpython.rtyper.lltypesystem.module import ll_math -from rpython.rtyper.lltypesystem.lloperation import llop -from rpython.rtyper.ootypesystem import rdict as oo_rdict -from rpython.rtyper.llinterp import LLInterpreter -from rpython.rtyper.extregistry import ExtRegistryEntry -from rpython.translator.simplify import get_funcobj -from rpython.translator.unsimplify import split_block + +from rpython.annotator import model as annmodel +from rpython.annotator.policy import AnnotatorPolicy from rpython.flowspace.model import Variable, Constant -from rpython.translator.translator import TranslationContext -from rpython.annotator.policy import AnnotatorPolicy -from rpython.annotator import model as annmodel -from rpython.rtyper.annlowlevel import MixLevelHelperAnnotator from rpython.jit.metainterp.typesystem import deref from rpython.rlib import rgc -from rpython.rlib.jit import elidable +from rpython.rlib.jit import elidable, oopspec from rpython.rlib.rarithmetic import r_longlong, r_ulonglong, r_uint, intmask +from rpython.rtyper import rlist +from rpython.rtyper.annlowlevel import MixLevelHelperAnnotator +from rpython.rtyper.extregistry import ExtRegistryEntry +from rpython.rtyper.llinterp import LLInterpreter +from rpython.rtyper.lltypesystem import lltype, rclass, rffi, llmemory, rstr as ll_rstr, rdict as ll_rdict +from rpython.rtyper.lltypesystem.lloperation import llop +from rpython.rtyper.lltypesystem.module import ll_math +from rpython.rtyper.ootypesystem import ootype, rdict as oo_rdict +from rpython.translator.simplify import get_funcobj +from rpython.translator.translator import TranslationContext +from rpython.translator.unsimplify import split_block + def getargtypes(annotator, values): if values is None: # for backend tests producing stand-alone exe's @@ -213,10 +213,12 @@ _ll_5_list_ll_arraycopy = rgc.ll_arraycopy + @elidable def _ll_1_gc_identityhash(x): return lltype.identityhash(x) + # the following function should not be "@elidable": I can think of # a corner case in which id(const) is constant-folded, and then 'const' # disappears and is collected too early (possibly causing another object @@ -224,6 +226,8 @@ def _ll_1_gc_id(ptr): return llop.gc_id(lltype.Signed, ptr) + + at oopspec("jit.force_virtual(inst)") def _ll_1_jit_force_virtual(inst): return llop.jit_force_virtual(lltype.typeOf(inst), inst) diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py --- a/rpython/jit/metainterp/history.py +++ b/rpython/jit/metainterp/history.py @@ -978,11 +978,9 @@ def get_all_jitcell_tokens(self): tokens = [t() for t in self.jitcell_token_wrefs] if None in tokens: - assert False, "get_all_jitcell_tokens will not work as "+\ - "loops have been freed" + assert False, ("get_all_jitcell_tokens will not work as " + "loops have been freed") return tokens - - def check_history(self, expected=None, **check): insns = {} diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -1,12 +1,13 @@ +from rpython.jit.codewriter.effectinfo import EffectInfo +from rpython.jit.metainterp.executor import execute from rpython.jit.codewriter.heaptracker import vtable2descr -from rpython.jit.metainterp.executor import execute from rpython.jit.metainterp.history import Const, ConstInt, BoxInt from rpython.jit.metainterp.optimizeopt import optimizer +from rpython.jit.metainterp.optimizeopt.optimizer import OptValue, REMOVED from rpython.jit.metainterp.optimizeopt.util import (make_dispatcher_method, descrlist_dict, sort_descrs) from rpython.jit.metainterp.resoperation import rop, ResOperation from rpython.rlib.objectmodel import we_are_translated -from rpython.jit.metainterp.optimizeopt.optimizer import OptValue class AbstractVirtualValue(optimizer.OptValue): @@ -386,6 +387,24 @@ self.make_equal_to(box, vvalue) return vvalue + def optimize_GUARD_NO_EXCEPTION(self, op): + if self.last_emitted_operation is REMOVED: + return + self.emit_operation(op) + + def optimize_GUARD_NOT_FORCED(self, op): + if self.last_emitted_operation is REMOVED: + return + self.emit_operation(op) + + def optimize_CALL_MAY_FORCE(self, op): + effectinfo = op.getdescr().get_extra_info() + oopspecindex = effectinfo.oopspecindex + if oopspecindex == EffectInfo.OS_JIT_FORCE_VIRTUAL: + if self._optimize_JIT_FORCE_VIRTUAL(op): + return + self.emit_operation(op) + def optimize_VIRTUAL_REF(self, op): indexbox = op.getarg(1) # @@ -429,7 +448,7 @@ # - set 'virtual_token' to TOKEN_NONE args = [op.getarg(0), ConstInt(vrefinfo.TOKEN_NONE)] seo(ResOperation(rop.SETFIELD_GC, args, None, - descr = vrefinfo.descr_virtual_token)) + descr=vrefinfo.descr_virtual_token)) # Note that in some cases the virtual in op.getarg(1) has been forced # already. This is fine. In that case, and *if* a residual # CALL_MAY_FORCE suddenly turns out to access it, then it will @@ -437,6 +456,20 @@ # will work too (but just be a little pointless, as the structure # was already forced). + def _optimize_JIT_FORCE_VIRTUAL(self, op): + vref = self.getvalue(op.getarg(1)) + vrefinfo = self.optimizer.metainterp_sd.virtualref_info + if vref.is_virtual(): + tokenvalue = vref.getfield(vrefinfo.descr_virtual_token, None) + if (tokenvalue is not None and tokenvalue.is_constant() and + tokenvalue.box.getint() == vrefinfo.TOKEN_NONE): + forcedvalue = vref.getfield(vrefinfo.descr_forced, None) + if forcedvalue is not None and not forcedvalue.is_null(): + self.make_equal_to(op.result, forcedvalue) + self.last_emitted_operation = REMOVED + return True + return False + def optimize_GETFIELD_GC(self, op): value = self.getvalue(op.getarg(0)) # If this is an immutable field (as indicated by op.is_always_pure()) diff --git a/rpython/jit/metainterp/test/support.py b/rpython/jit/metainterp/test/support.py --- a/rpython/jit/metainterp/test/support.py +++ b/rpython/jit/metainterp/test/support.py @@ -160,16 +160,17 @@ class JitMixin: basic = True + def check_resops(self, expected=None, **check): get_stats().check_resops(expected=expected, **check) + def check_simple_loop(self, expected=None, **check): get_stats().check_simple_loop(expected=expected, **check) - - def check_trace_count(self, count): # was check_loop_count # The number of traces compiled assert get_stats().compiled_count == count + def check_trace_count_at_most(self, count): assert get_stats().compiled_count <= count @@ -178,11 +179,12 @@ def check_target_token_count(self, count): tokens = get_stats().get_all_jitcell_tokens() - n = sum ([len(t.target_tokens) for t in tokens]) + n = sum([len(t.target_tokens) for t in tokens]) assert n == count def check_enter_count(self, count): assert get_stats().enter_count == count + def check_enter_count_at_most(self, count): assert get_stats().enter_count <= count @@ -192,6 +194,7 @@ def check_aborted_count(self, count): assert get_stats().aborted_count == count + def check_aborted_count_at_least(self, count): assert get_stats().aborted_count >= count diff --git a/rpython/jit/metainterp/test/test_virtualref.py b/rpython/jit/metainterp/test/test_virtualref.py --- a/rpython/jit/metainterp/test/test_virtualref.py +++ b/rpython/jit/metainterp/test/test_virtualref.py @@ -1,19 +1,20 @@ import py -from rpython.rtyper.lltypesystem import lltype, llmemory, lloperation + +from rpython.rtyper.lltypesystem import lltype, lloperation from rpython.rtyper.exceptiondata import UnknownException from rpython.rlib.jit import JitDriver, dont_look_inside, vref_None from rpython.rlib.jit import virtual_ref, virtual_ref_finish, InvalidVirtualRef from rpython.rlib.jit import non_virtual_ref from rpython.rlib.objectmodel import compute_unique_id -from rpython.jit.metainterp.test.support import LLJitMixin, OOJitMixin, _get_jitcodes +from rpython.jit.metainterp.test.support import LLJitMixin, _get_jitcodes from rpython.jit.metainterp.resoperation import rop from rpython.jit.metainterp.virtualref import VirtualRefInfo + debug_print = lloperation.llop.debug_print -class VRefTests: - +class VRefTests(object): def finish_setup_for_interp_operations(self): self.vrefinfo = VirtualRefInfo(self.warmrunnerstate) self.cw.setup_vrefinfo(self.vrefinfo) @@ -115,8 +116,8 @@ from rpython.jit.metainterp.resume import ResumeDataDirectReader cpu = self.metainterp.cpu cpu.get_latest_value_count = lambda df: len(guard_op.getfailargs()) - cpu.get_latest_value_int = lambda df,i:guard_op.getfailargs()[i].getint() - cpu.get_latest_value_ref = lambda df,i:guard_op.getfailargs()[i].getref_base() + cpu.get_latest_value_int = lambda df, i: guard_op.getfailargs()[i].getint() + cpu.get_latest_value_ref = lambda df, i: guard_op.getfailargs()[i].getref_base() cpu.clear_latest_values = lambda count: None class FakeMetaInterpSd: callinfocollection = None @@ -418,15 +419,18 @@ self.check_aborted_count(0) def test_jit_force_virtual_seen(self): - myjitdriver = JitDriver(greens = [], reds = ['n']) - # + myjitdriver = JitDriver(greens=[], reds=['n']) + A = lltype.GcArray(lltype.Signed) - class XY: + + class XY(object): pass - class ExCtx: + + class ExCtx(object): pass exctx = ExCtx() - # + escapes = [] + def f(n): while n > 0: myjitdriver.can_enter_jit(n=n) @@ -434,16 +438,16 @@ xy = XY() xy.n = n exctx.topframeref = vref = virtual_ref(xy) + escapes.append(xy) xy.next1 = lltype.malloc(A, 0) n = exctx.topframeref().n - 1 - xy.next1 = lltype.nullptr(A) exctx.topframeref = vref_None virtual_ref_finish(vref, xy) return 1 # res = self.meta_interp(f, [15]) assert res == 1 - self.check_resops(new_with_vtable=4, # vref, xy + self.check_resops(new_with_vtable=2, # xy new_array=2) # next1 self.check_aborted_count(0) @@ -656,6 +660,34 @@ res = self.meta_interp(f, [10]) assert res == 0 + def test_force_virtual_vref(self): + myjitdriver = JitDriver(greens=[], reds=['n', 'ec']) + + class ExecutionContext(object): + pass + + class Frame(object): + def __init__(self, x): + self.x = x + + def f(n): + ec = ExecutionContext() + while n > 0: + myjitdriver.jit_merge_point(n=n, ec=ec) + frame = Frame(1) + ec.topframeref = virtual_ref(frame) + n -= ec.topframeref().x + frame_vref = ec.topframeref + ec.topframeref = vref_None + virtual_ref_finish(frame_vref, frame) + return n + res = self.meta_interp(f, [10]) + assert res == 0 + self.check_resops({ + 'int_sub': 2, 'int_gt': 2, 'jump': 1, 'guard_true': 2, + 'force_token': 2, 'setfield_gc': 1 + }) + class TestLLtype(VRefTests, LLJitMixin): pass diff --git a/rpython/rlib/unicodedata/test/test_ucd.py b/rpython/rlib/unicodedata/test/test_ucd.py --- a/rpython/rlib/unicodedata/test/test_ucd.py +++ b/rpython/rlib/unicodedata/test/test_ucd.py @@ -1,9 +1,9 @@ +from rpython.rlib.runicode import code_to_unichr +from rpython.rlib.unicodedata import unicodedb_5_2_0 from rpython.rtyper.test.tool import BaseRtypingTest, LLRtypeMixin -from rpython.rlib.unicodedata import unicodedb_5_2_0 -from rpython.rlib.unicodedata.ucd import code_to_unichr + class TestTranslated(BaseRtypingTest, LLRtypeMixin): - def test_translated(self): def f(n): if n == 0: diff --git a/rpython/rlib/unicodedata/test/test_unicodedata.py b/rpython/rlib/unicodedata/test/test_unicodedata.py --- a/rpython/rlib/unicodedata/test/test_unicodedata.py +++ b/rpython/rlib/unicodedata/test/test_unicodedata.py @@ -1,11 +1,15 @@ +import random +import unicodedata + import py + from rpython.rlib.unicodedata import unicodedb_3_2_0, unicodedb_5_2_0 + class TestUnicodeData(object): def setup_class(cls): - import random, unicodedata if unicodedata.unidata_version != '5.2.0': - skip('Needs python with unicode 5.2.0 database.') + py.test.skip('Needs python with unicode 5.2.0 database.') seed = random.getrandbits(32) print "random seed: ", seed @@ -39,14 +43,12 @@ assert not unicodedb_5_2_0.isprintable(0xE0020) # TAG SPACE def test_compare_functions(self): - import unicodedata # CPython implementation - def getX(fun, code): try: return getattr(unicodedb_5_2_0, fun)(code) except KeyError: return -1 - + for code in range(0x10000): char = unichr(code) assert unicodedata.digit(char, -1) == getX('digit', code) @@ -83,5 +85,3 @@ assert unicodedb_5_2_0.lookup('BENZENE RING WITH CIRCLE') == 9187 py.test.raises(KeyError, unicodedb_3_2_0.lookup, 'BENZENE RING WITH CIRCLE') py.test.raises(KeyError, unicodedb_3_2_0.name, 9187) - - From noreply at buildbot.pypy.org Thu Jan 24 00:21:17 2013 From: noreply at buildbot.pypy.org (liquibits) Date: Thu, 24 Jan 2013 00:21:17 +0100 (CET) Subject: [pypy-commit] pypy default: updated translation docs after rpython split Message-ID: <20130123232117.84EC31C105D@cobra.cs.uni-duesseldorf.de> Author: Pawe? Piotr Przeradowski Branch: Changeset: r60411:0941291008cf Date: 2013-01-24 00:16 +0100 http://bitbucket.org/pypy/pypy/changeset/0941291008cf/ Log: updated translation docs after rpython split diff --git a/pypy/doc/getting-started-python.rst b/pypy/doc/getting-started-python.rst --- a/pypy/doc/getting-started-python.rst +++ b/pypy/doc/getting-started-python.rst @@ -86,8 +86,8 @@ 4. Run:: - cd pypy/translator/goal - python translate.py --opt=jit targetpypystandalone.py + cd pypy/goal + python ../../rpython/bin/rpython --opt=jit targetpypystandalone.py possibly replacing ``--opt=jit`` with another `optimization level`_ of your choice like ``--opt=2`` if you do not want to include the JIT From noreply at buildbot.pypy.org Thu Jan 24 00:30:39 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 24 Jan 2013 00:30:39 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: continue writing test until we hit a bug. some stuff still does not work though Message-ID: <20130123233039.022DC1C1069@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60412:b06b729af83e Date: 2013-01-24 01:30 +0200 http://bitbucket.org/pypy/pypy/changeset/b06b729af83e/ Log: continue writing test until we hit a bug. some stuff still does not work though diff --git a/rpython/jit/backend/llsupport/jitframe.py b/rpython/jit/backend/llsupport/jitframe.py --- a/rpython/jit/backend/llsupport/jitframe.py +++ b/rpython/jit/backend/llsupport/jitframe.py @@ -47,7 +47,6 @@ # guard_not_forced descr ('jf_force_descr', llmemory.GCREF), # a map of GC pointers - ('jf_comingfrom', llmemory.GCREF), ('jf_gcmap', lltype.Ptr(GCMAP)), # For the front-end: a GCREF for the savedata ('jf_savedata', llmemory.GCREF), diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -862,8 +862,6 @@ self.mc.RET() def _call_header_shadowstack(self, gcrootmap): - # we need to put two words into the shadowstack: the MARKER_FRAME - # and the address of the frame (ebp, actually) rst = gcrootmap.get_root_stack_top_addr() if rx86.fits_in_32bits(rst): self.mc.MOV_rj(eax.value, rst) # MOV eax, [rootstacktop] @@ -1252,7 +1250,6 @@ gcrootmap = self.cpu.gc_ll_descr.gcrootmap if gcrootmap and gcrootmap.is_shadow_stack: rst = gcrootmap.get_root_stack_top_addr() - mc.MOV(ecx, ebp) # debugging mc.MOV(edx, heap(rst)) mc.MOV(ebp, mem(edx, -WORD)) base_ofs = self.cpu.get_baseofs_of_frame_field() @@ -2009,6 +2006,8 @@ ofs = self.cpu.get_ofs_of_frame_field('jf_descr') base_ofs = self.cpu.get_baseofs_of_frame_field() self.mov(fail_descr_loc, RawStackLoc(ofs)) + gcmap = self._regalloc.get_gcmap() + self.push_gcmap(self.mc, gcmap, store=True) self.mc.LEA_rb(eax.value, -base_ofs) # exit function self._call_footer() diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py --- a/rpython/jit/backend/x86/runner.py +++ b/rpython/jit/backend/x86/runner.py @@ -119,15 +119,13 @@ #llop.debug_print(lltype.Void, ">>>> Entering", addr) frame_info = clt.frame_info frame = jitframe.JITFRAME.allocate(frame_info) - - from rpython.rtyper.annlowlevel import cast_instance_to_gcref - - frame.jf_comingfrom = cast_instance_to_gcref(executable_token) ll_frame = lltype.cast_opaque_ptr(llmemory.GCREF, frame) prev_interpreter = None # help flow space if not self.translate_support_code: prev_interpreter = LLInterpreter.current_interpreter LLInterpreter.current_interpreter = self.debug_ll_interpreter + if hasattr(self, 'register_frame'): + self.register_frame(frame) try: num = JITFRAME_FIXED_SIZE * WORD for i, kind in kinds: @@ -156,12 +154,6 @@ cast_ptr_to_int._annspecialcase_ = 'specialize:arglltype(0)' cast_ptr_to_int = staticmethod(cast_ptr_to_int) - all_null_registers = lltype.malloc(rffi.LONGP.TO, - IS_X86_32 and (16+8) # 16 + 8 regs - or (16+16), # 16 + 16 regs - flavor='raw', zero=True, - immortal=True) - def force(self, addr_of_force_token): descr = self.signedarraydescr ofs = self.unpack_arraydescr(descr) diff --git a/rpython/jit/backend/x86/test/test_gc_integration.py b/rpython/jit/backend/x86/test/test_gc_integration.py --- a/rpython/jit/backend/x86/test/test_gc_integration.py +++ b/rpython/jit/backend/x86/test/test_gc_integration.py @@ -3,7 +3,7 @@ """ from rpython.jit.metainterp.history import TargetToken, BasicFinalDescr,\ - JitCellToken + JitCellToken, BasicFailDescr from rpython.jit.backend.llsupport.gc import GcLLDescription, GcLLDescr_boehm,\ GcLLDescr_framework, GcCache from rpython.jit.backend.detect_cpu import getcpuclass @@ -15,6 +15,7 @@ from rpython.jit.backend.x86.test.test_regalloc import BaseTestRegalloc from rpython.jit.backend.x86.regalloc import gpr_reg_mgr_cls from rpython.jit.codewriter.effectinfo import EffectInfo +from rpython.rtyper.memory.gcheader import GCHeaderBuilder CPU = getcpuclass() @@ -324,6 +325,9 @@ layoutbuilder = None write_barrier_descr = None + class GCClass: + JIT_WB_IF_FLAG = 0 + def get_malloc_slowpath_addr(self): return 0 @@ -335,9 +339,16 @@ self.gcrootmap = MockShadowStackRootMap() def unpack_gcmap(frame): - pass - #for i in range(len(frame.jf_gcmap)): - # item = frame.jf_gcmap[item] + res = [] + val = 0 + for i in range(len(frame.jf_gcmap)): + item = frame.jf_gcmap[i] + while item != 0: + if item & 1: + res.append(val) + val += 1 + item >>= 1 + return res class TestGcShadowstackDirect(BaseTestRegalloc): @@ -352,20 +363,21 @@ def check(i): assert self.cpu.gc_ll_descr.gcrootmap.stack[0] == i - ofs frame = rffi.cast(jitframe.JITFRAMEPTR, i - ofs) - assert len(frame.jf_frame) == JITFRAME_FIXED_SIZE + assert len(frame.jf_frame) == JITFRAME_FIXED_SIZE + 4 # we "collect" frames.append(frame) new_frame = frame.copy() - self.cpu.gc_ll_descr.gcrootmap.stack[0] = rffi.cast(lltype.Signed, new_frame) + assert self.cpu.gc_ll_descr.gcrootmap.stack[0] == rffi.cast(lltype.Signed, frame) + hdrbuilder.new_header(new_frame) + #gc_ll_descr.gcrootmap.stack[0] = rffi.cast(lltype.Signed, new_frame) frames.append(new_frame) - import pdb - pdb.set_trace() + assert unpack_gcmap(frame) == [28, 29, 30] def check2(i): assert self.cpu.gc_ll_descr.gcrootmap.stack[0] == i - ofs frame = rffi.cast(jitframe.JITFRAMEPTR, i - ofs) - assert frame == frames[1] - assert frame != frames[0] + #assert frame == frames[1] + #assert frame != frames[0] CHECK = lltype.FuncType([lltype.Signed], lltype.Void) checkptr = llhelper(lltype.Ptr(CHECK), check) @@ -373,15 +385,36 @@ checkdescr = self.cpu.calldescrof(CHECK, CHECK.ARGS, CHECK.RESULT, EffectInfo.MOST_GENERAL) + S = lltype.GcStruct('S', + ('x', lltype.Ptr(lltype.GcArray(lltype.Signed)))) loop = self.parse(""" - [] + [p0, p1, p2] i0 = force_token() # this is a bit below the frame call(ConstClass(check_adr), i0, descr=checkdescr) # this can collect + p3 = getfield_gc(p0, descr=fielddescr) call(ConstClass(check2_adr), i0, descr=checkdescr) - finish(i0, descr=finaldescr) + guard_true(i0, descr=faildescr) [p0, p1, p2, p3] + p4 = getfield_gc(p0, descr=fielddescr) + finish(p4, descr=finaldescr) """, namespace={'finaldescr': BasicFinalDescr(), + 'faildescr': BasicFailDescr(), 'check_adr': checkptr, 'check2_adr': check2ptr, - 'checkdescr': checkdescr}) + 'checkdescr': checkdescr, + 'fielddescr': self.cpu.fielddescrof(S, 'x')}) token = JitCellToken() self.cpu.compile_loop(loop.inputargs, loop.operations, token) - frame = self.cpu.execute_token(token) + self.cpu.register_frame = lambda frame : hdrbuilder.new_header(frame) + HDR = lltype.Struct('HDR', ('tid', lltype.Signed)) + hdrbuilder = GCHeaderBuilder(HDR) + gc_ll_descr = self.cpu.gc_ll_descr + gc_ll_descr.gcheaderbuilder = hdrbuilder + gc_ll_descr.HDRPTR = lltype.Ptr(HDR) + p0 = lltype.malloc(S, zero=True) + p1 = lltype.malloc(S) + p2 = lltype.malloc(S) + hdrbuilder.new_header(p0) + hdrbuilder.new_header(p1) + hdrbuilder.new_header(p2) + frame = self.cpu.execute_token(token, p0, p1, p2) + gcmap = unpack_gcmap(lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, frame)) + assert gcmap == [11] From noreply at buildbot.pypy.org Thu Jan 24 09:10:40 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 24 Jan 2013 09:10:40 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: finish the test and make it pass Message-ID: <20130124081040.8D5F81C00A8@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60413:1f9aea6048aa Date: 2013-01-24 10:10 +0200 http://bitbucket.org/pypy/pypy/changeset/1f9aea6048aa/ Log: finish the test and make it pass diff --git a/rpython/jit/backend/llsupport/jitframe.py b/rpython/jit/backend/llsupport/jitframe.py --- a/rpython/jit/backend/llsupport/jitframe.py +++ b/rpython/jit/backend/llsupport/jitframe.py @@ -30,12 +30,6 @@ frame.jf_frame_info = frame_info return frame -def jitframe_copy(frame): - frame_info = frame.jf_frame_info - new_frame = lltype.malloc(JITFRAME, frame_info.jfi_frame_depth, zero=True) - new_frame.jf_frame_info = frame_info - return new_frame - JITFRAME = lltype.GcStruct( 'JITFRAME', ('jf_frame_info', lltype.Ptr(JITFRAMEINFO)), @@ -63,7 +57,6 @@ # about GCrefs here and not in frame info which might change adtmeths = { 'allocate': jitframe_allocate, - 'copy': jitframe_copy, }, rtti = True, ) diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -140,6 +140,9 @@ debug_start('jit-backend-counts') self.set_debug(have_debug_prints()) debug_stop('jit-backend-counts') + # when finishing, we only have one value at [0], the rest dies + self.gcmap_for_finish = lltype.malloc(jitframe.GCMAP, 1, zero=True) + self.gcmap_for_finish[0] = r_uint(1) def setup(self, looptoken): assert self.memcpy_addr != 0, "setup_once() not called?" @@ -2006,7 +2009,7 @@ ofs = self.cpu.get_ofs_of_frame_field('jf_descr') base_ofs = self.cpu.get_baseofs_of_frame_field() self.mov(fail_descr_loc, RawStackLoc(ofs)) - gcmap = self._regalloc.get_gcmap() + gcmap = self.gcmap_for_finish self.push_gcmap(self.mc, gcmap, store=True) self.mc.LEA_rb(eax.value, -base_ofs) # exit function diff --git a/rpython/jit/backend/x86/test/test_gc_integration.py b/rpython/jit/backend/x86/test/test_gc_integration.py --- a/rpython/jit/backend/x86/test/test_gc_integration.py +++ b/rpython/jit/backend/x86/test/test_gc_integration.py @@ -366,27 +366,31 @@ assert len(frame.jf_frame) == JITFRAME_FIXED_SIZE + 4 # we "collect" frames.append(frame) - new_frame = frame.copy() + new_frame = jitframe.JITFRAME.allocate(frame.jf_frame_info) + gcmap = unpack_gcmap(frame) + assert gcmap == [28, 29, 30] + for item, s in zip(gcmap, new_items): + new_frame.jf_frame[item] = rffi.cast(lltype.Signed, s) assert self.cpu.gc_ll_descr.gcrootmap.stack[0] == rffi.cast(lltype.Signed, frame) hdrbuilder.new_header(new_frame) - #gc_ll_descr.gcrootmap.stack[0] = rffi.cast(lltype.Signed, new_frame) + gc_ll_descr.gcrootmap.stack[0] = rffi.cast(lltype.Signed, new_frame) frames.append(new_frame) - assert unpack_gcmap(frame) == [28, 29, 30] def check2(i): assert self.cpu.gc_ll_descr.gcrootmap.stack[0] == i - ofs frame = rffi.cast(jitframe.JITFRAMEPTR, i - ofs) - #assert frame == frames[1] - #assert frame != frames[0] + assert frame == frames[1] + assert frame != frames[0] CHECK = lltype.FuncType([lltype.Signed], lltype.Void) checkptr = llhelper(lltype.Ptr(CHECK), check) check2ptr = llhelper(lltype.Ptr(CHECK), check2) checkdescr = self.cpu.calldescrof(CHECK, CHECK.ARGS, CHECK.RESULT, EffectInfo.MOST_GENERAL) - - S = lltype.GcStruct('S', - ('x', lltype.Ptr(lltype.GcArray(lltype.Signed)))) + + S = lltype.GcForwardReference() + S.become(lltype.GcStruct('S', + ('x', lltype.Ptr(S)))) loop = self.parse(""" [p0, p1, p2] i0 = force_token() # this is a bit below the frame @@ -412,9 +416,15 @@ p0 = lltype.malloc(S, zero=True) p1 = lltype.malloc(S) p2 = lltype.malloc(S) + new_items = [lltype.malloc(S), lltype.malloc(S), lltype.malloc(S)] + new_items[0].x = new_items[2] hdrbuilder.new_header(p0) hdrbuilder.new_header(p1) hdrbuilder.new_header(p2) frame = self.cpu.execute_token(token, p0, p1, p2) + frame = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, frame) gcmap = unpack_gcmap(lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, frame)) - assert gcmap == [11] + assert len(gcmap) == 1 + assert gcmap[0] < 29 + item = rffi.cast(lltype.Ptr(S), frame.jf_frame[gcmap[0]]) + assert item == new_items[2] From noreply at buildbot.pypy.org Thu Jan 24 10:17:20 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 24 Jan 2013 10:17:20 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: Zero the gcmap after reloading the frame Message-ID: <20130124091720.720E91C0246@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60414:15171e3e7dcd Date: 2013-01-24 11:16 +0200 http://bitbucket.org/pypy/pypy/changeset/15171e3e7dcd/ Log: Zero the gcmap after reloading the frame diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -234,6 +234,7 @@ assert not IS_X86_32 # the arg is already in edi if hasattr(self.cpu.gc_ll_descr, 'passes_frame'): + # for tests only base_ofs = self.cpu.get_baseofs_of_frame_field() mc.LEA_rb(esi.value, -base_ofs) mc.SUB_ri(esp.value, 16 - WORD) @@ -249,8 +250,8 @@ nursery_free_adr = self.cpu.gc_ll_descr.get_nursery_free_addr() mc.MOV(edi, heap(nursery_free_adr)) # load this in EDX # clear the gc pattern + self._reload_frame_if_necessary(mc) mc.MOV_bi(ofs, 0) - self._reload_frame_if_necessary(mc) mc.RET() # # If the slowpath malloc failed, we raise a MemoryError that @@ -1244,10 +1245,10 @@ remap_frame_layout(self, src_locs, dst_locs, X86_64_SCRATCH_REG) self.push_gcmap(self.mc, self._regalloc.get_gcmap(), store=True) self.mc.CALL(x) - self.pop_gcmap(self.mc) if align: self.mc.ADD_ri(esp.value, align * WORD) self._reload_frame_if_necessary(self.mc) + self.pop_gcmap(self.mc) def _reload_frame_if_necessary(self, mc): gcrootmap = self.cpu.gc_ll_descr.gcrootmap From noreply at buildbot.pypy.org Thu Jan 24 11:00:33 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 24 Jan 2013 11:00:33 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: a bit undertested write barrier for frames Message-ID: <20130124100033.02AFA1C05D7@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60415:2b69388c82c3 Date: 2013-01-24 12:00 +0200 http://bitbucket.org/pypy/pypy/changeset/2b69388c82c3/ Log: a bit undertested write barrier for frames diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -1258,6 +1258,13 @@ mc.MOV(ebp, mem(edx, -WORD)) base_ofs = self.cpu.get_baseofs_of_frame_field() mc.ADD_ri(ebp.value, base_ofs) + wbdescr = self.cpu.gc_ll_descr.write_barrier_descr + if gcrootmap and wbdescr: + ofs = self.cpu.get_baseofs_of_frame_field() + # frame never uses card marking, so we enforce this is not + # an array + self._write_barrier_fastpah(wbdescr, [ebp], array=False, + extra_ofs=-ofs) def call(self, addr, args, res): self._emit_call(imm(addr), args) @@ -2311,20 +2318,18 @@ self.mc.overwrite(jmp_location - 1, chr(offset)) self._emit_guard_not_forced(guard_token) - def genop_discard_cond_call_gc_wb(self, op, arglocs): + def _write_barrier_fastpah(self, descr, arglocs, array=False, extra_ofs=0): # Write code equivalent to write_barrier() in the GC: it checks # a flag in the object at arglocs[0], and if set, it calls a # helper piece of assembler. The latter saves registers as needed # and call the function jit_remember_young_pointer() from the GC. - descr = op.getdescr() if we_are_translated(): cls = self.cpu.gc_ll_descr.has_write_barrier_class() assert cls is not None and isinstance(descr, cls) # - opnum = op.getopnum() card_marking = False mask = descr.jit_wb_if_flag_singlebyte - if opnum == rop.COND_CALL_GC_WB_ARRAY and descr.jit_wb_cards_set != 0: + if array and descr.jit_wb_cards_set != 0: # assumptions the rest of the function depends on: assert (descr.jit_wb_cards_set_byteofs == descr.jit_wb_if_flag_byteofs) @@ -2333,7 +2338,8 @@ mask = descr.jit_wb_if_flag_singlebyte | -0x80 # loc_base = arglocs[0] - self.mc.TEST8(addr_add_const(loc_base, descr.jit_wb_if_flag_byteofs), + self.mc.TEST8(addr_add_const(loc_base, + descr.jit_wb_if_flag_byteofs + extra_ofs), imm(mask)) self.mc.J_il8(rx86.Conditions['Z'], 0) # patched later jz_location = self.mc.get_relative_pos() @@ -2418,7 +2424,11 @@ assert 0 < offset <= 127 self.mc.overwrite(jz_location-1, chr(offset)) - genop_discard_cond_call_gc_wb_array = genop_discard_cond_call_gc_wb + def genop_discard_cond_call_gc_wb(self, op, arglocs): + self._write_barrier_fastpah(op.getdescr(), arglocs) + + def genop_discard_cond_call_gc_wb_array(self, op, arglocs): + self._write_barrier_fastpah(op.getdescr(), arglocs, array=True) def not_implemented_op_discard(self, op, arglocs): not_implemented("not implemented operation: %s" % op.getopname()) diff --git a/rpython/jit/backend/x86/test/test_gc_integration.py b/rpython/jit/backend/x86/test/test_gc_integration.py --- a/rpython/jit/backend/x86/test/test_gc_integration.py +++ b/rpython/jit/backend/x86/test/test_gc_integration.py @@ -236,7 +236,7 @@ assert len(x) == 1 assert (bin(x[0]).count('1') == '0b1111100000000000000001111111011111'.count('1')) - # all but two + # all but two registers + some stuff on stack self.cpu = self.getcpu(check) S1 = lltype.GcStruct('S1') @@ -328,16 +328,22 @@ class GCClass: JIT_WB_IF_FLAG = 0 + def __init__(self, nursery_size=100): + GcCache.__init__(self, False, None) + self.gcrootmap = MockShadowStackRootMap() + self.nursery = lltype.malloc(rffi.CArray(lltype.Char), nursery_size, + flavor='raw') + self.nursery_addr = rffi.cast(lltype.Signed, self.nursery) + def get_malloc_slowpath_addr(self): return 0 def get_nursery_free_addr(self): return 0 + + def __del__(self): + lltype.free(self.nursery, flavor='raw') - def __init__(self): - GcCache.__init__(self, False, None) - self.gcrootmap = MockShadowStackRootMap() - def unpack_gcmap(frame): res = [] val = 0 @@ -351,17 +357,20 @@ return res class TestGcShadowstackDirect(BaseTestRegalloc): - - cpu = CPU(None, None) - cpu.gc_ll_descr = GCDescrShadowstackDirect() - cpu.setup_once() + + def setup_method(self, meth): + cpu = CPU(None, None) + cpu.gc_ll_descr = GCDescrShadowstackDirect() + cpu.setup_once() + self.cpu = cpu def test_shadowstack_call(self): - ofs = self.cpu.get_baseofs_of_frame_field() + cpu = self.cpu + ofs = cpu.get_baseofs_of_frame_field() frames = [] def check(i): - assert self.cpu.gc_ll_descr.gcrootmap.stack[0] == i - ofs + assert cpu.gc_ll_descr.gcrootmap.stack[0] == i - ofs frame = rffi.cast(jitframe.JITFRAMEPTR, i - ofs) assert len(frame.jf_frame) == JITFRAME_FIXED_SIZE + 4 # we "collect" @@ -371,13 +380,13 @@ assert gcmap == [28, 29, 30] for item, s in zip(gcmap, new_items): new_frame.jf_frame[item] = rffi.cast(lltype.Signed, s) - assert self.cpu.gc_ll_descr.gcrootmap.stack[0] == rffi.cast(lltype.Signed, frame) + assert cpu.gc_ll_descr.gcrootmap.stack[0] == rffi.cast(lltype.Signed, frame) hdrbuilder.new_header(new_frame) gc_ll_descr.gcrootmap.stack[0] = rffi.cast(lltype.Signed, new_frame) frames.append(new_frame) def check2(i): - assert self.cpu.gc_ll_descr.gcrootmap.stack[0] == i - ofs + assert cpu.gc_ll_descr.gcrootmap.stack[0] == i - ofs frame = rffi.cast(jitframe.JITFRAMEPTR, i - ofs) assert frame == frames[1] assert frame != frames[0] @@ -385,7 +394,7 @@ CHECK = lltype.FuncType([lltype.Signed], lltype.Void) checkptr = llhelper(lltype.Ptr(CHECK), check) check2ptr = llhelper(lltype.Ptr(CHECK), check2) - checkdescr = self.cpu.calldescrof(CHECK, CHECK.ARGS, CHECK.RESULT, + checkdescr = cpu.calldescrof(CHECK, CHECK.ARGS, CHECK.RESULT, EffectInfo.MOST_GENERAL) S = lltype.GcForwardReference() @@ -404,13 +413,13 @@ 'faildescr': BasicFailDescr(), 'check_adr': checkptr, 'check2_adr': check2ptr, 'checkdescr': checkdescr, - 'fielddescr': self.cpu.fielddescrof(S, 'x')}) + 'fielddescr': cpu.fielddescrof(S, 'x')}) token = JitCellToken() - self.cpu.compile_loop(loop.inputargs, loop.operations, token) - self.cpu.register_frame = lambda frame : hdrbuilder.new_header(frame) + cpu.compile_loop(loop.inputargs, loop.operations, token) + cpu.register_frame = lambda frame : hdrbuilder.new_header(frame) HDR = lltype.Struct('HDR', ('tid', lltype.Signed)) hdrbuilder = GCHeaderBuilder(HDR) - gc_ll_descr = self.cpu.gc_ll_descr + gc_ll_descr = cpu.gc_ll_descr gc_ll_descr.gcheaderbuilder = hdrbuilder gc_ll_descr.HDRPTR = lltype.Ptr(HDR) p0 = lltype.malloc(S, zero=True) @@ -421,10 +430,13 @@ hdrbuilder.new_header(p0) hdrbuilder.new_header(p1) hdrbuilder.new_header(p2) - frame = self.cpu.execute_token(token, p0, p1, p2) + frame = cpu.execute_token(token, p0, p1, p2) frame = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, frame) gcmap = unpack_gcmap(lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, frame)) assert len(gcmap) == 1 assert gcmap[0] < 29 item = rffi.cast(lltype.Ptr(S), frame.jf_frame[gcmap[0]]) assert item == new_items[2] + + def test_malloc_slowpath(self): + cpu = self.cpu From noreply at buildbot.pypy.org Thu Jan 24 11:06:34 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 24 Jan 2013 11:06:34 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: some real fixes and some rpython fixes Message-ID: <20130124100634.497EF1C05D7@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60416:873dbd9cc580 Date: 2013-01-24 12:06 +0200 http://bitbucket.org/pypy/pypy/changeset/873dbd9cc580/ Log: some real fixes and some rpython fixes diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -1263,7 +1263,7 @@ ofs = self.cpu.get_baseofs_of_frame_field() # frame never uses card marking, so we enforce this is not # an array - self._write_barrier_fastpah(wbdescr, [ebp], array=False, + self._write_barrier_fastpah(mc, wbdescr, [ebp], array=False, extra_ofs=-ofs) def call(self, addr, args, res): @@ -2318,7 +2318,8 @@ self.mc.overwrite(jmp_location - 1, chr(offset)) self._emit_guard_not_forced(guard_token) - def _write_barrier_fastpah(self, descr, arglocs, array=False, extra_ofs=0): + def _write_barrier_fastpah(self, mc, descr, arglocs, array=False, + extra_ofs=0): # Write code equivalent to write_barrier() in the GC: it checks # a flag in the object at arglocs[0], and if set, it calls a # helper piece of assembler. The latter saves registers as needed @@ -2338,19 +2339,19 @@ mask = descr.jit_wb_if_flag_singlebyte | -0x80 # loc_base = arglocs[0] - self.mc.TEST8(addr_add_const(loc_base, + mc.TEST8(addr_add_const(loc_base, descr.jit_wb_if_flag_byteofs + extra_ofs), imm(mask)) - self.mc.J_il8(rx86.Conditions['Z'], 0) # patched later - jz_location = self.mc.get_relative_pos() + mc.J_il8(rx86.Conditions['Z'], 0) # patched later + jz_location = mc.get_relative_pos() # for cond_call_gc_wb_array, also add another fast path: # if GCFLAG_CARDS_SET, then we can just set one bit and be done if card_marking: # GCFLAG_CARDS_SET is in this byte at 0x80, so this fact can # been checked by the status flags of the previous TEST8 - self.mc.J_il8(rx86.Conditions['S'], 0) # patched later - js_location = self.mc.get_relative_pos() + mc.J_il8(rx86.Conditions['S'], 0) # patched later + js_location = mc.get_relative_pos() else: js_location = 0 @@ -2358,7 +2359,7 @@ # argument the address of the structure we are writing into # (the first argument to COND_CALL_GC_WB). helper_num = card_marking - if self._regalloc.xrm.reg_bindings: + if self._regalloc is not None and self._regalloc.xrm.reg_bindings: helper_num += 2 if self.wb_slowpath[helper_num] == 0: # tests only assert not we_are_translated() @@ -2367,20 +2368,20 @@ bool(self._regalloc.xrm.reg_bindings)) assert self.wb_slowpath[helper_num] != 0 # - self.mc.PUSH(loc_base) - self.mc.CALL(imm(self.wb_slowpath[helper_num])) + mc.PUSH(loc_base) + mc.CALL(imm(self.wb_slowpath[helper_num])) if card_marking: # The helper ends again with a check of the flag in the object. # So here, we can simply write again a 'JNS', which will be # taken if GCFLAG_CARDS_SET is still not set. - self.mc.J_il8(rx86.Conditions['NS'], 0) # patched later - jns_location = self.mc.get_relative_pos() + mc.J_il8(rx86.Conditions['NS'], 0) # patched later + jns_location = mc.get_relative_pos() # # patch the JS above - offset = self.mc.get_relative_pos() - js_location + offset = mc.get_relative_pos() - js_location assert 0 < offset <= 127 - self.mc.overwrite(js_location-1, chr(offset)) + mc.overwrite(js_location-1, chr(offset)) # # case GCFLAG_CARDS_SET: emit a few instructions to do # directly the card flag setting @@ -2389,46 +2390,46 @@ if IS_X86_64 and isinstance(loc_base, RegLoc): # copy loc_index into r11 tmp1 = X86_64_SCRATCH_REG - self.mc.MOV_rr(tmp1.value, loc_index.value) + mc.MOV_rr(tmp1.value, loc_index.value) final_pop = False else: # must save the register loc_index before it is mutated - self.mc.PUSH_r(loc_index.value) + mc.PUSH_r(loc_index.value) tmp1 = loc_index final_pop = True # SHR tmp, card_page_shift - self.mc.SHR_ri(tmp1.value, descr.jit_wb_card_page_shift) + mc.SHR_ri(tmp1.value, descr.jit_wb_card_page_shift) # XOR tmp, -8 - self.mc.XOR_ri(tmp1.value, -8) + mc.XOR_ri(tmp1.value, -8) # BTS [loc_base], tmp - self.mc.BTS(addr_add_const(loc_base, 0), tmp1) + mc.BTS(addr_add_const(loc_base, 0), tmp1) # done if final_pop: - self.mc.POP_r(loc_index.value) + mc.POP_r(loc_index.value) # elif isinstance(loc_index, ImmedLoc): byte_index = loc_index.value >> descr.jit_wb_card_page_shift byte_ofs = ~(byte_index >> 3) byte_val = 1 << (byte_index & 7) - self.mc.OR8(addr_add_const(loc_base, byte_ofs), imm(byte_val)) + mc.OR8(addr_add_const(loc_base, byte_ofs), imm(byte_val)) else: raise AssertionError("index is neither RegLoc nor ImmedLoc") # # patch the JNS above - offset = self.mc.get_relative_pos() - jns_location + offset = mc.get_relative_pos() - jns_location assert 0 < offset <= 127 - self.mc.overwrite(jns_location-1, chr(offset)) + mc.overwrite(jns_location-1, chr(offset)) # patch the JZ above - offset = self.mc.get_relative_pos() - jz_location + offset = mc.get_relative_pos() - jz_location assert 0 < offset <= 127 - self.mc.overwrite(jz_location-1, chr(offset)) + mc.overwrite(jz_location-1, chr(offset)) def genop_discard_cond_call_gc_wb(self, op, arglocs): - self._write_barrier_fastpah(op.getdescr(), arglocs) + self._write_barrier_fastpah(self.mc, op.getdescr(), arglocs) def genop_discard_cond_call_gc_wb_array(self, op, arglocs): - self._write_barrier_fastpah(op.getdescr(), arglocs, array=True) + self._write_barrier_fastpah(self.mc, op.getdescr(), arglocs, array=True) def not_implemented_op_discard(self, op, arglocs): not_implemented("not implemented operation: %s" % op.getopname()) From noreply at buildbot.pypy.org Thu Jan 24 13:10:49 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 24 Jan 2013 13:10:49 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: enough work to avoid an assert Message-ID: <20130124121049.69EF71C05D7@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60417:d2c1694ddea7 Date: 2013-01-24 14:10 +0200 http://bitbucket.org/pypy/pypy/changeset/d2c1694ddea7/ Log: enough work to avoid an assert diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -2339,9 +2339,12 @@ mask = descr.jit_wb_if_flag_singlebyte | -0x80 # loc_base = arglocs[0] - mc.TEST8(addr_add_const(loc_base, - descr.jit_wb_if_flag_byteofs + extra_ofs), - imm(mask)) + if loc_base is ebp: + loc = raw_stack(descr.jit_wb_if_flag_byteofs + extra_ofs) + else: + loc = addr_add_const(loc_base, descr.jit_wb_if_flag_byteofs + + extra_ofs) + mc.TEST8(loc, imm(mask)) mc.J_il8(rx86.Conditions['Z'], 0) # patched later jz_location = mc.get_relative_pos() diff --git a/rpython/jit/backend/x86/rx86.py b/rpython/jit/backend/x86/rx86.py --- a/rpython/jit/backend/x86/rx86.py +++ b/rpython/jit/backend/x86/rx86.py @@ -582,6 +582,7 @@ CDQ = insn(rex_nw, '\x99') TEST8_mi = insn(rex_nw, '\xF6', orbyte(0<<3), mem_reg_plus_const(1), immediate(2, 'b')) + TEST8_bi = insn(rex_nw, '\xF6', orbyte(0<<3), stack_bp(1), immediate(2, 'b')) TEST8_ji = insn(rex_nw, '\xF6', orbyte(0<<3), abs_, immediate(1), immediate(2, 'b')) TEST_rr = insn(rex_w, '\x85', register(2,8), register(1), '\xC0') diff --git a/rpython/jit/backend/x86/test/test_gc_integration.py b/rpython/jit/backend/x86/test/test_gc_integration.py --- a/rpython/jit/backend/x86/test/test_gc_integration.py +++ b/rpython/jit/backend/x86/test/test_gc_integration.py @@ -3,14 +3,14 @@ """ from rpython.jit.metainterp.history import TargetToken, BasicFinalDescr,\ - JitCellToken, BasicFailDescr + JitCellToken, BasicFailDescr, AbstractDescr from rpython.jit.backend.llsupport.gc import GcLLDescription, GcLLDescr_boehm,\ GcLLDescr_framework, GcCache from rpython.jit.backend.detect_cpu import getcpuclass from rpython.jit.backend.x86.arch import WORD, JITFRAME_FIXED_SIZE from rpython.jit.backend.llsupport import jitframe from rpython.rtyper.lltypesystem import lltype, llmemory, rffi -from rpython.rtyper.annlowlevel import llhelper +from rpython.rtyper.annlowlevel import llhelper, llhelper_args from rpython.jit.backend.x86.test.test_regalloc import BaseTestRegalloc from rpython.jit.backend.x86.regalloc import gpr_reg_mgr_cls @@ -125,8 +125,8 @@ class GCDescrFastpathMalloc(GcLLDescription): gcrootmap = None + passes_frame = True write_barrier_descr = None - passes_frame = True def __init__(self, callback): GcLLDescription.__init__(self, None) @@ -321,19 +321,35 @@ def get_root_stack_top_addr(self): return rffi.cast(lltype.Signed, self.stack_addr) +class WriteBarrierDescr(AbstractDescr): + jit_wb_cards_set = 0 + jit_wb_if_flag_singlebyte = 1 + + def __init__(self, gc_ll_descr): + def write_barrier(x): + import pdb + pdb.set_trace() + + self.write_barrier_fn = llhelper_args(write_barrier, + [lltype.Signed], lltype.Void) + + def get_write_barrier_fn(self, cpu): + return self.write_barrier_fn + class GCDescrShadowstackDirect(GcLLDescr_framework): layoutbuilder = None - write_barrier_descr = None class GCClass: JIT_WB_IF_FLAG = 0 def __init__(self, nursery_size=100): GcCache.__init__(self, False, None) + self._generated_functions = [] self.gcrootmap = MockShadowStackRootMap() self.nursery = lltype.malloc(rffi.CArray(lltype.Char), nursery_size, flavor='raw') self.nursery_addr = rffi.cast(lltype.Signed, self.nursery) + self.write_barrier_descr = WriteBarrierDescr(self) def get_malloc_slowpath_addr(self): return 0 @@ -361,11 +377,19 @@ def setup_method(self, meth): cpu = CPU(None, None) cpu.gc_ll_descr = GCDescrShadowstackDirect() - cpu.setup_once() + wbd = cpu.gc_ll_descr.write_barrier_descr + wbd.jit_wb_if_flag_byteofs = 0 # directly into 'hdr' field + cpu.setup_once() + S = lltype.GcForwardReference() + S.become(lltype.GcStruct('S', + ('hdr', lltype.Signed), + ('x', lltype.Ptr(S)))) + self.S = S self.cpu = cpu def test_shadowstack_call(self): cpu = self.cpu + S = self.S ofs = cpu.get_baseofs_of_frame_field() frames = [] @@ -397,9 +421,6 @@ checkdescr = cpu.calldescrof(CHECK, CHECK.ARGS, CHECK.RESULT, EffectInfo.MOST_GENERAL) - S = lltype.GcForwardReference() - S.become(lltype.GcStruct('S', - ('x', lltype.Ptr(S)))) loop = self.parse(""" [p0, p1, p2] i0 = force_token() # this is a bit below the frame @@ -437,6 +458,3 @@ assert gcmap[0] < 29 item = rffi.cast(lltype.Ptr(S), frame.jf_frame[gcmap[0]]) assert item == new_items[2] - - def test_malloc_slowpath(self): - cpu = self.cpu diff --git a/rpython/rtyper/annlowlevel.py b/rpython/rtyper/annlowlevel.py --- a/rpython/rtyper/annlowlevel.py +++ b/rpython/rtyper/annlowlevel.py @@ -365,6 +365,8 @@ else: return lltype.functionptr(F.TO, f.func_name, _callable=f) +def llhelper_args(f, ARGS, RESULT): + return llhelper(lltype.Ptr(lltype.FuncType(ARGS, RESULT)), f) class LLHelperEntry(extregistry.ExtRegistryEntry): _about_ = llhelper From noreply at buildbot.pypy.org Thu Jan 24 14:01:42 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 24 Jan 2013 14:01:42 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: remove some hacks Message-ID: <20130124130142.B73DA1C105A@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60418:fbd8924eff85 Date: 2013-01-24 14:37 +0200 http://bitbucket.org/pypy/pypy/changeset/fbd8924eff85/ Log: remove some hacks diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py --- a/rpython/jit/backend/x86/runner.py +++ b/rpython/jit/backend/x86/runner.py @@ -124,8 +124,6 @@ if not self.translate_support_code: prev_interpreter = LLInterpreter.current_interpreter LLInterpreter.current_interpreter = self.debug_ll_interpreter - if hasattr(self, 'register_frame'): - self.register_frame(frame) try: num = JITFRAME_FIXED_SIZE * WORD for i, kind in kinds: diff --git a/rpython/jit/backend/x86/test/test_gc_integration.py b/rpython/jit/backend/x86/test/test_gc_integration.py --- a/rpython/jit/backend/x86/test/test_gc_integration.py +++ b/rpython/jit/backend/x86/test/test_gc_integration.py @@ -351,6 +351,9 @@ self.nursery_addr = rffi.cast(lltype.Signed, self.nursery) self.write_barrier_descr = WriteBarrierDescr(self) + def do_write_barrier(self, gcref_struct, gcref_newptr): + pass + def get_malloc_slowpath_addr(self): return 0 @@ -405,8 +408,7 @@ for item, s in zip(gcmap, new_items): new_frame.jf_frame[item] = rffi.cast(lltype.Signed, s) assert cpu.gc_ll_descr.gcrootmap.stack[0] == rffi.cast(lltype.Signed, frame) - hdrbuilder.new_header(new_frame) - gc_ll_descr.gcrootmap.stack[0] = rffi.cast(lltype.Signed, new_frame) + cpu.gc_ll_descr.gcrootmap.stack[0] = rffi.cast(lltype.Signed, new_frame) frames.append(new_frame) def check2(i): @@ -437,20 +439,11 @@ 'fielddescr': cpu.fielddescrof(S, 'x')}) token = JitCellToken() cpu.compile_loop(loop.inputargs, loop.operations, token) - cpu.register_frame = lambda frame : hdrbuilder.new_header(frame) - HDR = lltype.Struct('HDR', ('tid', lltype.Signed)) - hdrbuilder = GCHeaderBuilder(HDR) - gc_ll_descr = cpu.gc_ll_descr - gc_ll_descr.gcheaderbuilder = hdrbuilder - gc_ll_descr.HDRPTR = lltype.Ptr(HDR) p0 = lltype.malloc(S, zero=True) p1 = lltype.malloc(S) p2 = lltype.malloc(S) new_items = [lltype.malloc(S), lltype.malloc(S), lltype.malloc(S)] new_items[0].x = new_items[2] - hdrbuilder.new_header(p0) - hdrbuilder.new_header(p1) - hdrbuilder.new_header(p2) frame = cpu.execute_token(token, p0, p1, p2) frame = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, frame) gcmap = unpack_gcmap(lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, frame)) @@ -458,3 +451,9 @@ assert gcmap[0] < 29 item = rffi.cast(lltype.Ptr(S), frame.jf_frame[gcmap[0]]) assert item == new_items[2] + + def test_malloc_frame_writebarrier(self): + loop = self.parse(""" + [] + """, namespace={}) + loop From noreply at buildbot.pypy.org Thu Jan 24 14:01:44 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 24 Jan 2013 14:01:44 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: frame can definitely contain young pointers Message-ID: <20130124130144.0A5F51C105A@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60419:4d6db99cec1d Date: 2013-01-24 15:01 +0200 http://bitbucket.org/pypy/pypy/changeset/4d6db99cec1d/ Log: frame can definitely contain young pointers diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py --- a/rpython/jit/backend/llsupport/gc.py +++ b/rpython/jit/backend/llsupport/gc.py @@ -133,6 +133,13 @@ """ raise NotImplementedError # cpu specific + def malloc_jitframe(self, frame_info): + """ Allocate a new frame, overwritten by tests + """ + frame = jitframe.JITFRAME.allocate(frame_info) + llop.gc_assume_young_pointers(lltype.Void, frame) + return frame + class JitFrameDescrs: def _freeze_(self): return True diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py --- a/rpython/jit/backend/x86/runner.py +++ b/rpython/jit/backend/x86/runner.py @@ -118,7 +118,7 @@ func = rffi.cast(FUNCPTR, addr) #llop.debug_print(lltype.Void, ">>>> Entering", addr) frame_info = clt.frame_info - frame = jitframe.JITFRAME.allocate(frame_info) + frame = self.gc_ll_descr.malloc_jitframe(frame_info) ll_frame = lltype.cast_opaque_ptr(llmemory.GCREF, frame) prev_interpreter = None # help flow space if not self.translate_support_code: diff --git a/rpython/jit/backend/x86/test/test_gc_integration.py b/rpython/jit/backend/x86/test/test_gc_integration.py --- a/rpython/jit/backend/x86/test/test_gc_integration.py +++ b/rpython/jit/backend/x86/test/test_gc_integration.py @@ -348,8 +348,13 @@ self.gcrootmap = MockShadowStackRootMap() self.nursery = lltype.malloc(rffi.CArray(lltype.Char), nursery_size, flavor='raw') - self.nursery_addr = rffi.cast(lltype.Signed, self.nursery) + self.nursery_ptrs = lltype.malloc(rffi.CArray(lltype.Signed), 2, + flavor='raw') + self.nursery_ptrs[0] = rffi.cast(lltype.Signed, self.nursery) + self.nursery_ptrs[1] = self.nursery_ptrs[0] + nursery_size + self.nursery_addr = rffi.cast(lltype.Signed, self.nursery_ptrs) self.write_barrier_descr = WriteBarrierDescr(self) + self._initialize_for_tests() def do_write_barrier(self, gcref_struct, gcref_newptr): pass @@ -358,10 +363,14 @@ return 0 def get_nursery_free_addr(self): - return 0 + return self.nursery_addr + + def get_nursery_top_addr(self): + return self.nursery_addr + rffi.sizeof(lltype.Signed) def __del__(self): lltype.free(self.nursery, flavor='raw') + lltype.free(self.nursery_ptrs, flavor='raw') def unpack_gcmap(frame): res = [] @@ -387,6 +396,7 @@ S.become(lltype.GcStruct('S', ('hdr', lltype.Signed), ('x', lltype.Ptr(S)))) + cpu.gc_ll_descr.fielddescr_tid = cpu.fielddescrof(S, 'hdr') self.S = S self.cpu = cpu @@ -452,8 +462,21 @@ item = rffi.cast(lltype.Ptr(S), frame.jf_frame[gcmap[0]]) assert item == new_items[2] - def test_malloc_frame_writebarrier(self): + def test_malloc_1(self): + cpu = self.cpu + sizeof = cpu.sizeof(self.S) + sizeof.tid = 0 loop = self.parse(""" [] - """, namespace={}) - loop + p0 = new(descr=sizedescr) + finish(p0, descr=finaldescr) + """, namespace={'sizedescr': sizeof, + 'finaldescr': BasicFinalDescr()}) + token = JitCellToken() + cpu.compile_loop(loop.inputargs, loop.operations, token) + frame = cpu.execute_token(token) + # now we should be able to track everything from the frame + frame = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, frame) + thing = frame.jf_frame[unpack_gcmap(frame)[0]] + assert thing == rffi.cast(lltype.Signed, cpu.gc_ll_descr.nursery) + assert cpu.gc_ll_descr.nursery_ptrs[0] == thing + sizeof.size From noreply at buildbot.pypy.org Thu Jan 24 15:10:00 2013 From: noreply at buildbot.pypy.org (cfbolz) Date: Thu, 24 Jan 2013 15:10:00 +0100 (CET) Subject: [pypy-commit] pypy default: fix a regression: space.newdict(instance=True) was returning a StringDict. Message-ID: <20130124141000.E464B1C00A8@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: Changeset: r60420:5e86c989acc1 Date: 2013-01-24 14:57 +0100 http://bitbucket.org/pypy/pypy/changeset/5e86c989acc1/ Log: fix a regression: space.newdict(instance=True) was returning a StringDict. Write the 10 slightly obscure lines that fix that. diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py --- a/pypy/objspace/std/dictmultiobject.py +++ b/pypy/objspace/std/dictmultiobject.py @@ -54,6 +54,9 @@ # every module needs its own strategy, because the strategy stores # the version tag strategy = ModuleDictStrategy(space) + elif space.config.objspace.std.withmapdict and instance: + from pypy.objspace.std.mapdict import MapDictStrategy + strategy = space.fromcache(MapDictStrategy) elif instance or strdict or module: assert w_type is None diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py --- a/pypy/objspace/std/mapdict.py +++ b/pypy/objspace/std/mapdict.py @@ -593,6 +593,9 @@ # ____________________________________________________________ # dict implementation +def get_terminator_for_dicts(space): + return DictTerminator(space, None) + class MapDictStrategy(DictStrategy): erase, unerase = rerased.new_erasing_pair("map") @@ -602,13 +605,19 @@ def __init__(self, space): self.space = space + def get_empty_storage(self): + w_result = Object() + terminator = self.space.fromcache(get_terminator_for_dicts) + w_result._init_empty(terminator) + return self.erase(w_result) + def switch_to_object_strategy(self, w_dict): w_obj = self.unerase(w_dict.dstorage) strategy = self.space.fromcache(ObjectDictStrategy) dict_w = strategy.unerase(strategy.get_empty_storage()) w_dict.strategy = strategy w_dict.dstorage = strategy.erase(dict_w) - assert w_obj.getdict(self.space) is w_dict + assert w_obj.getdict(self.space) is w_dict or w_obj._get_mapdict_map().terminator.w_cls is None materialize_r_dict(self.space, w_obj, dict_w) def getitem(self, w_dict, w_key): diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py --- a/pypy/objspace/std/test/test_dictmultiobject.py +++ b/pypy/objspace/std/test/test_dictmultiobject.py @@ -954,6 +954,7 @@ withcelldict = False withmethodcache = False withidentitydict = False + withmapdict = False FakeSpace.config = Config() diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py --- a/pypy/objspace/std/test/test_mapdict.py +++ b/pypy/objspace/std/test/test_mapdict.py @@ -1,7 +1,17 @@ from pypy.objspace.std.test.test_dictmultiobject import FakeSpace, W_DictMultiObject from pypy.objspace.std.mapdict import * +class Config: + class objspace: + class std: + withsmalldicts = False + withcelldict = False + withmethodcache = False + withidentitydict = False + withmapdict = True + space = FakeSpace() +space.config = Config class Class(object): def __init__(self, hasdict=True): @@ -1061,3 +1071,11 @@ return A() """) assert w_dict.user_overridden_class + +def test_newdict_instance(): + w_dict = space.newdict(instance=True) + assert type(w_dict.strategy) is MapDictStrategy + +class TestMapDictImplementationUsingnewdict(BaseTestRDictImplementation): + StrategyClass = MapDictStrategy + # NB: the get_impl method is not overwritten here, as opposed to above From noreply at buildbot.pypy.org Thu Jan 24 15:10:02 2013 From: noreply at buildbot.pypy.org (cfbolz) Date: Thu, 24 Jan 2013 15:10:02 +0100 (CET) Subject: [pypy-commit] pypy default: a jit test for the fact that the TLS storage uses an instance dict Message-ID: <20130124141002.22EE91C00A8@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: Changeset: r60421:42affcc9488e Date: 2013-01-24 15:09 +0100 http://bitbucket.org/pypy/pypy/changeset/42affcc9488e/ Log: a jit test for the fact that the TLS storage uses an instance dict diff --git a/pypy/module/pypyjit/test_pypy_c/test_thread.py b/pypy/module/pypyjit/test_pypy_c/test_thread.py --- a/pypy/module/pypyjit/test_pypy_c/test_thread.py +++ b/pypy/module/pypyjit/test_pypy_c/test_thread.py @@ -26,3 +26,41 @@ --THREAD-TICK-- jump(..., descr=...) """) + + def test_tls(self): + def main(n): + import thread + local = thread._local() + local.x = 1 + i = 0 + while i < n: + i += local.x + return 0 + log = self.run(main, [500]) + assert round(log.result, 6) == round(main(500), 6) + loop, = log.loops_by_filename(self.filepath) + assert loop.match(""" + i60 = int_lt(i55, i27) + guard_true(i60, descr=...) + i61 = call(ConstClass(ll_dict_lookup_trampoline__v1134___simple_call__function_), p33, p32, i35, descr=...) + guard_no_exception(descr=...) + i62 = int_and(i61, -2147483648) + i63 = int_is_true(i62) + guard_false(i63, descr=...) + p64 = getinteriorfield_gc(p41, i61, descr=...) + guard_nonnull_class(p64, ConstClass(W_DictMultiObject), descr=...) + p65 = getfield_gc(p64, descr=...) + guard_class(p65, 176132160, descr=...) + p66 = getfield_gc(p64, descr=...) + guard_class(p66, 175975744, descr=...) + p67 = getfield_gc(p66, descr=...) + guard_value(p67, ConstPtr(ptr49), descr=...) + p68 = getfield_gc(p66, descr=...) + p69 = getarrayitem_gc(p68, 0, descr=...) + guard_nonnull_class(p69, ConstClass(W_IntObject), descr=...) + i70 = getfield_gc_pure(p69, descr=...) + i71 = int_add_ovf(i55, i70) + guard_no_overflow(descr=...) + --TICK-- + jump(p0, p1, p3, p5, p10, p12, p14, i71, i27, p33, p32, i35, p41, descr=...) + """) From noreply at buildbot.pypy.org Thu Jan 24 16:23:48 2013 From: noreply at buildbot.pypy.org (krono) Date: Thu, 24 Jan 2013 16:23:48 +0100 (CET) Subject: [pypy-commit] lang-smalltalk default: Cope for rpython split Message-ID: <20130124152348.E31491C00A8@cobra.cs.uni-duesseldorf.de> Author: Tobias Pape Branch: Changeset: r20:ee61936862c8 Date: 2013-01-24 14:26 +0100 http://bitbucket.org/pypy/lang-smalltalk/changeset/ee61936862c8/ Log: Cope for rpython split diff --git a/spyvm/fixedstack.py b/spyvm/fixedstack.py --- a/spyvm/fixedstack.py +++ b/spyvm/fixedstack.py @@ -4,7 +4,7 @@ import types -from pypy.rlib.rarithmetic import r_uint +from rpython.rlib.rarithmetic import r_uint class FixedStack(object): _annspecialcase_ = "specialize:ctr_location" # polymorphic diff --git a/spyvm/interpreter.py b/spyvm/interpreter.py --- a/spyvm/interpreter.py +++ b/spyvm/interpreter.py @@ -5,7 +5,7 @@ from spyvm import conftest from spyvm import wrapper -from pypy.rlib import objectmodel, unroll +from rpython.rlib import objectmodel, unroll class MissingBytecode(Exception): """Bytecode not implemented yet.""" diff --git a/spyvm/model.py b/spyvm/model.py --- a/spyvm/model.py +++ b/spyvm/model.py @@ -18,10 +18,10 @@ from spyvm.tool.bitmanipulation import splitter from spyvm import constants, error -from pypy.rlib import rrandom, objectmodel -from pypy.rlib.rarithmetic import intmask, r_uint -from pypy.tool.pairtype import extendabletype -from pypy.rlib.objectmodel import instantiate +from rpython.rlib import rrandom, objectmodel +from rpython.rlib.rarithmetic import intmask, r_uint +from rpython.tool.pairtype import extendabletype +from rpython.rlib.objectmodel import instantiate class W_Object(object): """Root of Squeak model, abstract.""" diff --git a/spyvm/objspace.py b/spyvm/objspace.py --- a/spyvm/objspace.py +++ b/spyvm/objspace.py @@ -2,8 +2,8 @@ from spyvm import model from spyvm import shadow from spyvm.error import UnwrappingError, WrappingError -from pypy.rlib.objectmodel import instantiate -from pypy.rlib.rarithmetic import intmask, r_uint +from rpython.rlib.objectmodel import instantiate +from rpython.rlib.rarithmetic import intmask, r_uint class ObjSpace(object): def __init__(self): diff --git a/spyvm/primitives.py b/spyvm/primitives.py --- a/spyvm/primitives.py +++ b/spyvm/primitives.py @@ -7,7 +7,7 @@ PrimitiveNotYetWrittenError from spyvm import wrapper -from pypy.rlib import rarithmetic, rfloat, unroll +from rpython.rlib import rarithmetic, rfloat, unroll def assert_bounds(n0, minimum, maximum): if not minimum <= n0 < maximum: @@ -45,7 +45,7 @@ def expose_primitive(code, unwrap_spec=None, no_result=False): # some serious magic, don't look - from pypy.rlib.unroll import unrolling_iterable + from rpython.rlib.unroll import unrolling_iterable # heuristics to give it a nice name name = None for key, value in globals().iteritems(): @@ -620,7 +620,7 @@ @expose_primitive(INC_GC, unwrap_spec=[object]) @expose_primitive(FULL_GC, unwrap_spec=[object]) def func(interp, w_arg): # Squeak pops the arg and ignores it ... go figure - from pypy.rlib import rgc + from rpython.rlib import rgc rgc.collect() return fake_bytes_left(interp) diff --git a/spyvm/shadow.py b/spyvm/shadow.py --- a/spyvm/shadow.py +++ b/spyvm/shadow.py @@ -1,7 +1,7 @@ import weakref from spyvm import model, constants, error -from pypy.tool.pairtype import extendabletype -from pypy.rlib import rarithmetic +from rpython.tool.pairtype import extendabletype +from rpython.rlib import rarithmetic class AbstractShadow(object): """A shadow is an optional extra bit of information that diff --git a/spyvm/squeakimage.py b/spyvm/squeakimage.py --- a/spyvm/squeakimage.py +++ b/spyvm/squeakimage.py @@ -4,7 +4,7 @@ from spyvm import model from spyvm.tool.bitmanipulation import splitter -from pypy.rlib import objectmodel +from rpython.rlib import objectmodel def chrs2int(b): assert len(b) == 4 @@ -360,7 +360,7 @@ w_pointersobject.hash = self.chunk.hash12 def fillin_wordsobject(self, w_wordsobject): - from pypy.rlib.rarithmetic import r_uint + from rpython.rlib.rarithmetic import r_uint w_wordsobject.words = [r_uint(x) for x in self.chunk.data] w_class = self.g_class.w_object assert isinstance(w_class, model.W_PointersObject) diff --git a/spyvm/test/test_objectspace.py b/spyvm/test/test_objectspace.py --- a/spyvm/test/test_objectspace.py +++ b/spyvm/test/test_objectspace.py @@ -27,7 +27,7 @@ assert w_Metaclass.w_class.w_class is w_Metaclass def test_ruint(): - from pypy.rlib.rarithmetic import r_uint + from rpython.rlib.rarithmetic import r_uint import sys for num in [0, 1, 41, 100, 2**31, sys.maxint + 1]: num = r_uint(num) diff --git a/spyvm/test/test_primitives.py b/spyvm/test/test_primitives.py --- a/spyvm/test/test_primitives.py +++ b/spyvm/test/test_primitives.py @@ -6,7 +6,7 @@ from spyvm import primitives from spyvm import objspace -from pypy.rlib.rfloat import INFINITY, NAN, isinf, isnan +from rpython.rlib.rfloat import INFINITY, NAN, isinf, isnan mockclass = objspace.bootstrap_class diff --git a/spyvm/tool/bitmanipulation.py b/spyvm/tool/bitmanipulation.py --- a/spyvm/tool/bitmanipulation.py +++ b/spyvm/tool/bitmanipulation.py @@ -1,4 +1,4 @@ -from pypy.rlib import unroll +from rpython.rlib import unroll class BitSplitter(dict): def __getitem__(self, lengths): From noreply at buildbot.pypy.org Thu Jan 24 16:41:28 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 24 Jan 2013 16:41:28 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: a fix and a failing test Message-ID: <20130124154128.33A341C00A8@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60422:019db8f1e708 Date: 2013-01-24 17:41 +0200 http://bitbucket.org/pypy/pypy/changeset/019db8f1e708/ Log: a fix and a failing test diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -151,6 +151,7 @@ self.newops.append(op0) # XXX for now it generates call_malloc_gc, instead of # call_malloc_nursery, because the array is strange + op1 = ResOperation(rop.NEW_ARRAY, [lgt_box], frame, descr=descrs.arraydescr) self.handle_new_array(descrs.arraydescr, op1) diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -866,6 +866,10 @@ self.mc.RET() def _call_header_shadowstack(self, gcrootmap): + # we don't *really* have to do it, since we have the frame + # being referenced by the caller. However, we still do it + # to provide a place where we can read the frame from, in case + # we need to reload it after a collection rst = gcrootmap.get_root_stack_top_addr() if rx86.fits_in_32bits(rst): self.mc.MOV_rj(eax.value, rst) # MOV eax, [rootstacktop] diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -891,6 +891,8 @@ size_box = op.getarg(0) assert isinstance(size_box, ConstInt) size = size_box.getint() + gcmap = self.get_gcmap() # allocate the gcmap *before* + # looking at the result self.rm.force_allocate_reg(op.result, selected_reg=eax) # # We need edx as a temporary, but otherwise don't save any more @@ -899,7 +901,6 @@ self.rm.force_allocate_reg(tmp_box, selected_reg=edi) self.rm.possibly_free_var(tmp_box) # - gcmap = self.get_gcmap() gc_ll_descr = self.assembler.cpu.gc_ll_descr self.assembler.malloc_cond( gc_ll_descr.get_nursery_free_addr(), diff --git a/rpython/jit/backend/x86/test/test_gc_integration.py b/rpython/jit/backend/x86/test/test_gc_integration.py --- a/rpython/jit/backend/x86/test/test_gc_integration.py +++ b/rpython/jit/backend/x86/test/test_gc_integration.py @@ -342,25 +342,54 @@ class GCClass: JIT_WB_IF_FLAG = 0 - def __init__(self, nursery_size=100): + def __init__(self): GcCache.__init__(self, False, None) self._generated_functions = [] self.gcrootmap = MockShadowStackRootMap() + self.write_barrier_descr = WriteBarrierDescr(self) + self._initialize_for_tests() + self.frames = [] + + def malloc_slowpath(size): + self._collect() + + self.malloc_slowpath_fnptr = llhelper_args(malloc_slowpath, + [lltype.Signed], + llmemory.GCREF) + self.all_nurseries = [] + + def init_nursery(self, nursery_size): self.nursery = lltype.malloc(rffi.CArray(lltype.Char), nursery_size, - flavor='raw') + flavor='raw', zero=True) self.nursery_ptrs = lltype.malloc(rffi.CArray(lltype.Signed), 2, flavor='raw') self.nursery_ptrs[0] = rffi.cast(lltype.Signed, self.nursery) self.nursery_ptrs[1] = self.nursery_ptrs[0] + nursery_size self.nursery_addr = rffi.cast(lltype.Signed, self.nursery_ptrs) - self.write_barrier_descr = WriteBarrierDescr(self) - self._initialize_for_tests() + self.all_nurseries.append(self.nursery) + self.collections.reverse() + + def _collect(self): + gcmap = unpack_gcmap(self.frames[-1]) + col = self.collections.pop() + frame = self.frames[-1].jf_frame + start = self.nursery_ptrs[0] + assert len(gcmap) == len(col) + for i in range(len(gcmap)): + assert col[i] + start == frame[gcmap[i]] + + def malloc_jitframe(self, frame_info): + """ Allocate a new frame, overwritten by tests + """ + frame = jitframe.JITFRAME.allocate(frame_info) + self.frames.append(frame) + return frame def do_write_barrier(self, gcref_struct, gcref_newptr): pass def get_malloc_slowpath_addr(self): - return 0 + return self.malloc_slowpath_fnptr def get_nursery_free_addr(self): return self.nursery_addr @@ -369,7 +398,8 @@ return self.nursery_addr + rffi.sizeof(lltype.Signed) def __del__(self): - lltype.free(self.nursery, flavor='raw') + for nursery in self.all_nurseries: + lltype.free(nursery, flavor='raw') lltype.free(self.nursery_ptrs, flavor='raw') def unpack_gcmap(frame): @@ -391,7 +421,6 @@ cpu.gc_ll_descr = GCDescrShadowstackDirect() wbd = cpu.gc_ll_descr.write_barrier_descr wbd.jit_wb_if_flag_byteofs = 0 # directly into 'hdr' field - cpu.setup_once() S = lltype.GcForwardReference() S.become(lltype.GcStruct('S', ('hdr', lltype.Signed), @@ -402,6 +431,8 @@ def test_shadowstack_call(self): cpu = self.cpu + cpu.gc_ll_descr.init_nursery(100) + cpu.setup_once() S = self.S ofs = cpu.get_baseofs_of_frame_field() frames = [] @@ -466,13 +497,21 @@ cpu = self.cpu sizeof = cpu.sizeof(self.S) sizeof.tid = 0 + size = sizeof.size loop = self.parse(""" [] - p0 = new(descr=sizedescr) + p0 = call_malloc_nursery(%d) + p1 = call_malloc_nursery(%d) + p2 = call_malloc_nursery(%d) # this overflows + guard_nonnull(p2, descr=faildescr) [p0, p1, p2] finish(p0, descr=finaldescr) - """, namespace={'sizedescr': sizeof, - 'finaldescr': BasicFinalDescr()}) + """ % (size, size, size), namespace={'sizedescr': sizeof, + 'finaldescr': BasicFinalDescr(), + 'faildescr': BasicFailDescr()}) token = JitCellToken() + cpu.gc_ll_descr.collections = [[0, sizeof.size]] + cpu.gc_ll_descr.init_nursery(2 * sizeof.size) + cpu.setup_once() cpu.compile_loop(loop.inputargs, loop.operations, token) frame = cpu.execute_token(token) # now we should be able to track everything from the frame From noreply at buildbot.pypy.org Thu Jan 24 17:08:08 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 24 Jan 2013 17:08:08 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: some fixes Message-ID: <20130124160808.43CC61C0246@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60423:3f77cd26741e Date: 2013-01-24 18:07 +0200 http://bitbucket.org/pypy/pypy/changeset/3f77cd26741e/ Log: some fixes diff --git a/pypy/TODO b/pypy/TODO --- a/pypy/TODO +++ b/pypy/TODO @@ -5,4 +5,6 @@ got potentially moved, which is after each potentially collecting call or slowpath malloc) * kill jit2gc on translator -* fix test_singlefloats in test_calling_conventions \ No newline at end of file +* fix test_singlefloats in test_calling_conventions +* slowpaths can have more variants depending on how many registers we're using + (like floats vs non-floats for failures) \ No newline at end of file diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -891,7 +891,6 @@ size_box = op.getarg(0) assert isinstance(size_box, ConstInt) size = size_box.getint() - gcmap = self.get_gcmap() # allocate the gcmap *before* # looking at the result self.rm.force_allocate_reg(op.result, selected_reg=eax) # @@ -899,6 +898,7 @@ # register. See comments in _build_malloc_slowpath(). tmp_box = TempBox() self.rm.force_allocate_reg(tmp_box, selected_reg=edi) + gcmap = self.get_gcmap([eax, edi]) # allocate the gcmap *before* self.rm.possibly_free_var(tmp_box) # gc_ll_descr = self.assembler.cpu.gc_ll_descr @@ -907,12 +907,14 @@ gc_ll_descr.get_nursery_top_addr(), size, gcmap) - def get_gcmap(self): + def get_gcmap(self, forbidden_regs): frame_depth = self.fm.get_frame_depth() size = frame_depth + JITFRAME_FIXED_SIZE gcmap = lltype.malloc(GCMAP, size // WORD // 8 + 1, zero=True) for box, loc in self.rm.reg_bindings.iteritems(): + if loc in forbidden_regs: + continue if box.type == REF: assert isinstance(loc, RegLoc) val = gpr_reg_mgr_cls.all_reg_indexes[loc.value] diff --git a/rpython/jit/backend/x86/test/test_gc_integration.py b/rpython/jit/backend/x86/test/test_gc_integration.py --- a/rpython/jit/backend/x86/test/test_gc_integration.py +++ b/rpython/jit/backend/x86/test/test_gc_integration.py @@ -347,22 +347,30 @@ self._generated_functions = [] self.gcrootmap = MockShadowStackRootMap() self.write_barrier_descr = WriteBarrierDescr(self) + self.nursery_ptrs = lltype.malloc(rffi.CArray(lltype.Signed), 2, + flavor='raw') self._initialize_for_tests() self.frames = [] def malloc_slowpath(size): self._collect() + res = self.nursery_ptrs[0] + self.nursery_ptrs[0] += size + return res self.malloc_slowpath_fnptr = llhelper_args(malloc_slowpath, [lltype.Signed], - llmemory.GCREF) + lltype.Signed) self.all_nurseries = [] - def init_nursery(self, nursery_size): + def init_nursery(self, nursery_size=None): + if nursery_size is None: + nursery_size = self.nursery_size + else: + self.nursery_size = nursery_size self.nursery = lltype.malloc(rffi.CArray(lltype.Char), nursery_size, - flavor='raw', zero=True) - self.nursery_ptrs = lltype.malloc(rffi.CArray(lltype.Signed), 2, - flavor='raw') + flavor='raw', zero=True, + track_allocation=False) self.nursery_ptrs[0] = rffi.cast(lltype.Signed, self.nursery) self.nursery_ptrs[1] = self.nursery_ptrs[0] + nursery_size self.nursery_addr = rffi.cast(lltype.Signed, self.nursery_ptrs) @@ -373,10 +381,13 @@ gcmap = unpack_gcmap(self.frames[-1]) col = self.collections.pop() frame = self.frames[-1].jf_frame - start = self.nursery_ptrs[0] + start = rffi.cast(lltype.Signed, self.nursery) assert len(gcmap) == len(col) + pos = [frame[item] for item in gcmap] + pos.sort() for i in range(len(gcmap)): - assert col[i] + start == frame[gcmap[i]] + assert col[i] + start == pos[i] + self.init_nursery() def malloc_jitframe(self, frame_info): """ Allocate a new frame, overwritten by tests @@ -399,7 +410,7 @@ def __del__(self): for nursery in self.all_nurseries: - lltype.free(nursery, flavor='raw') + lltype.free(nursery, flavor='raw', track_allocation=False) lltype.free(self.nursery_ptrs, flavor='raw') def unpack_gcmap(frame): @@ -516,6 +527,6 @@ frame = cpu.execute_token(token) # now we should be able to track everything from the frame frame = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, frame) - thing = frame.jf_frame[unpack_gcmap(frame)[0]] - assert thing == rffi.cast(lltype.Signed, cpu.gc_ll_descr.nursery) - assert cpu.gc_ll_descr.nursery_ptrs[0] == thing + sizeof.size + #thing = frame.jf_frame[unpack_gcmap(frame)[0]] + #assert thing == rffi.cast(lltype.Signed, cpu.gc_ll_descr.nursery) + #assert cpu.gc_ll_descr.nursery_ptrs[0] == thing + sizeof.size From noreply at buildbot.pypy.org Thu Jan 24 17:14:30 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 24 Jan 2013 17:14:30 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fixes Message-ID: <20130124161430.E9B6D1C0246@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60424:fdccfded154e Date: 2013-01-24 18:14 +0200 http://bitbucket.org/pypy/pypy/changeset/fdccfded154e/ Log: fixes diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -1247,7 +1247,7 @@ dst_locs.append(r10) x = r10 remap_frame_layout(self, src_locs, dst_locs, X86_64_SCRATCH_REG) - self.push_gcmap(self.mc, self._regalloc.get_gcmap(), store=True) + self.push_gcmap(self.mc, self._regalloc.get_gcmap([eax]), store=True) self.mc.CALL(x) if align: self.mc.ADD_ri(esp.value, align * WORD) diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -907,7 +907,7 @@ gc_ll_descr.get_nursery_top_addr(), size, gcmap) - def get_gcmap(self, forbidden_regs): + def get_gcmap(self, forbidden_regs=[]): frame_depth = self.fm.get_frame_depth() size = frame_depth + JITFRAME_FIXED_SIZE gcmap = lltype.malloc(GCMAP, size // WORD // 8 + 1, From noreply at buildbot.pypy.org Thu Jan 24 17:17:19 2013 From: noreply at buildbot.pypy.org (cfbolz) Date: Thu, 24 Jan 2013 17:17:19 +0100 (CET) Subject: [pypy-commit] pypy default: improve the JITting of thread-local storage Message-ID: <20130124161719.78A691C0246@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: Changeset: r60425:11342bf4b34e Date: 2013-01-24 17:17 +0100 http://bitbucket.org/pypy/pypy/changeset/11342bf4b34e/ Log: improve the JITting of thread-local storage use a slightly hacky workaround because we can use a loop invariant function: we only support loop-invariant functions without arguments atm. Instead have last-used cache on the local object. This helps in a lot of cases. diff --git a/pypy/module/pypyjit/test_pypy_c/test_thread.py b/pypy/module/pypyjit/test_pypy_c/test_thread.py --- a/pypy/module/pypyjit/test_pypy_c/test_thread.py +++ b/pypy/module/pypyjit/test_pypy_c/test_thread.py @@ -40,27 +40,11 @@ assert round(log.result, 6) == round(main(500), 6) loop, = log.loops_by_filename(self.filepath) assert loop.match(""" - i60 = int_lt(i55, i27) - guard_true(i60, descr=...) - i61 = call(ConstClass(ll_dict_lookup_trampoline__v1134___simple_call__function_), p33, p32, i35, descr=...) - guard_no_exception(descr=...) - i62 = int_and(i61, -2147483648) - i63 = int_is_true(i62) - guard_false(i63, descr=...) - p64 = getinteriorfield_gc(p41, i61, descr=...) - guard_nonnull_class(p64, ConstClass(W_DictMultiObject), descr=...) - p65 = getfield_gc(p64, descr=...) - guard_class(p65, 176132160, descr=...) - p66 = getfield_gc(p64, descr=...) - guard_class(p66, 175975744, descr=...) - p67 = getfield_gc(p66, descr=...) - guard_value(p67, ConstPtr(ptr49), descr=...) - p68 = getfield_gc(p66, descr=...) - p69 = getarrayitem_gc(p68, 0, descr=...) - guard_nonnull_class(p69, ConstClass(W_IntObject), descr=...) - i70 = getfield_gc_pure(p69, descr=...) - i71 = int_add_ovf(i55, i70) + i53 = int_lt(i48, i27) + guard_true(i53, descr=...) + i54 = int_add_ovf(i48, i47) guard_no_overflow(descr=...) --TICK-- - jump(p0, p1, p3, p5, p10, p12, p14, i71, i27, p33, p32, i35, p41, descr=...) + i58 = arraylen_gc(p43, descr=...) + jump(p0, p1, p3, p5, p10, p12, p14, i54, i27, i47, p45, p43, descr=...) """) diff --git a/pypy/module/thread/os_local.py b/pypy/module/thread/os_local.py --- a/pypy/module/thread/os_local.py +++ b/pypy/module/thread/os_local.py @@ -30,6 +30,9 @@ w_dict = space.newdict(instance=True) self.dicts[ec] = w_dict self._register_in_ec(ec) + # cache the last seen dict, works because we are protected by the GIL + self.last_dict = w_dict + self.last_ec = ec def _register_in_ec(self, ec): if not ec.space.config.translation.rweakref: @@ -60,10 +63,14 @@ def getdict(self, space): ec = space.getexecutioncontext() + if ec is self.last_ec: + return self.last_dict try: w_dict = self.dicts[ec] except KeyError: w_dict = self.create_new_dict(ec) + self.last_ec = ec + self.last_dict = w_dict return w_dict def descr_local__new__(space, w_subtype, __args__): @@ -91,3 +98,5 @@ local = wref() if local is not None: del local.dicts[ec] + local.last_dict = None + local.last_ec = None diff --git a/pypy/module/thread/test/test_local.py b/pypy/module/thread/test/test_local.py --- a/pypy/module/thread/test/test_local.py +++ b/pypy/module/thread/test/test_local.py @@ -108,3 +108,54 @@ gc.collect() assert done == ['ok', 'del'] done.append('shutdown') + +def test_local_caching(): + from pypy.module.thread.os_local import Local + class FakeSpace: + def getexecutioncontext(self): + return self.ec + + def getattr(*args): + pass + def call_obj_args(*args): + pass + def newdict(*args, **kwargs): + return {} + def wrap(self, obj): + return obj + def type(self, obj): + return type(obj) + class config: + class translation: + rweakref = True + + class FakeEC: + def __init__(self, space): + self.space = space + self._thread_local_objs = None + space = FakeSpace() + ec1 = FakeEC(space) + space.ec = ec1 + + l = Local(space, None) + assert l.last_dict is l.dicts[ec1] + assert l.last_ec is ec1 + d1 = l.getdict(space) + assert d1 is l.last_dict + + ec2 = space.ec = FakeEC(space) + d2 = l.getdict(space) + assert l.last_dict is d2 + assert d2 is l.dicts[ec2] + assert l.last_ec is ec2 + dicts = l.dicts + l.dicts = "nope" + assert l.getdict(space) is d2 + l.dicts = dicts + + space.ec = ec1 + assert l.getdict(space) is d1 + l.dicts = "nope" + assert l.getdict(space) is d1 + l.dicts = dicts + From noreply at buildbot.pypy.org Thu Jan 24 17:20:55 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 24 Jan 2013 17:20:55 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: make this test briefly pass Message-ID: <20130124162055.45F0E1C0246@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60426:0e6f39387238 Date: 2013-01-24 18:20 +0200 http://bitbucket.org/pypy/pypy/changeset/0e6f39387238/ Log: make this test briefly pass diff --git a/rpython/jit/backend/x86/test/test_gc_integration.py b/rpython/jit/backend/x86/test/test_gc_integration.py --- a/rpython/jit/backend/x86/test/test_gc_integration.py +++ b/rpython/jit/backend/x86/test/test_gc_integration.py @@ -515,7 +515,7 @@ p1 = call_malloc_nursery(%d) p2 = call_malloc_nursery(%d) # this overflows guard_nonnull(p2, descr=faildescr) [p0, p1, p2] - finish(p0, descr=finaldescr) + finish(p2, descr=finaldescr) """ % (size, size, size), namespace={'sizedescr': sizeof, 'finaldescr': BasicFinalDescr(), 'faildescr': BasicFailDescr()}) @@ -527,6 +527,6 @@ frame = cpu.execute_token(token) # now we should be able to track everything from the frame frame = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, frame) - #thing = frame.jf_frame[unpack_gcmap(frame)[0]] - #assert thing == rffi.cast(lltype.Signed, cpu.gc_ll_descr.nursery) - #assert cpu.gc_ll_descr.nursery_ptrs[0] == thing + sizeof.size + thing = frame.jf_frame[unpack_gcmap(frame)[0]] + assert thing == rffi.cast(lltype.Signed, cpu.gc_ll_descr.nursery) + assert cpu.gc_ll_descr.nursery_ptrs[0] == thing + sizeof.size From noreply at buildbot.pypy.org Thu Jan 24 18:43:02 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 24 Jan 2013 18:43:02 +0100 (CET) Subject: [pypy-commit] pypy fix-e4fa0b2: fix changes made in e4fa0b2: still need the check before because modulo wipes negative Message-ID: <20130124174302.881201C05D7@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: fix-e4fa0b2 Changeset: r60427:269fdb562619 Date: 2013-01-24 01:18 -0500 http://bitbucket.org/pypy/pypy/changeset/269fdb562619/ Log: fix changes made in e4fa0b2: still need the check before because modulo wipes negative diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py --- a/pypy/module/rctime/interp_time.py +++ b/pypy/module/rctime/interp_time.py @@ -448,6 +448,12 @@ raise OperationError(space.w_ValueError, space.wrap("year out of range")) + # tm_wday does not need checking of its upper-bound since taking "% + # 7" in gettmarg() automatically restricts the range. + if rffi.getintfield(glob_buf, 'c_tm_wday') < -1: + raise OperationError(space.w_ValueError, + space.wrap("day of week out of range")) + rffi.setintfield(glob_buf, 'c_tm_year', y - 1900) rffi.setintfield(glob_buf, 'c_tm_mon', rffi.getintfield(glob_buf, 'c_tm_mon') - 1) @@ -456,12 +462,6 @@ rffi.setintfield(glob_buf, 'c_tm_yday', rffi.getintfield(glob_buf, 'c_tm_yday') - 1) - # tm_wday does not need checking of its upper-bound since taking "% - # 7" in gettmarg() automatically restricts the range. - if rffi.getintfield(glob_buf, 'c_tm_wday') < 0: - raise OperationError(space.w_ValueError, - space.wrap("day of week out of range")) - return glob_buf def time(space): From noreply at buildbot.pypy.org Thu Jan 24 18:43:03 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 24 Jan 2013 18:43:03 +0100 (CET) Subject: [pypy-commit] pypy default: Merged in bdkearns/pypy/fix-e4fa0b2 (pull request #108) Message-ID: <20130124174303.D75981C1241@cobra.cs.uni-duesseldorf.de> Author: arigo Branch: Changeset: r60428:e576eb24d685 Date: 2013-01-24 18:42 +0100 http://bitbucket.org/pypy/pypy/changeset/e576eb24d685/ Log: Merged in bdkearns/pypy/fix-e4fa0b2 (pull request #108) fix changes made in e4fa0b2: still need the check before because modulo wipes negative diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py --- a/pypy/module/rctime/interp_time.py +++ b/pypy/module/rctime/interp_time.py @@ -448,6 +448,12 @@ raise OperationError(space.w_ValueError, space.wrap("year out of range")) + # tm_wday does not need checking of its upper-bound since taking "% + # 7" in gettmarg() automatically restricts the range. + if rffi.getintfield(glob_buf, 'c_tm_wday') < -1: + raise OperationError(space.w_ValueError, + space.wrap("day of week out of range")) + rffi.setintfield(glob_buf, 'c_tm_year', y - 1900) rffi.setintfield(glob_buf, 'c_tm_mon', rffi.getintfield(glob_buf, 'c_tm_mon') - 1) @@ -456,12 +462,6 @@ rffi.setintfield(glob_buf, 'c_tm_yday', rffi.getintfield(glob_buf, 'c_tm_yday') - 1) - # tm_wday does not need checking of its upper-bound since taking "% - # 7" in gettmarg() automatically restricts the range. - if rffi.getintfield(glob_buf, 'c_tm_wday') < 0: - raise OperationError(space.w_ValueError, - space.wrap("day of week out of range")) - return glob_buf def time(space): From noreply at buildbot.pypy.org Thu Jan 24 20:27:27 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Thu, 24 Jan 2013 20:27:27 +0100 (CET) Subject: [pypy-commit] pypy default: appdef doesn't handle multiline py.test.mark decorators, blerg Message-ID: <20130124192727.3180A1C00A8@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: Changeset: r60429:d2b91b922c6e Date: 2013-01-24 11:26 -0800 http://bitbucket.org/pypy/pypy/changeset/d2b91b922c6e/ Log: appdef doesn't handle multiline py.test.mark decorators, blerg diff --git a/pypy/module/unicodedata/test/test_unicodedata.py b/pypy/module/unicodedata/test/test_unicodedata.py --- a/pypy/module/unicodedata/test/test_unicodedata.py +++ b/pypy/module/unicodedata/test/test_unicodedata.py @@ -82,8 +82,7 @@ import unicodedata raises(TypeError, unicodedata.normalize, 'x') - @py.test.mark.skipif("sys.maxunicode < 0x10ffff", - reason="requires a 'wide' python build.") + @py.test.mark.skipif("sys.maxunicode < 0x10ffff") def test_normalize_wide(self): import unicodedata assert unicodedata.normalize('NFC', u'\U000110a5\U000110ba') == u'\U000110ab' From noreply at buildbot.pypy.org Thu Jan 24 22:50:36 2013 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 24 Jan 2013 22:50:36 +0100 (CET) Subject: [pypy-commit] pypy win32-cleanup3: reduce windows failures Message-ID: <20130124215036.611281C05D7@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: win32-cleanup3 Changeset: r60430:a889e96454c4 Date: 2013-01-24 22:52 +0200 http://bitbucket.org/pypy/pypy/changeset/a889e96454c4/ Log: reduce windows failures From noreply at buildbot.pypy.org Thu Jan 24 22:50:37 2013 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 24 Jan 2013 22:50:37 +0100 (CET) Subject: [pypy-commit] pypy win32-cleanup3: revert changeset 727405791c06 to pass tests on narrow builds Message-ID: <20130124215037.DC6421C1241@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: win32-cleanup3 Changeset: r60431:ae93f80293b8 Date: 2013-01-24 22:54 +0200 http://bitbucket.org/pypy/pypy/changeset/ae93f80293b8/ Log: revert changeset 727405791c06 to pass tests on narrow builds diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py --- a/rpython/rlib/runicode.py +++ b/rpython/rlib/runicode.py @@ -43,8 +43,30 @@ ORD._flowspace_rewrite_directly_as_ = ord else: - UNICHR = unichr - ORD = ord + #UNICHR = unichr + #ORD = ord + def UNICHR(c): + if c <= sys.maxunicode and c <= MAXUNICODE: + return unichr(c) + else: + c -= 0x10000 + return (unichr(0xD800 + (c >> 10)) + + unichr(0xDC00 + (c & 0x03FF))) + + def ORD(u): + assert isinstance(u, unicode) + if len(u) == 1: + return ord(u[0]) + elif len(u) == 2: + ch1 = ord(u[0]) + ch2 = ord(u[1]) + if 0xD800 <= ch1 <= 0xDBFF and 0xDC00 <= ch2 <= 0xDFFF: + return (((ch1 - 0xD800) << 10) | (ch2 - 0xDC00)) + 0x10000 + raise ValueError + +def _STORECHAR(result, CH, byteorder): + hi = chr(((CH) >> 8) & 0xff) + lo = chr((CH) & 0xff) if MAXUNICODE > 0xFFFF: def code_to_unichr(code): From noreply at buildbot.pypy.org Thu Jan 24 22:50:39 2013 From: noreply at buildbot.pypy.org (mattip) Date: Thu, 24 Jan 2013 22:50:39 +0100 (CET) Subject: [pypy-commit] pypy win32-cleanup3: copy from wide build implementations Message-ID: <20130124215039.227771C05D7@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: win32-cleanup3 Changeset: r60432:ac4223d5769f Date: 2013-01-24 23:49 +0200 http://bitbucket.org/pypy/pypy/changeset/ac4223d5769f/ Log: copy from wide build implementations diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py --- a/rpython/rlib/runicode.py +++ b/rpython/rlib/runicode.py @@ -46,14 +46,17 @@ #UNICHR = unichr #ORD = ord def UNICHR(c): + assert not we_are_translated() if c <= sys.maxunicode and c <= MAXUNICODE: return unichr(c) else: c -= 0x10000 return (unichr(0xD800 + (c >> 10)) + unichr(0xDC00 + (c & 0x03FF))) + UNICHR._flowspace_rewrite_directly_as_ = unichr def ORD(u): + assert not we_are_translated() assert isinstance(u, unicode) if len(u) == 1: return ord(u[0]) @@ -63,6 +66,7 @@ if 0xD800 <= ch1 <= 0xDBFF and 0xDC00 <= ch2 <= 0xDFFF: return (((ch1 - 0xD800) << 10) | (ch2 - 0xDC00)) + 0x10000 raise ValueError + ORD._flowspace_rewrite_directly_as_ = ord def _STORECHAR(result, CH, byteorder): hi = chr(((CH) >> 8) & 0xff) From noreply at buildbot.pypy.org Fri Jan 25 01:30:08 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Fri, 25 Jan 2013 01:30:08 +0100 (CET) Subject: [pypy-commit] pypy default: preallocate the bytearraydata via length_hint Message-ID: <20130125003008.AEDE01C05D7@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: Changeset: r60433:8f5beaa9c51a Date: 2013-01-24 16:27 -0800 http://bitbucket.org/pypy/pypy/changeset/8f5beaa9c51a/ Log: preallocate the bytearraydata via length_hint diff --git a/pypy/objspace/std/bytearraytype.py b/pypy/objspace/std/bytearraytype.py --- a/pypy/objspace/std/bytearraytype.py +++ b/pypy/objspace/std/bytearraytype.py @@ -15,6 +15,7 @@ str_splitlines, str_translate) from pypy.objspace.std.listtype import ( list_append, list_extend) +from rpython.rlib.objectmodel import newlist_hint, resizelist_hint bytearray_insert = SMM('insert', 3, @@ -87,8 +88,10 @@ return [c for c in string] # sequence of bytes - data = [] w_iter = space.iter(w_source) + length_hint = space.length_hint(w_source, 0) + data = newlist_hint(length_hint) + extended = 0 while True: try: w_item = space.next(w_iter) @@ -98,6 +101,9 @@ break value = getbytevalue(space, w_item) data.append(value) + extended += 1 + if extended < length_hint: + resizelist_hint(data, extended) return data def descr_bytearray__reduce__(space, w_self): From noreply at buildbot.pypy.org Fri Jan 25 01:30:09 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Fri, 25 Jan 2013 01:30:09 +0100 (CET) Subject: [pypy-commit] pypy default: fix iters that fail due to modification to return a length_hint of 0, and add a Message-ID: <20130125003009.E92091C1241@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: Changeset: r60434:cbb68962961a Date: 2013-01-24 16:28 -0800 http://bitbucket.org/pypy/pypy/changeset/cbb68962961a/ Log: fix iters that fail due to modification to return a length_hint of 0, and add a couple PEP 424 workarounds to make test_iterlen pass diff --git a/lib-python/2.7/test/test_iterlen.py b/lib-python/2.7/test/test_iterlen.py --- a/lib-python/2.7/test/test_iterlen.py +++ b/lib-python/2.7/test/test_iterlen.py @@ -94,7 +94,11 @@ def test_no_len_for_infinite_repeat(self): # The repeat() object can also be infinite - self.assertRaises(TypeError, len, repeat(None)) + if test_support.check_impl_detail(pypy=True): + # 3.4 (PEP 424) behavior + self.assertEqual(len(repeat(None)), NotImplemented) + else: + self.assertRaises(TypeError, len, repeat(None)) class TestXrange(TestInvariantWithoutMutations): @@ -230,6 +234,7 @@ self.assertRaises(RuntimeError, b.extend, BadLen()) self.assertRaises(RuntimeError, b.extend, BadLengthHint()) + @test_support.impl_detail("PEP 424 disallows None results", pypy=False) def test_invalid_hint(self): # Make sure an invalid result doesn't muck-up the works self.assertEqual(list(NoneLengthHint()), list(range(10))) diff --git a/lib-python/conftest.py b/lib-python/conftest.py --- a/lib-python/conftest.py +++ b/lib-python/conftest.py @@ -281,7 +281,6 @@ RegrTest('test_ioctl.py'), RegrTest('test_isinstance.py', core=True), RegrTest('test_iter.py', core=True), - RegrTest('test_iterlen.py', skip="undocumented internal API behavior __length_hint__"), RegrTest('test_itertools.py', core=True, usemodules="itertools struct"), RegrTest('test_json.py'), RegrTest('test_kqueue.py'), diff --git a/pypy/module/_collections/interp_deque.py b/pypy/module/_collections/interp_deque.py --- a/pypy/module/_collections/interp_deque.py +++ b/pypy/module/_collections/interp_deque.py @@ -521,7 +521,11 @@ return self.space.wrap(self.counter) def next(self): - self.deque.checklock(self.lock) + if self.lock is not self.deque.lock: + self.counter = 0 + raise OperationError( + self.space.w_RuntimeError, + self.space.wrap("deque mutated during iteration")) if self.counter == 0: raise OperationError(self.space.w_StopIteration, self.space.w_None) self.counter -= 1 @@ -560,7 +564,11 @@ return self.space.wrap(self.counter) def next(self): - self.deque.checklock(self.lock) + if self.lock is not self.deque.lock: + self.counter = 0 + raise OperationError( + self.space.w_RuntimeError, + self.space.wrap("deque mutated during iteration")) if self.counter == 0: raise OperationError(self.space.w_StopIteration, self.space.w_None) self.counter -= 1 diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py --- a/pypy/objspace/std/dictmultiobject.py +++ b/pypy/objspace/std/dictmultiobject.py @@ -352,7 +352,7 @@ self.pos = 0 def length(self): - if self.dictimplementation is not None: + if self.dictimplementation is not None and self.len != -1: return self.len - self.pos return 0 diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py --- a/pypy/objspace/std/setobject.py +++ b/pypy/objspace/std/setobject.py @@ -898,7 +898,7 @@ raise NotImplementedError def length(self): - if self.setimplementation is not None: + if self.setimplementation is not None and self.len != -1: return self.len - self.pos return 0 diff --git a/pypy/objspace/std/test/test_lengthhint.py b/pypy/objspace/std/test/test_lengthhint.py --- a/pypy/objspace/std/test/test_lengthhint.py +++ b/pypy/objspace/std/test/test_lengthhint.py @@ -6,7 +6,7 @@ SIZE = 4 ITEMS = range(SIZE) - def _test_length_hint(self, w_obj): + def _test_length_hint(self, w_obj, w_mutate=None): space = self.space assert space.length_hint(w_obj, 8) == self.SIZE @@ -18,6 +18,13 @@ space.next(w_iter) assert space.length_hint(w_iter, 8) == self.SIZE - 1 + if w_mutate is not None: + # Test handling of collections that enforce length + # immutability during iteration + space.call_function(w_mutate) + space.raises_w(space.w_RuntimeError, space.next, w_iter) + assert space.length_hint(w_iter, 8) == 0 + def test_bytearray(self): space = self.space w_bytearray = space.call_function(space.w_bytearray, @@ -31,16 +38,20 @@ self._test_length_hint(w_dict) def test_dict_iterkeys(self): - w_iterkeys = self.space.appexec([], """(): - return dict.fromkeys(%r).iterkeys() - """ % self.ITEMS) - self._test_length_hint(w_iterkeys) + space = self.space + w_iterkeys, w_mutate = space.fixedview(space.appexec([], """(): + d = dict.fromkeys(%r) + return d.iterkeys(), d.popitem + """ % self.ITEMS), 2) + self._test_length_hint(w_iterkeys, w_mutate) def test_dict_values(self): - w_itervalues = self.space.appexec([], """(): - return dict.fromkeys(%r).itervalues() - """ % self.ITEMS) - self._test_length_hint(w_itervalues) + space = self.space + w_itervalues, w_mutate = space.fixedview(space.appexec([], """(): + d = dict.fromkeys(%r) + return d.itervalues(), d.popitem + """ % self.ITEMS), 2) + self._test_length_hint(w_itervalues, w_mutate) def test_frozenset(self): space = self.space @@ -50,7 +61,8 @@ def test_set(self): space = self.space w_set = space.call_function(space.w_set, space.wrap(self.ITEMS)) - self._test_length_hint(w_set) + w_mutate = space.getattr(w_set, space.wrap('pop')) + self._test_length_hint(w_set, w_mutate) def test_list(self): self._test_length_hint(self.space.wrap(self.ITEMS)) @@ -101,12 +113,19 @@ self._test_length_hint(W_Repeat(space, space.wrap(22), space.wrap(self.SIZE))) - def test_collections_deque(self): + def _setup_deque(self): space = self.space w_deque = W_Deque(space) space.call_method(w_deque, '__init__', space.wrap(self.ITEMS)) - self._test_length_hint(w_deque) - self._test_length_hint(w_deque.reviter()) + w_mutate = space.getattr(w_deque, space.wrap('pop')) + return w_deque, w_mutate + + def test_collections_deque(self): + self._test_length_hint(*self._setup_deque()) + + def test_collections_deque_rev(self): + w_deque, w_mutate = self._setup_deque() + self._test_length_hint(w_deque.reviter(), w_mutate) def test_default(self): space = self.space From noreply at buildbot.pypy.org Fri Jan 25 01:36:56 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Fri, 25 Jan 2013 01:36:56 +0100 (CET) Subject: [pypy-commit] pypy default: woops, no longer skips but still needs an entry here Message-ID: <20130125003656.9EAA01C05D7@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: Changeset: r60435:f4ce1da212bf Date: 2013-01-24 16:36 -0800 http://bitbucket.org/pypy/pypy/changeset/f4ce1da212bf/ Log: woops, no longer skips but still needs an entry here diff --git a/lib-python/conftest.py b/lib-python/conftest.py --- a/lib-python/conftest.py +++ b/lib-python/conftest.py @@ -281,6 +281,7 @@ RegrTest('test_ioctl.py'), RegrTest('test_isinstance.py', core=True), RegrTest('test_iter.py', core=True), + RegrTest('test_iterlen.py', core=True, usemodules="_collections itertools"), RegrTest('test_itertools.py', core=True, usemodules="itertools struct"), RegrTest('test_json.py'), RegrTest('test_kqueue.py'), From noreply at buildbot.pypy.org Fri Jan 25 01:55:00 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Fri, 25 Jan 2013 01:55:00 +0100 (CET) Subject: [pypy-commit] pypy py3k: merge default Message-ID: <20130125005500.2A5771C00A8@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60436:38b9fe11e79d Date: 2013-01-24 16:52 -0800 http://bitbucket.org/pypy/pypy/changeset/38b9fe11e79d/ Log: merge default diff --git a/lib-python/2.7/test/test_iterlen.py b/lib-python/2.7/test/test_iterlen.py --- a/lib-python/2.7/test/test_iterlen.py +++ b/lib-python/2.7/test/test_iterlen.py @@ -94,7 +94,11 @@ def test_no_len_for_infinite_repeat(self): # The repeat() object can also be infinite - self.assertRaises(TypeError, len, repeat(None)) + if test_support.check_impl_detail(pypy=True): + # 3.4 (PEP 424) behavior + self.assertEqual(len(repeat(None)), NotImplemented) + else: + self.assertRaises(TypeError, len, repeat(None)) class TestXrange(TestInvariantWithoutMutations): @@ -230,6 +234,7 @@ self.assertRaises(RuntimeError, b.extend, BadLen()) self.assertRaises(RuntimeError, b.extend, BadLengthHint()) + @test_support.impl_detail("PEP 424 disallows None results", pypy=False) def test_invalid_hint(self): # Make sure an invalid result doesn't muck-up the works self.assertEqual(list(NoneLengthHint()), list(range(10))) diff --git a/lib-python/conftest.py b/lib-python/conftest.py --- a/lib-python/conftest.py +++ b/lib-python/conftest.py @@ -258,7 +258,7 @@ RegrTest('test_ioctl.py'), RegrTest('test_isinstance.py', core=True), RegrTest('test_iter.py', core=True), - RegrTest('test_iterlen.py'), + RegrTest('test_iterlen.py', core=True, usemodules="_collections itertools"), RegrTest('test_itertools.py', core=True, usemodules="itertools struct"), RegrTest('test_json.py'), RegrTest('test_keywordonlyarg.py'), diff --git a/pypy/doc/getting-started-python.rst b/pypy/doc/getting-started-python.rst --- a/pypy/doc/getting-started-python.rst +++ b/pypy/doc/getting-started-python.rst @@ -86,8 +86,8 @@ 4. Run:: - cd pypy/translator/goal - python translate.py --opt=jit targetpypystandalone.py + cd pypy/goal + python ../../rpython/bin/rpython --opt=jit targetpypystandalone.py possibly replacing ``--opt=jit`` with another `optimization level`_ of your choice like ``--opt=2`` if you do not want to include the JIT diff --git a/pypy/module/_collections/interp_deque.py b/pypy/module/_collections/interp_deque.py --- a/pypy/module/_collections/interp_deque.py +++ b/pypy/module/_collections/interp_deque.py @@ -521,7 +521,11 @@ return self.space.wrap(self.counter) def next(self): - self.deque.checklock(self.lock) + if self.lock is not self.deque.lock: + self.counter = 0 + raise OperationError( + self.space.w_RuntimeError, + self.space.wrap("deque mutated during iteration")) if self.counter == 0: raise OperationError(self.space.w_StopIteration, self.space.w_None) self.counter -= 1 @@ -560,7 +564,11 @@ return self.space.wrap(self.counter) def next(self): - self.deque.checklock(self.lock) + if self.lock is not self.deque.lock: + self.counter = 0 + raise OperationError( + self.space.w_RuntimeError, + self.space.wrap("deque mutated during iteration")) if self.counter == 0: raise OperationError(self.space.w_StopIteration, self.space.w_None) self.counter -= 1 diff --git a/pypy/module/pypyjit/test_pypy_c/test_thread.py b/pypy/module/pypyjit/test_pypy_c/test_thread.py --- a/pypy/module/pypyjit/test_pypy_c/test_thread.py +++ b/pypy/module/pypyjit/test_pypy_c/test_thread.py @@ -26,3 +26,25 @@ --THREAD-TICK-- jump(..., descr=...) """) + + def test_tls(self): + def main(n): + import thread + local = thread._local() + local.x = 1 + i = 0 + while i < n: + i += local.x + return 0 + log = self.run(main, [500]) + assert round(log.result, 6) == round(main(500), 6) + loop, = log.loops_by_filename(self.filepath) + assert loop.match(""" + i53 = int_lt(i48, i27) + guard_true(i53, descr=...) + i54 = int_add_ovf(i48, i47) + guard_no_overflow(descr=...) + --TICK-- + i58 = arraylen_gc(p43, descr=...) + jump(p0, p1, p3, p5, p10, p12, p14, i54, i27, i47, p45, p43, descr=...) + """) diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py --- a/pypy/module/rctime/interp_time.py +++ b/pypy/module/rctime/interp_time.py @@ -447,6 +447,12 @@ space.warn("Century info guessed for a 2-digit year.", space.w_DeprecationWarning) + # tm_wday does not need checking of its upper-bound since taking "% + # 7" in gettmarg() automatically restricts the range. + if rffi.getintfield(glob_buf, 'c_tm_wday') < -1: + raise OperationError(space.w_ValueError, + space.wrap("day of week out of range")) + rffi.setintfield(glob_buf, 'c_tm_year', y - 1900) rffi.setintfield(glob_buf, 'c_tm_mon', rffi.getintfield(glob_buf, 'c_tm_mon') - 1) @@ -455,12 +461,6 @@ rffi.setintfield(glob_buf, 'c_tm_yday', rffi.getintfield(glob_buf, 'c_tm_yday') - 1) - # tm_wday does not need checking of its upper-bound since taking "% - # 7" in gettmarg() automatically restricts the range. - if rffi.getintfield(glob_buf, 'c_tm_wday') < 0: - raise OperationError(space.w_ValueError, - space.wrap("day of week out of range")) - return glob_buf def time(space): diff --git a/pypy/module/thread/os_local.py b/pypy/module/thread/os_local.py --- a/pypy/module/thread/os_local.py +++ b/pypy/module/thread/os_local.py @@ -30,6 +30,9 @@ w_dict = space.newdict(instance=True) self.dicts[ec] = w_dict self._register_in_ec(ec) + # cache the last seen dict, works because we are protected by the GIL + self.last_dict = w_dict + self.last_ec = ec def _register_in_ec(self, ec): if not ec.space.config.translation.rweakref: @@ -60,10 +63,14 @@ def getdict(self, space): ec = space.getexecutioncontext() + if ec is self.last_ec: + return self.last_dict try: w_dict = self.dicts[ec] except KeyError: w_dict = self.create_new_dict(ec) + self.last_ec = ec + self.last_dict = w_dict return w_dict def descr_local__new__(space, w_subtype, __args__): @@ -91,3 +98,5 @@ local = wref() if local is not None: del local.dicts[ec] + local.last_dict = None + local.last_ec = None diff --git a/pypy/module/thread/test/test_local.py b/pypy/module/thread/test/test_local.py --- a/pypy/module/thread/test/test_local.py +++ b/pypy/module/thread/test/test_local.py @@ -107,3 +107,54 @@ gc.collect() assert done == ['ok', 'del'] done.append('shutdown') + +def test_local_caching(): + from pypy.module.thread.os_local import Local + class FakeSpace: + def getexecutioncontext(self): + return self.ec + + def getattr(*args): + pass + def call_obj_args(*args): + pass + def newdict(*args, **kwargs): + return {} + def wrap(self, obj): + return obj + def type(self, obj): + return type(obj) + class config: + class translation: + rweakref = True + + class FakeEC: + def __init__(self, space): + self.space = space + self._thread_local_objs = None + space = FakeSpace() + ec1 = FakeEC(space) + space.ec = ec1 + + l = Local(space, None) + assert l.last_dict is l.dicts[ec1] + assert l.last_ec is ec1 + d1 = l.getdict(space) + assert d1 is l.last_dict + + ec2 = space.ec = FakeEC(space) + d2 = l.getdict(space) + assert l.last_dict is d2 + assert d2 is l.dicts[ec2] + assert l.last_ec is ec2 + dicts = l.dicts + l.dicts = "nope" + assert l.getdict(space) is d2 + l.dicts = dicts + + space.ec = ec1 + assert l.getdict(space) is d1 + l.dicts = "nope" + assert l.getdict(space) is d1 + l.dicts = dicts + diff --git a/pypy/module/unicodedata/test/test_unicodedata.py b/pypy/module/unicodedata/test/test_unicodedata.py --- a/pypy/module/unicodedata/test/test_unicodedata.py +++ b/pypy/module/unicodedata/test/test_unicodedata.py @@ -82,8 +82,7 @@ import unicodedata raises(TypeError, unicodedata.normalize, 'x') - @py.test.mark.skipif("sys.maxunicode < 0x10ffff", - reason="requires a 'wide' python build.") + @py.test.mark.skipif("sys.maxunicode < 0x10ffff") def test_normalize_wide(self): import unicodedata assert unicodedata.normalize('NFC', '\U000110a5\U000110ba') == u'\U000110ab' diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py --- a/pypy/objspace/std/dictmultiobject.py +++ b/pypy/objspace/std/dictmultiobject.py @@ -54,6 +54,9 @@ # every module needs its own strategy, because the strategy stores # the version tag strategy = ModuleDictStrategy(space) + elif space.config.objspace.std.withmapdict and instance: + from pypy.objspace.std.mapdict import MapDictStrategy + strategy = space.fromcache(MapDictStrategy) # elif instance or strdict or module: # assert w_type is None @@ -352,7 +355,7 @@ self.pos = 0 def length(self): - if self.dictimplementation is not None: + if self.dictimplementation is not None and self.len != -1: return self.len - self.pos return 0 diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py --- a/pypy/objspace/std/mapdict.py +++ b/pypy/objspace/std/mapdict.py @@ -591,6 +591,9 @@ # ____________________________________________________________ # dict implementation +def get_terminator_for_dicts(space): + return DictTerminator(space, None) + class MapDictStrategy(DictStrategy): erase, unerase = rerased.new_erasing_pair("map") @@ -600,13 +603,19 @@ def __init__(self, space): self.space = space + def get_empty_storage(self): + w_result = Object() + terminator = self.space.fromcache(get_terminator_for_dicts) + w_result._init_empty(terminator) + return self.erase(w_result) + def switch_to_object_strategy(self, w_dict): w_obj = self.unerase(w_dict.dstorage) strategy = self.space.fromcache(ObjectDictStrategy) dict_w = strategy.unerase(strategy.get_empty_storage()) w_dict.strategy = strategy w_dict.dstorage = strategy.erase(dict_w) - assert w_obj.getdict(self.space) is w_dict + assert w_obj.getdict(self.space) is w_dict or w_obj._get_mapdict_map().terminator.w_cls is None materialize_r_dict(self.space, w_obj, dict_w) def getitem(self, w_dict, w_key): diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py --- a/pypy/objspace/std/setobject.py +++ b/pypy/objspace/std/setobject.py @@ -888,7 +888,7 @@ raise NotImplementedError def length(self): - if self.setimplementation is not None: + if self.setimplementation is not None and self.len != -1: return self.len - self.pos return 0 diff --git a/pypy/objspace/std/stringtype.py b/pypy/objspace/std/stringtype.py --- a/pypy/objspace/std/stringtype.py +++ b/pypy/objspace/std/stringtype.py @@ -4,7 +4,7 @@ from pypy.objspace.std.register_all import register_all from sys import maxint -from rpython.rlib.objectmodel import specialize +from rpython.rlib.objectmodel import newlist_hint, resizelist_hint, specialize from rpython.rlib.jit import we_are_jitted def wrapstr(space, s): @@ -306,8 +306,10 @@ space.wrap("cannot convert unicode object to bytes")) # sequence of bytes - data = [] w_iter = space.iter(w_source) + length_hint = space.length_hint(w_source, 0) + data = newlist_hint(length_hint) + extended = 0 while True: try: w_item = space.next(w_iter) @@ -317,6 +319,9 @@ break value = getbytevalue(space, w_item) data.append(value) + extended += 1 + if extended < length_hint: + resizelist_hint(data, extended) return data @unwrap_spec(encoding='str_or_None', errors='str_or_None') diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py --- a/pypy/objspace/std/test/test_dictmultiobject.py +++ b/pypy/objspace/std/test/test_dictmultiobject.py @@ -1118,6 +1118,7 @@ withcelldict = False withmethodcache = False withidentitydict = False + withmapdict = False FakeSpace.config = Config() diff --git a/pypy/objspace/std/test/test_lengthhint.py b/pypy/objspace/std/test/test_lengthhint.py --- a/pypy/objspace/std/test/test_lengthhint.py +++ b/pypy/objspace/std/test/test_lengthhint.py @@ -6,7 +6,7 @@ SIZE = 4 ITEMS = range(SIZE) - def _test_length_hint(self, w_obj): + def _test_length_hint(self, w_obj, w_mutate=None): space = self.space assert space.length_hint(w_obj, 8) == self.SIZE @@ -18,6 +18,13 @@ space.next(w_iter) assert space.length_hint(w_iter, 8) == self.SIZE - 1 + if w_mutate is not None: + # Test handling of collections that enforce length + # immutability during iteration + space.call_function(w_mutate) + space.raises_w(space.w_RuntimeError, space.next, w_iter) + assert space.length_hint(w_iter, 8) == 0 + def test_bytearray(self): space = self.space w_bytearray = space.call_function(space.w_bytearray, @@ -31,16 +38,20 @@ self._test_length_hint(w_dict) def test_dict_iterkeys(self): - w_iterkeys = self.space.appexec([], """(): - return dict.fromkeys(%r).iterkeys() - """ % self.ITEMS) - self._test_length_hint(w_iterkeys) + space = self.space + w_iterkeys, w_mutate = space.fixedview(space.appexec([], """(): + d = dict.fromkeys(%r) + return d.iterkeys(), d.popitem + """ % self.ITEMS), 2) + self._test_length_hint(w_iterkeys, w_mutate) def test_dict_values(self): - w_itervalues = self.space.appexec([], """(): - return dict.fromkeys(%r).itervalues() - """ % self.ITEMS) - self._test_length_hint(w_itervalues) + space = self.space + w_itervalues, w_mutate = space.fixedview(space.appexec([], """(): + d = dict.fromkeys(%r) + return d.itervalues(), d.popitem + """ % self.ITEMS), 2) + self._test_length_hint(w_itervalues, w_mutate) def test_frozenset(self): space = self.space @@ -50,7 +61,8 @@ def test_set(self): space = self.space w_set = space.call_function(space.w_set, space.wrap(self.ITEMS)) - self._test_length_hint(w_set) + w_mutate = space.getattr(w_set, space.wrap('pop')) + self._test_length_hint(w_set, w_mutate) def test_list(self): self._test_length_hint(self.space.wrap(self.ITEMS)) @@ -107,12 +119,19 @@ self._test_length_hint(W_Repeat(space, space.wrap(22), space.wrap(self.SIZE))) - def test_collections_deque(self): + def _setup_deque(self): space = self.space w_deque = W_Deque(space) space.call_method(w_deque, '__init__', space.wrap(self.ITEMS)) - self._test_length_hint(w_deque) - self._test_length_hint(w_deque.reviter()) + w_mutate = space.getattr(w_deque, space.wrap('pop')) + return w_deque, w_mutate + + def test_collections_deque(self): + self._test_length_hint(*self._setup_deque()) + + def test_collections_deque_rev(self): + w_deque, w_mutate = self._setup_deque() + self._test_length_hint(w_deque.reviter(), w_mutate) def test_default(self): space = self.space diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py --- a/pypy/objspace/std/test/test_mapdict.py +++ b/pypy/objspace/std/test/test_mapdict.py @@ -1,7 +1,17 @@ from pypy.objspace.std.test.test_dictmultiobject import FakeSpace, W_DictMultiObject from pypy.objspace.std.mapdict import * +class Config: + class objspace: + class std: + withsmalldicts = False + withcelldict = False + withmethodcache = False + withidentitydict = False + withmapdict = True + space = FakeSpace() +space.config = Config class Class(object): def __init__(self, hasdict=True): @@ -1069,3 +1079,11 @@ return A() """) assert w_dict.user_overridden_class + +def test_newdict_instance(): + w_dict = space.newdict(instance=True) + assert type(w_dict.strategy) is MapDictStrategy + +class TestMapDictImplementationUsingnewdict(BaseTestRDictImplementation): + StrategyClass = MapDictStrategy + # NB: the get_impl method is not overwritten here, as opposed to above From noreply at buildbot.pypy.org Fri Jan 25 01:55:01 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Fri, 25 Jan 2013 01:55:01 +0100 (CET) Subject: [pypy-commit] pypy py3k: apply pep 424 workarounds from 2.7 Message-ID: <20130125005501.836F81C00A8@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60437:c8fd4e300ee3 Date: 2013-01-24 16:54 -0800 http://bitbucket.org/pypy/pypy/changeset/c8fd4e300ee3/ Log: apply pep 424 workarounds from 2.7 diff --git a/lib-python/3.2/test/test_iterlen.py b/lib-python/3.2/test/test_iterlen.py --- a/lib-python/3.2/test/test_iterlen.py +++ b/lib-python/3.2/test/test_iterlen.py @@ -94,7 +94,11 @@ def test_no_len_for_infinite_repeat(self): # The repeat() object can also be infinite - self.assertRaises(TypeError, len, repeat(None)) + if support.check_impl_detail(pypy=True): + # 3.4 (PEP 424) behavior + self.assertEqual(len(repeat(None)), NotImplemented) + else: + self.assertRaises(TypeError, len, repeat(None)) class TestXrange(TestInvariantWithoutMutations): @@ -224,6 +228,7 @@ self.assertRaises(RuntimeError, b.extend, BadLen()) self.assertRaises(RuntimeError, b.extend, BadLengthHint()) + @support.impl_detail("PEP 424 disallows None results", pypy=False) def test_invalid_hint(self): # Make sure an invalid result doesn't muck-up the works self.assertEqual(list(NoneLengthHint()), list(range(10))) From noreply at buildbot.pypy.org Fri Jan 25 01:55:48 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Fri, 25 Jan 2013 01:55:48 +0100 (CET) Subject: [pypy-commit] pypy py3k: idna using tests require unicodedata Message-ID: <20130125005548.BE4EF1C00A8@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60438:3d2ce58b3bb1 Date: 2013-01-24 16:55 -0800 http://bitbucket.org/pypy/pypy/changeset/3d2ce58b3bb1/ Log: idna using tests require unicodedata diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -6,7 +6,8 @@ from rpython.rtyper.lltypesystem import lltype, rffi def setup_module(mod): - mod.space = gettestobjspace(usemodules=['_socket', 'array', 'struct']) + mod.space = gettestobjspace(usemodules=['_socket', 'array', 'struct', + 'unicodedata']) global socket import socket mod.w_socket = space.appexec([], "(): import _socket as m; return m") diff --git a/pypy/module/_ssl/test/test_ssl.py b/pypy/module/_ssl/test/test_ssl.py --- a/pypy/module/_ssl/test/test_ssl.py +++ b/pypy/module/_ssl/test/test_ssl.py @@ -67,7 +67,8 @@ class AppTestConnectedSSL: spaceconfig = { - "usemodules": ['_ssl', '_socket', 'struct', 'array', 'binascii'], + "usemodules": ['_ssl', '_socket', 'struct', 'array', 'binascii', + 'unicodedata'], } def setup_method(self, method): From noreply at buildbot.pypy.org Fri Jan 25 02:07:15 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Fri, 25 Jan 2013 02:07:15 +0100 (CET) Subject: [pypy-commit] pypy default: restore these checks which got deleted Message-ID: <20130125010715.106F71C00A8@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: Changeset: r60439:cc89614c9009 Date: 2013-01-24 19:07 -0600 http://bitbucket.org/pypy/pypy/changeset/cc89614c9009/ Log: restore these checks which got deleted diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -484,8 +484,11 @@ self.autoreds = True self.reds = [] self.numreds = None # see warmspot.autodetect_jit_markers_redvars - assert confirm_enter_jit is None, ( - "reds='auto' is not compatible with confirm_enter_jit") + for hook in [ + get_jitcell_at, set_jitcell_at, get_printable_location, + confirm_enter_jit + ]: + assert hook is None, "reds='auto' is not compatible with JitDriver hooks" else: if reds is not None: self.reds = reds From noreply at buildbot.pypy.org Fri Jan 25 02:10:15 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Fri, 25 Jan 2013 02:10:15 +0100 (CET) Subject: [pypy-commit] pypy default: document a branch, fix a test Message-ID: <20130125011015.AC9151C00A8@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: Changeset: r60440:9d0be76fc5ab Date: 2013-01-24 19:10 -0600 http://bitbucket.org/pypy/pypy/changeset/9d0be76fc5ab/ Log: document a branch, fix a test diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -43,3 +43,7 @@ .. branch: fix-lookinside-iff-oopspec Fixed the interaction between two internal tools for controlling the JIT. + +.. branch: inline-virtualref-2 +Better optimized certain types of frame accesses in the JIT, particularly +around exceptions that escape the function they were raised in. From noreply at buildbot.pypy.org Fri Jan 25 02:16:39 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Fri, 25 Jan 2013 02:16:39 +0100 (CET) Subject: [pypy-commit] pypy py3k: requires an explicit relative import on py3k Message-ID: <20130125011639.13E741C0246@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60441:850797658e3b Date: 2013-01-24 17:16 -0800 http://bitbucket.org/pypy/pypy/changeset/850797658e3b/ Log: requires an explicit relative import on py3k diff --git a/lib_pypy/ctypes_config_cache/dumpcache.py b/lib_pypy/ctypes_config_cache/dumpcache.py --- a/lib_pypy/ctypes_config_cache/dumpcache.py +++ b/lib_pypy/ctypes_config_cache/dumpcache.py @@ -19,7 +19,7 @@ # XXX relative import, should be removed together with # XXX the relative imports done e.g. by lib_pypy/pypy_test/test_hashlib mod = __import__("_%s_%%s_" %% (cpumodel,), - globals(), locals(), ["*"]) + globals(), locals(), ["*"], level=1) globals().update(mod.__dict__)\ ''' % (basename,) g.close() From noreply at buildbot.pypy.org Fri Jan 25 07:09:19 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Fri, 25 Jan 2013 07:09:19 +0100 (CET) Subject: [pypy-commit] pypy default: get_printable_location doesn't work w/ autoreds Message-ID: <20130125060919.411221C0041@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: Changeset: r60442:7cd4c46c8f85 Date: 2013-01-24 22:08 -0800 http://bitbucket.org/pypy/pypy/changeset/7cd4c46c8f85/ Log: get_printable_location doesn't work w/ autoreds diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -310,10 +310,7 @@ def decorate(func): from rpython.tool.sourcetools import compile2 # - def get_printable_location(): - return name - jitdriver = JitDriver(get_printable_location=get_printable_location, - greens=[], reds='auto', name=name) + jitdriver = JitDriver(greens=[], reds='auto', name=name) # args = ','.join(['a%d' % i for i in range(func.func_code.co_argcount)]) source = """def callback_with_jitdriver(%(args)s): From noreply at buildbot.pypy.org Fri Jan 25 12:05:39 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 25 Jan 2013 12:05:39 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: Fix until the write barrier on frame is called correctly Message-ID: <20130125110539.80C691C131B@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60443:35fe48777d21 Date: 2013-01-25 13:05 +0200 http://bitbucket.org/pypy/pypy/changeset/35fe48777d21/ Log: Fix until the write barrier on frame is called correctly diff --git a/rpython/jit/backend/llsupport/asmmemmgr.py b/rpython/jit/backend/llsupport/asmmemmgr.py --- a/rpython/jit/backend/llsupport/asmmemmgr.py +++ b/rpython/jit/backend/llsupport/asmmemmgr.py @@ -248,6 +248,12 @@ index += self.SUBBLOCK_SIZE block.data[index] = char + def overwrite32(self, index, val): + self.overwrite(index, chr(val & 0xff)) + self.overwrite(index + 1, chr((val >> 8) & 0xff)) + self.overwrite(index + 2, chr((val >> 16) & 0xff)) + self.overwrite(index + 3, chr((val >> 24) & 0xff)) + def get_relative_pos(self): return self._baserelpos + self._cursubindex diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -94,7 +94,7 @@ self.float_const_neg_addr = 0 self.float_const_abs_addr = 0 self.malloc_slowpath = 0 - self.wb_slowpath = [0, 0, 0, 0] + self.wb_slowpath = [0, 0, 0, 0, 0] self.memcpy_addr = 0 self.setup_failure_recovery() self._debug = False @@ -119,6 +119,8 @@ self._build_failure_recovery(True) self._build_wb_slowpath(False) self._build_wb_slowpath(True) + self._build_wb_slowpath(False, for_frame=True) + # only one of those self._build_stack_check_failure() if self.cpu.supports_floats: self._build_failure_recovery(False, withfloats=True) @@ -239,18 +241,18 @@ mc.LEA_rb(esi.value, -base_ofs) mc.SUB_ri(esp.value, 16 - WORD) mc.CALL(imm(addr)) - mc.ADD_ri(esp.value, 16 - WORD) # Note: we check this after the code above, just because the code # above is more than 127 bytes on 64-bits... - self._pop_all_regs_from_frame(mc, [eax, edi], self.cpu.supports_floats) mc.TEST_rr(eax.value, eax.value) - mc.J_il8(rx86.Conditions['Z'], 0) # patched later + mc.J_il(rx86.Conditions['Z'], 0xfffff) # patched later jz_location = mc.get_relative_pos() # nursery_free_adr = self.cpu.gc_ll_descr.get_nursery_free_addr() + self._reload_frame_if_necessary(mc) + self._pop_all_regs_from_frame(mc, [eax, edi], self.cpu.supports_floats) + mc.LEA_rs(esp.value, 16 - WORD) mc.MOV(edi, heap(nursery_free_adr)) # load this in EDX # clear the gc pattern - self._reload_frame_if_necessary(mc) mc.MOV_bi(ofs, 0) mc.RET() # @@ -260,8 +262,7 @@ # but the code we jump to will actually restore the stack # position based on EBP, which will get us out of here for free. offset = mc.get_relative_pos() - jz_location - assert 0 < offset <= 127 - mc.overwrite(jz_location-1, chr(offset)) + mc.overwrite32(jz_location-4, offset) mc.JMP(imm(self.propagate_exception_path)) # rawstart = mc.materialize(self.cpu.asmmemmgr, []) @@ -364,7 +365,7 @@ rawstart = mc.materialize(self.cpu.asmmemmgr, []) self.stack_check_slowpath = rawstart - def _build_wb_slowpath(self, withcards, withfloats=False): + def _build_wb_slowpath(self, withcards, withfloats=False, for_frame=False): descr = self.cpu.gc_ll_descr.write_barrier_descr if descr is None: return @@ -384,61 +385,44 @@ # accordingly. mc = codebuf.MachineCodeBlockWrapper() # - frame_size = (1 + # my argument, considered part of my frame - 1 + # my return address - len(gpr_reg_mgr_cls.save_around_call_regs)) - if withfloats: - frame_size += 16 # X86_32: 16 words for 8 registers; - # X86_64: just 16 registers + if not for_frame: + self._push_all_regs_to_frame(mc, [], withfloats, callee_only=True) + else: + mc.MOV_sr(0, eax.value) + if IS_X86_32: - frame_size += 1 # argument to pass to the call - # - # align to a multiple of 16 bytes - frame_size = (frame_size + (CALL_ALIGN-1)) & ~(CALL_ALIGN-1) - # - correct_esp_by = (frame_size - 2) * WORD - mc.SUB_ri(esp.value, correct_esp_by) - # - ofs = correct_esp_by - if withfloats: - for reg in xmm_reg_mgr_cls.save_around_call_regs: - ofs -= 8 - mc.MOVSD_sx(ofs, reg.value) - for reg in gpr_reg_mgr_cls.save_around_call_regs: - ofs -= WORD - mc.MOV_sr(ofs, reg.value) - # - if IS_X86_32: - mc.MOV_rs(eax.value, (frame_size - 1) * WORD) + XXX + mc.MOV_rs(eax.value, WORD) mc.MOV_sr(0, eax.value) elif IS_X86_64: - mc.MOV_rs(edi.value, (frame_size - 1) * WORD) + mc.MOV_rs(edi.value, WORD) mc.CALL(imm(func)) # if withcards: # A final TEST8 before the RET, for the caller. Careful to # not follow this instruction with another one that changes # the status of the CPU flags! - mc.MOV_rs(eax.value, (frame_size - 1) * WORD) + mc.MOV_rs(eax.value, WORD) mc.TEST8(addr_add_const(eax, descr.jit_wb_if_flag_byteofs), imm(-0x80)) # - ofs = correct_esp_by - if withfloats: - for reg in xmm_reg_mgr_cls.save_around_call_regs: - ofs -= 8 - mc.MOVSD_xs(reg.value, ofs) - for reg in gpr_reg_mgr_cls.save_around_call_regs: - ofs -= WORD - mc.MOV_rs(reg.value, ofs) + + if not for_frame: + self._pop_all_regs_from_frame(mc, [], withfloats, callee_only=True) + else: + mc.MOV_rs(eax.value, 0) # # ADD esp, correct_esp_by --- but cannot use ADD, because # of its effects on the CPU flags - mc.LEA_rs(esp.value, correct_esp_by) + + #mc.LEA_rs(esp.value, WORD) mc.RET16_i(WORD) # rawstart = mc.materialize(self.cpu.asmmemmgr, []) - self.wb_slowpath[withcards + 2 * withfloats] = rawstart + if for_frame: + self.wb_slowpath[4] = rawstart + else: + self.wb_slowpath[withcards + 2 * withfloats] = rawstart @staticmethod @rgc.no_collect @@ -1249,9 +1233,9 @@ remap_frame_layout(self, src_locs, dst_locs, X86_64_SCRATCH_REG) self.push_gcmap(self.mc, self._regalloc.get_gcmap([eax]), store=True) self.mc.CALL(x) + self._reload_frame_if_necessary(self.mc) if align: self.mc.ADD_ri(esp.value, align * WORD) - self._reload_frame_if_necessary(self.mc) self.pop_gcmap(self.mc) def _reload_frame_if_necessary(self, mc): @@ -1264,11 +1248,10 @@ mc.ADD_ri(ebp.value, base_ofs) wbdescr = self.cpu.gc_ll_descr.write_barrier_descr if gcrootmap and wbdescr: - ofs = self.cpu.get_baseofs_of_frame_field() # frame never uses card marking, so we enforce this is not # an array - self._write_barrier_fastpah(mc, wbdescr, [ebp], array=False, - extra_ofs=-ofs) + self._write_barrier_fastpath(mc, wbdescr, [ebp], array=False, + is_frame=True) def call(self, addr, args, res): self._emit_call(imm(addr), args) @@ -1951,9 +1934,14 @@ def setup_failure_recovery(self): self.failure_recovery_code = [0, 0, 0, 0] - def _push_all_regs_to_frame(self, mc, ignored_regs, withfloats): + def _push_all_regs_to_frame(self, mc, ignored_regs, withfloats, + callee_only=False): # Push all general purpose registers - for i, gpr in enumerate(gpr_reg_mgr_cls.all_regs): + if callee_only: + regs = gpr_reg_mgr_cls.save_around_call_regs + else: + regs = gpr_reg_mgr_cls.all_regs + for i, gpr in enumerate(regs): if gpr not in ignored_regs: mc.MOV_br(i * WORD, gpr.value) if withfloats: @@ -1962,9 +1950,14 @@ for i in range(len(xmm_reg_mgr_cls.all_regs)): mc.MOVSD_bx((ofs + i) * WORD, i) - def _pop_all_regs_from_frame(self, mc, ignored_regs, withfloats): + def _pop_all_regs_from_frame(self, mc, ignored_regs, withfloats, + callee_only=False): # Pop all general purpose registers - for i, gpr in enumerate(gpr_reg_mgr_cls.all_regs): + if callee_only: + regs = gpr_reg_mgr_cls.save_around_call_regs + else: + regs = gpr_reg_mgr_cls.all_regs + for i, gpr in enumerate(regs): if gpr not in ignored_regs: mc.MOV_rb(gpr.value, i * WORD) if withfloats: @@ -2322,8 +2315,8 @@ self.mc.overwrite(jmp_location - 1, chr(offset)) self._emit_guard_not_forced(guard_token) - def _write_barrier_fastpah(self, mc, descr, arglocs, array=False, - extra_ofs=0): + def _write_barrier_fastpath(self, mc, descr, arglocs, array=False, + is_frame=False): # Write code equivalent to write_barrier() in the GC: it checks # a flag in the object at arglocs[0], and if set, it calls a # helper piece of assembler. The latter saves registers as needed @@ -2343,11 +2336,13 @@ mask = descr.jit_wb_if_flag_singlebyte | -0x80 # loc_base = arglocs[0] - if loc_base is ebp: - loc = raw_stack(descr.jit_wb_if_flag_byteofs + extra_ofs) + if is_frame: + assert loc_base is ebp + extra_ofs = self.cpu.get_baseofs_of_frame_field() + loc = raw_stack(descr.jit_wb_if_flag_byteofs - extra_ofs) + loc_base = raw_stack(-extra_ofs) else: - loc = addr_add_const(loc_base, descr.jit_wb_if_flag_byteofs + - extra_ofs) + loc = addr_add_const(loc_base, descr.jit_wb_if_flag_byteofs) mc.TEST8(loc, imm(mask)) mc.J_il8(rx86.Conditions['Z'], 0) # patched later jz_location = mc.get_relative_pos() @@ -2433,10 +2428,11 @@ mc.overwrite(jz_location-1, chr(offset)) def genop_discard_cond_call_gc_wb(self, op, arglocs): - self._write_barrier_fastpah(self.mc, op.getdescr(), arglocs) + self._write_barrier_fastpath(self.mc, op.getdescr(), arglocs) def genop_discard_cond_call_gc_wb_array(self, op, arglocs): - self._write_barrier_fastpah(self.mc, op.getdescr(), arglocs, array=True) + self._write_barrier_fastpath(self.mc, op.getdescr(), arglocs, + array=True) def not_implemented_op_discard(self, op, arglocs): not_implemented("not implemented operation: %s" % op.getopname()) diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -38,18 +38,6 @@ save_around_call_regs = [eax, edx, ecx] frame_reg = ebp - REGLOC_TO_GCROOTMAP_REG_INDEX = { - ebx: 1, - esi: 2, - edi: 3, - } - #REGLOC_TO_COPY_AREA_OFS = { - # ecx: MY_COPY_OF_REGS + 0 * WORD, - # ebx: MY_COPY_OF_REGS + 1 * WORD, - # esi: MY_COPY_OF_REGS + 2 * WORD, - # edi: MY_COPY_OF_REGS + 3 * WORD, - #} - def call_result_location(self, v): return eax diff --git a/rpython/jit/backend/x86/test/test_gc_integration.py b/rpython/jit/backend/x86/test/test_gc_integration.py --- a/rpython/jit/backend/x86/test/test_gc_integration.py +++ b/rpython/jit/backend/x86/test/test_gc_integration.py @@ -5,7 +5,7 @@ from rpython.jit.metainterp.history import TargetToken, BasicFinalDescr,\ JitCellToken, BasicFailDescr, AbstractDescr from rpython.jit.backend.llsupport.gc import GcLLDescription, GcLLDescr_boehm,\ - GcLLDescr_framework, GcCache + GcLLDescr_framework, GcCache, JitFrameDescrs from rpython.jit.backend.detect_cpu import getcpuclass from rpython.jit.backend.x86.arch import WORD, JITFRAME_FIXED_SIZE from rpython.jit.backend.llsupport import jitframe @@ -327,8 +327,7 @@ def __init__(self, gc_ll_descr): def write_barrier(x): - import pdb - pdb.set_trace() + gc_ll_descr.write_barrier_on_frame_called = True self.write_barrier_fn = llhelper_args(write_barrier, [lltype.Signed], lltype.Void) @@ -336,6 +335,29 @@ def get_write_barrier_fn(self, cpu): return self.write_barrier_fn +# a copy of JITFRAM that has 'hdr' field for tests + +def jitframe_allocate(frame_info): + frame = lltype.malloc(JITFRAME, frame_info.jfi_frame_depth, zero=True) + frame.jf_frame_info = frame_info + return frame + +JITFRAME = lltype.GcStruct( + 'JITFRAME', + ('hdr', lltype.Signed), + ('jf_frame_info', lltype.Ptr(jitframe.JITFRAMEINFO)), + ('jf_descr', llmemory.GCREF), + ('jf_guard_exc', llmemory.GCREF), + ('jf_gcmap', lltype.Ptr(jitframe.GCMAP)), + ('jf_gc_trace_state', lltype.Signed), + ('jf_frame', lltype.Array(lltype.Signed)), + adtmeths = { + 'allocate': jitframe_allocate, + }, +) + +JITFRAMEPTR = lltype.Ptr(JITFRAME) + class GCDescrShadowstackDirect(GcLLDescr_framework): layoutbuilder = None @@ -387,15 +409,26 @@ pos.sort() for i in range(len(gcmap)): assert col[i] + start == pos[i] + self.frames[-1].hdr |= 1 self.init_nursery() def malloc_jitframe(self, frame_info): """ Allocate a new frame, overwritten by tests """ - frame = jitframe.JITFRAME.allocate(frame_info) + frame = JITFRAME.allocate(frame_info) self.frames.append(frame) return frame + def getframedescrs(self, cpu): + descrs = JitFrameDescrs() + descrs.arraydescr = cpu.arraydescrof(JITFRAME) + for name in ['jf_descr', 'jf_guard_exc', + 'jf_frame_info', 'jf_gcmap']: + setattr(descrs, name, cpu.fielddescrof(JITFRAME, name)) + descrs.jfi_frame_depth = cpu.fielddescrof(jitframe.JITFRAMEINFO, + 'jfi_frame_depth') + return descrs + def do_write_barrier(self, gcref_struct, gcref_newptr): pass @@ -450,11 +483,11 @@ def check(i): assert cpu.gc_ll_descr.gcrootmap.stack[0] == i - ofs - frame = rffi.cast(jitframe.JITFRAMEPTR, i - ofs) + frame = rffi.cast(JITFRAMEPTR, i - ofs) assert len(frame.jf_frame) == JITFRAME_FIXED_SIZE + 4 # we "collect" frames.append(frame) - new_frame = jitframe.JITFRAME.allocate(frame.jf_frame_info) + new_frame = JITFRAME.allocate(frame.jf_frame_info) gcmap = unpack_gcmap(frame) assert gcmap == [28, 29, 30] for item, s in zip(gcmap, new_items): @@ -465,7 +498,7 @@ def check2(i): assert cpu.gc_ll_descr.gcrootmap.stack[0] == i - ofs - frame = rffi.cast(jitframe.JITFRAMEPTR, i - ofs) + frame = rffi.cast(JITFRAMEPTR, i - ofs) assert frame == frames[1] assert frame != frames[0] @@ -497,8 +530,8 @@ new_items = [lltype.malloc(S), lltype.malloc(S), lltype.malloc(S)] new_items[0].x = new_items[2] frame = cpu.execute_token(token, p0, p1, p2) - frame = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, frame) - gcmap = unpack_gcmap(lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, frame)) + frame = lltype.cast_opaque_ptr(JITFRAMEPTR, frame) + gcmap = unpack_gcmap(lltype.cast_opaque_ptr(JITFRAMEPTR, frame)) assert len(gcmap) == 1 assert gcmap[0] < 29 item = rffi.cast(lltype.Ptr(S), frame.jf_frame[gcmap[0]]) @@ -526,7 +559,8 @@ cpu.compile_loop(loop.inputargs, loop.operations, token) frame = cpu.execute_token(token) # now we should be able to track everything from the frame - frame = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, frame) + frame = lltype.cast_opaque_ptr(JITFRAMEPTR, frame) thing = frame.jf_frame[unpack_gcmap(frame)[0]] assert thing == rffi.cast(lltype.Signed, cpu.gc_ll_descr.nursery) assert cpu.gc_ll_descr.nursery_ptrs[0] == thing + sizeof.size + assert cpu.gc_ll_descr.write_barrier_on_frame_called From noreply at buildbot.pypy.org Fri Jan 25 12:30:39 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 25 Jan 2013 12:30:39 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: tests and fixes Message-ID: <20130125113039.C55D91C134E@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60444:6d54d21d2cbe Date: 2013-01-25 13:30 +0200 http://bitbucket.org/pypy/pypy/changeset/6d54d21d2cbe/ Log: tests and fixes diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -241,8 +241,7 @@ mc.LEA_rb(esi.value, -base_ofs) mc.SUB_ri(esp.value, 16 - WORD) mc.CALL(imm(addr)) - # Note: we check this after the code above, just because the code - # above is more than 127 bytes on 64-bits... + mc.ADD_ri(esp.value, 16 - WORD) mc.TEST_rr(eax.value, eax.value) mc.J_il(rx86.Conditions['Z'], 0xfffff) # patched later jz_location = mc.get_relative_pos() @@ -250,7 +249,6 @@ nursery_free_adr = self.cpu.gc_ll_descr.get_nursery_free_addr() self._reload_frame_if_necessary(mc) self._pop_all_regs_from_frame(mc, [eax, edi], self.cpu.supports_floats) - mc.LEA_rs(esp.value, 16 - WORD) mc.MOV(edi, heap(nursery_free_adr)) # load this in EDX # clear the gc pattern mc.MOV_bi(ofs, 0) @@ -387,15 +385,20 @@ # if not for_frame: self._push_all_regs_to_frame(mc, [], withfloats, callee_only=True) + if IS_X86_32: + XXX + mc.MOV_rs(eax.value, WORD) + mc.MOV_sr(0, eax.value) + elif IS_X86_64: + mc.MOV_rs(edi.value, WORD) else: - mc.MOV_sr(0, eax.value) + # we're possibly called from the slowpath of malloc, so we have + # one extra CALL on the stack, but one less PUSH, + # save to store stuff 2 locations away on the stack. + mc.MOV_sr(3*WORD, eax.value) + base_ofs = self.cpu.get_baseofs_of_frame_field() + mc.LEA_rb(edi.value, -base_ofs) - if IS_X86_32: - XXX - mc.MOV_rs(eax.value, WORD) - mc.MOV_sr(0, eax.value) - elif IS_X86_64: - mc.MOV_rs(edi.value, WORD) mc.CALL(imm(func)) # if withcards: @@ -409,14 +412,15 @@ if not for_frame: self._pop_all_regs_from_frame(mc, [], withfloats, callee_only=True) + mc.RET16_i(WORD) else: - mc.MOV_rs(eax.value, 0) + mc.MOV_rs(eax.value, 3 * WORD) + mc.RET() # # ADD esp, correct_esp_by --- but cannot use ADD, because # of its effects on the CPU flags #mc.LEA_rs(esp.value, WORD) - mc.RET16_i(WORD) # rawstart = mc.materialize(self.cpu.asmmemmgr, []) if for_frame: @@ -2340,7 +2344,6 @@ assert loc_base is ebp extra_ofs = self.cpu.get_baseofs_of_frame_field() loc = raw_stack(descr.jit_wb_if_flag_byteofs - extra_ofs) - loc_base = raw_stack(-extra_ofs) else: loc = addr_add_const(loc_base, descr.jit_wb_if_flag_byteofs) mc.TEST8(loc, imm(mask)) @@ -2361,7 +2364,9 @@ # argument the address of the structure we are writing into # (the first argument to COND_CALL_GC_WB). helper_num = card_marking - if self._regalloc is not None and self._regalloc.xrm.reg_bindings: + if is_frame: + helper_num = 4 + elif self._regalloc is not None and self._regalloc.xrm.reg_bindings: helper_num += 2 if self.wb_slowpath[helper_num] == 0: # tests only assert not we_are_translated() @@ -2370,7 +2375,8 @@ bool(self._regalloc.xrm.reg_bindings)) assert self.wb_slowpath[helper_num] != 0 # - mc.PUSH(loc_base) + if not is_frame: + mc.PUSH(loc_base) mc.CALL(imm(self.wb_slowpath[helper_num])) if card_marking: diff --git a/rpython/jit/backend/x86/test/test_gc_integration.py b/rpython/jit/backend/x86/test/test_gc_integration.py --- a/rpython/jit/backend/x86/test/test_gc_integration.py +++ b/rpython/jit/backend/x86/test/test_gc_integration.py @@ -326,8 +326,8 @@ jit_wb_if_flag_singlebyte = 1 def __init__(self, gc_ll_descr): - def write_barrier(x): - gc_ll_descr.write_barrier_on_frame_called = True + def write_barrier(frame): + gc_ll_descr.write_barrier_on_frame_called = frame self.write_barrier_fn = llhelper_args(write_barrier, [lltype.Signed], lltype.Void) @@ -563,4 +563,4 @@ thing = frame.jf_frame[unpack_gcmap(frame)[0]] assert thing == rffi.cast(lltype.Signed, cpu.gc_ll_descr.nursery) assert cpu.gc_ll_descr.nursery_ptrs[0] == thing + sizeof.size - assert cpu.gc_ll_descr.write_barrier_on_frame_called + assert rffi.cast(JITFRAMEPTR, cpu.gc_ll_descr.write_barrier_on_frame_called) == frame From noreply at buildbot.pypy.org Fri Jan 25 13:03:13 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 25 Jan 2013 13:03:13 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: Remove redundant-by-now test Message-ID: <20130125120313.4FD851C130F@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60445:c9b557589c3c Date: 2013-01-25 14:02 +0200 http://bitbucket.org/pypy/pypy/changeset/c9b557589c3c/ Log: Remove redundant-by-now test diff --git a/rpython/jit/backend/x86/test/test_zrpy_platform.py b/rpython/jit/backend/x86/test/test_zrpy_platform.py deleted file mode 100644 --- a/rpython/jit/backend/x86/test/test_zrpy_platform.py +++ /dev/null @@ -1,95 +0,0 @@ -import sys -import py -from rpython.rtyper.lltypesystem import lltype, rffi -from rpython.rlib import jit -from rpython.jit.backend.x86.test.test_zrpy_gc import compile_and_run - -from rpython.jit.metainterp import pyjitpl - -#pyjitpl.DEBUG = 4 - - -def test_stack_alignment(): - # we can detect gcc 4.5.0 to test those as well - if sys.platform != 'darwin': - py.test.skip("tests darwin only stack alignment requirements") - - externs = [""" -extern void check0(); -extern void check1(int); -extern void check2(int, int); -extern void check3(int, int, int); -"""] - c_source = r""" -#include - -void check0() { - void *ip = __builtin_return_address(0); - void *fp = __builtin_frame_address(0); - printf("0 %p %p %u\n", ip, fp, (unsigned)fp % 16); -} - -void check1(int a) { - void *ip = __builtin_return_address(0); - void *fp = __builtin_frame_address(0); - printf("1 %p %p %u\n", ip, fp, (unsigned)fp % 16); -} - -void check2(int a, int b) { - void *ip = __builtin_return_address(0); - void *fp = __builtin_frame_address(0); - printf("2 %p %p %u\n", ip, fp, (unsigned)fp % 16); -} - -void check3(int a, int b, int c) { - void *ip = __builtin_return_address(0); - void *fp = __builtin_frame_address(0); - printf("3 %p %p %u\n", ip, fp, (unsigned)fp % 16); -} -""" - - eci = rffi.ExternalCompilationInfo(separate_module_sources=[c_source], - post_include_bits = externs, - # not ideal, would like to apply this only to the checkX - # functions - compile_extra=["-fno-omit-frame-pointer"]) - - check0 = rffi.llexternal('check0', [], lltype.Void, - compilation_info=eci, - _nowrapper=True) - check1 = rffi.llexternal('check1', [lltype.Signed], lltype.Void, - compilation_info=eci, - _nowrapper=True) - check2 = rffi.llexternal('check2', [lltype.Signed, lltype.Signed], - lltype.Void, - compilation_info=eci, - _nowrapper=True) - - check3 = rffi.llexternal('check3', [lltype.Signed, lltype.Signed, - lltype.Signed], - lltype.Void, - compilation_info=eci, - _nowrapper=True) - - myjitdriver = jit.JitDriver(greens = [], reds = ['n']) - - def entrypoint(argv): - jit.set_param(myjitdriver, 'threshold', 2) - jit.set_param(myjitdriver, 'trace_eagerness', 0) - n = 16 - while n > 0: - myjitdriver.can_enter_jit(n=n) - myjitdriver.jit_merge_point(n=n) - n -= 1 - check0() - check1(0) - check2(0, 1) - check3(0, 1, 2) - return 0 - - output = compile_and_run(entrypoint, 'boehm', jit=True) - for line in output.splitlines(): - print line - # ret ip + bp == 8 - assert int(line.split()[-1]) == 8 - From noreply at buildbot.pypy.org Fri Jan 25 13:07:08 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 25 Jan 2013 13:07:08 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: reshuffle tests to make them run faster and more in parallel Message-ID: <20130125120708.6947E1C134E@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60446:009a70442a4b Date: 2013-01-25 14:06 +0200 http://bitbucket.org/pypy/pypy/changeset/009a70442a4b/ Log: reshuffle tests to make them run faster and more in parallel diff --git a/rpython/jit/backend/x86/test/test_zrpy_gc.py b/rpython/jit/backend/x86/test/test_zrpy_gc.py --- a/rpython/jit/backend/x86/test/test_zrpy_gc.py +++ b/rpython/jit/backend/x86/test/test_zrpy_gc.py @@ -5,8 +5,7 @@ """ import weakref -import py, os -from rpython.annotator import policy as annpolicy +import os from rpython.rlib import rgc from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.rlib.jit import JitDriver, dont_look_inside @@ -14,6 +13,9 @@ from rpython.jit.backend.llsupport.gc import GcLLDescr_framework from rpython.tool.udir import udir from rpython.config.translationoption import DEFL_GC +from rpython.rlib.libffi import CDLL, types, ArgChain, clibffi +from rpython.rtyper.annlowlevel import llhelper +from rpython.rtyper.lltypesystem.ll2ctypes import libc_name class X(object): def __init__(self, x=0): @@ -151,7 +153,6 @@ class BaseFrameworkTests(object): - compile_kwds = {} def setup_class(cls): funcs = [] @@ -203,7 +204,7 @@ GcLLDescr_framework.DEBUG = True cls.cbuilder = compile(get_entry(allfuncs), DEFL_GC, gcrootfinder=cls.gcrootfinder, jit=True, - **cls.compile_kwds) + thread=True) finally: GcLLDescr_framework.DEBUG = OLD_DEBUG @@ -789,9 +790,94 @@ def test_compile_framework_minimal_size_in_nursery(self): self.run('compile_framework_minimal_size_in_nursery') + def define_simple_call_release_gil(self): + class Glob: + pass + glob = Glob() + # + def f42(n): + c_strchr = glob.c_strchr + raw = rffi.str2charp("foobar" + chr((n & 63) + 32)) + argchain = ArgChain() + argchain = argchain.arg(rffi.cast(lltype.Signed, raw)) + argchain = argchain.arg(rffi.cast(rffi.INT, ord('b'))) + res = c_strchr.call(argchain, rffi.CCHARP) + check(rffi.charp2str(res) == "bar" + chr((n & 63) + 32)) + rffi.free_charp(raw) + # + def before(n, x): + libc = CDLL(libc_name) + c_strchr = libc.getpointer('strchr', [types.pointer, types.sint], + types.pointer) + glob.c_strchr = c_strchr + return (n, None, None, None, None, None, + None, None, None, None, None, None) + # + def f(n, x, *args): + f42(n) + n -= 1 + return (n, x) + args + return before, f, None + + def test_simple_call_release_gil(self): + self.run('simple_call_release_gil') + + def define_close_stack(self): + # + class Glob(object): + pass + glob = Glob() + class X(object): + pass + # + def callback(p1, p2): + for i in range(100): + glob.lst.append(X()) + return rffi.cast(rffi.INT, 1) + CALLBACK = lltype.Ptr(lltype.FuncType([lltype.Signed, + lltype.Signed], rffi.INT)) + # + @dont_look_inside + def alloc1(): + return llmemory.raw_malloc(16) + @dont_look_inside + def free1(p): + llmemory.raw_free(p) + # + def f42(): + length = len(glob.lst) + c_qsort = glob.c_qsort + raw = alloc1() + fn = llhelper(CALLBACK, rffi._make_wrapper_for(CALLBACK, callback)) + argchain = ArgChain() + argchain = argchain.arg(rffi.cast(lltype.Signed, raw)) + argchain = argchain.arg(rffi.cast(rffi.SIZE_T, 2)) + argchain = argchain.arg(rffi.cast(rffi.SIZE_T, 8)) + argchain = argchain.arg(rffi.cast(lltype.Signed, fn)) + c_qsort.call(argchain, lltype.Void) + free1(raw) + check(len(glob.lst) > length) + del glob.lst[:] + # + def before(n, x): + libc = CDLL(libc_name) + types_size_t = clibffi.cast_type_to_ffitype(rffi.SIZE_T) + c_qsort = libc.getpointer('qsort', [types.pointer, types_size_t, + types_size_t, types.pointer], + types.void) + glob.c_qsort = c_qsort + glob.lst = [] + return (n, None, None, None, None, None, + None, None, None, None, None, None) + # + def f(n, x, *args): + f42() + n -= 1 + return (n, x) + args + return before, f, None + + def test_close_stack(self): + self.run('close_stack') class TestShadowStack(CompileFrameworkTests): gcrootfinder = "shadowstack" - -class TestAsmGcc(CompileFrameworkTests): - gcrootfinder = "asmgcc" diff --git a/rpython/jit/backend/x86/test/test_zrpy_gcasmgcc.py b/rpython/jit/backend/x86/test/test_zrpy_gcasmgcc.py new file mode 100644 --- /dev/null +++ b/rpython/jit/backend/x86/test/test_zrpy_gcasmgcc.py @@ -0,0 +1,5 @@ + +from rpython.jit.backend.x86.test.test_zrpy_gc import CompileFrameworkTests + +class TestAsmGcc(CompileFrameworkTests): + gcrootfinder = "asmgcc" diff --git a/rpython/jit/backend/x86/test/test_zrpy_releasegil.py b/rpython/jit/backend/x86/test/test_zrpy_releasegil.py --- a/rpython/jit/backend/x86/test/test_zrpy_releasegil.py +++ b/rpython/jit/backend/x86/test/test_zrpy_releasegil.py @@ -13,95 +13,6 @@ class ReleaseGILTests(BaseFrameworkTests): compile_kwds = dict(enable_opts=ALL_OPTS_NAMES, thread=True) - def define_simple(self): - class Glob: - pass - glob = Glob() - # - def f42(n): - c_strchr = glob.c_strchr - raw = rffi.str2charp("foobar" + chr((n & 63) + 32)) - argchain = ArgChain() - argchain = argchain.arg(rffi.cast(lltype.Signed, raw)) - argchain = argchain.arg(rffi.cast(rffi.INT, ord('b'))) - res = c_strchr.call(argchain, rffi.CCHARP) - check(rffi.charp2str(res) == "bar" + chr((n & 63) + 32)) - rffi.free_charp(raw) - # - def before(n, x): - libc = CDLL(libc_name) - c_strchr = libc.getpointer('strchr', [types.pointer, types.sint], - types.pointer) - glob.c_strchr = c_strchr - return (n, None, None, None, None, None, - None, None, None, None, None, None) - # - def f(n, x, *args): - f42(n) - n -= 1 - return (n, x) + args - return before, f, None - - def test_simple(self): - self.run('simple') - - def define_close_stack(self): - # - class Glob(object): - pass - glob = Glob() - class X(object): - pass - # - def callback(p1, p2): - for i in range(100): - glob.lst.append(X()) - return rffi.cast(rffi.INT, 1) - CALLBACK = lltype.Ptr(lltype.FuncType([lltype.Signed, - lltype.Signed], rffi.INT)) - # - @dont_look_inside - def alloc1(): - return llmemory.raw_malloc(16) - @dont_look_inside - def free1(p): - llmemory.raw_free(p) - # - def f42(): - length = len(glob.lst) - c_qsort = glob.c_qsort - raw = alloc1() - fn = llhelper(CALLBACK, rffi._make_wrapper_for(CALLBACK, callback)) - argchain = ArgChain() - argchain = argchain.arg(rffi.cast(lltype.Signed, raw)) - argchain = argchain.arg(rffi.cast(rffi.SIZE_T, 2)) - argchain = argchain.arg(rffi.cast(rffi.SIZE_T, 8)) - argchain = argchain.arg(rffi.cast(lltype.Signed, fn)) - c_qsort.call(argchain, lltype.Void) - free1(raw) - check(len(glob.lst) > length) - del glob.lst[:] - # - def before(n, x): - libc = CDLL(libc_name) - types_size_t = clibffi.cast_type_to_ffitype(rffi.SIZE_T) - c_qsort = libc.getpointer('qsort', [types.pointer, types_size_t, - types_size_t, types.pointer], - types.void) - glob.c_qsort = c_qsort - glob.lst = [] - return (n, None, None, None, None, None, - None, None, None, None, None, None) - # - def f(n, x, *args): - f42() - n -= 1 - return (n, x) + args - return before, f, None - - def test_close_stack(self): - self.run('close_stack') - class TestShadowStack(ReleaseGILTests): gcrootfinder = "shadowstack" From noreply at buildbot.pypy.org Fri Jan 25 13:16:09 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 25 Jan 2013 13:16:09 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: oops that might not fit in 32bit Message-ID: <20130125121609.B6F001C0041@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60447:df874d62b661 Date: 2013-01-25 14:15 +0200 http://bitbucket.org/pypy/pypy/changeset/df874d62b661/ Log: oops that might not fit in 32bit diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -2093,8 +2093,8 @@ def _store_force_index(self, guard_op): faildescr = guard_op.getdescr() ofs = self.cpu.get_ofs_of_frame_field('jf_force_descr') - self.mc.MOV_bi(ofs, rffi.cast(lltype.Signed, - cast_instance_to_gcref(faildescr))) + self.mc.MOV(ofs, imm(rffi.cast(lltype.Signed, + cast_instance_to_gcref(faildescr)))) def _emit_guard_not_forced(self, guard_token): ofs = self.cpu.get_ofs_of_frame_field('jf_descr') From noreply at buildbot.pypy.org Fri Jan 25 13:19:40 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 25 Jan 2013 13:19:40 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: er Message-ID: <20130125121940.39DF11C0041@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60448:102e62882ae6 Date: 2013-01-25 14:19 +0200 http://bitbucket.org/pypy/pypy/changeset/102e62882ae6/ Log: er diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -2093,8 +2093,8 @@ def _store_force_index(self, guard_op): faildescr = guard_op.getdescr() ofs = self.cpu.get_ofs_of_frame_field('jf_force_descr') - self.mc.MOV(ofs, imm(rffi.cast(lltype.Signed, - cast_instance_to_gcref(faildescr)))) + self.mc.MOV(raw_stack(ofs), imm(rffi.cast(lltype.Signed, + cast_instance_to_gcref(faildescr)))) def _emit_guard_not_forced(self, guard_token): ofs = self.cpu.get_ofs_of_frame_field('jf_descr') From noreply at buildbot.pypy.org Fri Jan 25 13:37:08 2013 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Fri, 25 Jan 2013 13:37:08 +0100 (CET) Subject: [pypy-commit] pypy pytest: a few more config.option fixes Message-ID: <20130125123708.1CF2B1C05D7@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: pytest Changeset: r60449:04895e55cede Date: 2013-01-25 13:35 +0100 http://bitbucket.org/pypy/pypy/changeset/04895e55cede/ Log: a few more config.option fixes diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py --- a/pypy/module/_file/test/test_file.py +++ b/pypy/module/_file/test/test_file.py @@ -442,12 +442,12 @@ assert s == 'bar\n' def test_flush_at_exit(): - from pypy import conftest + import py from pypy.tool.option import make_config, make_objspace from rpython.tool.udir import udir tmpfile = udir.join('test_flush_at_exit') - config = make_config(conftest.option) + config = make_config(py.test.config.option) space = make_objspace(config) space.appexec([space.wrap(str(tmpfile))], """(tmpfile): f = open(tmpfile, 'w') diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -1,5 +1,6 @@ from rpython.tool.udir import udir import os +import py class AppTestFileIO: spaceconfig = dict(usemodules=['_io']) @@ -161,12 +162,11 @@ assert repr(f) == "<_io.FileIO [closed]>" def test_flush_at_exit(): - from pypy import conftest from pypy.tool.option import make_config, make_objspace from rpython.tool.udir import udir tmpfile = udir.join('test_flush_at_exit') - config = make_config(conftest.option) + config = make_config(py.test.config.option) space = make_objspace(config) space.appexec([space.wrap(str(tmpfile))], """(tmpfile): import io @@ -182,7 +182,7 @@ from pypy import conftest from pypy.tool.option import make_config, make_objspace - config = make_config(conftest.option) + config = make_config(py.test.config.option) space = make_objspace(config) space.appexec([], """(): import io From noreply at buildbot.pypy.org Fri Jan 25 13:37:10 2013 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Fri, 25 Jan 2013 13:37:10 +0100 (CET) Subject: [pypy-commit] pypy pytest: merge from default Message-ID: <20130125123710.168EB1C1055@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: pytest Changeset: r60450:3395bce3ab2b Date: 2013-01-25 13:36 +0100 http://bitbucket.org/pypy/pypy/changeset/3395bce3ab2b/ Log: merge from default diff --git a/lib-python/2.7/test/test_iterlen.py b/lib-python/2.7/test/test_iterlen.py --- a/lib-python/2.7/test/test_iterlen.py +++ b/lib-python/2.7/test/test_iterlen.py @@ -94,7 +94,11 @@ def test_no_len_for_infinite_repeat(self): # The repeat() object can also be infinite - self.assertRaises(TypeError, len, repeat(None)) + if test_support.check_impl_detail(pypy=True): + # 3.4 (PEP 424) behavior + self.assertEqual(len(repeat(None)), NotImplemented) + else: + self.assertRaises(TypeError, len, repeat(None)) class TestXrange(TestInvariantWithoutMutations): @@ -230,6 +234,7 @@ self.assertRaises(RuntimeError, b.extend, BadLen()) self.assertRaises(RuntimeError, b.extend, BadLengthHint()) + @test_support.impl_detail("PEP 424 disallows None results", pypy=False) def test_invalid_hint(self): # Make sure an invalid result doesn't muck-up the works self.assertEqual(list(NoneLengthHint()), list(range(10))) diff --git a/lib-python/conftest.py b/lib-python/conftest.py --- a/lib-python/conftest.py +++ b/lib-python/conftest.py @@ -17,8 +17,8 @@ from pypy.conftest import option as pypy_option from pypy.tool.pytest import appsupport -from pypy.tool.pytest.confpath import pypydir, testdir, testresultdir -from pypy.config.parse import parse_info +from pypy.tool.pytest.confpath import pypydir, rpythondir, testdir, testresultdir +from rpython.config.parse import parse_info pytest_plugins = "resultlog", rsyncdirs = ['.', '../pypy/'] @@ -40,7 +40,7 @@ dest="unittest_filter", help="Similar to -k, XXX") def gettimeout(timeout): - from test import pystone + from rpython.translator.test import rpystone if timeout.endswith('mp'): megapystone = float(timeout[:-2]) t, stone = pystone.Proc0(10000) @@ -281,7 +281,7 @@ RegrTest('test_ioctl.py'), RegrTest('test_isinstance.py', core=True), RegrTest('test_iter.py', core=True), - RegrTest('test_iterlen.py', skip="undocumented internal API behavior __length_hint__"), + RegrTest('test_iterlen.py', core=True, usemodules="_collections itertools"), RegrTest('test_itertools.py', core=True, usemodules="itertools struct"), RegrTest('test_json.py'), RegrTest('test_kqueue.py'), @@ -584,7 +584,7 @@ watchdog_name = 'watchdog_nt.py' else: watchdog_name = 'watchdog.py' - watchdog_script = pypydir.join('tool', watchdog_name) + watchdog_script = rpythondir.join('tool', watchdog_name) regr_script = pypydir.join('tool', 'pytest', 'run-script', 'regrverbose.py') diff --git a/pypy/doc/arm.rst b/pypy/doc/arm.rst --- a/pypy/doc/arm.rst +++ b/pypy/doc/arm.rst @@ -130,15 +130,7 @@ export SB2=/srv/chroot/precise_arm export SB2OPT='-t ARM' -Once this is set, you can call the translator - -:: - - pypy ~/path_to_pypy_checkout/pypy/translator/goal/translate.py -O1 --platform=arm target.py - -If everything worked correctly this should yield an ARM binary. Running this binary in the ARM chroot or on an ARM device should produce the output ``"Hello World"``. - -.. _`this`: +Once this is set, you can call the translator. For example save this file :: @@ -148,3 +140,22 @@ def target(*args): return main, None + +and call the translator + +:: + + pypy ~/path_to_pypy_checkout/pypy/translator/goal/translate.py -O1 --platform=arm target.py + +If everything worked correctly this should yield an ARM binary. Running this binary in the ARM chroot or on an ARM device should produce the output ``"Hello World"``. + +To translate the full python pypy interpreter with a jit, you can cd into pypy/goal and call + +:: + + pypy /rpython/translator/goal/translate.py -Ojit --platform=arm --gcrootfinder=shadowstack --jit-backend=arm targetpypystandalone.py + +The gcrootfinder option is needed to work around `issue 1377`_ and the jit-backend works around `issue 1376`_ + +.. _`issue 1377`: https://bugs.pypy.org/issue1377 +.. _`issue 1376`: https://bugs.pypy.org/issue1376 diff --git a/pypy/doc/getting-started-python.rst b/pypy/doc/getting-started-python.rst --- a/pypy/doc/getting-started-python.rst +++ b/pypy/doc/getting-started-python.rst @@ -86,8 +86,8 @@ 4. Run:: - cd pypy/translator/goal - python translate.py --opt=jit targetpypystandalone.py + cd pypy/goal + python ../../rpython/bin/rpython --opt=jit targetpypystandalone.py possibly replacing ``--opt=jit`` with another `optimization level`_ of your choice like ``--opt=2`` if you do not want to include the JIT diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -5,6 +5,9 @@ .. this is a revision shortly after release-2.0-beta1 .. startrev: 0e6161a009c6 +.. branch: split-rpython +Split rpython and pypy into seperate directories + .. branch: callback-jit Callbacks from C are now better JITted @@ -40,3 +43,7 @@ .. branch: fix-lookinside-iff-oopspec Fixed the interaction between two internal tools for controlling the JIT. + +.. branch: inline-virtualref-2 +Better optimized certain types of frame accesses in the JIT, particularly +around exceptions that escape the function they were raised in. diff --git a/pypy/module/_collections/interp_deque.py b/pypy/module/_collections/interp_deque.py --- a/pypy/module/_collections/interp_deque.py +++ b/pypy/module/_collections/interp_deque.py @@ -521,7 +521,11 @@ return self.space.wrap(self.counter) def next(self): - self.deque.checklock(self.lock) + if self.lock is not self.deque.lock: + self.counter = 0 + raise OperationError( + self.space.w_RuntimeError, + self.space.wrap("deque mutated during iteration")) if self.counter == 0: raise OperationError(self.space.w_StopIteration, self.space.w_None) self.counter -= 1 @@ -560,7 +564,11 @@ return self.space.wrap(self.counter) def next(self): - self.deque.checklock(self.lock) + if self.lock is not self.deque.lock: + self.counter = 0 + raise OperationError( + self.space.w_RuntimeError, + self.space.wrap("deque mutated during iteration")) if self.counter == 0: raise OperationError(self.space.w_StopIteration, self.space.w_None) self.counter -= 1 diff --git a/pypy/module/pypyjit/test_pypy_c/test_thread.py b/pypy/module/pypyjit/test_pypy_c/test_thread.py --- a/pypy/module/pypyjit/test_pypy_c/test_thread.py +++ b/pypy/module/pypyjit/test_pypy_c/test_thread.py @@ -26,3 +26,25 @@ --THREAD-TICK-- jump(..., descr=...) """) + + def test_tls(self): + def main(n): + import thread + local = thread._local() + local.x = 1 + i = 0 + while i < n: + i += local.x + return 0 + log = self.run(main, [500]) + assert round(log.result, 6) == round(main(500), 6) + loop, = log.loops_by_filename(self.filepath) + assert loop.match(""" + i53 = int_lt(i48, i27) + guard_true(i53, descr=...) + i54 = int_add_ovf(i48, i47) + guard_no_overflow(descr=...) + --TICK-- + i58 = arraylen_gc(p43, descr=...) + jump(p0, p1, p3, p5, p10, p12, p14, i54, i27, i47, p45, p43, descr=...) + """) diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py --- a/pypy/module/rctime/interp_time.py +++ b/pypy/module/rctime/interp_time.py @@ -448,7 +448,9 @@ raise OperationError(space.w_ValueError, space.wrap("year out of range")) - if rffi.getintfield(glob_buf, 'c_tm_wday') < 0: + # tm_wday does not need checking of its upper-bound since taking "% + # 7" in gettmarg() automatically restricts the range. + if rffi.getintfield(glob_buf, 'c_tm_wday') < -1: raise OperationError(space.w_ValueError, space.wrap("day of week out of range")) diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py --- a/pypy/module/rctime/test/test_rctime.py +++ b/pypy/module/rctime/test/test_rctime.py @@ -113,6 +113,9 @@ if os.name != 'nt': assert rctime.mktime(rctime.localtime(-1)) == -1 + res = rctime.mktime((2000, 1, 1, 0, 0, 0, -1, -1, -1)) + assert rctime.ctime(res) == 'Sat Jan 1 00:00:00 2000' + def test_asctime(self): import time as rctime rctime.asctime() diff --git a/pypy/module/thread/os_local.py b/pypy/module/thread/os_local.py --- a/pypy/module/thread/os_local.py +++ b/pypy/module/thread/os_local.py @@ -30,6 +30,9 @@ w_dict = space.newdict(instance=True) self.dicts[ec] = w_dict self._register_in_ec(ec) + # cache the last seen dict, works because we are protected by the GIL + self.last_dict = w_dict + self.last_ec = ec def _register_in_ec(self, ec): if not ec.space.config.translation.rweakref: @@ -60,10 +63,14 @@ def getdict(self, space): ec = space.getexecutioncontext() + if ec is self.last_ec: + return self.last_dict try: w_dict = self.dicts[ec] except KeyError: w_dict = self.create_new_dict(ec) + self.last_ec = ec + self.last_dict = w_dict return w_dict def descr_local__new__(space, w_subtype, __args__): @@ -91,3 +98,5 @@ local = wref() if local is not None: del local.dicts[ec] + local.last_dict = None + local.last_ec = None diff --git a/pypy/module/thread/test/test_local.py b/pypy/module/thread/test/test_local.py --- a/pypy/module/thread/test/test_local.py +++ b/pypy/module/thread/test/test_local.py @@ -108,3 +108,54 @@ gc.collect() assert done == ['ok', 'del'] done.append('shutdown') + +def test_local_caching(): + from pypy.module.thread.os_local import Local + class FakeSpace: + def getexecutioncontext(self): + return self.ec + + def getattr(*args): + pass + def call_obj_args(*args): + pass + def newdict(*args, **kwargs): + return {} + def wrap(self, obj): + return obj + def type(self, obj): + return type(obj) + class config: + class translation: + rweakref = True + + class FakeEC: + def __init__(self, space): + self.space = space + self._thread_local_objs = None + space = FakeSpace() + ec1 = FakeEC(space) + space.ec = ec1 + + l = Local(space, None) + assert l.last_dict is l.dicts[ec1] + assert l.last_ec is ec1 + d1 = l.getdict(space) + assert d1 is l.last_dict + + ec2 = space.ec = FakeEC(space) + d2 = l.getdict(space) + assert l.last_dict is d2 + assert d2 is l.dicts[ec2] + assert l.last_ec is ec2 + dicts = l.dicts + l.dicts = "nope" + assert l.getdict(space) is d2 + l.dicts = dicts + + space.ec = ec1 + assert l.getdict(space) is d1 + l.dicts = "nope" + assert l.getdict(space) is d1 + l.dicts = dicts + diff --git a/pypy/module/unicodedata/test/test_unicodedata.py b/pypy/module/unicodedata/test/test_unicodedata.py --- a/pypy/module/unicodedata/test/test_unicodedata.py +++ b/pypy/module/unicodedata/test/test_unicodedata.py @@ -82,8 +82,7 @@ import unicodedata raises(TypeError, unicodedata.normalize, 'x') - @py.test.mark.skipif("sys.maxunicode < 0x10ffff", - reason="requires a 'wide' python build.") + @py.test.mark.skipif("sys.maxunicode < 0x10ffff") def test_normalize_wide(self): import unicodedata assert unicodedata.normalize('NFC', u'\U000110a5\U000110ba') == u'\U000110ab' diff --git a/pypy/objspace/std/bytearraytype.py b/pypy/objspace/std/bytearraytype.py --- a/pypy/objspace/std/bytearraytype.py +++ b/pypy/objspace/std/bytearraytype.py @@ -15,6 +15,7 @@ str_splitlines, str_translate) from pypy.objspace.std.listtype import ( list_append, list_extend) +from rpython.rlib.objectmodel import newlist_hint, resizelist_hint bytearray_insert = SMM('insert', 3, @@ -87,8 +88,10 @@ return [c for c in string] # sequence of bytes - data = [] w_iter = space.iter(w_source) + length_hint = space.length_hint(w_source, 0) + data = newlist_hint(length_hint) + extended = 0 while True: try: w_item = space.next(w_iter) @@ -98,6 +101,9 @@ break value = getbytevalue(space, w_item) data.append(value) + extended += 1 + if extended < length_hint: + resizelist_hint(data, extended) return data def descr_bytearray__reduce__(space, w_self): diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py --- a/pypy/objspace/std/dictmultiobject.py +++ b/pypy/objspace/std/dictmultiobject.py @@ -54,6 +54,9 @@ # every module needs its own strategy, because the strategy stores # the version tag strategy = ModuleDictStrategy(space) + elif space.config.objspace.std.withmapdict and instance: + from pypy.objspace.std.mapdict import MapDictStrategy + strategy = space.fromcache(MapDictStrategy) elif instance or strdict or module: assert w_type is None @@ -349,7 +352,7 @@ self.pos = 0 def length(self): - if self.dictimplementation is not None: + if self.dictimplementation is not None and self.len != -1: return self.len - self.pos return 0 diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py --- a/pypy/objspace/std/mapdict.py +++ b/pypy/objspace/std/mapdict.py @@ -593,6 +593,9 @@ # ____________________________________________________________ # dict implementation +def get_terminator_for_dicts(space): + return DictTerminator(space, None) + class MapDictStrategy(DictStrategy): erase, unerase = rerased.new_erasing_pair("map") @@ -602,13 +605,19 @@ def __init__(self, space): self.space = space + def get_empty_storage(self): + w_result = Object() + terminator = self.space.fromcache(get_terminator_for_dicts) + w_result._init_empty(terminator) + return self.erase(w_result) + def switch_to_object_strategy(self, w_dict): w_obj = self.unerase(w_dict.dstorage) strategy = self.space.fromcache(ObjectDictStrategy) dict_w = strategy.unerase(strategy.get_empty_storage()) w_dict.strategy = strategy w_dict.dstorage = strategy.erase(dict_w) - assert w_obj.getdict(self.space) is w_dict + assert w_obj.getdict(self.space) is w_dict or w_obj._get_mapdict_map().terminator.w_cls is None materialize_r_dict(self.space, w_obj, dict_w) def getitem(self, w_dict, w_key): diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py --- a/pypy/objspace/std/setobject.py +++ b/pypy/objspace/std/setobject.py @@ -898,7 +898,7 @@ raise NotImplementedError def length(self): - if self.setimplementation is not None: + if self.setimplementation is not None and self.len != -1: return self.len - self.pos return 0 diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py --- a/pypy/objspace/std/test/test_dictmultiobject.py +++ b/pypy/objspace/std/test/test_dictmultiobject.py @@ -954,6 +954,7 @@ withcelldict = False withmethodcache = False withidentitydict = False + withmapdict = False FakeSpace.config = Config() diff --git a/pypy/objspace/std/test/test_lengthhint.py b/pypy/objspace/std/test/test_lengthhint.py --- a/pypy/objspace/std/test/test_lengthhint.py +++ b/pypy/objspace/std/test/test_lengthhint.py @@ -6,7 +6,7 @@ SIZE = 4 ITEMS = range(SIZE) - def _test_length_hint(self, w_obj): + def _test_length_hint(self, w_obj, w_mutate=None): space = self.space assert space.length_hint(w_obj, 8) == self.SIZE @@ -18,6 +18,13 @@ space.next(w_iter) assert space.length_hint(w_iter, 8) == self.SIZE - 1 + if w_mutate is not None: + # Test handling of collections that enforce length + # immutability during iteration + space.call_function(w_mutate) + space.raises_w(space.w_RuntimeError, space.next, w_iter) + assert space.length_hint(w_iter, 8) == 0 + def test_bytearray(self): space = self.space w_bytearray = space.call_function(space.w_bytearray, @@ -31,16 +38,20 @@ self._test_length_hint(w_dict) def test_dict_iterkeys(self): - w_iterkeys = self.space.appexec([], """(): - return dict.fromkeys(%r).iterkeys() - """ % self.ITEMS) - self._test_length_hint(w_iterkeys) + space = self.space + w_iterkeys, w_mutate = space.fixedview(space.appexec([], """(): + d = dict.fromkeys(%r) + return d.iterkeys(), d.popitem + """ % self.ITEMS), 2) + self._test_length_hint(w_iterkeys, w_mutate) def test_dict_values(self): - w_itervalues = self.space.appexec([], """(): - return dict.fromkeys(%r).itervalues() - """ % self.ITEMS) - self._test_length_hint(w_itervalues) + space = self.space + w_itervalues, w_mutate = space.fixedview(space.appexec([], """(): + d = dict.fromkeys(%r) + return d.itervalues(), d.popitem + """ % self.ITEMS), 2) + self._test_length_hint(w_itervalues, w_mutate) def test_frozenset(self): space = self.space @@ -50,7 +61,8 @@ def test_set(self): space = self.space w_set = space.call_function(space.w_set, space.wrap(self.ITEMS)) - self._test_length_hint(w_set) + w_mutate = space.getattr(w_set, space.wrap('pop')) + self._test_length_hint(w_set, w_mutate) def test_list(self): self._test_length_hint(self.space.wrap(self.ITEMS)) @@ -101,12 +113,19 @@ self._test_length_hint(W_Repeat(space, space.wrap(22), space.wrap(self.SIZE))) - def test_collections_deque(self): + def _setup_deque(self): space = self.space w_deque = W_Deque(space) space.call_method(w_deque, '__init__', space.wrap(self.ITEMS)) - self._test_length_hint(w_deque) - self._test_length_hint(w_deque.reviter()) + w_mutate = space.getattr(w_deque, space.wrap('pop')) + return w_deque, w_mutate + + def test_collections_deque(self): + self._test_length_hint(*self._setup_deque()) + + def test_collections_deque_rev(self): + w_deque, w_mutate = self._setup_deque() + self._test_length_hint(w_deque.reviter(), w_mutate) def test_default(self): space = self.space diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py --- a/pypy/objspace/std/test/test_mapdict.py +++ b/pypy/objspace/std/test/test_mapdict.py @@ -1,7 +1,17 @@ from pypy.objspace.std.test.test_dictmultiobject import FakeSpace, W_DictMultiObject from pypy.objspace.std.mapdict import * +class Config: + class objspace: + class std: + withsmalldicts = False + withcelldict = False + withmethodcache = False + withidentitydict = False + withmapdict = True + space = FakeSpace() +space.config = Config class Class(object): def __init__(self, hasdict=True): @@ -1061,3 +1071,11 @@ return A() """) assert w_dict.user_overridden_class + +def test_newdict_instance(): + w_dict = space.newdict(instance=True) + assert type(w_dict.strategy) is MapDictStrategy + +class TestMapDictImplementationUsingnewdict(BaseTestRDictImplementation): + StrategyClass = MapDictStrategy + # NB: the get_impl method is not overwritten here, as opposed to above diff --git a/pypy/pytest-A.cfg b/pypy/pytest-A.cfg --- a/pypy/pytest-A.cfg +++ b/pypy/pytest-A.cfg @@ -1,5 +1,5 @@ cherrypick = ['interpreter', 'objspace/test', 'objspace/std', 'module'] -interp = ['translator/goal/pypy-c'] +interp = ['goal/pypy-c'] test_driver = ['test_all.py', '-A'] diff --git a/pypy/test_all.py b/pypy/test_all.py --- a/pypy/test_all.py +++ b/pypy/test_all.py @@ -23,7 +23,8 @@ if len(sys.argv) == 1 and os.path.dirname(sys.argv[0]) in '.': print >> sys.stderr, __doc__ sys.exit(2) - + #Add toplevel repository dir to sys.path + sys.path.insert(0,os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) import pytest import pytest_cov sys.exit(pytest.main(plugins=[pytest_cov])) diff --git a/pypy/tool/pytest/confpath.py b/pypy/tool/pytest/confpath.py --- a/pypy/tool/pytest/confpath.py +++ b/pypy/tool/pytest/confpath.py @@ -1,8 +1,10 @@ import py import pypy +import rpython from pypy.tool import lib_pypy pypydir = py.path.local(pypy.__file__).dirpath() +rpythondir = py.path.local(rpython.__file__).dirpath() distdir = pypydir.dirpath() testresultdir = distdir.join('testresult') assert pypydir.check(dir=1) diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -11,8 +11,10 @@ import shutil import sys +import os +#Add toplevel repository dir to sys.path +sys.path.insert(0,os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))) import py -import os import fnmatch from rpython.tool.udir import udir @@ -48,7 +50,7 @@ basename = 'pypy-c' if sys.platform == 'win32': basename += '.exe' - pypy_c = basedir.join(basename) + pypy_c = basedir.join('pypy', 'goal', basename) else: pypy_c = py.path.local(override_pypy_c) if not pypy_c.check(): diff --git a/pypy/bin/translatorshell.py b/rpython/bin/translatorshell.py rename from pypy/bin/translatorshell.py rename to rpython/bin/translatorshell.py diff --git a/rpython/jit/backend/arm/test/support.py b/rpython/jit/backend/arm/test/support.py --- a/rpython/jit/backend/arm/test/support.py +++ b/rpython/jit/backend/arm/test/support.py @@ -14,6 +14,9 @@ def check_jumps(self, maxcount): pass +if not getattr(os, 'uname', None): + pytest.skip('cannot run arm tests on non-posix platform') + if os.uname()[1] == 'llaima.local': AS = '~/Code/arm-jit/android/android-ndk-r4b//build/prebuilt/darwin-x86/arm-eabi-4.4.0/arm-eabi/bin/as' else: diff --git a/rpython/jit/codewriter/effectinfo.py b/rpython/jit/codewriter/effectinfo.py --- a/rpython/jit/codewriter/effectinfo.py +++ b/rpython/jit/codewriter/effectinfo.py @@ -79,9 +79,11 @@ OS_RAW_MALLOC_VARSIZE = 110 OS_RAW_FREE = 111 + OS_JIT_FORCE_VIRTUAL = 120 + # for debugging: _OS_CANRAISE = set([OS_NONE, OS_STR2UNICODE, OS_LIBFFI_CALL, - OS_RAW_MALLOC_VARSIZE]) + OS_RAW_MALLOC_VARSIZE, OS_JIT_FORCE_VIRTUAL]) def __new__(cls, readonly_descrs_fields, readonly_descrs_arrays, write_descrs_fields, write_descrs_arrays, diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -1393,6 +1393,8 @@ elif oopspec_name == 'jit.isvirtual': kind = getkind(args[0].concretetype) return SpaceOperation('%s_isvirtual' % kind, args, op.result) + elif oopspec_name == 'jit.force_virtual': + return self._handle_oopspec_call(op, args, EffectInfo.OS_JIT_FORCE_VIRTUAL, EffectInfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE) else: raise AssertionError("missing support for %r" % oopspec_name) diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py --- a/rpython/jit/codewriter/support.py +++ b/rpython/jit/codewriter/support.py @@ -1,24 +1,24 @@ import sys -from rpython.rtyper.lltypesystem import lltype, rclass, rffi, llmemory -from rpython.rtyper.ootypesystem import ootype -from rpython.rtyper import rlist -from rpython.rtyper.lltypesystem import rstr as ll_rstr, rdict as ll_rdict -from rpython.rtyper.lltypesystem.module import ll_math -from rpython.rtyper.lltypesystem.lloperation import llop -from rpython.rtyper.ootypesystem import rdict as oo_rdict -from rpython.rtyper.llinterp import LLInterpreter -from rpython.rtyper.extregistry import ExtRegistryEntry -from rpython.translator.simplify import get_funcobj -from rpython.translator.unsimplify import split_block + +from rpython.annotator import model as annmodel +from rpython.annotator.policy import AnnotatorPolicy from rpython.flowspace.model import Variable, Constant -from rpython.translator.translator import TranslationContext -from rpython.annotator.policy import AnnotatorPolicy -from rpython.annotator import model as annmodel -from rpython.rtyper.annlowlevel import MixLevelHelperAnnotator from rpython.jit.metainterp.typesystem import deref from rpython.rlib import rgc -from rpython.rlib.jit import elidable +from rpython.rlib.jit import elidable, oopspec from rpython.rlib.rarithmetic import r_longlong, r_ulonglong, r_uint, intmask +from rpython.rtyper import rlist +from rpython.rtyper.annlowlevel import MixLevelHelperAnnotator +from rpython.rtyper.extregistry import ExtRegistryEntry +from rpython.rtyper.llinterp import LLInterpreter +from rpython.rtyper.lltypesystem import lltype, rclass, rffi, llmemory, rstr as ll_rstr, rdict as ll_rdict +from rpython.rtyper.lltypesystem.lloperation import llop +from rpython.rtyper.lltypesystem.module import ll_math +from rpython.rtyper.ootypesystem import ootype, rdict as oo_rdict +from rpython.translator.simplify import get_funcobj +from rpython.translator.translator import TranslationContext +from rpython.translator.unsimplify import split_block + def getargtypes(annotator, values): if values is None: # for backend tests producing stand-alone exe's @@ -213,10 +213,12 @@ _ll_5_list_ll_arraycopy = rgc.ll_arraycopy + @elidable def _ll_1_gc_identityhash(x): return lltype.identityhash(x) + # the following function should not be "@elidable": I can think of # a corner case in which id(const) is constant-folded, and then 'const' # disappears and is collected too early (possibly causing another object @@ -224,6 +226,8 @@ def _ll_1_gc_id(ptr): return llop.gc_id(lltype.Signed, ptr) + + at oopspec("jit.force_virtual(inst)") def _ll_1_jit_force_virtual(inst): return llop.jit_force_virtual(lltype.typeOf(inst), inst) diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py --- a/rpython/jit/metainterp/history.py +++ b/rpython/jit/metainterp/history.py @@ -976,11 +976,9 @@ def get_all_jitcell_tokens(self): tokens = [t() for t in self.jitcell_token_wrefs] if None in tokens: - assert False, "get_all_jitcell_tokens will not work as "+\ - "loops have been freed" + assert False, ("get_all_jitcell_tokens will not work as " + "loops have been freed") return tokens - - def check_history(self, expected=None, **check): insns = {} diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -1,12 +1,13 @@ +from rpython.jit.codewriter.effectinfo import EffectInfo +from rpython.jit.metainterp.executor import execute from rpython.jit.codewriter.heaptracker import vtable2descr -from rpython.jit.metainterp.executor import execute from rpython.jit.metainterp.history import Const, ConstInt, BoxInt from rpython.jit.metainterp.optimizeopt import optimizer +from rpython.jit.metainterp.optimizeopt.optimizer import OptValue, REMOVED from rpython.jit.metainterp.optimizeopt.util import (make_dispatcher_method, descrlist_dict, sort_descrs) from rpython.jit.metainterp.resoperation import rop, ResOperation from rpython.rlib.objectmodel import we_are_translated -from rpython.jit.metainterp.optimizeopt.optimizer import OptValue class AbstractVirtualValue(optimizer.OptValue): @@ -386,6 +387,24 @@ self.make_equal_to(box, vvalue) return vvalue + def optimize_GUARD_NO_EXCEPTION(self, op): + if self.last_emitted_operation is REMOVED: + return + self.emit_operation(op) + + def optimize_GUARD_NOT_FORCED(self, op): + if self.last_emitted_operation is REMOVED: + return + self.emit_operation(op) + + def optimize_CALL_MAY_FORCE(self, op): + effectinfo = op.getdescr().get_extra_info() + oopspecindex = effectinfo.oopspecindex + if oopspecindex == EffectInfo.OS_JIT_FORCE_VIRTUAL: + if self._optimize_JIT_FORCE_VIRTUAL(op): + return + self.emit_operation(op) + def optimize_VIRTUAL_REF(self, op): indexbox = op.getarg(1) # @@ -429,7 +448,7 @@ # - set 'virtual_token' to TOKEN_NONE args = [op.getarg(0), ConstInt(vrefinfo.TOKEN_NONE)] seo(ResOperation(rop.SETFIELD_GC, args, None, - descr = vrefinfo.descr_virtual_token)) + descr=vrefinfo.descr_virtual_token)) # Note that in some cases the virtual in op.getarg(1) has been forced # already. This is fine. In that case, and *if* a residual # CALL_MAY_FORCE suddenly turns out to access it, then it will @@ -437,6 +456,20 @@ # will work too (but just be a little pointless, as the structure # was already forced). + def _optimize_JIT_FORCE_VIRTUAL(self, op): + vref = self.getvalue(op.getarg(1)) + vrefinfo = self.optimizer.metainterp_sd.virtualref_info + if vref.is_virtual(): + tokenvalue = vref.getfield(vrefinfo.descr_virtual_token, None) + if (tokenvalue is not None and tokenvalue.is_constant() and + tokenvalue.box.getint() == vrefinfo.TOKEN_NONE): + forcedvalue = vref.getfield(vrefinfo.descr_forced, None) + if forcedvalue is not None and not forcedvalue.is_null(): + self.make_equal_to(op.result, forcedvalue) + self.last_emitted_operation = REMOVED + return True + return False + def optimize_GETFIELD_GC(self, op): value = self.getvalue(op.getarg(0)) # If this is an immutable field (as indicated by op.is_always_pure()) diff --git a/rpython/jit/metainterp/test/support.py b/rpython/jit/metainterp/test/support.py --- a/rpython/jit/metainterp/test/support.py +++ b/rpython/jit/metainterp/test/support.py @@ -160,16 +160,17 @@ class JitMixin: basic = True + def check_resops(self, expected=None, **check): get_stats().check_resops(expected=expected, **check) + def check_simple_loop(self, expected=None, **check): get_stats().check_simple_loop(expected=expected, **check) - - def check_trace_count(self, count): # was check_loop_count # The number of traces compiled assert get_stats().compiled_count == count + def check_trace_count_at_most(self, count): assert get_stats().compiled_count <= count @@ -178,11 +179,12 @@ def check_target_token_count(self, count): tokens = get_stats().get_all_jitcell_tokens() - n = sum ([len(t.target_tokens) for t in tokens]) + n = sum([len(t.target_tokens) for t in tokens]) assert n == count def check_enter_count(self, count): assert get_stats().enter_count == count + def check_enter_count_at_most(self, count): assert get_stats().enter_count <= count @@ -192,6 +194,7 @@ def check_aborted_count(self, count): assert get_stats().aborted_count == count + def check_aborted_count_at_least(self, count): assert get_stats().aborted_count >= count diff --git a/rpython/jit/metainterp/test/test_virtualref.py b/rpython/jit/metainterp/test/test_virtualref.py --- a/rpython/jit/metainterp/test/test_virtualref.py +++ b/rpython/jit/metainterp/test/test_virtualref.py @@ -1,19 +1,20 @@ import py -from rpython.rtyper.lltypesystem import lltype, llmemory, lloperation + +from rpython.rtyper.lltypesystem import lltype, lloperation from rpython.rtyper.exceptiondata import UnknownException from rpython.rlib.jit import JitDriver, dont_look_inside, vref_None from rpython.rlib.jit import virtual_ref, virtual_ref_finish, InvalidVirtualRef from rpython.rlib.jit import non_virtual_ref from rpython.rlib.objectmodel import compute_unique_id -from rpython.jit.metainterp.test.support import LLJitMixin, OOJitMixin, _get_jitcodes +from rpython.jit.metainterp.test.support import LLJitMixin, _get_jitcodes from rpython.jit.metainterp.resoperation import rop from rpython.jit.metainterp.virtualref import VirtualRefInfo + debug_print = lloperation.llop.debug_print -class VRefTests: - +class VRefTests(object): def finish_setup_for_interp_operations(self): self.vrefinfo = VirtualRefInfo(self.warmrunnerstate) self.cw.setup_vrefinfo(self.vrefinfo) @@ -115,8 +116,8 @@ from rpython.jit.metainterp.resume import ResumeDataDirectReader cpu = self.metainterp.cpu cpu.get_latest_value_count = lambda df: len(guard_op.getfailargs()) - cpu.get_latest_value_int = lambda df,i:guard_op.getfailargs()[i].getint() - cpu.get_latest_value_ref = lambda df,i:guard_op.getfailargs()[i].getref_base() + cpu.get_latest_value_int = lambda df, i: guard_op.getfailargs()[i].getint() + cpu.get_latest_value_ref = lambda df, i: guard_op.getfailargs()[i].getref_base() cpu.clear_latest_values = lambda count: None class FakeMetaInterpSd: callinfocollection = None @@ -418,15 +419,18 @@ self.check_aborted_count(0) def test_jit_force_virtual_seen(self): - myjitdriver = JitDriver(greens = [], reds = ['n']) - # + myjitdriver = JitDriver(greens=[], reds=['n']) + A = lltype.GcArray(lltype.Signed) - class XY: + + class XY(object): pass - class ExCtx: + + class ExCtx(object): pass exctx = ExCtx() - # + escapes = [] + def f(n): while n > 0: myjitdriver.can_enter_jit(n=n) @@ -434,16 +438,16 @@ xy = XY() xy.n = n exctx.topframeref = vref = virtual_ref(xy) + escapes.append(xy) xy.next1 = lltype.malloc(A, 0) n = exctx.topframeref().n - 1 - xy.next1 = lltype.nullptr(A) exctx.topframeref = vref_None virtual_ref_finish(vref, xy) return 1 # res = self.meta_interp(f, [15]) assert res == 1 - self.check_resops(new_with_vtable=4, # vref, xy + self.check_resops(new_with_vtable=2, # xy new_array=2) # next1 self.check_aborted_count(0) @@ -656,6 +660,34 @@ res = self.meta_interp(f, [10]) assert res == 0 + def test_force_virtual_vref(self): + myjitdriver = JitDriver(greens=[], reds=['n', 'ec']) + + class ExecutionContext(object): + pass + + class Frame(object): + def __init__(self, x): + self.x = x + + def f(n): + ec = ExecutionContext() + while n > 0: + myjitdriver.jit_merge_point(n=n, ec=ec) + frame = Frame(1) + ec.topframeref = virtual_ref(frame) + n -= ec.topframeref().x + frame_vref = ec.topframeref + ec.topframeref = vref_None + virtual_ref_finish(frame_vref, frame) + return n + res = self.meta_interp(f, [10]) + assert res == 0 + self.check_resops({ + 'int_sub': 2, 'int_gt': 2, 'jump': 1, 'guard_true': 2, + 'force_token': 2, 'setfield_gc': 1 + }) + class TestLLtype(VRefTests, LLJitMixin): pass diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -310,10 +310,7 @@ def decorate(func): from rpython.tool.sourcetools import compile2 # - def get_printable_location(): - return name - jitdriver = JitDriver(get_printable_location=get_printable_location, - greens=[], reds='auto', name=name) + jitdriver = JitDriver(greens=[], reds='auto', name=name) # args = ','.join(['a%d' % i for i in range(func.func_code.co_argcount)]) source = """def callback_with_jitdriver(%(args)s): @@ -484,8 +481,11 @@ self.autoreds = True self.reds = [] self.numreds = None # see warmspot.autodetect_jit_markers_redvars - assert confirm_enter_jit is None, ( - "reds='auto' is not compatible with confirm_enter_jit") + for hook in [ + get_jitcell_at, set_jitcell_at, get_printable_location, + confirm_enter_jit + ]: + assert hook is None, "reds='auto' is not compatible with JitDriver hooks" else: if reds is not None: self.reds = reds diff --git a/rpython/rlib/unicodedata/test/test_ucd.py b/rpython/rlib/unicodedata/test/test_ucd.py --- a/rpython/rlib/unicodedata/test/test_ucd.py +++ b/rpython/rlib/unicodedata/test/test_ucd.py @@ -1,9 +1,9 @@ +from rpython.rlib.runicode import code_to_unichr +from rpython.rlib.unicodedata import unicodedb_5_2_0 from rpython.rtyper.test.tool import BaseRtypingTest, LLRtypeMixin -from rpython.rlib.unicodedata import unicodedb_5_2_0 -from rpython.rlib.unicodedata.ucd import code_to_unichr + class TestTranslated(BaseRtypingTest, LLRtypeMixin): - def test_translated(self): def f(n): if n == 0: diff --git a/rpython/rlib/unicodedata/test/test_unicodedata.py b/rpython/rlib/unicodedata/test/test_unicodedata.py --- a/rpython/rlib/unicodedata/test/test_unicodedata.py +++ b/rpython/rlib/unicodedata/test/test_unicodedata.py @@ -1,11 +1,15 @@ +import random +import unicodedata + import py + from rpython.rlib.unicodedata import unicodedb_3_2_0, unicodedb_5_2_0 + class TestUnicodeData(object): def setup_class(cls): - import random, unicodedata if unicodedata.unidata_version != '5.2.0': - skip('Needs python with unicode 5.2.0 database.') + py.test.skip('Needs python with unicode 5.2.0 database.') seed = random.getrandbits(32) print "random seed: ", seed @@ -29,14 +33,12 @@ py.test.raises(KeyError, unicodedb_5_2_0.name, ord(chr)) def test_compare_functions(self): - import unicodedata # CPython implementation - def getX(fun, code): try: return getattr(unicodedb_5_2_0, fun)(code) except KeyError: return -1 - + for code in range(0x10000): char = unichr(code) assert unicodedata.digit(char, -1) == getX('digit', code) @@ -73,5 +75,3 @@ assert unicodedb_5_2_0.lookup('BENZENE RING WITH CIRCLE') == 9187 py.test.raises(KeyError, unicodedb_3_2_0.lookup, 'BENZENE RING WITH CIRCLE') py.test.raises(KeyError, unicodedb_3_2_0.name, 9187) - - From noreply at buildbot.pypy.org Fri Jan 25 15:24:43 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 25 Jan 2013 15:24:43 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: use the default enable_opts here Message-ID: <20130125142443.A3CAF1C1055@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60451:e2304d98bbcc Date: 2013-01-25 16:24 +0200 http://bitbucket.org/pypy/pypy/changeset/e2304d98bbcc/ Log: use the default enable_opts here diff --git a/rpython/jit/backend/x86/test/test_zrpy_gc.py b/rpython/jit/backend/x86/test/test_zrpy_gc.py --- a/rpython/jit/backend/x86/test/test_zrpy_gc.py +++ b/rpython/jit/backend/x86/test/test_zrpy_gc.py @@ -83,7 +83,7 @@ return {(gc.GcLLDescr_framework, 'can_use_nursery_malloc'): can_use_nursery_malloc2} -def compile(f, gc, enable_opts='', **kwds): +def compile(f, gc, **kwds): from rpython.annotator.listdef import s_list_of_strings from rpython.translator.translator import TranslationContext from rpython.jit.metainterp.warmspot import apply_jit @@ -107,7 +107,7 @@ old_value[obj, attr] = getattr(obj, attr) setattr(obj, attr, value) # - apply_jit(t, enable_opts=enable_opts) + apply_jit(t) # finally: for (obj, attr), oldvalue in old_value.items(): From noreply at buildbot.pypy.org Fri Jan 25 15:52:15 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 25 Jan 2013 15:52:15 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: merge default Message-ID: <20130125145215.2353B1C1069@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60452:4c505286f934 Date: 2013-01-25 16:51 +0200 http://bitbucket.org/pypy/pypy/changeset/4c505286f934/ Log: merge default diff --git a/lib-python/2.7/test/test_iterlen.py b/lib-python/2.7/test/test_iterlen.py --- a/lib-python/2.7/test/test_iterlen.py +++ b/lib-python/2.7/test/test_iterlen.py @@ -94,7 +94,11 @@ def test_no_len_for_infinite_repeat(self): # The repeat() object can also be infinite - self.assertRaises(TypeError, len, repeat(None)) + if test_support.check_impl_detail(pypy=True): + # 3.4 (PEP 424) behavior + self.assertEqual(len(repeat(None)), NotImplemented) + else: + self.assertRaises(TypeError, len, repeat(None)) class TestXrange(TestInvariantWithoutMutations): @@ -230,6 +234,7 @@ self.assertRaises(RuntimeError, b.extend, BadLen()) self.assertRaises(RuntimeError, b.extend, BadLengthHint()) + @test_support.impl_detail("PEP 424 disallows None results", pypy=False) def test_invalid_hint(self): # Make sure an invalid result doesn't muck-up the works self.assertEqual(list(NoneLengthHint()), list(range(10))) diff --git a/lib-python/conftest.py b/lib-python/conftest.py --- a/lib-python/conftest.py +++ b/lib-python/conftest.py @@ -17,8 +17,8 @@ from pypy.conftest import option as pypy_option from pypy.tool.pytest import appsupport -from pypy.tool.pytest.confpath import pypydir, testdir, testresultdir -from pypy.config.parse import parse_info +from pypy.tool.pytest.confpath import pypydir, rpythondir, testdir, testresultdir +from rpython.config.parse import parse_info pytest_plugins = "resultlog", rsyncdirs = ['.', '../pypy/'] @@ -40,7 +40,7 @@ dest="unittest_filter", help="Similar to -k, XXX") def gettimeout(timeout): - from test import pystone + from rpython.translator.test import rpystone if timeout.endswith('mp'): megapystone = float(timeout[:-2]) t, stone = pystone.Proc0(10000) @@ -281,7 +281,7 @@ RegrTest('test_ioctl.py'), RegrTest('test_isinstance.py', core=True), RegrTest('test_iter.py', core=True), - RegrTest('test_iterlen.py', skip="undocumented internal API behavior __length_hint__"), + RegrTest('test_iterlen.py', core=True, usemodules="_collections itertools"), RegrTest('test_itertools.py', core=True, usemodules="itertools struct"), RegrTest('test_json.py'), RegrTest('test_kqueue.py'), @@ -584,7 +584,7 @@ watchdog_name = 'watchdog_nt.py' else: watchdog_name = 'watchdog.py' - watchdog_script = pypydir.join('tool', watchdog_name) + watchdog_script = rpythondir.join('tool', watchdog_name) regr_script = pypydir.join('tool', 'pytest', 'run-script', 'regrverbose.py') diff --git a/pypy/doc/arm.rst b/pypy/doc/arm.rst --- a/pypy/doc/arm.rst +++ b/pypy/doc/arm.rst @@ -130,15 +130,7 @@ export SB2=/srv/chroot/precise_arm export SB2OPT='-t ARM' -Once this is set, you can call the translator - -:: - - pypy ~/path_to_pypy_checkout/pypy/translator/goal/translate.py -O1 --platform=arm target.py - -If everything worked correctly this should yield an ARM binary. Running this binary in the ARM chroot or on an ARM device should produce the output ``"Hello World"``. - -.. _`this`: +Once this is set, you can call the translator. For example save this file :: @@ -148,3 +140,22 @@ def target(*args): return main, None + +and call the translator + +:: + + pypy ~/path_to_pypy_checkout/pypy/translator/goal/translate.py -O1 --platform=arm target.py + +If everything worked correctly this should yield an ARM binary. Running this binary in the ARM chroot or on an ARM device should produce the output ``"Hello World"``. + +To translate the full python pypy interpreter with a jit, you can cd into pypy/goal and call + +:: + + pypy /rpython/translator/goal/translate.py -Ojit --platform=arm --gcrootfinder=shadowstack --jit-backend=arm targetpypystandalone.py + +The gcrootfinder option is needed to work around `issue 1377`_ and the jit-backend works around `issue 1376`_ + +.. _`issue 1377`: https://bugs.pypy.org/issue1377 +.. _`issue 1376`: https://bugs.pypy.org/issue1376 diff --git a/pypy/doc/getting-started-python.rst b/pypy/doc/getting-started-python.rst --- a/pypy/doc/getting-started-python.rst +++ b/pypy/doc/getting-started-python.rst @@ -86,8 +86,8 @@ 4. Run:: - cd pypy/translator/goal - python translate.py --opt=jit targetpypystandalone.py + cd pypy/goal + python ../../rpython/bin/rpython --opt=jit targetpypystandalone.py possibly replacing ``--opt=jit`` with another `optimization level`_ of your choice like ``--opt=2`` if you do not want to include the JIT diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -5,6 +5,9 @@ .. this is a revision shortly after release-2.0-beta1 .. startrev: 0e6161a009c6 +.. branch: split-rpython +Split rpython and pypy into seperate directories + .. branch: callback-jit Callbacks from C are now better JITted @@ -40,3 +43,7 @@ .. branch: fix-lookinside-iff-oopspec Fixed the interaction between two internal tools for controlling the JIT. + +.. branch: inline-virtualref-2 +Better optimized certain types of frame accesses in the JIT, particularly +around exceptions that escape the function they were raised in. diff --git a/pypy/goal/__init__.py b/pypy/goal/__init__.py new file mode 100644 --- /dev/null +++ b/pypy/goal/__init__.py @@ -0,0 +1,1 @@ +#empty diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py --- a/pypy/interpreter/astcompiler/codegen.py +++ b/pypy/interpreter/astcompiler/codegen.py @@ -904,8 +904,9 @@ def visit_Set(self, s): self.update_position(s.lineno) + elt_count = len(s.elts) if s.elts is not None else 0 self.visit_sequence(s.elts) - self.emit_op_arg(ops.BUILD_SET, len(s.elts)) + self.emit_op_arg(ops.BUILD_SET, elt_count) def visit_Name(self, name): self.update_position(name.lineno) diff --git a/pypy/interpreter/test2/test_app_main.py b/pypy/interpreter/test2/test_app_main.py --- a/pypy/interpreter/test2/test_app_main.py +++ b/pypy/interpreter/test2/test_app_main.py @@ -84,7 +84,7 @@ def check(self, argv, **expected): import StringIO - from rpython.translator.goal import app_main + from pypy.interpreter import app_main saved_sys_argv = sys.argv[:] saved_sys_stdout = sys.stdout saved_sys_stderr = sys.stdout @@ -825,7 +825,7 @@ class TestAppMain: def test_print_info(self): - from rpython.translator.goal import app_main + from pypy.interpreter import app_main import sys, cStringIO prev_so = sys.stdout prev_ti = getattr(sys, 'pypy_translation_info', 'missing') diff --git a/pypy/interpreter/test2/test_targetpypy.py b/pypy/interpreter/test2/test_targetpypy.py --- a/pypy/interpreter/test2/test_targetpypy.py +++ b/pypy/interpreter/test2/test_targetpypy.py @@ -1,6 +1,6 @@ import py -from goal.targetpypystandalone import get_entry_point +from pypy.goal.targetpypystandalone import get_entry_point from pypy.config.pypyoption import get_pypy_config class TestTargetPyPy(object): diff --git a/pypy/module/_ast/test/test_ast.py b/pypy/module/_ast/test/test_ast.py --- a/pypy/module/_ast/test/test_ast.py +++ b/pypy/module/_ast/test/test_ast.py @@ -290,6 +290,12 @@ ]) exec compile(body, '', 'exec') + def test_empty_set(self): + import ast + m = ast.Module(body=[ast.Expr(value=ast.Set(elts=[]))]) + ast.fix_missing_locations(m) + compile(m, "", "exec") + def test_invalid_sum(self): import _ast as ast pos = dict(lineno=2, col_offset=3) diff --git a/pypy/module/_collections/interp_deque.py b/pypy/module/_collections/interp_deque.py --- a/pypy/module/_collections/interp_deque.py +++ b/pypy/module/_collections/interp_deque.py @@ -521,7 +521,11 @@ return self.space.wrap(self.counter) def next(self): - self.deque.checklock(self.lock) + if self.lock is not self.deque.lock: + self.counter = 0 + raise OperationError( + self.space.w_RuntimeError, + self.space.wrap("deque mutated during iteration")) if self.counter == 0: raise OperationError(self.space.w_StopIteration, self.space.w_None) self.counter -= 1 @@ -560,7 +564,11 @@ return self.space.wrap(self.counter) def next(self): - self.deque.checklock(self.lock) + if self.lock is not self.deque.lock: + self.counter = 0 + raise OperationError( + self.space.w_RuntimeError, + self.space.wrap("deque mutated during iteration")) if self.counter == 0: raise OperationError(self.space.w_StopIteration, self.space.w_None) self.counter -= 1 diff --git a/pypy/module/cpyext/classobject.py b/pypy/module/cpyext/classobject.py --- a/pypy/module/cpyext/classobject.py +++ b/pypy/module/cpyext/classobject.py @@ -22,6 +22,12 @@ w_result.setdict(space, w_dict) return w_result + at cpython_api([PyObject, PyObject, PyObject], PyObject) +def PyInstance_New(space, w_cls, w_arg, w_kw): + """Create a new instance of a specific class. The parameters arg and kw are + used as the positional and keyword parameters to the object's constructor.""" + return space.call(w_cls, w_arg, w_kw) + @cpython_api([PyObject, PyObject], PyObject, error=CANNOT_FAIL) def _PyInstance_Lookup(space, w_instance, w_name): name = space.str_w(w_name) diff --git a/pypy/module/cpyext/include/ceval.h b/pypy/module/cpyext/include/ceval.h new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/include/ceval.h @@ -0,0 +1,1 @@ +/* empty */ diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py --- a/pypy/module/cpyext/stubs.py +++ b/pypy/module/cpyext/stubs.py @@ -176,12 +176,6 @@ """Return true if klass is a subclass of base. Return false in all other cases.""" raise NotImplementedError - at cpython_api([PyObject, PyObject, PyObject], PyObject) -def PyInstance_New(space, cls, arg, kw): - """Create a new instance of a specific class. The parameters arg and kw are - used as the positional and keyword parameters to the object's constructor.""" - raise NotImplementedError - @cpython_api([PyObject], rffi.INT_real, error=-1) def PyCodec_Register(space, search_function): """Register a new codec search function. diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py --- a/pypy/module/cpyext/test/conftest.py +++ b/pypy/module/cpyext/test/conftest.py @@ -1,6 +1,15 @@ import py import pytest +def pytest_configure(config): + from pypy.tool.pytest.objspace import gettestobjspace + # For some reason (probably a ll2ctypes cache issue on linux64) + # it's necessary to run "import time" at least once before any + # other cpyext test, otherwise the same statement will fail in + # test_datetime.py. + space = gettestobjspace(usemodules=['rctime']) + space.getbuiltinmodule("time") + def pytest_ignore_collect(path, config): if config.option.runappdirect: return True # "cannot be run by py.test -A" diff --git a/pypy/module/cpyext/test/test_classobject.py b/pypy/module/cpyext/test/test_classobject.py --- a/pypy/module/cpyext/test/test_classobject.py +++ b/pypy/module/cpyext/test/test_classobject.py @@ -7,8 +7,10 @@ w_class = space.appexec([], """(): class C: x = None - def __init__(self): + def __init__(self, *args, **kwargs): self.x = 1 + self.args = args + self.__dict__.update(kwargs) return C """) @@ -22,6 +24,13 @@ assert space.getattr(w_instance, space.wrap('x')) is space.w_None assert space.unwrap(space.getattr(w_instance, space.wrap('a'))) == 3 + w_instance = api.PyInstance_New(w_class, + space.wrap((3,)), space.wrap(dict(y=2))) + assert space.unwrap(space.getattr(w_instance, space.wrap('x'))) == 1 + assert space.unwrap(space.getattr(w_instance, space.wrap('y'))) == 2 + assert space.unwrap(space.getattr(w_instance, space.wrap('args'))) == (3,) + + def test_lookup(self, space, api): w_instance = space.appexec([], """(): class C: diff --git a/pypy/module/micronumpy/arrayimpl/scalar.py b/pypy/module/micronumpy/arrayimpl/scalar.py --- a/pypy/module/micronumpy/arrayimpl/scalar.py +++ b/pypy/module/micronumpy/arrayimpl/scalar.py @@ -34,6 +34,9 @@ def get_shape(self): return [] + def get_strides(self): + return [] + def create_iter(self, shape=None): return ScalarIterator(self.value) diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -40,6 +40,10 @@ self.implementation = self.implementation.set_shape(space, get_shape_from_iterable(space, self.get_size(), w_new_shape)) + def descr_get_strides(self, space): + strides = self.implementation.get_strides() + return space.newtuple([space.wrap(i) for i in strides]) + def get_dtype(self): return self.implementation.dtype @@ -645,6 +649,7 @@ dtype = GetSetProperty(W_NDimArray.descr_get_dtype), shape = GetSetProperty(W_NDimArray.descr_get_shape, W_NDimArray.descr_set_shape), + strides = GetSetProperty(W_NDimArray.descr_get_strides), ndim = GetSetProperty(W_NDimArray.descr_get_ndim), size = GetSetProperty(W_NDimArray.descr_get_size), itemsize = GetSetProperty(W_NDimArray.descr_get_itemsize), diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -684,6 +684,18 @@ assert a.reshape([1]).shape == (1,) raises(ValueError, "a.reshape(3)") + def test_strides(self): + from _numpypy import array + a = array([[1.0, 2.0], + [3.0, 4.0]]) + assert a.strides == (16, 8) + assert a[1:].strides == (16, 8) + + def test_strides_scalar(self): + from _numpypy import array + a = array(42) + assert a.strides == () + def test_add(self): from _numpypy import array a = array(range(5)) diff --git a/pypy/module/pypyjit/test_pypy_c/test_thread.py b/pypy/module/pypyjit/test_pypy_c/test_thread.py --- a/pypy/module/pypyjit/test_pypy_c/test_thread.py +++ b/pypy/module/pypyjit/test_pypy_c/test_thread.py @@ -26,3 +26,25 @@ --THREAD-TICK-- jump(..., descr=...) """) + + def test_tls(self): + def main(n): + import thread + local = thread._local() + local.x = 1 + i = 0 + while i < n: + i += local.x + return 0 + log = self.run(main, [500]) + assert round(log.result, 6) == round(main(500), 6) + loop, = log.loops_by_filename(self.filepath) + assert loop.match(""" + i53 = int_lt(i48, i27) + guard_true(i53, descr=...) + i54 = int_add_ovf(i48, i47) + guard_no_overflow(descr=...) + --TICK-- + i58 = arraylen_gc(p43, descr=...) + jump(p0, p1, p3, p5, p10, p12, p14, i54, i27, i47, p45, p43, descr=...) + """) diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py --- a/pypy/module/rctime/interp_time.py +++ b/pypy/module/rctime/interp_time.py @@ -448,7 +448,9 @@ raise OperationError(space.w_ValueError, space.wrap("year out of range")) - if rffi.getintfield(glob_buf, 'c_tm_wday') < 0: + # tm_wday does not need checking of its upper-bound since taking "% + # 7" in gettmarg() automatically restricts the range. + if rffi.getintfield(glob_buf, 'c_tm_wday') < -1: raise OperationError(space.w_ValueError, space.wrap("day of week out of range")) diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py --- a/pypy/module/rctime/test/test_rctime.py +++ b/pypy/module/rctime/test/test_rctime.py @@ -113,6 +113,9 @@ if os.name != 'nt': assert rctime.mktime(rctime.localtime(-1)) == -1 + res = rctime.mktime((2000, 1, 1, 0, 0, 0, -1, -1, -1)) + assert rctime.ctime(res) == 'Sat Jan 1 00:00:00 2000' + def test_asctime(self): import time as rctime rctime.asctime() diff --git a/pypy/module/thread/os_local.py b/pypy/module/thread/os_local.py --- a/pypy/module/thread/os_local.py +++ b/pypy/module/thread/os_local.py @@ -30,6 +30,9 @@ w_dict = space.newdict(instance=True) self.dicts[ec] = w_dict self._register_in_ec(ec) + # cache the last seen dict, works because we are protected by the GIL + self.last_dict = w_dict + self.last_ec = ec def _register_in_ec(self, ec): if not ec.space.config.translation.rweakref: @@ -60,10 +63,14 @@ def getdict(self, space): ec = space.getexecutioncontext() + if ec is self.last_ec: + return self.last_dict try: w_dict = self.dicts[ec] except KeyError: w_dict = self.create_new_dict(ec) + self.last_ec = ec + self.last_dict = w_dict return w_dict def descr_local__new__(space, w_subtype, __args__): @@ -91,3 +98,5 @@ local = wref() if local is not None: del local.dicts[ec] + local.last_dict = None + local.last_ec = None diff --git a/pypy/module/thread/test/test_local.py b/pypy/module/thread/test/test_local.py --- a/pypy/module/thread/test/test_local.py +++ b/pypy/module/thread/test/test_local.py @@ -108,3 +108,54 @@ gc.collect() assert done == ['ok', 'del'] done.append('shutdown') + +def test_local_caching(): + from pypy.module.thread.os_local import Local + class FakeSpace: + def getexecutioncontext(self): + return self.ec + + def getattr(*args): + pass + def call_obj_args(*args): + pass + def newdict(*args, **kwargs): + return {} + def wrap(self, obj): + return obj + def type(self, obj): + return type(obj) + class config: + class translation: + rweakref = True + + class FakeEC: + def __init__(self, space): + self.space = space + self._thread_local_objs = None + space = FakeSpace() + ec1 = FakeEC(space) + space.ec = ec1 + + l = Local(space, None) + assert l.last_dict is l.dicts[ec1] + assert l.last_ec is ec1 + d1 = l.getdict(space) + assert d1 is l.last_dict + + ec2 = space.ec = FakeEC(space) + d2 = l.getdict(space) + assert l.last_dict is d2 + assert d2 is l.dicts[ec2] + assert l.last_ec is ec2 + dicts = l.dicts + l.dicts = "nope" + assert l.getdict(space) is d2 + l.dicts = dicts + + space.ec = ec1 + assert l.getdict(space) is d1 + l.dicts = "nope" + assert l.getdict(space) is d1 + l.dicts = dicts + diff --git a/pypy/module/unicodedata/test/test_unicodedata.py b/pypy/module/unicodedata/test/test_unicodedata.py new file mode 100644 --- /dev/null +++ b/pypy/module/unicodedata/test/test_unicodedata.py @@ -0,0 +1,104 @@ +import py +import sys + +class AppTestUnicodeData: + spaceconfig = dict(usemodules=('unicodedata',)) + + def test_hangul_syllables(self): + import unicodedata + # Test all leading, vowel and trailing jamo + # but not every combination of them. + for code, name in ((0xAC00, 'HANGUL SYLLABLE GA'), + (0xAE69, 'HANGUL SYLLABLE GGAEG'), + (0xB0D2, 'HANGUL SYLLABLE NYAGG'), + (0xB33B, 'HANGUL SYLLABLE DYAEGS'), + (0xB5A4, 'HANGUL SYLLABLE DDEON'), + (0xB80D, 'HANGUL SYLLABLE RENJ'), + (0xBA76, 'HANGUL SYLLABLE MYEONH'), + (0xBCDF, 'HANGUL SYLLABLE BYED'), + (0xBF48, 'HANGUL SYLLABLE BBOL'), + (0xC1B1, 'HANGUL SYLLABLE SWALG'), + (0xC41A, 'HANGUL SYLLABLE SSWAELM'), + (0xC683, 'HANGUL SYLLABLE OELB'), + (0xC8EC, 'HANGUL SYLLABLE JYOLS'), + (0xCB55, 'HANGUL SYLLABLE JJULT'), + (0xCDBE, 'HANGUL SYLLABLE CWEOLP'), + (0xD027, 'HANGUL SYLLABLE KWELH'), + (0xD290, 'HANGUL SYLLABLE TWIM'), + (0xD4F9, 'HANGUL SYLLABLE PYUB'), + (0xD762, 'HANGUL SYLLABLE HEUBS'), + (0xAE27, 'HANGUL SYLLABLE GYIS'), + (0xB090, 'HANGUL SYLLABLE GGISS'), + (0xB0AD, 'HANGUL SYLLABLE NANG'), + (0xB316, 'HANGUL SYLLABLE DAEJ'), + (0xB57F, 'HANGUL SYLLABLE DDYAC'), + (0xB7E8, 'HANGUL SYLLABLE RYAEK'), + (0xBA51, 'HANGUL SYLLABLE MEOT'), + (0xBCBA, 'HANGUL SYLLABLE BEP'), + (0xBF23, 'HANGUL SYLLABLE BBYEOH'), + (0xD7A3, 'HANGUL SYLLABLE HIH')): + assert unicodedata.name(unichr(code)) == name + assert unicodedata.lookup(name) == unichr(code) + # Test outside the range + raises(ValueError, unicodedata.name, unichr(0xAC00 - 1)) + raises(ValueError, unicodedata.name, unichr(0xD7A3 + 1)) + + def test_cjk(self): + import sys + import unicodedata + cases = ((0x3400, 0x4DB5), + (0x4E00, 0x9FA5)) + if unicodedata.unidata_version >= "5": # don't know the exact limit + cases = ((0x3400, 0x4DB5), + (0x4E00, 0x9FCB), + (0x20000, 0x2A6D6), + (0x2A700, 0x2B734)) + elif unicodedata.unidata_version >= "4.1": + cases = ((0x3400, 0x4DB5), + (0x4E00, 0x9FBB), + (0x20000, 0x2A6D6)) + for first, last in cases: + # Test at and inside the boundary + for i in (first, first + 1, last - 1, last): + charname = 'CJK UNIFIED IDEOGRAPH-%X'%i + char = ('\\U%08X' % i).decode('unicode-escape') + assert unicodedata.name(char) == charname + assert unicodedata.lookup(charname) == char + # Test outside the boundary + for i in first - 1, last + 1: + charname = 'CJK UNIFIED IDEOGRAPH-%X'%i + char = ('\\U%08X' % i).decode('unicode-escape') + try: + unicodedata.name(char) + except ValueError, e: + assert e.message == 'no such name' + raises(KeyError, unicodedata.lookup, charname) + + def test_bug_1704793(self): # from CPython + import unicodedata + assert unicodedata.lookup("GOTHIC LETTER FAIHU") == u'\U00010346' + + def test_normalize(self): + import unicodedata + raises(TypeError, unicodedata.normalize, 'x') + + @py.test.mark.skipif("sys.maxunicode < 0x10ffff") + def test_normalize_wide(self): + import unicodedata + assert unicodedata.normalize('NFC', u'\U000110a5\U000110ba') == u'\U000110ab' + + def test_linebreaks(self): + linebreaks = (0x0a, 0x0b, 0x0c, 0x0d, 0x85, + 0x1c, 0x1d, 0x1e, 0x2028, 0x2029) + for i in linebreaks: + for j in range(-2, 3): + lines = (unichr(i + j) + u'A').splitlines() + if i + j in linebreaks: + assert len(lines) == 2 + else: + assert len(lines) == 1 + + def test_mirrored(self): + import unicodedata + # For no reason, unicodedata.mirrored() returns an int, not a bool + assert repr(unicodedata.mirrored(u' ')) == '0' diff --git a/pypy/module/unicodedata/test_unicodedata.py b/pypy/module/unicodedata/test_unicodedata.py deleted file mode 100644 --- a/pypy/module/unicodedata/test_unicodedata.py +++ /dev/null @@ -1,103 +0,0 @@ - -class AppTestUnicodeData: - spaceconfig = dict(usemodules=('unicodedata',)) - - def test_hangul_syllables(self): - import unicodedata - # Test all leading, vowel and trailing jamo - # but not every combination of them. - for code, name in ((0xAC00, 'HANGUL SYLLABLE GA'), - (0xAE69, 'HANGUL SYLLABLE GGAEG'), - (0xB0D2, 'HANGUL SYLLABLE NYAGG'), - (0xB33B, 'HANGUL SYLLABLE DYAEGS'), - (0xB5A4, 'HANGUL SYLLABLE DDEON'), - (0xB80D, 'HANGUL SYLLABLE RENJ'), - (0xBA76, 'HANGUL SYLLABLE MYEONH'), - (0xBCDF, 'HANGUL SYLLABLE BYED'), - (0xBF48, 'HANGUL SYLLABLE BBOL'), - (0xC1B1, 'HANGUL SYLLABLE SWALG'), - (0xC41A, 'HANGUL SYLLABLE SSWAELM'), - (0xC683, 'HANGUL SYLLABLE OELB'), - (0xC8EC, 'HANGUL SYLLABLE JYOLS'), - (0xCB55, 'HANGUL SYLLABLE JJULT'), - (0xCDBE, 'HANGUL SYLLABLE CWEOLP'), - (0xD027, 'HANGUL SYLLABLE KWELH'), - (0xD290, 'HANGUL SYLLABLE TWIM'), - (0xD4F9, 'HANGUL SYLLABLE PYUB'), - (0xD762, 'HANGUL SYLLABLE HEUBS'), - (0xAE27, 'HANGUL SYLLABLE GYIS'), - (0xB090, 'HANGUL SYLLABLE GGISS'), - (0xB0AD, 'HANGUL SYLLABLE NANG'), - (0xB316, 'HANGUL SYLLABLE DAEJ'), - (0xB57F, 'HANGUL SYLLABLE DDYAC'), - (0xB7E8, 'HANGUL SYLLABLE RYAEK'), - (0xBA51, 'HANGUL SYLLABLE MEOT'), - (0xBCBA, 'HANGUL SYLLABLE BEP'), - (0xBF23, 'HANGUL SYLLABLE BBYEOH'), - (0xD7A3, 'HANGUL SYLLABLE HIH')): - assert unicodedata.name(unichr(code)) == name - assert unicodedata.lookup(name) == unichr(code) - # Test outside the range - raises(ValueError, unicodedata.name, unichr(0xAC00 - 1)) - raises(ValueError, unicodedata.name, unichr(0xD7A3 + 1)) - - def test_cjk(self): - import sys - import unicodedata - cases = ((0x3400, 0x4DB5), - (0x4E00, 0x9FA5)) - if unicodedata.unidata_version >= "5": # don't know the exact limit - cases = ((0x3400, 0x4DB5), - (0x4E00, 0x9FCB), - (0x20000, 0x2A6D6), - (0x2A700, 0x2B734)) - elif unicodedata.unidata_version >= "4.1": - cases = ((0x3400, 0x4DB5), - (0x4E00, 0x9FBB), - (0x20000, 0x2A6D6)) - for first, last in cases: - # Test at and inside the boundary - for i in (first, first + 1, last - 1, last): - charname = 'CJK UNIFIED IDEOGRAPH-%X'%i - char = ('\\U%08X' % i).decode('unicode-escape') - assert unicodedata.name(char) == charname - assert unicodedata.lookup(charname) == char - # Test outside the boundary - for i in first - 1, last + 1: - charname = 'CJK UNIFIED IDEOGRAPH-%X'%i - char = ('\\U%08X' % i).decode('unicode-escape') - try: - unicodedata.name(char) - except ValueError, e: - assert e.message == 'no such name' - raises(KeyError, unicodedata.lookup, charname) - - def test_bug_1704793(self): # from CPython - import unicodedata - assert unicodedata.lookup("GOTHIC LETTER FAIHU") == u'\U00010346' - - def test_normalize(self): - import unicodedata - raises(TypeError, unicodedata.normalize, 'x') - - def test_normalize_wide(self): - import sys, unicodedata - if sys.maxunicode < 0x10ffff: - skip("requires a 'wide' python build.") - assert unicodedata.normalize('NFC', u'\U000110a5\U000110ba') == u'\U000110ab' - - def test_linebreaks(self): - linebreaks = (0x0a, 0x0b, 0x0c, 0x0d, 0x85, - 0x1c, 0x1d, 0x1e, 0x2028, 0x2029) - for i in linebreaks: - for j in range(-2, 3): - lines = (unichr(i + j) + u'A').splitlines() - if i + j in linebreaks: - assert len(lines) == 2 - else: - assert len(lines) == 1 - - def test_mirrored(self): - import unicodedata - # For no reason, unicodedata.mirrored() returns an int, not a bool - assert repr(unicodedata.mirrored(u' ')) == '0' diff --git a/pypy/objspace/std/bytearraytype.py b/pypy/objspace/std/bytearraytype.py --- a/pypy/objspace/std/bytearraytype.py +++ b/pypy/objspace/std/bytearraytype.py @@ -15,6 +15,7 @@ str_splitlines, str_translate) from pypy.objspace.std.listtype import ( list_append, list_extend) +from rpython.rlib.objectmodel import newlist_hint, resizelist_hint bytearray_insert = SMM('insert', 3, @@ -87,8 +88,10 @@ return [c for c in string] # sequence of bytes - data = [] w_iter = space.iter(w_source) + length_hint = space.length_hint(w_source, 0) + data = newlist_hint(length_hint) + extended = 0 while True: try: w_item = space.next(w_iter) @@ -98,6 +101,9 @@ break value = getbytevalue(space, w_item) data.append(value) + extended += 1 + if extended < length_hint: + resizelist_hint(data, extended) return data def descr_bytearray__reduce__(space, w_self): diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py --- a/pypy/objspace/std/dictmultiobject.py +++ b/pypy/objspace/std/dictmultiobject.py @@ -54,6 +54,9 @@ # every module needs its own strategy, because the strategy stores # the version tag strategy = ModuleDictStrategy(space) + elif space.config.objspace.std.withmapdict and instance: + from pypy.objspace.std.mapdict import MapDictStrategy + strategy = space.fromcache(MapDictStrategy) elif instance or strdict or module: assert w_type is None @@ -349,7 +352,7 @@ self.pos = 0 def length(self): - if self.dictimplementation is not None: + if self.dictimplementation is not None and self.len != -1: return self.len - self.pos return 0 diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py --- a/pypy/objspace/std/mapdict.py +++ b/pypy/objspace/std/mapdict.py @@ -593,6 +593,9 @@ # ____________________________________________________________ # dict implementation +def get_terminator_for_dicts(space): + return DictTerminator(space, None) + class MapDictStrategy(DictStrategy): erase, unerase = rerased.new_erasing_pair("map") @@ -602,13 +605,19 @@ def __init__(self, space): self.space = space + def get_empty_storage(self): + w_result = Object() + terminator = self.space.fromcache(get_terminator_for_dicts) + w_result._init_empty(terminator) + return self.erase(w_result) + def switch_to_object_strategy(self, w_dict): w_obj = self.unerase(w_dict.dstorage) strategy = self.space.fromcache(ObjectDictStrategy) dict_w = strategy.unerase(strategy.get_empty_storage()) w_dict.strategy = strategy w_dict.dstorage = strategy.erase(dict_w) - assert w_obj.getdict(self.space) is w_dict + assert w_obj.getdict(self.space) is w_dict or w_obj._get_mapdict_map().terminator.w_cls is None materialize_r_dict(self.space, w_obj, dict_w) def getitem(self, w_dict, w_key): diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py --- a/pypy/objspace/std/setobject.py +++ b/pypy/objspace/std/setobject.py @@ -898,7 +898,7 @@ raise NotImplementedError def length(self): - if self.setimplementation is not None: + if self.setimplementation is not None and self.len != -1: return self.len - self.pos return 0 diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py --- a/pypy/objspace/std/test/test_dictmultiobject.py +++ b/pypy/objspace/std/test/test_dictmultiobject.py @@ -954,6 +954,7 @@ withcelldict = False withmethodcache = False withidentitydict = False + withmapdict = False FakeSpace.config = Config() diff --git a/pypy/objspace/std/test/test_lengthhint.py b/pypy/objspace/std/test/test_lengthhint.py --- a/pypy/objspace/std/test/test_lengthhint.py +++ b/pypy/objspace/std/test/test_lengthhint.py @@ -6,7 +6,7 @@ SIZE = 4 ITEMS = range(SIZE) - def _test_length_hint(self, w_obj): + def _test_length_hint(self, w_obj, w_mutate=None): space = self.space assert space.length_hint(w_obj, 8) == self.SIZE @@ -18,6 +18,13 @@ space.next(w_iter) assert space.length_hint(w_iter, 8) == self.SIZE - 1 + if w_mutate is not None: + # Test handling of collections that enforce length + # immutability during iteration + space.call_function(w_mutate) + space.raises_w(space.w_RuntimeError, space.next, w_iter) + assert space.length_hint(w_iter, 8) == 0 + def test_bytearray(self): space = self.space w_bytearray = space.call_function(space.w_bytearray, @@ -31,16 +38,20 @@ self._test_length_hint(w_dict) def test_dict_iterkeys(self): - w_iterkeys = self.space.appexec([], """(): - return dict.fromkeys(%r).iterkeys() - """ % self.ITEMS) - self._test_length_hint(w_iterkeys) + space = self.space + w_iterkeys, w_mutate = space.fixedview(space.appexec([], """(): + d = dict.fromkeys(%r) + return d.iterkeys(), d.popitem + """ % self.ITEMS), 2) + self._test_length_hint(w_iterkeys, w_mutate) def test_dict_values(self): - w_itervalues = self.space.appexec([], """(): - return dict.fromkeys(%r).itervalues() - """ % self.ITEMS) - self._test_length_hint(w_itervalues) + space = self.space + w_itervalues, w_mutate = space.fixedview(space.appexec([], """(): + d = dict.fromkeys(%r) + return d.itervalues(), d.popitem + """ % self.ITEMS), 2) + self._test_length_hint(w_itervalues, w_mutate) def test_frozenset(self): space = self.space @@ -50,7 +61,8 @@ def test_set(self): space = self.space w_set = space.call_function(space.w_set, space.wrap(self.ITEMS)) - self._test_length_hint(w_set) + w_mutate = space.getattr(w_set, space.wrap('pop')) + self._test_length_hint(w_set, w_mutate) def test_list(self): self._test_length_hint(self.space.wrap(self.ITEMS)) @@ -101,12 +113,19 @@ self._test_length_hint(W_Repeat(space, space.wrap(22), space.wrap(self.SIZE))) - def test_collections_deque(self): + def _setup_deque(self): space = self.space w_deque = W_Deque(space) space.call_method(w_deque, '__init__', space.wrap(self.ITEMS)) - self._test_length_hint(w_deque) - self._test_length_hint(w_deque.reviter()) + w_mutate = space.getattr(w_deque, space.wrap('pop')) + return w_deque, w_mutate + + def test_collections_deque(self): + self._test_length_hint(*self._setup_deque()) + + def test_collections_deque_rev(self): + w_deque, w_mutate = self._setup_deque() + self._test_length_hint(w_deque.reviter(), w_mutate) def test_default(self): space = self.space diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py --- a/pypy/objspace/std/test/test_mapdict.py +++ b/pypy/objspace/std/test/test_mapdict.py @@ -1,7 +1,17 @@ from pypy.objspace.std.test.test_dictmultiobject import FakeSpace, W_DictMultiObject from pypy.objspace.std.mapdict import * +class Config: + class objspace: + class std: + withsmalldicts = False + withcelldict = False + withmethodcache = False + withidentitydict = False + withmapdict = True + space = FakeSpace() +space.config = Config class Class(object): def __init__(self, hasdict=True): @@ -1061,3 +1071,11 @@ return A() """) assert w_dict.user_overridden_class + +def test_newdict_instance(): + w_dict = space.newdict(instance=True) + assert type(w_dict.strategy) is MapDictStrategy + +class TestMapDictImplementationUsingnewdict(BaseTestRDictImplementation): + StrategyClass = MapDictStrategy + # NB: the get_impl method is not overwritten here, as opposed to above diff --git a/pypy/pytest-A.cfg b/pypy/pytest-A.cfg --- a/pypy/pytest-A.cfg +++ b/pypy/pytest-A.cfg @@ -1,5 +1,5 @@ cherrypick = ['interpreter', 'objspace/test', 'objspace/std', 'module'] -interp = ['translator/goal/pypy-c'] +interp = ['goal/pypy-c'] test_driver = ['test_all.py', '-A'] diff --git a/pypy/test_all.py b/pypy/test_all.py --- a/pypy/test_all.py +++ b/pypy/test_all.py @@ -23,7 +23,8 @@ if len(sys.argv) == 1 and os.path.dirname(sys.argv[0]) in '.': print >> sys.stderr, __doc__ sys.exit(2) - + #Add toplevel repository dir to sys.path + sys.path.insert(0,os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) import pytest import pytest_cov sys.exit(pytest.main(plugins=[pytest_cov])) diff --git a/pypy/tool/pytest/confpath.py b/pypy/tool/pytest/confpath.py --- a/pypy/tool/pytest/confpath.py +++ b/pypy/tool/pytest/confpath.py @@ -1,8 +1,10 @@ import py import pypy +import rpython from pypy.tool import lib_pypy pypydir = py.path.local(pypy.__file__).dirpath() +rpythondir = py.path.local(rpython.__file__).dirpath() distdir = pypydir.dirpath() testresultdir = distdir.join('testresult') assert pypydir.check(dir=1) diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -11,8 +11,10 @@ import shutil import sys +import os +#Add toplevel repository dir to sys.path +sys.path.insert(0,os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))) import py -import os import fnmatch from rpython.tool.udir import udir @@ -48,7 +50,7 @@ basename = 'pypy-c' if sys.platform == 'win32': basename += '.exe' - pypy_c = basedir.join(basename) + pypy_c = basedir.join('pypy', 'goal', basename) else: pypy_c = py.path.local(override_pypy_c) if not pypy_c.check(): diff --git a/pypy/bin/translatorshell.py b/rpython/bin/translatorshell.py rename from pypy/bin/translatorshell.py rename to rpython/bin/translatorshell.py diff --git a/rpython/jit/backend/arm/test/support.py b/rpython/jit/backend/arm/test/support.py --- a/rpython/jit/backend/arm/test/support.py +++ b/rpython/jit/backend/arm/test/support.py @@ -14,6 +14,9 @@ def check_jumps(self, maxcount): pass +if not getattr(os, 'uname', None): + pytest.skip('cannot run arm tests on non-posix platform') + if os.uname()[1] == 'llaima.local': AS = '~/Code/arm-jit/android/android-ndk-r4b//build/prebuilt/darwin-x86/arm-eabi-4.4.0/arm-eabi/bin/as' else: diff --git a/rpython/jit/codewriter/effectinfo.py b/rpython/jit/codewriter/effectinfo.py --- a/rpython/jit/codewriter/effectinfo.py +++ b/rpython/jit/codewriter/effectinfo.py @@ -79,9 +79,11 @@ OS_RAW_MALLOC_VARSIZE = 110 OS_RAW_FREE = 111 + OS_JIT_FORCE_VIRTUAL = 120 + # for debugging: _OS_CANRAISE = set([OS_NONE, OS_STR2UNICODE, OS_LIBFFI_CALL, - OS_RAW_MALLOC_VARSIZE]) + OS_RAW_MALLOC_VARSIZE, OS_JIT_FORCE_VIRTUAL]) def __new__(cls, readonly_descrs_fields, readonly_descrs_arrays, write_descrs_fields, write_descrs_arrays, diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py --- a/rpython/jit/codewriter/jtransform.py +++ b/rpython/jit/codewriter/jtransform.py @@ -1393,6 +1393,8 @@ elif oopspec_name == 'jit.isvirtual': kind = getkind(args[0].concretetype) return SpaceOperation('%s_isvirtual' % kind, args, op.result) + elif oopspec_name == 'jit.force_virtual': + return self._handle_oopspec_call(op, args, EffectInfo.OS_JIT_FORCE_VIRTUAL, EffectInfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE) else: raise AssertionError("missing support for %r" % oopspec_name) diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py --- a/rpython/jit/codewriter/support.py +++ b/rpython/jit/codewriter/support.py @@ -1,24 +1,24 @@ import sys -from rpython.rtyper.lltypesystem import lltype, rclass, rffi, llmemory -from rpython.rtyper.ootypesystem import ootype -from rpython.rtyper import rlist -from rpython.rtyper.lltypesystem import rstr as ll_rstr, rdict as ll_rdict -from rpython.rtyper.lltypesystem.module import ll_math -from rpython.rtyper.lltypesystem.lloperation import llop -from rpython.rtyper.ootypesystem import rdict as oo_rdict -from rpython.rtyper.llinterp import LLInterpreter -from rpython.rtyper.extregistry import ExtRegistryEntry -from rpython.translator.simplify import get_funcobj -from rpython.translator.unsimplify import split_block + +from rpython.annotator import model as annmodel +from rpython.annotator.policy import AnnotatorPolicy from rpython.flowspace.model import Variable, Constant -from rpython.translator.translator import TranslationContext -from rpython.annotator.policy import AnnotatorPolicy -from rpython.annotator import model as annmodel -from rpython.rtyper.annlowlevel import MixLevelHelperAnnotator from rpython.jit.metainterp.typesystem import deref from rpython.rlib import rgc -from rpython.rlib.jit import elidable +from rpython.rlib.jit import elidable, oopspec from rpython.rlib.rarithmetic import r_longlong, r_ulonglong, r_uint, intmask +from rpython.rtyper import rlist +from rpython.rtyper.annlowlevel import MixLevelHelperAnnotator +from rpython.rtyper.extregistry import ExtRegistryEntry +from rpython.rtyper.llinterp import LLInterpreter +from rpython.rtyper.lltypesystem import lltype, rclass, rffi, llmemory, rstr as ll_rstr, rdict as ll_rdict +from rpython.rtyper.lltypesystem.lloperation import llop +from rpython.rtyper.lltypesystem.module import ll_math +from rpython.rtyper.ootypesystem import ootype, rdict as oo_rdict +from rpython.translator.simplify import get_funcobj +from rpython.translator.translator import TranslationContext +from rpython.translator.unsimplify import split_block + def getargtypes(annotator, values): if values is None: # for backend tests producing stand-alone exe's @@ -213,10 +213,12 @@ _ll_5_list_ll_arraycopy = rgc.ll_arraycopy + @elidable def _ll_1_gc_identityhash(x): return lltype.identityhash(x) + # the following function should not be "@elidable": I can think of # a corner case in which id(const) is constant-folded, and then 'const' # disappears and is collected too early (possibly causing another object @@ -224,6 +226,8 @@ def _ll_1_gc_id(ptr): return llop.gc_id(lltype.Signed, ptr) + + at oopspec("jit.force_virtual(inst)") def _ll_1_jit_force_virtual(inst): return llop.jit_force_virtual(lltype.typeOf(inst), inst) diff --git a/rpython/jit/metainterp/history.py b/rpython/jit/metainterp/history.py --- a/rpython/jit/metainterp/history.py +++ b/rpython/jit/metainterp/history.py @@ -985,11 +985,9 @@ def get_all_jitcell_tokens(self): tokens = [t() for t in self.jitcell_token_wrefs] if None in tokens: - assert False, "get_all_jitcell_tokens will not work as "+\ - "loops have been freed" + assert False, ("get_all_jitcell_tokens will not work as " + "loops have been freed") return tokens - - def check_history(self, expected=None, **check): insns = {} diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -1,12 +1,13 @@ +from rpython.jit.codewriter.effectinfo import EffectInfo +from rpython.jit.metainterp.executor import execute from rpython.jit.codewriter.heaptracker import vtable2descr -from rpython.jit.metainterp.executor import execute from rpython.jit.metainterp.history import Const, ConstInt, BoxInt from rpython.jit.metainterp.optimizeopt import optimizer +from rpython.jit.metainterp.optimizeopt.optimizer import OptValue, REMOVED from rpython.jit.metainterp.optimizeopt.util import (make_dispatcher_method, descrlist_dict, sort_descrs) from rpython.jit.metainterp.resoperation import rop, ResOperation from rpython.rlib.objectmodel import we_are_translated -from rpython.jit.metainterp.optimizeopt.optimizer import OptValue class AbstractVirtualValue(optimizer.OptValue): @@ -386,6 +387,24 @@ self.make_equal_to(box, vvalue) return vvalue + def optimize_GUARD_NO_EXCEPTION(self, op): + if self.last_emitted_operation is REMOVED: + return + self.emit_operation(op) + + def optimize_GUARD_NOT_FORCED(self, op): + if self.last_emitted_operation is REMOVED: + return + self.emit_operation(op) + + def optimize_CALL_MAY_FORCE(self, op): + effectinfo = op.getdescr().get_extra_info() + oopspecindex = effectinfo.oopspecindex + if oopspecindex == EffectInfo.OS_JIT_FORCE_VIRTUAL: + if self._optimize_JIT_FORCE_VIRTUAL(op): + return + self.emit_operation(op) + def optimize_VIRTUAL_REF(self, op): indexbox = op.getarg(1) # @@ -429,7 +448,7 @@ # - set 'virtual_token' to TOKEN_NONE args = [op.getarg(0), ConstInt(vrefinfo.TOKEN_NONE)] seo(ResOperation(rop.SETFIELD_GC, args, None, - descr = vrefinfo.descr_virtual_token)) + descr=vrefinfo.descr_virtual_token)) # Note that in some cases the virtual in op.getarg(1) has been forced # already. This is fine. In that case, and *if* a residual # CALL_MAY_FORCE suddenly turns out to access it, then it will @@ -437,6 +456,20 @@ # will work too (but just be a little pointless, as the structure # was already forced). + def _optimize_JIT_FORCE_VIRTUAL(self, op): + vref = self.getvalue(op.getarg(1)) + vrefinfo = self.optimizer.metainterp_sd.virtualref_info + if vref.is_virtual(): + tokenvalue = vref.getfield(vrefinfo.descr_virtual_token, None) + if (tokenvalue is not None and tokenvalue.is_constant() and + tokenvalue.box.getint() == vrefinfo.TOKEN_NONE): + forcedvalue = vref.getfield(vrefinfo.descr_forced, None) + if forcedvalue is not None and not forcedvalue.is_null(): + self.make_equal_to(op.result, forcedvalue) + self.last_emitted_operation = REMOVED + return True + return False + def optimize_GETFIELD_GC(self, op): value = self.getvalue(op.getarg(0)) # If this is an immutable field (as indicated by op.is_always_pure()) diff --git a/rpython/jit/metainterp/test/support.py b/rpython/jit/metainterp/test/support.py --- a/rpython/jit/metainterp/test/support.py +++ b/rpython/jit/metainterp/test/support.py @@ -160,16 +160,17 @@ class JitMixin: basic = True + def check_resops(self, expected=None, **check): get_stats().check_resops(expected=expected, **check) + def check_simple_loop(self, expected=None, **check): get_stats().check_simple_loop(expected=expected, **check) - - def check_trace_count(self, count): # was check_loop_count # The number of traces compiled assert get_stats().compiled_count == count + def check_trace_count_at_most(self, count): assert get_stats().compiled_count <= count @@ -178,11 +179,12 @@ def check_target_token_count(self, count): tokens = get_stats().get_all_jitcell_tokens() - n = sum ([len(t.target_tokens) for t in tokens]) + n = sum([len(t.target_tokens) for t in tokens]) assert n == count def check_enter_count(self, count): assert get_stats().enter_count == count + def check_enter_count_at_most(self, count): assert get_stats().enter_count <= count @@ -192,6 +194,7 @@ def check_aborted_count(self, count): assert get_stats().aborted_count == count + def check_aborted_count_at_least(self, count): assert get_stats().aborted_count >= count diff --git a/rpython/jit/metainterp/test/test_virtualref.py b/rpython/jit/metainterp/test/test_virtualref.py --- a/rpython/jit/metainterp/test/test_virtualref.py +++ b/rpython/jit/metainterp/test/test_virtualref.py @@ -1,19 +1,20 @@ import py -from rpython.rtyper.lltypesystem import lltype, llmemory, lloperation + +from rpython.rtyper.lltypesystem import lltype, lloperation from rpython.rtyper.exceptiondata import UnknownException from rpython.rlib.jit import JitDriver, dont_look_inside, vref_None from rpython.rlib.jit import virtual_ref, virtual_ref_finish, InvalidVirtualRef from rpython.rlib.jit import non_virtual_ref from rpython.rlib.objectmodel import compute_unique_id -from rpython.jit.metainterp.test.support import LLJitMixin, OOJitMixin, _get_jitcodes +from rpython.jit.metainterp.test.support import LLJitMixin, _get_jitcodes from rpython.jit.metainterp.resoperation import rop from rpython.jit.metainterp.virtualref import VirtualRefInfo + debug_print = lloperation.llop.debug_print -class VRefTests: - +class VRefTests(object): def finish_setup_for_interp_operations(self): self.vrefinfo = VirtualRefInfo(self.warmrunnerstate) self.cw.setup_vrefinfo(self.vrefinfo) @@ -116,7 +117,6 @@ cpu = self.metainterp.cpu cpu.get_int_value = lambda df,i:guard_op.getfailargs()[i].getint() cpu.get_ref_value = lambda df,i:guard_op.getfailargs()[i].getref_base() - cpu.clear_latest_values = lambda count: None class FakeMetaInterpSd: callinfocollection = None FakeMetaInterpSd.cpu = cpu @@ -417,15 +417,18 @@ self.check_aborted_count(0) def test_jit_force_virtual_seen(self): - myjitdriver = JitDriver(greens = [], reds = ['n']) - # + myjitdriver = JitDriver(greens=[], reds=['n']) + A = lltype.GcArray(lltype.Signed) - class XY: + + class XY(object): pass - class ExCtx: + + class ExCtx(object): pass exctx = ExCtx() - # + escapes = [] + def f(n): while n > 0: myjitdriver.can_enter_jit(n=n) @@ -433,16 +436,16 @@ xy = XY() xy.n = n exctx.topframeref = vref = virtual_ref(xy) + escapes.append(xy) xy.next1 = lltype.malloc(A, 0) n = exctx.topframeref().n - 1 - xy.next1 = lltype.nullptr(A) exctx.topframeref = vref_None virtual_ref_finish(vref, xy) return 1 # res = self.meta_interp(f, [15]) assert res == 1 - self.check_resops(new_with_vtable=4, # vref, xy + self.check_resops(new_with_vtable=2, # xy new_array=2) # next1 self.check_aborted_count(0) @@ -655,6 +658,34 @@ res = self.meta_interp(f, [10]) assert res == 0 + def test_force_virtual_vref(self): + myjitdriver = JitDriver(greens=[], reds=['n', 'ec']) + + class ExecutionContext(object): + pass + + class Frame(object): + def __init__(self, x): + self.x = x + + def f(n): + ec = ExecutionContext() + while n > 0: + myjitdriver.jit_merge_point(n=n, ec=ec) + frame = Frame(1) + ec.topframeref = virtual_ref(frame) + n -= ec.topframeref().x + frame_vref = ec.topframeref + ec.topframeref = vref_None + virtual_ref_finish(frame_vref, frame) + return n + res = self.meta_interp(f, [10]) + assert res == 0 + self.check_resops({ + 'int_sub': 2, 'int_gt': 2, 'jump': 1, 'guard_true': 2, + 'force_token': 2, 'setfield_gc': 1 + }) + class TestLLtype(VRefTests, LLJitMixin): pass diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -310,10 +310,7 @@ def decorate(func): from rpython.tool.sourcetools import compile2 # - def get_printable_location(): - return name - jitdriver = JitDriver(get_printable_location=get_printable_location, - greens=[], reds='auto', name=name) + jitdriver = JitDriver(greens=[], reds='auto', name=name) # args = ','.join(['a%d' % i for i in range(func.func_code.co_argcount)]) source = """def callback_with_jitdriver(%(args)s): @@ -484,8 +481,11 @@ self.autoreds = True self.reds = [] self.numreds = None # see warmspot.autodetect_jit_markers_redvars - assert confirm_enter_jit is None, ( - "reds='auto' is not compatible with confirm_enter_jit") + for hook in [ + get_jitcell_at, set_jitcell_at, get_printable_location, + confirm_enter_jit + ]: + assert hook is None, "reds='auto' is not compatible with JitDriver hooks" else: if reds is not None: self.reds = reds diff --git a/rpython/rlib/parsing/test/test_pcre_regtest.py b/rpython/rlib/parsing/test/test_pcre_regtest.py --- a/rpython/rlib/parsing/test/test_pcre_regtest.py +++ b/rpython/rlib/parsing/test/test_pcre_regtest.py @@ -84,8 +84,7 @@ from rpython.rlib.parsing.regexparse import make_runner, unescape import string import re -import os -this_dir = py.path.local(os.path.realpath(os.path.dirname(__file__))) +this_dir = py.path.local(__file__).join('..') #py.test.skip("Still in progress") diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py --- a/rpython/rlib/runicode.py +++ b/rpython/rlib/runicode.py @@ -58,26 +58,6 @@ # generate surrogates for large codes return UNICHR(code) - -def UNICHR(c): - if c <= sys.maxunicode and c <= MAXUNICODE: - return unichr(c) - else: - c -= 0x10000 - return (unichr(0xD800 + (c >> 10)) + - unichr(0xDC00 + (c & 0x03FF))) - -def ORD(u): - assert isinstance(u, unicode) - if len(u) == 1: - return ord(u[0]) - elif len(u) == 2: - ch1 = ord(u[0]) - ch2 = ord(u[1]) - if 0xD800 <= ch1 <= 0xDBFF and 0xDC00 <= ch2 <= 0xDFFF: - return (((ch1 - 0xD800) << 10) | (ch2 - 0xDC00)) + 0x10000 - raise ValueError - def _STORECHAR(result, CH, byteorder): hi = chr(((CH) >> 8) & 0xff) lo = chr((CH) & 0xff) diff --git a/rpython/rlib/unicodedata/test/test_ucd.py b/rpython/rlib/unicodedata/test/test_ucd.py --- a/rpython/rlib/unicodedata/test/test_ucd.py +++ b/rpython/rlib/unicodedata/test/test_ucd.py @@ -1,9 +1,9 @@ +from rpython.rlib.runicode import code_to_unichr +from rpython.rlib.unicodedata import unicodedb_5_2_0 from rpython.rtyper.test.tool import BaseRtypingTest, LLRtypeMixin -from rpython.rlib.unicodedata import unicodedb_5_2_0 -from rpython.rlib.unicodedata.ucd import code_to_unichr + class TestTranslated(BaseRtypingTest, LLRtypeMixin): - def test_translated(self): def f(n): if n == 0: diff --git a/rpython/rlib/unicodedata/test/test_unicodedata.py b/rpython/rlib/unicodedata/test/test_unicodedata.py --- a/rpython/rlib/unicodedata/test/test_unicodedata.py +++ b/rpython/rlib/unicodedata/test/test_unicodedata.py @@ -1,11 +1,15 @@ +import random +import unicodedata + import py + from rpython.rlib.unicodedata import unicodedb_3_2_0, unicodedb_5_2_0 + class TestUnicodeData(object): def setup_class(cls): - import random, unicodedata if unicodedata.unidata_version != '5.2.0': - skip('Needs python with unicode 5.2.0 database.') + py.test.skip('Needs python with unicode 5.2.0 database.') seed = random.getrandbits(32) print "random seed: ", seed @@ -29,14 +33,12 @@ py.test.raises(KeyError, unicodedb_5_2_0.name, ord(chr)) def test_compare_functions(self): - import unicodedata # CPython implementation - def getX(fun, code): try: return getattr(unicodedb_5_2_0, fun)(code) except KeyError: return -1 - + for code in range(0x10000): char = unichr(code) assert unicodedata.digit(char, -1) == getX('digit', code) @@ -73,5 +75,3 @@ assert unicodedb_5_2_0.lookup('BENZENE RING WITH CIRCLE') == 9187 py.test.raises(KeyError, unicodedb_3_2_0.lookup, 'BENZENE RING WITH CIRCLE') py.test.raises(KeyError, unicodedb_3_2_0.name, 9187) - - diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py --- a/rpython/translator/c/genc.py +++ b/rpython/translator/c/genc.py @@ -131,7 +131,7 @@ self.secondary_entrypoints = secondary_entrypoints def get_eci(self): - pypy_include_dir = py.path.local(os.path.realpath(os.path.dirname(__file__))) + pypy_include_dir = py.path.local(__file__).join('..') include_dirs = [pypy_include_dir] return ExternalCompilationInfo(include_dirs=include_dirs) @@ -753,7 +753,7 @@ defines['PYPY_LONGLONG_BIT'] = LONGLONG_BIT def add_extra_files(eci): - srcdir = py.path.local(os.path.realpath(os.path.dirname(__file__))).join('src') + srcdir = py.path.local(__file__).join('..', 'src') files = [ srcdir / 'entrypoint.c', # ifdef PYPY_STANDALONE srcdir / 'allocator.c', # ifdef PYPY_STANDALONE diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py --- a/rpython/translator/platform/posix.py +++ b/rpython/translator/platform/posix.py @@ -5,7 +5,7 @@ from rpython.translator.platform import Platform, log, _run_subprocess import rpython -rpydir = os.path.dirname(rpython.__file__) +rpydir = str(py.path.local(rpython.__file__).join('..')) class BasePosix(Platform): exe_ext = '' diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -7,7 +7,7 @@ from rpython.translator.platform import Platform, posix import rpython -rpydir = os.path.dirname(rpython.__file__) +rpydir = str(py.path.local(rpython.__file__).join('..')) def _get_compiler_type(cc, x64_flag): import subprocess From noreply at buildbot.pypy.org Fri Jan 25 15:58:57 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 25 Jan 2013 15:58:57 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: update the TODO Message-ID: <20130125145857.541351C1069@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60453:9187ef7004a1 Date: 2013-01-25 16:58 +0200 http://bitbucket.org/pypy/pypy/changeset/9187ef7004a1/ Log: update the TODO diff --git a/pypy/TODO b/pypy/TODO --- a/pypy/TODO +++ b/pypy/TODO @@ -1,9 +1,8 @@ * 32bit x86 * ARM -* shadowstack + asmgcc (shadowstack needs reloading of ebp after frame - got potentially moved, which is after each potentially collecting call - or slowpath malloc) +* asmgcc +* shadowstack details - slowpath of stack check and call_release_gil * kill jit2gc on translator * fix test_singlefloats in test_calling_conventions * slowpaths can have more variants depending on how many registers we're using From noreply at buildbot.pypy.org Fri Jan 25 16:24:42 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 25 Jan 2013 16:24:42 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fix pypyjit module Message-ID: <20130125152442.247101C047D@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60454:da25aa3490be Date: 2013-01-25 17:24 +0200 http://bitbucket.org/pypy/pypy/changeset/da25aa3490be/ Log: fix pypyjit module diff --git a/pypy/TODO b/pypy/TODO --- a/pypy/TODO +++ b/pypy/TODO @@ -6,4 +6,5 @@ * kill jit2gc on translator * fix test_singlefloats in test_calling_conventions * slowpaths can have more variants depending on how many registers we're using - (like floats vs non-floats for failures) \ No newline at end of file + (like floats vs non-floats for failures) +* fix jit hooks \ No newline at end of file diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py --- a/pypy/module/pypyjit/interp_resop.py +++ b/pypy/module/pypyjit/interp_resop.py @@ -5,14 +5,14 @@ from pypy.interpreter.gateway import unwrap_spec, interp2app from pypy.interpreter.pycode import PyCode from pypy.interpreter.error import OperationError -from rpython.rtyper.lltypesystem import lltype, llmemory +from rpython.rtyper.lltypesystem import lltype from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance, hlstr from rpython.rtyper.lltypesystem.rclass import OBJECT -from rpython.jit.metainterp.resoperation import rop, AbstractResOp +from rpython.jit.metainterp.resoperation import rop from rpython.rlib.nonconst import NonConstant from rpython.rlib import jit_hooks from rpython.rlib.jit import Counters -from rpython.rlib.rarithmetic import r_uint +from rpython.rlib.objectmodel import compute_unique_id from pypy.module.pypyjit.interp_jit import pypyjitdriver class Cache(object): @@ -270,7 +270,8 @@ self.jd_name = debug_info.get_jitdriver().name self.type = debug_info.type if is_bridge: - self.bridge_no = debug_info.fail_descr_no + self.bridge_no = compute_unique_id(debug_info.fail_descr) + #self.bridge_no = debug_info.fail_descr_no self.w_green_key = space.w_None else: self.w_green_key = wrap_greenkey(space, diff --git a/pypy/module/pypyjit/test/test_jit_hook.py b/pypy/module/pypyjit/test/test_jit_hook.py --- a/pypy/module/pypyjit/test/test_jit_hook.py +++ b/pypy/module/pypyjit/test/test_jit_hook.py @@ -2,7 +2,8 @@ import py from pypy.interpreter.gateway import interp2app from pypy.interpreter.pycode import PyCode -from rpython.jit.metainterp.history import JitCellToken, ConstInt, ConstPtr +from rpython.jit.metainterp.history import JitCellToken, ConstInt, ConstPtr,\ + BasicFailDescr from rpython.jit.metainterp.resoperation import rop from rpython.jit.metainterp.logger import Logger from rpython.rtyper.annlowlevel import (cast_instance_to_base_ptr, @@ -69,7 +70,7 @@ oplist, 'loop', greenkey) di_loop.asminfo = AsmInfo(offset, 0, 0) di_bridge = JitDebugInfo(MockJitDriverSD, logger, JitCellToken(), - oplist, 'bridge', fail_descr_no=0) + oplist, 'bridge', fail_descr=BasicFailDescr()) di_bridge.asminfo = AsmInfo(offset, 0, 0) def interp_on_compile(): diff --git a/rpython/jit/backend/x86/test/test_gc_integration.py b/rpython/jit/backend/x86/test/test_gc_integration.py --- a/rpython/jit/backend/x86/test/test_gc_integration.py +++ b/rpython/jit/backend/x86/test/test_gc_integration.py @@ -564,3 +564,17 @@ assert thing == rffi.cast(lltype.Signed, cpu.gc_ll_descr.nursery) assert cpu.gc_ll_descr.nursery_ptrs[0] == thing + sizeof.size assert rffi.cast(JITFRAMEPTR, cpu.gc_ll_descr.write_barrier_on_frame_called) == frame + + def test_call_release_gil(self): + cpu = self.cpu + + def f(x): + import pdb + pdb.set_trace() + + loop = self.parse(""" + [f0] + f1 = call_release_gil(ConstClass(fptr), f0, descr=calldescr) + finish(f1) + """, namespace={ + }) From noreply at buildbot.pypy.org Fri Jan 25 16:47:42 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 25 Jan 2013 16:47:42 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: write a failing test and remove tests that don't test anything (especially Message-ID: <20130125154742.98CBF1C0041@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60455:a84a39aa0758 Date: 2013-01-25 17:47 +0200 http://bitbucket.org/pypy/pypy/changeset/a84a39aa0758/ Log: write a failing test and remove tests that don't test anything (especially not what they advertise to test) diff --git a/rpython/jit/backend/x86/test/test_gc_integration.py b/rpython/jit/backend/x86/test/test_gc_integration.py --- a/rpython/jit/backend/x86/test/test_gc_integration.py +++ b/rpython/jit/backend/x86/test/test_gc_integration.py @@ -397,7 +397,8 @@ self.nursery_ptrs[1] = self.nursery_ptrs[0] + nursery_size self.nursery_addr = rffi.cast(lltype.Signed, self.nursery_ptrs) self.all_nurseries.append(self.nursery) - self.collections.reverse() + if hasattr(self, 'collections'): + self.collections.reverse() def _collect(self): gcmap = unpack_gcmap(self.frames[-1]) @@ -571,10 +572,20 @@ def f(x): import pdb pdb.set_trace() - + + FUNC = lltype.FuncType([lltype.Float], lltype.Float) + fptr = llhelper(lltype.Ptr(FUNC), f) + calldescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, + EffectInfo.MOST_GENERAL) loop = self.parse(""" [f0] f1 = call_release_gil(ConstClass(fptr), f0, descr=calldescr) + guard_not_forced(descr=faildescr) [] finish(f1) - """, namespace={ - }) + """, namespace={'fptr': fptr, 'calldescr':calldescr, + 'faildescr': BasicFailDescr()}) + token = JitCellToken() + cpu.gc_ll_descr.init_nursery(100) + cpu.setup_once() + cpu.compile_loop(loop.inputargs, loop.operations, token) + cpu.execute_token(token, 1.3) diff --git a/rpython/jit/backend/x86/test/test_zrpy_gc.py b/rpython/jit/backend/x86/test/test_zrpy_gc.py --- a/rpython/jit/backend/x86/test/test_zrpy_gc.py +++ b/rpython/jit/backend/x86/test/test_zrpy_gc.py @@ -790,94 +790,5 @@ def test_compile_framework_minimal_size_in_nursery(self): self.run('compile_framework_minimal_size_in_nursery') - def define_simple_call_release_gil(self): - class Glob: - pass - glob = Glob() - # - def f42(n): - c_strchr = glob.c_strchr - raw = rffi.str2charp("foobar" + chr((n & 63) + 32)) - argchain = ArgChain() - argchain = argchain.arg(rffi.cast(lltype.Signed, raw)) - argchain = argchain.arg(rffi.cast(rffi.INT, ord('b'))) - res = c_strchr.call(argchain, rffi.CCHARP) - check(rffi.charp2str(res) == "bar" + chr((n & 63) + 32)) - rffi.free_charp(raw) - # - def before(n, x): - libc = CDLL(libc_name) - c_strchr = libc.getpointer('strchr', [types.pointer, types.sint], - types.pointer) - glob.c_strchr = c_strchr - return (n, None, None, None, None, None, - None, None, None, None, None, None) - # - def f(n, x, *args): - f42(n) - n -= 1 - return (n, x) + args - return before, f, None - - def test_simple_call_release_gil(self): - self.run('simple_call_release_gil') - - def define_close_stack(self): - # - class Glob(object): - pass - glob = Glob() - class X(object): - pass - # - def callback(p1, p2): - for i in range(100): - glob.lst.append(X()) - return rffi.cast(rffi.INT, 1) - CALLBACK = lltype.Ptr(lltype.FuncType([lltype.Signed, - lltype.Signed], rffi.INT)) - # - @dont_look_inside - def alloc1(): - return llmemory.raw_malloc(16) - @dont_look_inside - def free1(p): - llmemory.raw_free(p) - # - def f42(): - length = len(glob.lst) - c_qsort = glob.c_qsort - raw = alloc1() - fn = llhelper(CALLBACK, rffi._make_wrapper_for(CALLBACK, callback)) - argchain = ArgChain() - argchain = argchain.arg(rffi.cast(lltype.Signed, raw)) - argchain = argchain.arg(rffi.cast(rffi.SIZE_T, 2)) - argchain = argchain.arg(rffi.cast(rffi.SIZE_T, 8)) - argchain = argchain.arg(rffi.cast(lltype.Signed, fn)) - c_qsort.call(argchain, lltype.Void) - free1(raw) - check(len(glob.lst) > length) - del glob.lst[:] - # - def before(n, x): - libc = CDLL(libc_name) - types_size_t = clibffi.cast_type_to_ffitype(rffi.SIZE_T) - c_qsort = libc.getpointer('qsort', [types.pointer, types_size_t, - types_size_t, types.pointer], - types.void) - glob.c_qsort = c_qsort - glob.lst = [] - return (n, None, None, None, None, None, - None, None, None, None, None, None) - # - def f(n, x, *args): - f42() - n -= 1 - return (n, x) + args - return before, f, None - - def test_close_stack(self): - self.run('close_stack') - class TestShadowStack(CompileFrameworkTests): gcrootfinder = "shadowstack" From noreply at buildbot.pypy.org Fri Jan 25 17:04:56 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 25 Jan 2013 17:04:56 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: finish the test and fix the code Message-ID: <20130125160456.4245F1C0041@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60456:a57b539e5294 Date: 2013-01-25 18:04 +0200 http://bitbucket.org/pypy/pypy/changeset/a57b539e5294/ Log: finish the test and fix the code diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -2112,16 +2112,12 @@ # first, close the stack in the sense of the asmgcc GC root tracker gcrootmap = self.cpu.gc_ll_descr.gcrootmap if gcrootmap: - raise NotImplementedError # we need tests - xxx self.call_release_gil(gcrootmap, arglocs) # do the call self._store_force_index(guard_op) self.genop_call(op, arglocs, result_loc) # then reopen the stack if gcrootmap: - raise NotImplementedError # we need tests - xxx self.call_reacquire_gil(gcrootmap, result_loc) # finally, the guard_not_forced self._emit_guard_not_forced(guard_token) @@ -2199,7 +2195,6 @@ if reg in save_registers: self.mc.MOV_rs(reg.value, p) p += WORD - self._regalloc.needed_extra_stack_locations(p//WORD) def call_reacquire_gil(self, gcrootmap, save_loc): # save the previous result (eax/xmm0) into the stack temporarily. @@ -2211,6 +2206,7 @@ if gcrootmap.is_shadow_stack: args = [] else: + xxx assert self.gcrootmap_retaddr_forced == -1, ( "missing mark_gc_roots() in CALL_RELEASE_GIL") self.gcrootmap_retaddr_forced = 0 @@ -2226,7 +2222,6 @@ # restore the result from the stack if isinstance(save_loc, RegLoc) and not save_loc.is_xmm: self.mc.MOV_rs(save_loc.value, WORD) - self._regalloc.needed_extra_stack_locations(2) def genop_guard_call_assembler(self, op, guard_op, guard_token, arglocs, result_loc): diff --git a/rpython/jit/backend/x86/test/test_gc_integration.py b/rpython/jit/backend/x86/test/test_gc_integration.py --- a/rpython/jit/backend/x86/test/test_gc_integration.py +++ b/rpython/jit/backend/x86/test/test_gc_integration.py @@ -13,9 +13,8 @@ from rpython.rtyper.annlowlevel import llhelper, llhelper_args from rpython.jit.backend.x86.test.test_regalloc import BaseTestRegalloc -from rpython.jit.backend.x86.regalloc import gpr_reg_mgr_cls from rpython.jit.codewriter.effectinfo import EffectInfo -from rpython.rtyper.memory.gcheader import GCHeaderBuilder +from rpython.rlib.objectmodel import invoke_around_extcall CPU = getcpuclass() @@ -347,6 +346,7 @@ ('hdr', lltype.Signed), ('jf_frame_info', lltype.Ptr(jitframe.JITFRAMEINFO)), ('jf_descr', llmemory.GCREF), + ('jf_force_descr', llmemory.GCREF), ('jf_guard_exc', llmemory.GCREF), ('jf_gcmap', lltype.Ptr(jitframe.GCMAP)), ('jf_gc_trace_state', lltype.Signed), @@ -423,7 +423,7 @@ def getframedescrs(self, cpu): descrs = JitFrameDescrs() descrs.arraydescr = cpu.arraydescrof(JITFRAME) - for name in ['jf_descr', 'jf_guard_exc', + for name in ['jf_descr', 'jf_guard_exc', 'jf_force_descr', 'jf_frame_info', 'jf_gcmap']: setattr(descrs, name, cpu.fielddescrof(JITFRAME, name)) descrs.jfi_frame_depth = cpu.fielddescrof(jitframe.JITFRAMEINFO, @@ -567,25 +567,40 @@ assert rffi.cast(JITFRAMEPTR, cpu.gc_ll_descr.write_barrier_on_frame_called) == frame def test_call_release_gil(self): + # note that we can't test floats here because when untranslated + # people actually wreck xmm registers cpu = self.cpu + l = [] + + def before(): + l.append("before") + + def after(): + l.append("after") + + invoke_around_extcall(before, after) def f(x): - import pdb - pdb.set_trace() - - FUNC = lltype.FuncType([lltype.Float], lltype.Float) + assert x == 1 + return 2 + + FUNC = lltype.FuncType([lltype.Signed], lltype.Signed) fptr = llhelper(lltype.Ptr(FUNC), f) calldescr = cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, EffectInfo.MOST_GENERAL) loop = self.parse(""" - [f0] - f1 = call_release_gil(ConstClass(fptr), f0, descr=calldescr) + [i0] + i1 = call_release_gil(ConstClass(fptr), i0, descr=calldescr) guard_not_forced(descr=faildescr) [] - finish(f1) + finish(i1, descr=finaldescr) """, namespace={'fptr': fptr, 'calldescr':calldescr, - 'faildescr': BasicFailDescr()}) + 'faildescr': BasicFailDescr(), + 'finaldescr': BasicFinalDescr()}) token = JitCellToken() cpu.gc_ll_descr.init_nursery(100) cpu.setup_once() cpu.compile_loop(loop.inputargs, loop.operations, token) - cpu.execute_token(token, 1.3) + frame = cpu.execute_token(token, 1) + frame = rffi.cast(JITFRAMEPTR, frame) + assert frame.jf_frame[0] == 2 + assert l == ['before', 'after'] From noreply at buildbot.pypy.org Fri Jan 25 17:05:31 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 25 Jan 2013 17:05:31 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: this is done Message-ID: <20130125160531.5AAE81C0041@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60457:a342648c488b Date: 2013-01-25 18:05 +0200 http://bitbucket.org/pypy/pypy/changeset/a342648c488b/ Log: this is done diff --git a/pypy/TODO b/pypy/TODO --- a/pypy/TODO +++ b/pypy/TODO @@ -2,7 +2,7 @@ * 32bit x86 * ARM * asmgcc -* shadowstack details - slowpath of stack check and call_release_gil +* shadowstack details - slowpath of stack check * kill jit2gc on translator * fix test_singlefloats in test_calling_conventions * slowpaths can have more variants depending on how many registers we're using From noreply at buildbot.pypy.org Fri Jan 25 17:12:27 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 25 Jan 2013 17:12:27 +0100 (CET) Subject: [pypy-commit] jitviewer default: update imports Message-ID: <20130125161227.AED0D1C0041@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r207:edf3efa8ac6e Date: 2013-01-25 18:11 +0200 http://bitbucket.org/pypy/jitviewer/changeset/edf3efa8ac6e/ Log: update imports diff --git a/_jitviewer/app.py b/_jitviewer/app.py --- a/_jitviewer/app.py +++ b/_jitviewer/app.py @@ -44,9 +44,9 @@ import inspect import threading import time -from pypy.tool.logparser import extract_category -from pypy.tool.jitlogparser.storage import LoopStorage -from pypy.tool.jitlogparser.parser import adjust_bridges, import_log,\ +from rpython.tool.logparser import extract_category +from rpython.tool.jitlogparser.storage import LoopStorage +from rpython.tool.jitlogparser.parser import adjust_bridges, import_log,\ parse_log_counts # from _jitviewer.parser import ParserWithHtmlRepr, FunctionHtml diff --git a/_jitviewer/parser.py b/_jitviewer/parser.py --- a/_jitviewer/parser.py +++ b/_jitviewer/parser.py @@ -1,6 +1,6 @@ import re import cgi -from pypy.tool.jitlogparser import parser +from rpython.tool.jitlogparser import parser def cssclass(cls, s, **kwds): cls = re.sub("[^\w]", "_", cls) From noreply at buildbot.pypy.org Fri Jan 25 17:12:28 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 25 Jan 2013 17:12:28 +0100 (CET) Subject: [pypy-commit] jitviewer default: merge Message-ID: <20130125161228.D09D81C0041@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r208:635770d24676 Date: 2013-01-25 18:12 +0200 http://bitbucket.org/pypy/jitviewer/changeset/635770d24676/ Log: merge diff --git a/_jitviewer/app.py b/_jitviewer/app.py --- a/_jitviewer/app.py +++ b/_jitviewer/app.py @@ -147,7 +147,7 @@ loop.startlineno))) loop = loop.chunks[int(e)] path_so_far.append(e) - callstack.append((','.join(path_so_far), '%s in %s at %d' % (loop.name, + callstack.append((','.join(path_so_far), '%s in %s:%d' % (loop.name, loop.filename, loop.startlineno))) if not loop.has_valid_code() or loop.filename is None: diff --git a/_jitviewer/static/script.js b/_jitviewer/static/script.js --- a/_jitviewer/static/script.js +++ b/_jitviewer/static/script.js @@ -1,6 +1,7 @@ var glob_bridge_state = { 'asm': false, + 'bytecodepos': false, 'op': true, }; @@ -34,6 +35,9 @@ if (!glob_bridge_state.asm) { $(".asm").hide(); } + if (!glob_bridge_state.bytecodepos) { + $(".bytecodepos").hide(); + } }); } @@ -83,6 +87,9 @@ if (!glob_bridge_state.asm) { $(".asm").hide(); } + if (!glob_bridge_state.bytecodepos) { + $(".bytecodepos").hide(); + } $.scrollTo($("#loop-" + bridge_id), {axis:'y'}); }); }); @@ -103,6 +110,21 @@ } } +function bytecodepos_toggle() +{ + var e = $("#bytecodepos_toggler"); + var e2 = $(".bytecodepos"); + if (e.html().search("Show") != -1) { + glob_bridge_state.bytecodepos = true; + e.html("Hide bytecode position"); + e2.show(); + } else { + glob_bridge_state.bytecodepos = false; + e.html("Show bytecode position"); + e2.hide(); + } +} + function highlight_var(elem) { $('.' + elem.className).addClass("variable_highlight"); diff --git a/_jitviewer/templates/index.html b/_jitviewer/templates/index.html --- a/_jitviewer/templates/index.html +++ b/_jitviewer/templates/index.html @@ -17,7 +17,8 @@
Menu
- Show assembler + Show assembler
+ Show bytecode position
diff --git a/_jitviewer/templates/loop.html b/_jitviewer/templates/loop.html --- a/_jitviewer/templates/loop.html +++ b/_jitviewer/templates/loop.html @@ -9,7 +9,7 @@
{% for chunk in sourceline.chunks %} {% if chunk.is_bytecode %} - {{chunk.html_repr()}}
+ {{chunk.bytecode_no}} {{chunk.html_repr()}}
{% for op in chunk.operations %} {% if op.name != "debug_merge_point" %} {% if op.bridge %} From noreply at buildbot.pypy.org Fri Jan 25 17:14:05 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 25 Jan 2013 17:14:05 +0100 (CET) Subject: [pypy-commit] jitviewer default: I was wrong, jitlogparser is still in pypy Message-ID: <20130125161405.8FBCF1C0041@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r209:4aa321f2491e Date: 2013-01-25 18:13 +0200 http://bitbucket.org/pypy/jitviewer/changeset/4aa321f2491e/ Log: I was wrong, jitlogparser is still in pypy diff --git a/_jitviewer/app.py b/_jitviewer/app.py --- a/_jitviewer/app.py +++ b/_jitviewer/app.py @@ -45,8 +45,8 @@ import threading import time from rpython.tool.logparser import extract_category -from rpython.tool.jitlogparser.storage import LoopStorage -from rpython.tool.jitlogparser.parser import adjust_bridges, import_log,\ +from pypy.tool.jitlogparser.storage import LoopStorage +from pypy.tool.jitlogparser.parser import adjust_bridges, import_log,\ parse_log_counts # from _jitviewer.parser import ParserWithHtmlRepr, FunctionHtml diff --git a/_jitviewer/parser.py b/_jitviewer/parser.py --- a/_jitviewer/parser.py +++ b/_jitviewer/parser.py @@ -1,6 +1,6 @@ import re import cgi -from rpython.tool.jitlogparser import parser +from pypy.tool.jitlogparser import parser def cssclass(cls, s, **kwds): cls = re.sub("[^\w]", "_", cls) From noreply at buildbot.pypy.org Fri Jan 25 17:32:55 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Fri, 25 Jan 2013 17:32:55 +0100 (CET) Subject: [pypy-commit] pypy default: Reverted cc89614c9009 Message-ID: <20130125163255.222F11C047D@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: Changeset: r60458:b0869953bfc2 Date: 2013-01-25 00:33 -0600 http://bitbucket.org/pypy/pypy/changeset/b0869953bfc2/ Log: Reverted cc89614c9009 diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -484,11 +484,8 @@ self.autoreds = True self.reds = [] self.numreds = None # see warmspot.autodetect_jit_markers_redvars - for hook in [ - get_jitcell_at, set_jitcell_at, get_printable_location, - confirm_enter_jit - ]: - assert hook is None, "reds='auto' is not compatible with JitDriver hooks" + assert confirm_enter_jit is None, ( + "reds='auto' is not compatible with confirm_enter_jit") else: if reds is not None: self.reds = reds From noreply at buildbot.pypy.org Fri Jan 25 17:32:56 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Fri, 25 Jan 2013 17:32:56 +0100 (CET) Subject: [pypy-commit] pypy default: merged upstream Message-ID: <20130125163256.5C3571C1055@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: Changeset: r60459:0d0920964cd1 Date: 2013-01-25 10:32 -0600 http://bitbucket.org/pypy/pypy/changeset/0d0920964cd1/ Log: merged upstream diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -310,10 +310,7 @@ def decorate(func): from rpython.tool.sourcetools import compile2 # - def get_printable_location(): - return name - jitdriver = JitDriver(get_printable_location=get_printable_location, - greens=[], reds='auto', name=name) + jitdriver = JitDriver(greens=[], reds='auto', name=name) # args = ','.join(['a%d' % i for i in range(func.func_code.co_argcount)]) source = """def callback_with_jitdriver(%(args)s): From noreply at buildbot.pypy.org Fri Jan 25 17:36:01 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Fri, 25 Jan 2013 17:36:01 +0100 (CET) Subject: [pypy-commit] pypy default: rewrite list inplace mul in terms of ll_arraycopy, less code and probably faster Message-ID: <20130125163601.64FEA1C047D@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: Changeset: r60460:1d664ad717ce Date: 2013-01-25 10:35 -0600 http://bitbucket.org/pypy/pypy/changeset/1d664ad717ce/ Log: rewrite list inplace mul in terms of ll_arraycopy, less code and probably faster diff --git a/rpython/rtyper/rlist.py b/rpython/rtyper/rlist.py --- a/rpython/rtyper/rlist.py +++ b/rpython/rtyper/rlist.py @@ -1025,11 +1025,7 @@ res._ll_resize(resultlen) j = length while j < resultlen: - i = 0 - while i < length: - p = j + i - res.ll_setitem_fast(p, l.ll_getitem_fast(i)) - i += 1 + ll_arraycopy(l, res, 0, j, length) j += length return res ll_inplace_mul.oopspec = 'list.inplace_mul(l, factor)' From noreply at buildbot.pypy.org Fri Jan 25 17:39:14 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Fri, 25 Jan 2013 17:39:14 +0100 (CET) Subject: [pypy-commit] pypy default: Also rewrite list mul to use arraycopy Message-ID: <20130125163914.0F90D1C047D@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: Changeset: r60461:dd91b79534e6 Date: 2013-01-25 10:39 -0600 http://bitbucket.org/pypy/pypy/changeset/dd91b79534e6/ Log: Also rewrite list mul to use arraycopy diff --git a/rpython/rtyper/rlist.py b/rpython/rtyper/rlist.py --- a/rpython/rtyper/rlist.py +++ b/rpython/rtyper/rlist.py @@ -1042,11 +1042,7 @@ res = RESLIST.ll_newlist(resultlen) j = 0 while j < resultlen: - i = 0 - while i < length: - p = j + i - res.ll_setitem_fast(p, l.ll_getitem_fast(i)) - i += 1 + ll_arraycopy(l, res, 0, j, length) j += length return res # not inlined by the JIT -- contains a loop From noreply at buildbot.pypy.org Fri Jan 25 17:40:30 2013 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 25 Jan 2013 17:40:30 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: Fix a few tests, skip the others Message-ID: <20130125164030.303DF1C047D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r60462:50eabd063701 Date: 2013-01-21 17:12 +0100 http://bitbucket.org/pypy/pypy/changeset/50eabd063701/ Log: Fix a few tests, skip the others diff --git a/pypy/rpython/memory/gc/stmshared.py b/pypy/rpython/memory/gc/stmshared.py --- a/pypy/rpython/memory/gc/stmshared.py +++ b/pypy/rpython/memory/gc/stmshared.py @@ -33,7 +33,7 @@ def add_regular(self, obj): """After malloc_object(), register the object in the internal chained - list. For objects whose 'version' field is not otherwise needed.""" + list. For objects whose 'revision' field is not otherwise needed.""" self.gc.set_obj_revision(obj, self.chained_list) self.chained_list = obj diff --git a/pypy/rpython/memory/gc/test/test_stmgc.py b/pypy/rpython/memory/gc/test/test_stmgc.py --- a/pypy/rpython/memory/gc/test/test_stmgc.py +++ b/pypy/rpython/memory/gc/test/test_stmgc.py @@ -1,10 +1,11 @@ import py +py.test.skip("XXX fix or kill") from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup, rffi from pypy.rpython.memory.gc.stmgc import StmGC, WORD from pypy.rpython.memory.gc.stmgc import GCFLAG_GLOBAL, GCFLAG_NOT_WRITTEN from pypy.rpython.memory.gc.stmgc import GCFLAG_POSSIBLY_OUTDATED from pypy.rpython.memory.gc.stmgc import GCFLAG_LOCAL_COPY, GCFLAG_VISITED -from pypy.rpython.memory.gc.stmgc import GCFLAG_HASH_FIELD, REV_FLAG_NEW_HASH +from pypy.rpython.memory.gc.stmgc import GCFLAG_HASH_FIELD from pypy.rpython.memory.gc.stmgc import hdr_revision, set_hdr_revision from pypy.rpython.memory.support import mangle_hash @@ -91,25 +92,6 @@ == key) callback(tls, value) - def stm_HashObject(self, P): - # see et.c - hdr = self._gc.header(P) - if hdr.tid & (GCFLAG_GLOBAL|GCFLAG_LOCAL_COPY) == 0: - hdr.revision |= REV_FLAG_NEW_HASH - return P - - while True: - hdr = self._gc.header(P) - if hdr.tid & GCFLAG_HASH_FIELD: - return P - v = hdr.revision - if isinstance(v, llmemory.AddressAsInt): # "is a pointer" - P = llmemory.cast_int_to_adr(v) - else: - # add the flag without caring about atomicity here - hdr.revision = v | REV_FLAG_NEW_HASH - return P - def fake_get_size(obj): TYPE = obj.ptr._TYPE.TO diff --git a/pypy/rpython/memory/gc/test/test_stmtls.py b/pypy/rpython/memory/gc/test/test_stmtls.py --- a/pypy/rpython/memory/gc/test/test_stmtls.py +++ b/pypy/rpython/memory/gc/test/test_stmtls.py @@ -1,7 +1,7 @@ import py +from pypy.rlib.rarithmetic import r_uint from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup from pypy.rpython.memory.gc.stmtls import StmGCTLS, WORD -from pypy.rpython.memory.gc.test.test_stmgc import StmGCTests from pypy.rpython.memory.support import get_address_stack, get_address_deque from pypy.rpython.memory.gcheader import GCHeaderBuilder @@ -75,7 +75,7 @@ sharedarea = FakeSharedArea() root_walker = FakeRootWalker() HDR = lltype.Struct('header', ('tid', lltype.Signed), - ('version', llmemory.Address)) + ('revision', lltype.Unsigned)) gcheaderbuilder = GCHeaderBuilder(HDR) maximum_extra_threshold = 0 @@ -86,6 +86,10 @@ def get_size(self, addr): return llmemory.sizeof(lltype.typeOf(addr.ptr).TO) + def set_obj_revision(self, addr, nrevision): + hdr = self.header(addr) + hdr.revision = llmemory.cast_adr_to_uint_symbolic(nrevision) + def trace(self, obj, callback, arg): TYPE = obj.ptr._TYPE.TO if TYPE == S: @@ -129,7 +133,7 @@ obj = adr + size_gc_header hdr = self.gc.header(obj) hdr.tid = 0 - hdr.version = NULL + hdr.revision = r_uint(0) return llmemory.cast_adr_to_ptr(obj, lltype.Ptr(STRUCT)) # ---------- From noreply at buildbot.pypy.org Fri Jan 25 17:40:31 2013 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 25 Jan 2013 17:40:31 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: Fix some tests Message-ID: <20130125164031.5AFE91C047D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r60463:2673042290e6 Date: 2013-01-21 18:04 +0100 http://bitbucket.org/pypy/pypy/changeset/2673042290e6/ Log: Fix some tests diff --git a/pypy/rpython/memory/gc/test/test_stmgc.py b/pypy/rpython/memory/gc/test/test_stmgc.py --- a/pypy/rpython/memory/gc/test/test_stmgc.py +++ b/pypy/rpython/memory/gc/test/test_stmgc.py @@ -1,7 +1,7 @@ import py -py.test.skip("XXX fix or kill") +from pypy.rlib.rarithmetic import r_uint from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup, rffi -from pypy.rpython.memory.gc.stmgc import StmGC, WORD +from pypy.rpython.memory.gc.stmgc import StmGC, WORD, REV_INITIAL from pypy.rpython.memory.gc.stmgc import GCFLAG_GLOBAL, GCFLAG_NOT_WRITTEN from pypy.rpython.memory.gc.stmgc import GCFLAG_POSSIBLY_OUTDATED from pypy.rpython.memory.gc.stmgc import GCFLAG_LOCAL_COPY, GCFLAG_VISITED @@ -172,6 +172,7 @@ addr = adr1 + self.gc.gcheaderbuilder.size_gc_header self.gc.header(addr).tid = self.gc.combine( tid, GCFLAG_GLOBAL | GCFLAG_NOT_WRITTEN) + self.gc.header(addr).revision = REV_INITIAL realobj = llmemory.cast_adr_to_ptr(addr, lltype.Ptr(STRUCT)) else: gcref = self.gc.malloc_fixedsize_clear(tid, size, @@ -198,6 +199,29 @@ if must_have_version != '?': assert hdr.version == must_have_version + def stm_readbarrier(self, P): + if lltype.typeOf(P) != llmemory.Address: + P = llmemory.cast_ptr_to_adr(P) + hdr = self.gc.header(P) + if hdr.tid & GCFLAG_GLOBAL == 0: + # already a local object + R = P + else: + R = self.stm_latest_global_rev(P) + L = self.gc.stm_operations.tldict_lookup(R) + if hdr.tid & GCFLAG_POSSIBLY_OUTDATED == 0: + assert not L + elif L: + xxx + return R.ptr + + def stm_latest_global_rev(self, G): + hdr = self.gc.header(G) + assert hdr.tid & GCFLAG_GLOBAL != 0 + while hdr.revision != REV_INITIAL: + xxx + return G + def stm_writebarrier(self, P): if lltype.typeOf(P) != llmemory.Address: P = llmemory.cast_ptr_to_adr(P) @@ -328,8 +352,9 @@ pending = list(source_objects) found = set(obj._obj for obj in pending) for x in pending: + x = self.stm_readbarrier(x) for name in ('sr2', 'sr3'): - obj = self.gc.stm_operations.read_attribute(x, name) + obj = getattr(x, name) if obj and obj._obj not in found: found.add(obj._obj) pending.append(obj) @@ -340,10 +365,12 @@ pending = [source_object] found = {source_object._obj: 0} for x in pending: + x_orig = x + x = self.stm_readbarrier(x) + if not can_be_indirect: + assert x == x_orig for name in ('sr2', 'sr3'): - obj = self.gc.stm_operations.read_attribute(x, name) - if not can_be_indirect: - assert obj == getattr(x, name) + obj = getattr(x, name) if not obj: shape.append(None) else: @@ -386,7 +413,7 @@ dstobj = random.choice(missing_objects) name = random.choice(('sr2', 'sr3')) src_adr = llmemory.cast_ptr_to_adr(srcobj) - adr2 = self.gc.stm_writebarrier(src_adr) + adr2 = self.stm_writebarrier(src_adr) obj2 = llmemory.cast_adr_to_ptr(adr2, lltype.Ptr(SR)) setattr(obj2, name, dstobj) # From noreply at buildbot.pypy.org Fri Jan 25 17:40:32 2013 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 25 Jan 2013 17:40:32 +0100 (CET) Subject: [pypy-commit] pypy default: Add a quick hack to prevent py.test from taking forever to display certain Message-ID: <20130125164032.82AEB1C047D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r60464:dc81116ff4b7 Date: 2013-01-25 17:38 +0100 http://bitbucket.org/pypy/pypy/changeset/dc81116ff4b7/ Log: Add a quick hack to prevent py.test from taking forever to display certain tracebacks. diff --git a/py/_code/source.py b/py/_code/source.py --- a/py/_code/source.py +++ b/py/_code/source.py @@ -141,6 +141,8 @@ trysource = self[start:end] if trysource.isparseable(): return start, end + if end == start + 100: # XXX otherwise, it takes forever + break # XXX if end is None: raise IndexError("no valid source range around line %d " % (lineno,)) return start, end From noreply at buildbot.pypy.org Fri Jan 25 17:40:33 2013 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 25 Jan 2013 17:40:33 +0100 (CET) Subject: [pypy-commit] pypy default: Oups. This is needed to correctly initialize the GC; otherwise, if we're Message-ID: <20130125164033.A2B5A1C047D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r60465:a1998ce7cac6 Date: 2013-01-25 17:39 +0100 http://bitbucket.org/pypy/pypy/changeset/a1998ce7cac6/ Log: Oups. This is needed to correctly initialize the GC; otherwise, if we're unluckly, the first grab_frame_values() will trigger collection because we didn't lower the limit so far. diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py --- a/rpython/jit/backend/x86/runner.py +++ b/rpython/jit/backend/x86/runner.py @@ -64,6 +64,7 @@ def setup_once(self): self.profile_agent.startup() self.assembler.setup_once() + self.gc_set_extra_threshold() def finish_once(self): self.assembler.finish_once() From noreply at buildbot.pypy.org Fri Jan 25 17:40:34 2013 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 25 Jan 2013 17:40:34 +0100 (CET) Subject: [pypy-commit] pypy default: merge heads Message-ID: <20130125164034.C41A31C047D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r60466:4787278e28d8 Date: 2013-01-25 17:40 +0100 http://bitbucket.org/pypy/pypy/changeset/4787278e28d8/ Log: merge heads diff --git a/rpython/rtyper/rlist.py b/rpython/rtyper/rlist.py --- a/rpython/rtyper/rlist.py +++ b/rpython/rtyper/rlist.py @@ -1042,11 +1042,7 @@ res = RESLIST.ll_newlist(resultlen) j = 0 while j < resultlen: - i = 0 - while i < length: - p = j + i - res.ll_setitem_fast(p, l.ll_getitem_fast(i)) - i += 1 + ll_arraycopy(l, res, 0, j, length) j += length return res # not inlined by the JIT -- contains a loop From noreply at buildbot.pypy.org Fri Jan 25 18:00:39 2013 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Fri, 25 Jan 2013 18:00:39 +0100 (CET) Subject: [pypy-commit] pypy pytest: merge in default Message-ID: <20130125170039.B00B91C047D@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: pytest Changeset: r60467:75f9b6ea28b9 Date: 2013-01-25 17:53 +0100 http://bitbucket.org/pypy/pypy/changeset/75f9b6ea28b9/ Log: merge in default diff --git a/py/_code/source.py b/py/_code/source.py --- a/py/_code/source.py +++ b/py/_code/source.py @@ -141,6 +141,8 @@ trysource = self[start:end] if trysource.isparseable(): return start, end + if end == start + 100: # XXX otherwise, it takes forever + break # XXX if end is None: raise IndexError("no valid source range around line %d " % (lineno,)) return start, end diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py --- a/rpython/jit/backend/x86/runner.py +++ b/rpython/jit/backend/x86/runner.py @@ -64,6 +64,7 @@ def setup_once(self): self.profile_agent.startup() self.assembler.setup_once() + self.gc_set_extra_threshold() def finish_once(self): self.assembler.finish_once() diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -481,11 +481,8 @@ self.autoreds = True self.reds = [] self.numreds = None # see warmspot.autodetect_jit_markers_redvars - for hook in [ - get_jitcell_at, set_jitcell_at, get_printable_location, - confirm_enter_jit - ]: - assert hook is None, "reds='auto' is not compatible with JitDriver hooks" + assert confirm_enter_jit is None, ( + "reds='auto' is not compatible with confirm_enter_jit") else: if reds is not None: self.reds = reds diff --git a/rpython/rtyper/rlist.py b/rpython/rtyper/rlist.py --- a/rpython/rtyper/rlist.py +++ b/rpython/rtyper/rlist.py @@ -1025,11 +1025,7 @@ res._ll_resize(resultlen) j = length while j < resultlen: - i = 0 - while i < length: - p = j + i - res.ll_setitem_fast(p, l.ll_getitem_fast(i)) - i += 1 + ll_arraycopy(l, res, 0, j, length) j += length return res ll_inplace_mul.oopspec = 'list.inplace_mul(l, factor)' @@ -1046,11 +1042,7 @@ res = RESLIST.ll_newlist(resultlen) j = 0 while j < resultlen: - i = 0 - while i < length: - p = j + i - res.ll_setitem_fast(p, l.ll_getitem_fast(i)) - i += 1 + ll_arraycopy(l, res, 0, j, length) j += length return res # not inlined by the JIT -- contains a loop From noreply at buildbot.pypy.org Fri Jan 25 18:00:41 2013 From: noreply at buildbot.pypy.org (RonnyPfannschmidt) Date: Fri, 25 Jan 2013 18:00:41 +0100 (CET) Subject: [pypy-commit] pypy pytest: kill more conftest.option uses Message-ID: <20130125170041.065E51C047D@cobra.cs.uni-duesseldorf.de> Author: Ronny Pfannschmidt Branch: pytest Changeset: r60468:1d71a7df8c7f Date: 2013-01-25 18:00 +0100 http://bitbucket.org/pypy/pypy/changeset/1d71a7df8c7f/ Log: kill more conftest.option uses diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py --- a/pypy/module/imp/test/test_import.py +++ b/pypy/module/imp/test/test_import.py @@ -13,7 +13,6 @@ from pypy.module.imp import importing -from pypy import conftest def setuppkg(pkgname, **entries): p = udir.join('impsubdir') @@ -93,7 +92,7 @@ # create compiled/x.py and a corresponding pyc file p = setuppkg("compiled", x = "x = 84") - if conftest.option.runappdirect: + if py.test.config.option.runappdirect: import marshal, stat, struct, os, imp code = py.code.Source(p.join("x.py").read()).compile() s3 = marshal.dumps(code) @@ -153,7 +152,7 @@ } def setup_class(cls): - cls.w_runappdirect = cls.space.wrap(conftest.option.runappdirect) + cls.w_runappdirect = cls.space.wrap(py.test.config.option.runappdirect) cls.saved_modules = _setup(cls.space) #XXX Compile class diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py --- a/pypy/objspace/std/test/test_bytearrayobject.py +++ b/pypy/objspace/std/test/test_bytearrayobject.py @@ -1,8 +1,8 @@ -from pypy import conftest +import py class AppTestBytesArray: def setup_class(cls): - cls.w_runappdirect = cls.space.wrap(conftest.option.runappdirect) + cls.w_runappdirect = cls.space.wrap(py.test.config.option.runappdirect) def test_basics(self): b = bytearray() From noreply at buildbot.pypy.org Fri Jan 25 18:09:30 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 25 Jan 2013 18:09:30 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: (arigo, fijal) test and fix Message-ID: <20130125170930.97F321C0041@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60469:fc34a23fa97c Date: 2013-01-25 19:08 +0200 http://bitbucket.org/pypy/pypy/changeset/fc34a23fa97c/ Log: (arigo, fijal) test and fix diff --git a/rpython/jit/backend/llsupport/jitframe.py b/rpython/jit/backend/llsupport/jitframe.py --- a/rpython/jit/backend/llsupport/jitframe.py +++ b/rpython/jit/backend/llsupport/jitframe.py @@ -99,12 +99,12 @@ # 32 possible bits state = (fld >> 3) & 0x1f no = fld >> (3 + 5) - MAX = 31 + MAX = 32 else: # 64 possible bits state = (fld >> 3) & 0x3f no = fld >> (3 + 6) - MAX = 63 + MAX = 64 gcmap = (obj_addr + getofs('jf_gcmap')).address[0] gcmap_lgt = (gcmap + GCMAPLENGTHOFS).signed[0] while no < gcmap_lgt: diff --git a/rpython/jit/backend/llsupport/test/test_gc.py b/rpython/jit/backend/llsupport/test/test_gc.py --- a/rpython/jit/backend/llsupport/test/test_gc.py +++ b/rpython/jit/backend/llsupport/test/test_gc.py @@ -1,3 +1,4 @@ +import sys from rpython.rtyper.lltypesystem import lltype, llmemory, rstr from rpython.rtyper.lltypesystem.lloperation import llop from rpython.rtyper.annlowlevel import llhelper @@ -277,7 +278,11 @@ frame = lltype.malloc(jitframe.JITFRAME, 15, zero=True) frame.jf_frame_info = frame_info frame.jf_gcmap = lltype.malloc(jitframe.GCMAP, 2) - frame.jf_gcmap[0] = r_uint(1 | 2 | 8 | 32 | 128) + if sys.maxint == 2**31 - 1: + max = r_uint(2 ** 31) + else: + max = r_uint(2 ** 63) + frame.jf_gcmap[0] = r_uint(1 | 2 | 8 | 32 | 128) | max frame.jf_gcmap[1] = r_uint(2 | 16 | 32 | 128) frame_adr = llmemory.cast_ptr_to_adr(frame) all_addrs = [] @@ -297,10 +302,11 @@ assert all_addrs[8] == indexof(3) assert all_addrs[9] == indexof(5) assert all_addrs[10] == indexof(7) + assert all_addrs[11] == indexof(63) # XXX 32bit - assert all_addrs[11] == indexof(65) + assert all_addrs[12] == indexof(65) - assert len(all_addrs) == 6 + 5 + 4 + assert len(all_addrs) == 6 + 6 + 4 # 6 static fields, 4 addresses from gcmap, 2 from gcpattern finally: jitframe.STATICSIZE = PREV_STATICSIZE From noreply at buildbot.pypy.org Fri Jan 25 18:09:31 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 25 Jan 2013 18:09:31 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: remove dead code Message-ID: <20130125170931.D68C11C0041@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60470:8ccfdb07a4fb Date: 2013-01-25 19:08 +0200 http://bitbucket.org/pypy/pypy/changeset/8ccfdb07a4fb/ Log: remove dead code diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -58,27 +58,6 @@ no_lower_byte_regs = [] save_around_call_regs = [eax, ecx, edx, esi, edi, r8, r9, r10] - REGLOC_TO_GCROOTMAP_REG_INDEX = { - ebx: 1, - r12: 2, - r13: 3, - r14: 4, - r15: 5, - } - #REGLOC_TO_COPY_AREA_OFS = { - # ecx: MY_COPY_OF_REGS + 0 * WORD, - # ebx: MY_COPY_OF_REGS + 1 * WORD, - # esi: MY_COPY_OF_REGS + 2 * WORD, - # edi: MY_COPY_OF_REGS + 3 * WORD, - # r8: MY_COPY_OF_REGS + 4 * WORD, - # r9: MY_COPY_OF_REGS + 5 * WORD, - # r10: MY_COPY_OF_REGS + 6 * WORD, - # r12: MY_COPY_OF_REGS + 7 * WORD, - # r13: MY_COPY_OF_REGS + 8 * WORD, - # r14: MY_COPY_OF_REGS + 9 * WORD, - #3 r15: MY_COPY_OF_REGS + 10 * WORD, - #} - class X86XMMRegisterManager(RegisterManager): box_types = [FLOAT] From noreply at buildbot.pypy.org Fri Jan 25 18:16:46 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 25 Jan 2013 18:16:46 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: Fix those tests. With more dilligence it will be spotted earlier, but *before* Message-ID: <20130125171646.B75261C0041@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60471:e8c5b7052e29 Date: 2013-01-25 19:16 +0200 http://bitbucket.org/pypy/pypy/changeset/e8c5b7052e29/ Log: Fix those tests. With more dilligence it will be spotted earlier, but *before* malloc we don't store in gcmap the result of the malloc (it would be confusing) diff --git a/rpython/jit/backend/x86/test/test_gc_integration.py b/rpython/jit/backend/x86/test/test_gc_integration.py --- a/rpython/jit/backend/x86/test/test_gc_integration.py +++ b/rpython/jit/backend/x86/test/test_gc_integration.py @@ -205,7 +205,7 @@ def test_malloc_slowpath(self): def check(frame): assert len(frame.jf_gcmap) == 1 - assert frame.jf_gcmap[0] == (2 | (1<<29) | (1 << 30)) + assert frame.jf_gcmap[0] == (1<<29) | (1 << 30) self.cpu = self.getcpu(check) ops = ''' @@ -234,7 +234,7 @@ x = frame.jf_gcmap assert len(x) == 1 assert (bin(x[0]).count('1') == - '0b1111100000000000000001111111011111'.count('1')) + '0b1111100000000000000001111111011110'.count('1')) # all but two registers + some stuff on stack self.cpu = self.getcpu(check) From noreply at buildbot.pypy.org Fri Jan 25 18:31:54 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 25 Jan 2013 18:31:54 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: apparently is_shadow_stack is not constant folded away Message-ID: <20130125173154.CA0951C1055@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60472:204591cade37 Date: 2013-01-25 19:31 +0200 http://bitbucket.org/pypy/pypy/changeset/204591cade37/ Log: apparently is_shadow_stack is not constant folded away diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -2146,6 +2146,7 @@ # like %eax that would be destroyed by this call, *and* they are # used by arglocs for the *next* call, then trouble; for now we # will just push/pop them. + raise NotImplementedError xxx from rpython.rtyper.memory.gctransform import asmgcroot css = self._regalloc.close_stack_struct From noreply at buildbot.pypy.org Fri Jan 25 18:47:57 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 25 Jan 2013 18:47:57 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: if the gcmap is 0, just don't trace it. Also 0 it in finish Message-ID: <20130125174757.D10071C047D@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60473:0aeca905ae04 Date: 2013-01-25 19:47 +0200 http://bitbucket.org/pypy/pypy/changeset/0aeca905ae04/ Log: if the gcmap is 0, just don't trace it. Also 0 it in finish diff --git a/rpython/jit/backend/llsupport/jitframe.py b/rpython/jit/backend/llsupport/jitframe.py --- a/rpython/jit/backend/llsupport/jitframe.py +++ b/rpython/jit/backend/llsupport/jitframe.py @@ -106,6 +106,8 @@ no = fld >> (3 + 6) MAX = 64 gcmap = (obj_addr + getofs('jf_gcmap')).address[0] + if not gcmap: + return llmemory.NULL gcmap_lgt = (gcmap + GCMAPLENGTHOFS).signed[0] while no < gcmap_lgt: cur = (gcmap + GCMAPBASEOFS + UNSIGN_SIZE * no).unsigned[0] diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -2019,7 +2019,13 @@ base_ofs = self.cpu.get_baseofs_of_frame_field() self.mov(fail_descr_loc, RawStackLoc(ofs)) gcmap = self.gcmap_for_finish - self.push_gcmap(self.mc, gcmap, store=True) + if op.getarg(0).type == REF: + self.push_gcmap(self.mc, gcmap, store=True) + else: + # note that the 0 here is redundant, but I would rather + # keep that one and kill all the others + ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') + self.mc.MOV_bi(ofs, 0) self.mc.LEA_rb(eax.value, -base_ofs) # exit function self._call_footer() From noreply at buildbot.pypy.org Fri Jan 25 18:50:56 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 25 Jan 2013 18:50:56 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: ugh, how is this part getting annotated? Message-ID: <20130125175056.D71E91C047D@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60474:5007b0fbabda Date: 2013-01-25 19:50 +0200 http://bitbucket.org/pypy/pypy/changeset/5007b0fbabda/ Log: ugh, how is this part getting annotated? diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -149,8 +149,6 @@ op0 = ResOperation(rop.GETFIELD_GC, [history.ConstPtr(llref)], lgt_box, descr=descrs.jfi_frame_depth) self.newops.append(op0) - # XXX for now it generates call_malloc_gc, instead of - # call_malloc_nursery, because the array is strange op1 = ResOperation(rop.NEW_ARRAY, [lgt_box], frame, descr=descrs.arraydescr) @@ -168,7 +166,7 @@ assert isinstance(descr, JitCellToken) jd = descr.outermost_jitdriver_sd args = [frame] - if jd.index_of_virtualizable >= 0: + if jd and jd.index_of_virtualizable >= 0: args = [frame, arglist[jd.index_of_virtualizable]] else: args = [frame] @@ -264,6 +262,8 @@ """Try to generate or update a CALL_MALLOC_NURSERY. If that fails, generate a plain CALL_MALLOC_GC instead. """ + import pdb + pdb.set_trace() size = self.round_up_for_allocation(size) if not self.gc_ll_descr.can_use_nursery_malloc(size): return False diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py --- a/rpython/jit/backend/llsupport/test/test_rewrite.py +++ b/rpython/jit/backend/llsupport/test/test_rewrite.py @@ -738,7 +738,7 @@ """, """ [i0, f0] i1 = getfield_gc(ConstPtr(ll_frame_info), descr=jfi_frame_depth) - p1 = call_malloc_gc(ConstClass(malloc_array_nonstandard), 1, 2, 0, 0, i1, descr=malloc_array_nonstandard_descr) + p1 = call_malloc_nursery(13) setfield_gc(p1, ConstPtr(ll_frame_info), descr=jf_frame_info) setarrayitem_gc(p1, 0, i0, descr=signedframedescr) setarrayitem_gc(p1, 1, f0, descr=floatframedescr) diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -2213,6 +2213,7 @@ if gcrootmap.is_shadow_stack: args = [] else: + raise NotImplementedError xxx assert self.gcrootmap_retaddr_forced == -1, ( "missing mark_gc_roots() in CALL_RELEASE_GIL") From noreply at buildbot.pypy.org Fri Jan 25 18:54:35 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 25 Jan 2013 18:54:35 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: remove pdb, it was not supposed to go in Message-ID: <20130125175435.04B6D1C047D@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60475:68a884c17ef2 Date: 2013-01-25 19:54 +0200 http://bitbucket.org/pypy/pypy/changeset/68a884c17ef2/ Log: remove pdb, it was not supposed to go in diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -262,8 +262,6 @@ """Try to generate or update a CALL_MALLOC_NURSERY. If that fails, generate a plain CALL_MALLOC_GC instead. """ - import pdb - pdb.set_trace() size = self.round_up_for_allocation(size) if not self.gc_ll_descr.can_use_nursery_malloc(size): return False From noreply at buildbot.pypy.org Fri Jan 25 18:56:03 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 25 Jan 2013 18:56:03 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: leave some comments Message-ID: <20130125175603.248C01C1069@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60476:d17f7b6d6256 Date: 2013-01-25 19:55 +0200 http://bitbucket.org/pypy/pypy/changeset/d17f7b6d6256/ Log: leave some comments diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -203,9 +203,11 @@ base_ofs = self.cpu.get_baseofs_of_frame_field() self._push_all_regs_to_frame(mc, [], self.cpu.supports_floats) assert not IS_X86_32 + # this is the gcmap stored by push_gcmap(mov=True) in _check_stack_frame mc.MOV_rs(ecx.value, WORD) gcmap_ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') mc.MOV_br(gcmap_ofs, ecx.value) + # this is size that we're after, sanity checking only mc.MOV_rs(esi.value, WORD*2) # push first arg mc.LEA_rb(edi.value, -base_ofs) From noreply at buildbot.pypy.org Fri Jan 25 18:57:36 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 25 Jan 2013 18:57:36 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: comment fix Message-ID: <20130125175736.786F41C1069@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60477:ec5b34476bdc Date: 2013-01-25 19:57 +0200 http://bitbucket.org/pypy/pypy/changeset/ec5b34476bdc/ Log: comment fix diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -223,7 +223,7 @@ def _build_malloc_slowpath(self): """ While arriving on slowpath, we have a gcpattern on stack, - nursery_head in eax and the size in edx - eax + nursery_head in eax and the size in edi - eax """ mc = codebuf.MachineCodeBlockWrapper() self._push_all_regs_to_frame(mc, [eax, edi], self.cpu.supports_floats) From noreply at buildbot.pypy.org Fri Jan 25 19:01:13 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 25 Jan 2013 19:01:13 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: armin spotting off comments Message-ID: <20130125180113.7B2311C1069@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60478:bd478fb14dab Date: 2013-01-25 20:00 +0200 http://bitbucket.org/pypy/pypy/changeset/bd478fb14dab/ Log: armin spotting off comments diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -251,7 +251,7 @@ nursery_free_adr = self.cpu.gc_ll_descr.get_nursery_free_addr() self._reload_frame_if_necessary(mc) self._pop_all_regs_from_frame(mc, [eax, edi], self.cpu.supports_floats) - mc.MOV(edi, heap(nursery_free_adr)) # load this in EDX + mc.MOV(edi, heap(nursery_free_adr)) # load this in EDI # clear the gc pattern mc.MOV_bi(ofs, 0) mc.RET() From noreply at buildbot.pypy.org Fri Jan 25 19:23:09 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Fri, 25 Jan 2013 19:23:09 +0100 (CET) Subject: [pypy-commit] pypy default: fix translating with an exe_name thats in a nested directory Message-ID: <20130125182309.B1E801C0041@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: Changeset: r60479:0f2efc923987 Date: 2013-01-25 12:22 -0600 http://bitbucket.org/pypy/pypy/changeset/0f2efc923987/ Log: fix translating with an exe_name thats in a nested directory diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py --- a/pypy/module/sys/vm.py +++ b/pypy/module/sys/vm.py @@ -95,7 +95,7 @@ def exc_info_with_tb(space): operror = space.getexecutioncontext().sys_exc_info() if operror is None: - return space.newtuple([space.w_None,space.w_None,space.w_None]) + return space.newtuple([space.w_None, space.w_None, space.w_None]) else: return space.newtuple([operror.w_type, operror.get_w_value(space), space.wrap(operror.get_traceback())]) diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py --- a/rpython/translator/platform/posix.py +++ b/rpython/translator/platform/posix.py @@ -115,7 +115,7 @@ cflags = self.cflags + self.standalone_only m = GnuMakefile(path) - m.exe_name = exe_name + m.exe_name = path.join(target_name) m.eci = eci def rpyrel(fpath): diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -261,8 +261,15 @@ else: exe_name = exe_name.new(ext=self.exe_ext) + if shared: + so_name = exe_name.new(purebasename='lib' + exe_name.purebasename, + ext=self.so_ext) + target_name = so_name.basename + else: + target_name = exe_name.basename + m = NMakefile(path) - m.exe_name = exe_name + m.exe_name = path.join(target_name) m.eci = eci linkflags = list(self.link_flags) @@ -274,13 +281,6 @@ # This is required for the JIT. linkflags.append('/opt:noicf') - if shared: - so_name = exe_name.new(purebasename='lib' + exe_name.purebasename, - ext=self.so_ext) - target_name = so_name.basename - else: - target_name = exe_name.basename - def rpyrel(fpath): rel = py.path.local(fpath).relto(rpypath) if rel: From noreply at buildbot.pypy.org Fri Jan 25 20:14:22 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 25 Jan 2013 20:14:22 +0100 (CET) Subject: [pypy-commit] pypy default: haha, os.path is a bad joke Message-ID: <20130125191422.1ED421C0041@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r60480:9d14d0af5a62 Date: 2013-01-25 21:13 +0200 http://bitbucket.org/pypy/pypy/changeset/9d14d0af5a62/ Log: haha, os.path is a bad joke diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -13,7 +13,7 @@ import sys import os #Add toplevel repository dir to sys.path -sys.path.insert(0,os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))) +sys.path.insert(0,os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))) import py import fnmatch from rpython.tool.udir import udir From noreply at buildbot.pypy.org Fri Jan 25 20:26:12 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 25 Jan 2013 20:26:12 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: small fixes Message-ID: <20130125192612.69AFF1C0041@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60481:4a98ccf02e16 Date: 2013-01-25 21:25 +0200 http://bitbucket.org/pypy/pypy/changeset/4a98ccf02e16/ Log: small fixes diff --git a/pypy/tool/jitlogparser/parser.py b/pypy/tool/jitlogparser/parser.py --- a/pypy/tool/jitlogparser/parser.py +++ b/pypy/tool/jitlogparser/parser.py @@ -365,7 +365,7 @@ m = re.search('has address ([-\da-f]+)', entry) addr = int(m.group(1), 16) entry = entry.lower() - m = re.search('guard \d+', entry) + m = re.search('guard [\da-f]+', entry) name = m.group(0) else: name = entry[:entry.find('(') - 1].lower() @@ -388,8 +388,8 @@ comm = loop.comment comm = comm.lower() if comm.startswith('# bridge'): - m = re.search('guard \d+', comm) - name = m.group(0) + m = re.search('guard (\d+)', comm) + name = 'guard ' + hex(int(m.group(1)))[2:] elif "(" in comm: name = comm[2:comm.find('(')-1] else: @@ -437,3 +437,7 @@ if line: num, count = line.split(':', 2) mapping[num].count = int(count) + +if __name__ == '__main__': + import_log(sys.argv[1]) + diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -527,7 +527,6 @@ rgc._make_sure_does_not_move(lltype.cast_opaque_ptr(llmemory.GCREF, clt.frame_info)) looppos = self.mc.get_relative_pos() - looptoken._x86_loop_code = looppos frame_depth = self._assemble(regalloc, inputargs, operations) self.update_frame_depth(frame_depth + JITFRAME_FIXED_SIZE) # @@ -536,6 +535,7 @@ full_size = self.mc.get_relative_pos() # rawstart = self.materialize_loop(looptoken) + looptoken._x86_loop_code = looppos + rawstart debug_start("jit-backend-addr") debug_print("Loop %d (%s) has address %x to %x (bootstrap %x)" % ( looptoken.number, loopname, From noreply at buildbot.pypy.org Fri Jan 25 20:34:09 2013 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 25 Jan 2013 20:34:09 +0100 (CET) Subject: [pypy-commit] pypy.org extradoc: Update for the current trunk Message-ID: <20130125193409.BECC41C0041@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r376:0694de9c9e9d Date: 2013-01-25 20:33 +0100 http://bitbucket.org/pypy/pypy.org/changeset/0694de9c9e9d/ Log: Update for the current trunk diff --git a/download.html b/download.html --- a/download.html +++ b/download.html @@ -150,10 +150,12 @@

  • Enter the goal directory:

    -cd pypy/pypy/translator/goal
+cd pypy/pypy/translator/goal      # release 2.0-beta-1
+cd pypy/pypy/goal                 # current trunk
    • -

  • Run the translate.py script. Here are the common combinations +

  • Run the translate.py script (called rpython.py in the current +trunk). Here are the common combinations of options (works also with python instead of pypy):

     pypy translate.py -Ojit                # get the JIT version
@@ -161,6 +163,10 @@
 pypy translate.py -O2 --sandbox        # get the sandbox version
 pypy translate.py -Ojit --backend=cli  # only for branch/cli-jit
    +

    Or with the current trunk:

    +
    +../../rpython/bin/rpython -Ojit        # or -O2, etc.
+

  • Enjoy Mandelbrot :-) It takes on the order of half an hour to finish the translation, and 2.x GB of RAM on a 32-bit system diff --git a/source/download.txt b/source/download.txt --- a/source/download.txt +++ b/source/download.txt @@ -156,9 +156,11 @@ 3. Enter the ``goal`` directory:: - cd pypy/pypy/translator/goal + cd pypy/pypy/translator/goal # release 2.0-beta-1 + cd pypy/pypy/goal # current trunk -4. Run the ``translate.py`` script. Here are the common combinations +4. Run the ``translate.py`` script (called ``rpython.py`` in the current + trunk). Here are the common combinations of options (works also with ``python`` instead of ``pypy``):: pypy translate.py -Ojit # get the JIT version @@ -166,6 +168,10 @@ pypy translate.py -O2 --sandbox # get the sandbox version pypy translate.py -Ojit --backend=cli # only for branch/cli-jit + Or with the current trunk:: + + ../../rpython/bin/rpython -Ojit # or -O2, etc. + 5. Enjoy Mandelbrot ``:-)`` It takes on the order of half an hour to finish the translation, and 2.x GB of RAM on a 32-bit system and 4.x GB on 64-bit systems. (Do not start a translation on a From noreply at buildbot.pypy.org Fri Jan 25 20:52:42 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 25 Jan 2013 20:52:42 +0100 (CET) Subject: [pypy-commit] pypy default: a test and a fix Message-ID: <20130125195242.752A51C047D@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r60482:ee9892ab4807 Date: 2013-01-25 21:52 +0200 http://bitbucket.org/pypy/pypy/changeset/ee9892ab4807/ Log: a test and a fix diff --git a/pypy/tool/jitlogparser/parser.py b/pypy/tool/jitlogparser/parser.py --- a/pypy/tool/jitlogparser/parser.py +++ b/pypy/tool/jitlogparser/parser.py @@ -247,12 +247,15 @@ if not stack: stack.append([]) else: - if bc.inline_level is not None and bc.inline_level + 1 != len(stack): - if bc.inline_level < len(stack): - last = stack.pop() - stack[-1].append(cls(last, getpath(stack), storage)) + if bc.inline_level is not None: + if bc.inline_level == len(stack) - 1: + pass + elif bc.inline_level > len(stack) - 1: + stack.append([]) else: - stack.append([]) + while bc.inline_level + 1 < len(stack): + last = stack.pop() + stack[-1].append(cls(last, getpath(stack), storage)) stack[-1].append(bc) so_far = [] diff --git a/pypy/tool/jitlogparser/test/test_parser.py b/pypy/tool/jitlogparser/test/test_parser.py --- a/pypy/tool/jitlogparser/test/test_parser.py +++ b/pypy/tool/jitlogparser/test/test_parser.py @@ -354,3 +354,14 @@ f = Function.from_operations(loop.operations, LoopStorage()) assert f.chunks[-1].filename == 'x.py' assert f.filename is None + +def test_parse_2_levels_up(): + loop = parse(""" + [] + debug_merge_point(0, 0, 'one') + debug_merge_point(1, 0, 'two') + debug_merge_point(2, 0, 'three') + debug_merge_point(0, 0, 'one') + """) + f = Function.from_operations(loop.operations, LoopStorage()) + assert len(f.chunks) == 3 From noreply at buildbot.pypy.org Fri Jan 25 20:52:43 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 25 Jan 2013 20:52:43 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: merge default Message-ID: <20130125195243.C4EF91C047D@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60483:bd89a27a688b Date: 2013-01-25 21:52 +0200 http://bitbucket.org/pypy/pypy/changeset/bd89a27a688b/ Log: merge default diff --git a/py/_code/source.py b/py/_code/source.py --- a/py/_code/source.py +++ b/py/_code/source.py @@ -141,6 +141,8 @@ trysource = self[start:end] if trysource.isparseable(): return start, end + if end == start + 100: # XXX otherwise, it takes forever + break # XXX if end is None: raise IndexError("no valid source range around line %d " % (lineno,)) return start, end diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py --- a/pypy/module/sys/vm.py +++ b/pypy/module/sys/vm.py @@ -95,7 +95,7 @@ def exc_info_with_tb(space): operror = space.getexecutioncontext().sys_exc_info() if operror is None: - return space.newtuple([space.w_None,space.w_None,space.w_None]) + return space.newtuple([space.w_None, space.w_None, space.w_None]) else: return space.newtuple([operror.w_type, operror.get_w_value(space), space.wrap(operror.get_traceback())]) diff --git a/pypy/tool/jitlogparser/parser.py b/pypy/tool/jitlogparser/parser.py --- a/pypy/tool/jitlogparser/parser.py +++ b/pypy/tool/jitlogparser/parser.py @@ -247,12 +247,15 @@ if not stack: stack.append([]) else: - if bc.inline_level is not None and bc.inline_level + 1 != len(stack): - if bc.inline_level < len(stack): - last = stack.pop() - stack[-1].append(cls(last, getpath(stack), storage)) + if bc.inline_level is not None: + if bc.inline_level == len(stack) - 1: + pass + elif bc.inline_level > len(stack) - 1: + stack.append([]) else: - stack.append([]) + while bc.inline_level + 1 < len(stack): + last = stack.pop() + stack[-1].append(cls(last, getpath(stack), storage)) stack[-1].append(bc) so_far = [] diff --git a/pypy/tool/jitlogparser/test/test_parser.py b/pypy/tool/jitlogparser/test/test_parser.py --- a/pypy/tool/jitlogparser/test/test_parser.py +++ b/pypy/tool/jitlogparser/test/test_parser.py @@ -354,3 +354,14 @@ f = Function.from_operations(loop.operations, LoopStorage()) assert f.chunks[-1].filename == 'x.py' assert f.filename is None + +def test_parse_2_levels_up(): + loop = parse(""" + [] + debug_merge_point(0, 0, 'one') + debug_merge_point(1, 0, 'two') + debug_merge_point(2, 0, 'three') + debug_merge_point(0, 0, 'one') + """) + f = Function.from_operations(loop.operations, LoopStorage()) + assert len(f.chunks) == 3 diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -13,7 +13,7 @@ import sys import os #Add toplevel repository dir to sys.path -sys.path.insert(0,os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))) +sys.path.insert(0,os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))) import py import fnmatch from rpython.tool.udir import udir diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py --- a/rpython/jit/backend/x86/runner.py +++ b/rpython/jit/backend/x86/runner.py @@ -64,6 +64,7 @@ def setup_once(self): self.profile_agent.startup() self.assembler.setup_once() + self.gc_set_extra_threshold() def finish_once(self): self.assembler.finish_once() diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -481,11 +481,8 @@ self.autoreds = True self.reds = [] self.numreds = None # see warmspot.autodetect_jit_markers_redvars - for hook in [ - get_jitcell_at, set_jitcell_at, get_printable_location, - confirm_enter_jit - ]: - assert hook is None, "reds='auto' is not compatible with JitDriver hooks" + assert confirm_enter_jit is None, ( + "reds='auto' is not compatible with confirm_enter_jit") else: if reds is not None: self.reds = reds diff --git a/rpython/rtyper/rlist.py b/rpython/rtyper/rlist.py --- a/rpython/rtyper/rlist.py +++ b/rpython/rtyper/rlist.py @@ -1025,11 +1025,7 @@ res._ll_resize(resultlen) j = length while j < resultlen: - i = 0 - while i < length: - p = j + i - res.ll_setitem_fast(p, l.ll_getitem_fast(i)) - i += 1 + ll_arraycopy(l, res, 0, j, length) j += length return res ll_inplace_mul.oopspec = 'list.inplace_mul(l, factor)' @@ -1046,11 +1042,7 @@ res = RESLIST.ll_newlist(resultlen) j = 0 while j < resultlen: - i = 0 - while i < length: - p = j + i - res.ll_setitem_fast(p, l.ll_getitem_fast(i)) - i += 1 + ll_arraycopy(l, res, 0, j, length) j += length return res # not inlined by the JIT -- contains a loop diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py --- a/rpython/translator/platform/posix.py +++ b/rpython/translator/platform/posix.py @@ -115,7 +115,7 @@ cflags = self.cflags + self.standalone_only m = GnuMakefile(path) - m.exe_name = exe_name + m.exe_name = path.join(target_name) m.eci = eci def rpyrel(fpath): diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -261,8 +261,15 @@ else: exe_name = exe_name.new(ext=self.exe_ext) + if shared: + so_name = exe_name.new(purebasename='lib' + exe_name.purebasename, + ext=self.so_ext) + target_name = so_name.basename + else: + target_name = exe_name.basename + m = NMakefile(path) - m.exe_name = exe_name + m.exe_name = path.join(target_name) m.eci = eci linkflags = list(self.link_flags) @@ -274,13 +281,6 @@ # This is required for the JIT. linkflags.append('/opt:noicf') - if shared: - so_name = exe_name.new(purebasename='lib' + exe_name.purebasename, - ext=self.so_ext) - target_name = so_name.basename - else: - target_name = exe_name.basename - def rpyrel(fpath): rel = py.path.local(fpath).relto(rpypath) if rel: From noreply at buildbot.pypy.org Fri Jan 25 21:20:32 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 25 Jan 2013 21:20:32 +0100 (CET) Subject: [pypy-commit] jitviewer default: "fix" Message-ID: <20130125202032.E3C871C0041@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r210:783713a57056 Date: 2013-01-25 22:20 +0200 http://bitbucket.org/pypy/jitviewer/changeset/783713a57056/ Log: "fix" diff --git a/_jitviewer/display.py b/_jitviewer/display.py --- a/_jitviewer/display.py +++ b/_jitviewer/display.py @@ -48,7 +48,10 @@ last_lineno = no else: no = last_lineno - self.lines[no - self.firstlineno].chunks.append(chunk) - + try: + self.lines[no - self.firstlineno].chunks.append(chunk) + except IndexError: + self.lines[-1].chunks.append(chunk) # too bad, just stash it there + From noreply at buildbot.pypy.org Fri Jan 25 21:36:04 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 25 Jan 2013 21:36:04 +0100 (CET) Subject: [pypy-commit] pypy default: a real fix Message-ID: <20130125203604.8CB8A1C1055@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r60484:4bce2d983bf8 Date: 2013-01-25 22:35 +0200 http://bitbucket.org/pypy/pypy/changeset/4bce2d983bf8/ Log: a real fix diff --git a/pypy/tool/jitlogparser/parser.py b/pypy/tool/jitlogparser/parser.py --- a/pypy/tool/jitlogparser/parser.py +++ b/pypy/tool/jitlogparser/parser.py @@ -215,6 +215,7 @@ _lineset = None is_bytecode = False inline_level = None + bytecode_name = None # factory method TraceForOpcode = TraceForOpcode @@ -244,26 +245,29 @@ return ",".join([str(len(v)) for v in stack]) def append_to_res(bc): - if not stack: - stack.append([]) - else: - if bc.inline_level is not None: - if bc.inline_level == len(stack) - 1: - pass - elif bc.inline_level > len(stack) - 1: - stack.append([]) - else: - while bc.inline_level + 1 < len(stack): - last = stack.pop() - stack[-1].append(cls(last, getpath(stack), storage)) + if bc.inline_level is not None: + if bc.inline_level == len(stack) - 1: + pass + elif bc.inline_level > len(stack) - 1: + stack.append([]) + else: + while bc.inline_level + 1 < len(stack): + last = stack.pop() + stack[-1].append(cls(last, getpath(stack), storage)) stack[-1].append(bc) so_far = [] stack = [] + nothing_yet = True for op in operations: if op.name == 'debug_merge_point': if so_far: - append_to_res(cls.TraceForOpcode(so_far, storage, loopname)) + opc = cls.TraceForOpcode(so_far, storage, loopname) + if nothing_yet: + nothing_yet = False + for i in xrange(opc.inline_level + 1): + stack.append([]) + append_to_res(opc) if limit: break so_far = [] diff --git a/pypy/tool/jitlogparser/test/test_parser.py b/pypy/tool/jitlogparser/test/test_parser.py --- a/pypy/tool/jitlogparser/test/test_parser.py +++ b/pypy/tool/jitlogparser/test/test_parser.py @@ -365,3 +365,14 @@ """) f = Function.from_operations(loop.operations, LoopStorage()) assert len(f.chunks) == 3 + +def test_parse_from_inside(): + loop = parse(""" + [] + debug_merge_point(1, 0, 'two') + debug_merge_point(2, 0, 'three') + debug_merge_point(0, 0, 'one') + """) + f = Function.from_operations(loop.operations, LoopStorage()) + assert len(f.chunks) == 2 + From noreply at buildbot.pypy.org Fri Jan 25 21:36:37 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 25 Jan 2013 21:36:37 +0100 (CET) Subject: [pypy-commit] jitviewer default: revert this hack Message-ID: <20130125203637.7835E1C1055@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r211:a30511ad22d4 Date: 2013-01-25 22:36 +0200 http://bitbucket.org/pypy/jitviewer/changeset/a30511ad22d4/ Log: revert this hack diff --git a/_jitviewer/display.py b/_jitviewer/display.py --- a/_jitviewer/display.py +++ b/_jitviewer/display.py @@ -48,10 +48,4 @@ last_lineno = no else: no = last_lineno - try: - self.lines[no - self.firstlineno].chunks.append(chunk) - except IndexError: - self.lines[-1].chunks.append(chunk) # too bad, just stash it there - - - + self.lines[no - self.firstlineno].chunks.append(chunk) From noreply at buildbot.pypy.org Fri Jan 25 21:40:00 2013 From: noreply at buildbot.pypy.org (fijal) Date: Fri, 25 Jan 2013 21:40:00 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: merge default Message-ID: <20130125204000.BE0C71C1055@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60485:2bc4c9896967 Date: 2013-01-25 22:39 +0200 http://bitbucket.org/pypy/pypy/changeset/2bc4c9896967/ Log: merge default diff --git a/pypy/tool/jitlogparser/parser.py b/pypy/tool/jitlogparser/parser.py --- a/pypy/tool/jitlogparser/parser.py +++ b/pypy/tool/jitlogparser/parser.py @@ -215,6 +215,7 @@ _lineset = None is_bytecode = False inline_level = None + bytecode_name = None # factory method TraceForOpcode = TraceForOpcode @@ -244,26 +245,29 @@ return ",".join([str(len(v)) for v in stack]) def append_to_res(bc): - if not stack: - stack.append([]) - else: - if bc.inline_level is not None: - if bc.inline_level == len(stack) - 1: - pass - elif bc.inline_level > len(stack) - 1: - stack.append([]) - else: - while bc.inline_level + 1 < len(stack): - last = stack.pop() - stack[-1].append(cls(last, getpath(stack), storage)) + if bc.inline_level is not None: + if bc.inline_level == len(stack) - 1: + pass + elif bc.inline_level > len(stack) - 1: + stack.append([]) + else: + while bc.inline_level + 1 < len(stack): + last = stack.pop() + stack[-1].append(cls(last, getpath(stack), storage)) stack[-1].append(bc) so_far = [] stack = [] + nothing_yet = True for op in operations: if op.name == 'debug_merge_point': if so_far: - append_to_res(cls.TraceForOpcode(so_far, storage, loopname)) + opc = cls.TraceForOpcode(so_far, storage, loopname) + if nothing_yet: + nothing_yet = False + for i in xrange(opc.inline_level + 1): + stack.append([]) + append_to_res(opc) if limit: break so_far = [] diff --git a/pypy/tool/jitlogparser/test/test_parser.py b/pypy/tool/jitlogparser/test/test_parser.py --- a/pypy/tool/jitlogparser/test/test_parser.py +++ b/pypy/tool/jitlogparser/test/test_parser.py @@ -365,3 +365,14 @@ """) f = Function.from_operations(loop.operations, LoopStorage()) assert len(f.chunks) == 3 + +def test_parse_from_inside(): + loop = parse(""" + [] + debug_merge_point(1, 0, 'two') + debug_merge_point(2, 0, 'three') + debug_merge_point(0, 0, 'one') + """) + f = Function.from_operations(loop.operations, LoopStorage()) + assert len(f.chunks) == 2 + From noreply at buildbot.pypy.org Fri Jan 25 22:44:20 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Fri, 25 Jan 2013 22:44:20 +0100 (CET) Subject: [pypy-commit] pypy py3k: abridge Message-ID: <20130125214420.7A0871C0041@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60486:e75dafc63125 Date: 2013-01-25 13:30 -0800 http://bitbucket.org/pypy/pypy/changeset/e75dafc63125/ Log: abridge diff --git a/pypy/objspace/std/longobject.py b/pypy/objspace/std/longobject.py --- a/pypy/objspace/std/longobject.py +++ b/pypy/objspace/std/longobject.py @@ -198,7 +198,7 @@ f = w_long1.num.truediv(w_long2.num) except ZeroDivisionError: raise OperationError(space.w_ZeroDivisionError, - space.wrap("integer division or modulo by zero")) + space.wrap("division by zero")) except OverflowError: raise OperationError( space.w_OverflowError, From noreply at buildbot.pypy.org Fri Jan 25 22:44:21 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Fri, 25 Jan 2013 22:44:21 +0100 (CET) Subject: [pypy-commit] pypy py3k: fix tests under -A, attrgetter now fails fast Message-ID: <20130125214421.EDB201C0041@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60487:8bdff3f40f7f Date: 2013-01-25 13:30 -0800 http://bitbucket.org/pypy/pypy/changeset/8bdff3f40f7f/ Log: fix tests under -A, attrgetter now fails fast diff --git a/pypy/module/operator/app_operator.py b/pypy/module/operator/app_operator.py --- a/pypy/module/operator/app_operator.py +++ b/pypy/module/operator/app_operator.py @@ -35,10 +35,8 @@ def single_attr_getter(attr): if not isinstance(attr, str): - def _raise_typeerror(obj): - raise TypeError("argument must be a string, not %r" % - (type(attr).__name__,)) - return _raise_typeerror + raise TypeError("attribute name must be a string, not {!r}".format( + type(attr).__name__)) # def make_getter(name, prevfn=None): if prevfn is None: diff --git a/pypy/module/operator/test/test_operator.py b/pypy/module/operator/test/test_operator.py --- a/pypy/module/operator/test/test_operator.py +++ b/pypy/module/operator/test/test_operator.py @@ -14,8 +14,6 @@ assert a.getx(a) == 5 assert a.get3("foobar") == "b" assert a.getx(*(a,)) == 5 - assert a.get3(obj="foobar") == "b" - def test_getter_multiple_gest(self): import operator @@ -29,12 +27,16 @@ a.z = 'Z' assert operator.attrgetter('x','z','y')(a) == ('X', 'Z', 'Y') - raises(TypeError, operator.attrgetter('x', (), 'y'), a) + raises(TypeError, operator.attrgetter, ('x', (), 'y')) data = list(map(str, range(20))) assert operator.itemgetter(2,10,5)(data) == ('2', '10', '5') raises(TypeError, operator.itemgetter(2, 'x', 5), data) + def test_attrgetter(self): + import operator + raises(TypeError, operator.attrgetter, 2) + def test_dotted_attrgetter(self): from operator import attrgetter class A: From noreply at buildbot.pypy.org Fri Jan 25 22:44:23 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Fri, 25 Jan 2013 22:44:23 +0100 (CET) Subject: [pypy-commit] pypy py3k: fix gc_collects Message-ID: <20130125214423.2855D1C0041@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60488:75c14c291d34 Date: 2013-01-25 13:31 -0800 http://bitbucket.org/pypy/pypy/changeset/75c14c291d34/ Log: fix gc_collects diff --git a/lib-python/3.2/test/test_exceptions.py b/lib-python/3.2/test/test_exceptions.py --- a/lib-python/3.2/test/test_exceptions.py +++ b/lib-python/3.2/test/test_exceptions.py @@ -497,8 +497,8 @@ # must clear the latter manually for our test to succeed. e.__context__ = None obj = None + gc_collect() obj = wr() - gc_collect() self.assertTrue(obj is None, "%s" % obj) # Some complicated construct @@ -839,6 +839,7 @@ self.assertNotEqual(wr(), None) else: self.fail("MemoryError not raised") + gc_collect() self.assertEqual(wr(), None) def test_recursion_error_cleanup(self): From noreply at buildbot.pypy.org Fri Jan 25 22:44:24 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Fri, 25 Jan 2013 22:44:24 +0100 (CET) Subject: [pypy-commit] pypy py3k: kill Message-ID: <20130125214424.596A21C0041@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60489:f611af118b2d Date: 2013-01-25 13:31 -0800 http://bitbucket.org/pypy/pypy/changeset/f611af118b2d/ Log: kill diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py --- a/pypy/objspace/std/test/test_typeobject.py +++ b/pypy/objspace/std/test/test_typeobject.py @@ -901,12 +901,6 @@ pass class B(A): pass - A.__eq__ - A.__ne__ - assert A.__eq__(A) - assert not A.__eq__(B) - assert A.__ne__(B) - assert not A.__ne__(A) assert A == A assert A != B assert not A == B From noreply at buildbot.pypy.org Fri Jan 25 22:44:25 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Fri, 25 Jan 2013 22:44:25 +0100 (CET) Subject: [pypy-commit] pypy py3k: convert LookupErrors from find_module into SyntaxErrors Message-ID: <20130125214425.89C7D1C0041@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60490:16d93617dc0c Date: 2013-01-25 13:32 -0800 http://bitbucket.org/pypy/pypy/changeset/16d93617dc0c/ Log: convert LookupErrors from find_module into SyntaxErrors diff --git a/pypy/module/imp/interp_imp.py b/pypy/module/imp/interp_imp.py --- a/pypy/module/imp/interp_imp.py +++ b/pypy/module/imp/interp_imp.py @@ -87,8 +87,14 @@ # object doesn't have a name attached. We do the same in PyPy, because # there is no easy way to attach the filename -- too bad fd = stream.try_to_find_file_descriptor() - w_fileobj = interp_io.open(space, space.wrap(fd), find_info.filemode, - encoding=encoding) + try: + w_fileobj = interp_io.open(space, space.wrap(fd), + find_info.filemode, encoding=encoding) + except OperationError as e: + if e.match(space, space.w_LookupError): + raise OperationError(space.w_SyntaxError, + space.str(e.get_w_value(space))) + raise else: w_fileobj = space.w_None w_import_info = space.newtuple( diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py --- a/pypy/module/imp/test/test_import.py +++ b/pypy/module/imp/test/test_import.py @@ -92,7 +92,8 @@ 'a=15\nb=16\rc="""foo\r\nbar"""\r', mode='wb') setuppkg("encoded", # actually a line 2, setuppkg() sets up a line1 - line2 = "# encoding: iso-8859-1\n") + line2 = "# encoding: iso-8859-1\n", + bad = "# encoding: uft-8\n") # create compiled/x.py and a corresponding pyc file p = setuppkg("compiled", x = "x = 84") @@ -668,6 +669,11 @@ fd = imp.find_module('line2', encoded.__path__)[0] assert fd.encoding == 'iso-8859-1' + def test_bad_source_encoding(self): + import imp + import encoded + raises(SyntaxError, imp.find_module, 'bad', encoded.__path__) + class TestAbi: def test_abi_tag(self): From noreply at buildbot.pypy.org Fri Jan 25 22:44:26 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Fri, 25 Jan 2013 22:44:26 +0100 (CET) Subject: [pypy-commit] pypy py3k: backout b1ab47f92c6e Message-ID: <20130125214426.CFDB61C0041@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60491:5d354f43b715 Date: 2013-01-25 13:41 -0800 http://bitbucket.org/pypy/pypy/changeset/5d354f43b715/ Log: backout b1ab47f92c6e matching cpython 3.2 isn't important here because importlib as import behaves like this anyway diff --git a/pypy/module/imp/importing.py b/pypy/module/imp/importing.py --- a/pypy/module/imp/importing.py +++ b/pypy/module/imp/importing.py @@ -650,7 +650,7 @@ else: # ImportError msg = "No module named %s" - raise operationerrfmt(space.w_ImportError, msg, partname) + raise operationerrfmt(space.w_ImportError, msg, modulename) @jit.dont_look_inside def reload(space, w_module): diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py --- a/pypy/module/imp/test/test_import.py +++ b/pypy/module/imp/test/test_import.py @@ -410,13 +410,13 @@ def imp(): from pkg import relative_f exc = raises(ImportError, imp) - assert exc.value.args[0] == "No module named imp" + assert exc.value.args[0] == "No module named pkg.imp" def test_no_relative_import_bug(self): def imp(): from pkg import relative_g exc = raises(ImportError, imp) - assert exc.value.args[0] == "No module named imp" + assert exc.value.args[0] == "No module named pkg.imp" def test_future_relative_import_level_1(self): from pkg import relative_c From noreply at buildbot.pypy.org Fri Jan 25 22:44:28 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Fri, 25 Jan 2013 22:44:28 +0100 (CET) Subject: [pypy-commit] pypy py3k: further test for this error message Message-ID: <20130125214428.17CDC1C0041@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60492:282c08ad3c0a Date: 2013-01-25 13:41 -0800 http://bitbucket.org/pypy/pypy/changeset/282c08ad3c0a/ Log: further test for this error message diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py --- a/pypy/module/imp/test/test_import.py +++ b/pypy/module/imp/test/test_import.py @@ -418,6 +418,12 @@ exc = raises(ImportError, imp) assert exc.value.args[0] == "No module named pkg.imp" + def test_import_msg(self): + def imp(): + import pkg.i_am_not_here.neither_am_i + exc = raises(ImportError, imp) + assert exc.value.args[0] == "No module named i_am_not_here.neither_am_i" + def test_future_relative_import_level_1(self): from pkg import relative_c assert relative_c.inpackage == 1 From noreply at buildbot.pypy.org Fri Jan 25 22:44:29 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Fri, 25 Jan 2013 22:44:29 +0100 (CET) Subject: [pypy-commit] pypy py3k: use 3.3's expected results as we already match importlib's err message Message-ID: <20130125214429.44B7A1C0041@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60493:11fdb7f93f1d Date: 2013-01-25 13:43 -0800 http://bitbucket.org/pypy/pypy/changeset/11fdb7f93f1d/ Log: use 3.3's expected results as we already match importlib's err message diff --git a/lib-python/3.2/test/test_pydoc.py b/lib-python/3.2/test/test_pydoc.py --- a/lib-python/3.2/test/test_pydoc.py +++ b/lib-python/3.2/test/test_pydoc.py @@ -392,11 +392,10 @@ modname = 'testmod_xyzzy' testpairs = ( ('i_am_not_here', 'i_am_not_here'), - ('test.i_am_not_here_either', 'i_am_not_here_either'), - ('test.i_am_not_here.neither_am_i', 'i_am_not_here.neither_am_i'), - ('i_am_not_here.{}'.format(modname), - 'i_am_not_here.{}'.format(modname)), - ('test.{}'.format(modname), modname), + ('test.i_am_not_here_either', 'test.i_am_not_here_either'), + ('test.i_am_not_here.neither_am_i', 'test.i_am_not_here'), + ('i_am_not_here.{}'.format(modname), 'i_am_not_here'), + ('test.{}'.format(modname), 'test.{}'.format(modname)), ) sourcefn = os.path.join(TESTFN, modname) + os.extsep + "py" From noreply at buildbot.pypy.org Fri Jan 25 22:54:34 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Fri, 25 Jan 2013 22:54:34 +0100 (CET) Subject: [pypy-commit] pypy default: backout 7cd4c46 Message-ID: <20130125215434.F0FFF1C0041@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: Changeset: r60494:2ae4a9e25e32 Date: 2013-01-25 13:52 -0800 http://bitbucket.org/pypy/pypy/changeset/2ae4a9e25e32/ Log: backout 7cd4c46 diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -310,7 +310,10 @@ def decorate(func): from rpython.tool.sourcetools import compile2 # - jitdriver = JitDriver(greens=[], reds='auto', name=name) + def get_printable_location(): + return name + jitdriver = JitDriver(get_printable_location=get_printable_location, + greens=[], reds='auto', name=name) # args = ','.join(['a%d' % i for i in range(func.func_code.co_argcount)]) source = """def callback_with_jitdriver(%(args)s): From noreply at buildbot.pypy.org Fri Jan 25 22:54:36 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Fri, 25 Jan 2013 22:54:36 +0100 (CET) Subject: [pypy-commit] pypy default: arigo lightened this restriction Message-ID: <20130125215436.22B5F1C0041@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: Changeset: r60495:c009f60a2307 Date: 2013-01-25 13:54 -0800 http://bitbucket.org/pypy/pypy/changeset/c009f60a2307/ Log: arigo lightened this restriction diff --git a/rpython/rlib/test/test_jit.py b/rpython/rlib/test/test_jit.py --- a/rpython/rlib/test/test_jit.py +++ b/rpython/rlib/test/test_jit.py @@ -21,8 +21,6 @@ assert driver.reds == [] assert driver.numreds is None py.test.raises(TypeError, "driver.can_enter_jit(foo='something')") - # - py.test.raises(AssertionError, "JitDriver(greens=['foo'], reds='auto', get_printable_location='something')") py.test.raises(AssertionError, "JitDriver(greens=['foo'], reds='auto', confirm_enter_jit='something')") def test_jitdriver_numreds(): From noreply at buildbot.pypy.org Fri Jan 25 23:40:23 2013 From: noreply at buildbot.pypy.org (amauryfa) Date: Fri, 25 Jan 2013 23:40:23 +0100 (CET) Subject: [pypy-commit] pypy default: Merge 25f5ab9 from py3k: Message-ID: <20130125224023.EA33A1C0041@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: Changeset: r60496:d4c30109a272 Date: 2013-01-24 00:32 +0100 http://bitbucket.org/pypy/pypy/changeset/d4c30109a272/ Log: Merge 25f5ab9 from py3k: annotation of range(): if start and step are non-negative, then the item is non-negative as well. Used in py3k function array._array_reconstructor, which uses a list comprehension with a variable (positive) stride. diff --git a/rpython/annotator/builtin.py b/rpython/annotator/builtin.py --- a/rpython/annotator/builtin.py +++ b/rpython/annotator/builtin.py @@ -79,7 +79,7 @@ s_item = s_ImpossibleValue else: nonneg = False # so far - if step > 0: + if step > 0 or s_step.nonneg: nonneg = s_start.nonneg elif step < 0: nonneg = s_stop.nonneg or (s_stop.is_constant() and diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py --- a/rpython/annotator/test/test_annrpython.py +++ b/rpython/annotator/test/test_annrpython.py @@ -2694,6 +2694,23 @@ assert isinstance(s, annmodel.SomeInteger) assert s.nonneg + def test_range_nonneg_variablestep(self): + def get_step(n): + if n == 1: + return 2 + else: + return 3 + def fun(n, k): + step = get_step(n) + for i in range(0, n, step): + if k == 17: + return i + return 0 + a = self.RPythonAnnotator() + s = a.build_types(fun, [int, int]) + assert isinstance(s, annmodel.SomeInteger) + assert s.nonneg + def test_reverse_range_nonneg(self): def fun(n, k): for i in range(n-1, -1, -1): From noreply at buildbot.pypy.org Fri Jan 25 23:40:25 2013 From: noreply at buildbot.pypy.org (amauryfa) Date: Fri, 25 Jan 2013 23:40:25 +0100 (CET) Subject: [pypy-commit] pypy default: Propagate no_nul property in str.replace(). Message-ID: <20130125224025.205CA1C0041@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: Changeset: r60497:9fe829a66345 Date: 2013-01-24 00:59 +0100 http://bitbucket.org/pypy/pypy/changeset/9fe829a66345/ Log: Propagate no_nul property in str.replace(). diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py --- a/rpython/annotator/test/test_annrpython.py +++ b/rpython/annotator/test/test_annrpython.py @@ -3271,6 +3271,7 @@ a = self.RPythonAnnotator() s = a.build_types(f, [str]) assert isinstance(s, annmodel.SomeString) + assert s.no_nul def f(x): return u'a'.replace(x, u'b') diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py --- a/rpython/annotator/unaryop.py +++ b/rpython/annotator/unaryop.py @@ -509,7 +509,7 @@ return getbookkeeper().newlist(s_item) def method_replace(str, s1, s2): - return str.basestringclass() + return str.basestringclass(no_nul=str.no_nul and s2.no_nul) def getslice(str, s_start, s_stop): check_negative_slice(s_start, s_stop) From noreply at buildbot.pypy.org Fri Jan 25 23:40:26 2013 From: noreply at buildbot.pypy.org (amauryfa) Date: Fri, 25 Jan 2013 23:40:26 +0100 (CET) Subject: [pypy-commit] pypy default: Issue1381 resolved: Prevent random.jumpahead() to yield random values above 1.0. Message-ID: <20130125224026.40A261C0041@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: Changeset: r60498:0ec3d77645a2 Date: 2013-01-25 22:02 +0100 http://bitbucket.org/pypy/pypy/changeset/0ec3d77645a2/ Log: Issue1381 resolved: Prevent random.jumpahead() to yield random values above 1.0. Similar to CPython issue14591, even if we don't use exactly the same algorithm (The n%i operation operates on C ints, when CPython uses Python numbers) diff --git a/rpython/rlib/rrandom.py b/rpython/rlib/rrandom.py --- a/rpython/rlib/rrandom.py +++ b/rpython/rlib/rrandom.py @@ -100,9 +100,21 @@ def jumpahead(self, n): mt = self.state - for i in range(N - 1, 0, -1): + for i in range(N - 1, 1, -1): j = n % i mt[i], mt[j] = mt[j], mt[i] - for i in range(N): + nonzero = False + for i in range(1, N): mt[i] += r_uint(i + 1) + mt[i] &= r_uint(0xffffffff) + nonzero |= bool(mt[i]) + # Ensure the state is nonzero: in the unlikely event that mt[1] through + # mt[N-1] are all zero, set the MSB of mt[0] (see issue #14591). In the + # normal case, we fall back to the pre-issue 14591 behaviour for mt[0]. + if nonzero: + mt[0] += r_uint(1) + mt[0] &= r_uint(0xffffffff) + else: + mt[0] = r_uint(0x80000000) + print mt[:3] self.index = N diff --git a/rpython/rlib/test/test_rrandom.py b/rpython/rlib/test/test_rrandom.py --- a/rpython/rlib/test/test_rrandom.py +++ b/rpython/rlib/test/test_rrandom.py @@ -46,6 +46,15 @@ assert tuple(rnd.state) + (rnd.index, ) == cpyrandom.getstate() +def test_jumpahead_badstate(): + rnd = Random() + s, j = 4043161618, 2674112291824205302 + rnd.init_by_array([s]) + rnd.jumpahead(j) + for i in range(500): + r = rnd.random() + assert r <= 1.0, (r, i) + def test_translate(): def f(x, y): x = r_uint(x) From noreply at buildbot.pypy.org Fri Jan 25 23:40:27 2013 From: noreply at buildbot.pypy.org (amauryfa) Date: Fri, 25 Jan 2013 23:40:27 +0100 (CET) Subject: [pypy-commit] pypy default: Add a cross-platform rsocket.dup(), which also works on win32. Message-ID: <20130125224027.710831C0041@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: Changeset: r60499:ea8199da498f Date: 2013-01-25 22:58 +0100 http://bitbucket.org/pypy/pypy/changeset/ea8199da498f/ Log: Add a cross-platform rsocket.dup(), which also works on win32. py3k needs it, but it's not exposed in any way on 2.7. diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py --- a/rpython/rlib/_rsocket_rffi.py +++ b/rpython/rlib/_rsocket_rffi.py @@ -343,6 +343,12 @@ ('iErrorCode', rffi.CFixedArray(rffi.INT, 10)), #FD_MAX_EVENTS ]) + CConfig.WSAPROTOCOL_INFO = platform.Struct( + 'struct WSAPROTOCOL_INFO', + []) # Struct is just passed between functions + CConfig.FROM_PROTOCOL_INFO = platform.DefinedConstantInteger( + 'FROM_PROTOCOL_INFO') + CConfig.timeval = platform.Struct('struct timeval', [('tv_sec', rffi.LONG), ('tv_usec', rffi.LONG)]) @@ -602,6 +608,18 @@ rffi.INT) tcp_keepalive = cConfig.tcp_keepalive + WSAPROTOCOL_INFO = cConfig.WSAPROTOCOL_INFO + FROM_PROTOCOL_INFO = cConfig.FROM_PROTOCOL_INFO + WSADuplicateSocket = external('WSADuplicateSocket', + [socketfd_type, rwin32.DWORD, + lltype.Ptr(WSAPROTOCOL_INFO)], + rffi.INT) + WSASocket = external('WSASocket', + [rffi.INT, rffi.INT, rffi.INT, + lltype.Ptr(WSAPROTOCOL_INFO), + rffi.DWORD, rffi.DWORD], + socketfd_type) + if WIN32: WSAData = cConfig.WSAData WSAStartup = external('WSAStartup', [rffi.INT, lltype.Ptr(WSAData)], diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py --- a/rpython/rlib/rsocket.py +++ b/rpython/rlib/rsocket.py @@ -1044,7 +1044,21 @@ return (make_socket(fd0, family, type, proto, SocketClass), make_socket(fd1, family, type, proto, SocketClass)) -if hasattr(_c, 'dup'): +if _c.WIN32: + def dup(fd): + with lltype.scoped_alloc(_c.WSAData, zero=True) as info: + if _c.WSADuplicateSocket(fd, rwin32.GetCurrentProcessId(), info): + raise last_error() + result = _c.WSASocket( + _c.FROM_PROTOCOL_INFO, _c.FROM_PROTOCOL_INFO, + _c.FROM_PROTOCOL_INFO, info, 0, 0) + if result == INVALID_SOCKET: + raise last_error() + return result +else: + def dup(fd): + return _c.dup(fd) + def fromfd(fd, family, type, proto=0, SocketClass=RSocket): # Dup the fd so it and the socket can be closed independently fd = _c.dup(fd) diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py --- a/rpython/rlib/test/test_rsocket.py +++ b/rpython/rlib/test/test_rsocket.py @@ -373,6 +373,15 @@ assert s.fd != s2.fd assert s.getsockname().eq(s2.getsockname()) +def test_c_dup(): + # rsocket.dup() duplicates fd, it also works on Windows + # (but only on socket handles!) + s = RSocket(AF_INET, SOCK_STREAM) + s.setsockopt_int(SOL_SOCKET, SO_REUSEADDR, 1) + s.bind(INETAddress('localhost', 50007)) + fd2 = dup(s.fd) + assert s.fd != fd2 + def test_inet_aton(): assert inet_aton('1.2.3.4') == '\x01\x02\x03\x04' assert inet_aton('127.0.0.1') == '\x7f\x00\x00\x01' From noreply at buildbot.pypy.org Fri Jan 25 23:40:59 2013 From: noreply at buildbot.pypy.org (amauryfa) Date: Fri, 25 Jan 2013 23:40:59 +0100 (CET) Subject: [pypy-commit] pypy py3k: re-import a full copy of rpython/flowspace from default branch. Message-ID: <20130125224059.A77561C0041@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r60500:85f0de783ee0 Date: 2013-01-24 01:04 +0100 http://bitbucket.org/pypy/pypy/changeset/85f0de783ee0/ Log: re-import a full copy of rpython/flowspace from default branch. diff --git a/rpython/flowspace/argument.py b/rpython/flowspace/argument.py --- a/rpython/flowspace/argument.py +++ b/rpython/flowspace/argument.py @@ -201,11 +201,9 @@ elif num_args > co_argcount: raise ArgErrCount(num_args, num_kwds, signature, defaults_w, 0) - # if a **kwargs argument is needed, create the dict - w_kwds = None + # if a **kwargs argument is needed, explode if signature.has_kwarg(): - w_kwds = self.space.newdict(kwargs=True) - scope_w[co_argcount + signature.has_vararg()] = w_kwds + raise TypeError("Keyword arguments as **kwargs is not supported by RPython") # handle keyword arguments num_remainingkwds = 0 @@ -236,24 +234,10 @@ num_remainingkwds -= 1 if num_remainingkwds: - if w_kwds is not None: - # collect extra keyword arguments into the **kwarg - limit = len(keywords) - if self.keyword_names_w is not None: - limit -= len(self.keyword_names_w) - for i in range(len(keywords)): - if i in kwds_mapping: - continue - if i < limit: - w_key = self.space.wrap(keywords[i]) - else: - w_key = self.keyword_names_w[i - limit] - self.space.setitem(w_kwds, w_key, keywords_w[i]) - else: - if co_argcount == 0: - raise ArgErrCount(num_args, num_kwds, signature, defaults_w, 0) - raise ArgErrUnknownKwds(self.space, num_remainingkwds, keywords, - kwds_mapping, self.keyword_names_w) + if co_argcount == 0: + raise ArgErrCount(num_args, num_kwds, signature, defaults_w, 0) + raise ArgErrUnknownKwds(self.space, num_remainingkwds, keywords, + kwds_mapping, self.keyword_names_w) # check for missing arguments and fill them from the kwds, # or with defaults, if available diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py --- a/rpython/flowspace/bytecode.py +++ b/rpython/flowspace/bytecode.py @@ -32,15 +32,13 @@ """ opnames = host_bytecode_spec.method_names - def __init__(self, argcount, kwonlyargcount, nlocals, stacksize, flags, + def __init__(self, argcount, nlocals, stacksize, flags, code, consts, names, varnames, filename, name, firstlineno, lnotab, freevars): """Initialize a new code object""" self.co_name = name assert nlocals >= 0 self.co_argcount = argcount - # note that this is ignored, as HostCode represents a python2 bytecode - self.co_kwonlyargcount = kwonlyargcount self.co_nlocals = nlocals self.co_stacksize = stacksize self.co_flags = flags @@ -60,7 +58,6 @@ """Initialize the code object from a real (CPython) one. """ return cls(code.co_argcount, - getattr(code, 'co_kwonlyargcount', 0), code.co_nlocals, code.co_stacksize, code.co_flags, diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py --- a/rpython/flowspace/flowcontext.py +++ b/rpython/flowspace/flowcontext.py @@ -662,8 +662,8 @@ self.last_exception = operr raise operr else: - raise FSException(space.w_RuntimeError, - space.wrap("No active exception to reraise")) + raise FSException(space.w_TypeError, + space.wrap("raise: no active exception to re-raise")) w_value = w_traceback = space.w_None if nbargs >= 3: @@ -1164,75 +1164,6 @@ LOAD_CLOSURE = BAD_OPCODE MAKE_CLOSURE = BAD_OPCODE - # opcodes removed or heavily changed in python 3 - - def slice(self, w_start, w_end): - w_obj = self.popvalue() - w_result = self.space.getslice(w_obj, w_start, w_end) - self.pushvalue(w_result) - - def SLICE_0(self, oparg, next_instr): - self.slice(self.space.w_None, self.space.w_None) - - def SLICE_1(self, oparg, next_instr): - w_start = self.popvalue() - self.slice(w_start, self.space.w_None) - - def SLICE_2(self, oparg, next_instr): - w_end = self.popvalue() - self.slice(self.space.w_None, w_end) - - def SLICE_3(self, oparg, next_instr): - w_end = self.popvalue() - w_start = self.popvalue() - self.slice(w_start, w_end) - - def storeslice(self, w_start, w_end): - w_obj = self.popvalue() - w_newvalue = self.popvalue() - self.space.setslice(w_obj, w_start, w_end, w_newvalue) - - def STORE_SLICE_0(self, oparg, next_instr): - self.storeslice(self.space.w_None, self.space.w_None) - - def STORE_SLICE_1(self, oparg, next_instr): - w_start = self.popvalue() - self.storeslice(w_start, self.space.w_None) - - def STORE_SLICE_2(self, oparg, next_instr): - w_end = self.popvalue() - self.storeslice(self.space.w_None, w_end) - - def STORE_SLICE_3(self, oparg, next_instr): - w_end = self.popvalue() - w_start = self.popvalue() - self.storeslice(w_start, w_end) - - def deleteslice(self, w_start, w_end): - w_obj = self.popvalue() - self.space.delslice(w_obj, w_start, w_end) - - def DELETE_SLICE_0(self, oparg, next_instr): - self.deleteslice(self.space.w_None, self.space.w_None) - - def DELETE_SLICE_1(self, oparg, next_instr): - w_start = self.popvalue() - self.deleteslice(w_start, self.space.w_None) - - def DELETE_SLICE_2(self, oparg, next_instr): - w_end = self.popvalue() - self.deleteslice(self.space.w_None, w_end) - - def DELETE_SLICE_3(self, oparg, next_instr): - w_end = self.popvalue() - w_start = self.popvalue() - self.deleteslice(w_start, w_end) - - def make_arguments(self, nargs): - return ArgumentsForTranslation(self.space, self.peekvalues(nargs)) - def argument_factory(self, *args): - return ArgumentsForTranslation(self.space, *args) - ### Frame blocks ### class SuspendedUnroller(object): diff --git a/rpython/flowspace/objspace.py b/rpython/flowspace/objspace.py --- a/rpython/flowspace/objspace.py +++ b/rpython/flowspace/objspace.py @@ -75,7 +75,6 @@ clsname = exc.__name__ locals()['w_' + clsname] = Constant(exc) - py3k = False # the RPython bytecode is still python2 # the following exceptions should not show up # during flow graph construction w_NameError = 'NameError' @@ -159,20 +158,6 @@ return val return self.unwrap(w_obj) - identifier_w = str_w # RPython interprets Python2, where identifier_w is - # equivalent to str_w - - def unicode_w(self, w_obj): - if isinstance(w_obj, Constant): - val = w_obj.value - if type(val) is str: - return val.decode('ascii') - elif type(val) is unicode: - return val - else: - raise TypeError("expected unicode: " + repr(w_obj)) - return self.unwrap(w_obj) - def float_w(self, w_obj): if isinstance(w_obj, Constant): val = w_obj.value diff --git a/rpython/flowspace/operation.py b/rpython/flowspace/operation.py --- a/rpython/flowspace/operation.py +++ b/rpython/flowspace/operation.py @@ -374,79 +374,3 @@ _add_exceptions("""pow""", OverflowError) # for the float case del _add_exceptions, _add_except_ovf - -def make_op(fs, name, symbol, arity, specialnames): - if getattr(fs, name, None) is not None: - return - - op = None - skip = False - arithmetic = False - - if (name.startswith('del') or - name.startswith('set') or - name.startswith('inplace_')): - # skip potential mutators - skip = True - elif name in ('id', 'hash', 'iter', 'userdel'): - # skip potential runtime context dependecies - skip = True - elif name in ('repr', 'str'): - rep = getattr(__builtin__, name) - def op(obj): - s = rep(obj) - if "at 0x" in s: - print >>sys.stderr, "Warning: captured address may be awkward" - return s - else: - op = FunctionByName[name] - arithmetic = (name + '_ovf') in FunctionByName - - if not op and not skip: - raise ValueError("XXX missing operator: %s" % (name,)) - - def generic_operator(self, *args_w): - assert len(args_w) == arity, name + " got the wrong number of arguments" - if op: - args = [] - for w_arg in args_w: - try: - arg = self.unwrap_for_computation(w_arg) - except model.UnwrapException: - break - else: - args.append(arg) - else: - # All arguments are constants: call the operator now - try: - result = op(*args) - except Exception, e: - etype = e.__class__ - msg = "generated by a constant operation:\n\t%s%r" % ( - name, tuple(args)) - raise OperationThatShouldNotBePropagatedError( - self.wrap(etype), self.wrap(msg)) - else: - # don't try to constant-fold operations giving a 'long' - # result. The result is probably meant to be sent to - # an intmask(), but the 'long' constant confuses the - # annotator a lot. - if arithmetic and type(result) is long: - pass - # don't constant-fold getslice on lists, either - elif name == 'getslice' and type(result) is list: - pass - # otherwise, fine - else: - try: - return self.wrap(result) - except model.WrapException: - # type cannot sanely appear in flow graph, - # store operation with variable result instead - pass - w_result = self.do_operation_with_implicit_exceptions(name, *args_w) - return w_result - - setattr(fs, name, generic_operator) - - diff --git a/rpython/flowspace/test/test_argument.py b/rpython/flowspace/test/test_argument.py --- a/rpython/flowspace/test/test_argument.py +++ b/rpython/flowspace/test/test_argument.py @@ -99,12 +99,6 @@ def str_w(self, s): return str(s) - def unicode_w(self, s): - return unicode(s) - - def identifier_w(self, s): - return self.unicode_w(s).encode('utf-8') - def len(self, x): return len(x) diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py --- a/rpython/flowspace/test/test_objspace.py +++ b/rpython/flowspace/test/test_objspace.py @@ -702,13 +702,6 @@ for op in block.operations: assert not op.opname == "call_args" - def test_keyword_arguments(self): - def g(a, b): - pass - def f(): - return g(a=1, b=2) - self.codetest(f) - def test_catch_importerror_1(self): def f(): try: From noreply at buildbot.pypy.org Fri Jan 25 23:41:01 2013 From: noreply at buildbot.pypy.org (amauryfa) Date: Fri, 25 Jan 2013 23:41:01 +0100 (CET) Subject: [pypy-commit] pypy py3k: Let rpython's float.__hash__ be different from interp-level W_FloatObject, Message-ID: <20130125224101.0573C1C0041@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r60501:811fe28a50d5 Date: 2013-01-25 23:39 +0100 http://bitbucket.org/pypy/pypy/changeset/811fe28a50d5/ Log: Let rpython's float.__hash__ be different from interp-level W_FloatObject, and revert the rpython one to the default branch. diff --git a/pypy/module/sys/system.py b/pypy/module/sys/system.py --- a/pypy/module/sys/system.py +++ b/pypy/module/sys/system.py @@ -2,7 +2,8 @@ from pypy.interpreter import gateway from rpython.rlib import rfloat, rbigint from rpython.rtyper.lltypesystem import rffi, lltype -from rpython.rlib.objectmodel import HASH_INF, HASH_NAN, HASH_IMAG +from pypy.objspace.std.floatobject import HASH_INF, HASH_NAN +from pypy.objspace.std.complexobject import HASH_IMAG app = gateway.applevel(""" diff --git a/pypy/objspace/std/complexobject.py b/pypy/objspace/std/complexobject.py --- a/pypy/objspace/std/complexobject.py +++ b/pypy/objspace/std/complexobject.py @@ -8,12 +8,13 @@ from rpython.rlib.rbigint import rbigint from rpython.rlib.rfloat import ( formatd, DTSF_STR_PRECISION, isinf, isnan, copysign) -from rpython.rlib.objectmodel import HASH_IMAG from rpython.rlib import jit, rcomplex from rpython.rlib.rarithmetic import intmask import math +HASH_IMAG = 1000003 + class W_AbstractComplexObject(W_Object): __slots__ = () diff --git a/pypy/objspace/std/floatobject.py b/pypy/objspace/std/floatobject.py --- a/pypy/objspace/std/floatobject.py +++ b/pypy/objspace/std/floatobject.py @@ -12,7 +12,6 @@ isinf, isnan, isfinite, INFINITY, NAN, copysign, formatd, DTSF_ADD_DOT_0, DTSF_STR_PRECISION) from rpython.rlib.rbigint import rbigint -from rpython.rlib.objectmodel import HASH_INF, HASH_NAN from rpython.rlib import rfloat from rpython.tool.sourcetools import func_with_new_name @@ -20,6 +19,9 @@ import math from pypy.objspace.std.intobject import W_IntObject +HASH_INF = 314159 +HASH_NAN = 0 + class W_AbstractFloatObject(W_Object): __slots__ = () diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py --- a/rpython/rlib/objectmodel.py +++ b/rpython/rlib/objectmodel.py @@ -451,10 +451,6 @@ x ^= length return intmask(x) -HASH_INF = 314159 -HASH_NAN = 0 -HASH_IMAG = 1000003 - def _hash_float(f): """The algorithm behind compute_hash() for a float. This implementation is identical to the CPython implementation, @@ -466,11 +462,11 @@ if not isfinite(f): if isinf(f): if f < 0.0: - return -HASH_INF + return -271828 else: - return HASH_INF + return 314159 else: #isnan(f): - return HASH_NAN + return 0 v, expo = math.frexp(f) v *= TAKE_NEXT hipart = int(v) diff --git a/rpython/rlib/test/test_objectmodel.py b/rpython/rlib/test/test_objectmodel.py --- a/rpython/rlib/test/test_objectmodel.py +++ b/rpython/rlib/test/test_objectmodel.py @@ -163,8 +163,8 @@ def test_compute_hash_float(): from rpython.rlib.rfloat import INFINITY, NAN - assert compute_hash(INFINITY) == HASH_INF - assert compute_hash(-INFINITY) == -HASH_INF + assert compute_hash(INFINITY) == 314159 + assert compute_hash(-INFINITY) == -271828 assert compute_hash(NAN) == 0 def test_compute_identity_hash(): @@ -335,8 +335,8 @@ q = Foo() assert compute_hash(q) == compute_identity_hash(q) from rpython.rlib.rfloat import INFINITY, NAN - assert compute_hash(INFINITY) == HASH_INF - assert compute_hash(-INFINITY) == -HASH_INF + assert compute_hash(INFINITY) == 314159 + assert compute_hash(-INFINITY) == -271828 assert compute_hash(NAN) == 0 return i*2 res = self.interpret(f, [42]) From noreply at buildbot.pypy.org Fri Jan 25 23:52:34 2013 From: noreply at buildbot.pypy.org (amauryfa) Date: Fri, 25 Jan 2013 23:52:34 +0100 (CET) Subject: [pypy-commit] pypy default: Remove extra print statement, thank you bdk! Message-ID: <20130125225234.1D12A1C0041@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: Changeset: r60502:dd27a638f3ab Date: 2013-01-25 23:52 +0100 http://bitbucket.org/pypy/pypy/changeset/dd27a638f3ab/ Log: Remove extra print statement, thank you bdk! diff --git a/rpython/rlib/rrandom.py b/rpython/rlib/rrandom.py --- a/rpython/rlib/rrandom.py +++ b/rpython/rlib/rrandom.py @@ -116,5 +116,4 @@ mt[0] &= r_uint(0xffffffff) else: mt[0] = r_uint(0x80000000) - print mt[:3] self.index = N From noreply at buildbot.pypy.org Sat Jan 26 00:24:53 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Sat, 26 Jan 2013 00:24:53 +0100 (CET) Subject: [pypy-commit] pypy default: Moved pypy.tool.version to rpython; reverted udir Message-ID: <20130125232453.7A3A41C047D@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: Changeset: r60503:64f878febd11 Date: 2013-01-26 00:24 +0100 http://bitbucket.org/pypy/pypy/changeset/64f878febd11/ Log: Moved pypy.tool.version to rpython; reverted udir diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -29,7 +29,7 @@ import pypy pypydir = os.path.dirname(os.path.abspath(pypy.__file__)) del pypy -from pypy.tool.version import get_repo_version_info +from rpython.tool.version import get_repo_version_info import time as t gmtime = t.gmtime() diff --git a/pypy/tool/test/test_version.py b/rpython/tool/test/test_version.py rename from pypy/tool/test/test_version.py rename to rpython/tool/test/test_version.py --- a/pypy/tool/test/test_version.py +++ b/rpython/tool/test/test_version.py @@ -1,6 +1,6 @@ import os, sys import py -from pypy.tool.version import get_repo_version_info, _get_hg_archive_version +from rpython.tool.version import get_repo_version_info, _get_hg_archive_version def test_hg_archival_version(tmpdir): def version_for(name, **kw): @@ -11,14 +11,14 @@ assert version_for('release', tag='release-123', node='000', - ) == ('PyPy', 'release-123', '000') + ) == ('RPython', 'release-123', '000') assert version_for('somebranch', node='000', branch='something', - ) == ('PyPy', 'something', '000') + ) == ('RPython', 'something', '000') def test_get_repo_version_info(): assert get_repo_version_info(None) - assert get_repo_version_info(os.devnull) == ('PyPy', '?', '?') - assert get_repo_version_info(sys.executable) == ('PyPy', '?', '?') + assert get_repo_version_info(os.devnull) == ('RPython', '?', '?') + assert get_repo_version_info(sys.executable) == ('RPython', '?', '?') diff --git a/rpython/tool/udir.py b/rpython/tool/udir.py --- a/rpython/tool/udir.py +++ b/rpython/tool/udir.py @@ -20,6 +20,7 @@ import os, sys import py +from rpython.tool.version import get_repo_version_info from py.path import local PYPY_KEEP = int(os.environ.get('PYPY_USESSION_KEEP', '3')) @@ -28,7 +29,17 @@ if dir is not None: dir = local(dir) if basename is None: - basename = '' + info = get_repo_version_info() + if info: + project, hgtag, hgid = info + basename = hgtag + if basename == '?': + basename = 'unknown' # directories with ? are not fun + # especially on windows + if isinstance(basename, unicode): + basename = basename.encode(sys.getdefaultencoding()) + else: + basename = '' basename = basename.replace('/', '--') if not basename.startswith('-'): basename = '-' + basename diff --git a/pypy/tool/version.py b/rpython/tool/version.py rename from pypy/tool/version.py rename to rpython/tool/version.py --- a/pypy/tool/version.py +++ b/rpython/tool/version.py @@ -1,10 +1,10 @@ import py import os from subprocess import Popen, PIPE -import pypy -pypydir = os.path.dirname(os.path.abspath(pypy.__file__)) -pypyroot = os.path.dirname(pypydir) -default_retval = 'PyPy', '?', '?' +import rpython +rpythondir = os.path.dirname(os.path.abspath(rpython.__file__)) +rpythonroot = os.path.dirname(rpythondir) +default_retval = 'RPython', '?', '?' def maywarn(err, repo_type='Mercurial'): if not err: @@ -20,16 +20,16 @@ # Try to see if we can get info from Git if hgexe is not specified. if not hgexe: - if os.path.isdir(os.path.join(pypyroot, '.git')): + if os.path.isdir(os.path.join(rpythonroot, '.git')): return _get_git_version() # Fallback to trying Mercurial. if hgexe is None: hgexe = py.path.local.sysfind('hg') - if os.path.isfile(os.path.join(pypyroot, '.hg_archival.txt')): - return _get_hg_archive_version(os.path.join(pypyroot, '.hg_archival.txt')) - elif not os.path.isdir(os.path.join(pypyroot, '.hg')): + if os.path.isfile(os.path.join(rpythonroot, '.hg_archival.txt')): + return _get_hg_archive_version(os.path.join(rpythonroot, '.hg_archival.txt')) + elif not os.path.isdir(os.path.join(rpythonroot, '.hg')): maywarn('Not running from a Mercurial repository!') return default_retval elif not hgexe: @@ -57,14 +57,14 @@ maywarn('command does not identify itself as Mercurial') return default_retval - p = Popen([str(hgexe), 'id', '-i', pypyroot], + p = Popen([str(hgexe), 'id', '-i', rpythonroot], stdout=PIPE, stderr=PIPE, env=env) hgid = p.stdout.read().strip() maywarn(p.stderr.read()) if p.wait() != 0: hgid = '?' - p = Popen([str(hgexe), 'id', '-t', pypyroot], + p = Popen([str(hgexe), 'id', '-t', rpythonroot], stdout=PIPE, stderr=PIPE, env=env) hgtags = [t for t in p.stdout.read().strip().split() if t != 'tip'] maywarn(p.stderr.read()) @@ -72,15 +72,15 @@ hgtags = ['?'] if hgtags: - return 'PyPy', hgtags[0], hgid + return 'RPython', hgtags[0], hgid else: # use the branch instead - p = Popen([str(hgexe), 'id', '-b', pypyroot], + p = Popen([str(hgexe), 'id', '-b', rpythonroot], stdout=PIPE, stderr=PIPE, env=env) hgbranch = p.stdout.read().strip() maywarn(p.stderr.read()) - return 'PyPy', hgbranch, hgid + return 'RPython', hgbranch, hgid def _get_hg_archive_version(path): @@ -90,9 +90,9 @@ finally: fp.close() if 'tag' in data: - return 'PyPy', data['tag'], data['node'] + return 'RPython', data['tag'], data['node'] else: - return 'PyPy', data['branch'], data['node'] + return 'RPython', data['branch'], data['node'] def _get_git_version(): @@ -105,7 +105,7 @@ try: p = Popen( [str(gitexe), 'rev-parse', 'HEAD'], - stdout=PIPE, stderr=PIPE, cwd=pypyroot + stdout=PIPE, stderr=PIPE, cwd=rpythonroot ) except OSError, e: maywarn(e, 'Git') @@ -116,16 +116,16 @@ revision_id = p.stdout.read().strip()[:12] p = Popen( [str(gitexe), 'describe', '--tags', '--exact-match'], - stdout=PIPE, stderr=PIPE, cwd=pypyroot + stdout=PIPE, stderr=PIPE, cwd=rpythonroot ) if p.wait() != 0: p = Popen( [str(gitexe), 'branch'], stdout=PIPE, stderr=PIPE, - cwd=pypyroot + cwd=rpythonroot ) if p.wait() != 0: maywarn(p.stderr.read(), 'Git') - return 'PyPy', '?', revision_id + return 'RPython', '?', revision_id branch = '?' for line in p.stdout.read().strip().split('\n'): if line.startswith('* '): @@ -133,8 +133,8 @@ if branch == '(no branch)': branch = '?' break - return 'PyPy', branch, revision_id - return 'PyPy', p.stdout.read().strip(), revision_id + return 'RPython', branch, revision_id + return 'RPython', p.stdout.read().strip(), revision_id if __name__ == '__main__': From noreply at buildbot.pypy.org Sat Jan 26 00:45:26 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sat, 26 Jan 2013 00:45:26 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fix Message-ID: <20130125234526.DEF571C0041@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60504:e98c84761a4d Date: 2013-01-26 01:44 +0200 http://bitbucket.org/pypy/pypy/changeset/e98c84761a4d/ Log: fix diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -2021,7 +2021,8 @@ base_ofs = self.cpu.get_baseofs_of_frame_field() self.mov(fail_descr_loc, RawStackLoc(ofs)) gcmap = self.gcmap_for_finish - if op.getarg(0).type == REF: + arglist = op.getarglist() + if arglist and arglist[0].type == REF: self.push_gcmap(self.mc, gcmap, store=True) else: # note that the 0 here is redundant, but I would rather diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py --- a/rpython/jit/backend/x86/runner.py +++ b/rpython/jit/backend/x86/runner.py @@ -64,7 +64,6 @@ def setup_once(self): self.profile_agent.startup() self.assembler.setup_once() - self.gc_set_extra_threshold() def finish_once(self): self.assembler.finish_once() From noreply at buildbot.pypy.org Sat Jan 26 17:23:41 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 26 Jan 2013 17:23:41 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: in-progress Message-ID: <20130126162341.AAE9D1C039A@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r60505:f823b3f06916 Date: 2013-01-26 17:21 +0100 http://bitbucket.org/pypy/pypy/changeset/f823b3f06916/ Log: in-progress diff --git a/pypy/rpython/memory/gc/test/test_stmgc.py b/pypy/rpython/memory/gc/test/test_stmgc.py --- a/pypy/rpython/memory/gc/test/test_stmgc.py +++ b/pypy/rpython/memory/gc/test/test_stmgc.py @@ -178,8 +178,7 @@ gcref = self.gc.malloc_fixedsize_clear(tid, size, contains_weakptr=weakref) realobj = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), gcref) - addr = llmemory.cast_ptr_to_adr(realobj) - return realobj, addr + return realobj def select_thread(self, threadnum): self.gc.stm_operations.threadnum = threadnum if threadnum not in self.gc.stm_operations._tls_dict: @@ -199,10 +198,18 @@ if must_have_version != '?': assert hdr.version == must_have_version - def stm_readbarrier(self, P): + def header(self, P): if lltype.typeOf(P) != llmemory.Address: P = llmemory.cast_ptr_to_adr(P) - hdr = self.gc.header(P) + return self.gc.header(P) + + def set_hdr_revision(self, hdr, P): + if lltype.typeOf(P) != llmemory.Address: + P = llmemory.cast_ptr_to_adr(P) + set_hdr_revision(hdr, P) + + def stm_readbarrier(self, P): + hdr = self.header(P) if hdr.tid & GCFLAG_GLOBAL == 0: # already a local object R = P @@ -223,9 +230,7 @@ return G def stm_writebarrier(self, P): - if lltype.typeOf(P) != llmemory.Address: - P = llmemory.cast_ptr_to_adr(P) - hdr = self.gc.header(P) + hdr = self.header(P) if hdr.tid & GCFLAG_NOT_WRITTEN == 0: # already a local, written-to object assert hdr.tid & GCFLAG_GLOBAL == 0 @@ -295,22 +300,22 @@ def test_write_barrier_exists(self): self.select_thread(1) - t, t_adr = self.malloc(S) - obj = self.stm_writebarrier(t_adr) # local object - assert obj == t_adr + t = self.malloc(S) + obj = self.stm_writebarrier(t) # local object + assert obj == t # self.select_thread(0) - s, s_adr = self.malloc(S) + s = self.malloc(S) # self.select_thread(1) - assert self.gc.header(s_adr).tid & GCFLAG_GLOBAL != 0 - assert self.gc.header(t_adr).tid & GCFLAG_GLOBAL == 0 - self.gc.header(s_adr).tid |= GCFLAG_POSSIBLY_OUTDATED - self.gc.header(t_adr).tid |= GCFLAG_LOCAL_COPY | GCFLAG_VISITED - set_hdr_revision(self.gc.header(t_adr), s_adr) + assert self.header(s).tid & GCFLAG_GLOBAL != 0 + assert self.header(t).tid & GCFLAG_GLOBAL == 0 + self.header(s).tid |= GCFLAG_POSSIBLY_OUTDATED + self.header(t).tid |= GCFLAG_LOCAL_COPY | GCFLAG_VISITED + self.set_hdr_revision(self.header(t), s) self.gc.stm_operations._tldicts[1][s_adr] = t_adr - obj = self.stm_writebarrier(s_adr) # global copied object - assert obj == t_adr + obj = self.stm_writebarrier(s) # global copied object + assert obj == t assert self.gc.stm_operations._transactional_copies == [] def test_write_barrier_new(self): @@ -450,20 +455,20 @@ from pypy.rpython.memory.gc.test import test_stmtls self.gc.root_walker = test_stmtls.FakeRootWalker() # - tr1, tr1_adr = self.malloc(SR, globl=True) # three prebuilt objects - tr2, tr2_adr = self.malloc(SR, globl=True) - tr3, tr3_adr = self.malloc(SR, globl=True) + tr1 = self.malloc(SR, globl=True) # three prebuilt objects + tr2 = self.malloc(SR, globl=True) + tr3 = self.malloc(SR, globl=True) tr1.sr2 = tr2 self.gc.root_walker.push(tr1) - sr1_adr = self.gc.stm_writebarrier(tr1_adr) - assert sr1_adr != tr1_adr - sr2_adr = self.gc.stm_writebarrier(tr2_adr) - assert sr2_adr != tr2_adr - sr3_adr = self.gc.stm_writebarrier(tr3_adr) - assert sr3_adr != tr3_adr - self.checkflags(sr1_adr, False, True) # sr1 is local - self.checkflags(sr2_adr, False, True) # sr2 is local - self.checkflags(sr3_adr, False, True) # sr3 is local + sr1 = self.stm_writebarrier(tr1) + assert sr1 != tr1 + sr2 = self.stm_writebarrier(tr2) + assert sr2 != tr2 + sr3 = self.gc.stm_writebarrier(tr3) + assert sr3 != tr3 + self.checkflags(sr1, False, True) # sr1 is local + self.checkflags(sr2, False, True) # sr2 is local + self.checkflags(sr3, False, True) # sr3 is local # self.gc.stop_transaction() self.gc.start_transaction() From noreply at buildbot.pypy.org Sat Jan 26 17:23:42 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 26 Jan 2013 17:23:42 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: Split interp_signal in three. rlib/rsignal is like on trunk. sigaction.py is the non-stm action classes. Message-ID: <20130126162342.EA0E81C1069@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r60506:ff1b216bf13d Date: 2013-01-26 17:22 +0100 http://bitbucket.org/pypy/pypy/changeset/ff1b216bf13d/ Log: Split interp_signal in three. rlib/rsignal is like on trunk. sigaction.py is the non-stm action classes. diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -351,6 +351,8 @@ assert isinstance(action, PeriodicAsyncAction) self._periodic_actions.append(action) if use_bytecode_counter: + assert not action.space.config.translation.stm, ( + "Cannot use the bytecode counter with STM") self.has_bytecode_counter = True self._rebuild_action_dispatcher() diff --git a/pypy/module/signal/__init__.py b/pypy/module/signal/__init__.py --- a/pypy/module/signal/__init__.py +++ b/pypy/module/signal/__init__.py @@ -30,9 +30,9 @@ } def buildloaders(cls): - from pypy.module.signal import interp_signal - for name in interp_signal.signal_names: - signum = getattr(interp_signal, name) + from pypy.rlib import rsignal + for name in rsignal.signal_names: + signum = getattr(rsignal, name) if signum is not None: Module.interpleveldefs[name] = 'space.wrap(%d)' % (signum,) super(Module, cls).buildloaders() @@ -40,11 +40,11 @@ def __init__(self, space, *args): "NOT_RPYTHON" - from pypy.module.signal import interp_signal + from pypy.module.signal import sigaction MixedModule.__init__(self, space, *args) # add the signal-checking callback as an action on the space - space.check_signal_action = interp_signal.CheckSignalAction(space) + space.check_signal_action = sigaction.CheckSignalAction(space) space.actionflag.register_periodic_action(space.check_signal_action, use_bytecode_counter=False) - space.actionflag.__class__ = interp_signal.SignalActionFlag + space.actionflag.__class__ = sigaction.SignalActionFlag # xxx yes I know the previous line is a hack diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py --- a/pypy/module/signal/interp_signal.py +++ b/pypy/module/signal/interp_signal.py @@ -1,228 +1,10 @@ from __future__ import with_statement from pypy.interpreter.error import OperationError, exception_from_errno -from pypy.interpreter.executioncontext import AsyncAction, AbstractActionFlag -from pypy.interpreter.executioncontext import PeriodicAsyncAction from pypy.interpreter.gateway import unwrap_spec -import signal as cpy_signal from pypy.rpython.lltypesystem import lltype, rffi -from pypy.rpython.tool import rffi_platform -from pypy.translator.tool.cbuild import ExternalCompilationInfo -import py -import sys -from pypy.tool import autopath from pypy.rlib import jit, rposix -from pypy.rlib.rarithmetic import intmask, is_valid_int - -def setup(): - for key, value in cpy_signal.__dict__.items(): - if (key.startswith('SIG') or key.startswith('CTRL_')) and \ - is_valid_int(value) and \ - key != 'SIG_DFL' and key != 'SIG_IGN': - globals()[key] = value - yield key - -NSIG = cpy_signal.NSIG -SIG_DFL = cpy_signal.SIG_DFL -SIG_IGN = cpy_signal.SIG_IGN -signal_names = list(setup()) -signal_values = {} -for key in signal_names: - signal_values[globals()[key]] = None -if sys.platform == 'win32' and not hasattr(cpy_signal,'CTRL_C_EVENT'): - # XXX Hack to revive values that went missing, - # Remove this once we are sure the host cpy module has them. - signal_values[0] = None - signal_values[1] = None - signal_names.append('CTRL_C_EVENT') - signal_names.append('CTRL_BREAK_EVENT') - CTRL_C_EVENT = 0 - CTRL_BREAK_EVENT = 1 -includes = ['stdlib.h', 'src/signals.h'] -if sys.platform != 'win32': - includes.append('sys/time.h') - -cdir = py.path.local(autopath.pypydir).join('translator', 'c') - -eci = ExternalCompilationInfo( - includes = includes, - separate_module_files = [cdir / 'src' / 'signals.c'], - include_dirs = [str(cdir)], - export_symbols = ['pypysig_poll', 'pypysig_default', - 'pypysig_ignore', 'pypysig_setflag', - 'pypysig_reinstall', - 'pypysig_set_wakeup_fd', - 'pypysig_getaddr_occurred'], -) - -class CConfig: - _compilation_info_ = eci - -if sys.platform != 'win32': - for name in """ITIMER_REAL ITIMER_VIRTUAL ITIMER_PROF""".split(): - setattr(CConfig, name, rffi_platform.DefinedConstantInteger(name)) - - CConfig.timeval = rffi_platform.Struct( - 'struct timeval', - [('tv_sec', rffi.LONG), - ('tv_usec', rffi.LONG)]) - - CConfig.itimerval = rffi_platform.Struct( - 'struct itimerval', - [('it_value', CConfig.timeval), - ('it_interval', CConfig.timeval)]) - -for k, v in rffi_platform.configure(CConfig).items(): - globals()[k] = v - -def external(name, args, result, **kwds): - return rffi.llexternal(name, args, result, compilation_info=eci, - sandboxsafe=True, **kwds) - -pypysig_ignore = external('pypysig_ignore', [rffi.INT], lltype.Void) -pypysig_default = external('pypysig_default', [rffi.INT], lltype.Void) -pypysig_setflag = external('pypysig_setflag', [rffi.INT], lltype.Void) -pypysig_reinstall = external('pypysig_reinstall', [rffi.INT], lltype.Void) -pypysig_set_wakeup_fd = external('pypysig_set_wakeup_fd', [rffi.INT], rffi.INT) -pypysig_poll = external('pypysig_poll', [], rffi.INT, threadsafe=False) -# don't bother releasing the GIL around a call to pypysig_poll: it's -# pointless and a performance issue - -# don't use rffi.LONGP because the JIT doesn't support raw arrays so far -struct_name = 'pypysig_long_struct' -LONG_STRUCT = lltype.Struct(struct_name, ('c_value', lltype.Signed), - hints={'c_name' : struct_name, 'external' : 'C'}) -del struct_name - -pypysig_getaddr_occurred = external('pypysig_getaddr_occurred', [], - lltype.Ptr(LONG_STRUCT), _nowrapper=True, - elidable_function=True) -c_alarm = external('alarm', [rffi.INT], rffi.INT) -c_pause = external('pause', [], rffi.INT, threadsafe=True) -c_siginterrupt = external('siginterrupt', [rffi.INT, rffi.INT], rffi.INT) - -if sys.platform != 'win32': - itimervalP = rffi.CArrayPtr(itimerval) - c_setitimer = external('setitimer', - [rffi.INT, itimervalP, itimervalP], rffi.INT) - c_getitimer = external('getitimer', [rffi.INT, itimervalP], rffi.INT) - - -class SignalActionFlag(AbstractActionFlag): - # This class uses the C-level pypysig_counter variable as the tick - # counter. The C-level signal handler will reset it to -1 whenever - # a signal is received. - - def get_ticker(self): - p = pypysig_getaddr_occurred() - return p.c_value - - def reset_ticker(self, value): - p = pypysig_getaddr_occurred() - p.c_value = value - - def decrement_ticker(self, by): - p = pypysig_getaddr_occurred() - value = p.c_value - if self.has_bytecode_counter: # this 'if' is constant-folded - if jit.isconstant(by) and by == 0: - pass # normally constant-folded too - else: - value -= by - p.c_value = value - return value - - -class CheckSignalAction(PeriodicAsyncAction): - """An action that is automatically invoked when a signal is received.""" - - def __init__(self, space): - AsyncAction.__init__(self, space) - self.handlers_w = {} - if space.config.objspace.usemodules.thread: - # need a helper action in case signals arrive in a non-main thread - self.pending_signals = {} - self.reissue_signal_action = ReissueSignalAction(space) - else: - self.reissue_signal_action = None - - @jit.dont_look_inside - def perform(self, executioncontext, frame): - while True: - n = pypysig_poll() - if n < 0: - break - self.perform_signal(executioncontext, n) - - @jit.dont_look_inside - def perform_signal(self, executioncontext, n): - if self.reissue_signal_action is None: - # no threads: we can report the signal immediately - self.report_signal(n) - else: - main_ec = self.space.threadlocals.getmainthreadvalue() - if executioncontext is main_ec: - # running in the main thread: we can report the - # signal immediately - self.report_signal(n) - else: - # running in another thread: we need to hack a bit - self.pending_signals[n] = None - self.reissue_signal_action.fire_after_thread_switch() - - @jit.dont_look_inside - def set_interrupt(self): - "Simulates the effect of a SIGINT signal arriving" - ec = self.space.getexecutioncontext() - self.perform_signal(ec, cpy_signal.SIGINT) - - @jit.dont_look_inside - def report_signal(self, n): - try: - w_handler = self.handlers_w[n] - except KeyError: - return # no handler, ignore signal - space = self.space - if not space.is_true(space.callable(w_handler)): - return # w_handler is SIG_IGN or SIG_DFL? - # re-install signal handler, for OSes that clear it - pypysig_reinstall(n) - # invoke the app-level handler - ec = space.getexecutioncontext() - w_frame = space.wrap(ec.gettopframe_nohidden()) - space.call_function(w_handler, space.wrap(n), w_frame) - - @jit.dont_look_inside - def report_pending_signals(self): - # XXX this logic isn't so complicated but I have no clue how - # to test it :-( - pending_signals = self.pending_signals.keys() - self.pending_signals.clear() - try: - while pending_signals: - self.report_signal(pending_signals.pop()) - finally: - # in case of exception, put the undelivered signals back - # into the dict instead of silently swallowing them - if pending_signals: - for n in pending_signals: - self.pending_signals[n] = None - self.reissue_signal_action.fire() - - -class ReissueSignalAction(AsyncAction): - """A special action to help deliver signals to the main thread. If - a non-main thread caught a signal, this action fires after every - thread switch until we land in the main thread. - """ - - def perform(self, executioncontext, frame): - main_ec = self.space.threadlocals.getmainthreadvalue() - if executioncontext is main_ec: - # now running in the main thread: we can really report the signals - self.space.check_signal_action.report_pending_signals() - else: - # still running in some other thread: try again later - self.fire_after_thread_switch() +from pypy.rlib.rarithmetic import intmask +from pypy.rlib.rsignal import * @unwrap_spec(signum=int) diff --git a/pypy/module/signal/sigaction.py b/pypy/module/signal/sigaction.py new file mode 100644 --- /dev/null +++ b/pypy/module/signal/sigaction.py @@ -0,0 +1,123 @@ +from pypy.interpreter.executioncontext import AsyncAction, AbstractActionFlag +from pypy.interpreter.executioncontext import PeriodicAsyncAction +from pypy.rlib import jit +from pypy.rlib.rsignal import pypysig_getaddr_occurred +from pypy.rlib.rsignal import pypysig_poll, pypysig_reinstall + + +class SignalActionFlag(AbstractActionFlag): + # This class uses the C-level pypysig_counter variable as the tick + # counter. The C-level signal handler will reset it to -1 whenever + # a signal is received. + + def get_ticker(self): + p = pypysig_getaddr_occurred() + return p.c_value + + def reset_ticker(self, value): + p = pypysig_getaddr_occurred() + p.c_value = value + + def decrement_ticker(self, by): + p = pypysig_getaddr_occurred() + value = p.c_value + if self.has_bytecode_counter: # this 'if' is constant-folded + if jit.isconstant(by) and by == 0: + pass # normally constant-folded too + else: + value -= by + p.c_value = value + return value + + +class CheckSignalAction(PeriodicAsyncAction): + """An action that is automatically invoked when a signal is received.""" + + def __init__(self, space): + AsyncAction.__init__(self, space) + self.handlers_w = {} + if space.config.objspace.usemodules.thread: + # need a helper action in case signals arrive in a non-main thread + self.pending_signals = {} + self.reissue_signal_action = ReissueSignalAction(space) + else: + self.reissue_signal_action = None + + @jit.dont_look_inside + def perform(self, executioncontext, frame): + while True: + n = pypysig_poll() + if n < 0: + break + self.perform_signal(executioncontext, n) + + @jit.dont_look_inside + def perform_signal(self, executioncontext, n): + if self.reissue_signal_action is None: + # no threads: we can report the signal immediately + self.report_signal(n) + else: + main_ec = self.space.threadlocals.getmainthreadvalue() + if executioncontext is main_ec: + # running in the main thread: we can report the + # signal immediately + self.report_signal(n) + else: + # running in another thread: we need to hack a bit + self.pending_signals[n] = None + self.reissue_signal_action.fire_after_thread_switch() + + @jit.dont_look_inside + def set_interrupt(self): + "Simulates the effect of a SIGINT signal arriving" + ec = self.space.getexecutioncontext() + self.perform_signal(ec, cpy_signal.SIGINT) + + @jit.dont_look_inside + def report_signal(self, n): + try: + w_handler = self.handlers_w[n] + except KeyError: + return # no handler, ignore signal + space = self.space + if not space.is_true(space.callable(w_handler)): + return # w_handler is SIG_IGN or SIG_DFL? + # re-install signal handler, for OSes that clear it + pypysig_reinstall(n) + # invoke the app-level handler + ec = space.getexecutioncontext() + w_frame = space.wrap(ec.gettopframe_nohidden()) + space.call_function(w_handler, space.wrap(n), w_frame) + + @jit.dont_look_inside + def report_pending_signals(self): + # XXX this logic isn't so complicated but I have no clue how + # to test it :-( + pending_signals = self.pending_signals.keys() + self.pending_signals.clear() + try: + while pending_signals: + self.report_signal(pending_signals.pop()) + finally: + # in case of exception, put the undelivered signals back + # into the dict instead of silently swallowing them + if pending_signals: + for n in pending_signals: + self.pending_signals[n] = None + self.reissue_signal_action.fire() + + +class ReissueSignalAction(AsyncAction): + """A special action to help deliver signals to the main thread. If + a non-main thread caught a signal, this action fires after every + thread switch until we land in the main thread. + """ + + def perform(self, executioncontext, frame): + main_ec = self.space.threadlocals.getmainthreadvalue() + if executioncontext is main_ec: + # now running in the main thread: we can really report the signals + self.space.check_signal_action.report_pending_signals() + else: + # still running in some other thread: try again later + self.fire_after_thread_switch() diff --git a/pypy/rlib/rsignal.py b/pypy/rlib/rsignal.py new file mode 100644 --- /dev/null +++ b/pypy/rlib/rsignal.py @@ -0,0 +1,101 @@ +import signal as cpy_signal +import sys +import py +from pypy.tool import autopath +from pypy.rpython.tool import rffi_platform +from pypy.rpython.lltypesystem import lltype, rffi +from pypy.translator.tool.cbuild import ExternalCompilationInfo +from pypy.rlib.rarithmetic import is_valid_int + +def setup(): + for key, value in cpy_signal.__dict__.items(): + if (key.startswith('SIG') or key.startswith('CTRL_')) and \ + is_valid_int(value) and \ + key != 'SIG_DFL' and key != 'SIG_IGN': + globals()[key] = value + yield key + +NSIG = cpy_signal.NSIG +SIG_DFL = cpy_signal.SIG_DFL +SIG_IGN = cpy_signal.SIG_IGN +signal_names = list(setup()) +signal_values = {} +for key in signal_names: + signal_values[globals()[key]] = None +if sys.platform == 'win32' and not hasattr(cpy_signal,'CTRL_C_EVENT'): + # XXX Hack to revive values that went missing, + # Remove this once we are sure the host cpy module has them. + signal_values[0] = None + signal_values[1] = None + signal_names.append('CTRL_C_EVENT') + signal_names.append('CTRL_BREAK_EVENT') + CTRL_C_EVENT = 0 + CTRL_BREAK_EVENT = 1 +includes = ['stdlib.h', 'src/signals.h'] +if sys.platform != 'win32': + includes.append('sys/time.h') + +cdir = py.path.local(autopath.pypydir).join('translator', 'c') + +eci = ExternalCompilationInfo( + includes = includes, + separate_module_files = [cdir / 'src' / 'signals.c'], + include_dirs = [str(cdir)], + export_symbols = ['pypysig_poll', 'pypysig_default', + 'pypysig_ignore', 'pypysig_setflag', + 'pypysig_reinstall', + 'pypysig_set_wakeup_fd', + 'pypysig_getaddr_occurred'], +) + +class CConfig: + _compilation_info_ = eci + +if sys.platform != 'win32': + for name in """ITIMER_REAL ITIMER_VIRTUAL ITIMER_PROF""".split(): + setattr(CConfig, name, rffi_platform.DefinedConstantInteger(name)) + + CConfig.timeval = rffi_platform.Struct( + 'struct timeval', + [('tv_sec', rffi.LONG), + ('tv_usec', rffi.LONG)]) + + CConfig.itimerval = rffi_platform.Struct( + 'struct itimerval', + [('it_value', CConfig.timeval), + ('it_interval', CConfig.timeval)]) + +for k, v in rffi_platform.configure(CConfig).items(): + globals()[k] = v + +def external(name, args, result, **kwds): + return rffi.llexternal(name, args, result, compilation_info=eci, + sandboxsafe=True, **kwds) + +pypysig_ignore = external('pypysig_ignore', [rffi.INT], lltype.Void) +pypysig_default = external('pypysig_default', [rffi.INT], lltype.Void) +pypysig_setflag = external('pypysig_setflag', [rffi.INT], lltype.Void) +pypysig_reinstall = external('pypysig_reinstall', [rffi.INT], lltype.Void) +pypysig_set_wakeup_fd = external('pypysig_set_wakeup_fd', [rffi.INT], rffi.INT) +pypysig_poll = external('pypysig_poll', [], rffi.INT, threadsafe=False) +# don't bother releasing the GIL around a call to pypysig_poll: it's +# pointless and a performance issue + +# don't use rffi.LONGP because the JIT doesn't support raw arrays so far +struct_name = 'pypysig_long_struct' +LONG_STRUCT = lltype.Struct(struct_name, ('c_value', lltype.Signed), + hints={'c_name' : struct_name, 'external' : 'C'}) +del struct_name + +pypysig_getaddr_occurred = external('pypysig_getaddr_occurred', [], + lltype.Ptr(LONG_STRUCT), _nowrapper=True, + elidable_function=True) +c_alarm = external('alarm', [rffi.INT], rffi.INT) +c_pause = external('pause', [], rffi.INT, threadsafe=True) +c_siginterrupt = external('siginterrupt', [rffi.INT, rffi.INT], rffi.INT) + +if sys.platform != 'win32': + itimervalP = rffi.CArrayPtr(itimerval) + c_setitimer = external('setitimer', + [rffi.INT, itimervalP, itimervalP], rffi.INT) + c_getitimer = external('getitimer', [rffi.INT, itimervalP], rffi.INT) From noreply at buildbot.pypy.org Sun Jan 27 00:26:30 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Sun, 27 Jan 2013 00:26:30 +0100 (CET) Subject: [pypy-commit] pypy py3k: merge default Message-ID: <20130126232630.1B2031C1055@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60507:331eda1c7211 Date: 2013-01-26 15:25 -0800 http://bitbucket.org/pypy/pypy/changeset/331eda1c7211/ Log: merge default diff --git a/py/_code/source.py b/py/_code/source.py --- a/py/_code/source.py +++ b/py/_code/source.py @@ -141,6 +141,8 @@ trysource = self[start:end] if trysource.isparseable(): return start, end + if end == start + 100: # XXX otherwise, it takes forever + break # XXX if end is None: raise IndexError("no valid source range around line %d " % (lineno,)) return start, end diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -43,3 +43,7 @@ .. branch: fix-lookinside-iff-oopspec Fixed the interaction between two internal tools for controlling the JIT. + +.. branch: inline-virtualref-2 +Better optimized certain types of frame accesses in the JIT, particularly +around exceptions that escape the function they were raised in. diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -29,7 +29,7 @@ import pypy pypydir = os.path.dirname(os.path.abspath(pypy.__file__)) del pypy -from pypy.tool.version import get_repo_version_info +from rpython.tool.version import get_repo_version_info import time as t gmtime = t.gmtime() diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py --- a/pypy/module/sys/vm.py +++ b/pypy/module/sys/vm.py @@ -95,7 +95,7 @@ def exc_info_with_tb(space): operror = space.getexecutioncontext().sys_exc_info() if operror is None: - return space.newtuple([space.w_None,space.w_None,space.w_None]) + return space.newtuple([space.w_None, space.w_None, space.w_None]) else: return space.newtuple([operror.w_type, operror.get_w_value(space), space.wrap(operror.get_traceback())]) diff --git a/pypy/tool/jitlogparser/parser.py b/pypy/tool/jitlogparser/parser.py --- a/pypy/tool/jitlogparser/parser.py +++ b/pypy/tool/jitlogparser/parser.py @@ -215,6 +215,7 @@ _lineset = None is_bytecode = False inline_level = None + bytecode_name = None # factory method TraceForOpcode = TraceForOpcode @@ -244,23 +245,29 @@ return ",".join([str(len(v)) for v in stack]) def append_to_res(bc): - if not stack: - stack.append([]) - else: - if bc.inline_level is not None and bc.inline_level + 1 != len(stack): - if bc.inline_level < len(stack): + if bc.inline_level is not None: + if bc.inline_level == len(stack) - 1: + pass + elif bc.inline_level > len(stack) - 1: + stack.append([]) + else: + while bc.inline_level + 1 < len(stack): last = stack.pop() stack[-1].append(cls(last, getpath(stack), storage)) - else: - stack.append([]) stack[-1].append(bc) so_far = [] stack = [] + nothing_yet = True for op in operations: if op.name == 'debug_merge_point': if so_far: - append_to_res(cls.TraceForOpcode(so_far, storage, loopname)) + opc = cls.TraceForOpcode(so_far, storage, loopname) + if nothing_yet: + nothing_yet = False + for i in xrange(opc.inline_level + 1): + stack.append([]) + append_to_res(opc) if limit: break so_far = [] diff --git a/pypy/tool/jitlogparser/test/test_parser.py b/pypy/tool/jitlogparser/test/test_parser.py --- a/pypy/tool/jitlogparser/test/test_parser.py +++ b/pypy/tool/jitlogparser/test/test_parser.py @@ -354,3 +354,25 @@ f = Function.from_operations(loop.operations, LoopStorage()) assert f.chunks[-1].filename == 'x.py' assert f.filename is None + +def test_parse_2_levels_up(): + loop = parse(""" + [] + debug_merge_point(0, 0, 'one') + debug_merge_point(1, 0, 'two') + debug_merge_point(2, 0, 'three') + debug_merge_point(0, 0, 'one') + """) + f = Function.from_operations(loop.operations, LoopStorage()) + assert len(f.chunks) == 3 + +def test_parse_from_inside(): + loop = parse(""" + [] + debug_merge_point(1, 0, 'two') + debug_merge_point(2, 0, 'three') + debug_merge_point(0, 0, 'one') + """) + f = Function.from_operations(loop.operations, LoopStorage()) + assert len(f.chunks) == 2 + diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -13,7 +13,7 @@ import sys import os #Add toplevel repository dir to sys.path -sys.path.insert(0,os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))) +sys.path.insert(0,os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))) import py import fnmatch from rpython.tool.udir import udir diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py --- a/rpython/jit/backend/x86/runner.py +++ b/rpython/jit/backend/x86/runner.py @@ -64,6 +64,7 @@ def setup_once(self): self.profile_agent.startup() self.assembler.setup_once() + self.gc_set_extra_threshold() def finish_once(self): self.assembler.finish_once() diff --git a/rpython/rlib/rrandom.py b/rpython/rlib/rrandom.py --- a/rpython/rlib/rrandom.py +++ b/rpython/rlib/rrandom.py @@ -100,9 +100,20 @@ def jumpahead(self, n): mt = self.state - for i in range(N - 1, 0, -1): + for i in range(N - 1, 1, -1): j = n % i mt[i], mt[j] = mt[j], mt[i] - for i in range(N): + nonzero = False + for i in range(1, N): mt[i] += r_uint(i + 1) + mt[i] &= r_uint(0xffffffff) + nonzero |= bool(mt[i]) + # Ensure the state is nonzero: in the unlikely event that mt[1] through + # mt[N-1] are all zero, set the MSB of mt[0] (see issue #14591). In the + # normal case, we fall back to the pre-issue 14591 behaviour for mt[0]. + if nonzero: + mt[0] += r_uint(1) + mt[0] &= r_uint(0xffffffff) + else: + mt[0] = r_uint(0x80000000) self.index = N diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py --- a/rpython/rlib/rsocket.py +++ b/rpython/rlib/rsocket.py @@ -1059,6 +1059,13 @@ def dup(fd): return _c.dup(fd) + def fromfd(fd, family, type, proto=0, SocketClass=RSocket): + # Dup the fd so it and the socket can be closed independently + fd = _c.dup(fd) + if fd < 0: + raise last_error() + return make_socket(fd, family, type, proto, SocketClass) + def getdefaulttimeout(): return defaults.timeout diff --git a/rpython/rlib/test/test_jit.py b/rpython/rlib/test/test_jit.py --- a/rpython/rlib/test/test_jit.py +++ b/rpython/rlib/test/test_jit.py @@ -21,8 +21,6 @@ assert driver.reds == [] assert driver.numreds is None py.test.raises(TypeError, "driver.can_enter_jit(foo='something')") - # - py.test.raises(AssertionError, "JitDriver(greens=['foo'], reds='auto', get_printable_location='something')") py.test.raises(AssertionError, "JitDriver(greens=['foo'], reds='auto', confirm_enter_jit='something')") def test_jitdriver_numreds(): diff --git a/rpython/rlib/test/test_rrandom.py b/rpython/rlib/test/test_rrandom.py --- a/rpython/rlib/test/test_rrandom.py +++ b/rpython/rlib/test/test_rrandom.py @@ -46,6 +46,15 @@ assert tuple(rnd.state) + (rnd.index, ) == cpyrandom.getstate() +def test_jumpahead_badstate(): + rnd = Random() + s, j = 4043161618, 2674112291824205302 + rnd.init_by_array([s]) + rnd.jumpahead(j) + for i in range(500): + r = rnd.random() + assert r <= 1.0, (r, i) + def test_translate(): def f(x, y): x = r_uint(x) diff --git a/rpython/rlib/test/test_rsocket.py b/rpython/rlib/test/test_rsocket.py --- a/rpython/rlib/test/test_rsocket.py +++ b/rpython/rlib/test/test_rsocket.py @@ -373,6 +373,15 @@ assert s.fd != s2.fd assert s.getsockname().eq(s2.getsockname()) +def test_c_dup(): + # rsocket.dup() duplicates fd, it also works on Windows + # (but only on socket handles!) + s = RSocket(AF_INET, SOCK_STREAM) + s.setsockopt_int(SOL_SOCKET, SO_REUSEADDR, 1) + s.bind(INETAddress('localhost', 50007)) + fd2 = dup(s.fd) + assert s.fd != fd2 + def test_inet_aton(): assert inet_aton('1.2.3.4') == '\x01\x02\x03\x04' assert inet_aton('127.0.0.1') == '\x7f\x00\x00\x01' diff --git a/rpython/rtyper/rlist.py b/rpython/rtyper/rlist.py --- a/rpython/rtyper/rlist.py +++ b/rpython/rtyper/rlist.py @@ -1025,11 +1025,7 @@ res._ll_resize(resultlen) j = length while j < resultlen: - i = 0 - while i < length: - p = j + i - res.ll_setitem_fast(p, l.ll_getitem_fast(i)) - i += 1 + ll_arraycopy(l, res, 0, j, length) j += length return res ll_inplace_mul.oopspec = 'list.inplace_mul(l, factor)' @@ -1046,11 +1042,7 @@ res = RESLIST.ll_newlist(resultlen) j = 0 while j < resultlen: - i = 0 - while i < length: - p = j + i - res.ll_setitem_fast(p, l.ll_getitem_fast(i)) - i += 1 + ll_arraycopy(l, res, 0, j, length) j += length return res # not inlined by the JIT -- contains a loop diff --git a/pypy/tool/test/test_version.py b/rpython/tool/test/test_version.py rename from pypy/tool/test/test_version.py rename to rpython/tool/test/test_version.py --- a/pypy/tool/test/test_version.py +++ b/rpython/tool/test/test_version.py @@ -1,6 +1,6 @@ import os, sys import py -from pypy.tool.version import get_repo_version_info, _get_hg_archive_version +from rpython.tool.version import get_repo_version_info, _get_hg_archive_version def test_hg_archival_version(tmpdir): def version_for(name, **kw): @@ -11,14 +11,14 @@ assert version_for('release', tag='release-123', node='000', - ) == ('PyPy', 'release-123', '000') + ) == ('RPython', 'release-123', '000') assert version_for('somebranch', node='000', branch='something', - ) == ('PyPy', 'something', '000') + ) == ('RPython', 'something', '000') def test_get_repo_version_info(): assert get_repo_version_info(None) - assert get_repo_version_info(os.devnull) == ('PyPy', '?', '?') - assert get_repo_version_info(sys.executable) == ('PyPy', '?', '?') + assert get_repo_version_info(os.devnull) == ('RPython', '?', '?') + assert get_repo_version_info(sys.executable) == ('RPython', '?', '?') diff --git a/rpython/tool/udir.py b/rpython/tool/udir.py --- a/rpython/tool/udir.py +++ b/rpython/tool/udir.py @@ -20,6 +20,7 @@ import os, sys import py +from rpython.tool.version import get_repo_version_info from py.path import local PYPY_KEEP = int(os.environ.get('PYPY_USESSION_KEEP', '3')) @@ -28,7 +29,17 @@ if dir is not None: dir = local(dir) if basename is None: - basename = '' + info = get_repo_version_info() + if info: + project, hgtag, hgid = info + basename = hgtag + if basename == '?': + basename = 'unknown' # directories with ? are not fun + # especially on windows + if isinstance(basename, unicode): + basename = basename.encode(sys.getdefaultencoding()) + else: + basename = '' basename = basename.replace('/', '--') if not basename.startswith('-'): basename = '-' + basename diff --git a/pypy/tool/version.py b/rpython/tool/version.py rename from pypy/tool/version.py rename to rpython/tool/version.py --- a/pypy/tool/version.py +++ b/rpython/tool/version.py @@ -1,10 +1,10 @@ import py import os from subprocess import Popen, PIPE -import pypy -pypydir = os.path.dirname(os.path.abspath(pypy.__file__)) -pypyroot = os.path.dirname(pypydir) -default_retval = 'PyPy', '?', '?' +import rpython +rpythondir = os.path.dirname(os.path.abspath(rpython.__file__)) +rpythonroot = os.path.dirname(rpythondir) +default_retval = 'RPython', '?', '?' def maywarn(err, repo_type='Mercurial'): if not err: @@ -20,16 +20,16 @@ # Try to see if we can get info from Git if hgexe is not specified. if not hgexe: - if os.path.isdir(os.path.join(pypyroot, '.git')): + if os.path.isdir(os.path.join(rpythonroot, '.git')): return _get_git_version() # Fallback to trying Mercurial. if hgexe is None: hgexe = py.path.local.sysfind('hg') - if os.path.isfile(os.path.join(pypyroot, '.hg_archival.txt')): - return _get_hg_archive_version(os.path.join(pypyroot, '.hg_archival.txt')) - elif not os.path.isdir(os.path.join(pypyroot, '.hg')): + if os.path.isfile(os.path.join(rpythonroot, '.hg_archival.txt')): + return _get_hg_archive_version(os.path.join(rpythonroot, '.hg_archival.txt')) + elif not os.path.isdir(os.path.join(rpythonroot, '.hg')): maywarn('Not running from a Mercurial repository!') return default_retval elif not hgexe: @@ -57,14 +57,14 @@ maywarn('command does not identify itself as Mercurial') return default_retval - p = Popen([str(hgexe), 'id', '-i', pypyroot], + p = Popen([str(hgexe), 'id', '-i', rpythonroot], stdout=PIPE, stderr=PIPE, env=env) hgid = p.stdout.read().strip() maywarn(p.stderr.read()) if p.wait() != 0: hgid = '?' - p = Popen([str(hgexe), 'id', '-t', pypyroot], + p = Popen([str(hgexe), 'id', '-t', rpythonroot], stdout=PIPE, stderr=PIPE, env=env) hgtags = [t for t in p.stdout.read().strip().split() if t != 'tip'] maywarn(p.stderr.read()) @@ -72,15 +72,15 @@ hgtags = ['?'] if hgtags: - return 'PyPy', hgtags[0], hgid + return 'RPython', hgtags[0], hgid else: # use the branch instead - p = Popen([str(hgexe), 'id', '-b', pypyroot], + p = Popen([str(hgexe), 'id', '-b', rpythonroot], stdout=PIPE, stderr=PIPE, env=env) hgbranch = p.stdout.read().strip() maywarn(p.stderr.read()) - return 'PyPy', hgbranch, hgid + return 'RPython', hgbranch, hgid def _get_hg_archive_version(path): @@ -90,9 +90,9 @@ finally: fp.close() if 'tag' in data: - return 'PyPy', data['tag'], data['node'] + return 'RPython', data['tag'], data['node'] else: - return 'PyPy', data['branch'], data['node'] + return 'RPython', data['branch'], data['node'] def _get_git_version(): @@ -105,7 +105,7 @@ try: p = Popen( [str(gitexe), 'rev-parse', 'HEAD'], - stdout=PIPE, stderr=PIPE, cwd=pypyroot + stdout=PIPE, stderr=PIPE, cwd=rpythonroot ) except OSError, e: maywarn(e, 'Git') @@ -116,16 +116,16 @@ revision_id = p.stdout.read().strip()[:12] p = Popen( [str(gitexe), 'describe', '--tags', '--exact-match'], - stdout=PIPE, stderr=PIPE, cwd=pypyroot + stdout=PIPE, stderr=PIPE, cwd=rpythonroot ) if p.wait() != 0: p = Popen( [str(gitexe), 'branch'], stdout=PIPE, stderr=PIPE, - cwd=pypyroot + cwd=rpythonroot ) if p.wait() != 0: maywarn(p.stderr.read(), 'Git') - return 'PyPy', '?', revision_id + return 'RPython', '?', revision_id branch = '?' for line in p.stdout.read().strip().split('\n'): if line.startswith('* '): @@ -133,8 +133,8 @@ if branch == '(no branch)': branch = '?' break - return 'PyPy', branch, revision_id - return 'PyPy', p.stdout.read().strip(), revision_id + return 'RPython', branch, revision_id + return 'RPython', p.stdout.read().strip(), revision_id if __name__ == '__main__': diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py --- a/rpython/translator/platform/posix.py +++ b/rpython/translator/platform/posix.py @@ -115,7 +115,7 @@ cflags = self.cflags + self.standalone_only m = GnuMakefile(path) - m.exe_name = exe_name + m.exe_name = path.join(target_name) m.eci = eci def rpyrel(fpath): diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -261,8 +261,15 @@ else: exe_name = exe_name.new(ext=self.exe_ext) + if shared: + so_name = exe_name.new(purebasename='lib' + exe_name.purebasename, + ext=self.so_ext) + target_name = so_name.basename + else: + target_name = exe_name.basename + m = NMakefile(path) - m.exe_name = exe_name + m.exe_name = path.join(target_name) m.eci = eci linkflags = list(self.link_flags) @@ -274,13 +281,6 @@ # This is required for the JIT. linkflags.append('/opt:noicf') - if shared: - so_name = exe_name.new(purebasename='lib' + exe_name.purebasename, - ext=self.so_ext) - target_name = so_name.basename - else: - target_name = exe_name.basename - def rpyrel(fpath): rel = py.path.local(fpath).relto(rpypath) if rel: From noreply at buildbot.pypy.org Sun Jan 27 08:50:32 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Sun, 27 Jan 2013 08:50:32 +0100 (CET) Subject: [pypy-commit] pypy default: fix unnecessary skip in test_signal on unix due to missing usemodule Message-ID: <20130127075032.AD2341C12E1@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60508:35e7bb0e4580 Date: 2013-01-25 19:31 -0500 http://bitbucket.org/pypy/pypy/changeset/35e7bb0e4580/ Log: fix unnecessary skip in test_signal on unix due to missing usemodule diff --git a/pypy/module/signal/test/test_signal.py b/pypy/module/signal/test/test_signal.py --- a/pypy/module/signal/test/test_signal.py +++ b/pypy/module/signal/test/test_signal.py @@ -36,7 +36,7 @@ class AppTestSignal: spaceconfig = { - "usemodules": ['signal', 'rctime'], + "usemodules": ['signal', 'rctime'] + (['fcntl'] if os.name != 'nt' else []), } def setup_class(cls): From noreply at buildbot.pypy.org Sun Jan 27 08:50:33 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Sun, 27 Jan 2013 08:50:33 +0100 (CET) Subject: [pypy-commit] pypy default: demonstrate more consistently the intermittent failure in untranslated test_interrupt_main Message-ID: <20130127075033.D02FD1C12E1@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60509:47b3810089ca Date: 2013-01-26 14:50 -0500 http://bitbucket.org/pypy/pypy/changeset/47b3810089ca/ Log: demonstrate more consistently the intermittent failure in untranslated test_interrupt_main diff --git a/pypy/module/thread/test/test_thread.py b/pypy/module/thread/test/test_thread.py --- a/pypy/module/thread/test/test_thread.py +++ b/pypy/module/thread/test/test_thread.py @@ -231,5 +231,6 @@ # This is normally called by app_main.py signal.signal(signal.SIGINT, signal.default_int_handler) - thread.start_new_thread(f, ()) - raises(KeyboardInterrupt, busy_wait) + for i in range(100): + thread.start_new_thread(f, ()) + raises(KeyboardInterrupt, busy_wait) From noreply at buildbot.pypy.org Sun Jan 27 08:50:34 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Sun, 27 Jan 2013 08:50:34 +0100 (CET) Subject: [pypy-commit] pypy default: fix untranslated test_interrupt_main by making sure we don't interrupt too early Message-ID: <20130127075034.EE8021C12E1@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60510:943dfd91fc9d Date: 2013-01-26 14:55 -0500 http://bitbucket.org/pypy/pypy/changeset/943dfd91fc9d/ Log: fix untranslated test_interrupt_main by making sure we don't interrupt too early diff --git a/pypy/module/thread/test/test_thread.py b/pypy/module/thread/test/test_thread.py --- a/pypy/module/thread/test/test_thread.py +++ b/pypy/module/thread/test/test_thread.py @@ -220,17 +220,24 @@ import signal def f(): - time.sleep(0.5) - thread.interrupt_main() + for x in range(50): + if waiting: + thread.interrupt_main() + return + print 'tock...', x # <-force the GIL to be released, as + time.sleep(0.01) # time.sleep doesn't do non-translated def busy_wait(): - for x in range(1000): + waiting.append(None) + for x in range(100): print 'tick...', x # <-force the GIL to be released, as time.sleep(0.01) # time.sleep doesn't do non-translated + waiting.pop() # This is normally called by app_main.py signal.signal(signal.SIGINT, signal.default_int_handler) for i in range(100): + waiting = [] thread.start_new_thread(f, ()) raises(KeyboardInterrupt, busy_wait) From noreply at buildbot.pypy.org Sun Jan 27 08:50:36 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Sun, 27 Jan 2013 08:50:36 +0100 (CET) Subject: [pypy-commit] pypy default: fix pypy_find_stdlib(): app_main.py was moved up a level in split-rpython Message-ID: <20130127075036.110631C12E1@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60511:1fd8960987fb Date: 2013-01-26 18:50 -0500 http://bitbucket.org/pypy/pypy/changeset/1fd8960987fb/ Log: fix pypy_find_stdlib(): app_main.py was moved up a level in split- rpython diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -713,7 +713,7 @@ def pypy_find_stdlib(s): from os.path import abspath, join, dirname as dn thisfile = abspath(__file__) - root = dn(dn(dn(dn(thisfile)))) + root = dn(dn(dn(thisfile))) return [join(root, 'lib-python', '2.7'), join(root, 'lib_pypy')] From noreply at buildbot.pypy.org Sun Jan 27 08:50:37 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Sun, 27 Jan 2013 08:50:37 +0100 (CET) Subject: [pypy-commit] pypy default: fix test_package for changes made in split-rpython Message-ID: <20130127075037.2A97A1C12E1@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60512:6dfc65223172 Date: 2013-01-26 19:46 -0500 http://bitbucket.org/pypy/pypy/changeset/6dfc65223172/ Log: fix test_package for changes made in split-rpython diff --git a/pypy/tool/release/test/test_package.py b/pypy/tool/release/test/test_package.py --- a/pypy/tool/release/test/test_package.py +++ b/pypy/tool/release/test/test_package.py @@ -15,7 +15,7 @@ basename = 'pypy-c' rename_pypy_c = 'pypy' exe_name_in_archive = 'bin/pypy' - pypy_c = py.path.local(pypydir).join('..', basename) + pypy_c = py.path.local(pypydir).join('goal', basename) if not pypy_c.check(): os.system("echo faked_pypy_c> %s" % (pypy_c,)) pypy_c.chmod(0755) From noreply at buildbot.pypy.org Sun Jan 27 08:50:38 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Sun, 27 Jan 2013 08:50:38 +0100 (CET) Subject: [pypy-commit] pypy default: fix another path in test_app_main.py for split-rpython changes Message-ID: <20130127075038.4BC771C12E1@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60513:96eb2daca099 Date: 2013-01-26 20:12 -0500 http://bitbucket.org/pypy/pypy/changeset/96eb2daca099/ Log: fix another path in test_app_main.py for split-rpython changes diff --git a/pypy/interpreter/test2/test_app_main.py b/pypy/interpreter/test2/test_app_main.py --- a/pypy/interpreter/test2/test_app_main.py +++ b/pypy/interpreter/test2/test_app_main.py @@ -547,7 +547,7 @@ assert line.rstrip() == 'Not at all. They could be carried.' print 'A five ounce bird could not carry a one pound coconut.' """) - py_py = os.path.join(pypydir, 'bin', 'py.py') + py_py = os.path.join(pypydir, 'bin', 'pyinteractive.py') child = self._spawn(sys.executable, [py_py, '-S', path]) child.expect('Are you suggesting coconuts migrate?', timeout=120) child.sendline('Not at all. They could be carried.') From noreply at buildbot.pypy.org Sun Jan 27 08:50:39 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Sun, 27 Jan 2013 08:50:39 +0100 (CET) Subject: [pypy-commit] pypy default: fix more paths for split-python changes Message-ID: <20130127075039.77CCE1C12E1@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60514:aebc29980baf Date: 2013-01-26 20:39 -0500 http://bitbucket.org/pypy/pypy/changeset/aebc29980baf/ Log: fix more paths for split-python changes diff --git a/pypy/interpreter/test2/test_app_main.py b/pypy/interpreter/test2/test_app_main.py --- a/pypy/interpreter/test2/test_app_main.py +++ b/pypy/interpreter/test2/test_app_main.py @@ -907,7 +907,7 @@ sys.path.append(self.goal_dir) try: import app_main - pypy_c = os.path.join(self.trunkdir, 'pypy', 'translator', 'goal', 'pypy-c') + pypy_c = os.path.join(self.trunkdir, 'pypy', 'goal', 'pypy-c') app_main.setup_bootstrap_path(pypy_c) newpath = sys.path[:] # we get at least lib_pypy @@ -925,7 +925,7 @@ sys.path.append(self.goal_dir) try: import app_main - pypy_c = os.path.join(self.trunkdir, 'pypy', 'translator', 'goal', 'pypy-c') + pypy_c = os.path.join(self.trunkdir, 'pypy', 'goal', 'pypy-c') app_main.entry_point(pypy_c, [self.foo_py]) # assert it did not crash finally: From noreply at buildbot.pypy.org Sun Jan 27 08:50:40 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Sun, 27 Jan 2013 08:50:40 +0100 (CET) Subject: [pypy-commit] pypy default: update some paths in comments Message-ID: <20130127075040.8C2821C12E1@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60515:d411fd8682b7 Date: 2013-01-26 20:45 -0500 http://bitbucket.org/pypy/pypy/changeset/d411fd8682b7/ Log: update some paths in comments diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -719,7 +719,7 @@ def pypy_resolvedirof(s): # we ignore the issue of symlinks; for tests, the executable is always - # translator/goal/app_main.py anyway + # interpreter/app_main.py anyway import os return os.path.abspath(os.path.join(s, '..')) diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -1,6 +1,6 @@ #!/usr/bin/env python """ A sample script that packages PyPy, provided that it's already built. -It uses 'pypy/translator/goal/pypy-c' and parts of the rest of the working +It uses 'pypy/goal/pypy-c' and parts of the rest of the working copy. Usage: package.py root-pypy-dir [name-of-archive] [name-of-pypy-c] [destination-for-tarball] [pypy-c-path] diff --git a/pypy/tool/rundictbenchmarks.py b/pypy/tool/rundictbenchmarks.py --- a/pypy/tool/rundictbenchmarks.py +++ b/pypy/tool/rundictbenchmarks.py @@ -3,7 +3,7 @@ # this file runs some benchmarks with a pypy-c that is assumed to be # built using the MeasuringDictImplementation. -# it should be run with pypy/translator/goal as the cwd, and you'll +# it should be run with pypy/goal as the cwd, and you'll # need to hack a copy of rst2html for yourself (svn docutils # required). From noreply at buildbot.pypy.org Sun Jan 27 12:53:10 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 27 Jan 2013 12:53:10 +0100 (CET) Subject: [pypy-commit] pypy default: Fix the default goal. Unsure if it's better to keep it at all or completely remove it. Message-ID: <20130127115310.956A01C067C@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r60516:cf3939f3c5ed Date: 2013-01-27 12:52 +0100 http://bitbucket.org/pypy/pypy/changeset/cf3939f3c5ed/ Log: Fix the default goal. Unsure if it's better to keep it at all or completely remove it. diff --git a/rpython/translator/goal/translate.py b/rpython/translator/goal/translate.py --- a/rpython/translator/goal/translate.py +++ b/rpython/translator/goal/translate.py @@ -50,7 +50,7 @@ return result translate_optiondescr = OptionDescription("translate", "XXX", [ - StrOption("targetspec", "XXX", default='targetpypystandalone', + StrOption("targetspec", "XXX", default='../../pypy/goal/targetpypystandalone', cmdline=None), ChoiceOption("opt", "optimization level", OPT_LEVELS, default=DEFAULT_OPT_LEVEL, From noreply at buildbot.pypy.org Sun Jan 27 13:34:35 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 27 Jan 2013 13:34:35 +0100 (CET) Subject: [pypy-commit] pypy better-log-parser: Start a simpler version of jitlogparser Message-ID: <20130127123435.16F4F1C125A@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: better-log-parser Changeset: r60517:0f876ed9d38e Date: 2013-01-26 14:40 +0200 http://bitbucket.org/pypy/pypy/changeset/0f876ed9d38e/ Log: Start a simpler version of jitlogparser diff --git a/rpython/jit/tool/oparser_model.py b/rpython/jit/tool/oparser_model.py --- a/rpython/jit/tool/oparser_model.py +++ b/rpython/jit/tool/oparser_model.py @@ -84,7 +84,8 @@ return str(self.value) class ConstInt(Const): - pass + def getint(self): + return self.value class ConstPtr(Const): pass From noreply at buildbot.pypy.org Sun Jan 27 13:34:36 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 27 Jan 2013 13:34:36 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: put force_token into a normal register, so it's no longer tied to a strange ebp Message-ID: <20130127123436.5D9B61C125A@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60518:7712c8416bac Date: 2013-01-27 14:32 +0200 http://bitbucket.org/pypy/pypy/changeset/7712c8416bac/ Log: put force_token into a normal register, so it's no longer tied to a strange ebp location. That removes quite a bit of hacks. diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -1874,12 +1874,13 @@ fail_descr = rffi.cast(lltype.Signed, fail_descr) positions = [0] * len(guardtok.fail_locs) for i, loc in enumerate(guardtok.fail_locs): - if loc is None or loc is ebp: # frame + if loc is None: positions[i] = -1 elif isinstance(loc, StackLoc): positions[i] = loc.value else: assert isinstance(loc, RegLoc) + assert loc is not ebp # for now if loc.is_xmm: v = len(gpr_reg_mgr_cls.all_regs) + loc.value else: @@ -2485,6 +2486,11 @@ self.mc.overwrite(jmp_adr-1, chr(offset)) self.mc.MOV(heap(nursery_free_adr), edi) + def force_token(self, reg): + base_ofs = self.cpu.get_baseofs_of_frame_field() + assert isinstance(reg, RegLoc) + self.mc.LEA_rb(reg.value, -base_ofs) + genop_discard_list = [Assembler386.not_implemented_op_discard] * rop._LAST genop_list = [Assembler386.not_implemented_op] * rop._LAST genop_llong_list = {} diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -289,17 +289,17 @@ self.rm._check_invariants() self.xrm._check_invariants() - def Perform(self, op, arglocs, result_loc): + def perform(self, op, arglocs, result_loc): if not we_are_translated(): self.assembler.dump('%s <- %s(%s)' % (result_loc, op, arglocs)) self.assembler.regalloc_perform(op, arglocs, result_loc) - def PerformLLong(self, op, arglocs, result_loc): + def perform_llong(self, op, arglocs, result_loc): if not we_are_translated(): self.assembler.dump('%s <- %s(%s)' % (result_loc, op, arglocs)) self.assembler.regalloc_perform_llong(op, arglocs, result_loc) - def PerformMath(self, op, arglocs, result_loc): + def perform_math(self, op, arglocs, result_loc): if not we_are_translated(): self.assembler.dump('%s <- %s(%s)' % (result_loc, op, arglocs)) self.assembler.regalloc_perform_math(op, arglocs, result_loc) @@ -329,7 +329,7 @@ self.fm.get_frame_depth()) self.possibly_free_vars(guard_op.getfailargs()) - def PerformDiscard(self, op, arglocs): + def perform_discard(self, op, arglocs): if not we_are_translated(): self.assembler.dump('%s(%s)' % (op, arglocs)) self.assembler.regalloc_perform_discard(op, arglocs) @@ -427,7 +427,7 @@ locs = [loc, imm(fail_descr)] else: locs = [imm(fail_descr)] - self.Perform(op, locs, None) + self.perform(op, locs, None) def consider_guard_no_exception(self, op): self.perform_guard(op, [], None) @@ -480,12 +480,12 @@ def _consider_binop(self, op): loc, argloc = self._consider_binop_part(op) - self.Perform(op, [loc, argloc], loc) + self.perform(op, [loc, argloc], loc) def _consider_lea(self, op, loc): argloc = self.loc(op.getarg(1)) resloc = self.force_allocate_reg(op.result) - self.Perform(op, [loc, argloc], resloc) + self.perform(op, [loc, argloc], resloc) def consider_int_add(self, op): loc = self.loc(op.getarg(0)) @@ -520,7 +520,7 @@ def consider_int_neg(self, op): res = self.rm.force_result_in_reg(op.result, op.getarg(0)) - self.Perform(op, [res], res) + self.perform(op, [res], res) consider_int_invert = consider_int_neg @@ -531,7 +531,7 @@ loc2 = self.rm.make_sure_var_in_reg(op.getarg(1), selected_reg=ecx) args = op.getarglist() loc1 = self.rm.force_result_in_reg(op.result, op.getarg(0), args) - self.Perform(op, [loc1, loc2], loc1) + self.perform(op, [loc1, loc2], loc1) consider_int_rshift = consider_int_lshift consider_uint_rshift = consider_int_lshift @@ -551,11 +551,11 @@ def consider_int_mod(self, op): self._consider_int_div_or_mod(op, edx, eax) - self.Perform(op, [eax, ecx], edx) + self.perform(op, [eax, ecx], edx) def consider_int_floordiv(self, op): self._consider_int_div_or_mod(op, eax, edx) - self.Perform(op, [eax, ecx], eax) + self.perform(op, [eax, ecx], eax) consider_uint_floordiv = consider_int_floordiv @@ -572,7 +572,7 @@ if guard_op is None: loc = self.rm.force_allocate_reg(op.result, args, need_lower_byte=True) - self.Perform(op, arglocs, loc) + self.perform(op, arglocs, loc) else: self.perform_with_guard(op, guard_op, arglocs, None) @@ -593,7 +593,7 @@ loc1 = self.xrm.loc(op.getarg(1)) args = op.getarglist() loc0 = self.xrm.force_result_in_reg(op.result, op.getarg(0), args) - self.Perform(op, [loc0, loc1], loc0) + self.perform(op, [loc0, loc1], loc0) consider_float_add = _consider_float_op consider_float_sub = _consider_float_op @@ -612,7 +612,7 @@ arglocs[0] = self.xrm.make_sure_var_in_reg(vx) if guard_op is None: res = self.rm.force_allocate_reg(op.result, need_lower_byte=True) - self.Perform(op, arglocs, res) + self.perform(op, arglocs, res) else: self.perform_with_guard(op, guard_op, arglocs, None) @@ -625,7 +625,7 @@ def _consider_float_unary_op(self, op): loc0 = self.xrm.force_result_in_reg(op.result, op.getarg(0)) - self.Perform(op, [loc0], loc0) + self.perform(op, [loc0], loc0) consider_float_neg = _consider_float_unary_op consider_float_abs = _consider_float_unary_op @@ -633,12 +633,12 @@ def consider_cast_float_to_int(self, op): loc0 = self.xrm.make_sure_var_in_reg(op.getarg(0)) loc1 = self.rm.force_allocate_reg(op.result) - self.Perform(op, [loc0], loc1) + self.perform(op, [loc0], loc1) def consider_cast_int_to_float(self, op): loc0 = self.rm.make_sure_var_in_reg(op.getarg(0)) loc1 = self.xrm.force_allocate_reg(op.result) - self.Perform(op, [loc0], loc1) + self.perform(op, [loc0], loc1) def consider_cast_float_to_singlefloat(self, op): loc0 = self.xrm.make_sure_var_in_reg(op.getarg(0)) @@ -646,7 +646,7 @@ tmpxvar = TempBox() loctmp = self.xrm.force_allocate_reg(tmpxvar) # may be equal to loc0 self.xrm.possibly_free_var(tmpxvar) - self.Perform(op, [loc0, loctmp], loc1) + self.perform(op, [loc0, loctmp], loc1) consider_cast_singlefloat_to_float = consider_cast_int_to_float @@ -654,23 +654,23 @@ if longlong.is_64_bit: loc0 = self.xrm.make_sure_var_in_reg(op.getarg(0)) loc1 = self.rm.force_allocate_reg(op.result) - self.Perform(op, [loc0], loc1) + self.perform(op, [loc0], loc1) else: arg0 = op.getarg(0) loc0 = self.xrm.loc(arg0) loc1 = self.xrm.force_allocate_reg(op.result, forbidden_vars=[arg0]) - self.Perform(op, [loc0], loc1) + self.perform(op, [loc0], loc1) def consider_convert_longlong_bytes_to_float(self, op): if longlong.is_64_bit: loc0 = self.rm.make_sure_var_in_reg(op.getarg(0)) loc1 = self.xrm.force_allocate_reg(op.result) - self.Perform(op, [loc0], loc1) + self.perform(op, [loc0], loc1) else: arg0 = op.getarg(0) loc0 = self.xrm.make_sure_var_in_reg(arg0) loc1 = self.xrm.force_allocate_reg(op.result, forbidden_vars=[arg0]) - self.Perform(op, [loc0], loc1) + self.perform(op, [loc0], loc1) def _consider_llong_binop_xx(self, op): # must force both arguments into xmm registers, because we don't @@ -679,7 +679,7 @@ args = [op.getarg(1), op.getarg(2)] loc1 = self.load_xmm_aligned_16_bytes(args[1]) loc0 = self.xrm.force_result_in_reg(op.result, args[0], args) - self.PerformLLong(op, [loc0, loc1], loc0) + self.perform_llong(op, [loc0, loc1], loc0) def _consider_llong_eq_ne_xx(self, op): # must force both arguments into xmm registers, because we don't @@ -692,7 +692,7 @@ loc3 = self.xrm.force_allocate_reg(tmpxvar, args) self.xrm.possibly_free_var(tmpxvar) loc0 = self.rm.force_allocate_reg(op.result, need_lower_byte=True) - self.PerformLLong(op, [loc1, loc2, loc3], loc0) + self.perform_llong(op, [loc1, loc2, loc3], loc0) def _maybe_consider_llong_lt(self, op): # XXX just a special case for now @@ -706,14 +706,14 @@ assert isinstance(box, BoxFloat) loc1 = self.xrm.make_sure_var_in_reg(box) loc0 = self.rm.force_allocate_reg(op.result) - self.PerformLLong(op, [loc1], loc0) + self.perform_llong(op, [loc1], loc0) return True def _consider_llong_to_int(self, op): # accept an argument in a xmm register or in the stack loc1 = self.xrm.loc(op.getarg(1)) loc0 = self.rm.force_allocate_reg(op.result) - self.PerformLLong(op, [loc1], loc0) + self.perform_llong(op, [loc1], loc0) def _loc_of_const_longlong(self, value64): c = ConstFloat(value64) @@ -731,17 +731,17 @@ tmpxvar = TempBox() loc2 = self.xrm.force_allocate_reg(tmpxvar, [op.result]) self.xrm.possibly_free_var(tmpxvar) - self.PerformLLong(op, [loc1, loc2], loc0) + self.perform_llong(op, [loc1, loc2], loc0) def _consider_llong_from_uint(self, op): assert IS_X86_32 loc0 = self.xrm.force_allocate_reg(op.result) loc1 = self.rm.make_sure_var_in_reg(op.getarg(1)) - self.PerformLLong(op, [loc1], loc0) + self.perform_llong(op, [loc1], loc0) def _consider_math_sqrt(self, op): loc0 = self.xrm.force_result_in_reg(op.result, op.getarg(1)) - self.PerformMath(op, [loc0], loc0) + self.perform_math(op, [loc0], loc0) def _call(self, op, arglocs, force_store=[], guard_not_forced_op=None): # we need to save registers on the stack: @@ -774,7 +774,7 @@ if guard_not_forced_op is not None: self.perform_with_guard(op, guard_not_forced_op, arglocs, resloc) else: - self.Perform(op, arglocs, resloc) + self.perform(op, arglocs, resloc) def _consider_call(self, op, guard_not_forced_op=None): calldescr = op.getdescr() @@ -850,7 +850,7 @@ # or setarrayitem_gc. It avoids loading it twice from the memory. arglocs = [self.rm.make_sure_var_in_reg(op.getarg(i), args) for i in range(N)] - self.PerformDiscard(op, arglocs) + self.perform_discard(op, arglocs) consider_cond_call_gc_wb_array = consider_cond_call_gc_wb @@ -907,7 +907,7 @@ base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args) value_loc = self.make_sure_var_in_reg(op.getarg(1), args, need_lower_byte=need_lower_byte) - self.PerformDiscard(op, [base_loc, ofs_loc, size_loc, value_loc]) + self.perform_discard(op, [base_loc, ofs_loc, size_loc, value_loc]) consider_setfield_raw = consider_setfield_gc @@ -939,7 +939,7 @@ self.rm.possibly_free_var(box_index) self.rm.possibly_free_var(box_base) self.possibly_free_var(box_value) - self.PerformDiscard(op, [base_loc, ofs, itemsize, fieldsize, + self.perform_discard(op, [base_loc, ofs, itemsize, fieldsize, index_loc, temp_loc, value_loc]) consider_setinteriorfield_raw = consider_setinteriorfield_gc @@ -950,7 +950,7 @@ ofs_loc = self.rm.make_sure_var_in_reg(op.getarg(1), args) value_loc = self.rm.make_sure_var_in_reg(op.getarg(2), args, need_lower_byte=True) - self.PerformDiscard(op, [base_loc, ofs_loc, value_loc]) + self.perform_discard(op, [base_loc, ofs_loc, value_loc]) consider_unicodesetitem = consider_strsetitem @@ -965,7 +965,7 @@ value_loc = self.make_sure_var_in_reg(op.getarg(2), args, need_lower_byte=need_lower_byte) ofs_loc = self.rm.make_sure_var_in_reg(op.getarg(1), args) - self.PerformDiscard(op, [base_loc, ofs_loc, value_loc, + self.perform_discard(op, [base_loc, ofs_loc, value_loc, imm(itemsize), imm(ofs)]) consider_setarrayitem_raw = consider_setarrayitem_gc @@ -982,7 +982,7 @@ sign_loc = imm1 else: sign_loc = imm0 - self.Perform(op, [base_loc, ofs_loc, size_loc, sign_loc], result_loc) + self.perform(op, [base_loc, ofs_loc, size_loc, sign_loc], result_loc) consider_getfield_raw = consider_getfield_gc consider_getfield_raw_pure = consider_getfield_gc @@ -998,7 +998,7 @@ sign_loc = imm1 else: sign_loc = imm0 - self.Perform(op, [base_loc, ofs_loc, imm(itemsize), imm(ofs), + self.perform(op, [base_loc, ofs_loc, imm(itemsize), imm(ofs), sign_loc], result_loc) consider_getarrayitem_raw = consider_getarrayitem_gc @@ -1031,7 +1031,7 @@ tempvar = TempBox() temp_loc = self.rm.force_allocate_reg(tempvar, [op.getarg(0)]) self.rm.possibly_free_var(tempvar) - self.Perform(op, [base_loc, ofs, itemsize, fieldsize, + self.perform(op, [base_loc, ofs, itemsize, fieldsize, index_loc, temp_loc, sign_loc], result_loc) def consider_int_is_true(self, op, guard_op): @@ -1041,27 +1041,27 @@ self.perform_with_guard(op, guard_op, [argloc], None) else: resloc = self.rm.force_allocate_reg(op.result, need_lower_byte=True) - self.Perform(op, [argloc], resloc) + self.perform(op, [argloc], resloc) consider_int_is_zero = consider_int_is_true def consider_same_as(self, op): argloc = self.loc(op.getarg(0)) resloc = self.force_allocate_reg(op.result) - self.Perform(op, [argloc], resloc) + self.perform(op, [argloc], resloc) consider_cast_ptr_to_int = consider_same_as consider_cast_int_to_ptr = consider_same_as def consider_int_force_ge_zero(self, op): argloc = self.make_sure_var_in_reg(op.getarg(0)) resloc = self.force_allocate_reg(op.result, [op.getarg(0)]) - self.Perform(op, [argloc], resloc) + self.perform(op, [argloc], resloc) def consider_strlen(self, op): args = op.getarglist() base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args) result_loc = self.rm.force_allocate_reg(op.result) - self.Perform(op, [base_loc], result_loc) + self.perform(op, [base_loc], result_loc) consider_unicodelen = consider_strlen @@ -1072,14 +1072,14 @@ args = op.getarglist() base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args) result_loc = self.rm.force_allocate_reg(op.result) - self.Perform(op, [base_loc, imm(ofs)], result_loc) + self.perform(op, [base_loc, imm(ofs)], result_loc) def consider_strgetitem(self, op): args = op.getarglist() base_loc = self.rm.make_sure_var_in_reg(op.getarg(0), args) ofs_loc = self.rm.make_sure_var_in_reg(op.getarg(1), args) result_loc = self.rm.force_allocate_reg(op.result) - self.Perform(op, [base_loc, ofs_loc], result_loc) + self.perform(op, [base_loc, ofs_loc], result_loc) consider_unicodegetitem = consider_strgetitem @@ -1163,7 +1163,7 @@ # result in rdx result_loc = self.rm.force_allocate_reg(op.result, selected_reg=edx) - self.Perform(op, [], result_loc) + self.perform(op, [], result_loc) else: # on 32-bit, use both eax and edx as temporary registers, # use a temporary xmm register, and returns the result in @@ -1173,7 +1173,7 @@ xmmtmpbox = TempBox() xmmtmploc = self.xrm.force_allocate_reg(xmmtmpbox) result_loc = self.xrm.force_allocate_reg(op.result) - self.Perform(op, [xmmtmploc], result_loc) + self.perform(op, [xmmtmploc], result_loc) self.xrm.possibly_free_var(xmmtmpbox) self.rm.possibly_free_var(tmpbox_low) self.rm.possibly_free_var(tmpbox_high) @@ -1259,8 +1259,9 @@ self.force_spill_var(op.getarg(0)) def consider_force_token(self, op): - # the FORCE_TOKEN operation returns directly 'ebp' - self.rm.force_allocate_frame_reg(op.result) + # XXX for now we return a regular reg + #self.rm.force_allocate_frame_reg(op.result) + self.assembler.force_token(self.rm.force_allocate_reg(op.result)) def consider_label(self, op): descr = op.getdescr() From noreply at buildbot.pypy.org Sun Jan 27 13:35:25 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 27 Jan 2013 13:35:25 +0100 (CET) Subject: [pypy-commit] pypy default: Remove again this file, which was resurrected at some point. Message-ID: <20130127123525.35B501C125A@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r60519:96504ced9bca Date: 2013-01-27 13:35 +0100 http://bitbucket.org/pypy/pypy/changeset/96504ced9bca/ Log: Remove again this file, which was resurrected at some point. diff --git a/rpython/jit/backend/llgraph/llimpl.py b/rpython/jit/backend/llgraph/llimpl.py deleted file mode 100644 --- a/rpython/jit/backend/llgraph/llimpl.py +++ /dev/null @@ -1,1968 +0,0 @@ -""" -The non-RPythonic part of the llgraph backend. -This contains all the code that is directly run -when executing on top of the llinterpreter. -""" - -import weakref -from rpython.flowspace.model import Variable, Constant -from rpython.annotator import model as annmodel -from rpython.jit.metainterp.history import REF, INT, FLOAT -from rpython.jit.metainterp import history -from rpython.jit.codewriter import heaptracker -from rpython.rtyper.lltypesystem import lltype, llmemory, rclass, rstr, rffi -from rpython.rtyper.ootypesystem import ootype -from rpython.rtyper.module.support import LLSupport, OOSupport -from rpython.rtyper.llinterp import LLException -from rpython.rtyper.extregistry import ExtRegistryEntry - -from rpython.jit.metainterp import resoperation -from rpython.jit.metainterp.resoperation import rop -from rpython.jit.backend.llgraph import symbolic -from rpython.jit.codewriter import longlong -from rpython.jit.codewriter.effectinfo import EffectInfo - -from rpython.rlib.objectmodel import ComputedIntSymbolic, we_are_translated -from rpython.rlib.rarithmetic import ovfcheck -from rpython.rlib.rarithmetic import r_longlong, r_ulonglong, r_uint -from rpython.rlib.rtimer import read_timestamp - -import py -from rpython.tool.ansi_print import ansi_log -log = py.log.Producer('runner') -py.log.setconsumer('runner', ansi_log) - -IS_32_BIT = r_ulonglong is not r_uint - - -def _from_opaque(opq): - return opq._obj.externalobj - -_TO_OPAQUE = {} - -def _to_opaque(value): - try: - return value._the_opaque_pointer - except AttributeError: - op = lltype.opaqueptr(_TO_OPAQUE[value.__class__], 'opaque', - externalobj=value) - value._the_opaque_pointer = op - return op - -def _normalize(value): - if isinstance(value, lltype._ptr): - value = lltype.top_container(value._obj) - return value - -def from_opaque_string(s): - if isinstance(s, str): - return s - elif isinstance(s, ootype._string): - return OOSupport.from_rstr(s) - else: - return LLSupport.from_rstr(s) - -FLOAT_ARRAY_TP = lltype.Ptr(lltype.Array(lltype.Float, hints={"nolength": True})) -def maybe_uncast(TP, array): - if array._TYPE.TO.OF != lltype.Float: - # array._TYPE.TO._hints.get("uncast_on_llgraph"): - array = rffi.cast(TP, array) - return array - -# a list of argtypes of all operations - couldn't find any and it's -# very useful. Note however that the table is half-broken here and -# there, in ways that are sometimes a bit hard to fix; that's why -# it is not "official". -TYPES = { - 'int_add' : (('int', 'int'), 'int'), - 'int_sub' : (('int', 'int'), 'int'), - 'int_mul' : (('int', 'int'), 'int'), - 'int_floordiv' : (('int', 'int'), 'int'), - 'int_mod' : (('int', 'int'), 'int'), - 'int_and' : (('int', 'int'), 'int'), - 'int_or' : (('int', 'int'), 'int'), - 'int_xor' : (('int', 'int'), 'int'), - 'int_lshift' : (('int', 'int'), 'int'), - 'int_rshift' : (('int', 'int'), 'int'), - 'int_lt' : (('int', 'int'), 'bool'), - 'int_gt' : (('int', 'int'), 'bool'), - 'int_ge' : (('int', 'int'), 'bool'), - 'int_le' : (('int', 'int'), 'bool'), - 'int_eq' : (('int', 'int'), 'bool'), - 'int_ne' : (('int', 'int'), 'bool'), - 'int_is_true' : (('int',), 'bool'), - 'int_is_zero' : (('int',), 'bool'), - 'int_neg' : (('int',), 'int'), - 'int_invert' : (('int',), 'int'), - 'int_add_ovf' : (('int', 'int'), 'int'), - 'int_sub_ovf' : (('int', 'int'), 'int'), - 'int_mul_ovf' : (('int', 'int'), 'int'), - 'int_force_ge_zero':(('int',), 'int'), - 'uint_add' : (('int', 'int'), 'int'), - 'uint_sub' : (('int', 'int'), 'int'), - 'uint_mul' : (('int', 'int'), 'int'), - 'uint_lt' : (('int', 'int'), 'bool'), - 'uint_le' : (('int', 'int'), 'bool'), - 'uint_eq' : (('int', 'int'), 'bool'), - 'uint_ne' : (('int', 'int'), 'bool'), - 'uint_gt' : (('int', 'int'), 'bool'), - 'uint_ge' : (('int', 'int'), 'bool'), - 'uint_xor' : (('int', 'int'), 'int'), - 'uint_rshift' : (('int', 'int'), 'int'), - 'uint_floordiv' : (('int', 'int'), 'int'), - 'float_add' : (('float', 'float'), 'float'), - 'float_sub' : (('float', 'float'), 'float'), - 'float_mul' : (('float', 'float'), 'float'), - 'float_truediv' : (('float', 'float'), 'float'), - 'float_lt' : (('float', 'float'), 'bool'), - 'float_le' : (('float', 'float'), 'bool'), - 'float_eq' : (('float', 'float'), 'bool'), - 'float_ne' : (('float', 'float'), 'bool'), - 'float_gt' : (('float', 'float'), 'bool'), - 'float_ge' : (('float', 'float'), 'bool'), - 'float_neg' : (('float',), 'float'), - 'float_abs' : (('float',), 'float'), - 'cast_float_to_int':(('float',), 'int'), - 'cast_int_to_float':(('int',), 'float'), - 'same_as' : (('int',), 'int'), # could also be ptr=>ptr - 'new_with_vtable' : (('ref',), 'ref'), - 'new' : ((), 'ref'), - 'new_array' : (('int',), 'ref'), - 'oois' : (('ref', 'ref'), 'bool'), - 'ooisnot' : (('ref', 'ref'), 'bool'), - 'instanceof' : (('ref',), 'bool'), - 'subclassof' : (('ref', 'ref'), 'bool'), - 'runtimenew' : (('ref',), 'ref'), - 'setfield_gc' : (('ref', 'intorptr'), None), - 'getfield_gc' : (('ref',), 'intorptr'), - 'getfield_gc_pure': (('ref',), 'intorptr'), - 'setfield_raw' : (('ref', 'intorptr'), None), - 'getfield_raw' : (('ref',), 'intorptr'), - 'getfield_raw_pure': (('ref',), 'intorptr'), - 'setarrayitem_gc' : (('ref', 'int', 'intorptr'), None), - 'getarrayitem_gc' : (('ref', 'int'), 'intorptr'), - 'getarrayitem_gc_pure' : (('ref', 'int'), 'intorptr'), - 'setarrayitem_raw' : (('ref', 'int', 'intorptr'), None), - 'getarrayitem_raw' : (('ref', 'int'), 'intorptr'), - 'getarrayitem_raw_pure' : (('ref', 'int'), 'intorptr'), - 'arraylen_gc' : (('ref',), 'int'), - 'call' : (('ref', 'varargs'), 'intorptr'), - 'call_assembler' : (('varargs',), 'intorptr'), - 'cond_call_gc_wb' : (('ptr', 'ptr'), None), - 'cond_call_gc_wb_array': (('ptr', 'int', 'ptr'), None), - 'oosend' : (('varargs',), 'intorptr'), - 'oosend_pure' : (('varargs',), 'intorptr'), - 'guard_true' : (('bool',), None), - 'guard_false' : (('bool',), None), - 'guard_value' : (('int', 'int'), None), - 'guard_class' : (('ref', 'ref'), None), - 'guard_no_exception' : ((), None), - 'guard_exception' : (('ref',), 'ref'), - 'guard_no_overflow' : ((), None), - 'guard_overflow' : ((), None), - 'guard_nonnull' : (('ref',), None), - 'guard_isnull' : (('ref',), None), - 'guard_nonnull_class' : (('ref', 'ref'), None), - 'newstr' : (('int',), 'ref'), - 'strlen' : (('ref',), 'int'), - 'strgetitem' : (('ref', 'int'), 'int'), - 'strsetitem' : (('ref', 'int', 'int'), None), - 'newunicode' : (('int',), 'ref'), - 'unicodelen' : (('ref',), 'int'), - 'unicodegetitem' : (('ref', 'int'), 'int'), - 'unicodesetitem' : (('ref', 'int', 'int'), 'int'), - 'cast_ptr_to_int' : (('ref',), 'int'), - 'cast_int_to_ptr' : (('int',), 'ref'), - 'debug_merge_point': (('ref', 'int', 'int'), None), - 'force_token' : ((), 'int'), - 'call_may_force' : (('int', 'varargs'), 'intorptr'), - 'guard_not_forced': ((), None), -} - -# ____________________________________________________________ - -class CompiledLoop(object): - has_been_freed = False - invalid = False - - def __init__(self): - self.inputargs = [] - self.operations = [] - - def getargtypes(self): - return [v.concretetype for v in self.inputargs] - - def __repr__(self): - lines = [] - self.as_text(lines, 1) - return 'CompiledLoop %s:\n%s' % (self.inputargs, '\n'.join(lines)) - - def as_text(self, lines, indent): - for op in self.operations: - lines.append('\t'*indent + repr(op)) - -class Operation(object): - result = None - descr = None - jump_target = None - fail_args = None - - def __init__(self, opnum): - self.opnum = opnum - self.args = [] - - def __repr__(self): - if self.result is not None: - sres = repr0(self.result) + ' = ' - else: - sres = '' - return '{%s%s(%s)}' % (sres, self.getopname(), - ', '.join(map(repr0, self.args))) - - def getopname(self): - try: - return resoperation.opname[self.opnum] - except KeyError: - return '<%d>' % self.opnum - - def is_guard(self): - return rop._GUARD_FIRST <= self.opnum <= rop._GUARD_LAST - - def is_final(self): - return rop._FINAL_FIRST <= self.opnum <= rop._FINAL_LAST - -def repr0(x): - if isinstance(x, list): - return '[' + ', '.join(repr0(y) for y in x) + ']' - elif isinstance(x, Constant): - return '(' + repr0(x.value) + ')' - elif isinstance(x, lltype._ptr): - x = llmemory.cast_ptr_to_adr(x) - if x.ptr: - try: - container = x.ptr._obj._normalizedcontainer() - return '* %s' % (container._TYPE._short_name(),) - except AttributeError: - return repr(x) - else: - return 'NULL' - else: - return repr(x) - -def repr_list(lst, types): - res_l = [] - if types and types[-1] == 'varargs': - types = types[:-1] + ('int',) * (len(lst) - len(types) + 1) - assert len(types) == len(lst) - for elem, tp in zip(lst, types): - if isinstance(elem, Constant): - res_l.append('(%s)' % repr1(elem, tp)) - else: - res_l.append(repr1(elem, tp)) - return '[%s]' % (', '.join(res_l)) - -def repr1(x, tp): - if tp == "intorptr": - TYPE = lltype.typeOf(x) - if isinstance(TYPE, lltype.Ptr) and TYPE.TO._gckind == 'gc': - tp = "ref" - else: - tp = "int" - if tp == 'int': - return str(x) - elif tp == 'void': - return '---' - elif tp == 'ref': - if not x: - return '(* None)' - if isinstance(x, int): - # XXX normalize? - ptr = str(llmemory.cast_int_to_adr(x)) - elif isinstance(ootype.typeOf(x), ootype.OOType): - return repr(x) - else: - if getattr(x, '_fake', None): - return repr(x) - if lltype.typeOf(x) == llmemory.GCREF: - TP = lltype.Ptr(lltype.typeOf(x._obj.container)) - ptr = lltype.cast_opaque_ptr(TP, x) - else: - ptr = x - try: - container = ptr._obj._normalizedcontainer() - return '(* %s)' % (container._TYPE._short_name(),) - except AttributeError: - return '(%r)' % (ptr,) - elif tp == 'bool': - assert x == 0 or x == 1 - return str(bool(x)) - #elif tp == 'fieldname': - # return str(symbolic.TokenToField[x...][1]) - elif tp == 'float': - return str(x) - else: - raise NotImplementedError("tp = %s" % tp) - -_variables = [] - -def compile_start(): - del _variables[:] - return _to_opaque(CompiledLoop()) - -def mark_as_free(loop): - loop = _from_opaque(loop) - assert not loop.has_been_freed - loop.has_been_freed = True - -def compile_start_int_var(loop): - return compile_start_ref_var(loop, lltype.Signed) - -def compile_start_float_var(loop): - return compile_start_ref_var(loop, longlong.FLOATSTORAGE) - -def compile_start_ref_var(loop, TYPE): - loop = _from_opaque(loop) - assert not loop.operations - v = Variable() - v.concretetype = TYPE - loop.inputargs.append(v) - r = len(_variables) - _variables.append(v) - return r - -def compile_started_vars(clt): - if not hasattr(clt, '_debug_argtypes'): # only when compiling the loop - argtypes = [v.concretetype for v in _variables] - try: - clt._debug_argtypes = argtypes - except AttributeError: # when 'clt' is actually a translated - pass # GcStruct - -def compile_add(loop, opnum): - loop = _from_opaque(loop) - loop.operations.append(Operation(opnum)) - -def compile_add_descr(loop, ofs, type, arg_types, extrainfo, width): - from rpython.jit.backend.llgraph.runner import Descr - loop = _from_opaque(loop) - op = loop.operations[-1] - assert isinstance(type, str) and len(type) == 1 - op.descr = Descr(ofs, type, arg_types=arg_types, extrainfo=extrainfo, width=width) - -def compile_add_descr_arg(loop, ofs, type, arg_types): - from rpython.jit.backend.llgraph.runner import Descr - loop = _from_opaque(loop) - op = loop.operations[-1] - assert isinstance(type, str) and len(type) == 1 - op.args.append(Descr(ofs, type, arg_types=arg_types)) - -def compile_add_loop_token(loop, descr): - if we_are_translated(): - raise ValueError("CALL_ASSEMBLER not supported") - loop = _from_opaque(loop) - op = loop.operations[-1] - op.descr = weakref.ref(descr) - -TARGET_TOKENS = weakref.WeakKeyDictionary() - -def compile_add_target_token(loop, descr, clt): - # here, 'clt' is the compiled_loop_token of the original loop that - # we are compiling - loop = _from_opaque(loop) - op = loop.operations[-1] - descrobj = _normalize(descr) - TARGET_TOKENS[descrobj] = loop, len(loop.operations), op.args, clt - -def compile_add_var(loop, intvar): - loop = _from_opaque(loop) - op = loop.operations[-1] - op.args.append(_variables[intvar]) - -def compile_add_int_const(loop, value): - compile_add_ref_const(loop, value, lltype.Signed) - -def compile_add_float_const(loop, value): - compile_add_ref_const(loop, value, longlong.FLOATSTORAGE) - -def compile_add_ref_const(loop, value, TYPE): - loop = _from_opaque(loop) - const = Constant(value) - const.concretetype = TYPE - op = loop.operations[-1] - op.args.append(const) - -def compile_add_int_result(loop): - return compile_add_ref_result(loop, lltype.Signed) - -def compile_add_float_result(loop): - return compile_add_ref_result(loop, longlong.FLOATSTORAGE) - -def compile_add_ref_result(loop, TYPE): - loop = _from_opaque(loop) - v = Variable() - v.concretetype = TYPE - op = loop.operations[-1] - op.result = v - r = len(_variables) - _variables.append(v) - return r - -def compile_add_jump_target(loop, targettoken, source_clt): - loop = _from_opaque(loop) - descrobj = _normalize(targettoken) - (loop_target, target_opindex, target_inputargs, target_clt - ) = TARGET_TOKENS[descrobj] - # - try: - assert source_clt._debug_argtypes == target_clt._debug_argtypes - except AttributeError: # when translated - pass - # - op = loop.operations[-1] - op.jump_target = loop_target - op.jump_target_opindex = target_opindex - op.jump_target_inputargs = target_inputargs - assert op.opnum == rop.JUMP - assert [v.concretetype for v in op.args] == ( - [v.concretetype for v in target_inputargs]) - # - if loop_target == loop: - log.info("compiling new loop") - else: - log.info("compiling new bridge") - -def compile_add_guard_jump_target(loop, loop_target): - loop = _from_opaque(loop) - loop_target = _from_opaque(loop_target) - op = loop.operations[-1] - assert op.is_guard() - op.jump_target = loop_target - -def compile_add_fail(loop, fail_index): - loop = _from_opaque(loop) - index = len(loop.operations)-1 - op = loop.operations[index] - op.fail_index = fail_index - return index - -def compile_add_fail_arg(loop, intvar): - loop = _from_opaque(loop) - op = loop.operations[-1] - if op.fail_args is None: - op.fail_args = [] - if intvar == -1: - op.fail_args.append(None) - else: - op.fail_args.append(_variables[intvar]) - -def compile_redirect_fail(old_loop, old_index, new_loop): - old_loop = _from_opaque(old_loop) - new_loop = _from_opaque(new_loop) - guard_op = old_loop.operations[old_index] - assert guard_op.is_guard() - guard_op.jump_target = new_loop - # check that the bridge's inputargs are of the correct number and - # kind for the guard - if guard_op.fail_args is not None: - argkinds = [v.concretetype for v in guard_op.fail_args if v] - else: - argkinds = [] - assert argkinds == [v.concretetype for v in new_loop.inputargs] - -# ------------------------------ - -class Frame(object): - OPHANDLERS = [None] * (rop._LAST+1) - - def __init__(self, cpu): - self.verbose = False - self.cpu = cpu - self.opindex = 1 - self._forced = False - self._may_force = -1 - - def getenv(self, v): - from rpython.jit.backend.llgraph.runner import Descr - if isinstance(v, Constant): - return v.value - elif isinstance(v, Descr): - return v - else: - return self.env[v] - - def _populate_fail_args(self, op, skip=None): - fail_args = [] - if op.fail_args: - for fail_arg in op.fail_args: - if fail_arg is None: - fail_args.append(None) - elif fail_arg is skip: - fail_args.append(fail_arg.concretetype._defl()) - else: - fail_args.append(self.getenv(fail_arg)) - self.fail_args = fail_args - self.fail_index = op.fail_index - - def execute(self): - """Execute all operations in a loop, - possibly following to other loops as well. - """ - global _last_exception - assert _last_exception is None, "exception left behind" - verbose = True - self.opindex = 0 - while True: - assert not self.loop.has_been_freed - op = self.loop.operations[self.opindex] - args = [self.getenv(v) for v in op.args] - if not op.is_final(): - try: - result = self.execute_operation(op.opnum, args, op.descr, - verbose) - except GuardFailed: - assert op.is_guard() - _stats.exec_conditional_jumps += 1 - if op.jump_target is not None: - # a patched guard, pointing to further code - if op.fail_args: - args = [self.getenv(v) for v in op.fail_args if v] - else: - args = [] - assert len(op.jump_target.inputargs) == len(args) - self.env = dict(zip(op.jump_target.inputargs, args)) - self.loop = op.jump_target - self.opindex = 0 - continue - else: - self._populate_fail_args(op) - # a non-patched guard - if self.verbose: - log.trace('failed: %s' % ( - ', '.join(map(str, fail_args)),)) - return op.fail_index - #verbose = self.verbose - assert (result is None) == (op.result is None) - if op.result is not None: - RESTYPE = op.result.concretetype - if RESTYPE is lltype.Signed: - x = self.as_int(result) - elif RESTYPE is llmemory.GCREF: - x = self.as_ptr(result) - elif RESTYPE is ootype.Object: - x = self.as_object(result) - elif RESTYPE is longlong.FLOATSTORAGE: - x = self.as_floatstorage(result) - else: - raise Exception("op.result.concretetype is %r" - % (RESTYPE,)) - self.env[op.result] = x - self.opindex += 1 - continue - if op.opnum == rop.JUMP: - inputargs = op.jump_target_inputargs - assert len(inputargs) == len(args) - self.env = dict(zip(inputargs, args)) - self.loop = op.jump_target - self.opindex = op.jump_target_opindex - _stats.exec_jumps += 1 - elif op.opnum == rop.FINISH: - if self.verbose: - log.trace('finished: %s' % ( - ', '.join(map(str, args)),)) - self.fail_args = args - return op.fail_index - - else: - assert 0, "unknown final operation %d" % (op.opnum,) - - def execute_operation(self, opnum, values, descr, verbose): - """Execute a single operation. - """ - ophandler = self.OPHANDLERS[opnum] - if ophandler is None: - self._define_impl(opnum) - ophandler = self.OPHANDLERS[opnum] - assert ophandler is not None, "missing impl for op %d" % opnum - opname = resoperation.opname[opnum].lower() - exec_counters = _stats.exec_counters - exec_counters[opname] = exec_counters.get(opname, 0) + 1 - for i in range(len(values)): - if isinstance(values[i], ComputedIntSymbolic): - values[i] = values[i].compute_fn() - res = NotImplemented - try: - res = ophandler(self, descr, *values) - finally: - if 0: # if verbose: - argtypes, restype = TYPES[opname] - if res is None: - resdata = '' - elif res is NotImplemented: - resdata = '*fail*' - else: - resdata = '-> ' + repr1(res, restype) - # fish the types - log.cpu('\t%s %s %s' % (opname, repr_list(values, argtypes), - resdata)) - return res - - def as_int(self, x): - return cast_to_int(x) - - def as_ptr(self, x): - return cast_to_ptr(x) - - def as_object(self, x): - return ootype.cast_to_object(x) - - def as_floatstorage(self, x): - return cast_to_floatstorage(x) - - def log_progress(self): - count = sum(_stats.exec_counters.values()) - count_jumps = _stats.exec_jumps - log.trace('ran %d operations, %d jumps' % (count, count_jumps)) - - # ---------- - - @classmethod - def _define_impl(cls, opnum): - opname = resoperation.opname[opnum] - try: - op = getattr(cls, 'op_' + opname.lower()) # op_guard_true etc. - except AttributeError: - try: - impl = globals()['do_' + opname.lower()] # do_arraylen_gc etc. - def op(self, descr, *args): - if descr is None: - return impl(*args) - else: - return impl(descr, *args) - except KeyError: - op = cls._make_impl_from_blackhole_interp(opname) - cls.OPHANDLERS[opnum] = op - - @classmethod - def _make_impl_from_blackhole_interp(cls, opname): - from rpython.jit.metainterp.blackhole import BlackholeInterpreter - name = 'bhimpl_' + opname.lower() - func = BlackholeInterpreter.__dict__[name] - for argtype in func.argtypes: - assert argtype in ('i', 'r', 'f') - # - def _op_default_implementation(self, descr, *args): - # for all operations implemented in the blackhole interpreter - return func(*args) - # - return _op_default_implementation - - def op_label(self, _, *args): - op = self.loop.operations[self.opindex] - assert op.opnum == rop.LABEL - assert len(op.args) == len(args) - newenv = {} - for v, value in zip(op.args, args): - newenv[v] = value - self.env = newenv - - def op_debug_merge_point(self, _, *args): - from rpython.jit.metainterp.warmspot import get_stats - try: - stats = get_stats() - except AttributeError: - pass - else: - stats.add_merge_point_location(args[1:]) - pass - - def op_guard_true(self, _, value): - if not value: - raise GuardFailed - - def op_guard_false(self, _, value): - if value: - raise GuardFailed - - op_guard_nonnull = op_guard_true - op_guard_isnull = op_guard_false - - def op_guard_class(self, _, value, expected_class): - value = lltype.cast_opaque_ptr(rclass.OBJECTPTR, value) - expected_class = llmemory.cast_adr_to_ptr( - llmemory.cast_int_to_adr(expected_class), - rclass.CLASSTYPE) - if value.typeptr != expected_class: - raise GuardFailed - - def op_guard_nonnull_class(self, _, value, expected_class): - if not value: - raise GuardFailed - self.op_guard_class(_, value, expected_class) - - def op_guard_value(self, _, value, expected_value): - if value != expected_value: - raise GuardFailed - - def op_guard_no_exception(self, _): - if _last_exception: - raise GuardFailed - - def _check_exception(self, expected_exception): - global _last_exception - expected_exception = self._cast_exception(expected_exception) - assert expected_exception - exc = _last_exception - if exc: - got = exc.args[0] - # exact match! - if got != expected_exception: - return False - return True - else: - return False - - def _cast_exception(self, exception): - return llmemory.cast_adr_to_ptr( - llmemory.cast_int_to_adr(exception), - rclass.CLASSTYPE) - - def _issubclass(self, cls1, cls2): - return rclass.ll_issubclass(cls1, cls2) - - def op_guard_exception(self, _, expected_exception): - global _last_exception - if not self._check_exception(expected_exception): - raise GuardFailed - res = _last_exception[1] - _last_exception = None - return res - - def op_guard_no_overflow(self, _): - flag = self.overflow_flag - del self.overflow_flag - if flag: - raise GuardFailed - - def op_guard_overflow(self, _): - flag = self.overflow_flag - del self.overflow_flag - if not flag: - raise GuardFailed - - def op_int_add_ovf(self, _, x, y): - try: - z = ovfcheck(x + y) - except OverflowError: - ovf = True - z = 0 - else: - ovf = False - self.overflow_flag = ovf - return z - - def op_int_sub_ovf(self, _, x, y): - try: - z = ovfcheck(x - y) - except OverflowError: - ovf = True - z = 0 - else: - ovf = False - self.overflow_flag = ovf - return z - - def op_int_mul_ovf(self, _, x, y): - try: - z = ovfcheck(x * y) - except OverflowError: - ovf = True - z = 0 - else: - ovf = False - self.overflow_flag = ovf - return z - - def op_keepalive(self, _, x): - pass - - # ---------- - # delegating to the builtins do_xxx() (done automatically for simple cases) - - def op_getarrayitem_gc(self, arraydescr, array, index): - if arraydescr.typeinfo == REF: - return do_getarrayitem_gc_ptr(array, index) - elif arraydescr.typeinfo == INT: - return do_getarrayitem_gc_int(array, index) - elif arraydescr.typeinfo == FLOAT: - return do_getarrayitem_gc_float(array, index) - else: - raise NotImplementedError - - op_getarrayitem_gc_pure = op_getarrayitem_gc - - def op_getarrayitem_raw(self, arraydescr, array, index): - if arraydescr.typeinfo == REF: - raise NotImplementedError("getarrayitem_raw -> gcref") - elif arraydescr.typeinfo == INT: - return do_getarrayitem_raw_int(array, index, arraydescr.ofs) - elif arraydescr.typeinfo == FLOAT: - return do_getarrayitem_raw_float(array, index) - else: - raise NotImplementedError - - op_getarrayitem_raw_pure = op_getarrayitem_raw - - def op_getfield_gc(self, fielddescr, struct): - if fielddescr.typeinfo == REF: - return do_getfield_gc_ptr(struct, fielddescr.ofs) - elif fielddescr.typeinfo == INT: - return do_getfield_gc_int(struct, fielddescr.ofs) - elif fielddescr.typeinfo == FLOAT: - return do_getfield_gc_float(struct, fielddescr.ofs) - else: - raise NotImplementedError - - op_getfield_gc_pure = op_getfield_gc - - def op_getfield_raw(self, fielddescr, struct): - if fielddescr.typeinfo == REF: - return do_getfield_raw_ptr(struct, fielddescr.ofs) - elif fielddescr.typeinfo == INT: - return do_getfield_raw_int(struct, fielddescr.ofs) - elif fielddescr.typeinfo == FLOAT: - return do_getfield_raw_float(struct, fielddescr.ofs) - else: - raise NotImplementedError - - op_getfield_raw_pure = op_getfield_raw - - def op_raw_store(self, arraydescr, addr, offset, value): - if arraydescr.typeinfo == REF: - raise AssertionError("cannot store GC pointer in raw storage") - elif arraydescr.typeinfo == INT: - do_raw_store_int(addr, offset, arraydescr.ofs, value) - elif arraydescr.typeinfo == FLOAT: - do_raw_store_float(addr, offset, value) - else: - raise NotImplementedError - - def op_raw_load(self, arraydescr, addr, offset): - if arraydescr.typeinfo == REF: - raise AssertionError("cannot store GC pointer in raw storage") - elif arraydescr.typeinfo == INT: - return do_raw_load_int(addr, offset, arraydescr.ofs) - elif arraydescr.typeinfo == FLOAT: - return do_raw_load_float(addr, offset) - else: - raise NotImplementedError - - def op_new(self, size): - return do_new(size.ofs) - - def op_new_with_vtable(self, descr, vtable): - assert descr is None - descr = heaptracker.vtable2descr(self.cpu, vtable) - result = do_new(descr.ofs) - value = lltype.cast_opaque_ptr(rclass.OBJECTPTR, result) - value.typeptr = cast_from_int(rclass.CLASSTYPE, vtable) - return result - - def op_setarrayitem_gc(self, arraydescr, array, index, newvalue): - if arraydescr.typeinfo == REF: - do_setarrayitem_gc_ptr(array, index, newvalue) - elif arraydescr.typeinfo == INT: - do_setarrayitem_gc_int(array, index, newvalue) - elif arraydescr.typeinfo == FLOAT: - do_setarrayitem_gc_float(array, index, newvalue) - else: - raise NotImplementedError - - def op_setarrayitem_raw(self, arraydescr, array, index, newvalue): - if arraydescr.typeinfo == REF: - raise NotImplementedError("setarrayitem_raw <- gcref") - elif arraydescr.typeinfo == INT: - do_setarrayitem_raw_int(array, index, newvalue, arraydescr.ofs) - elif arraydescr.typeinfo == FLOAT: - do_setarrayitem_raw_float(array, index, newvalue) - else: - raise NotImplementedError - - def op_getinteriorfield_gc(self, descr, array, index): - if descr.typeinfo == REF: - return do_getinteriorfield_gc_ptr(array, index, descr.ofs) - elif descr.typeinfo == INT: - return do_getinteriorfield_gc_int(array, index, descr.ofs) - elif descr.typeinfo == FLOAT: - return do_getinteriorfield_gc_float(array, index, descr.ofs) - else: - raise NotImplementedError - - def op_getinteriorfield_raw(self, descr, array, index): - if descr.typeinfo == REF: - return do_getinteriorfield_raw_ptr(array, index, descr.width, descr.ofs) - elif descr.typeinfo == INT: - return do_getinteriorfield_raw_int(array, index, descr.width, descr.ofs) - elif descr.typeinfo == FLOAT: - return do_getinteriorfield_raw_float(array, index, descr.width, descr.ofs) - else: - raise NotImplementedError - - def op_setinteriorfield_gc(self, descr, array, index, newvalue): - if descr.typeinfo == REF: - return do_setinteriorfield_gc_ptr(array, index, descr.ofs, - newvalue) - elif descr.typeinfo == INT: - return do_setinteriorfield_gc_int(array, index, descr.ofs, - newvalue) - elif descr.typeinfo == FLOAT: - return do_setinteriorfield_gc_float(array, index, descr.ofs, - newvalue) - else: - raise NotImplementedError - - def op_setinteriorfield_raw(self, descr, array, index, newvalue): - if descr.typeinfo == REF: - return do_setinteriorfield_raw_ptr(array, index, newvalue, descr.width, descr.ofs) - elif descr.typeinfo == INT: - return do_setinteriorfield_raw_int(array, index, newvalue, descr.width, descr.ofs) - elif descr.typeinfo == FLOAT: - return do_setinteriorfield_raw_float(array, index, newvalue, descr.width, descr.ofs) - else: - raise NotImplementedError - - def op_setfield_gc(self, fielddescr, struct, newvalue): - if fielddescr.typeinfo == REF: - do_setfield_gc_ptr(struct, fielddescr.ofs, newvalue) - elif fielddescr.typeinfo == INT: - do_setfield_gc_int(struct, fielddescr.ofs, newvalue) - elif fielddescr.typeinfo == FLOAT: - do_setfield_gc_float(struct, fielddescr.ofs, newvalue) - else: - raise NotImplementedError - - def op_setfield_raw(self, fielddescr, struct, newvalue): - if fielddescr.typeinfo == REF: - do_setfield_raw_ptr(struct, fielddescr.ofs, newvalue) - elif fielddescr.typeinfo == INT: - do_setfield_raw_int(struct, fielddescr.ofs, newvalue) - elif fielddescr.typeinfo == FLOAT: - do_setfield_raw_float(struct, fielddescr.ofs, newvalue) - else: - raise NotImplementedError - - def op_call(self, calldescr, func, *args): - effectinfo = calldescr.get_extra_info() - if effectinfo is not None and hasattr(effectinfo, 'oopspecindex'): - oopspecindex = effectinfo.oopspecindex - if oopspecindex == EffectInfo.OS_MATH_SQRT: - return do_math_sqrt(args[0]) - return self._do_call(calldescr, func, args, call_with_llptr=False) - - def op_call_release_gil(self, calldescr, func, *args): - return self._do_call(calldescr, func, args, call_with_llptr=True) - - def _do_call(self, calldescr, func, args, call_with_llptr): - global _last_exception - assert _last_exception is None, "exception left behind" - assert _call_args_i == _call_args_r == _call_args_f == [] - args_in_order = [] - for x in args: - T = lltype.typeOf(x) - if T is lltype.Signed: - args_in_order.append('i') - _call_args_i.append(x) - elif T == llmemory.GCREF: - args_in_order.append('r') - _call_args_r.append(x) - elif T is longlong.FLOATSTORAGE: - args_in_order.append('f') - _call_args_f.append(x) - else: - raise TypeError(x) - try: - return _do_call_common(func, args_in_order, calldescr, - call_with_llptr) - except LLException, lle: - _last_exception = lle - d = {'v': None, - REF: lltype.nullptr(llmemory.GCREF.TO), - INT: 0, - FLOAT: 0.0} - return d[calldescr.typeinfo] - - def op_cond_call_gc_wb(self, descr, a, b): - py.test.skip("cond_call_gc_wb not supported") - - def op_cond_call_gc_wb_array(self, descr, a, b, c): - py.test.skip("cond_call_gc_wb_array not supported") - - def op_oosend(self, descr, obj, *args): - raise NotImplementedError("oosend for lltype backend??") - - op_oosend_pure = op_oosend - - def op_new_array(self, arraydescr, count): - return do_new_array(arraydescr.ofs, count) - - def op_force_token(self, descr): - opaque_frame = _to_opaque(self) - return llmemory.cast_ptr_to_adr(opaque_frame) - - def op_read_timestamp(self, descr): - return read_timestamp() - - def op_call_may_force(self, calldescr, func, *args): - assert not self._forced - self._may_force = self.opindex - try: - return self.op_call(calldescr, func, *args) - finally: - self._may_force = -1 - - def op_call_assembler(self, wref_loop_token, *args): - if we_are_translated(): - raise ValueError("CALL_ASSEMBLER not supported") - return self._do_call_assembler(wref_loop_token, *args) - - def _do_call_assembler(self, wref_loop_token, *args): - global _last_exception - loop_token = wref_loop_token() - assert loop_token, "CALL_ASSEMBLER to a target that already died" - ctl = loop_token.compiled_loop_token - if hasattr(ctl, 'redirected'): - return self._do_call_assembler(ctl.redirected, *args) - assert not self._forced - self._may_force = self.opindex - try: - inpargs = _from_opaque(ctl.compiled_version).inputargs - assert len(inpargs) == len(args) - for i, inparg in enumerate(inpargs): - TYPE = inparg.concretetype - if TYPE is lltype.Signed: - set_future_value_int(i, args[i]) - elif isinstance(TYPE, lltype.Ptr): - set_future_value_ref(i, args[i]) - elif TYPE is longlong.FLOATSTORAGE: - set_future_value_float(i, args[i]) - else: - raise Exception("Nonsense type %s" % TYPE) - - failindex = self.cpu._execute_token(loop_token) - jd = loop_token.outermost_jitdriver_sd - assert jd is not None, ("call_assembler(): the loop_token needs " - "to have 'outermost_jitdriver_sd'") - if jd.index_of_virtualizable != -1: - vable = args[jd.index_of_virtualizable] - else: - vable = lltype.nullptr(llmemory.GCREF.TO) - # - # Emulate the fast path - if failindex == self.cpu.done_with_this_frame_int_v: - reset_vable(jd, vable) - return self.cpu.get_latest_value_int(0) - if failindex == self.cpu.done_with_this_frame_ref_v: - reset_vable(jd, vable) - return self.cpu.get_latest_value_ref(0) - if failindex == self.cpu.done_with_this_frame_float_v: - reset_vable(jd, vable) - return self.cpu.get_latest_value_float(0) - if failindex == self.cpu.done_with_this_frame_void_v: - reset_vable(jd, vable) - return None - # - assembler_helper_ptr = jd.assembler_helper_adr.ptr # fish - try: - return assembler_helper_ptr(failindex, vable) - except LLException, lle: - assert _last_exception is None, "exception left behind" - _last_exception = lle - # fish op - op = self.loop.operations[self.opindex] - if op.result is not None: - return 0 - finally: - self._may_force = -1 - - def op_guard_not_forced(self, descr): - forced = self._forced - self._forced = False - if forced: - raise GuardFailed - - def op_guard_not_invalidated(self, descr): - if self.loop.invalid: - raise GuardFailed - -class OOFrame(Frame): - - OPHANDLERS = [None] * (rop._LAST+1) - - def op_new_with_vtable(self, descr, vtable): - assert descr is None - typedescr = get_class_size(self.memocast, vtable) - return ootype.cast_to_object(ootype.new(typedescr.TYPE)) - - def op_new_array(self, typedescr, count): - res = ootype.oonewarray(typedescr.ARRAY, count) - return ootype.cast_to_object(res) - - def op_getfield_gc(self, fielddescr, obj): - TYPE = fielddescr.TYPE - fieldname = fielddescr.fieldname - _, T = TYPE._lookup_field(fieldname) - obj = ootype.cast_from_object(TYPE, obj) - res = getattr(obj, fieldname) - if isinstance(T, ootype.OOType): - return ootype.cast_to_object(res) - return res - - op_getfield_gc_pure = op_getfield_gc - - def op_setfield_gc(self, fielddescr, obj, newvalue): - TYPE = fielddescr.TYPE - fieldname = fielddescr.fieldname - _, T = TYPE._lookup_field(fieldname) - obj = ootype.cast_from_object(TYPE, obj) - if isinstance(ootype.typeOf(newvalue), ootype.OOType): - newvalue = ootype.cast_from_object(T, newvalue) - elif isinstance(T, lltype.Primitive): - newvalue = lltype.cast_primitive(T, newvalue) - setattr(obj, fieldname, newvalue) - - def op_getarrayitem_gc(self, typedescr, obj, index): - array = ootype.cast_from_object(typedescr.ARRAY, obj) - res = array.ll_getitem_fast(index) - if isinstance(typedescr.TYPE, ootype.OOType): - return ootype.cast_to_object(res) - return res - - op_getarrayitem_gc_pure = op_getarrayitem_gc - - def op_setarrayitem_gc(self, typedescr, obj, index, objnewvalue): - array = ootype.cast_from_object(typedescr.ARRAY, obj) - if ootype.typeOf(objnewvalue) == ootype.Object: - newvalue = ootype.cast_from_object(typedescr.TYPE, objnewvalue) - else: - newvalue = objnewvalue - array.ll_setitem_fast(index, newvalue) - - def op_arraylen_gc(self, typedescr, obj): - array = ootype.cast_from_object(typedescr.ARRAY, obj) - return array.ll_length() - - def op_call(self, calldescr, func, *args): - sm = ootype.cast_from_object(calldescr.FUNC, func) - newargs = cast_call_args(calldescr.FUNC.ARGS, args) - res = call_maybe_on_top_of_llinterp(sm, newargs) - if isinstance(calldescr.FUNC.RESULT, ootype.OOType): - return ootype.cast_to_object(res) - return res - - def op_oosend(self, descr, obj, *args): - METH = descr.METH - obj = ootype.cast_from_object(descr.SELFTYPE, obj) - meth = getattr(obj, descr.methname) - newargs = cast_call_args(METH.ARGS, args) - res = call_maybe_on_top_of_llinterp(meth, newargs) - if isinstance(METH.RESULT, ootype.OOType): - return ootype.cast_to_object(res) - return res - - op_oosend_pure = op_oosend - - def op_guard_class(self, _, value, expected_class): - value = ootype.cast_from_object(ootype.ROOT, value) - expected_class = ootype.cast_from_object(ootype.Class, expected_class) - if ootype.classof(value) is not expected_class: - raise GuardFailed - - def op_runtimenew(self, _, cls): - cls = ootype.cast_from_object(ootype.Class, cls) - res = ootype.runtimenew(cls) - return ootype.cast_to_object(res) - - def op_instanceof(self, typedescr, obj): - inst = ootype.cast_from_object(ootype.ROOT, obj) - return ootype.instanceof(inst, typedescr.TYPE) - - def op_subclassof(self, _, obj1, obj2): - cls1 = ootype.cast_from_object(ootype.Class, obj1) - cls2 = ootype.cast_from_object(ootype.Class, obj2) - return ootype.subclassof(cls1, cls2) - - def _cast_exception(self, exception): - return ootype.cast_from_object(ootype.Class, exception) - - def _issubclass(self, cls1, cls2): - return ootype.subclassof(cls1, cls2) - -# ____________________________________________________________ - -def cast_to_int(x): - TP = lltype.typeOf(x) - if isinstance(TP, lltype.Ptr): - return heaptracker.adr2int(llmemory.cast_ptr_to_adr(x)) - if TP == llmemory.Address: - return heaptracker.adr2int(x) - if TP is lltype.SingleFloat: - return longlong.singlefloat2int(x) - return lltype.cast_primitive(lltype.Signed, x) - -def cast_from_int(TYPE, x): - if isinstance(TYPE, lltype.Ptr): - if isinstance(x, (int, long, llmemory.AddressAsInt)): - x = llmemory.cast_int_to_adr(x) - if TYPE is rffi.VOIDP or ( - hasattr(TYPE.TO, '_hints') and - TYPE.TO._hints.get("uncast_on_llgraph")): - # assume that we want a "C-style" cast, without typechecking the value - return rffi.cast(TYPE, x) - return llmemory.cast_adr_to_ptr(x, TYPE) - elif TYPE == llmemory.Address: - if isinstance(x, (int, long, llmemory.AddressAsInt)): - x = llmemory.cast_int_to_adr(x) - assert lltype.typeOf(x) == llmemory.Address - return x - elif TYPE is lltype.SingleFloat: - assert lltype.typeOf(x) is lltype.Signed - return longlong.int2singlefloat(x) - else: - if lltype.typeOf(x) == llmemory.Address: - x = heaptracker.adr2int(x) - return lltype.cast_primitive(TYPE, x) - -def cast_to_ptr(x): - assert isinstance(lltype.typeOf(x), lltype.Ptr) - return lltype.cast_opaque_ptr(llmemory.GCREF, x) - -def cast_from_ptr(TYPE, x): - return lltype.cast_opaque_ptr(TYPE, x) - -def cast_to_floatstorage(x): - if isinstance(x, float): - return longlong.getfloatstorage(x) # common case - if IS_32_BIT: - assert longlong.supports_longlong - if isinstance(x, r_longlong): - return x - if isinstance(x, r_ulonglong): - return rffi.cast(lltype.SignedLongLong, x) - raise TypeError(type(x)) - -def cast_from_floatstorage(TYPE, x): - assert isinstance(x, longlong.r_float_storage) - if TYPE is lltype.Float: - return longlong.getrealfloat(x) - if longlong.is_longlong(TYPE): - return rffi.cast(TYPE, x) - raise TypeError(TYPE) - - -def new_frame(is_oo, cpu): - if is_oo: - frame = OOFrame(cpu) - else: - frame = Frame(cpu) - return _to_opaque(frame) - -_future_values = [] - -def frame_clear(frame, loop): - frame = _from_opaque(frame) - loop = _from_opaque(loop) - assert len(_future_values) == len(loop.inputargs) - frame.loop = loop - frame.env = {} - for i in range(len(loop.inputargs)): - expected_type = loop.inputargs[i].concretetype - assert lltype.typeOf(_future_values[i]) == expected_type - frame.env[loop.inputargs[i]] = _future_values[i] - del _future_values[:] - -def set_future_value_int(index, value): - assert lltype.typeOf(value) is lltype.Signed - set_future_value_ref(index, value) - -def set_future_value_float(index, value): - assert isinstance(value, longlong.r_float_storage) - set_future_value_ref(index, value) - -def set_future_value_ref(index, value): - del _future_values[index:] - assert len(_future_values) == index - _future_values.append(value) - -def frame_execute(frame): - frame = _from_opaque(frame) - if frame.verbose: - values = [frame.env[v] for v in frame.loop.inputargs] - log.trace('Entering CPU frame <- %r' % (values,)) - try: - result = frame.execute() - if frame.verbose: - log.trace('Leaving CPU frame -> #%d' % (result,)) - frame.log_progress() - except Exception, e: - log.ERROR('%s in CPU frame: %s' % (e.__class__.__name__, e)) - # Only invoke pdb when io capturing is not on otherwise py.io complains. - if py.test.config.option.capture == 'no': - import sys, pdb - pdb.post_mortem(sys.exc_info()[2]) - raise - del frame.env - return result - -def frame_int_getvalue(frame, num): - frame = _from_opaque(frame) - assert num >= 0 - x = frame.fail_args[num] - assert lltype.typeOf(x) is lltype.Signed - return x - -def frame_float_getvalue(frame, num): - frame = _from_opaque(frame) - assert num >= 0 - x = frame.fail_args[num] - assert lltype.typeOf(x) is longlong.FLOATSTORAGE - return x - -def frame_ptr_getvalue(frame, num): - frame = _from_opaque(frame) - assert num >= 0 - x = frame.fail_args[num] - assert lltype.typeOf(x) == llmemory.GCREF - return x - -def frame_get_value_count(frame): - frame = _from_opaque(frame) - return len(frame.fail_args) - -def frame_clear_latest_values(frame, count): - frame = _from_opaque(frame) - assert count == len(frame.fail_args) - del frame.fail_args - -_last_exception = None - -def grab_exc_value(): - global _last_exception - if _last_exception is not None: - result = _last_exception.args[1] - _last_exception = None - return lltype.cast_opaque_ptr(llmemory.GCREF, result) - else: - return lltype.nullptr(llmemory.GCREF.TO) - -##_pseudo_exceptions = {} - -##def _get_error(Class): -## if _llinterp.typer is not None: -## llframe = _llinterp.frame_class(None, None, _llinterp) -## try: -## llframe.make_llexception(Class()) -## except LLException, e: -## return e -## else: -## assert 0, "should have raised" -## else: -## # for tests, a random emulated ll_inst will do -## if Class not in _pseudo_exceptions: -## ll_inst = lltype.malloc(rclass.OBJECT, zero=True) -## ll_inst.typeptr = lltype.malloc(rclass.OBJECT_VTABLE, -## immortal=True) -## _pseudo_exceptions[Class] = LLException(ll_inst.typeptr, ll_inst) -## return _pseudo_exceptions[Class] - -##def get_overflow_error_value(): -## return lltype.cast_opaque_ptr(llmemory.GCREF, -## _get_error(OverflowError).args[1]) - -def force(opaque_frame): - frame = _from_opaque(opaque_frame) - assert not frame._forced - frame._forced = True - assert frame._may_force >= 0 - call_op = frame.loop.operations[frame._may_force] - guard_op = frame.loop.operations[frame._may_force+1] - opnum = call_op.opnum - assert opnum == rop.CALL_MAY_FORCE or opnum == rop.CALL_ASSEMBLER - frame._populate_fail_args(guard_op, skip=call_op.result) - return frame.fail_index - -def get_forced_token_frame(force_token): - opaque_frame = llmemory.cast_adr_to_ptr(force_token, - lltype.Ptr(_TO_OPAQUE[Frame])) - return opaque_frame - -def get_frame_forced_token(opaque_frame): - return llmemory.cast_ptr_to_adr(opaque_frame) - -##def cast_adr_to_int(memocast, adr): -## # xxx slow -## assert lltype.typeOf(adr) == llmemory.Address -## memocast = _from_opaque(memocast) -## addresses = memocast.addresses -## for i in xrange(len(addresses)-1, -1, -1): -## if addresses[i] == adr: -## return i -## i = len(addresses) -## addresses.append(adr) -## return i - -##def cast_int_to_adr(memocast, int): -## memocast = _from_opaque(memocast) -## assert 0 <= int < len(memocast.addresses) -## return memocast.addresses[int] - -##def get_class_size(memocast, vtable): -## memocast = _from_opaque(memocast) -## return memocast.vtable_to_size[vtable] - -##def set_class_size(memocast, vtable, size): -## memocast = _from_opaque(memocast) -## memocast.vtable_to_size[vtable] = size - -class GuardFailed(Exception): - pass - -# ____________________________________________________________ - - -def do_same_as(x): - return x - -def do_arraylen_gc(arraydescr, array): - array = array._obj.container - return array.getlength() - -def do_strlen(string): - str = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string) - return len(str.chars) - -def do_strgetitem(string, index): - str = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string) - return ord(str.chars[index]) - -def do_unicodelen(string): - uni = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), string) - return len(uni.chars) - -def do_unicodegetitem(string, index): - uni = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), string) - return ord(uni.chars[index]) - -def do_getarrayitem_gc_int(array, index): - array = array._obj.container - return cast_to_int(array.getitem(index)) - -def do_getarrayitem_raw_int(array, index, itemsize): - array = array.adr.ptr - ITEMTYPE = lltype.typeOf(array).TO.OF - TYPE = symbolic.Size2Type[itemsize] - if TYPE.OF != ITEMTYPE: - array = rffi.cast(lltype.Ptr(TYPE), array) - return cast_to_int(array._obj.getitem(index)) - -def do_getarrayitem_gc_float(array, index): - array = array._obj.container - return cast_to_floatstorage(array.getitem(index)) - -def do_getarrayitem_raw_float(array, index): - array = maybe_uncast(FLOAT_ARRAY_TP, array.adr.ptr) - return cast_to_floatstorage(array._obj.getitem(index)) - -def do_getarrayitem_gc_ptr(array, index): - array = array._obj.container - return cast_to_ptr(array.getitem(index)) - -def _getfield_gc(struct, fieldnum): - STRUCT, fieldname = symbolic.TokenToField[fieldnum] - ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct) - return getattr(ptr, fieldname) - -def do_getfield_gc_int(struct, fieldnum): - return cast_to_int(_getfield_gc(struct, fieldnum)) - -def do_getfield_gc_float(struct, fieldnum): - return cast_to_floatstorage(_getfield_gc(struct, fieldnum)) - -def do_getfield_gc_ptr(struct, fieldnum): - return cast_to_ptr(_getfield_gc(struct, fieldnum)) - -def _getinteriorfield_gc(struct, fieldnum): - STRUCT, fieldname = symbolic.TokenToField[fieldnum] - return getattr(struct, fieldname) - -def do_getinteriorfield_gc_int(array, index, fieldnum): - struct = array._obj.container.getitem(index) - return cast_to_int(_getinteriorfield_gc(struct, fieldnum)) - -def do_getinteriorfield_gc_float(array, index, fieldnum): - struct = array._obj.container.getitem(index) - return cast_to_floatstorage(_getinteriorfield_gc(struct, fieldnum)) - -def do_getinteriorfield_gc_ptr(array, index, fieldnum): - struct = array._obj.container.getitem(index) - return cast_to_ptr(_getinteriorfield_gc(struct, fieldnum)) - -def _getfield_raw(struct, fieldnum): - STRUCT, fieldname = symbolic.TokenToField[fieldnum] - ptr = cast_from_int(lltype.Ptr(STRUCT), struct) - return getattr(ptr, fieldname) - -def do_getfield_raw_int(struct, fieldnum): - return cast_to_int(_getfield_raw(struct, fieldnum)) - -def do_getfield_raw_float(struct, fieldnum): - return cast_to_floatstorage(_getfield_raw(struct, fieldnum)) - -def do_getfield_raw_ptr(struct, fieldnum): - return cast_to_ptr(_getfield_raw(struct, fieldnum)) - -def do_raw_load_int(struct, offset, descrofs): - TYPE = symbolic.Size2Type[descrofs] - ll_p = rffi.cast(rffi.CCHARP, struct) - ll_p = rffi.cast(lltype.Ptr(TYPE), rffi.ptradd(ll_p, offset)) - value = ll_p[0] - return rffi.cast(lltype.Signed, value) - -def do_raw_load_float(struct, offset): - ll_p = rffi.cast(rffi.CCHARP, struct) - ll_p = rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), - rffi.ptradd(ll_p, offset)) - value = ll_p[0] - return value - -def do_raw_store_int(struct, offset, descrofs, value): - TYPE = symbolic.Size2Type[descrofs] - ll_p = rffi.cast(rffi.CCHARP, struct) - ll_p = rffi.cast(lltype.Ptr(TYPE), rffi.ptradd(ll_p, offset)) - ll_p[0] = rffi.cast(TYPE.OF, value) - -def do_raw_store_float(struct, offset, value): - ll_p = rffi.cast(rffi.CCHARP, struct) - ll_p = rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), - rffi.ptradd(ll_p, offset)) - ll_p[0] = value - -def do_new(size): - TYPE = symbolic.Size2Type[size] - x = lltype.malloc(TYPE, zero=True) - return cast_to_ptr(x) - -def do_new_array(arraynum, count): - TYPE = symbolic.Size2Type[arraynum] - assert count >= 0 # explode if it's not - x = lltype.malloc(TYPE, count, zero=True) - return cast_to_ptr(x) - -def do_setarrayitem_gc_int(array, index, newvalue): - array = array._obj.container - ITEMTYPE = lltype.typeOf(array).OF - newvalue = cast_from_int(ITEMTYPE, newvalue) - array.setitem(index, newvalue) - -def do_setarrayitem_raw_int(array, index, newvalue, itemsize): - array = array.adr.ptr - ITEMTYPE = lltype.typeOf(array).TO.OF - TYPE = symbolic.Size2Type[itemsize] - if TYPE.OF != ITEMTYPE: - array = rffi.cast(lltype.Ptr(TYPE), array) - newvalue = cast_from_int(TYPE.OF, newvalue) - array._obj.setitem(index, newvalue) - -def do_setarrayitem_gc_float(array, index, newvalue): - array = array._obj.container - ITEMTYPE = lltype.typeOf(array).OF - newvalue = cast_from_floatstorage(ITEMTYPE, newvalue) - array.setitem(index, newvalue) - - -def do_setarrayitem_raw_float(array, index, newvalue): - array = maybe_uncast(FLOAT_ARRAY_TP, array.adr.ptr) - ITEMTYPE = lltype.typeOf(array).TO.OF - newvalue = cast_from_floatstorage(ITEMTYPE, newvalue) - array._obj.setitem(index, newvalue) - -def do_setarrayitem_gc_ptr(array, index, newvalue): - array = array._obj.container - ITEMTYPE = lltype.typeOf(array).OF - newvalue = cast_from_ptr(ITEMTYPE, newvalue) - array.setitem(index, newvalue) - -def new_setfield_gc(cast_func): - def do_setfield_gc(struct, fieldnum, newvalue): - STRUCT, fieldname = symbolic.TokenToField[fieldnum] - ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct) - FIELDTYPE = getattr(STRUCT, fieldname) - newvalue = cast_func(FIELDTYPE, newvalue) - setattr(ptr, fieldname, newvalue) - return do_setfield_gc -do_setfield_gc_int = new_setfield_gc(cast_from_int) -do_setfield_gc_float = new_setfield_gc(cast_from_floatstorage) -do_setfield_gc_ptr = new_setfield_gc(cast_from_ptr) - -def new_setinteriorfield_gc(cast_func): - def do_setinteriorfield_gc(array, index, fieldnum, newvalue): - STRUCT, fieldname = symbolic.TokenToField[fieldnum] - struct = array._obj.container.getitem(index) - FIELDTYPE = getattr(STRUCT, fieldname) - setattr(struct, fieldname, cast_func(FIELDTYPE, newvalue)) - return do_setinteriorfield_gc -do_setinteriorfield_gc_int = new_setinteriorfield_gc(cast_from_int) -do_setinteriorfield_gc_float = new_setinteriorfield_gc(cast_from_floatstorage) -do_setinteriorfield_gc_ptr = new_setinteriorfield_gc(cast_from_ptr) - -def do_setfield_raw_int(struct, fieldnum, newvalue): - STRUCT, fieldname = symbolic.TokenToField[fieldnum] - ptr = cast_from_int(lltype.Ptr(STRUCT), struct) - FIELDTYPE = getattr(STRUCT, fieldname) - newvalue = cast_from_int(FIELDTYPE, newvalue) - setattr(ptr, fieldname, newvalue) - -def do_setfield_raw_float(struct, fieldnum, newvalue): - STRUCT, fieldname = symbolic.TokenToField[fieldnum] - ptr = cast_from_int(lltype.Ptr(STRUCT), struct) - FIELDTYPE = getattr(STRUCT, fieldname) - newvalue = cast_from_floatstorage(FIELDTYPE, newvalue) - setattr(ptr, fieldname, newvalue) - -def do_setfield_raw_ptr(struct, fieldnum, newvalue): - STRUCT, fieldname = symbolic.TokenToField[fieldnum] - ptr = cast_from_int(lltype.Ptr(STRUCT), struct) - FIELDTYPE = getattr(STRUCT, fieldname) - newvalue = cast_from_ptr(FIELDTYPE, newvalue) - setattr(ptr, fieldname, newvalue) - -def do_newstr(length): - x = rstr.mallocstr(length) - return cast_to_ptr(x) - -def do_newunicode(length): - return cast_to_ptr(rstr.mallocunicode(length)) - -def do_strsetitem(string, index, newvalue): - str = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string) - str.chars[index] = chr(newvalue) - -def do_unicodesetitem(string, index, newvalue): - uni = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), string) - uni.chars[index] = unichr(newvalue) - -def do_copystrcontent(src, dst, srcstart, dststart, length): - src = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), src) - dst = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), dst) - assert 0 <= srcstart <= srcstart + length <= len(src.chars) - assert 0 <= dststart <= dststart + length <= len(dst.chars) - rstr.copy_string_contents(src, dst, srcstart, dststart, length) - -def do_copyunicodecontent(src, dst, srcstart, dststart, length): - src = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), src) - dst = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), dst) - assert 0 <= srcstart <= srcstart + length <= len(src.chars) - assert 0 <= dststart <= dststart + length <= len(dst.chars) - rstr.copy_unicode_contents(src, dst, srcstart, dststart, length) - -def do_math_sqrt(value): - import math - y = cast_from_floatstorage(lltype.Float, value) - x = math.sqrt(y) - return cast_to_floatstorage(x) - -# ---------- call ---------- - -_call_args_i = [] -_call_args_r = [] -_call_args_f = [] - -def do_call_pushint(x): - _call_args_i.append(x) - -def do_call_pushptr(x): - _call_args_r.append(x) - -def do_call_pushfloat(x): - _call_args_f.append(x) - -kind2TYPE = { - 'i': lltype.Signed, - 'f': lltype.Float, - 'L': lltype.SignedLongLong, - 'S': lltype.SingleFloat, - 'v': lltype.Void, - } - -def _do_call_common(f, args_in_order=None, calldescr=None, - call_with_llptr=False): - ptr = llmemory.cast_int_to_adr(f).ptr - PTR = lltype.typeOf(ptr) - if PTR == rffi.VOIDP: - # it's a pointer to a C function, so we don't have a precise - # signature: create one from the descr - assert call_with_llptr is True - ARGS = map(kind2TYPE.get, calldescr.arg_types) - RESULT = kind2TYPE[calldescr.typeinfo] - FUNC = lltype.FuncType(ARGS, RESULT) - func_to_call = rffi.cast(lltype.Ptr(FUNC), ptr) - else: - assert call_with_llptr is False - FUNC = PTR.TO - ARGS = FUNC.ARGS - func_to_call = ptr._obj._callable - args = cast_call_args(ARGS, _call_args_i, _call_args_r, _call_args_f, - args_in_order) - del _call_args_i[:] - del _call_args_r[:] - del _call_args_f[:] - assert len(ARGS) == len(args) - if hasattr(ptr._obj, 'graph'): - llinterp = _llinterp # it's a global set here by CPU.__init__() - result = llinterp.eval_graph(ptr._obj.graph, args) - # ^^^ may raise, in which case we get an LLException - else: - result = func_to_call(*args) - return result - -def do_call_void(f): - _do_call_common(f) - -def do_call_int(f): - x = _do_call_common(f) - return cast_to_int(x) - -def do_call_float(f): - x = _do_call_common(f) - return cast_to_floatstorage(x) - -def do_call_ptr(f): - x = _do_call_common(f) - return cast_to_ptr(x) - -def cast_call_args(ARGS, args_i, args_r, args_f, args_in_order=None): - argsiter_i = iter(args_i) - argsiter_r = iter(args_r) - argsiter_f = iter(args_f) - if args_in_order is not None: - orderiter = iter(args_in_order) - args = [] - for TYPE in ARGS: - if TYPE is lltype.Void: - x = None - else: - if isinstance(TYPE, ootype.OOType): - if args_in_order is not None: - n = orderiter.next() - assert n == 'r' - x = argsiter_r.next() - x = ootype.cast_from_object(TYPE, x) - elif isinstance(TYPE, lltype.Ptr) and TYPE.TO._gckind == 'gc': - if args_in_order is not None: - n = orderiter.next() - assert n == 'r' - x = argsiter_r.next() - x = cast_from_ptr(TYPE, x) - elif TYPE is lltype.Float or longlong.is_longlong(TYPE): - if args_in_order is not None: - n = orderiter.next() - assert n == 'f' - x = argsiter_f.next() - x = cast_from_floatstorage(TYPE, x) - else: - if args_in_order is not None: - n = orderiter.next() - assert n == 'i' - x = argsiter_i.next() - x = cast_from_int(TYPE, x) - args.append(x) - assert list(argsiter_i) == [] - assert list(argsiter_r) == [] - assert list(argsiter_f) == [] - return args - - -# for ootype meth and staticmeth -def call_maybe_on_top_of_llinterp(meth, args): - global _last_exception - if isinstance(meth, ootype._bound_meth): - mymethod = meth.meth - myargs = [meth.inst] + list(args) - else: - mymethod = meth - myargs = args - try: - if hasattr(mymethod, 'graph'): - llinterp = _llinterp # it's a global set here by CPU.__init__() - result = llinterp.eval_graph(mymethod.graph, myargs) - else: - result = meth(*args) - except XXX-LLException, e: - _last_exception = e - result = get_err_result_for_type(mymethod._TYPE.RESULT) - return result - -def get_err_result_for_type(T): - if T is ootype.Void: - return None - elif isinstance(T, ootype.OOType): - return ootype.null(T) - else: - return 0 - -def reset_vable(jd, vable): - if jd.index_of_virtualizable != -1: - fielddescr = jd.vable_token_descr - do_setfield_gc_int(vable, fielddescr.ofs, 0) - -def redirect_call_assembler(cpu, oldlooptoken, newlooptoken): - oldclt = oldlooptoken.compiled_loop_token - newclt = newlooptoken.compiled_loop_token - OLD = _from_opaque(oldclt.compiled_version).getargtypes() - NEW = _from_opaque(newclt.compiled_version).getargtypes() - assert OLD == NEW - assert not hasattr(oldclt, 'redirected') - oldclt.redirected = weakref.ref(newlooptoken) - -# ____________________________________________________________ - - -def setannotation(func, annotation, specialize_as_constant=False): - - class Entry(ExtRegistryEntry): - "Annotation and specialization for calls to 'func'." - _about_ = func - - if annotation is None or isinstance(annotation, annmodel.SomeObject): - s_result_annotation = annotation - else: - def compute_result_annotation(self, *args_s): - return annotation(*args_s) - - if specialize_as_constant: - def specialize_call(self, hop): - llvalue = func(hop.args_s[0].const) - hop.exception_cannot_occur() - return hop.inputconst(lltype.typeOf(llvalue), llvalue) - else: - # specialize as direct_call - def specialize_call(self, hop): - ARGS = [r.lowleveltype for r in hop.args_r] - RESULT = hop.r_result.lowleveltype - if hop.rtyper.type_system.name == 'lltypesystem': - FUNCTYPE = lltype.FuncType(ARGS, RESULT) - funcptr = lltype.functionptr(FUNCTYPE, func.__name__, - _callable=func, _debugexc=True) - cfunc = hop.inputconst(lltype.Ptr(FUNCTYPE), funcptr) - else: - FUNCTYPE = ootype.StaticMethod(ARGS, RESULT) - sm = ootype._static_meth(FUNCTYPE, _name=func.__name__, _callable=func) - cfunc = hop.inputconst(FUNCTYPE, sm) - args_v = hop.inputargs(*hop.args_r) - hop.exception_is_here() - return hop.genop('direct_call', [cfunc] + args_v, hop.r_result) - - -COMPILEDLOOP = lltype.Ptr(lltype.OpaqueType("CompiledLoop")) -FRAME = lltype.Ptr(lltype.OpaqueType("Frame")) -OOFRAME = lltype.Ptr(lltype.OpaqueType("OOFrame")) - -_TO_OPAQUE[CompiledLoop] = COMPILEDLOOP.TO -_TO_OPAQUE[Frame] = FRAME.TO -_TO_OPAQUE[OOFrame] = OOFRAME.TO - -s_CompiledLoop = annmodel.SomePtr(COMPILEDLOOP) -s_Frame = annmodel.SomePtr(FRAME) - -if longlong.FLOATSTORAGE is lltype.Float: - s_FloatStorage = annmodel.SomeFloat() -elif longlong.FLOATSTORAGE is lltype.SignedLongLong: - s_FloatStorage = annmodel.SomeInteger(knowntype=longlong.r_float_storage) -else: - assert 0 - -setannotation(compile_start, s_CompiledLoop) -setannotation(compile_start_int_var, annmodel.SomeInteger()) -setannotation(compile_start_ref_var, annmodel.SomeInteger()) -setannotation(compile_start_float_var, annmodel.SomeInteger()) -setannotation(compile_started_vars, annmodel.s_None) -setannotation(compile_add, annmodel.s_None) -setannotation(compile_add_descr, annmodel.s_None) -setannotation(compile_add_descr_arg, annmodel.s_None) -setannotation(compile_add_target_token, annmodel.s_None) -setannotation(compile_add_var, annmodel.s_None) -setannotation(compile_add_int_const, annmodel.s_None) -setannotation(compile_add_ref_const, annmodel.s_None) -setannotation(compile_add_float_const, annmodel.s_None) -setannotation(compile_add_int_result, annmodel.SomeInteger()) -setannotation(compile_add_ref_result, annmodel.SomeInteger()) -setannotation(compile_add_float_result, annmodel.SomeInteger()) -setannotation(compile_add_jump_target, annmodel.s_None) -setannotation(compile_add_guard_jump_target, annmodel.s_None) -setannotation(compile_add_fail, annmodel.SomeInteger()) -setannotation(compile_add_fail_arg, annmodel.s_None) -setannotation(compile_redirect_fail, annmodel.s_None) -setannotation(mark_as_free, annmodel.s_None) - -setannotation(new_frame, s_Frame) -setannotation(frame_clear, annmodel.s_None) -setannotation(set_future_value_int, annmodel.s_None) -setannotation(set_future_value_ref, annmodel.s_None) -setannotation(set_future_value_float, annmodel.s_None) -setannotation(frame_execute, annmodel.SomeInteger()) -setannotation(frame_int_getvalue, annmodel.SomeInteger()) -setannotation(frame_ptr_getvalue, annmodel.SomePtr(llmemory.GCREF)) -setannotation(frame_float_getvalue, s_FloatStorage) -setannotation(frame_get_value_count, annmodel.SomeInteger()) -setannotation(frame_clear_latest_values, annmodel.s_None) - -setannotation(grab_exc_value, annmodel.SomePtr(llmemory.GCREF)) -setannotation(force, annmodel.SomeInteger()) -setannotation(get_forced_token_frame, s_Frame) -setannotation(get_frame_forced_token, annmodel.SomeAddress()) - -setannotation(do_arraylen_gc, annmodel.SomeInteger()) -setannotation(do_strlen, annmodel.SomeInteger()) -setannotation(do_strgetitem, annmodel.SomeInteger()) -setannotation(do_unicodelen, annmodel.SomeInteger()) -setannotation(do_unicodegetitem, annmodel.SomeInteger()) -setannotation(do_getarrayitem_gc_int, annmodel.SomeInteger()) -setannotation(do_getarrayitem_gc_ptr, annmodel.SomePtr(llmemory.GCREF)) -setannotation(do_getarrayitem_gc_float, s_FloatStorage) -setannotation(do_getarrayitem_raw_int, annmodel.SomeInteger()) -setannotation(do_getarrayitem_raw_float, s_FloatStorage) -setannotation(do_getfield_gc_int, annmodel.SomeInteger()) -setannotation(do_getfield_gc_ptr, annmodel.SomePtr(llmemory.GCREF)) -setannotation(do_getfield_gc_float, s_FloatStorage) -setannotation(do_getfield_raw_int, annmodel.SomeInteger()) -setannotation(do_getfield_raw_ptr, annmodel.SomePtr(llmemory.GCREF)) -setannotation(do_getfield_raw_float, s_FloatStorage) -setannotation(do_getinteriorfield_gc_int, annmodel.SomeInteger()) -setannotation(do_getinteriorfield_gc_ptr, annmodel.SomePtr(llmemory.GCREF)) -setannotation(do_getinteriorfield_gc_float, s_FloatStorage) -setannotation(do_raw_load_int, annmodel.SomeInteger()) -setannotation(do_new, annmodel.SomePtr(llmemory.GCREF)) -setannotation(do_new_array, annmodel.SomePtr(llmemory.GCREF)) -setannotation(do_setarrayitem_gc_int, annmodel.s_None) -setannotation(do_setarrayitem_gc_ptr, annmodel.s_None) -setannotation(do_setarrayitem_gc_float, annmodel.s_None) -setannotation(do_setarrayitem_raw_int, annmodel.s_None) -setannotation(do_setarrayitem_raw_float, annmodel.s_None) -setannotation(do_setfield_gc_int, annmodel.s_None) -setannotation(do_setfield_gc_ptr, annmodel.s_None) -setannotation(do_setfield_gc_float, annmodel.s_None) -setannotation(do_setfield_raw_int, annmodel.s_None) -setannotation(do_setfield_raw_ptr, annmodel.s_None) -setannotation(do_setfield_raw_float, annmodel.s_None) -setannotation(do_setinteriorfield_gc_int, annmodel.s_None) -setannotation(do_setinteriorfield_gc_ptr, annmodel.s_None) -setannotation(do_setinteriorfield_gc_float, annmodel.s_None) -setannotation(do_raw_store_int, annmodel.s_None) -setannotation(do_newstr, annmodel.SomePtr(llmemory.GCREF)) -setannotation(do_strsetitem, annmodel.s_None) -setannotation(do_newunicode, annmodel.SomePtr(llmemory.GCREF)) -setannotation(do_unicodesetitem, annmodel.s_None) -setannotation(do_call_pushint, annmodel.s_None) -setannotation(do_call_pushptr, annmodel.s_None) -setannotation(do_call_int, annmodel.SomeInteger()) -setannotation(do_call_ptr, annmodel.SomePtr(llmemory.GCREF)) -setannotation(do_call_float, s_FloatStorage) -setannotation(do_call_void, annmodel.s_None) From noreply at buildbot.pypy.org Sun Jan 27 13:35:43 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 27 Jan 2013 13:35:43 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fix Message-ID: <20130127123543.740B21C125A@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60520:0c439825d278 Date: 2013-01-27 14:35 +0200 http://bitbucket.org/pypy/pypy/changeset/0c439825d278/ Log: fix diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py --- a/rpython/jit/backend/x86/runner.py +++ b/rpython/jit/backend/x86/runner.py @@ -153,9 +153,7 @@ cast_ptr_to_int = staticmethod(cast_ptr_to_int) def force(self, addr_of_force_token): - descr = self.signedarraydescr - ofs = self.unpack_arraydescr(descr) - frame = rffi.cast(jitframe.JITFRAMEPTR, addr_of_force_token - ofs) + frame = rffi.cast(jitframe.JITFRAMEPTR, addr_of_force_token) frame.jf_descr = frame.jf_force_descr return lltype.cast_opaque_ptr(llmemory.GCREF, frame) From noreply at buildbot.pypy.org Sun Jan 27 14:09:21 2013 From: noreply at buildbot.pypy.org (arigo) Date: Sun, 27 Jan 2013 14:09:21 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: Change the vable_token and virtual_token fields to be GC references Message-ID: <20130127130921.53B911C039A@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: jitframe-on-heap Changeset: r60521:13a7d58be534 Date: 2013-01-27 14:09 +0100 http://bitbucket.org/pypy/pypy/changeset/13a7d58be534/ Log: Change the vable_token and virtual_token fields to be GC references to the live frame. diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py --- a/rpython/jit/backend/llgraph/runner.py +++ b/rpython/jit/backend/llgraph/runner.py @@ -582,7 +582,7 @@ class LLFrame(object): - _TYPE = lltype.Signed + _TYPE = llmemory.GCREF forced_deadframe = None overflow_flag = False @@ -595,6 +595,22 @@ for box, arg in zip(argboxes, args): self.setenv(box, arg) + def __eq__(self, other): + # this is here to avoid crashes in 'token == TOKEN_TRACING_RESCALL' + from rpython.jit.metainterp.virtualizable import TOKEN_NONE + from rpython.jit.metainterp.virtualizable import TOKEN_TRACING_RESCALL + if isinstance(other, LLFrame): + return self is other + if other == TOKEN_NONE or other == TOKEN_TRACING_RESCALL: + return False + assert 0 + + def __ne__(self, other): + return not (self == other) + + def _identityhash(self): + return hash(self) + def setenv(self, box, arg): if box.type == INT: # typecheck the result @@ -863,7 +879,8 @@ def reset_vable(jd, vable): if jd.index_of_virtualizable != -1: fielddescr = jd.vable_token_descr - self.cpu.bh_setfield_gc(vable, 0, fielddescr) + NULL = lltype.nullptr(llmemory.GCREF.TO) + self.cpu.bh_setfield_gc(vable, NULL, fielddescr) faildescr = self.cpu.get_latest_descr(pframe) if faildescr == self.cpu.done_with_this_frame_descr_int: reset_vable(jd, vable) diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py --- a/rpython/jit/backend/test/runner_test.py +++ b/rpython/jit/backend/test/runner_test.py @@ -2220,7 +2220,7 @@ values.append(self.cpu.get_int_value(deadframe, 1)) self.cpu.set_savedata_ref(deadframe, random_gcref) - FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Void) + FUNC = self.FuncType([llmemory.GCREF, lltype.Signed], lltype.Void) func_ptr = llhelper(lltype.Ptr(FUNC), maybe_force) funcbox = self.get_funcbox(self.cpu, func_ptr).constbox() calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, @@ -2228,7 +2228,7 @@ cpu = self.cpu i0 = BoxInt() i1 = BoxInt() - tok = BoxInt() + tok = BoxPtr() faildescr = BasicFailDescr(1) ops = [ ResOperation(rop.FORCE_TOKEN, [], tok), @@ -2265,7 +2265,7 @@ self.cpu.set_savedata_ref(deadframe, random_gcref) return 42 - FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Signed) + FUNC = self.FuncType([llmemory.GCREF, lltype.Signed], lltype.Signed) func_ptr = llhelper(lltype.Ptr(FUNC), maybe_force) funcbox = self.get_funcbox(self.cpu, func_ptr).constbox() calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, @@ -2274,7 +2274,7 @@ i0 = BoxInt() i1 = BoxInt() i2 = BoxInt() - tok = BoxInt() + tok = BoxPtr() faildescr = BasicFailDescr(1) ops = [ ResOperation(rop.FORCE_TOKEN, [], tok), @@ -2313,7 +2313,7 @@ self.cpu.set_savedata_ref(deadframe, random_gcref) return 42.5 - FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Float) + FUNC = self.FuncType([llmemory.GCREF, lltype.Signed], lltype.Float) func_ptr = llhelper(lltype.Ptr(FUNC), maybe_force) funcbox = self.get_funcbox(self.cpu, func_ptr).constbox() calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, @@ -2322,7 +2322,7 @@ i0 = BoxInt() i1 = BoxInt() f2 = BoxFloat() - tok = BoxInt() + tok = BoxPtr() faildescr = BasicFailDescr(1) ops = [ ResOperation(rop.FORCE_TOKEN, [], tok), @@ -3697,7 +3697,7 @@ values.append(self.cpu.get_int_value(deadframe, 0)) return 42 - FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Signed) + FUNC = self.FuncType([llmemory.GCREF, lltype.Signed], lltype.Signed) func_ptr = llhelper(lltype.Ptr(FUNC), maybe_force) funcbox = self.get_funcbox(self.cpu, func_ptr).constbox() calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, @@ -3705,7 +3705,7 @@ i0 = BoxInt() i1 = BoxInt() i2 = BoxInt() - tok = BoxInt() + tok = BoxPtr() faildescr = BasicFailDescr(23) ops = [ ResOperation(rop.FORCE_TOKEN, [], tok), diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -2954,7 +2954,7 @@ """ expected = """ [p1] - i0 = force_token() + p0 = force_token() jump(p1) """ self.optimize_loop(ops, expected) @@ -2969,12 +2969,12 @@ """ expected = """ [p1] - i0 = force_token() + p0 = force_token() p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) - setfield_gc(p2, i0, descr=virtualtokendescr) + setfield_gc(p2, p0, descr=virtualtokendescr) escape(p2) setfield_gc(p2, p1, descr=virtualforceddescr) - setfield_gc(p2, -3, descr=virtualtokendescr) + setfield_gc(p2, NULL, descr=virtualtokendescr) jump(p1) """ # XXX we should optimize a bit more the case of a nonvirtual. @@ -3000,10 +3000,10 @@ """ expected = """ [p0, i1] - i3 = force_token() + p3 = force_token() # p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) - setfield_gc(p2, i3, descr=virtualtokendescr) + setfield_gc(p2, p3, descr=virtualtokendescr) setfield_gc(p0, p2, descr=nextdescr) # call_may_force(i1, descr=mayforcevirtdescr) @@ -3015,7 +3015,7 @@ setfield_gc(p1b, 252, descr=valuedescr) setfield_gc(p1, p1b, descr=nextdescr) setfield_gc(p2, p1, descr=virtualforceddescr) - setfield_gc(p2, -3, descr=virtualtokendescr) + setfield_gc(p2, NULL, descr=virtualtokendescr) jump(p0, i1) """ self.optimize_loop(ops, expected) @@ -3039,10 +3039,10 @@ """ expected = """ [p0, i1] - i3 = force_token() + p3 = force_token() # p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) - setfield_gc(p2, i3, descr=virtualtokendescr) + setfield_gc(p2, p3, descr=virtualtokendescr) setfield_gc(p0, p2, descr=nextdescr) # call_may_force(i1, descr=mayforcevirtdescr) @@ -3054,7 +3054,7 @@ setfield_gc(p1b, i1, descr=valuedescr) setfield_gc(p1, p1b, descr=nextdescr) setfield_gc(p2, p1, descr=virtualforceddescr) - setfield_gc(p2, -3, descr=virtualtokendescr) + setfield_gc(p2, NULL, descr=virtualtokendescr) jump(p0, i1) """ # the point of this test is that 'i1' should show up in the fail_args @@ -3084,21 +3084,21 @@ """ expected = """ [p0, i1] - i3 = force_token() + p3 = force_token() call(i1, descr=nonwritedescr) - guard_no_exception(descr=fdescr) [i3, i1, p0] + guard_no_exception(descr=fdescr) [p3, i1, p0] setfield_gc(p0, NULL, descr=refdescr) jump(p0, i1) """ self.optimize_loop(ops, expected) - # the fail_args contain [i3, i1, p0]: - # - i3 is from the virtual expansion of p2 + # the fail_args contain [p3, i1, p0]: + # - p3 is from the virtual expansion of p2 # - i1 is from the virtual expansion of p1 # - p0 is from the extra pendingfields self.loop.inputargs[0].value = self.nodeobjvalue self.check_expanded_fail_descr('''p2, p1 p0.refdescr = p2 - where p2 is a jit_virtual_ref_vtable, virtualtokendescr=i3 + where p2 is a jit_virtual_ref_vtable, virtualtokendescr=p3 where p1 is a node_vtable, nextdescr=p1b where p1b is a node_vtable, valuedescr=i1 ''', rop.GUARD_NO_EXCEPTION) @@ -3116,13 +3116,13 @@ """ expected = """ [i1] - i3 = force_token() + p3 = force_token() p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) - setfield_gc(p2, i3, descr=virtualtokendescr) + setfield_gc(p2, p3, descr=virtualtokendescr) escape(p2) p1 = new_with_vtable(ConstClass(node_vtable)) setfield_gc(p2, p1, descr=virtualforceddescr) - setfield_gc(p2, -3, descr=virtualtokendescr) + setfield_gc(p2, NULL, descr=virtualtokendescr) call_may_force(i1, descr=mayforcevirtdescr) guard_not_forced() [] jump(i1) @@ -3141,12 +3141,12 @@ """ expected = """ [i1, p1] - i3 = force_token() + p3 = force_token() p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) - setfield_gc(p2, i3, descr=virtualtokendescr) + setfield_gc(p2, p3, descr=virtualtokendescr) escape(p2) setfield_gc(p2, p1, descr=virtualforceddescr) - setfield_gc(p2, -3, descr=virtualtokendescr) + setfield_gc(p2, NULL, descr=virtualtokendescr) call_may_force(i1, descr=mayforcevirtdescr) guard_not_forced() [i1] jump(i1, p1) @@ -3671,7 +3671,7 @@ i5 = int_gt(i4, i22) guard_false(i5) [] i6 = int_add(i4, 1) - i331 = force_token() + p331 = force_token() i7 = int_sub(i6, 1) setfield_gc(p0, i7, descr=valuedescr) jump(p0, i22) @@ -3682,7 +3682,7 @@ i2 = int_gt(i1, i22) guard_false(i2) [] i3 = int_add(i1, 1) - i331 = force_token() + p331 = force_token() jump(p0, i22) """ self.optimize_loop(ops, expected) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -3424,7 +3424,7 @@ """ expected = """ [p1] - i0 = force_token() + p0 = force_token() jump(p1) """ self.optimize_loop(ops, expected, expected) @@ -3439,12 +3439,12 @@ """ expected = """ [p1] - i0 = force_token() + p0 = force_token() p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) - setfield_gc(p2, i0, descr=virtualtokendescr) + setfield_gc(p2, p0, descr=virtualtokendescr) escape(p2) setfield_gc(p2, p1, descr=virtualforceddescr) - setfield_gc(p2, -3, descr=virtualtokendescr) + setfield_gc(p2, NULL, descr=virtualtokendescr) jump(p1) """ # XXX we should optimize a bit more the case of a nonvirtual. @@ -3470,10 +3470,10 @@ """ expected = """ [p0, i1] - i3 = force_token() + p3 = force_token() # p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) - setfield_gc(p2, i3, descr=virtualtokendescr) + setfield_gc(p2, p3, descr=virtualtokendescr) setfield_gc(p0, p2, descr=nextdescr) # call_may_force(i1, descr=mayforcevirtdescr) @@ -3485,7 +3485,7 @@ setfield_gc(p1b, 252, descr=valuedescr) setfield_gc(p1, p1b, descr=nextdescr) setfield_gc(p2, p1, descr=virtualforceddescr) - setfield_gc(p2, -3, descr=virtualtokendescr) + setfield_gc(p2, NULL, descr=virtualtokendescr) jump(p0, i1) """ self.optimize_loop(ops, expected, expected) @@ -3509,10 +3509,10 @@ """ expected = """ [p0, i1] - i3 = force_token() + p3 = force_token() # p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) - setfield_gc(p2, i3, descr=virtualtokendescr) + setfield_gc(p2, p3, descr=virtualtokendescr) setfield_gc(p0, p2, descr=nextdescr) # call_may_force(i1, descr=mayforcevirtdescr) @@ -3524,7 +3524,7 @@ setfield_gc(p1b, i1, descr=valuedescr) setfield_gc(p1, p1b, descr=nextdescr) setfield_gc(p2, p1, descr=virtualforceddescr) - setfield_gc(p2, -3, descr=virtualtokendescr) + setfield_gc(p2, NULL, descr=virtualtokendescr) jump(p0, i1) """ # the point of this test is that 'i1' should show up in the fail_args @@ -3555,31 +3555,31 @@ """ preamble = """ [p0, i1] - i3 = force_token() + p3 = force_token() call(i1, descr=nonwritedescr) - guard_no_exception(descr=fdescr) [i3, i1, p0] + guard_no_exception(descr=fdescr) [p3, i1, p0] setfield_gc(p0, NULL, descr=refdescr) escape() jump(p0, i1) """ expected = """ [p0, i1] - i3 = force_token() + p3 = force_token() call(i1, descr=nonwritedescr) - guard_no_exception(descr=fdescr2) [i3, i1, p0] + guard_no_exception(descr=fdescr2) [p3, i1, p0] setfield_gc(p0, NULL, descr=refdescr) escape() jump(p0, i1) """ self.optimize_loop(ops, expected, preamble) - # the fail_args contain [i3, i1, p0]: - # - i3 is from the virtual expansion of p2 + # the fail_args contain [p3, i1, p0]: + # - p3 is from the virtual expansion of p2 # - i1 is from the virtual expansion of p1 # - p0 is from the extra pendingfields #self.loop.inputargs[0].value = self.nodeobjvalue #self.check_expanded_fail_descr('''p2, p1 # p0.refdescr = p2 - # where p2 is a jit_virtual_ref_vtable, virtualtokendescr=i3 + # where p2 is a jit_virtual_ref_vtable, virtualtokendescr=p3 # where p1 is a node_vtable, nextdescr=p1b # where p1b is a node_vtable, valuedescr=i1 # ''', rop.GUARD_NO_EXCEPTION) @@ -3597,13 +3597,13 @@ """ expected = """ [i1] - i3 = force_token() + p3 = force_token() p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) - setfield_gc(p2, i3, descr=virtualtokendescr) + setfield_gc(p2, p3, descr=virtualtokendescr) escape(p2) p1 = new_with_vtable(ConstClass(node_vtable)) setfield_gc(p2, p1, descr=virtualforceddescr) - setfield_gc(p2, -3, descr=virtualtokendescr) + setfield_gc(p2, NULL, descr=virtualtokendescr) call_may_force(i1, descr=mayforcevirtdescr) guard_not_forced() [] jump(i1) @@ -3622,12 +3622,12 @@ """ expected = """ [i1, p1] - i3 = force_token() + p3 = force_token() p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) - setfield_gc(p2, i3, descr=virtualtokendescr) + setfield_gc(p2, p3, descr=virtualtokendescr) escape(p2) setfield_gc(p2, p1, descr=virtualforceddescr) - setfield_gc(p2, -3, descr=virtualtokendescr) + setfield_gc(p2, NULL, descr=virtualtokendescr) call_may_force(i1, descr=mayforcevirtdescr) guard_not_forced() [i1] jump(i1, p1) @@ -4312,14 +4312,14 @@ i5 = int_gt(i4, i22) guard_false(i5) [] i6 = int_add(i4, 1) - i331 = force_token() + p331 = force_token() i7 = int_sub(i6, 1) setfield_gc(p0, i7, descr=valuedescr) jump(p0, i22) """ expected = """ [p0, i22] - i331 = force_token() + p331 = force_token() jump(p0, i22) """ self.optimize_loop(ops, expected) diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -2,6 +2,7 @@ from rpython.jit.metainterp.executor import execute from rpython.jit.codewriter.heaptracker import vtable2descr from rpython.jit.metainterp.history import Const, ConstInt, BoxInt +from rpython.jit.metainterp.history import CONST_NULL, BoxPtr from rpython.jit.metainterp.optimizeopt import optimizer from rpython.jit.metainterp.optimizeopt.optimizer import OptValue, REMOVED from rpython.jit.metainterp.optimizeopt.util import (make_dispatcher_method, @@ -418,7 +419,7 @@ # but the point is that doing so does not force the original structure. op = ResOperation(rop.NEW_WITH_VTABLE, [c_cls], op.result) vrefvalue = self.make_virtual(c_cls, op.result, op) - tokenbox = BoxInt() + tokenbox = BoxPtr() self.emit_operation(ResOperation(rop.FORCE_TOKEN, [], tokenbox)) vrefvalue.setfield(descr_virtual_token, self.getvalue(tokenbox)) @@ -441,12 +442,12 @@ # - set 'forced' to point to the real object objbox = op.getarg(1) - if not self.optimizer.cpu.ts.CONST_NULL.same_constant(objbox): + if not CONST_NULL.same_constant(objbox): seo(ResOperation(rop.SETFIELD_GC, op.getarglist(), None, descr = vrefinfo.descr_forced)) - # - set 'virtual_token' to TOKEN_NONE - args = [op.getarg(0), ConstInt(vrefinfo.TOKEN_NONE)] + # - set 'virtual_token' to TOKEN_NONE (== NULL) + args = [op.getarg(0), CONST_NULL] seo(ResOperation(rop.SETFIELD_GC, args, None, descr=vrefinfo.descr_virtual_token)) # Note that in some cases the virtual in op.getarg(1) has been forced @@ -462,7 +463,7 @@ if vref.is_virtual(): tokenvalue = vref.getfield(vrefinfo.descr_virtual_token, None) if (tokenvalue is not None and tokenvalue.is_constant() and - tokenvalue.box.getint() == vrefinfo.TOKEN_NONE): + not tokenvalue.box.nonnull()): forcedvalue = vref.getfield(vrefinfo.descr_forced, None) if forcedvalue is not None and not forcedvalue.is_null(): self.make_equal_to(op.result, forcedvalue) diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -2283,7 +2283,7 @@ virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) vinfo.tracing_before_residual_call(virtualizable) # - force_token_box = history.BoxInt() + force_token_box = history.BoxPtr() self.history.record(rop.FORCE_TOKEN, [], force_token_box) self.history.record(rop.SETFIELD_GC, [virtualizable_box, force_token_box], @@ -2376,7 +2376,7 @@ self.virtualizable_boxes = virtualizable_boxes # just jumped away from assembler (case 4 in the comment in # virtualizable.py) into tracing (case 2); check that vable_token - # is and stays 0. Note the call to reset_vable_token() in + # is and stays NULL. Note the call to reset_vable_token() in # warmstate.py. virtualizable_box = self.virtualizable_boxes[-1] virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py --- a/rpython/jit/metainterp/resume.py +++ b/rpython/jit/metainterp/resume.py @@ -1110,7 +1110,7 @@ else: # just jumped away from assembler (case 4 in the comment in # virtualizable.py) into tracing (case 2); check that vable_token - # is and stays 0. Note the call to reset_vable_token() in + # is and stays NULL. Note the call to reset_vable_token() in # warmstate.py. assert not vinfo.is_token_nonnull_gcref(virtualizable) return vinfo.write_from_resume_data_partial(virtualizable, self, numb) diff --git a/rpython/jit/metainterp/test/test_recursive.py b/rpython/jit/metainterp/test/test_recursive.py --- a/rpython/jit/metainterp/test/test_recursive.py +++ b/rpython/jit/metainterp/test/test_recursive.py @@ -823,7 +823,8 @@ # at the level 2 is set to a non-zero value when doing the # call to the level 3 only. This used to fail when the test # is run via rpython.jit.backend.x86.test.test_recursive. - assert ll_subframe.vable_token == 0 + from rpython.jit.metainterp.virtualizable import TOKEN_NONE + assert ll_subframe.vable_token == TOKEN_NONE def main(codeno): frame = Frame() diff --git a/rpython/jit/metainterp/test/test_virtualizable.py b/rpython/jit/metainterp/test/test_virtualizable.py --- a/rpython/jit/metainterp/test/test_virtualizable.py +++ b/rpython/jit/metainterp/test/test_virtualizable.py @@ -41,7 +41,7 @@ XY = lltype.GcStruct( 'XY', ('parent', rclass.OBJECT), - ('vable_token', lltype.Signed), + ('vable_token', llmemory.GCREF), ('inst_x', lltype.Signed), ('inst_node', lltype.Ptr(LLtypeMixin.NODE)), hints = {'virtualizable2_accessor': FieldListAccessor()}) @@ -56,7 +56,7 @@ def setup(self): xy = lltype.malloc(self.XY) - xy.vable_token = 0 + xy.vable_token = lltype.nullptr(llmemory.GCREF.TO) xy.parent.typeptr = self.xy_vtable return xy @@ -206,7 +206,7 @@ XY2 = lltype.GcStruct( 'XY2', ('parent', rclass.OBJECT), - ('vable_token', lltype.Signed), + ('vable_token', llmemory.GCREF), ('inst_x', lltype.Signed), ('inst_l1', lltype.Ptr(lltype.GcArray(lltype.Signed))), ('inst_l2', lltype.Ptr(lltype.GcArray(lltype.Signed))), @@ -220,7 +220,7 @@ def setup2(self): xy2 = lltype.malloc(self.XY2) - xy2.vable_token = 0 + xy2.vable_token = lltype.nullptr(llmemory.GCREF.TO) xy2.parent.typeptr = self.xy2_vtable return xy2 @@ -393,7 +393,7 @@ def setup2sub(self): xy2 = lltype.malloc(self.XY2SUB) - xy2.parent.vable_token = 0 + xy2.parent.vable_token = lltype.nullptr(llmemory.GCREF.TO) xy2.parent.parent.typeptr = self.xy2_vtable return xy2 diff --git a/rpython/jit/metainterp/test/test_virtualref.py b/rpython/jit/metainterp/test/test_virtualref.py --- a/rpython/jit/metainterp/test/test_virtualref.py +++ b/rpython/jit/metainterp/test/test_virtualref.py @@ -1,6 +1,6 @@ import py -from rpython.rtyper.lltypesystem import lltype, lloperation +from rpython.rtyper.lltypesystem import lltype, llmemory, lloperation from rpython.rtyper.exceptiondata import UnknownException from rpython.rlib.jit import JitDriver, dont_look_inside, vref_None from rpython.rlib.jit import virtual_ref, virtual_ref_finish, InvalidVirtualRef @@ -110,7 +110,10 @@ if str(box._getrepr_()).endswith('JitVirtualRef')] assert len(bxs2) == 1 JIT_VIRTUAL_REF = self.vrefinfo.JIT_VIRTUAL_REF - bxs2[0].getref(lltype.Ptr(JIT_VIRTUAL_REF)).virtual_token = 1234567 + FOO = lltype.GcStruct('FOO') + foo = lltype.malloc(FOO) + tok = lltype.cast_opaque_ptr(llmemory.GCREF, foo) + bxs2[0].getref(lltype.Ptr(JIT_VIRTUAL_REF)).virtual_token = tok # # try reloading from blackhole.py's point of view from rpython.jit.metainterp.resume import ResumeDataDirectReader diff --git a/rpython/jit/metainterp/virtualizable.py b/rpython/jit/metainterp/virtualizable.py --- a/rpython/jit/metainterp/virtualizable.py +++ b/rpython/jit/metainterp/virtualizable.py @@ -10,9 +10,8 @@ from rpython.jit.metainterp.warmstate import wrap, unwrap from rpython.rlib.objectmodel import specialize + class VirtualizableInfo(object): - TOKEN_NONE = 0 # must be 0 -- see also x86.call_assembler - TOKEN_TRACING_RESCALL = -1 def __init__(self, warmrunnerdesc, VTYPEPTR): self.warmrunnerdesc = warmrunnerdesc @@ -217,7 +216,7 @@ self.cast_gcref_to_vtype = cast_gcref_to_vtype def reset_vable_token(virtualizable): - virtualizable.vable_token = VirtualizableInfo.TOKEN_NONE + virtualizable.vable_token = TOKEN_NONE self.reset_vable_token = reset_vable_token def clear_vable_token(virtualizable): @@ -230,7 +229,7 @@ def tracing_before_residual_call(virtualizable): virtualizable = cast_gcref_to_vtype(virtualizable) assert not virtualizable.vable_token - virtualizable.vable_token = VirtualizableInfo.TOKEN_TRACING_RESCALL + virtualizable.vable_token = TOKEN_TRACING_RESCALL self.tracing_before_residual_call = tracing_before_residual_call def tracing_after_residual_call(virtualizable): @@ -238,8 +237,8 @@ if virtualizable.vable_token: # not modified by the residual call; assert that it is still # set to TOKEN_TRACING_RESCALL and clear it. - assert virtualizable.vable_token == VirtualizableInfo.TOKEN_TRACING_RESCALL - virtualizable.vable_token = VirtualizableInfo.TOKEN_NONE + assert virtualizable.vable_token == TOKEN_TRACING_RESCALL + virtualizable.vable_token = TOKEN_NONE return False else: # marker "modified during residual call" set. @@ -248,16 +247,16 @@ def force_now(virtualizable): token = virtualizable.vable_token - if token == VirtualizableInfo.TOKEN_TRACING_RESCALL: + if token == TOKEN_TRACING_RESCALL: # The values in the virtualizable are always correct during # tracing. We only need to reset vable_token to TOKEN_NONE # as a marker for the tracing, to tell it that this # virtualizable escapes. - virtualizable.vable_token = VirtualizableInfo.TOKEN_NONE + virtualizable.vable_token = TOKEN_NONE else: from rpython.jit.metainterp.compile import ResumeGuardForcedDescr ResumeGuardForcedDescr.force_now(cpu, token) - assert virtualizable.vable_token == VirtualizableInfo.TOKEN_NONE + assert virtualizable.vable_token == TOKEN_NONE force_now._dont_inline_ = True self.force_now = force_now @@ -268,7 +267,7 @@ def reset_token_gcref(virtualizable): virtualizable = cast_gcref_to_vtype(virtualizable) - virtualizable.vable_token = VirtualizableInfo.TOKEN_NONE + virtualizable.vable_token = TOKEN_NONE self.reset_token_gcref = reset_token_gcref def _freeze_(self): @@ -297,16 +296,23 @@ # ____________________________________________________________ # -# The 'vable_token' field of a virtualizable is either 0, -1, or points -# into the CPU stack to a particular field in the current frame. It is: +# The 'vable_token' field of a virtualizable is either NULL, points +# to the JITFRAME object for the current assembler frame, or is +# the special value TOKEN_TRACING_RESCALL. It is: # -# 1. 0 (TOKEN_NONE) if not in the JIT at all, except as described below. +# 1. NULL (TOKEN_NONE) if not in the JIT at all, except as described below. # -# 2. equal to 0 when tracing is in progress; except: +# 2. NULL when tracing is in progress; except: # -# 3. equal to -1 (TOKEN_TRACING_RESCALL) during tracing when we do a +# 3. equal to TOKEN_TRACING_RESCALL during tracing when we do a # residual call, calling random unknown other parts of the interpreter; -# it is reset to 0 as soon as something occurs to the virtualizable. +# it is reset to NULL as soon as something occurs to the virtualizable. # # 4. when running the machine code with a virtualizable, it is set -# to the address in the CPU stack by the FORCE_TOKEN operation. +# to the JITFRAME, as obtained with the FORCE_TOKEN operation. + +_DUMMY = lltype.GcStruct('JITFRAME_DUMMY') +_dummy = lltype.malloc(_DUMMY) + +TOKEN_NONE = lltype.nullptr(llmemory.GCREF.TO) +TOKEN_TRACING_RESCALL = lltype.cast_opaque_ptr(llmemory.GCREF, _dummy) diff --git a/rpython/jit/metainterp/virtualref.py b/rpython/jit/metainterp/virtualref.py --- a/rpython/jit/metainterp/virtualref.py +++ b/rpython/jit/metainterp/virtualref.py @@ -1,6 +1,8 @@ from rpython.rtyper.rmodel import inputconst, log from rpython.rtyper.lltypesystem import lltype, llmemory, rclass from rpython.jit.metainterp import history +from rpython.jit.metainterp.virtualizable import TOKEN_NONE +from rpython.jit.metainterp.virtualizable import TOKEN_TRACING_RESCALL from rpython.jit.codewriter import heaptracker from rpython.rlib.jit import InvalidVirtualRef @@ -12,7 +14,7 @@ # we make the low-level type of an RPython class directly self.JIT_VIRTUAL_REF = lltype.GcStruct('JitVirtualRef', ('super', rclass.OBJECT), - ('virtual_token', lltype.Signed), + ('virtual_token', llmemory.GCREF), ('forced', rclass.OBJECTPTR)) self.jit_virtual_ref_vtable = lltype.malloc(rclass.OBJECT_VTABLE, zero=True, flavor='raw', @@ -69,19 +71,17 @@ # The 'virtual_token' field has the same meaning as the 'vable_token' field # of a virtualizable. It is equal to: - # * -3 (TOKEN_NONE) when tracing, except as described below; - # * -1 (TOKEN_TRACING_RESCALL) during tracing when we do a residual call; - # * addr in the CPU stack (set by FORCE_TOKEN) when running the assembler; - # * -3 (TOKEN_NONE) after the virtual is forced, if it is forced at all. - TOKEN_NONE = -3 - TOKEN_TRACING_RESCALL = -1 + # * TOKEN_NONE when tracing, except as described below; + # * TOKEN_TRACING_RESCALL during tracing when we do a residual call; + # * the JITFRAME (set by FORCE_TOKEN) when running the assembler; + # * TOKEN_NONE after the virtual is forced, if it is forced at all. def virtual_ref_during_tracing(self, real_object): assert real_object vref = lltype.malloc(self.JIT_VIRTUAL_REF) p = lltype.cast_pointer(rclass.OBJECTPTR, vref) p.typeptr = self.jit_virtual_ref_vtable - vref.virtual_token = self.TOKEN_NONE + vref.virtual_token = TOKEN_NONE vref.forced = lltype.cast_opaque_ptr(rclass.OBJECTPTR, real_object) return lltype.cast_opaque_ptr(llmemory.GCREF, vref) @@ -95,19 +95,19 @@ if not self.is_virtual_ref(gcref): return vref = lltype.cast_opaque_ptr(lltype.Ptr(self.JIT_VIRTUAL_REF), gcref) - assert vref.virtual_token == self.TOKEN_NONE - vref.virtual_token = self.TOKEN_TRACING_RESCALL + assert vref.virtual_token == TOKEN_NONE + vref.virtual_token = TOKEN_TRACING_RESCALL def tracing_after_residual_call(self, gcref): if not self.is_virtual_ref(gcref): return False vref = lltype.cast_opaque_ptr(lltype.Ptr(self.JIT_VIRTUAL_REF), gcref) assert vref.forced - if vref.virtual_token != self.TOKEN_NONE: + if vref.virtual_token != TOKEN_NONE: # not modified by the residual call; assert that it is still # set to TOKEN_TRACING_RESCALL and clear it. - assert vref.virtual_token == self.TOKEN_TRACING_RESCALL - vref.virtual_token = self.TOKEN_NONE + assert vref.virtual_token == TOKEN_TRACING_RESCALL + vref.virtual_token = TOKEN_NONE return False else: # marker "modified during residual call" set. @@ -118,8 +118,8 @@ return assert real_object vref = lltype.cast_opaque_ptr(lltype.Ptr(self.JIT_VIRTUAL_REF), gcref) - assert vref.virtual_token != self.TOKEN_TRACING_RESCALL - vref.virtual_token = self.TOKEN_NONE + assert vref.virtual_token != TOKEN_TRACING_RESCALL + vref.virtual_token = TOKEN_NONE vref.forced = lltype.cast_opaque_ptr(rclass.OBJECTPTR, real_object) # ____________________________________________________________ @@ -151,19 +151,19 @@ def force_virtual(self, inst): vref = lltype.cast_pointer(lltype.Ptr(self.JIT_VIRTUAL_REF), inst) token = vref.virtual_token - if token != self.TOKEN_NONE: - if token == self.TOKEN_TRACING_RESCALL: + if token != TOKEN_NONE: + if token == TOKEN_TRACING_RESCALL: # The "virtual" is not a virtual at all during tracing. # We only need to reset virtual_token to TOKEN_NONE # as a marker for the tracing, to tell it that this # "virtual" escapes. assert vref.forced - vref.virtual_token = self.TOKEN_NONE + vref.virtual_token = TOKEN_NONE else: assert not vref.forced from rpython.jit.metainterp.compile import ResumeGuardForcedDescr ResumeGuardForcedDescr.force_now(self.cpu, token) - assert vref.virtual_token == self.TOKEN_NONE + assert vref.virtual_token == TOKEN_NONE assert vref.forced elif not vref.forced: # token == TOKEN_NONE and the vref was not forced: it's invalid diff --git a/rpython/rtyper/lltypesystem/rvirtualizable2.py b/rpython/rtyper/lltypesystem/rvirtualizable2.py --- a/rpython/rtyper/lltypesystem/rvirtualizable2.py +++ b/rpython/rtyper/lltypesystem/rvirtualizable2.py @@ -9,15 +9,19 @@ def _setup_repr_llfields(self): llfields = [] if self.top_of_virtualizable_hierarchy: - llfields.append(('vable_token', lltype.Signed)) + llfields.append(('vable_token', llmemory.GCREF)) return llfields - def set_vable(self, llops, vinst, force_cast=False): - if self.top_of_virtualizable_hierarchy: - if force_cast: - vinst = llops.genop('cast_pointer', [vinst], resulttype=self) - cname = inputconst(lltype.Void, 'vable_token') - cvalue = inputconst(lltype.Signed, 0) - llops.genop('setfield', [vinst, cname, cvalue]) - else: - self.rbase.set_vable(llops, vinst, force_cast=True) +## The code below is commented out because vtable_token is always +## initialized to NULL anyway. +## +## def set_vable(self, llops, vinst, force_cast=False): +## if self.top_of_virtualizable_hierarchy: +## if force_cast: +## vinst = llops.genop('cast_pointer', [vinst], resulttype=self) +## cname = inputconst(lltype.Void, 'vable_token') +## cvalue = inputconst(llmemory.GCREF, +## lltype.nullptr(llmemory.GCREF.TO)) +## llops.genop('setfield', [vinst, cname, cvalue]) +## else: +## self.rbase.set_vable(llops, vinst, force_cast=True) diff --git a/rpython/rtyper/rvirtualizable2.py b/rpython/rtyper/rvirtualizable2.py --- a/rpython/rtyper/rvirtualizable2.py +++ b/rpython/rtyper/rvirtualizable2.py @@ -23,8 +23,8 @@ def _setup_repr_llfields(self): raise NotImplementedError - def set_vable(self, llops, vinst, force_cast=False): - raise NotImplementedError +## def set_vable(self, llops, vinst, force_cast=False): +## raise NotImplementedError def _setup_repr(self): if self.top_of_virtualizable_hierarchy: @@ -43,10 +43,10 @@ # not need it, but it doesn't hurt to have it anyway self.my_redirected_fields = self.rbase.my_redirected_fields - def new_instance(self, llops, classcallhop=None): - vptr = self._super().new_instance(llops, classcallhop) - self.set_vable(llops, vptr) - return vptr +## def new_instance(self, llops, classcallhop=None): +## vptr = self._super().new_instance(llops, classcallhop) +## self.set_vable(llops, vptr) +## return vptr def hook_access_field(self, vinst, cname, llops, flags): #if not flags.get('access_directly'): diff --git a/rpython/rtyper/test/test_rvirtualizable2.py b/rpython/rtyper/test/test_rvirtualizable2.py --- a/rpython/rtyper/test/test_rvirtualizable2.py +++ b/rpython/rtyper/test/test_rvirtualizable2.py @@ -1,5 +1,5 @@ import py -from rpython.rtyper.lltypesystem import lltype +from rpython.rtyper.lltypesystem import lltype, llmemory from rpython.rtyper.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin from rpython.rtyper.rvirtualizable2 import replace_force_virtualizable_with_call from rpython.rlib.jit import hint @@ -373,7 +373,7 @@ assert res.item1 == 42 res = lltype.normalizeptr(res.item0) assert res.inst_v == 42 - assert res.vable_token == 0 + assert res.vable_token == lltype.nullptr(llmemory.GCREF.TO) class TestOOtype(OORtypeMixin, BaseTest): prefix = 'o' From noreply at buildbot.pypy.org Sun Jan 27 18:14:46 2013 From: noreply at buildbot.pypy.org (rlamy) Date: Sun, 27 Jan 2013 18:14:46 +0100 (CET) Subject: [pypy-commit] pypy kill-flowobjspace: Move do_operation* from FlowObjSpace to FSFrame Message-ID: <20130127171446.E43371C039A@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: kill-flowobjspace Changeset: r60522:d45f1bd16894 Date: 2013-01-26 20:40 +0000 http://bitbucket.org/pypy/pypy/changeset/d45f1bd16894/ Log: Move do_operation* from FlowObjSpace to FSFrame diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py --- a/rpython/flowspace/flowcontext.py +++ b/rpython/flowspace/flowcontext.py @@ -9,11 +9,12 @@ from rpython.tool.stdlib_opcode import host_bytecode_spec from rpython.flowspace.argument import ArgumentsForTranslation from rpython.flowspace.model import (Constant, Variable, Block, Link, - UnwrapException, c_last_exception) + UnwrapException, c_last_exception, SpaceOperation) from rpython.flowspace.framestate import (FrameState, recursively_unflatten, recursively_flatten) from rpython.flowspace.specialcase import (rpython_print_item, rpython_print_newline) +from rpython.flowspace.operation import implicit_exceptions class FlowingError(Exception): """ Signals invalid RPython in the function being analysed""" @@ -471,6 +472,17 @@ def guessbool(self, w_condition, **kwds): return self.recorder.guessbool(self, w_condition, **kwds) + def do_operation(self, name, *args_w): + spaceop = SpaceOperation(name, args_w, Variable()) + spaceop.offset = self.last_instr + self.record(spaceop) + return spaceop.result + + def do_operation_with_implicit_exceptions(self, name, *args_w): + w_result = self.do_operation(name, *args_w) + self.handle_implicit_exceptions(implicit_exceptions.get(name)) + return w_result + def handle_implicit_exceptions(self, exceptions): """ Catch possible exceptions implicitly. @@ -740,7 +752,7 @@ def YIELD_VALUE(self, _, next_instr): assert self.pycode.is_generator w_result = self.popvalue() - self.space.do_operation('yield', w_result) + self.do_operation('yield', w_result) # XXX yield expressions not supported. This will blow up if the value # isn't popped straightaway. self.pushvalue(None) @@ -751,7 +763,7 @@ def PRINT_ITEM(self, oparg, next_instr): w_item = self.popvalue() - w_s = self.space.do_operation('str', w_item) + w_s = self.do_operation('str', w_item) self.space.appcall(rpython_print_item, w_s) def PRINT_NEWLINE(self, oparg, next_instr): diff --git a/rpython/flowspace/objspace.py b/rpython/flowspace/objspace.py --- a/rpython/flowspace/objspace.py +++ b/rpython/flowspace/objspace.py @@ -9,7 +9,7 @@ from rpython.flowspace.argument import ArgumentsForTranslation from rpython.flowspace.model import (Constant, Variable, WrapException, - UnwrapException, checkgraph, SpaceOperation) + UnwrapException, checkgraph) from rpython.flowspace.bytecode import HostCode from rpython.flowspace import operation from rpython.flowspace.flowcontext import (FlowSpaceFrame, fixeggblocks, @@ -91,21 +91,21 @@ id = None # real version added by add_operations() def newdict(self, module="ignored"): - return self.do_operation('newdict') + return self.frame.do_operation('newdict') def newtuple(self, args_w): try: content = [self.unwrap(w_arg) for w_arg in args_w] except UnwrapException: - return self.do_operation('newtuple', *args_w) + return self.frame.do_operation('newtuple', *args_w) else: return Constant(tuple(content)) def newlist(self, args_w, sizehint=None): - return self.do_operation('newlist', *args_w) + return self.frame.do_operation('newlist', *args_w) def newslice(self, w_start, w_stop, w_step): - return self.do_operation('newslice', w_start, w_stop, w_step) + return self.frame.do_operation('newslice', w_start, w_stop, w_step) def newbool(self, b): if b: @@ -299,22 +299,10 @@ if not self.is_true(w_correct): e = self.exc_from_raise(self.w_ValueError, self.w_None) raise e - return [self.do_operation('getitem', w_iterable, self.wrap(i)) + return [self.frame.do_operation('getitem', w_iterable, self.wrap(i)) for i in range(expected_length)] # ____________________________________________________________ - def do_operation(self, name, *args_w): - spaceop = SpaceOperation(name, args_w, Variable()) - spaceop.offset = self.frame.last_instr - self.frame.record(spaceop) - return spaceop.result - - def do_operation_with_implicit_exceptions(self, name, *args_w): - w_result = self.do_operation(name, *args_w) - self.frame.handle_implicit_exceptions( - operation.implicit_exceptions.get(name)) - return w_result - def not_(self, w_obj): return self.wrap(not self.is_true(w_obj)) @@ -325,7 +313,7 @@ pass else: return bool(obj) - w_truthvalue = self.do_operation('is_true', w_obj) + w_truthvalue = self.frame.do_operation('is_true', w_obj) return self.frame.guessbool(w_truthvalue) def iter(self, w_iterable): @@ -336,7 +324,7 @@ else: if isinstance(iterable, unrolling_iterable): return self.wrap(iterable.get_unroller()) - w_iter = self.do_operation("iter", w_iterable) + w_iter = self.frame.do_operation("iter", w_iterable) return w_iter def next(self, w_iter): @@ -354,7 +342,7 @@ else: frame.replace_in_stack(it, next_unroller) return self.wrap(v) - w_item = self.do_operation("next", w_iter) + w_item = frame.do_operation("next", w_iter) frame.handle_implicit_exceptions([StopIteration, RuntimeError]) return w_item @@ -363,8 +351,8 @@ if w_obj is self.frame.w_globals: raise FlowingError(self.frame, "Attempting to modify global variable %r." % (w_key)) - return self.do_operation_with_implicit_exceptions('setitem', w_obj, - w_key, w_val) + return self.frame.do_operation_with_implicit_exceptions('setitem', + w_obj, w_key, w_val) def setitem_str(self, w_obj, key, w_value): return self.setitem(w_obj, self.wrap(key), w_value) @@ -375,7 +363,7 @@ if w_obj in self.not_really_const: const_w = self.not_really_const[w_obj] if w_name not in const_w: - return self.do_operation_with_implicit_exceptions('getattr', + return self.frame.do_operation_with_implicit_exceptions('getattr', w_obj, w_name) try: obj = self.unwrap_for_computation(w_obj) @@ -394,7 +382,7 @@ return self.wrap(result) except WrapException: pass - return self.do_operation_with_implicit_exceptions('getattr', + return self.frame.do_operation_with_implicit_exceptions('getattr', w_obj, w_name) def isinstance_w(self, w_obj, w_type): @@ -414,7 +402,7 @@ if w_module in self.not_really_const: const_w = self.not_really_const[w_obj] if w_name not in const_w: - return self.do_operation_with_implicit_exceptions('getattr', + return self.frame.do_operation_with_implicit_exceptions('getattr', w_obj, w_name) try: return self.wrap(getattr(w_module.value, w_name.value)) @@ -433,7 +421,7 @@ def appcall(self, func, *args_w): """Call an app-level RPython function directly""" w_func = self.wrap(func) - return self.do_operation('simple_call', w_func, *args_w) + return self.frame.do_operation('simple_call', w_func, *args_w) def call_args(self, w_callable, args): try: @@ -454,11 +442,11 @@ # NOTE: annrpython needs to know about the following two operations! if not kwds_w: # simple case - w_res = self.do_operation('simple_call', w_callable, *args_w) + w_res = self.frame.do_operation('simple_call', w_callable, *args_w) else: # general case shape, args_w = args.flatten() - w_res = self.do_operation('call_args', w_callable, Constant(shape), + w_res = self.frame.do_operation('call_args', w_callable, Constant(shape), *args_w) # maybe the call has generated an exception (any one) @@ -562,7 +550,7 @@ # type cannot sanely appear in flow graph, # store operation with variable result instead pass - w_result = self.do_operation_with_implicit_exceptions(name, *args_w) + w_result = self.frame.do_operation_with_implicit_exceptions(name, *args_w) return w_result setattr(FlowObjSpace, name, generic_operator) diff --git a/rpython/flowspace/specialcase.py b/rpython/flowspace/specialcase.py --- a/rpython/flowspace/specialcase.py +++ b/rpython/flowspace/specialcase.py @@ -18,7 +18,7 @@ if opname == 'pow' and len(args_w) == 2: args_w = args_w + [Constant(None)] elif opname == 'getattr' and len(args_w) == 3: - return space.do_operation('simple_call', Constant(getattr), *args_w) + return space.frame.do_operation('simple_call', Constant(getattr), *args_w) else: raise Exception, "should call %r with exactly %d arguments" % ( fn, Arity[opname]) @@ -60,7 +60,7 @@ [w_value] = args_w if isinstance(w_value, Constant): return Constant(r_uint(w_value.value)) - return space.do_operation('simple_call', space.wrap(r_uint), w_value) + return space.frame.do_operation('simple_call', space.wrap(r_uint), w_value) def sc_we_are_translated(space, we_are_translated, args): return Constant(True) From noreply at buildbot.pypy.org Sun Jan 27 18:14:48 2013 From: noreply at buildbot.pypy.org (rlamy) Date: Sun, 27 Jan 2013 18:14:48 +0100 (CET) Subject: [pypy-commit] pypy kill-flowobjspace: move build_flow() out of FlowObjSpace Message-ID: <20130127171448.4CC4D1C125A@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: kill-flowobjspace Changeset: r60523:ec3af57266f4 Date: 2013-01-27 01:56 +0000 http://bitbucket.org/pypy/pypy/changeset/ec3af57266f4/ Log: move build_flow() out of FlowObjSpace diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py --- a/rpython/annotator/test/test_annrpython.py +++ b/rpython/annotator/test/test_annrpython.py @@ -13,7 +13,7 @@ from rpython.rlib.rarithmetic import r_uint, base_int, r_longlong, r_ulonglong from rpython.rlib.rarithmetic import r_singlefloat from rpython.rlib import objectmodel -from rpython.flowspace.objspace import FlowObjSpace, FlowingError +from rpython.flowspace.objspace import build_flow, FlowingError from rpython.translator.test import snippet @@ -40,9 +40,6 @@ class TestAnnotateTestCase: - def setup_class(cls): - cls.space = FlowObjSpace() - def teardown_method(self, meth): assert annmodel.s_Bool == annmodel.SomeBool() @@ -60,7 +57,7 @@ except AttributeError: pass name = func.func_name - funcgraph = self.space.build_flow(func) + funcgraph = build_flow(func) funcgraph.source = inspect.getsource(func) return funcgraph @@ -775,7 +772,7 @@ assert isinstance(s, annmodel.SomePBC) assert s.const == myobj - def test_cleanup_protocol(self): + def test_cleanup_protocol(self): class Stuff: def __init__(self): self.called = False @@ -2068,7 +2065,7 @@ s = a.build_types(f, [annmodel.SomeString(no_nul=True)]) assert isinstance(s, annmodel.SomeString) assert s.no_nul - + def test_non_none_and_none_with_isinstance(self): class A(object): @@ -3707,7 +3704,7 @@ def test_join_none_and_nonnull(self): from rpython.rlib.rstring import assert_str0 - + def f(i): a = str(i) a = assert_str0(a) @@ -3751,7 +3748,7 @@ class A(object): def __iter__(self): return self - + def fn(): return iter(A()) @@ -3810,7 +3807,7 @@ return True x = X() - + def f(i): if i: x1 = x diff --git a/rpython/flowspace/objspace.py b/rpython/flowspace/objspace.py --- a/rpython/flowspace/objspace.py +++ b/rpython/flowspace/objspace.py @@ -84,6 +84,9 @@ # objects which should keep their SomeObjectness not_really_const = NOT_REALLY_CONST + def build_flow(self, func): + return build_flow(func, self) + def is_w(self, w_one, w_two): return self.is_true(self.is_(w_one, w_two)) @@ -262,28 +265,6 @@ w_type = w_instclass return FSException(w_type, w_value) - def build_flow(self, func): - """ - """ - _assert_rpythonic(func) - code = HostCode._from_code(func.func_code) - if (code.is_generator and - not hasattr(func, '_generator_next_method_of_')): - graph = PyGraph(func, code) - block = graph.startblock - for name, w_value in zip(code.co_varnames, block.framestate.mergeable): - if isinstance(w_value, Variable): - w_value.rename(name) - return bootstrap_generator(graph) - graph = PyGraph(func, code) - frame = self.frame = FlowSpaceFrame(self, graph, code) - frame.build_flow() - fixeggblocks(graph) - checkgraph(graph) - if code.is_generator: - tweak_generator_graph(graph) - return graph - def unpackiterable(self, w_iterable, expected_length=None): if not isinstance(w_iterable, Variable): l = list(self.unwrap(w_iterable)) @@ -555,6 +536,29 @@ setattr(FlowObjSpace, name, generic_operator) - for (name, symbol, arity, specialnames) in operation.MethodTable: make_op(name, arity) + + +def build_flow(func, space=FlowObjSpace()): + """ + Create the flow graph for the function. + """ + _assert_rpythonic(func) + code = HostCode._from_code(func.func_code) + if (code.is_generator and + not hasattr(func, '_generator_next_method_of_')): + graph = PyGraph(func, code) + block = graph.startblock + for name, w_value in zip(code.co_varnames, block.framestate.mergeable): + if isinstance(w_value, Variable): + w_value.rename(name) + return bootstrap_generator(graph) + graph = PyGraph(func, code) + frame = space.frame = FlowSpaceFrame(space, graph, code) + frame.build_flow() + fixeggblocks(graph) + checkgraph(graph) + if code.is_generator: + tweak_generator_graph(graph) + return graph diff --git a/rpython/flowspace/test/test_framestate.py b/rpython/flowspace/test/test_framestate.py --- a/rpython/flowspace/test/test_framestate.py +++ b/rpython/flowspace/test/test_framestate.py @@ -6,9 +6,6 @@ from rpython.flowspace.pygraph import PyGraph class TestFrameState: - def setup_class(cls): - cls.space = FlowObjSpace() - def getframe(self, func): try: func = func.im_func @@ -16,7 +13,7 @@ pass code = HostCode._from_code(func.func_code) graph = PyGraph(func, code) - frame = FlowSpaceFrame(self.space, graph, code) + frame = FlowSpaceFrame(FlowObjSpace(), graph, code) # hack the frame frame.setstate(graph.startblock.framestate) frame.locals_stack_w[frame.pycode.co_nlocals-1] = Constant(None) diff --git a/rpython/flowspace/test/test_generator.py b/rpython/flowspace/test/test_generator.py --- a/rpython/flowspace/test/test_generator.py +++ b/rpython/flowspace/test/test_generator.py @@ -1,5 +1,5 @@ from rpython.conftest import option -from rpython.flowspace.objspace import FlowObjSpace +from rpython.flowspace.objspace import build_flow from rpython.flowspace.model import Variable from rpython.flowspace.generator import (make_generatoriterator_class, replace_graph_with_bootstrap, get_variable_names, attach_next_method) @@ -67,7 +67,7 @@ yield n yield n # - graph = FlowObjSpace().build_flow(func) + graph = build_flow(func) if option.view: graph.show() block = graph.startblock @@ -93,8 +93,7 @@ yield n + 1 z -= 10 # - space = FlowObjSpace() - graph = space.build_flow(f) + graph = build_flow(f) GeneratorIterator = make_generatoriterator_class(graph) replace_graph_with_bootstrap(GeneratorIterator, graph) func1 = attach_next_method(GeneratorIterator, graph) @@ -104,12 +103,12 @@ assert func1._generator_next_method_of_ is GeneratorIterator assert hasattr(GeneratorIterator, 'next') # - graph_next = space.build_flow(GeneratorIterator.next.im_func) + graph_next = build_flow(GeneratorIterator.next.im_func) join_blocks(graph_next) if option.view: graph_next.show() # - graph1 = space.build_flow(func1) + graph1 = build_flow(func1) if option.view: graph1.show() @@ -119,7 +118,7 @@ yield n + 1 z -= 10 # - graph = FlowObjSpace().build_flow(f) + graph = build_flow(f) if option.view: graph.show() block = graph.startblock diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py --- a/rpython/flowspace/test/test_objspace.py +++ b/rpython/flowspace/test/test_objspace.py @@ -5,7 +5,7 @@ from rpython.flowspace.model import Constant, mkentrymap, c_last_exception from rpython.translator.simplify import simplify_graph -from rpython.flowspace.objspace import FlowObjSpace +from rpython.flowspace.objspace import build_flow from rpython.flowspace.flowcontext import FlowingError, FlowSpaceFrame from rpython.conftest import option from rpython.tool.stdlib_opcode import host_bytecode_spec @@ -34,7 +34,7 @@ except AttributeError: pass #name = func.func_name - graph = self.space.build_flow(func, **kwds) + graph = build_flow(func, **kwds) graph.source = inspect.getsource(func) self.show(graph) return graph @@ -43,9 +43,6 @@ if option.view: graph.show() - def setup_class(cls): - cls.space = FlowObjSpace() - def all_operations(self, graph): result = {} for node in graph.iterblocks(): @@ -740,7 +737,7 @@ def test_relative_import(self): def f(): - from ..test.test_objspace import FlowObjSpace + from ..objspace import build_flow # Check that the function works in Python assert f() is None self.codetest(f) diff --git a/rpython/rlib/test/test_nonconst.py b/rpython/rlib/test/test_nonconst.py --- a/rpython/rlib/test/test_nonconst.py +++ b/rpython/rlib/test/test_nonconst.py @@ -4,7 +4,6 @@ from rpython.rlib.nonconst import NonConstant -from rpython.flowspace.objspace import FlowObjSpace from rpython.annotator.annrpython import RPythonAnnotator from rpython.conftest import option from rpython.annotator.model import SomeInstance @@ -13,20 +12,20 @@ def nonconst_f(): a = NonConstant(3) return a - + a = RPythonAnnotator() s = a.build_types(nonconst_f, []) assert s.knowntype is int assert not hasattr(s, 'const') #rtyper = a.translator.buildrtyper(type_system="ootype") #rtyper.specialize() - + def test_nonconst_list(): def nonconst_l(): a = NonConstant([1, 2, 3]) return a[0] - + a = RPythonAnnotator() s = a.build_types(nonconst_l, []) assert s.knowntype is int @@ -36,7 +35,7 @@ class A: pass a = A() - + def nonconst_i(): return NonConstant(a) @@ -51,9 +50,9 @@ def test_bool_nonconst(): def fn(): return bool(NonConstant(False)) - + assert not fn() - + a = RPythonAnnotator() s = a.build_types(fn, []) assert s.knowntype is bool diff --git a/rpython/rtyper/ootypesystem/test/test_ooann.py b/rpython/rtyper/ootypesystem/test/test_ooann.py --- a/rpython/rtyper/ootypesystem/test/test_ooann.py +++ b/rpython/rtyper/ootypesystem/test/test_ooann.py @@ -1,7 +1,6 @@ import py from rpython.rtyper.ootypesystem.ootype import * from rpython.annotator import model as annmodel -from rpython.flowspace.objspace import FlowObjSpace from rpython.annotator.annrpython import RPythonAnnotator import exceptions from rpython.rtyper.ootypesystem import ooregistry # side effects @@ -9,7 +8,7 @@ def test_simple_new(): C = Instance("test", ROOT, {'a': Signed}) - + def oof(): c = new(C) c.a = 5 @@ -23,7 +22,7 @@ def test_simple_instanceof(): C = Instance("test", ROOT, {'a': Signed}) - + def oof(): c = new(C) return instanceof(c, C) @@ -36,7 +35,7 @@ def test_simple_null(): I = Instance("test", ROOT, {'a': Signed}) - + def oof(): i = null(I) return i @@ -49,7 +48,7 @@ def test_simple_classof(): I = Instance("test", ROOT, {'a': Signed}) - + def oof(): i = new(I) return classof(i) @@ -62,8 +61,8 @@ def test_subclassof(): I = Instance("test", ROOT, {'a': Signed}) - I1 = Instance("test1", I) - + I1 = Instance("test1", I) + def oof(): i = new(I) i1 = new(I1) @@ -77,7 +76,7 @@ def test_simple_runtimenew(): I = Instance("test", ROOT, {'a': Signed}) - + def oof(): i = new(I) c = classof(i) @@ -94,7 +93,7 @@ I = Instance("test", ROOT, {'a': Signed}) J = Instance("test2", I, {'b': Signed}) K = Instance("test2", I, {'b': Signed}) - + def oof(x): k = new(K) j = new(J) @@ -124,7 +123,7 @@ def oof(): c = new(C) return c.m(c) - + a = RPythonAnnotator() s = a.build_types(oof, []) # a.translator.view() @@ -152,7 +151,7 @@ def test_record(): R = Record({'foo': Signed}) r = new(R) - + def oof(): return r @@ -184,7 +183,7 @@ a = RPythonAnnotator() s = a.build_types(oof, []) - + assert s == annmodel.SomeOOStaticMeth(F) def test_truth_value(): @@ -285,7 +284,7 @@ return a a = RPythonAnnotator() assert isinstance(a.build_types(f, []), annmodel.SomeFloat) - + def test_overload_reannotate_unrelated(): py.test.skip("Maybe we want this to work") # this test fails because the result type of c.foo(mylist[0]) diff --git a/rpython/rtyper/ootypesystem/test/test_oortype.py b/rpython/rtyper/ootypesystem/test/test_oortype.py --- a/rpython/rtyper/ootypesystem/test/test_oortype.py +++ b/rpython/rtyper/ootypesystem/test/test_oortype.py @@ -5,7 +5,6 @@ from rpython.rtyper.ootypesystem.rlist import ListRepr from rpython.rtyper.rint import signed_repr from rpython.annotator import model as annmodel -from rpython.flowspace.objspace import FlowObjSpace from rpython.translator.translator import TranslationContext, graphof from rpython.rtyper.test.test_llinterp import interpret from rpython.rlib.objectmodel import r_dict @@ -25,7 +24,7 @@ def test_simple_class(): C = Instance("test", ROOT, {'a': Signed}) - + def f(): c = new(C) return c @@ -33,10 +32,10 @@ g = gengraph(f) rettype = g.getreturnvar().concretetype assert rettype == C - + def test_simple_field(): C = Instance("test", ROOT, {'a': (Signed, 3)}) - + def f(): c = new(C) c.a = 5 @@ -45,7 +44,7 @@ g = gengraph(f) rettype = g.getreturnvar().concretetype assert rettype == Signed - + def test_simple_method(): C = Instance("test", ROOT, {'a': (Signed, 3)}) M = Meth([], Signed) @@ -53,7 +52,7 @@ return self.a m = meth(M, _name="m", _callable=m_) addMethods(C, {"m": m}) - + def f(): c = new(C) return c.m() @@ -83,7 +82,7 @@ def test_simple_classof(): I = Instance("test", ROOT, {'a': Signed}) - + def oof(): i = new(I) return classof(i) @@ -94,8 +93,8 @@ def test_subclassof(): I = Instance("test", ROOT, {'a': Signed}) - I1 = Instance("test1", I) - + I1 = Instance("test1", I) + def oof(): i = new(I) i1 = new(I1) @@ -171,7 +170,7 @@ class A: def method(self, number): return number + 2 - + def oof(): a = A() return a.method(3) @@ -188,8 +187,8 @@ t.view() rtyper = t.buildrtyper(type_system="ootype") rtyper.specialize() - graph = graphof(t, f) - + graph = graphof(t, f) + INST = graph.getreturnvar().concretetype assert rtyper.exceptiondata.is_exception_instance(INST) @@ -297,7 +296,7 @@ def oof(): return r.foo - + res = interpret(oof, [], type_system='ootype') assert res == 42 @@ -347,7 +346,7 @@ lst = L.ll_newlist(1) lst.ll_setitem_fast(0, 42) return wrapper(lst, 0) - + res = interpret(fn, [], type_system='ootype') assert res == 42 diff --git a/rpython/rtyper/test/test_llann.py b/rpython/rtyper/test/test_llann.py --- a/rpython/rtyper/test/test_llann.py +++ b/rpython/rtyper/test/test_llann.py @@ -2,7 +2,6 @@ from rpython.annotator import model as annmodel from rpython.conftest import option -from rpython.flowspace.objspace import FlowObjSpace from rpython.rtyper.annlowlevel import (annotate_lowlevel_helper, MixLevelHelperAnnotator, PseudoHighLevelCallable, llhelper, cast_instance_to_base_ptr, cast_base_ptr_to_instance, base_ptr_lltype) @@ -31,9 +30,6 @@ class TestLowLevelAnnotateTestCase: - def setup_class(cls): - cls.space = FlowObjSpace() - from rpython.annotator.annrpython import RPythonAnnotator def annotate(self, ll_function, argtypes): diff --git a/rpython/translator/tool/make_dot.py b/rpython/translator/tool/make_dot.py --- a/rpython/translator/tool/make_dot.py +++ b/rpython/translator/tool/make_dot.py @@ -1,7 +1,7 @@ import os import inspect, linecache from rpython.flowspace.model import * -from rpython.flowspace.objspace import FlowObjSpace as Space +from rpython.flowspace.objspace import build_flow from rpython.tool.udir import udir from py.process import cmdexec from rpython.tool.error import offset2lineno @@ -44,9 +44,9 @@ def leave_subgraph(self): self.emit("}") - def emit_edge(self, name1, name2, label="", - style="dashed", - color="black", + def emit_edge(self, name1, name2, label="", + style="dashed", + color="black", dir="forward", weight="5", ports=None, @@ -61,11 +61,11 @@ else: self.emit('%s -> %s' % (safename(name1), safename(name2))) - def emit_node(self, name, - shape="diamond", - label="", + def emit_node(self, name, + shape="diamond", + label="", color="black", - fillcolor="white", + fillcolor="white", style="filled", width="0.75", ): @@ -134,7 +134,7 @@ data = "BROKEN BLOCK\\n%r" % (block,) self.emit_node(name, label=data) return - + lines = [] for op in block.operations: lines.extend(repr(op).split('\n')) @@ -247,6 +247,5 @@ i += 1 return i - space = Space() - graph = space.build_flow(f) + graph = build_flow(f) make_dot('f', graph) diff --git a/rpython/translator/translator.py b/rpython/translator/translator.py --- a/rpython/translator/translator.py +++ b/rpython/translator/translator.py @@ -8,7 +8,7 @@ from rpython.translator import simplify from rpython.flowspace.model import FunctionGraph, checkgraph, Block -from rpython.flowspace.objspace import FlowObjSpace +from rpython.flowspace.objspace import build_flow from rpython.tool.ansi_print import ansi_log from rpython.tool.sourcetools import nice_repr_for_func from rpython.config.translationoption import get_combined_translation_config @@ -52,8 +52,7 @@ else: if self.config.translation.verbose: log.start(nice_repr_for_func(func)) - space = FlowObjSpace() - graph = space.build_flow(func) + graph = build_flow(func) if self.config.translation.simplifying: simplify.simplify_graph(graph) if self.config.translation.list_comprehension_operations: From noreply at buildbot.pypy.org Sun Jan 27 18:14:49 2013 From: noreply at buildbot.pypy.org (rlamy) Date: Sun, 27 Jan 2013 18:14:49 +0100 (CET) Subject: [pypy-commit] pypy kill-flowobjspace: flowspace: Use (for now trivial) specialisation CallSpec of ArgumentsForTranslation Message-ID: <20130127171449.887101C12E1@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: kill-flowobjspace Changeset: r60524:95a4332784d5 Date: 2013-01-27 16:24 +0000 http://bitbucket.org/pypy/pypy/changeset/95a4332784d5/ Log: flowspace: Use (for now trivial) specialisation CallSpec of ArgumentsForTranslation diff --git a/rpython/flowspace/argument.py b/rpython/flowspace/argument.py --- a/rpython/flowspace/argument.py +++ b/rpython/flowspace/argument.py @@ -374,6 +374,11 @@ return args._rawshape(nextra) +class CallSpec(ArgumentsForTranslation): + """Represents the arguments passed into a function call, i.e. the + `a, b, *c, **d` part in `return func(a, b, *c, **d)`. + """ + # # ArgErr family of exceptions raised in case of argument mismatch. # We try to give error messages following CPython's, which are very informative. diff --git a/rpython/flowspace/bytecode.py b/rpython/flowspace/bytecode.py --- a/rpython/flowspace/bytecode.py +++ b/rpython/flowspace/bytecode.py @@ -36,7 +36,6 @@ code, consts, names, varnames, filename, name, firstlineno, lnotab, freevars): """Initialize a new code object""" - self.co_name = name assert nlocals >= 0 self.co_argcount = argcount self.co_nlocals = nlocals diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py --- a/rpython/flowspace/flowcontext.py +++ b/rpython/flowspace/flowcontext.py @@ -7,7 +7,7 @@ from rpython.tool.error import source_lines from rpython.tool.stdlib_opcode import host_bytecode_spec -from rpython.flowspace.argument import ArgumentsForTranslation +from rpython.flowspace.argument import CallSpec from rpython.flowspace.model import (Constant, Variable, Block, Link, UnwrapException, c_last_exception, SpaceOperation) from rpython.flowspace.framestate import (FrameState, recursively_unflatten, @@ -989,7 +989,7 @@ keywords = None keywords_w = None arguments = self.popvalues(n_arguments) - args = ArgumentsForTranslation(self.space, arguments, keywords, + args = CallSpec(self.space, arguments, keywords, keywords_w, w_star, w_starstar) w_function = self.popvalue() w_result = self.space.call_args(w_function, args) diff --git a/rpython/flowspace/objspace.py b/rpython/flowspace/objspace.py --- a/rpython/flowspace/objspace.py +++ b/rpython/flowspace/objspace.py @@ -7,7 +7,7 @@ import types from inspect import CO_NEWLOCALS -from rpython.flowspace.argument import ArgumentsForTranslation +from rpython.flowspace.argument import CallSpec from rpython.flowspace.model import (Constant, Variable, WrapException, UnwrapException, checkgraph) from rpython.flowspace.bytecode import HostCode @@ -396,7 +396,7 @@ return self.call_function(w_meth, *arg_w) def call_function(self, w_func, *args_w): - args = ArgumentsForTranslation(self, list(args_w)) + args = CallSpec(self, list(args_w)) return self.call_args(w_func, args) def appcall(self, func, *args_w): From noreply at buildbot.pypy.org Sun Jan 27 18:14:50 2013 From: noreply at buildbot.pypy.org (rlamy) Date: Sun, 27 Jan 2013 18:14:50 +0100 (CET) Subject: [pypy-commit] pypy kill-flowobjspace: raise FlowingError on **-unpacking Message-ID: <20130127171450.BB2111C12F9@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: kill-flowobjspace Changeset: r60525:352b1a959932 Date: 2013-01-27 16:40 +0000 http://bitbucket.org/pypy/pypy/changeset/352b1a959932/ Log: raise FlowingError on **-unpacking diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py --- a/rpython/flowspace/flowcontext.py +++ b/rpython/flowspace/flowcontext.py @@ -971,6 +971,8 @@ self.pushvalue(last_val) def call_function(self, oparg, w_star=None, w_starstar=None): + if w_starstar is not None: + raise FlowingError(self, "Dict-unpacking is not RPython") n_arguments = oparg & 0xff n_keywords = (oparg>>8) & 0xff if n_keywords: diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py --- a/rpython/flowspace/test/test_objspace.py +++ b/rpython/flowspace/test/test_objspace.py @@ -699,6 +699,23 @@ for op in block.operations: assert not op.opname == "call_args" + def test_starstar_call(self): + """Check that CALL_FUNCTION_KW and CALL_FUNCTION_VAR_KW raise a + useful error. + """ + def g(a, b, c): + return a*b*c + def f1(): + return g(**{'a':0}) + with py.test.raises(FlowingError) as excinfo: + graph = self.codetest(f1) + assert 'Dict-unpacking' in str(excinfo.value) + def f2(): + return g(*(0,), **{'c':3}) + with py.test.raises(FlowingError) as excinfo: + graph = self.codetest(f2) + assert 'Dict-unpackinga' in str(excinfo.value) + def test_catch_importerror_1(self): def f(): try: From noreply at buildbot.pypy.org Sun Jan 27 18:14:51 2013 From: noreply at buildbot.pypy.org (rlamy) Date: Sun, 27 Jan 2013 18:14:51 +0100 (CET) Subject: [pypy-commit] pypy kill-flowobjspace: Kill unused argument CallSpec.w_starstararg Message-ID: <20130127171451.DC8141C12FA@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: kill-flowobjspace Changeset: r60526:ab253504ce34 Date: 2013-01-27 16:58 +0000 http://bitbucket.org/pypy/pypy/changeset/ab253504ce34/ Log: Kill unused argument CallSpec.w_starstararg diff --git a/rpython/flowspace/argument.py b/rpython/flowspace/argument.py --- a/rpython/flowspace/argument.py +++ b/rpython/flowspace/argument.py @@ -378,6 +378,45 @@ """Represents the arguments passed into a function call, i.e. the `a, b, *c, **d` part in `return func(a, b, *c, **d)`. """ + def __init__(self, space, args_w, keywords=None, keywords_w=None, + w_stararg=None, w_starstararg=None): + self.w_stararg = w_stararg + assert w_starstararg is None, "No **-unpacking in RPython" + self.combine_has_happened = False + self.space = space + assert isinstance(args_w, list) + self.arguments_w = args_w + self.keywords = keywords + self.keywords_w = keywords_w + self.keyword_names_w = None + + def copy(self): + return CallSpec(self.space, self.arguments_w, + self.keywords, self.keywords_w, self.w_stararg) + + def combine_if_necessary(self): + if self.combine_has_happened: + return + self._combine_wrapped(self.w_stararg) + self.combine_has_happened = True + + def _combine_wrapped(self, w_stararg): + "unpack the *arg and **kwd into arguments_w and keywords_w" + if w_stararg is not None: + self._combine_starargs_wrapped(w_stararg) + + def _rawshape(self, nextra=0): + assert not self.combine_has_happened + shape_cnt = len(self.arguments_w)+nextra # Number of positional args + if self.keywords: + shape_keys = self.keywords[:] # List of keywords (strings) + shape_keys.sort() + else: + shape_keys = [] + shape_star = self.w_stararg is not None # Flag: presence of *arg + shape_stst = False # Flag: presence of **kwds + return shape_cnt, tuple(shape_keys), shape_star, shape_stst # shape_keys are sorted + # # ArgErr family of exceptions raised in case of argument mismatch. diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py --- a/rpython/flowspace/test/test_objspace.py +++ b/rpython/flowspace/test/test_objspace.py @@ -714,7 +714,7 @@ return g(*(0,), **{'c':3}) with py.test.raises(FlowingError) as excinfo: graph = self.codetest(f2) - assert 'Dict-unpackinga' in str(excinfo.value) + assert 'Dict-unpacking' in str(excinfo.value) def test_catch_importerror_1(self): def f(): From noreply at buildbot.pypy.org Sun Jan 27 19:07:42 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 27 Jan 2013 19:07:42 +0100 (CET) Subject: [pypy-commit] pypy default: Help with unicode issues Message-ID: <20130127180742.805021C1306@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r60527:53723bf32fd5 Date: 2013-01-27 20:02 +0200 http://bitbucket.org/pypy/pypy/changeset/53723bf32fd5/ Log: Help with unicode issues diff --git a/pypy/module/unicodedata/interp_ucd.py b/pypy/module/unicodedata/interp_ucd.py --- a/pypy/module/unicodedata/interp_ucd.py +++ b/pypy/module/unicodedata/interp_ucd.py @@ -9,7 +9,7 @@ from rpython.rlib.objectmodel import we_are_translated from rpython.rlib.runicode import MAXUNICODE from rpython.rlib.unicodedata import unicodedb_5_2_0, unicodedb_3_2_0 -from rpython.rlib.runicode import code_to_unichr, ORD +from rpython.rlib.runicode import code_to_unichr, ord_accepts_surrogate import sys @@ -28,8 +28,6 @@ # handling: on narrow unicode builds, a surrogate pair is considered as one # unicode code point. -# The functions below are subtly different from the ones in runicode.py. -# When PyPy implements Python 3 they should be merged. if MAXUNICODE > 0xFFFF: # Target is wide build @@ -41,7 +39,7 @@ if not we_are_translated() and sys.maxunicode == 0xFFFF: # Host CPython is narrow build, accept surrogates try: - return ORD(space.unicode_w(w_unichr)) + return ord_accepts_surrogate(space.unicode_w(w_unichr)) except ValueError: raise OperationError(space.w_TypeError, space.wrap( 'need a single Unicode character as parameter')) @@ -68,7 +66,7 @@ else: # Accept surrogates try: - return ORD(space.unicode_w(w_unichr)) + return ord_accepts_surrogate(space.unicode_w(w_unichr)) except ValueError: raise OperationError(space.w_TypeError, space.wrap( 'need a single Unicode character as parameter')) diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py --- a/rpython/rlib/runicode.py +++ b/rpython/rlib/runicode.py @@ -13,6 +13,26 @@ BYTEORDER = sys.byteorder +# python 2.7 has a preview of py3k behavior, so those functions +# are used either when we're testing wide pypy on narrow cpython +# or in unicodedata in pypy + +def unichr_returns_surrogate(c): + if c <= sys.maxunicode or c > MAXUNICODE: + return unichr(c) + else: + c -= 0x10000 + return (unichr(0xD800 + (c >> 10)) + + unichr(0xDC00 + (c & 0x03FF))) + +def ord_accepts_surrogate(u): + if isinstance(u, unicode) and len(u) == 2: + ch1 = ord(u[0]) + ch2 = ord(u[1]) + if 0xD800 <= ch1 <= 0xDBFF and 0xDC00 <= ch2 <= 0xDFFF: + return (((ch1 - 0xD800) << 10) | (ch2 - 0xDC00)) + 0x10000 + return ord(u) + if MAXUNICODE > sys.maxunicode: # A version of unichr which allows codes outside the BMP # even on narrow unicode builds. @@ -21,12 +41,7 @@ # Note that Python3 uses a similar implementation. def UNICHR(c): assert not we_are_translated() - if c <= sys.maxunicode or c > MAXUNICODE: - return unichr(c) - else: - c -= 0x10000 - return (unichr(0xD800 + (c >> 10)) + - unichr(0xDC00 + (c & 0x03FF))) + return unichr_returns_surrogate(c) UNICHR._flowspace_rewrite_directly_as_ = unichr # ^^^ NB.: for translation, it's essential to use this hack instead # of calling unichr() from UNICHR(), because unichr() detects if there @@ -34,12 +49,7 @@ def ORD(u): assert not we_are_translated() - if isinstance(u, unicode) and len(u) == 2: - ch1 = ord(u[0]) - ch2 = ord(u[1]) - if 0xD800 <= ch1 <= 0xDBFF and 0xDC00 <= ch2 <= 0xDFFF: - return (((ch1 - 0xD800) << 10) | (ch2 - 0xDC00)) + 0x10000 - return ord(u) + return ord_accepts_surrogate(u) ORD._flowspace_rewrite_directly_as_ = ord else: From noreply at buildbot.pypy.org Sun Jan 27 19:07:44 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 27 Jan 2013 19:07:44 +0100 (CET) Subject: [pypy-commit] pypy default: merge Message-ID: <20130127180744.0A00A1C1306@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r60528:4c07d8ff67fd Date: 2013-01-27 20:04 +0200 http://bitbucket.org/pypy/pypy/changeset/4c07d8ff67fd/ Log: merge diff too long, truncating to 2000 out of 2528 lines diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -713,13 +713,13 @@ def pypy_find_stdlib(s): from os.path import abspath, join, dirname as dn thisfile = abspath(__file__) - root = dn(dn(dn(dn(thisfile)))) + root = dn(dn(dn(thisfile))) return [join(root, 'lib-python', '2.7'), join(root, 'lib_pypy')] def pypy_resolvedirof(s): # we ignore the issue of symlinks; for tests, the executable is always - # translator/goal/app_main.py anyway + # interpreter/app_main.py anyway import os return os.path.abspath(os.path.join(s, '..')) diff --git a/pypy/interpreter/test2/test_app_main.py b/pypy/interpreter/test2/test_app_main.py --- a/pypy/interpreter/test2/test_app_main.py +++ b/pypy/interpreter/test2/test_app_main.py @@ -547,7 +547,7 @@ assert line.rstrip() == 'Not at all. They could be carried.' print 'A five ounce bird could not carry a one pound coconut.' """) - py_py = os.path.join(pypydir, 'bin', 'py.py') + py_py = os.path.join(pypydir, 'bin', 'pyinteractive.py') child = self._spawn(sys.executable, [py_py, '-S', path]) child.expect('Are you suggesting coconuts migrate?', timeout=120) child.sendline('Not at all. They could be carried.') @@ -907,7 +907,7 @@ sys.path.append(self.goal_dir) try: import app_main - pypy_c = os.path.join(self.trunkdir, 'pypy', 'translator', 'goal', 'pypy-c') + pypy_c = os.path.join(self.trunkdir, 'pypy', 'goal', 'pypy-c') app_main.setup_bootstrap_path(pypy_c) newpath = sys.path[:] # we get at least lib_pypy @@ -925,7 +925,7 @@ sys.path.append(self.goal_dir) try: import app_main - pypy_c = os.path.join(self.trunkdir, 'pypy', 'translator', 'goal', 'pypy-c') + pypy_c = os.path.join(self.trunkdir, 'pypy', 'goal', 'pypy-c') app_main.entry_point(pypy_c, [self.foo_py]) # assert it did not crash finally: diff --git a/pypy/module/signal/test/test_signal.py b/pypy/module/signal/test/test_signal.py --- a/pypy/module/signal/test/test_signal.py +++ b/pypy/module/signal/test/test_signal.py @@ -36,7 +36,7 @@ class AppTestSignal: spaceconfig = { - "usemodules": ['signal', 'rctime'], + "usemodules": ['signal', 'rctime'] + (['fcntl'] if os.name != 'nt' else []), } def setup_class(cls): diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -29,7 +29,7 @@ import pypy pypydir = os.path.dirname(os.path.abspath(pypy.__file__)) del pypy -from pypy.tool.version import get_repo_version_info +from rpython.tool.version import get_repo_version_info import time as t gmtime = t.gmtime() diff --git a/pypy/module/thread/test/test_thread.py b/pypy/module/thread/test/test_thread.py --- a/pypy/module/thread/test/test_thread.py +++ b/pypy/module/thread/test/test_thread.py @@ -220,16 +220,24 @@ import signal def f(): - time.sleep(0.5) - thread.interrupt_main() + for x in range(50): + if waiting: + thread.interrupt_main() + return + print 'tock...', x # <-force the GIL to be released, as + time.sleep(0.01) # time.sleep doesn't do non-translated def busy_wait(): - for x in range(1000): + waiting.append(None) + for x in range(100): print 'tick...', x # <-force the GIL to be released, as time.sleep(0.01) # time.sleep doesn't do non-translated + waiting.pop() # This is normally called by app_main.py signal.signal(signal.SIGINT, signal.default_int_handler) - thread.start_new_thread(f, ()) - raises(KeyboardInterrupt, busy_wait) + for i in range(100): + waiting = [] + thread.start_new_thread(f, ()) + raises(KeyboardInterrupt, busy_wait) diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -1,6 +1,6 @@ #!/usr/bin/env python """ A sample script that packages PyPy, provided that it's already built. -It uses 'pypy/translator/goal/pypy-c' and parts of the rest of the working +It uses 'pypy/goal/pypy-c' and parts of the rest of the working copy. Usage: package.py root-pypy-dir [name-of-archive] [name-of-pypy-c] [destination-for-tarball] [pypy-c-path] diff --git a/pypy/tool/release/test/test_package.py b/pypy/tool/release/test/test_package.py --- a/pypy/tool/release/test/test_package.py +++ b/pypy/tool/release/test/test_package.py @@ -15,7 +15,7 @@ basename = 'pypy-c' rename_pypy_c = 'pypy' exe_name_in_archive = 'bin/pypy' - pypy_c = py.path.local(pypydir).join('..', basename) + pypy_c = py.path.local(pypydir).join('goal', basename) if not pypy_c.check(): os.system("echo faked_pypy_c> %s" % (pypy_c,)) pypy_c.chmod(0755) diff --git a/pypy/tool/rundictbenchmarks.py b/pypy/tool/rundictbenchmarks.py --- a/pypy/tool/rundictbenchmarks.py +++ b/pypy/tool/rundictbenchmarks.py @@ -3,7 +3,7 @@ # this file runs some benchmarks with a pypy-c that is assumed to be # built using the MeasuringDictImplementation. -# it should be run with pypy/translator/goal as the cwd, and you'll +# it should be run with pypy/goal as the cwd, and you'll # need to hack a copy of rst2html for yourself (svn docutils # required). diff --git a/rpython/annotator/builtin.py b/rpython/annotator/builtin.py --- a/rpython/annotator/builtin.py +++ b/rpython/annotator/builtin.py @@ -79,7 +79,7 @@ s_item = s_ImpossibleValue else: nonneg = False # so far - if step > 0: + if step > 0 or s_step.nonneg: nonneg = s_start.nonneg elif step < 0: nonneg = s_stop.nonneg or (s_stop.is_constant() and diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py --- a/rpython/annotator/test/test_annrpython.py +++ b/rpython/annotator/test/test_annrpython.py @@ -2694,6 +2694,23 @@ assert isinstance(s, annmodel.SomeInteger) assert s.nonneg + def test_range_nonneg_variablestep(self): + def get_step(n): + if n == 1: + return 2 + else: + return 3 + def fun(n, k): + step = get_step(n) + for i in range(0, n, step): + if k == 17: + return i + return 0 + a = self.RPythonAnnotator() + s = a.build_types(fun, [int, int]) + assert isinstance(s, annmodel.SomeInteger) + assert s.nonneg + def test_reverse_range_nonneg(self): def fun(n, k): for i in range(n-1, -1, -1): @@ -3254,6 +3271,7 @@ a = self.RPythonAnnotator() s = a.build_types(f, [str]) assert isinstance(s, annmodel.SomeString) + assert s.no_nul def f(x): return u'a'.replace(x, u'b') diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py --- a/rpython/annotator/unaryop.py +++ b/rpython/annotator/unaryop.py @@ -509,7 +509,7 @@ return getbookkeeper().newlist(s_item) def method_replace(str, s1, s2): - return str.basestringclass() + return str.basestringclass(no_nul=str.no_nul and s2.no_nul) def getslice(str, s_start, s_stop): check_negative_slice(s_start, s_stop) diff --git a/rpython/jit/backend/llgraph/llimpl.py b/rpython/jit/backend/llgraph/llimpl.py deleted file mode 100644 --- a/rpython/jit/backend/llgraph/llimpl.py +++ /dev/null @@ -1,1968 +0,0 @@ -""" -The non-RPythonic part of the llgraph backend. -This contains all the code that is directly run -when executing on top of the llinterpreter. -""" - -import weakref -from rpython.flowspace.model import Variable, Constant -from rpython.annotator import model as annmodel -from rpython.jit.metainterp.history import REF, INT, FLOAT -from rpython.jit.metainterp import history -from rpython.jit.codewriter import heaptracker -from rpython.rtyper.lltypesystem import lltype, llmemory, rclass, rstr, rffi -from rpython.rtyper.ootypesystem import ootype -from rpython.rtyper.module.support import LLSupport, OOSupport -from rpython.rtyper.llinterp import LLException -from rpython.rtyper.extregistry import ExtRegistryEntry - -from rpython.jit.metainterp import resoperation -from rpython.jit.metainterp.resoperation import rop -from rpython.jit.backend.llgraph import symbolic -from rpython.jit.codewriter import longlong -from rpython.jit.codewriter.effectinfo import EffectInfo - -from rpython.rlib.objectmodel import ComputedIntSymbolic, we_are_translated -from rpython.rlib.rarithmetic import ovfcheck -from rpython.rlib.rarithmetic import r_longlong, r_ulonglong, r_uint -from rpython.rlib.rtimer import read_timestamp - -import py -from rpython.tool.ansi_print import ansi_log -log = py.log.Producer('runner') -py.log.setconsumer('runner', ansi_log) - -IS_32_BIT = r_ulonglong is not r_uint - - -def _from_opaque(opq): - return opq._obj.externalobj - -_TO_OPAQUE = {} - -def _to_opaque(value): - try: - return value._the_opaque_pointer - except AttributeError: - op = lltype.opaqueptr(_TO_OPAQUE[value.__class__], 'opaque', - externalobj=value) - value._the_opaque_pointer = op - return op - -def _normalize(value): - if isinstance(value, lltype._ptr): - value = lltype.top_container(value._obj) - return value - -def from_opaque_string(s): - if isinstance(s, str): - return s - elif isinstance(s, ootype._string): - return OOSupport.from_rstr(s) - else: - return LLSupport.from_rstr(s) - -FLOAT_ARRAY_TP = lltype.Ptr(lltype.Array(lltype.Float, hints={"nolength": True})) -def maybe_uncast(TP, array): - if array._TYPE.TO.OF != lltype.Float: - # array._TYPE.TO._hints.get("uncast_on_llgraph"): - array = rffi.cast(TP, array) - return array - -# a list of argtypes of all operations - couldn't find any and it's -# very useful. Note however that the table is half-broken here and -# there, in ways that are sometimes a bit hard to fix; that's why -# it is not "official". -TYPES = { - 'int_add' : (('int', 'int'), 'int'), - 'int_sub' : (('int', 'int'), 'int'), - 'int_mul' : (('int', 'int'), 'int'), - 'int_floordiv' : (('int', 'int'), 'int'), - 'int_mod' : (('int', 'int'), 'int'), - 'int_and' : (('int', 'int'), 'int'), - 'int_or' : (('int', 'int'), 'int'), - 'int_xor' : (('int', 'int'), 'int'), - 'int_lshift' : (('int', 'int'), 'int'), - 'int_rshift' : (('int', 'int'), 'int'), - 'int_lt' : (('int', 'int'), 'bool'), - 'int_gt' : (('int', 'int'), 'bool'), - 'int_ge' : (('int', 'int'), 'bool'), - 'int_le' : (('int', 'int'), 'bool'), - 'int_eq' : (('int', 'int'), 'bool'), - 'int_ne' : (('int', 'int'), 'bool'), - 'int_is_true' : (('int',), 'bool'), - 'int_is_zero' : (('int',), 'bool'), - 'int_neg' : (('int',), 'int'), - 'int_invert' : (('int',), 'int'), - 'int_add_ovf' : (('int', 'int'), 'int'), - 'int_sub_ovf' : (('int', 'int'), 'int'), - 'int_mul_ovf' : (('int', 'int'), 'int'), - 'int_force_ge_zero':(('int',), 'int'), - 'uint_add' : (('int', 'int'), 'int'), - 'uint_sub' : (('int', 'int'), 'int'), - 'uint_mul' : (('int', 'int'), 'int'), - 'uint_lt' : (('int', 'int'), 'bool'), - 'uint_le' : (('int', 'int'), 'bool'), - 'uint_eq' : (('int', 'int'), 'bool'), - 'uint_ne' : (('int', 'int'), 'bool'), - 'uint_gt' : (('int', 'int'), 'bool'), - 'uint_ge' : (('int', 'int'), 'bool'), - 'uint_xor' : (('int', 'int'), 'int'), - 'uint_rshift' : (('int', 'int'), 'int'), - 'uint_floordiv' : (('int', 'int'), 'int'), - 'float_add' : (('float', 'float'), 'float'), - 'float_sub' : (('float', 'float'), 'float'), - 'float_mul' : (('float', 'float'), 'float'), - 'float_truediv' : (('float', 'float'), 'float'), - 'float_lt' : (('float', 'float'), 'bool'), - 'float_le' : (('float', 'float'), 'bool'), - 'float_eq' : (('float', 'float'), 'bool'), - 'float_ne' : (('float', 'float'), 'bool'), - 'float_gt' : (('float', 'float'), 'bool'), - 'float_ge' : (('float', 'float'), 'bool'), - 'float_neg' : (('float',), 'float'), - 'float_abs' : (('float',), 'float'), - 'cast_float_to_int':(('float',), 'int'), - 'cast_int_to_float':(('int',), 'float'), - 'same_as' : (('int',), 'int'), # could also be ptr=>ptr - 'new_with_vtable' : (('ref',), 'ref'), - 'new' : ((), 'ref'), - 'new_array' : (('int',), 'ref'), - 'oois' : (('ref', 'ref'), 'bool'), - 'ooisnot' : (('ref', 'ref'), 'bool'), - 'instanceof' : (('ref',), 'bool'), - 'subclassof' : (('ref', 'ref'), 'bool'), - 'runtimenew' : (('ref',), 'ref'), - 'setfield_gc' : (('ref', 'intorptr'), None), - 'getfield_gc' : (('ref',), 'intorptr'), - 'getfield_gc_pure': (('ref',), 'intorptr'), - 'setfield_raw' : (('ref', 'intorptr'), None), - 'getfield_raw' : (('ref',), 'intorptr'), - 'getfield_raw_pure': (('ref',), 'intorptr'), - 'setarrayitem_gc' : (('ref', 'int', 'intorptr'), None), - 'getarrayitem_gc' : (('ref', 'int'), 'intorptr'), - 'getarrayitem_gc_pure' : (('ref', 'int'), 'intorptr'), - 'setarrayitem_raw' : (('ref', 'int', 'intorptr'), None), - 'getarrayitem_raw' : (('ref', 'int'), 'intorptr'), - 'getarrayitem_raw_pure' : (('ref', 'int'), 'intorptr'), - 'arraylen_gc' : (('ref',), 'int'), - 'call' : (('ref', 'varargs'), 'intorptr'), - 'call_assembler' : (('varargs',), 'intorptr'), - 'cond_call_gc_wb' : (('ptr', 'ptr'), None), - 'cond_call_gc_wb_array': (('ptr', 'int', 'ptr'), None), - 'oosend' : (('varargs',), 'intorptr'), - 'oosend_pure' : (('varargs',), 'intorptr'), - 'guard_true' : (('bool',), None), - 'guard_false' : (('bool',), None), - 'guard_value' : (('int', 'int'), None), - 'guard_class' : (('ref', 'ref'), None), - 'guard_no_exception' : ((), None), - 'guard_exception' : (('ref',), 'ref'), - 'guard_no_overflow' : ((), None), - 'guard_overflow' : ((), None), - 'guard_nonnull' : (('ref',), None), - 'guard_isnull' : (('ref',), None), - 'guard_nonnull_class' : (('ref', 'ref'), None), - 'newstr' : (('int',), 'ref'), - 'strlen' : (('ref',), 'int'), - 'strgetitem' : (('ref', 'int'), 'int'), - 'strsetitem' : (('ref', 'int', 'int'), None), - 'newunicode' : (('int',), 'ref'), - 'unicodelen' : (('ref',), 'int'), - 'unicodegetitem' : (('ref', 'int'), 'int'), - 'unicodesetitem' : (('ref', 'int', 'int'), 'int'), - 'cast_ptr_to_int' : (('ref',), 'int'), - 'cast_int_to_ptr' : (('int',), 'ref'), - 'debug_merge_point': (('ref', 'int', 'int'), None), - 'force_token' : ((), 'int'), - 'call_may_force' : (('int', 'varargs'), 'intorptr'), - 'guard_not_forced': ((), None), -} - -# ____________________________________________________________ - -class CompiledLoop(object): - has_been_freed = False - invalid = False - - def __init__(self): - self.inputargs = [] - self.operations = [] - - def getargtypes(self): - return [v.concretetype for v in self.inputargs] - - def __repr__(self): - lines = [] - self.as_text(lines, 1) - return 'CompiledLoop %s:\n%s' % (self.inputargs, '\n'.join(lines)) - - def as_text(self, lines, indent): - for op in self.operations: - lines.append('\t'*indent + repr(op)) - -class Operation(object): - result = None - descr = None - jump_target = None - fail_args = None - - def __init__(self, opnum): - self.opnum = opnum - self.args = [] - - def __repr__(self): - if self.result is not None: - sres = repr0(self.result) + ' = ' - else: - sres = '' - return '{%s%s(%s)}' % (sres, self.getopname(), - ', '.join(map(repr0, self.args))) - - def getopname(self): - try: - return resoperation.opname[self.opnum] - except KeyError: - return '<%d>' % self.opnum - - def is_guard(self): - return rop._GUARD_FIRST <= self.opnum <= rop._GUARD_LAST - - def is_final(self): - return rop._FINAL_FIRST <= self.opnum <= rop._FINAL_LAST - -def repr0(x): - if isinstance(x, list): - return '[' + ', '.join(repr0(y) for y in x) + ']' - elif isinstance(x, Constant): - return '(' + repr0(x.value) + ')' - elif isinstance(x, lltype._ptr): - x = llmemory.cast_ptr_to_adr(x) - if x.ptr: - try: - container = x.ptr._obj._normalizedcontainer() - return '* %s' % (container._TYPE._short_name(),) - except AttributeError: - return repr(x) - else: - return 'NULL' - else: - return repr(x) - -def repr_list(lst, types): - res_l = [] - if types and types[-1] == 'varargs': - types = types[:-1] + ('int',) * (len(lst) - len(types) + 1) - assert len(types) == len(lst) - for elem, tp in zip(lst, types): - if isinstance(elem, Constant): - res_l.append('(%s)' % repr1(elem, tp)) - else: - res_l.append(repr1(elem, tp)) - return '[%s]' % (', '.join(res_l)) - -def repr1(x, tp): - if tp == "intorptr": - TYPE = lltype.typeOf(x) - if isinstance(TYPE, lltype.Ptr) and TYPE.TO._gckind == 'gc': - tp = "ref" - else: - tp = "int" - if tp == 'int': - return str(x) - elif tp == 'void': - return '---' - elif tp == 'ref': - if not x: - return '(* None)' - if isinstance(x, int): - # XXX normalize? - ptr = str(llmemory.cast_int_to_adr(x)) - elif isinstance(ootype.typeOf(x), ootype.OOType): - return repr(x) - else: - if getattr(x, '_fake', None): - return repr(x) - if lltype.typeOf(x) == llmemory.GCREF: - TP = lltype.Ptr(lltype.typeOf(x._obj.container)) - ptr = lltype.cast_opaque_ptr(TP, x) - else: - ptr = x - try: - container = ptr._obj._normalizedcontainer() - return '(* %s)' % (container._TYPE._short_name(),) - except AttributeError: - return '(%r)' % (ptr,) - elif tp == 'bool': - assert x == 0 or x == 1 - return str(bool(x)) - #elif tp == 'fieldname': - # return str(symbolic.TokenToField[x...][1]) - elif tp == 'float': - return str(x) - else: - raise NotImplementedError("tp = %s" % tp) - -_variables = [] - -def compile_start(): - del _variables[:] - return _to_opaque(CompiledLoop()) - -def mark_as_free(loop): - loop = _from_opaque(loop) - assert not loop.has_been_freed - loop.has_been_freed = True - -def compile_start_int_var(loop): - return compile_start_ref_var(loop, lltype.Signed) - -def compile_start_float_var(loop): - return compile_start_ref_var(loop, longlong.FLOATSTORAGE) - -def compile_start_ref_var(loop, TYPE): - loop = _from_opaque(loop) - assert not loop.operations - v = Variable() - v.concretetype = TYPE - loop.inputargs.append(v) - r = len(_variables) - _variables.append(v) - return r - -def compile_started_vars(clt): - if not hasattr(clt, '_debug_argtypes'): # only when compiling the loop - argtypes = [v.concretetype for v in _variables] - try: - clt._debug_argtypes = argtypes - except AttributeError: # when 'clt' is actually a translated - pass # GcStruct - -def compile_add(loop, opnum): - loop = _from_opaque(loop) - loop.operations.append(Operation(opnum)) - -def compile_add_descr(loop, ofs, type, arg_types, extrainfo, width): - from rpython.jit.backend.llgraph.runner import Descr - loop = _from_opaque(loop) - op = loop.operations[-1] - assert isinstance(type, str) and len(type) == 1 - op.descr = Descr(ofs, type, arg_types=arg_types, extrainfo=extrainfo, width=width) - -def compile_add_descr_arg(loop, ofs, type, arg_types): - from rpython.jit.backend.llgraph.runner import Descr - loop = _from_opaque(loop) - op = loop.operations[-1] - assert isinstance(type, str) and len(type) == 1 - op.args.append(Descr(ofs, type, arg_types=arg_types)) - -def compile_add_loop_token(loop, descr): - if we_are_translated(): - raise ValueError("CALL_ASSEMBLER not supported") - loop = _from_opaque(loop) - op = loop.operations[-1] - op.descr = weakref.ref(descr) - -TARGET_TOKENS = weakref.WeakKeyDictionary() - -def compile_add_target_token(loop, descr, clt): - # here, 'clt' is the compiled_loop_token of the original loop that - # we are compiling - loop = _from_opaque(loop) - op = loop.operations[-1] - descrobj = _normalize(descr) - TARGET_TOKENS[descrobj] = loop, len(loop.operations), op.args, clt - -def compile_add_var(loop, intvar): - loop = _from_opaque(loop) - op = loop.operations[-1] - op.args.append(_variables[intvar]) - -def compile_add_int_const(loop, value): - compile_add_ref_const(loop, value, lltype.Signed) - -def compile_add_float_const(loop, value): - compile_add_ref_const(loop, value, longlong.FLOATSTORAGE) - -def compile_add_ref_const(loop, value, TYPE): - loop = _from_opaque(loop) - const = Constant(value) - const.concretetype = TYPE - op = loop.operations[-1] - op.args.append(const) - -def compile_add_int_result(loop): - return compile_add_ref_result(loop, lltype.Signed) - -def compile_add_float_result(loop): - return compile_add_ref_result(loop, longlong.FLOATSTORAGE) - -def compile_add_ref_result(loop, TYPE): - loop = _from_opaque(loop) - v = Variable() - v.concretetype = TYPE - op = loop.operations[-1] - op.result = v - r = len(_variables) - _variables.append(v) - return r - -def compile_add_jump_target(loop, targettoken, source_clt): - loop = _from_opaque(loop) - descrobj = _normalize(targettoken) - (loop_target, target_opindex, target_inputargs, target_clt - ) = TARGET_TOKENS[descrobj] - # - try: - assert source_clt._debug_argtypes == target_clt._debug_argtypes - except AttributeError: # when translated - pass - # - op = loop.operations[-1] - op.jump_target = loop_target - op.jump_target_opindex = target_opindex - op.jump_target_inputargs = target_inputargs - assert op.opnum == rop.JUMP - assert [v.concretetype for v in op.args] == ( - [v.concretetype for v in target_inputargs]) - # - if loop_target == loop: - log.info("compiling new loop") - else: - log.info("compiling new bridge") - -def compile_add_guard_jump_target(loop, loop_target): - loop = _from_opaque(loop) - loop_target = _from_opaque(loop_target) - op = loop.operations[-1] - assert op.is_guard() - op.jump_target = loop_target - -def compile_add_fail(loop, fail_index): - loop = _from_opaque(loop) - index = len(loop.operations)-1 - op = loop.operations[index] - op.fail_index = fail_index - return index - -def compile_add_fail_arg(loop, intvar): - loop = _from_opaque(loop) - op = loop.operations[-1] - if op.fail_args is None: - op.fail_args = [] - if intvar == -1: - op.fail_args.append(None) - else: - op.fail_args.append(_variables[intvar]) - -def compile_redirect_fail(old_loop, old_index, new_loop): - old_loop = _from_opaque(old_loop) - new_loop = _from_opaque(new_loop) - guard_op = old_loop.operations[old_index] - assert guard_op.is_guard() - guard_op.jump_target = new_loop - # check that the bridge's inputargs are of the correct number and - # kind for the guard - if guard_op.fail_args is not None: - argkinds = [v.concretetype for v in guard_op.fail_args if v] - else: - argkinds = [] - assert argkinds == [v.concretetype for v in new_loop.inputargs] - -# ------------------------------ - -class Frame(object): - OPHANDLERS = [None] * (rop._LAST+1) - - def __init__(self, cpu): - self.verbose = False - self.cpu = cpu - self.opindex = 1 - self._forced = False - self._may_force = -1 - - def getenv(self, v): - from rpython.jit.backend.llgraph.runner import Descr - if isinstance(v, Constant): - return v.value - elif isinstance(v, Descr): - return v - else: - return self.env[v] - - def _populate_fail_args(self, op, skip=None): - fail_args = [] - if op.fail_args: - for fail_arg in op.fail_args: - if fail_arg is None: - fail_args.append(None) - elif fail_arg is skip: - fail_args.append(fail_arg.concretetype._defl()) - else: - fail_args.append(self.getenv(fail_arg)) - self.fail_args = fail_args - self.fail_index = op.fail_index - - def execute(self): - """Execute all operations in a loop, - possibly following to other loops as well. - """ - global _last_exception - assert _last_exception is None, "exception left behind" - verbose = True - self.opindex = 0 - while True: - assert not self.loop.has_been_freed - op = self.loop.operations[self.opindex] - args = [self.getenv(v) for v in op.args] - if not op.is_final(): - try: - result = self.execute_operation(op.opnum, args, op.descr, - verbose) - except GuardFailed: - assert op.is_guard() - _stats.exec_conditional_jumps += 1 - if op.jump_target is not None: - # a patched guard, pointing to further code - if op.fail_args: - args = [self.getenv(v) for v in op.fail_args if v] - else: - args = [] - assert len(op.jump_target.inputargs) == len(args) - self.env = dict(zip(op.jump_target.inputargs, args)) - self.loop = op.jump_target - self.opindex = 0 - continue - else: - self._populate_fail_args(op) - # a non-patched guard - if self.verbose: - log.trace('failed: %s' % ( - ', '.join(map(str, fail_args)),)) - return op.fail_index - #verbose = self.verbose - assert (result is None) == (op.result is None) - if op.result is not None: - RESTYPE = op.result.concretetype - if RESTYPE is lltype.Signed: - x = self.as_int(result) - elif RESTYPE is llmemory.GCREF: - x = self.as_ptr(result) - elif RESTYPE is ootype.Object: - x = self.as_object(result) - elif RESTYPE is longlong.FLOATSTORAGE: - x = self.as_floatstorage(result) - else: - raise Exception("op.result.concretetype is %r" - % (RESTYPE,)) - self.env[op.result] = x - self.opindex += 1 - continue - if op.opnum == rop.JUMP: - inputargs = op.jump_target_inputargs - assert len(inputargs) == len(args) - self.env = dict(zip(inputargs, args)) - self.loop = op.jump_target - self.opindex = op.jump_target_opindex - _stats.exec_jumps += 1 - elif op.opnum == rop.FINISH: - if self.verbose: - log.trace('finished: %s' % ( - ', '.join(map(str, args)),)) - self.fail_args = args - return op.fail_index - - else: - assert 0, "unknown final operation %d" % (op.opnum,) - - def execute_operation(self, opnum, values, descr, verbose): - """Execute a single operation. - """ - ophandler = self.OPHANDLERS[opnum] - if ophandler is None: - self._define_impl(opnum) - ophandler = self.OPHANDLERS[opnum] - assert ophandler is not None, "missing impl for op %d" % opnum - opname = resoperation.opname[opnum].lower() - exec_counters = _stats.exec_counters - exec_counters[opname] = exec_counters.get(opname, 0) + 1 - for i in range(len(values)): - if isinstance(values[i], ComputedIntSymbolic): - values[i] = values[i].compute_fn() - res = NotImplemented - try: - res = ophandler(self, descr, *values) - finally: - if 0: # if verbose: - argtypes, restype = TYPES[opname] - if res is None: - resdata = '' - elif res is NotImplemented: - resdata = '*fail*' - else: - resdata = '-> ' + repr1(res, restype) - # fish the types - log.cpu('\t%s %s %s' % (opname, repr_list(values, argtypes), - resdata)) - return res - - def as_int(self, x): - return cast_to_int(x) - - def as_ptr(self, x): - return cast_to_ptr(x) - - def as_object(self, x): - return ootype.cast_to_object(x) - - def as_floatstorage(self, x): - return cast_to_floatstorage(x) - - def log_progress(self): - count = sum(_stats.exec_counters.values()) - count_jumps = _stats.exec_jumps - log.trace('ran %d operations, %d jumps' % (count, count_jumps)) - - # ---------- - - @classmethod - def _define_impl(cls, opnum): - opname = resoperation.opname[opnum] - try: - op = getattr(cls, 'op_' + opname.lower()) # op_guard_true etc. - except AttributeError: - try: - impl = globals()['do_' + opname.lower()] # do_arraylen_gc etc. - def op(self, descr, *args): - if descr is None: - return impl(*args) - else: - return impl(descr, *args) - except KeyError: - op = cls._make_impl_from_blackhole_interp(opname) - cls.OPHANDLERS[opnum] = op - - @classmethod - def _make_impl_from_blackhole_interp(cls, opname): - from rpython.jit.metainterp.blackhole import BlackholeInterpreter - name = 'bhimpl_' + opname.lower() - func = BlackholeInterpreter.__dict__[name] - for argtype in func.argtypes: - assert argtype in ('i', 'r', 'f') - # - def _op_default_implementation(self, descr, *args): - # for all operations implemented in the blackhole interpreter - return func(*args) - # - return _op_default_implementation - - def op_label(self, _, *args): - op = self.loop.operations[self.opindex] - assert op.opnum == rop.LABEL - assert len(op.args) == len(args) - newenv = {} - for v, value in zip(op.args, args): - newenv[v] = value - self.env = newenv - - def op_debug_merge_point(self, _, *args): - from rpython.jit.metainterp.warmspot import get_stats - try: - stats = get_stats() - except AttributeError: - pass - else: - stats.add_merge_point_location(args[1:]) - pass - - def op_guard_true(self, _, value): - if not value: - raise GuardFailed - - def op_guard_false(self, _, value): - if value: - raise GuardFailed - - op_guard_nonnull = op_guard_true - op_guard_isnull = op_guard_false - - def op_guard_class(self, _, value, expected_class): - value = lltype.cast_opaque_ptr(rclass.OBJECTPTR, value) - expected_class = llmemory.cast_adr_to_ptr( - llmemory.cast_int_to_adr(expected_class), - rclass.CLASSTYPE) - if value.typeptr != expected_class: - raise GuardFailed - - def op_guard_nonnull_class(self, _, value, expected_class): - if not value: - raise GuardFailed - self.op_guard_class(_, value, expected_class) - - def op_guard_value(self, _, value, expected_value): - if value != expected_value: - raise GuardFailed - - def op_guard_no_exception(self, _): - if _last_exception: - raise GuardFailed - - def _check_exception(self, expected_exception): - global _last_exception - expected_exception = self._cast_exception(expected_exception) - assert expected_exception - exc = _last_exception - if exc: - got = exc.args[0] - # exact match! - if got != expected_exception: - return False - return True - else: - return False - - def _cast_exception(self, exception): - return llmemory.cast_adr_to_ptr( - llmemory.cast_int_to_adr(exception), - rclass.CLASSTYPE) - - def _issubclass(self, cls1, cls2): - return rclass.ll_issubclass(cls1, cls2) - - def op_guard_exception(self, _, expected_exception): - global _last_exception - if not self._check_exception(expected_exception): - raise GuardFailed - res = _last_exception[1] - _last_exception = None - return res - - def op_guard_no_overflow(self, _): - flag = self.overflow_flag - del self.overflow_flag - if flag: - raise GuardFailed - - def op_guard_overflow(self, _): - flag = self.overflow_flag - del self.overflow_flag - if not flag: - raise GuardFailed - - def op_int_add_ovf(self, _, x, y): - try: - z = ovfcheck(x + y) - except OverflowError: - ovf = True - z = 0 - else: - ovf = False - self.overflow_flag = ovf - return z - - def op_int_sub_ovf(self, _, x, y): - try: - z = ovfcheck(x - y) - except OverflowError: - ovf = True - z = 0 - else: - ovf = False - self.overflow_flag = ovf - return z - - def op_int_mul_ovf(self, _, x, y): - try: - z = ovfcheck(x * y) - except OverflowError: - ovf = True - z = 0 - else: - ovf = False - self.overflow_flag = ovf - return z - - def op_keepalive(self, _, x): - pass - - # ---------- - # delegating to the builtins do_xxx() (done automatically for simple cases) - - def op_getarrayitem_gc(self, arraydescr, array, index): - if arraydescr.typeinfo == REF: - return do_getarrayitem_gc_ptr(array, index) - elif arraydescr.typeinfo == INT: - return do_getarrayitem_gc_int(array, index) - elif arraydescr.typeinfo == FLOAT: - return do_getarrayitem_gc_float(array, index) - else: - raise NotImplementedError - - op_getarrayitem_gc_pure = op_getarrayitem_gc - - def op_getarrayitem_raw(self, arraydescr, array, index): - if arraydescr.typeinfo == REF: - raise NotImplementedError("getarrayitem_raw -> gcref") - elif arraydescr.typeinfo == INT: - return do_getarrayitem_raw_int(array, index, arraydescr.ofs) - elif arraydescr.typeinfo == FLOAT: - return do_getarrayitem_raw_float(array, index) - else: - raise NotImplementedError - - op_getarrayitem_raw_pure = op_getarrayitem_raw - - def op_getfield_gc(self, fielddescr, struct): - if fielddescr.typeinfo == REF: - return do_getfield_gc_ptr(struct, fielddescr.ofs) - elif fielddescr.typeinfo == INT: - return do_getfield_gc_int(struct, fielddescr.ofs) - elif fielddescr.typeinfo == FLOAT: - return do_getfield_gc_float(struct, fielddescr.ofs) - else: - raise NotImplementedError - - op_getfield_gc_pure = op_getfield_gc - - def op_getfield_raw(self, fielddescr, struct): - if fielddescr.typeinfo == REF: - return do_getfield_raw_ptr(struct, fielddescr.ofs) - elif fielddescr.typeinfo == INT: - return do_getfield_raw_int(struct, fielddescr.ofs) - elif fielddescr.typeinfo == FLOAT: - return do_getfield_raw_float(struct, fielddescr.ofs) - else: - raise NotImplementedError - - op_getfield_raw_pure = op_getfield_raw - - def op_raw_store(self, arraydescr, addr, offset, value): - if arraydescr.typeinfo == REF: - raise AssertionError("cannot store GC pointer in raw storage") - elif arraydescr.typeinfo == INT: - do_raw_store_int(addr, offset, arraydescr.ofs, value) - elif arraydescr.typeinfo == FLOAT: - do_raw_store_float(addr, offset, value) - else: - raise NotImplementedError - - def op_raw_load(self, arraydescr, addr, offset): - if arraydescr.typeinfo == REF: - raise AssertionError("cannot store GC pointer in raw storage") - elif arraydescr.typeinfo == INT: - return do_raw_load_int(addr, offset, arraydescr.ofs) - elif arraydescr.typeinfo == FLOAT: - return do_raw_load_float(addr, offset) - else: - raise NotImplementedError - - def op_new(self, size): - return do_new(size.ofs) - - def op_new_with_vtable(self, descr, vtable): - assert descr is None - descr = heaptracker.vtable2descr(self.cpu, vtable) - result = do_new(descr.ofs) - value = lltype.cast_opaque_ptr(rclass.OBJECTPTR, result) - value.typeptr = cast_from_int(rclass.CLASSTYPE, vtable) - return result - - def op_setarrayitem_gc(self, arraydescr, array, index, newvalue): - if arraydescr.typeinfo == REF: - do_setarrayitem_gc_ptr(array, index, newvalue) - elif arraydescr.typeinfo == INT: - do_setarrayitem_gc_int(array, index, newvalue) - elif arraydescr.typeinfo == FLOAT: - do_setarrayitem_gc_float(array, index, newvalue) - else: - raise NotImplementedError - - def op_setarrayitem_raw(self, arraydescr, array, index, newvalue): - if arraydescr.typeinfo == REF: - raise NotImplementedError("setarrayitem_raw <- gcref") - elif arraydescr.typeinfo == INT: - do_setarrayitem_raw_int(array, index, newvalue, arraydescr.ofs) - elif arraydescr.typeinfo == FLOAT: - do_setarrayitem_raw_float(array, index, newvalue) - else: - raise NotImplementedError - - def op_getinteriorfield_gc(self, descr, array, index): - if descr.typeinfo == REF: - return do_getinteriorfield_gc_ptr(array, index, descr.ofs) - elif descr.typeinfo == INT: - return do_getinteriorfield_gc_int(array, index, descr.ofs) - elif descr.typeinfo == FLOAT: - return do_getinteriorfield_gc_float(array, index, descr.ofs) - else: - raise NotImplementedError - - def op_getinteriorfield_raw(self, descr, array, index): - if descr.typeinfo == REF: - return do_getinteriorfield_raw_ptr(array, index, descr.width, descr.ofs) - elif descr.typeinfo == INT: - return do_getinteriorfield_raw_int(array, index, descr.width, descr.ofs) - elif descr.typeinfo == FLOAT: - return do_getinteriorfield_raw_float(array, index, descr.width, descr.ofs) - else: - raise NotImplementedError - - def op_setinteriorfield_gc(self, descr, array, index, newvalue): - if descr.typeinfo == REF: - return do_setinteriorfield_gc_ptr(array, index, descr.ofs, - newvalue) - elif descr.typeinfo == INT: - return do_setinteriorfield_gc_int(array, index, descr.ofs, - newvalue) - elif descr.typeinfo == FLOAT: - return do_setinteriorfield_gc_float(array, index, descr.ofs, - newvalue) - else: - raise NotImplementedError - - def op_setinteriorfield_raw(self, descr, array, index, newvalue): - if descr.typeinfo == REF: - return do_setinteriorfield_raw_ptr(array, index, newvalue, descr.width, descr.ofs) - elif descr.typeinfo == INT: - return do_setinteriorfield_raw_int(array, index, newvalue, descr.width, descr.ofs) - elif descr.typeinfo == FLOAT: - return do_setinteriorfield_raw_float(array, index, newvalue, descr.width, descr.ofs) - else: - raise NotImplementedError - - def op_setfield_gc(self, fielddescr, struct, newvalue): - if fielddescr.typeinfo == REF: - do_setfield_gc_ptr(struct, fielddescr.ofs, newvalue) - elif fielddescr.typeinfo == INT: - do_setfield_gc_int(struct, fielddescr.ofs, newvalue) - elif fielddescr.typeinfo == FLOAT: - do_setfield_gc_float(struct, fielddescr.ofs, newvalue) - else: - raise NotImplementedError - - def op_setfield_raw(self, fielddescr, struct, newvalue): - if fielddescr.typeinfo == REF: - do_setfield_raw_ptr(struct, fielddescr.ofs, newvalue) - elif fielddescr.typeinfo == INT: - do_setfield_raw_int(struct, fielddescr.ofs, newvalue) - elif fielddescr.typeinfo == FLOAT: - do_setfield_raw_float(struct, fielddescr.ofs, newvalue) - else: - raise NotImplementedError - - def op_call(self, calldescr, func, *args): - effectinfo = calldescr.get_extra_info() - if effectinfo is not None and hasattr(effectinfo, 'oopspecindex'): - oopspecindex = effectinfo.oopspecindex - if oopspecindex == EffectInfo.OS_MATH_SQRT: - return do_math_sqrt(args[0]) - return self._do_call(calldescr, func, args, call_with_llptr=False) - - def op_call_release_gil(self, calldescr, func, *args): - return self._do_call(calldescr, func, args, call_with_llptr=True) - - def _do_call(self, calldescr, func, args, call_with_llptr): - global _last_exception - assert _last_exception is None, "exception left behind" - assert _call_args_i == _call_args_r == _call_args_f == [] - args_in_order = [] - for x in args: - T = lltype.typeOf(x) - if T is lltype.Signed: - args_in_order.append('i') - _call_args_i.append(x) - elif T == llmemory.GCREF: - args_in_order.append('r') - _call_args_r.append(x) - elif T is longlong.FLOATSTORAGE: - args_in_order.append('f') - _call_args_f.append(x) - else: - raise TypeError(x) - try: - return _do_call_common(func, args_in_order, calldescr, - call_with_llptr) - except LLException, lle: - _last_exception = lle - d = {'v': None, - REF: lltype.nullptr(llmemory.GCREF.TO), - INT: 0, - FLOAT: 0.0} - return d[calldescr.typeinfo] - - def op_cond_call_gc_wb(self, descr, a, b): - py.test.skip("cond_call_gc_wb not supported") - - def op_cond_call_gc_wb_array(self, descr, a, b, c): - py.test.skip("cond_call_gc_wb_array not supported") - - def op_oosend(self, descr, obj, *args): - raise NotImplementedError("oosend for lltype backend??") - - op_oosend_pure = op_oosend - - def op_new_array(self, arraydescr, count): - return do_new_array(arraydescr.ofs, count) - - def op_force_token(self, descr): - opaque_frame = _to_opaque(self) - return llmemory.cast_ptr_to_adr(opaque_frame) - - def op_read_timestamp(self, descr): - return read_timestamp() - - def op_call_may_force(self, calldescr, func, *args): - assert not self._forced - self._may_force = self.opindex - try: - return self.op_call(calldescr, func, *args) - finally: - self._may_force = -1 - - def op_call_assembler(self, wref_loop_token, *args): - if we_are_translated(): - raise ValueError("CALL_ASSEMBLER not supported") - return self._do_call_assembler(wref_loop_token, *args) - - def _do_call_assembler(self, wref_loop_token, *args): - global _last_exception - loop_token = wref_loop_token() - assert loop_token, "CALL_ASSEMBLER to a target that already died" - ctl = loop_token.compiled_loop_token - if hasattr(ctl, 'redirected'): - return self._do_call_assembler(ctl.redirected, *args) - assert not self._forced - self._may_force = self.opindex - try: - inpargs = _from_opaque(ctl.compiled_version).inputargs - assert len(inpargs) == len(args) - for i, inparg in enumerate(inpargs): - TYPE = inparg.concretetype - if TYPE is lltype.Signed: - set_future_value_int(i, args[i]) - elif isinstance(TYPE, lltype.Ptr): - set_future_value_ref(i, args[i]) - elif TYPE is longlong.FLOATSTORAGE: - set_future_value_float(i, args[i]) - else: - raise Exception("Nonsense type %s" % TYPE) - - failindex = self.cpu._execute_token(loop_token) - jd = loop_token.outermost_jitdriver_sd - assert jd is not None, ("call_assembler(): the loop_token needs " - "to have 'outermost_jitdriver_sd'") - if jd.index_of_virtualizable != -1: - vable = args[jd.index_of_virtualizable] - else: - vable = lltype.nullptr(llmemory.GCREF.TO) - # - # Emulate the fast path - if failindex == self.cpu.done_with_this_frame_int_v: - reset_vable(jd, vable) - return self.cpu.get_latest_value_int(0) - if failindex == self.cpu.done_with_this_frame_ref_v: - reset_vable(jd, vable) - return self.cpu.get_latest_value_ref(0) - if failindex == self.cpu.done_with_this_frame_float_v: - reset_vable(jd, vable) - return self.cpu.get_latest_value_float(0) - if failindex == self.cpu.done_with_this_frame_void_v: - reset_vable(jd, vable) - return None - # - assembler_helper_ptr = jd.assembler_helper_adr.ptr # fish - try: - return assembler_helper_ptr(failindex, vable) - except LLException, lle: - assert _last_exception is None, "exception left behind" - _last_exception = lle - # fish op - op = self.loop.operations[self.opindex] - if op.result is not None: - return 0 - finally: - self._may_force = -1 - - def op_guard_not_forced(self, descr): - forced = self._forced - self._forced = False - if forced: - raise GuardFailed - - def op_guard_not_invalidated(self, descr): - if self.loop.invalid: - raise GuardFailed - -class OOFrame(Frame): - - OPHANDLERS = [None] * (rop._LAST+1) - - def op_new_with_vtable(self, descr, vtable): - assert descr is None - typedescr = get_class_size(self.memocast, vtable) - return ootype.cast_to_object(ootype.new(typedescr.TYPE)) - - def op_new_array(self, typedescr, count): - res = ootype.oonewarray(typedescr.ARRAY, count) - return ootype.cast_to_object(res) - - def op_getfield_gc(self, fielddescr, obj): - TYPE = fielddescr.TYPE - fieldname = fielddescr.fieldname - _, T = TYPE._lookup_field(fieldname) - obj = ootype.cast_from_object(TYPE, obj) - res = getattr(obj, fieldname) - if isinstance(T, ootype.OOType): - return ootype.cast_to_object(res) - return res - - op_getfield_gc_pure = op_getfield_gc - - def op_setfield_gc(self, fielddescr, obj, newvalue): - TYPE = fielddescr.TYPE - fieldname = fielddescr.fieldname - _, T = TYPE._lookup_field(fieldname) - obj = ootype.cast_from_object(TYPE, obj) - if isinstance(ootype.typeOf(newvalue), ootype.OOType): - newvalue = ootype.cast_from_object(T, newvalue) - elif isinstance(T, lltype.Primitive): - newvalue = lltype.cast_primitive(T, newvalue) - setattr(obj, fieldname, newvalue) - - def op_getarrayitem_gc(self, typedescr, obj, index): - array = ootype.cast_from_object(typedescr.ARRAY, obj) - res = array.ll_getitem_fast(index) - if isinstance(typedescr.TYPE, ootype.OOType): - return ootype.cast_to_object(res) - return res - - op_getarrayitem_gc_pure = op_getarrayitem_gc - - def op_setarrayitem_gc(self, typedescr, obj, index, objnewvalue): - array = ootype.cast_from_object(typedescr.ARRAY, obj) - if ootype.typeOf(objnewvalue) == ootype.Object: - newvalue = ootype.cast_from_object(typedescr.TYPE, objnewvalue) - else: - newvalue = objnewvalue - array.ll_setitem_fast(index, newvalue) - - def op_arraylen_gc(self, typedescr, obj): - array = ootype.cast_from_object(typedescr.ARRAY, obj) - return array.ll_length() - - def op_call(self, calldescr, func, *args): - sm = ootype.cast_from_object(calldescr.FUNC, func) - newargs = cast_call_args(calldescr.FUNC.ARGS, args) - res = call_maybe_on_top_of_llinterp(sm, newargs) - if isinstance(calldescr.FUNC.RESULT, ootype.OOType): - return ootype.cast_to_object(res) - return res - - def op_oosend(self, descr, obj, *args): - METH = descr.METH - obj = ootype.cast_from_object(descr.SELFTYPE, obj) - meth = getattr(obj, descr.methname) - newargs = cast_call_args(METH.ARGS, args) - res = call_maybe_on_top_of_llinterp(meth, newargs) - if isinstance(METH.RESULT, ootype.OOType): - return ootype.cast_to_object(res) - return res - - op_oosend_pure = op_oosend - - def op_guard_class(self, _, value, expected_class): - value = ootype.cast_from_object(ootype.ROOT, value) - expected_class = ootype.cast_from_object(ootype.Class, expected_class) - if ootype.classof(value) is not expected_class: - raise GuardFailed - - def op_runtimenew(self, _, cls): - cls = ootype.cast_from_object(ootype.Class, cls) - res = ootype.runtimenew(cls) - return ootype.cast_to_object(res) - - def op_instanceof(self, typedescr, obj): - inst = ootype.cast_from_object(ootype.ROOT, obj) - return ootype.instanceof(inst, typedescr.TYPE) - - def op_subclassof(self, _, obj1, obj2): - cls1 = ootype.cast_from_object(ootype.Class, obj1) - cls2 = ootype.cast_from_object(ootype.Class, obj2) - return ootype.subclassof(cls1, cls2) - - def _cast_exception(self, exception): - return ootype.cast_from_object(ootype.Class, exception) - - def _issubclass(self, cls1, cls2): - return ootype.subclassof(cls1, cls2) - -# ____________________________________________________________ - -def cast_to_int(x): - TP = lltype.typeOf(x) - if isinstance(TP, lltype.Ptr): - return heaptracker.adr2int(llmemory.cast_ptr_to_adr(x)) - if TP == llmemory.Address: - return heaptracker.adr2int(x) - if TP is lltype.SingleFloat: - return longlong.singlefloat2int(x) - return lltype.cast_primitive(lltype.Signed, x) - -def cast_from_int(TYPE, x): - if isinstance(TYPE, lltype.Ptr): - if isinstance(x, (int, long, llmemory.AddressAsInt)): - x = llmemory.cast_int_to_adr(x) - if TYPE is rffi.VOIDP or ( - hasattr(TYPE.TO, '_hints') and - TYPE.TO._hints.get("uncast_on_llgraph")): - # assume that we want a "C-style" cast, without typechecking the value - return rffi.cast(TYPE, x) - return llmemory.cast_adr_to_ptr(x, TYPE) - elif TYPE == llmemory.Address: - if isinstance(x, (int, long, llmemory.AddressAsInt)): - x = llmemory.cast_int_to_adr(x) - assert lltype.typeOf(x) == llmemory.Address - return x - elif TYPE is lltype.SingleFloat: - assert lltype.typeOf(x) is lltype.Signed - return longlong.int2singlefloat(x) - else: - if lltype.typeOf(x) == llmemory.Address: - x = heaptracker.adr2int(x) - return lltype.cast_primitive(TYPE, x) - -def cast_to_ptr(x): - assert isinstance(lltype.typeOf(x), lltype.Ptr) - return lltype.cast_opaque_ptr(llmemory.GCREF, x) - -def cast_from_ptr(TYPE, x): - return lltype.cast_opaque_ptr(TYPE, x) - -def cast_to_floatstorage(x): - if isinstance(x, float): - return longlong.getfloatstorage(x) # common case - if IS_32_BIT: - assert longlong.supports_longlong - if isinstance(x, r_longlong): - return x - if isinstance(x, r_ulonglong): - return rffi.cast(lltype.SignedLongLong, x) - raise TypeError(type(x)) - -def cast_from_floatstorage(TYPE, x): - assert isinstance(x, longlong.r_float_storage) - if TYPE is lltype.Float: - return longlong.getrealfloat(x) - if longlong.is_longlong(TYPE): - return rffi.cast(TYPE, x) - raise TypeError(TYPE) - - -def new_frame(is_oo, cpu): - if is_oo: - frame = OOFrame(cpu) - else: - frame = Frame(cpu) - return _to_opaque(frame) - -_future_values = [] - -def frame_clear(frame, loop): - frame = _from_opaque(frame) - loop = _from_opaque(loop) - assert len(_future_values) == len(loop.inputargs) - frame.loop = loop - frame.env = {} - for i in range(len(loop.inputargs)): - expected_type = loop.inputargs[i].concretetype - assert lltype.typeOf(_future_values[i]) == expected_type - frame.env[loop.inputargs[i]] = _future_values[i] - del _future_values[:] - -def set_future_value_int(index, value): - assert lltype.typeOf(value) is lltype.Signed - set_future_value_ref(index, value) - -def set_future_value_float(index, value): - assert isinstance(value, longlong.r_float_storage) - set_future_value_ref(index, value) - -def set_future_value_ref(index, value): - del _future_values[index:] - assert len(_future_values) == index - _future_values.append(value) - -def frame_execute(frame): - frame = _from_opaque(frame) - if frame.verbose: - values = [frame.env[v] for v in frame.loop.inputargs] - log.trace('Entering CPU frame <- %r' % (values,)) - try: - result = frame.execute() - if frame.verbose: - log.trace('Leaving CPU frame -> #%d' % (result,)) - frame.log_progress() - except Exception, e: - log.ERROR('%s in CPU frame: %s' % (e.__class__.__name__, e)) - # Only invoke pdb when io capturing is not on otherwise py.io complains. - if py.test.config.option.capture == 'no': - import sys, pdb - pdb.post_mortem(sys.exc_info()[2]) - raise - del frame.env - return result - -def frame_int_getvalue(frame, num): - frame = _from_opaque(frame) - assert num >= 0 - x = frame.fail_args[num] - assert lltype.typeOf(x) is lltype.Signed - return x - -def frame_float_getvalue(frame, num): - frame = _from_opaque(frame) - assert num >= 0 - x = frame.fail_args[num] - assert lltype.typeOf(x) is longlong.FLOATSTORAGE - return x - -def frame_ptr_getvalue(frame, num): - frame = _from_opaque(frame) - assert num >= 0 - x = frame.fail_args[num] - assert lltype.typeOf(x) == llmemory.GCREF - return x - -def frame_get_value_count(frame): - frame = _from_opaque(frame) - return len(frame.fail_args) - -def frame_clear_latest_values(frame, count): - frame = _from_opaque(frame) - assert count == len(frame.fail_args) - del frame.fail_args - -_last_exception = None - -def grab_exc_value(): - global _last_exception - if _last_exception is not None: - result = _last_exception.args[1] - _last_exception = None - return lltype.cast_opaque_ptr(llmemory.GCREF, result) - else: - return lltype.nullptr(llmemory.GCREF.TO) - -##_pseudo_exceptions = {} - -##def _get_error(Class): -## if _llinterp.typer is not None: -## llframe = _llinterp.frame_class(None, None, _llinterp) -## try: -## llframe.make_llexception(Class()) -## except LLException, e: -## return e -## else: -## assert 0, "should have raised" -## else: -## # for tests, a random emulated ll_inst will do -## if Class not in _pseudo_exceptions: -## ll_inst = lltype.malloc(rclass.OBJECT, zero=True) -## ll_inst.typeptr = lltype.malloc(rclass.OBJECT_VTABLE, -## immortal=True) -## _pseudo_exceptions[Class] = LLException(ll_inst.typeptr, ll_inst) -## return _pseudo_exceptions[Class] - -##def get_overflow_error_value(): -## return lltype.cast_opaque_ptr(llmemory.GCREF, -## _get_error(OverflowError).args[1]) - -def force(opaque_frame): - frame = _from_opaque(opaque_frame) - assert not frame._forced - frame._forced = True - assert frame._may_force >= 0 - call_op = frame.loop.operations[frame._may_force] - guard_op = frame.loop.operations[frame._may_force+1] - opnum = call_op.opnum - assert opnum == rop.CALL_MAY_FORCE or opnum == rop.CALL_ASSEMBLER - frame._populate_fail_args(guard_op, skip=call_op.result) - return frame.fail_index - -def get_forced_token_frame(force_token): - opaque_frame = llmemory.cast_adr_to_ptr(force_token, - lltype.Ptr(_TO_OPAQUE[Frame])) - return opaque_frame - -def get_frame_forced_token(opaque_frame): - return llmemory.cast_ptr_to_adr(opaque_frame) - -##def cast_adr_to_int(memocast, adr): -## # xxx slow -## assert lltype.typeOf(adr) == llmemory.Address -## memocast = _from_opaque(memocast) -## addresses = memocast.addresses -## for i in xrange(len(addresses)-1, -1, -1): -## if addresses[i] == adr: -## return i -## i = len(addresses) -## addresses.append(adr) -## return i - -##def cast_int_to_adr(memocast, int): -## memocast = _from_opaque(memocast) -## assert 0 <= int < len(memocast.addresses) -## return memocast.addresses[int] - -##def get_class_size(memocast, vtable): -## memocast = _from_opaque(memocast) -## return memocast.vtable_to_size[vtable] - -##def set_class_size(memocast, vtable, size): -## memocast = _from_opaque(memocast) -## memocast.vtable_to_size[vtable] = size - -class GuardFailed(Exception): - pass - -# ____________________________________________________________ - - -def do_same_as(x): - return x - -def do_arraylen_gc(arraydescr, array): - array = array._obj.container - return array.getlength() - -def do_strlen(string): - str = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string) - return len(str.chars) - -def do_strgetitem(string, index): - str = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string) - return ord(str.chars[index]) - -def do_unicodelen(string): - uni = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), string) - return len(uni.chars) - -def do_unicodegetitem(string, index): - uni = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), string) - return ord(uni.chars[index]) - -def do_getarrayitem_gc_int(array, index): - array = array._obj.container - return cast_to_int(array.getitem(index)) - -def do_getarrayitem_raw_int(array, index, itemsize): - array = array.adr.ptr - ITEMTYPE = lltype.typeOf(array).TO.OF - TYPE = symbolic.Size2Type[itemsize] - if TYPE.OF != ITEMTYPE: - array = rffi.cast(lltype.Ptr(TYPE), array) - return cast_to_int(array._obj.getitem(index)) - -def do_getarrayitem_gc_float(array, index): - array = array._obj.container - return cast_to_floatstorage(array.getitem(index)) - -def do_getarrayitem_raw_float(array, index): - array = maybe_uncast(FLOAT_ARRAY_TP, array.adr.ptr) - return cast_to_floatstorage(array._obj.getitem(index)) - -def do_getarrayitem_gc_ptr(array, index): - array = array._obj.container - return cast_to_ptr(array.getitem(index)) - -def _getfield_gc(struct, fieldnum): - STRUCT, fieldname = symbolic.TokenToField[fieldnum] - ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct) - return getattr(ptr, fieldname) - -def do_getfield_gc_int(struct, fieldnum): - return cast_to_int(_getfield_gc(struct, fieldnum)) - -def do_getfield_gc_float(struct, fieldnum): - return cast_to_floatstorage(_getfield_gc(struct, fieldnum)) - -def do_getfield_gc_ptr(struct, fieldnum): - return cast_to_ptr(_getfield_gc(struct, fieldnum)) - -def _getinteriorfield_gc(struct, fieldnum): - STRUCT, fieldname = symbolic.TokenToField[fieldnum] - return getattr(struct, fieldname) - -def do_getinteriorfield_gc_int(array, index, fieldnum): - struct = array._obj.container.getitem(index) - return cast_to_int(_getinteriorfield_gc(struct, fieldnum)) - -def do_getinteriorfield_gc_float(array, index, fieldnum): - struct = array._obj.container.getitem(index) - return cast_to_floatstorage(_getinteriorfield_gc(struct, fieldnum)) - -def do_getinteriorfield_gc_ptr(array, index, fieldnum): - struct = array._obj.container.getitem(index) - return cast_to_ptr(_getinteriorfield_gc(struct, fieldnum)) - -def _getfield_raw(struct, fieldnum): - STRUCT, fieldname = symbolic.TokenToField[fieldnum] - ptr = cast_from_int(lltype.Ptr(STRUCT), struct) - return getattr(ptr, fieldname) - -def do_getfield_raw_int(struct, fieldnum): - return cast_to_int(_getfield_raw(struct, fieldnum)) - -def do_getfield_raw_float(struct, fieldnum): - return cast_to_floatstorage(_getfield_raw(struct, fieldnum)) - -def do_getfield_raw_ptr(struct, fieldnum): - return cast_to_ptr(_getfield_raw(struct, fieldnum)) - -def do_raw_load_int(struct, offset, descrofs): - TYPE = symbolic.Size2Type[descrofs] - ll_p = rffi.cast(rffi.CCHARP, struct) - ll_p = rffi.cast(lltype.Ptr(TYPE), rffi.ptradd(ll_p, offset)) - value = ll_p[0] - return rffi.cast(lltype.Signed, value) - -def do_raw_load_float(struct, offset): - ll_p = rffi.cast(rffi.CCHARP, struct) - ll_p = rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), - rffi.ptradd(ll_p, offset)) - value = ll_p[0] - return value - -def do_raw_store_int(struct, offset, descrofs, value): - TYPE = symbolic.Size2Type[descrofs] - ll_p = rffi.cast(rffi.CCHARP, struct) - ll_p = rffi.cast(lltype.Ptr(TYPE), rffi.ptradd(ll_p, offset)) - ll_p[0] = rffi.cast(TYPE.OF, value) - -def do_raw_store_float(struct, offset, value): - ll_p = rffi.cast(rffi.CCHARP, struct) - ll_p = rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), - rffi.ptradd(ll_p, offset)) - ll_p[0] = value - -def do_new(size): - TYPE = symbolic.Size2Type[size] - x = lltype.malloc(TYPE, zero=True) - return cast_to_ptr(x) - -def do_new_array(arraynum, count): - TYPE = symbolic.Size2Type[arraynum] - assert count >= 0 # explode if it's not - x = lltype.malloc(TYPE, count, zero=True) - return cast_to_ptr(x) - -def do_setarrayitem_gc_int(array, index, newvalue): - array = array._obj.container - ITEMTYPE = lltype.typeOf(array).OF - newvalue = cast_from_int(ITEMTYPE, newvalue) - array.setitem(index, newvalue) - -def do_setarrayitem_raw_int(array, index, newvalue, itemsize): - array = array.adr.ptr - ITEMTYPE = lltype.typeOf(array).TO.OF - TYPE = symbolic.Size2Type[itemsize] - if TYPE.OF != ITEMTYPE: - array = rffi.cast(lltype.Ptr(TYPE), array) - newvalue = cast_from_int(TYPE.OF, newvalue) - array._obj.setitem(index, newvalue) - -def do_setarrayitem_gc_float(array, index, newvalue): - array = array._obj.container - ITEMTYPE = lltype.typeOf(array).OF - newvalue = cast_from_floatstorage(ITEMTYPE, newvalue) - array.setitem(index, newvalue) - - -def do_setarrayitem_raw_float(array, index, newvalue): - array = maybe_uncast(FLOAT_ARRAY_TP, array.adr.ptr) - ITEMTYPE = lltype.typeOf(array).TO.OF - newvalue = cast_from_floatstorage(ITEMTYPE, newvalue) - array._obj.setitem(index, newvalue) - -def do_setarrayitem_gc_ptr(array, index, newvalue): - array = array._obj.container - ITEMTYPE = lltype.typeOf(array).OF - newvalue = cast_from_ptr(ITEMTYPE, newvalue) - array.setitem(index, newvalue) - -def new_setfield_gc(cast_func): - def do_setfield_gc(struct, fieldnum, newvalue): - STRUCT, fieldname = symbolic.TokenToField[fieldnum] - ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct) - FIELDTYPE = getattr(STRUCT, fieldname) - newvalue = cast_func(FIELDTYPE, newvalue) - setattr(ptr, fieldname, newvalue) - return do_setfield_gc -do_setfield_gc_int = new_setfield_gc(cast_from_int) -do_setfield_gc_float = new_setfield_gc(cast_from_floatstorage) -do_setfield_gc_ptr = new_setfield_gc(cast_from_ptr) - -def new_setinteriorfield_gc(cast_func): - def do_setinteriorfield_gc(array, index, fieldnum, newvalue): - STRUCT, fieldname = symbolic.TokenToField[fieldnum] - struct = array._obj.container.getitem(index) - FIELDTYPE = getattr(STRUCT, fieldname) - setattr(struct, fieldname, cast_func(FIELDTYPE, newvalue)) - return do_setinteriorfield_gc -do_setinteriorfield_gc_int = new_setinteriorfield_gc(cast_from_int) -do_setinteriorfield_gc_float = new_setinteriorfield_gc(cast_from_floatstorage) -do_setinteriorfield_gc_ptr = new_setinteriorfield_gc(cast_from_ptr) - -def do_setfield_raw_int(struct, fieldnum, newvalue): - STRUCT, fieldname = symbolic.TokenToField[fieldnum] - ptr = cast_from_int(lltype.Ptr(STRUCT), struct) - FIELDTYPE = getattr(STRUCT, fieldname) - newvalue = cast_from_int(FIELDTYPE, newvalue) - setattr(ptr, fieldname, newvalue) - -def do_setfield_raw_float(struct, fieldnum, newvalue): - STRUCT, fieldname = symbolic.TokenToField[fieldnum] - ptr = cast_from_int(lltype.Ptr(STRUCT), struct) - FIELDTYPE = getattr(STRUCT, fieldname) - newvalue = cast_from_floatstorage(FIELDTYPE, newvalue) - setattr(ptr, fieldname, newvalue) - -def do_setfield_raw_ptr(struct, fieldnum, newvalue): - STRUCT, fieldname = symbolic.TokenToField[fieldnum] - ptr = cast_from_int(lltype.Ptr(STRUCT), struct) - FIELDTYPE = getattr(STRUCT, fieldname) - newvalue = cast_from_ptr(FIELDTYPE, newvalue) - setattr(ptr, fieldname, newvalue) - -def do_newstr(length): - x = rstr.mallocstr(length) - return cast_to_ptr(x) - -def do_newunicode(length): - return cast_to_ptr(rstr.mallocunicode(length)) - -def do_strsetitem(string, index, newvalue): - str = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string) - str.chars[index] = chr(newvalue) - -def do_unicodesetitem(string, index, newvalue): - uni = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), string) - uni.chars[index] = unichr(newvalue) - -def do_copystrcontent(src, dst, srcstart, dststart, length): - src = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), src) - dst = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), dst) - assert 0 <= srcstart <= srcstart + length <= len(src.chars) - assert 0 <= dststart <= dststart + length <= len(dst.chars) - rstr.copy_string_contents(src, dst, srcstart, dststart, length) - -def do_copyunicodecontent(src, dst, srcstart, dststart, length): - src = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), src) - dst = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), dst) - assert 0 <= srcstart <= srcstart + length <= len(src.chars) - assert 0 <= dststart <= dststart + length <= len(dst.chars) - rstr.copy_unicode_contents(src, dst, srcstart, dststart, length) - -def do_math_sqrt(value): - import math - y = cast_from_floatstorage(lltype.Float, value) - x = math.sqrt(y) - return cast_to_floatstorage(x) - -# ---------- call ---------- - -_call_args_i = [] -_call_args_r = [] -_call_args_f = [] - -def do_call_pushint(x): - _call_args_i.append(x) - -def do_call_pushptr(x): - _call_args_r.append(x) - -def do_call_pushfloat(x): - _call_args_f.append(x) - -kind2TYPE = { - 'i': lltype.Signed, - 'f': lltype.Float, - 'L': lltype.SignedLongLong, - 'S': lltype.SingleFloat, - 'v': lltype.Void, - } - -def _do_call_common(f, args_in_order=None, calldescr=None, - call_with_llptr=False): - ptr = llmemory.cast_int_to_adr(f).ptr - PTR = lltype.typeOf(ptr) - if PTR == rffi.VOIDP: - # it's a pointer to a C function, so we don't have a precise - # signature: create one from the descr - assert call_with_llptr is True - ARGS = map(kind2TYPE.get, calldescr.arg_types) - RESULT = kind2TYPE[calldescr.typeinfo] - FUNC = lltype.FuncType(ARGS, RESULT) - func_to_call = rffi.cast(lltype.Ptr(FUNC), ptr) - else: - assert call_with_llptr is False - FUNC = PTR.TO - ARGS = FUNC.ARGS - func_to_call = ptr._obj._callable - args = cast_call_args(ARGS, _call_args_i, _call_args_r, _call_args_f, - args_in_order) - del _call_args_i[:] - del _call_args_r[:] - del _call_args_f[:] - assert len(ARGS) == len(args) - if hasattr(ptr._obj, 'graph'): - llinterp = _llinterp # it's a global set here by CPU.__init__() - result = llinterp.eval_graph(ptr._obj.graph, args) - # ^^^ may raise, in which case we get an LLException - else: - result = func_to_call(*args) - return result - -def do_call_void(f): - _do_call_common(f) - -def do_call_int(f): - x = _do_call_common(f) - return cast_to_int(x) - -def do_call_float(f): - x = _do_call_common(f) - return cast_to_floatstorage(x) - -def do_call_ptr(f): - x = _do_call_common(f) - return cast_to_ptr(x) - -def cast_call_args(ARGS, args_i, args_r, args_f, args_in_order=None): - argsiter_i = iter(args_i) - argsiter_r = iter(args_r) - argsiter_f = iter(args_f) - if args_in_order is not None: - orderiter = iter(args_in_order) - args = [] - for TYPE in ARGS: - if TYPE is lltype.Void: - x = None - else: - if isinstance(TYPE, ootype.OOType): - if args_in_order is not None: - n = orderiter.next() - assert n == 'r' - x = argsiter_r.next() - x = ootype.cast_from_object(TYPE, x) - elif isinstance(TYPE, lltype.Ptr) and TYPE.TO._gckind == 'gc': - if args_in_order is not None: - n = orderiter.next() - assert n == 'r' - x = argsiter_r.next() - x = cast_from_ptr(TYPE, x) - elif TYPE is lltype.Float or longlong.is_longlong(TYPE): - if args_in_order is not None: - n = orderiter.next() - assert n == 'f' - x = argsiter_f.next() - x = cast_from_floatstorage(TYPE, x) - else: - if args_in_order is not None: - n = orderiter.next() - assert n == 'i' - x = argsiter_i.next() - x = cast_from_int(TYPE, x) - args.append(x) - assert list(argsiter_i) == [] - assert list(argsiter_r) == [] - assert list(argsiter_f) == [] - return args - - -# for ootype meth and staticmeth -def call_maybe_on_top_of_llinterp(meth, args): - global _last_exception - if isinstance(meth, ootype._bound_meth): - mymethod = meth.meth - myargs = [meth.inst] + list(args) - else: - mymethod = meth - myargs = args - try: - if hasattr(mymethod, 'graph'): - llinterp = _llinterp # it's a global set here by CPU.__init__() - result = llinterp.eval_graph(mymethod.graph, myargs) From noreply at buildbot.pypy.org Sun Jan 27 19:07:45 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 27 Jan 2013 19:07:45 +0100 (CET) Subject: [pypy-commit] pypy default: one more "try to bring it to the previous state" thanks armin Message-ID: <20130127180745.36A681C1306@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r60529:05c1d0284431 Date: 2013-01-27 20:07 +0200 http://bitbucket.org/pypy/pypy/changeset/05c1d0284431/ Log: one more "try to bring it to the previous state" thanks armin diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py --- a/rpython/rlib/runicode.py +++ b/rpython/rlib/runicode.py @@ -60,13 +60,13 @@ def code_to_unichr(code): if not we_are_translated() and sys.maxunicode == 0xFFFF: # Host CPython is narrow build, generate surrogates - return UNICHR(code) + return unichr_returns_surrogate(code) else: return unichr(code) else: def code_to_unichr(code): # generate surrogates for large codes - return UNICHR(code) + return unichr_returns_surrogate(code) def _STORECHAR(result, CH, byteorder): hi = chr(((CH) >> 8) & 0xff) From noreply at buildbot.pypy.org Sun Jan 27 19:29:14 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Sun, 27 Jan 2013 19:29:14 +0100 (CET) Subject: [pypy-commit] pypy win32-fixes: see if this fixes win32 translation Message-ID: <20130127182914.CB26F1C125A@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: win32-fixes Changeset: r60530:3865dce660fd Date: 2013-01-27 13:28 -0500 http://bitbucket.org/pypy/pypy/changeset/3865dce660fd/ Log: see if this fixes win32 translation diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py --- a/rpython/rlib/_rsocket_rffi.py +++ b/rpython/rlib/_rsocket_rffi.py @@ -344,7 +344,7 @@ ]) CConfig.WSAPROTOCOL_INFO = platform.Struct( - 'struct WSAPROTOCOL_INFO', + 'struct _WSAPROTOCOL_INFO', []) # Struct is just passed between functions CConfig.FROM_PROTOCOL_INFO = platform.DefinedConstantInteger( 'FROM_PROTOCOL_INFO') From noreply at buildbot.pypy.org Sun Jan 27 19:32:16 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Sun, 27 Jan 2013 19:32:16 +0100 (CET) Subject: [pypy-commit] pypy win32-fixes: ok, another try Message-ID: <20130127183216.95C0D1C125A@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: win32-fixes Changeset: r60531:1929c19109d0 Date: 2013-01-27 13:32 -0500 http://bitbucket.org/pypy/pypy/changeset/1929c19109d0/ Log: ok, another try diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py --- a/rpython/rlib/_rsocket_rffi.py +++ b/rpython/rlib/_rsocket_rffi.py @@ -344,7 +344,7 @@ ]) CConfig.WSAPROTOCOL_INFO = platform.Struct( - 'struct _WSAPROTOCOL_INFO', + 'WSAPROTOCOL_INFO', []) # Struct is just passed between functions CConfig.FROM_PROTOCOL_INFO = platform.DefinedConstantInteger( 'FROM_PROTOCOL_INFO') From noreply at buildbot.pypy.org Sun Jan 27 19:35:35 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Sun, 27 Jan 2013 19:35:35 +0100 (CET) Subject: [pypy-commit] pypy win32-fixes: more fixes Message-ID: <20130127183535.A06031C125A@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: win32-fixes Changeset: r60532:a0aa9cfc1a17 Date: 2013-01-27 13:35 -0500 http://bitbucket.org/pypy/pypy/changeset/a0aa9cfc1a17/ Log: more fixes diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py --- a/rpython/rlib/_rsocket_rffi.py +++ b/rpython/rlib/_rsocket_rffi.py @@ -617,7 +617,7 @@ WSASocket = external('WSASocket', [rffi.INT, rffi.INT, rffi.INT, lltype.Ptr(WSAPROTOCOL_INFO), - rffi.DWORD, rffi.DWORD], + rwin32.DWORD, rwin32.DWORD], socketfd_type) if WIN32: From noreply at buildbot.pypy.org Sun Jan 27 19:53:15 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Sun, 27 Jan 2013 19:53:15 +0100 (CET) Subject: [pypy-commit] pypy default: merge win32-fixes for rsocket Message-ID: <20130127185315.DB1101C1055@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60533:e6f15c5f03b0 Date: 2013-01-27 13:52 -0500 http://bitbucket.org/pypy/pypy/changeset/e6f15c5f03b0/ Log: merge win32-fixes for rsocket diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py --- a/rpython/rlib/_rsocket_rffi.py +++ b/rpython/rlib/_rsocket_rffi.py @@ -344,7 +344,7 @@ ]) CConfig.WSAPROTOCOL_INFO = platform.Struct( - 'struct WSAPROTOCOL_INFO', + 'WSAPROTOCOL_INFO', []) # Struct is just passed between functions CConfig.FROM_PROTOCOL_INFO = platform.DefinedConstantInteger( 'FROM_PROTOCOL_INFO') @@ -617,7 +617,7 @@ WSASocket = external('WSASocket', [rffi.INT, rffi.INT, rffi.INT, lltype.Ptr(WSAPROTOCOL_INFO), - rffi.DWORD, rffi.DWORD], + rwin32.DWORD, rwin32.DWORD], socketfd_type) if WIN32: From noreply at buildbot.pypy.org Sun Jan 27 19:54:49 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 27 Jan 2013 19:54:49 +0100 (CET) Subject: [pypy-commit] pypy default: fix rpython Message-ID: <20130127185449.D2B021C1055@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r60534:b30e913b44ff Date: 2013-01-27 20:53 +0200 http://bitbucket.org/pypy/pypy/changeset/b30e913b44ff/ Log: fix rpython diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py --- a/rpython/rlib/runicode.py +++ b/rpython/rlib/runicode.py @@ -31,7 +31,8 @@ ch2 = ord(u[1]) if 0xD800 <= ch1 <= 0xDBFF and 0xDC00 <= ch2 <= 0xDFFF: return (((ch1 - 0xD800) << 10) | (ch2 - 0xDC00)) + 0x10000 - return ord(u) + assert len(u) == 1 + return ord(u[0]) if MAXUNICODE > sys.maxunicode: # A version of unichr which allows codes outside the BMP From noreply at buildbot.pypy.org Sun Jan 27 19:54:50 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 27 Jan 2013 19:54:50 +0100 (CET) Subject: [pypy-commit] pypy default: merge Message-ID: <20130127185450.EF8C71C1055@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r60535:d701c9bcb992 Date: 2013-01-27 20:54 +0200 http://bitbucket.org/pypy/pypy/changeset/d701c9bcb992/ Log: merge diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py --- a/rpython/rlib/_rsocket_rffi.py +++ b/rpython/rlib/_rsocket_rffi.py @@ -344,7 +344,7 @@ ]) CConfig.WSAPROTOCOL_INFO = platform.Struct( - 'struct WSAPROTOCOL_INFO', + 'WSAPROTOCOL_INFO', []) # Struct is just passed between functions CConfig.FROM_PROTOCOL_INFO = platform.DefinedConstantInteger( 'FROM_PROTOCOL_INFO') @@ -617,7 +617,7 @@ WSASocket = external('WSASocket', [rffi.INT, rffi.INT, rffi.INT, lltype.Ptr(WSAPROTOCOL_INFO), - rffi.DWORD, rffi.DWORD], + rwin32.DWORD, rwin32.DWORD], socketfd_type) if WIN32: From noreply at buildbot.pypy.org Sun Jan 27 19:59:08 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Sun, 27 Jan 2013 19:59:08 +0100 (CET) Subject: [pypy-commit] pypy default: fix whatsnew for merged branches Message-ID: <20130127185908.956591C1055@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60536:60c48ee502e1 Date: 2013-01-27 13:58 -0500 http://bitbucket.org/pypy/pypy/changeset/60c48ee502e1/ Log: fix whatsnew for merged branches diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -34,6 +34,8 @@ .. branch: kill-faking .. branch: improved_ebnfparse_error .. branch: task-decorator +.. branch: fix-e4fa0b2 +.. branch: win32-fixes .. branch: release-2.0-beta1 From noreply at buildbot.pypy.org Sun Jan 27 19:59:09 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Sun, 27 Jan 2013 19:59:09 +0100 (CET) Subject: [pypy-commit] pypy default: merge Message-ID: <20130127185909.DCB0B1C1055@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60537:ce89f6bbee1e Date: 2013-01-27 13:58 -0500 http://bitbucket.org/pypy/pypy/changeset/ce89f6bbee1e/ Log: merge diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py --- a/rpython/rlib/runicode.py +++ b/rpython/rlib/runicode.py @@ -31,7 +31,8 @@ ch2 = ord(u[1]) if 0xD800 <= ch1 <= 0xDBFF and 0xDC00 <= ch2 <= 0xDFFF: return (((ch1 - 0xD800) << 10) | (ch2 - 0xDC00)) + 0x10000 - return ord(u) + assert len(u) == 1 + return ord(u[0]) if MAXUNICODE > sys.maxunicode: # A version of unichr which allows codes outside the BMP From noreply at buildbot.pypy.org Sun Jan 27 20:47:24 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 27 Jan 2013 20:47:24 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: implement malloc_cond_varsize_small for the JIT, not used so far Message-ID: <20130127194724.BFC331C067C@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60539:537d5ac910a9 Date: 2013-01-27 21:45 +0200 http://bitbucket.org/pypy/pypy/changeset/537d5ac910a9/ Log: implement malloc_cond_varsize_small for the JIT, not used so far diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -2486,6 +2486,22 @@ self.mc.overwrite(jmp_adr-1, chr(offset)) self.mc.MOV(heap(nursery_free_adr), edi) + def malloc_cond_varsize_small(self, nursery_free_adr, nursery_top_adr, + sizeloc, gcmap): + self.mc.MOV(edi, heap(nursery_free_adr)) + self.mc.MOV(eax, edi) + self.mc.ADD(edi, sizeloc) + self.mc.CMP(edi, heap(nursery_top_adr)) + self.mc.J_il8(rx86.Conditions['NA'], 0) # patched later + jmp_adr = self.mc.get_relative_pos() + # save the gcmap + self.push_gcmap(self.mc, gcmap, mov=True) + self.mc.CALL(imm(self.malloc_slowpath)) + offset = self.mc.get_relative_pos() - jmp_adr + assert 0 < offset <= 127 + self.mc.overwrite(jmp_adr-1, chr(offset)) + self.mc.MOV(heap(nursery_free_adr), edi) + def force_token(self, reg): base_ofs = self.cpu.get_baseofs_of_frame_field() assert isinstance(reg, RegLoc) diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -4,7 +4,7 @@ import os from rpython.jit.metainterp.history import (Box, Const, ConstInt, ConstPtr, - BoxPtr, ConstFloat, + BoxPtr, ConstFloat, BoxInt, BoxFloat, INT, REF, FLOAT, TargetToken, JitCellToken) from rpython.jit.backend.x86.regloc import * @@ -874,6 +874,26 @@ gc_ll_descr.get_nursery_top_addr(), size, gcmap) + def consider_call_malloc_nursery_varsize_small(self, op): + size_box = op.getarg(0) + assert isinstance(size_box, BoxInt) # we cannot have a const here! + # looking at the result + self.rm.force_allocate_reg(op.result, selected_reg=eax) + # + # We need edx as a temporary, but otherwise don't save any more + # register. See comments in _build_malloc_slowpath(). + tmp_box = TempBox() + self.rm.force_allocate_reg(tmp_box, selected_reg=edi) + sizeloc = self.rm.make_sure_var_in_reg(size_box, [op.result, tmp_box]) + gcmap = self.get_gcmap([eax, edi]) # allocate the gcmap *before* + self.rm.possibly_free_var(tmp_box) + # + gc_ll_descr = self.assembler.cpu.gc_ll_descr + self.assembler.malloc_cond_varsize_small( + gc_ll_descr.get_nursery_free_addr(), + gc_ll_descr.get_nursery_top_addr(), + sizeloc, gcmap) + def get_gcmap(self, forbidden_regs=[]): frame_depth = self.fm.get_frame_depth() size = frame_depth + JITFRAME_FIXED_SIZE diff --git a/rpython/jit/backend/x86/test/test_gc_integration.py b/rpython/jit/backend/x86/test/test_gc_integration.py --- a/rpython/jit/backend/x86/test/test_gc_integration.py +++ b/rpython/jit/backend/x86/test/test_gc_integration.py @@ -202,6 +202,29 @@ # slowpath never called assert gc_ll_descr.calls == [] + def test_malloc_nursery_varsize_small(self): + self.cpu = self.getcpu(None) + ops = ''' + [i0, i1, i2] + p0 = call_malloc_nursery_varsize_small(i0) + p1 = call_malloc_nursery_varsize_small(i1) + p2 = call_malloc_nursery_varsize_small(i2) + guard_true(i0) [p0, p1, p2] + ''' + self.interpret(ops, [16, 32, 16]) + # check the returned pointers + gc_ll_descr = self.cpu.gc_ll_descr + nurs_adr = rffi.cast(lltype.Signed, gc_ll_descr.nursery) + ref = lambda n: self.cpu.get_ref_value(self.deadframe, n) + assert rffi.cast(lltype.Signed, ref(0)) == nurs_adr + 0 + assert rffi.cast(lltype.Signed, ref(1)) == nurs_adr + 16 + assert rffi.cast(lltype.Signed, ref(2)) == nurs_adr + 48 + # check the nursery content and state + gc_ll_descr.check_nothing_in_nursery() + assert gc_ll_descr.addrs[0] == nurs_adr + 64 + # slowpath never called + assert gc_ll_descr.calls == [] + def test_malloc_slowpath(self): def check(frame): assert len(frame.jf_gcmap) == 1 diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py --- a/rpython/jit/metainterp/resoperation.py +++ b/rpython/jit/metainterp/resoperation.py @@ -529,6 +529,10 @@ 'CALL_PURE/*d', # removed before it's passed to the backend 'CALL_MALLOC_GC/*d', # like CALL, but NULL => propagate MemoryError 'CALL_MALLOC_NURSERY/1', # nursery malloc, const number of bytes, zeroed + 'CALL_MALLOC_NURSERY_VARSIZE_SMALL/1', + # nursery malloc, non-const number of bytes, zeroed + # note that the number of bytes must be well known to be small enough + # to fulfill allocating in the nursery rules (and no card markings) '_CALL_LAST', '_CANRAISE_LAST', # ----- end of can_raise operations ----- From noreply at buildbot.pypy.org Sun Jan 27 20:47:23 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 27 Jan 2013 20:47:23 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fix the test Message-ID: <20130127194723.8D0511C039A@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60538:26beb40a6f7e Date: 2013-01-27 21:06 +0200 http://bitbucket.org/pypy/pypy/changeset/26beb40a6f7e/ Log: fix the test diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -2021,9 +2021,9 @@ ofs = self.cpu.get_ofs_of_frame_field('jf_descr') base_ofs = self.cpu.get_baseofs_of_frame_field() self.mov(fail_descr_loc, RawStackLoc(ofs)) - gcmap = self.gcmap_for_finish arglist = op.getarglist() if arglist and arglist[0].type == REF: + gcmap = self.gcmap_for_finish self.push_gcmap(self.mc, gcmap, store=True) else: # note that the 0 here is redundant, but I would rather diff --git a/rpython/jit/backend/x86/test/test_gc_integration.py b/rpython/jit/backend/x86/test/test_gc_integration.py --- a/rpython/jit/backend/x86/test/test_gc_integration.py +++ b/rpython/jit/backend/x86/test/test_gc_integration.py @@ -479,12 +479,11 @@ cpu.gc_ll_descr.init_nursery(100) cpu.setup_once() S = self.S - ofs = cpu.get_baseofs_of_frame_field() frames = [] def check(i): - assert cpu.gc_ll_descr.gcrootmap.stack[0] == i - ofs - frame = rffi.cast(JITFRAMEPTR, i - ofs) + assert cpu.gc_ll_descr.gcrootmap.stack[0] == i + frame = rffi.cast(JITFRAMEPTR, i) assert len(frame.jf_frame) == JITFRAME_FIXED_SIZE + 4 # we "collect" frames.append(frame) @@ -498,8 +497,8 @@ frames.append(new_frame) def check2(i): - assert cpu.gc_ll_descr.gcrootmap.stack[0] == i - ofs - frame = rffi.cast(JITFRAMEPTR, i - ofs) + assert cpu.gc_ll_descr.gcrootmap.stack[0] == i + frame = rffi.cast(JITFRAMEPTR, i) assert frame == frames[1] assert frame != frames[0] @@ -511,11 +510,12 @@ loop = self.parse(""" [p0, p1, p2] - i0 = force_token() # this is a bit below the frame - call(ConstClass(check_adr), i0, descr=checkdescr) # this can collect + pf = force_token() # this is a bit below the frame + call(ConstClass(check_adr), pf, descr=checkdescr) # this can collect p3 = getfield_gc(p0, descr=fielddescr) - call(ConstClass(check2_adr), i0, descr=checkdescr) - guard_true(i0, descr=faildescr) [p0, p1, p2, p3] + pf2 = force_token() + call(ConstClass(check2_adr), pf2, descr=checkdescr) + guard_nonnull(p3, descr=faildescr) [p0, p1, p2, p3] p4 = getfield_gc(p0, descr=fielddescr) finish(p4, descr=finaldescr) """, namespace={'finaldescr': BasicFinalDescr(), @@ -534,7 +534,7 @@ frame = lltype.cast_opaque_ptr(JITFRAMEPTR, frame) gcmap = unpack_gcmap(lltype.cast_opaque_ptr(JITFRAMEPTR, frame)) assert len(gcmap) == 1 - assert gcmap[0] < 29 + assert gcmap[0] < JITFRAME_FIXED_SIZE item = rffi.cast(lltype.Ptr(S), frame.jf_frame[gcmap[0]]) assert item == new_items[2] From noreply at buildbot.pypy.org Sun Jan 27 20:47:26 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 27 Jan 2013 20:47:26 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: merge Message-ID: <20130127194726.20C1A1C039A@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60540:73a4b46a1381 Date: 2013-01-27 21:46 +0200 http://bitbucket.org/pypy/pypy/changeset/73a4b46a1381/ Log: merge diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py --- a/rpython/jit/backend/llgraph/runner.py +++ b/rpython/jit/backend/llgraph/runner.py @@ -582,7 +582,7 @@ class LLFrame(object): - _TYPE = lltype.Signed + _TYPE = llmemory.GCREF forced_deadframe = None overflow_flag = False @@ -595,6 +595,22 @@ for box, arg in zip(argboxes, args): self.setenv(box, arg) + def __eq__(self, other): + # this is here to avoid crashes in 'token == TOKEN_TRACING_RESCALL' + from rpython.jit.metainterp.virtualizable import TOKEN_NONE + from rpython.jit.metainterp.virtualizable import TOKEN_TRACING_RESCALL + if isinstance(other, LLFrame): + return self is other + if other == TOKEN_NONE or other == TOKEN_TRACING_RESCALL: + return False + assert 0 + + def __ne__(self, other): + return not (self == other) + + def _identityhash(self): + return hash(self) + def setenv(self, box, arg): if box.type == INT: # typecheck the result @@ -863,7 +879,8 @@ def reset_vable(jd, vable): if jd.index_of_virtualizable != -1: fielddescr = jd.vable_token_descr - self.cpu.bh_setfield_gc(vable, 0, fielddescr) + NULL = lltype.nullptr(llmemory.GCREF.TO) + self.cpu.bh_setfield_gc(vable, NULL, fielddescr) faildescr = self.cpu.get_latest_descr(pframe) if faildescr == self.cpu.done_with_this_frame_descr_int: reset_vable(jd, vable) diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py --- a/rpython/jit/backend/test/runner_test.py +++ b/rpython/jit/backend/test/runner_test.py @@ -2220,7 +2220,7 @@ values.append(self.cpu.get_int_value(deadframe, 1)) self.cpu.set_savedata_ref(deadframe, random_gcref) - FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Void) + FUNC = self.FuncType([llmemory.GCREF, lltype.Signed], lltype.Void) func_ptr = llhelper(lltype.Ptr(FUNC), maybe_force) funcbox = self.get_funcbox(self.cpu, func_ptr).constbox() calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, @@ -2228,7 +2228,7 @@ cpu = self.cpu i0 = BoxInt() i1 = BoxInt() - tok = BoxInt() + tok = BoxPtr() faildescr = BasicFailDescr(1) ops = [ ResOperation(rop.FORCE_TOKEN, [], tok), @@ -2265,7 +2265,7 @@ self.cpu.set_savedata_ref(deadframe, random_gcref) return 42 - FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Signed) + FUNC = self.FuncType([llmemory.GCREF, lltype.Signed], lltype.Signed) func_ptr = llhelper(lltype.Ptr(FUNC), maybe_force) funcbox = self.get_funcbox(self.cpu, func_ptr).constbox() calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, @@ -2274,7 +2274,7 @@ i0 = BoxInt() i1 = BoxInt() i2 = BoxInt() - tok = BoxInt() + tok = BoxPtr() faildescr = BasicFailDescr(1) ops = [ ResOperation(rop.FORCE_TOKEN, [], tok), @@ -2313,7 +2313,7 @@ self.cpu.set_savedata_ref(deadframe, random_gcref) return 42.5 - FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Float) + FUNC = self.FuncType([llmemory.GCREF, lltype.Signed], lltype.Float) func_ptr = llhelper(lltype.Ptr(FUNC), maybe_force) funcbox = self.get_funcbox(self.cpu, func_ptr).constbox() calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, @@ -2322,7 +2322,7 @@ i0 = BoxInt() i1 = BoxInt() f2 = BoxFloat() - tok = BoxInt() + tok = BoxPtr() faildescr = BasicFailDescr(1) ops = [ ResOperation(rop.FORCE_TOKEN, [], tok), @@ -3697,7 +3697,7 @@ values.append(self.cpu.get_int_value(deadframe, 0)) return 42 - FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Signed) + FUNC = self.FuncType([llmemory.GCREF, lltype.Signed], lltype.Signed) func_ptr = llhelper(lltype.Ptr(FUNC), maybe_force) funcbox = self.get_funcbox(self.cpu, func_ptr).constbox() calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, @@ -3705,7 +3705,7 @@ i0 = BoxInt() i1 = BoxInt() i2 = BoxInt() - tok = BoxInt() + tok = BoxPtr() faildescr = BasicFailDescr(23) ops = [ ResOperation(rop.FORCE_TOKEN, [], tok), diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py @@ -2954,7 +2954,7 @@ """ expected = """ [p1] - i0 = force_token() + p0 = force_token() jump(p1) """ self.optimize_loop(ops, expected) @@ -2969,12 +2969,12 @@ """ expected = """ [p1] - i0 = force_token() + p0 = force_token() p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) - setfield_gc(p2, i0, descr=virtualtokendescr) + setfield_gc(p2, p0, descr=virtualtokendescr) escape(p2) setfield_gc(p2, p1, descr=virtualforceddescr) - setfield_gc(p2, -3, descr=virtualtokendescr) + setfield_gc(p2, NULL, descr=virtualtokendescr) jump(p1) """ # XXX we should optimize a bit more the case of a nonvirtual. @@ -3000,10 +3000,10 @@ """ expected = """ [p0, i1] - i3 = force_token() + p3 = force_token() # p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) - setfield_gc(p2, i3, descr=virtualtokendescr) + setfield_gc(p2, p3, descr=virtualtokendescr) setfield_gc(p0, p2, descr=nextdescr) # call_may_force(i1, descr=mayforcevirtdescr) @@ -3015,7 +3015,7 @@ setfield_gc(p1b, 252, descr=valuedescr) setfield_gc(p1, p1b, descr=nextdescr) setfield_gc(p2, p1, descr=virtualforceddescr) - setfield_gc(p2, -3, descr=virtualtokendescr) + setfield_gc(p2, NULL, descr=virtualtokendescr) jump(p0, i1) """ self.optimize_loop(ops, expected) @@ -3039,10 +3039,10 @@ """ expected = """ [p0, i1] - i3 = force_token() + p3 = force_token() # p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) - setfield_gc(p2, i3, descr=virtualtokendescr) + setfield_gc(p2, p3, descr=virtualtokendescr) setfield_gc(p0, p2, descr=nextdescr) # call_may_force(i1, descr=mayforcevirtdescr) @@ -3054,7 +3054,7 @@ setfield_gc(p1b, i1, descr=valuedescr) setfield_gc(p1, p1b, descr=nextdescr) setfield_gc(p2, p1, descr=virtualforceddescr) - setfield_gc(p2, -3, descr=virtualtokendescr) + setfield_gc(p2, NULL, descr=virtualtokendescr) jump(p0, i1) """ # the point of this test is that 'i1' should show up in the fail_args @@ -3084,21 +3084,21 @@ """ expected = """ [p0, i1] - i3 = force_token() + p3 = force_token() call(i1, descr=nonwritedescr) - guard_no_exception(descr=fdescr) [i3, i1, p0] + guard_no_exception(descr=fdescr) [p3, i1, p0] setfield_gc(p0, NULL, descr=refdescr) jump(p0, i1) """ self.optimize_loop(ops, expected) - # the fail_args contain [i3, i1, p0]: - # - i3 is from the virtual expansion of p2 + # the fail_args contain [p3, i1, p0]: + # - p3 is from the virtual expansion of p2 # - i1 is from the virtual expansion of p1 # - p0 is from the extra pendingfields self.loop.inputargs[0].value = self.nodeobjvalue self.check_expanded_fail_descr('''p2, p1 p0.refdescr = p2 - where p2 is a jit_virtual_ref_vtable, virtualtokendescr=i3 + where p2 is a jit_virtual_ref_vtable, virtualtokendescr=p3 where p1 is a node_vtable, nextdescr=p1b where p1b is a node_vtable, valuedescr=i1 ''', rop.GUARD_NO_EXCEPTION) @@ -3116,13 +3116,13 @@ """ expected = """ [i1] - i3 = force_token() + p3 = force_token() p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) - setfield_gc(p2, i3, descr=virtualtokendescr) + setfield_gc(p2, p3, descr=virtualtokendescr) escape(p2) p1 = new_with_vtable(ConstClass(node_vtable)) setfield_gc(p2, p1, descr=virtualforceddescr) - setfield_gc(p2, -3, descr=virtualtokendescr) + setfield_gc(p2, NULL, descr=virtualtokendescr) call_may_force(i1, descr=mayforcevirtdescr) guard_not_forced() [] jump(i1) @@ -3141,12 +3141,12 @@ """ expected = """ [i1, p1] - i3 = force_token() + p3 = force_token() p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) - setfield_gc(p2, i3, descr=virtualtokendescr) + setfield_gc(p2, p3, descr=virtualtokendescr) escape(p2) setfield_gc(p2, p1, descr=virtualforceddescr) - setfield_gc(p2, -3, descr=virtualtokendescr) + setfield_gc(p2, NULL, descr=virtualtokendescr) call_may_force(i1, descr=mayforcevirtdescr) guard_not_forced() [i1] jump(i1, p1) @@ -3671,7 +3671,7 @@ i5 = int_gt(i4, i22) guard_false(i5) [] i6 = int_add(i4, 1) - i331 = force_token() + p331 = force_token() i7 = int_sub(i6, 1) setfield_gc(p0, i7, descr=valuedescr) jump(p0, i22) @@ -3682,7 +3682,7 @@ i2 = int_gt(i1, i22) guard_false(i2) [] i3 = int_add(i1, 1) - i331 = force_token() + p331 = force_token() jump(p0, i22) """ self.optimize_loop(ops, expected) diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -3424,7 +3424,7 @@ """ expected = """ [p1] - i0 = force_token() + p0 = force_token() jump(p1) """ self.optimize_loop(ops, expected, expected) @@ -3439,12 +3439,12 @@ """ expected = """ [p1] - i0 = force_token() + p0 = force_token() p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) - setfield_gc(p2, i0, descr=virtualtokendescr) + setfield_gc(p2, p0, descr=virtualtokendescr) escape(p2) setfield_gc(p2, p1, descr=virtualforceddescr) - setfield_gc(p2, -3, descr=virtualtokendescr) + setfield_gc(p2, NULL, descr=virtualtokendescr) jump(p1) """ # XXX we should optimize a bit more the case of a nonvirtual. @@ -3470,10 +3470,10 @@ """ expected = """ [p0, i1] - i3 = force_token() + p3 = force_token() # p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) - setfield_gc(p2, i3, descr=virtualtokendescr) + setfield_gc(p2, p3, descr=virtualtokendescr) setfield_gc(p0, p2, descr=nextdescr) # call_may_force(i1, descr=mayforcevirtdescr) @@ -3485,7 +3485,7 @@ setfield_gc(p1b, 252, descr=valuedescr) setfield_gc(p1, p1b, descr=nextdescr) setfield_gc(p2, p1, descr=virtualforceddescr) - setfield_gc(p2, -3, descr=virtualtokendescr) + setfield_gc(p2, NULL, descr=virtualtokendescr) jump(p0, i1) """ self.optimize_loop(ops, expected, expected) @@ -3509,10 +3509,10 @@ """ expected = """ [p0, i1] - i3 = force_token() + p3 = force_token() # p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) - setfield_gc(p2, i3, descr=virtualtokendescr) + setfield_gc(p2, p3, descr=virtualtokendescr) setfield_gc(p0, p2, descr=nextdescr) # call_may_force(i1, descr=mayforcevirtdescr) @@ -3524,7 +3524,7 @@ setfield_gc(p1b, i1, descr=valuedescr) setfield_gc(p1, p1b, descr=nextdescr) setfield_gc(p2, p1, descr=virtualforceddescr) - setfield_gc(p2, -3, descr=virtualtokendescr) + setfield_gc(p2, NULL, descr=virtualtokendescr) jump(p0, i1) """ # the point of this test is that 'i1' should show up in the fail_args @@ -3555,31 +3555,31 @@ """ preamble = """ [p0, i1] - i3 = force_token() + p3 = force_token() call(i1, descr=nonwritedescr) - guard_no_exception(descr=fdescr) [i3, i1, p0] + guard_no_exception(descr=fdescr) [p3, i1, p0] setfield_gc(p0, NULL, descr=refdescr) escape() jump(p0, i1) """ expected = """ [p0, i1] - i3 = force_token() + p3 = force_token() call(i1, descr=nonwritedescr) - guard_no_exception(descr=fdescr2) [i3, i1, p0] + guard_no_exception(descr=fdescr2) [p3, i1, p0] setfield_gc(p0, NULL, descr=refdescr) escape() jump(p0, i1) """ self.optimize_loop(ops, expected, preamble) - # the fail_args contain [i3, i1, p0]: - # - i3 is from the virtual expansion of p2 + # the fail_args contain [p3, i1, p0]: + # - p3 is from the virtual expansion of p2 # - i1 is from the virtual expansion of p1 # - p0 is from the extra pendingfields #self.loop.inputargs[0].value = self.nodeobjvalue #self.check_expanded_fail_descr('''p2, p1 # p0.refdescr = p2 - # where p2 is a jit_virtual_ref_vtable, virtualtokendescr=i3 + # where p2 is a jit_virtual_ref_vtable, virtualtokendescr=p3 # where p1 is a node_vtable, nextdescr=p1b # where p1b is a node_vtable, valuedescr=i1 # ''', rop.GUARD_NO_EXCEPTION) @@ -3597,13 +3597,13 @@ """ expected = """ [i1] - i3 = force_token() + p3 = force_token() p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) - setfield_gc(p2, i3, descr=virtualtokendescr) + setfield_gc(p2, p3, descr=virtualtokendescr) escape(p2) p1 = new_with_vtable(ConstClass(node_vtable)) setfield_gc(p2, p1, descr=virtualforceddescr) - setfield_gc(p2, -3, descr=virtualtokendescr) + setfield_gc(p2, NULL, descr=virtualtokendescr) call_may_force(i1, descr=mayforcevirtdescr) guard_not_forced() [] jump(i1) @@ -3622,12 +3622,12 @@ """ expected = """ [i1, p1] - i3 = force_token() + p3 = force_token() p2 = new_with_vtable(ConstClass(jit_virtual_ref_vtable)) - setfield_gc(p2, i3, descr=virtualtokendescr) + setfield_gc(p2, p3, descr=virtualtokendescr) escape(p2) setfield_gc(p2, p1, descr=virtualforceddescr) - setfield_gc(p2, -3, descr=virtualtokendescr) + setfield_gc(p2, NULL, descr=virtualtokendescr) call_may_force(i1, descr=mayforcevirtdescr) guard_not_forced() [i1] jump(i1, p1) @@ -4312,14 +4312,14 @@ i5 = int_gt(i4, i22) guard_false(i5) [] i6 = int_add(i4, 1) - i331 = force_token() + p331 = force_token() i7 = int_sub(i6, 1) setfield_gc(p0, i7, descr=valuedescr) jump(p0, i22) """ expected = """ [p0, i22] - i331 = force_token() + p331 = force_token() jump(p0, i22) """ self.optimize_loop(ops, expected) diff --git a/rpython/jit/metainterp/optimizeopt/virtualize.py b/rpython/jit/metainterp/optimizeopt/virtualize.py --- a/rpython/jit/metainterp/optimizeopt/virtualize.py +++ b/rpython/jit/metainterp/optimizeopt/virtualize.py @@ -2,6 +2,7 @@ from rpython.jit.metainterp.executor import execute from rpython.jit.codewriter.heaptracker import vtable2descr from rpython.jit.metainterp.history import Const, ConstInt, BoxInt +from rpython.jit.metainterp.history import CONST_NULL, BoxPtr from rpython.jit.metainterp.optimizeopt import optimizer from rpython.jit.metainterp.optimizeopt.optimizer import OptValue, REMOVED from rpython.jit.metainterp.optimizeopt.util import (make_dispatcher_method, @@ -418,7 +419,7 @@ # but the point is that doing so does not force the original structure. op = ResOperation(rop.NEW_WITH_VTABLE, [c_cls], op.result) vrefvalue = self.make_virtual(c_cls, op.result, op) - tokenbox = BoxInt() + tokenbox = BoxPtr() self.emit_operation(ResOperation(rop.FORCE_TOKEN, [], tokenbox)) vrefvalue.setfield(descr_virtual_token, self.getvalue(tokenbox)) @@ -441,12 +442,12 @@ # - set 'forced' to point to the real object objbox = op.getarg(1) - if not self.optimizer.cpu.ts.CONST_NULL.same_constant(objbox): + if not CONST_NULL.same_constant(objbox): seo(ResOperation(rop.SETFIELD_GC, op.getarglist(), None, descr = vrefinfo.descr_forced)) - # - set 'virtual_token' to TOKEN_NONE - args = [op.getarg(0), ConstInt(vrefinfo.TOKEN_NONE)] + # - set 'virtual_token' to TOKEN_NONE (== NULL) + args = [op.getarg(0), CONST_NULL] seo(ResOperation(rop.SETFIELD_GC, args, None, descr=vrefinfo.descr_virtual_token)) # Note that in some cases the virtual in op.getarg(1) has been forced @@ -462,7 +463,7 @@ if vref.is_virtual(): tokenvalue = vref.getfield(vrefinfo.descr_virtual_token, None) if (tokenvalue is not None and tokenvalue.is_constant() and - tokenvalue.box.getint() == vrefinfo.TOKEN_NONE): + not tokenvalue.box.nonnull()): forcedvalue = vref.getfield(vrefinfo.descr_forced, None) if forcedvalue is not None and not forcedvalue.is_null(): self.make_equal_to(op.result, forcedvalue) diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -2283,7 +2283,7 @@ virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) vinfo.tracing_before_residual_call(virtualizable) # - force_token_box = history.BoxInt() + force_token_box = history.BoxPtr() self.history.record(rop.FORCE_TOKEN, [], force_token_box) self.history.record(rop.SETFIELD_GC, [virtualizable_box, force_token_box], @@ -2376,7 +2376,7 @@ self.virtualizable_boxes = virtualizable_boxes # just jumped away from assembler (case 4 in the comment in # virtualizable.py) into tracing (case 2); check that vable_token - # is and stays 0. Note the call to reset_vable_token() in + # is and stays NULL. Note the call to reset_vable_token() in # warmstate.py. virtualizable_box = self.virtualizable_boxes[-1] virtualizable = vinfo.unwrap_virtualizable_box(virtualizable_box) diff --git a/rpython/jit/metainterp/resume.py b/rpython/jit/metainterp/resume.py --- a/rpython/jit/metainterp/resume.py +++ b/rpython/jit/metainterp/resume.py @@ -1110,7 +1110,7 @@ else: # just jumped away from assembler (case 4 in the comment in # virtualizable.py) into tracing (case 2); check that vable_token - # is and stays 0. Note the call to reset_vable_token() in + # is and stays NULL. Note the call to reset_vable_token() in # warmstate.py. assert not vinfo.is_token_nonnull_gcref(virtualizable) return vinfo.write_from_resume_data_partial(virtualizable, self, numb) diff --git a/rpython/jit/metainterp/test/test_recursive.py b/rpython/jit/metainterp/test/test_recursive.py --- a/rpython/jit/metainterp/test/test_recursive.py +++ b/rpython/jit/metainterp/test/test_recursive.py @@ -823,7 +823,8 @@ # at the level 2 is set to a non-zero value when doing the # call to the level 3 only. This used to fail when the test # is run via rpython.jit.backend.x86.test.test_recursive. - assert ll_subframe.vable_token == 0 + from rpython.jit.metainterp.virtualizable import TOKEN_NONE + assert ll_subframe.vable_token == TOKEN_NONE def main(codeno): frame = Frame() diff --git a/rpython/jit/metainterp/test/test_virtualizable.py b/rpython/jit/metainterp/test/test_virtualizable.py --- a/rpython/jit/metainterp/test/test_virtualizable.py +++ b/rpython/jit/metainterp/test/test_virtualizable.py @@ -41,7 +41,7 @@ XY = lltype.GcStruct( 'XY', ('parent', rclass.OBJECT), - ('vable_token', lltype.Signed), + ('vable_token', llmemory.GCREF), ('inst_x', lltype.Signed), ('inst_node', lltype.Ptr(LLtypeMixin.NODE)), hints = {'virtualizable2_accessor': FieldListAccessor()}) @@ -56,7 +56,7 @@ def setup(self): xy = lltype.malloc(self.XY) - xy.vable_token = 0 + xy.vable_token = lltype.nullptr(llmemory.GCREF.TO) xy.parent.typeptr = self.xy_vtable return xy @@ -206,7 +206,7 @@ XY2 = lltype.GcStruct( 'XY2', ('parent', rclass.OBJECT), - ('vable_token', lltype.Signed), + ('vable_token', llmemory.GCREF), ('inst_x', lltype.Signed), ('inst_l1', lltype.Ptr(lltype.GcArray(lltype.Signed))), ('inst_l2', lltype.Ptr(lltype.GcArray(lltype.Signed))), @@ -220,7 +220,7 @@ def setup2(self): xy2 = lltype.malloc(self.XY2) - xy2.vable_token = 0 + xy2.vable_token = lltype.nullptr(llmemory.GCREF.TO) xy2.parent.typeptr = self.xy2_vtable return xy2 @@ -393,7 +393,7 @@ def setup2sub(self): xy2 = lltype.malloc(self.XY2SUB) - xy2.parent.vable_token = 0 + xy2.parent.vable_token = lltype.nullptr(llmemory.GCREF.TO) xy2.parent.parent.typeptr = self.xy2_vtable return xy2 diff --git a/rpython/jit/metainterp/test/test_virtualref.py b/rpython/jit/metainterp/test/test_virtualref.py --- a/rpython/jit/metainterp/test/test_virtualref.py +++ b/rpython/jit/metainterp/test/test_virtualref.py @@ -1,6 +1,6 @@ import py -from rpython.rtyper.lltypesystem import lltype, lloperation +from rpython.rtyper.lltypesystem import lltype, llmemory, lloperation from rpython.rtyper.exceptiondata import UnknownException from rpython.rlib.jit import JitDriver, dont_look_inside, vref_None from rpython.rlib.jit import virtual_ref, virtual_ref_finish, InvalidVirtualRef @@ -110,7 +110,10 @@ if str(box._getrepr_()).endswith('JitVirtualRef')] assert len(bxs2) == 1 JIT_VIRTUAL_REF = self.vrefinfo.JIT_VIRTUAL_REF - bxs2[0].getref(lltype.Ptr(JIT_VIRTUAL_REF)).virtual_token = 1234567 + FOO = lltype.GcStruct('FOO') + foo = lltype.malloc(FOO) + tok = lltype.cast_opaque_ptr(llmemory.GCREF, foo) + bxs2[0].getref(lltype.Ptr(JIT_VIRTUAL_REF)).virtual_token = tok # # try reloading from blackhole.py's point of view from rpython.jit.metainterp.resume import ResumeDataDirectReader diff --git a/rpython/jit/metainterp/virtualizable.py b/rpython/jit/metainterp/virtualizable.py --- a/rpython/jit/metainterp/virtualizable.py +++ b/rpython/jit/metainterp/virtualizable.py @@ -10,9 +10,8 @@ from rpython.jit.metainterp.warmstate import wrap, unwrap from rpython.rlib.objectmodel import specialize + class VirtualizableInfo(object): - TOKEN_NONE = 0 # must be 0 -- see also x86.call_assembler - TOKEN_TRACING_RESCALL = -1 def __init__(self, warmrunnerdesc, VTYPEPTR): self.warmrunnerdesc = warmrunnerdesc @@ -217,7 +216,7 @@ self.cast_gcref_to_vtype = cast_gcref_to_vtype def reset_vable_token(virtualizable): - virtualizable.vable_token = VirtualizableInfo.TOKEN_NONE + virtualizable.vable_token = TOKEN_NONE self.reset_vable_token = reset_vable_token def clear_vable_token(virtualizable): @@ -230,7 +229,7 @@ def tracing_before_residual_call(virtualizable): virtualizable = cast_gcref_to_vtype(virtualizable) assert not virtualizable.vable_token - virtualizable.vable_token = VirtualizableInfo.TOKEN_TRACING_RESCALL + virtualizable.vable_token = TOKEN_TRACING_RESCALL self.tracing_before_residual_call = tracing_before_residual_call def tracing_after_residual_call(virtualizable): @@ -238,8 +237,8 @@ if virtualizable.vable_token: # not modified by the residual call; assert that it is still # set to TOKEN_TRACING_RESCALL and clear it. - assert virtualizable.vable_token == VirtualizableInfo.TOKEN_TRACING_RESCALL - virtualizable.vable_token = VirtualizableInfo.TOKEN_NONE + assert virtualizable.vable_token == TOKEN_TRACING_RESCALL + virtualizable.vable_token = TOKEN_NONE return False else: # marker "modified during residual call" set. @@ -248,16 +247,16 @@ def force_now(virtualizable): token = virtualizable.vable_token - if token == VirtualizableInfo.TOKEN_TRACING_RESCALL: + if token == TOKEN_TRACING_RESCALL: # The values in the virtualizable are always correct during # tracing. We only need to reset vable_token to TOKEN_NONE # as a marker for the tracing, to tell it that this # virtualizable escapes. - virtualizable.vable_token = VirtualizableInfo.TOKEN_NONE + virtualizable.vable_token = TOKEN_NONE else: from rpython.jit.metainterp.compile import ResumeGuardForcedDescr ResumeGuardForcedDescr.force_now(cpu, token) - assert virtualizable.vable_token == VirtualizableInfo.TOKEN_NONE + assert virtualizable.vable_token == TOKEN_NONE force_now._dont_inline_ = True self.force_now = force_now @@ -268,7 +267,7 @@ def reset_token_gcref(virtualizable): virtualizable = cast_gcref_to_vtype(virtualizable) - virtualizable.vable_token = VirtualizableInfo.TOKEN_NONE + virtualizable.vable_token = TOKEN_NONE self.reset_token_gcref = reset_token_gcref def _freeze_(self): @@ -297,16 +296,23 @@ # ____________________________________________________________ # -# The 'vable_token' field of a virtualizable is either 0, -1, or points -# into the CPU stack to a particular field in the current frame. It is: +# The 'vable_token' field of a virtualizable is either NULL, points +# to the JITFRAME object for the current assembler frame, or is +# the special value TOKEN_TRACING_RESCALL. It is: # -# 1. 0 (TOKEN_NONE) if not in the JIT at all, except as described below. +# 1. NULL (TOKEN_NONE) if not in the JIT at all, except as described below. # -# 2. equal to 0 when tracing is in progress; except: +# 2. NULL when tracing is in progress; except: # -# 3. equal to -1 (TOKEN_TRACING_RESCALL) during tracing when we do a +# 3. equal to TOKEN_TRACING_RESCALL during tracing when we do a # residual call, calling random unknown other parts of the interpreter; -# it is reset to 0 as soon as something occurs to the virtualizable. +# it is reset to NULL as soon as something occurs to the virtualizable. # # 4. when running the machine code with a virtualizable, it is set -# to the address in the CPU stack by the FORCE_TOKEN operation. +# to the JITFRAME, as obtained with the FORCE_TOKEN operation. + +_DUMMY = lltype.GcStruct('JITFRAME_DUMMY') +_dummy = lltype.malloc(_DUMMY) + +TOKEN_NONE = lltype.nullptr(llmemory.GCREF.TO) +TOKEN_TRACING_RESCALL = lltype.cast_opaque_ptr(llmemory.GCREF, _dummy) diff --git a/rpython/jit/metainterp/virtualref.py b/rpython/jit/metainterp/virtualref.py --- a/rpython/jit/metainterp/virtualref.py +++ b/rpython/jit/metainterp/virtualref.py @@ -1,6 +1,8 @@ from rpython.rtyper.rmodel import inputconst, log from rpython.rtyper.lltypesystem import lltype, llmemory, rclass from rpython.jit.metainterp import history +from rpython.jit.metainterp.virtualizable import TOKEN_NONE +from rpython.jit.metainterp.virtualizable import TOKEN_TRACING_RESCALL from rpython.jit.codewriter import heaptracker from rpython.rlib.jit import InvalidVirtualRef @@ -12,7 +14,7 @@ # we make the low-level type of an RPython class directly self.JIT_VIRTUAL_REF = lltype.GcStruct('JitVirtualRef', ('super', rclass.OBJECT), - ('virtual_token', lltype.Signed), + ('virtual_token', llmemory.GCREF), ('forced', rclass.OBJECTPTR)) self.jit_virtual_ref_vtable = lltype.malloc(rclass.OBJECT_VTABLE, zero=True, flavor='raw', @@ -69,19 +71,17 @@ # The 'virtual_token' field has the same meaning as the 'vable_token' field # of a virtualizable. It is equal to: - # * -3 (TOKEN_NONE) when tracing, except as described below; - # * -1 (TOKEN_TRACING_RESCALL) during tracing when we do a residual call; - # * addr in the CPU stack (set by FORCE_TOKEN) when running the assembler; - # * -3 (TOKEN_NONE) after the virtual is forced, if it is forced at all. - TOKEN_NONE = -3 - TOKEN_TRACING_RESCALL = -1 + # * TOKEN_NONE when tracing, except as described below; + # * TOKEN_TRACING_RESCALL during tracing when we do a residual call; + # * the JITFRAME (set by FORCE_TOKEN) when running the assembler; + # * TOKEN_NONE after the virtual is forced, if it is forced at all. def virtual_ref_during_tracing(self, real_object): assert real_object vref = lltype.malloc(self.JIT_VIRTUAL_REF) p = lltype.cast_pointer(rclass.OBJECTPTR, vref) p.typeptr = self.jit_virtual_ref_vtable - vref.virtual_token = self.TOKEN_NONE + vref.virtual_token = TOKEN_NONE vref.forced = lltype.cast_opaque_ptr(rclass.OBJECTPTR, real_object) return lltype.cast_opaque_ptr(llmemory.GCREF, vref) @@ -95,19 +95,19 @@ if not self.is_virtual_ref(gcref): return vref = lltype.cast_opaque_ptr(lltype.Ptr(self.JIT_VIRTUAL_REF), gcref) - assert vref.virtual_token == self.TOKEN_NONE - vref.virtual_token = self.TOKEN_TRACING_RESCALL + assert vref.virtual_token == TOKEN_NONE + vref.virtual_token = TOKEN_TRACING_RESCALL def tracing_after_residual_call(self, gcref): if not self.is_virtual_ref(gcref): return False vref = lltype.cast_opaque_ptr(lltype.Ptr(self.JIT_VIRTUAL_REF), gcref) assert vref.forced - if vref.virtual_token != self.TOKEN_NONE: + if vref.virtual_token != TOKEN_NONE: # not modified by the residual call; assert that it is still # set to TOKEN_TRACING_RESCALL and clear it. - assert vref.virtual_token == self.TOKEN_TRACING_RESCALL - vref.virtual_token = self.TOKEN_NONE + assert vref.virtual_token == TOKEN_TRACING_RESCALL + vref.virtual_token = TOKEN_NONE return False else: # marker "modified during residual call" set. @@ -118,8 +118,8 @@ return assert real_object vref = lltype.cast_opaque_ptr(lltype.Ptr(self.JIT_VIRTUAL_REF), gcref) - assert vref.virtual_token != self.TOKEN_TRACING_RESCALL - vref.virtual_token = self.TOKEN_NONE + assert vref.virtual_token != TOKEN_TRACING_RESCALL + vref.virtual_token = TOKEN_NONE vref.forced = lltype.cast_opaque_ptr(rclass.OBJECTPTR, real_object) # ____________________________________________________________ @@ -151,19 +151,19 @@ def force_virtual(self, inst): vref = lltype.cast_pointer(lltype.Ptr(self.JIT_VIRTUAL_REF), inst) token = vref.virtual_token - if token != self.TOKEN_NONE: - if token == self.TOKEN_TRACING_RESCALL: + if token != TOKEN_NONE: + if token == TOKEN_TRACING_RESCALL: # The "virtual" is not a virtual at all during tracing. # We only need to reset virtual_token to TOKEN_NONE # as a marker for the tracing, to tell it that this # "virtual" escapes. assert vref.forced - vref.virtual_token = self.TOKEN_NONE + vref.virtual_token = TOKEN_NONE else: assert not vref.forced from rpython.jit.metainterp.compile import ResumeGuardForcedDescr ResumeGuardForcedDescr.force_now(self.cpu, token) - assert vref.virtual_token == self.TOKEN_NONE + assert vref.virtual_token == TOKEN_NONE assert vref.forced elif not vref.forced: # token == TOKEN_NONE and the vref was not forced: it's invalid diff --git a/rpython/rtyper/lltypesystem/rvirtualizable2.py b/rpython/rtyper/lltypesystem/rvirtualizable2.py --- a/rpython/rtyper/lltypesystem/rvirtualizable2.py +++ b/rpython/rtyper/lltypesystem/rvirtualizable2.py @@ -9,15 +9,19 @@ def _setup_repr_llfields(self): llfields = [] if self.top_of_virtualizable_hierarchy: - llfields.append(('vable_token', lltype.Signed)) + llfields.append(('vable_token', llmemory.GCREF)) return llfields - def set_vable(self, llops, vinst, force_cast=False): - if self.top_of_virtualizable_hierarchy: - if force_cast: - vinst = llops.genop('cast_pointer', [vinst], resulttype=self) - cname = inputconst(lltype.Void, 'vable_token') - cvalue = inputconst(lltype.Signed, 0) - llops.genop('setfield', [vinst, cname, cvalue]) - else: - self.rbase.set_vable(llops, vinst, force_cast=True) +## The code below is commented out because vtable_token is always +## initialized to NULL anyway. +## +## def set_vable(self, llops, vinst, force_cast=False): +## if self.top_of_virtualizable_hierarchy: +## if force_cast: +## vinst = llops.genop('cast_pointer', [vinst], resulttype=self) +## cname = inputconst(lltype.Void, 'vable_token') +## cvalue = inputconst(llmemory.GCREF, +## lltype.nullptr(llmemory.GCREF.TO)) +## llops.genop('setfield', [vinst, cname, cvalue]) +## else: +## self.rbase.set_vable(llops, vinst, force_cast=True) diff --git a/rpython/rtyper/rvirtualizable2.py b/rpython/rtyper/rvirtualizable2.py --- a/rpython/rtyper/rvirtualizable2.py +++ b/rpython/rtyper/rvirtualizable2.py @@ -23,8 +23,8 @@ def _setup_repr_llfields(self): raise NotImplementedError - def set_vable(self, llops, vinst, force_cast=False): - raise NotImplementedError +## def set_vable(self, llops, vinst, force_cast=False): +## raise NotImplementedError def _setup_repr(self): if self.top_of_virtualizable_hierarchy: @@ -43,10 +43,10 @@ # not need it, but it doesn't hurt to have it anyway self.my_redirected_fields = self.rbase.my_redirected_fields - def new_instance(self, llops, classcallhop=None): - vptr = self._super().new_instance(llops, classcallhop) - self.set_vable(llops, vptr) - return vptr +## def new_instance(self, llops, classcallhop=None): +## vptr = self._super().new_instance(llops, classcallhop) +## self.set_vable(llops, vptr) +## return vptr def hook_access_field(self, vinst, cname, llops, flags): #if not flags.get('access_directly'): diff --git a/rpython/rtyper/test/test_rvirtualizable2.py b/rpython/rtyper/test/test_rvirtualizable2.py --- a/rpython/rtyper/test/test_rvirtualizable2.py +++ b/rpython/rtyper/test/test_rvirtualizable2.py @@ -1,5 +1,5 @@ import py -from rpython.rtyper.lltypesystem import lltype +from rpython.rtyper.lltypesystem import lltype, llmemory from rpython.rtyper.test.tool import BaseRtypingTest, LLRtypeMixin, OORtypeMixin from rpython.rtyper.rvirtualizable2 import replace_force_virtualizable_with_call from rpython.rlib.jit import hint @@ -373,7 +373,7 @@ assert res.item1 == 42 res = lltype.normalizeptr(res.item0) assert res.inst_v == 42 - assert res.vable_token == 0 + assert res.vable_token == lltype.nullptr(llmemory.GCREF.TO) class TestOOtype(OORtypeMixin, BaseTest): prefix = 'o' From noreply at buildbot.pypy.org Sun Jan 27 21:22:25 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 27 Jan 2013 21:22:25 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: implement rewriting of frame malloc Message-ID: <20130127202225.422A41C039A@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60541:cf0aed3ee46e Date: 2013-01-27 22:21 +0200 http://bitbucket.org/pypy/pypy/changeset/cf0aed3ee46e/ Log: implement rewriting of frame malloc diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py --- a/rpython/jit/backend/llsupport/gc.py +++ b/rpython/jit/backend/llsupport/gc.py @@ -122,6 +122,8 @@ for name in ['jf_descr', 'jf_guard_exc', 'jf_force_descr', 'jf_frame_info', 'jf_gcmap']: setattr(descrs, name, cpu.fielddescrof(jitframe.JITFRAME, name)) + descrs.jfi_frame_size = cpu.fielddescrof(jitframe.JITFRAMEINFO, + 'jfi_frame_size') descrs.jfi_frame_depth = cpu.fielddescrof(jitframe.JITFRAMEINFO, 'jfi_frame_depth') return descrs diff --git a/rpython/jit/backend/llsupport/jitframe.py b/rpython/jit/backend/llsupport/jitframe.py --- a/rpython/jit/backend/llsupport/jitframe.py +++ b/rpython/jit/backend/llsupport/jitframe.py @@ -14,10 +14,19 @@ GCMAP = lltype.GcArray(lltype.Unsigned) NULLGCMAP = lltype.nullptr(GCMAP) +def jitframeinfo_set_depth(jfi, new_depth): + jfi.jfi_frame_depth = new_depth + jfi.jfi_frame_size = STATICSIZE + new_depth * SIZEOFSIGNED + JITFRAMEINFO = lltype.GcStruct( 'JITFRAMEINFO', - # the depth of frame + # the depth of the frame ('jfi_frame_depth', lltype.Signed), + # the total size of the frame, in bytes + ('jfi_frame_size', lltype.Signed), + adtmeths = { + 'set_frame_depth': jitframeinfo_set_depth, + }, ) NULLFRAMEINFO = lltype.nullptr(JITFRAMEINFO) diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -137,22 +137,42 @@ else: raise NotImplementedError(op.getopname()) + def gen_malloc_frame(self, frame_info, frame, size_box): + descrs = self.gc_ll_descr.getframedescrs(self.cpu) + if self.gc_ll_descr.kind == 'boehm': + op0 = ResOperation(rop.GETFIELD_GC, [history.ConstPtr(frame_info)], + size_box, + descr=descrs.jfi_frame_depth) + self.newops.append(op0) + op1 = ResOperation(rop.NEW_ARRAY, [size_box], frame, + descr=descrs.arraydescr) + self.handle_new_array(descrs.arraydescr, op1) + else: + # we read size in bytes here, not the length + op0 = ResOperation(rop.GETFIELD_GC, [history.ConstPtr(frame_info)], + size_box, + descr=descrs.jfi_frame_size) + self.newops.append(op0) + self.gen_malloc_nursery_varsize(size_box, frame, is_small=True) + self.gen_initialize_tid(frame, descrs.arraydescr.tid) + length_box = history.BoxInt() + op1 = ResOperation(rop.GETFIELD_GC, [history.ConstPtr(frame_info)], + length_box, + descr=descrs.jfi_frame_depth) + self.newops.append(op1) + self.gen_initialize_len(frame, length_box, + descrs.arraydescr.lendescr) + def handle_call_assembler(self, op): descrs = self.gc_ll_descr.getframedescrs(self.cpu) loop_token = op.getdescr() assert isinstance(loop_token, history.JitCellToken) - lgt_box = history.BoxInt() - frame = history.BoxPtr() jfi = loop_token.compiled_loop_token.frame_info llref = lltype.cast_opaque_ptr(llmemory.GCREF, jfi) rgc._make_sure_does_not_move(llref) - op0 = ResOperation(rop.GETFIELD_GC, [history.ConstPtr(llref)], lgt_box, - descr=descrs.jfi_frame_depth) - self.newops.append(op0) - - op1 = ResOperation(rop.NEW_ARRAY, [lgt_box], frame, - descr=descrs.arraydescr) - self.handle_new_array(descrs.arraydescr, op1) + size_box = history.BoxInt() + frame = history.BoxPtr() + self.gen_malloc_frame(llref, frame, size_box) op2 = ResOperation(rop.SETFIELD_GC, [frame, history.ConstPtr(llref)], None, descr=descrs.jf_frame_info) self.newops.append(op2) @@ -258,6 +278,18 @@ self._gen_call_malloc_gc([ConstInt(addr), v_num_elem], v_result, self.gc_ll_descr.malloc_unicode_descr) + def gen_malloc_nursery_varsize(self, sizebox, v_result, is_small=False): + """ Generate CALL_MALLOC_NURSERY_VARSIZE_SMALL + """ + assert is_small + self.emitting_an_operation_that_can_collect() + op = ResOperation(rop.CALL_MALLOC_NURSERY_VARSIZE_SMALL, + [sizebox], + v_result) + + self.newops.append(op) + self.recent_mallocs[v_result] = None + def gen_malloc_nursery(self, size, v_result): """Try to generate or update a CALL_MALLOC_NURSERY. If that fails, generate a plain CALL_MALLOC_GC instead. diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py --- a/rpython/jit/backend/llsupport/test/test_rewrite.py +++ b/rpython/jit/backend/llsupport/test/test_rewrite.py @@ -76,8 +76,11 @@ ll_frame_info = lltype.cast_opaque_ptr(llmemory.GCREF, frame_info) clt.frame_info = frame_info frame_info.jfi_frame_depth = 13 + frame_info.jfi_frame_size = 255 framedescrs = self.gc_ll_descr.getframedescrs(self.cpu) + framelendescr = framedescrs.arraydescr.lendescr jfi_frame_depth = framedescrs.jfi_frame_depth + jfi_frame_size = framedescrs.jfi_frame_size jf_frame_info = framedescrs.jf_frame_info signedframedescr = self.cpu.signedframedescr floatframedescr = self.cpu.floatframedescr @@ -737,12 +740,13 @@ i2 = call_assembler(i0, f0, descr=casmdescr) """, """ [i0, f0] - i1 = getfield_gc(ConstPtr(ll_frame_info), descr=jfi_frame_depth) - p1 = call_malloc_nursery(13) + i1 = getfield_gc(ConstPtr(ll_frame_info), descr=jfi_frame_size) + p1 = call_malloc_nursery_varsize_small(i1) + setfield_gc(p1, 0, descr=tiddescr) + i2 = getfield_gc(ConstPtr(ll_frame_info), descr=jfi_frame_depth) + setfield_gc(p1, i2, descr=framelendescr) setfield_gc(p1, ConstPtr(ll_frame_info), descr=jf_frame_info) setarrayitem_gc(p1, 0, i0, descr=signedframedescr) setarrayitem_gc(p1, 1, f0, descr=floatframedescr) - i2 = call_assembler(p1, descr=casmdescr) + i3 = call_assembler(p1, descr=casmdescr) """) - # XXX we want call_malloc_nursery actually, but let's not care - # for now, the array is a bit non-standard diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -507,7 +507,7 @@ clt = CompiledLoopToken(self.cpu, looptoken.number) clt.frame_info = lltype.malloc(jitframe.JITFRAMEINFO) clt.allgcrefs = [] - clt.frame_info.jfi_frame_depth = 0 # for now + clt.frame_info.set_frame_depth(0) # for now looptoken.compiled_loop_token = clt clt._debug_nbargs = len(inputargs) if not we_are_translated(): @@ -666,13 +666,13 @@ mc.copy_to_raw_memory(rawstart + pos_after_jz - 4) def update_frame_depth(self, frame_depth): - self.current_clt.frame_info.jfi_frame_depth = frame_depth + self.current_clt.frame_info.set_frame_depth(frame_depth) new_jumping_to = [] for wref in self.current_clt.jumping_to: clt = wref() if clt is not None: - clt.frame_info.jfi_frame_depth = max(frame_depth, - clt.frame_info.jfi_frame_depth) + clt.frame_info.set_frame_depth(max(frame_depth, + clt.frame_info.jfi_frame_depth)) new_jumping_to.append(weakref.ref(clt)) self.current_clt.jumping_to = new_jumping_to @@ -898,7 +898,7 @@ # copy frame-info data old_fi = oldlooptoken.compiled_loop_token.frame_info new_fi = newlooptoken.compiled_loop_token.frame_info - old_fi.jfi_frame_depth = new_fi.jfi_frame_depth + old_fi.set_frame_depth(new_fi.jfi_frame_depth) mc = codebuf.MachineCodeBlockWrapper() mc.JMP(imm(target)) if WORD == 4: # keep in sync with prepare_loop() diff --git a/rpython/jit/backend/x86/test/test_gc_integration.py b/rpython/jit/backend/x86/test/test_gc_integration.py --- a/rpython/jit/backend/x86/test/test_gc_integration.py +++ b/rpython/jit/backend/x86/test/test_gc_integration.py @@ -451,6 +451,8 @@ setattr(descrs, name, cpu.fielddescrof(JITFRAME, name)) descrs.jfi_frame_depth = cpu.fielddescrof(jitframe.JITFRAMEINFO, 'jfi_frame_depth') + descrs.jfi_frame_size = cpu.fielddescrof(jitframe.JITFRAMEINFO, + 'jfi_frame_size') return descrs def do_write_barrier(self, gcref_struct, gcref_newptr): diff --git a/rpython/jit/backend/x86/test/test_zrpy_gc.py b/rpython/jit/backend/x86/test/test_zrpy_gc.py --- a/rpython/jit/backend/x86/test/test_zrpy_gc.py +++ b/rpython/jit/backend/x86/test/test_zrpy_gc.py @@ -790,5 +790,11 @@ def test_compile_framework_minimal_size_in_nursery(self): self.run('compile_framework_minimal_size_in_nursery') + #def define_compile_framework_call_assembler(self): + # xxx + + #def test_compile_framework_call_assembler(self): + # self.run('compile_framework_call_assembler') + class TestShadowStack(CompileFrameworkTests): gcrootfinder = "shadowstack" diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py --- a/rpython/jit/metainterp/executor.py +++ b/rpython/jit/metainterp/executor.py @@ -354,6 +354,7 @@ rop.QUASIIMMUT_FIELD, rop.CALL_MALLOC_GC, rop.CALL_MALLOC_NURSERY, + rop.CALL_MALLOC_NURSERY_VARSIZE_SMALL, rop.LABEL, ): # list of opcodes never executed by pyjitpl continue From noreply at buildbot.pypy.org Sun Jan 27 21:28:26 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 27 Jan 2013 21:28:26 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: an attempt to have a call assembler somewhere in zrpy_gc tests Message-ID: <20130127202826.9750F1C039A@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60542:f0fddff075fb Date: 2013-01-27 22:27 +0200 http://bitbucket.org/pypy/pypy/changeset/f0fddff075fb/ Log: an attempt to have a call assembler somewhere in zrpy_gc tests diff --git a/rpython/jit/backend/x86/test/test_zrpy_gc.py b/rpython/jit/backend/x86/test/test_zrpy_gc.py --- a/rpython/jit/backend/x86/test/test_zrpy_gc.py +++ b/rpython/jit/backend/x86/test/test_zrpy_gc.py @@ -790,11 +790,25 @@ def test_compile_framework_minimal_size_in_nursery(self): self.run('compile_framework_minimal_size_in_nursery') - #def define_compile_framework_call_assembler(self): - # xxx + def define_compile_framework_call_assembler(self): + S = lltype.GcForwardReference() + S.become(lltype.GcStruct('S', ('s', lltype.Ptr(S)))) + driver = JitDriver(greens = [], reds = ['x', 'x0', 'x2']) - #def test_compile_framework_call_assembler(self): - # self.run('compile_framework_call_assembler') + def f(n, x, x0, x1, x2, x3, x4, x5, x6, x7, l, s0): + driver.jit_merge_point(x=x, x0=x0, x2=x2) + i = 0 + prev_s = lltype.nullptr(S) + while i < 100: + s = lltype.malloc(S) + s.s = prev_s + prev_s = s + return n - 1, x, x0, x1, x2, x3, x4, x5, x6, x7, l, s0 + + return None, f, None + + def test_compile_framework_call_assembler(self): + self.run('compile_framework_call_assembler') class TestShadowStack(CompileFrameworkTests): gcrootfinder = "shadowstack" From noreply at buildbot.pypy.org Sun Jan 27 21:33:11 2013 From: noreply at buildbot.pypy.org (mattip) Date: Sun, 27 Jan 2013 21:33:11 +0100 (CET) Subject: [pypy-commit] pypy win32-cleanup3: abandon, fixed differently Message-ID: <20130127203311.D61BA1C039A@cobra.cs.uni-duesseldorf.de> Author: Matti Picus Branch: win32-cleanup3 Changeset: r60543:2eb0f70e009a Date: 2013-01-27 22:18 +0200 http://bitbucket.org/pypy/pypy/changeset/2eb0f70e009a/ Log: abandon, fixed differently From noreply at buildbot.pypy.org Sun Jan 27 21:33:14 2013 From: noreply at buildbot.pypy.org (mattip) Date: Sun, 27 Jan 2013 21:33:14 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: merge default into about-to-be-reviewed branch Message-ID: <20130127203314.05D261C039A@cobra.cs.uni-duesseldorf.de> Author: Matti Picus Branch: missing-ndarray-attributes Changeset: r60544:421ad500fd66 Date: 2013-01-27 22:22 +0200 http://bitbucket.org/pypy/pypy/changeset/421ad500fd66/ Log: merge default into about-to-be-reviewed branch diff too long, truncating to 2000 out of 3725 lines diff --git a/lib-python/2.7/test/test_iterlen.py b/lib-python/2.7/test/test_iterlen.py --- a/lib-python/2.7/test/test_iterlen.py +++ b/lib-python/2.7/test/test_iterlen.py @@ -94,7 +94,11 @@ def test_no_len_for_infinite_repeat(self): # The repeat() object can also be infinite - self.assertRaises(TypeError, len, repeat(None)) + if test_support.check_impl_detail(pypy=True): + # 3.4 (PEP 424) behavior + self.assertEqual(len(repeat(None)), NotImplemented) + else: + self.assertRaises(TypeError, len, repeat(None)) class TestXrange(TestInvariantWithoutMutations): @@ -230,6 +234,7 @@ self.assertRaises(RuntimeError, b.extend, BadLen()) self.assertRaises(RuntimeError, b.extend, BadLengthHint()) + @test_support.impl_detail("PEP 424 disallows None results", pypy=False) def test_invalid_hint(self): # Make sure an invalid result doesn't muck-up the works self.assertEqual(list(NoneLengthHint()), list(range(10))) diff --git a/lib-python/conftest.py b/lib-python/conftest.py --- a/lib-python/conftest.py +++ b/lib-python/conftest.py @@ -281,7 +281,7 @@ RegrTest('test_ioctl.py'), RegrTest('test_isinstance.py', core=True), RegrTest('test_iter.py', core=True), - RegrTest('test_iterlen.py', skip="undocumented internal API behavior __length_hint__"), + RegrTest('test_iterlen.py', core=True, usemodules="_collections itertools"), RegrTest('test_itertools.py', core=True, usemodules="itertools struct"), RegrTest('test_json.py'), RegrTest('test_kqueue.py'), diff --git a/py/_code/source.py b/py/_code/source.py --- a/py/_code/source.py +++ b/py/_code/source.py @@ -141,6 +141,8 @@ trysource = self[start:end] if trysource.isparseable(): return start, end + if end == start + 100: # XXX otherwise, it takes forever + break # XXX if end is None: raise IndexError("no valid source range around line %d " % (lineno,)) return start, end diff --git a/pypy/doc/getting-started-python.rst b/pypy/doc/getting-started-python.rst --- a/pypy/doc/getting-started-python.rst +++ b/pypy/doc/getting-started-python.rst @@ -86,8 +86,8 @@ 4. Run:: - cd pypy/translator/goal - python translate.py --opt=jit targetpypystandalone.py + cd pypy/goal + python ../../rpython/bin/rpython --opt=jit targetpypystandalone.py possibly replacing ``--opt=jit`` with another `optimization level`_ of your choice like ``--opt=2`` if you do not want to include the JIT diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -34,6 +34,8 @@ .. branch: kill-faking .. branch: improved_ebnfparse_error .. branch: task-decorator +.. branch: fix-e4fa0b2 +.. branch: win32-fixes .. branch: release-2.0-beta1 @@ -43,3 +45,7 @@ .. branch: fix-lookinside-iff-oopspec Fixed the interaction between two internal tools for controlling the JIT. + +.. branch: inline-virtualref-2 +Better optimized certain types of frame accesses in the JIT, particularly +around exceptions that escape the function they were raised in. diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -713,13 +713,13 @@ def pypy_find_stdlib(s): from os.path import abspath, join, dirname as dn thisfile = abspath(__file__) - root = dn(dn(dn(dn(thisfile)))) + root = dn(dn(dn(thisfile))) return [join(root, 'lib-python', '2.7'), join(root, 'lib_pypy')] def pypy_resolvedirof(s): # we ignore the issue of symlinks; for tests, the executable is always - # translator/goal/app_main.py anyway + # interpreter/app_main.py anyway import os return os.path.abspath(os.path.join(s, '..')) diff --git a/pypy/interpreter/test2/test_app_main.py b/pypy/interpreter/test2/test_app_main.py --- a/pypy/interpreter/test2/test_app_main.py +++ b/pypy/interpreter/test2/test_app_main.py @@ -547,7 +547,7 @@ assert line.rstrip() == 'Not at all. They could be carried.' print 'A five ounce bird could not carry a one pound coconut.' """) - py_py = os.path.join(pypydir, 'bin', 'py.py') + py_py = os.path.join(pypydir, 'bin', 'pyinteractive.py') child = self._spawn(sys.executable, [py_py, '-S', path]) child.expect('Are you suggesting coconuts migrate?', timeout=120) child.sendline('Not at all. They could be carried.') @@ -907,7 +907,7 @@ sys.path.append(self.goal_dir) try: import app_main - pypy_c = os.path.join(self.trunkdir, 'pypy', 'translator', 'goal', 'pypy-c') + pypy_c = os.path.join(self.trunkdir, 'pypy', 'goal', 'pypy-c') app_main.setup_bootstrap_path(pypy_c) newpath = sys.path[:] # we get at least lib_pypy @@ -925,7 +925,7 @@ sys.path.append(self.goal_dir) try: import app_main - pypy_c = os.path.join(self.trunkdir, 'pypy', 'translator', 'goal', 'pypy-c') + pypy_c = os.path.join(self.trunkdir, 'pypy', 'goal', 'pypy-c') app_main.entry_point(pypy_c, [self.foo_py]) # assert it did not crash finally: diff --git a/pypy/module/_collections/interp_deque.py b/pypy/module/_collections/interp_deque.py --- a/pypy/module/_collections/interp_deque.py +++ b/pypy/module/_collections/interp_deque.py @@ -521,7 +521,11 @@ return self.space.wrap(self.counter) def next(self): - self.deque.checklock(self.lock) + if self.lock is not self.deque.lock: + self.counter = 0 + raise OperationError( + self.space.w_RuntimeError, + self.space.wrap("deque mutated during iteration")) if self.counter == 0: raise OperationError(self.space.w_StopIteration, self.space.w_None) self.counter -= 1 @@ -560,7 +564,11 @@ return self.space.wrap(self.counter) def next(self): - self.deque.checklock(self.lock) + if self.lock is not self.deque.lock: + self.counter = 0 + raise OperationError( + self.space.w_RuntimeError, + self.space.wrap("deque mutated during iteration")) if self.counter == 0: raise OperationError(self.space.w_StopIteration, self.space.w_None) self.counter -= 1 diff --git a/pypy/module/pypyjit/test_pypy_c/test_thread.py b/pypy/module/pypyjit/test_pypy_c/test_thread.py --- a/pypy/module/pypyjit/test_pypy_c/test_thread.py +++ b/pypy/module/pypyjit/test_pypy_c/test_thread.py @@ -26,3 +26,25 @@ --THREAD-TICK-- jump(..., descr=...) """) + + def test_tls(self): + def main(n): + import thread + local = thread._local() + local.x = 1 + i = 0 + while i < n: + i += local.x + return 0 + log = self.run(main, [500]) + assert round(log.result, 6) == round(main(500), 6) + loop, = log.loops_by_filename(self.filepath) + assert loop.match(""" + i53 = int_lt(i48, i27) + guard_true(i53, descr=...) + i54 = int_add_ovf(i48, i47) + guard_no_overflow(descr=...) + --TICK-- + i58 = arraylen_gc(p43, descr=...) + jump(p0, p1, p3, p5, p10, p12, p14, i54, i27, i47, p45, p43, descr=...) + """) diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py --- a/pypy/module/rctime/interp_time.py +++ b/pypy/module/rctime/interp_time.py @@ -448,6 +448,12 @@ raise OperationError(space.w_ValueError, space.wrap("year out of range")) + # tm_wday does not need checking of its upper-bound since taking "% + # 7" in gettmarg() automatically restricts the range. + if rffi.getintfield(glob_buf, 'c_tm_wday') < -1: + raise OperationError(space.w_ValueError, + space.wrap("day of week out of range")) + rffi.setintfield(glob_buf, 'c_tm_year', y - 1900) rffi.setintfield(glob_buf, 'c_tm_mon', rffi.getintfield(glob_buf, 'c_tm_mon') - 1) @@ -456,12 +462,6 @@ rffi.setintfield(glob_buf, 'c_tm_yday', rffi.getintfield(glob_buf, 'c_tm_yday') - 1) - # tm_wday does not need checking of its upper-bound since taking "% - # 7" in gettmarg() automatically restricts the range. - if rffi.getintfield(glob_buf, 'c_tm_wday') < 0: - raise OperationError(space.w_ValueError, - space.wrap("day of week out of range")) - return glob_buf def time(space): diff --git a/pypy/module/signal/test/test_signal.py b/pypy/module/signal/test/test_signal.py --- a/pypy/module/signal/test/test_signal.py +++ b/pypy/module/signal/test/test_signal.py @@ -36,7 +36,7 @@ class AppTestSignal: spaceconfig = { - "usemodules": ['signal', 'rctime'], + "usemodules": ['signal', 'rctime'] + (['fcntl'] if os.name != 'nt' else []), } def setup_class(cls): diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -29,7 +29,7 @@ import pypy pypydir = os.path.dirname(os.path.abspath(pypy.__file__)) del pypy -from pypy.tool.version import get_repo_version_info +from rpython.tool.version import get_repo_version_info import time as t gmtime = t.gmtime() diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py --- a/pypy/module/sys/vm.py +++ b/pypy/module/sys/vm.py @@ -95,7 +95,7 @@ def exc_info_with_tb(space): operror = space.getexecutioncontext().sys_exc_info() if operror is None: - return space.newtuple([space.w_None,space.w_None,space.w_None]) + return space.newtuple([space.w_None, space.w_None, space.w_None]) else: return space.newtuple([operror.w_type, operror.get_w_value(space), space.wrap(operror.get_traceback())]) diff --git a/pypy/module/thread/os_local.py b/pypy/module/thread/os_local.py --- a/pypy/module/thread/os_local.py +++ b/pypy/module/thread/os_local.py @@ -30,6 +30,9 @@ w_dict = space.newdict(instance=True) self.dicts[ec] = w_dict self._register_in_ec(ec) + # cache the last seen dict, works because we are protected by the GIL + self.last_dict = w_dict + self.last_ec = ec def _register_in_ec(self, ec): if not ec.space.config.translation.rweakref: @@ -60,10 +63,14 @@ def getdict(self, space): ec = space.getexecutioncontext() + if ec is self.last_ec: + return self.last_dict try: w_dict = self.dicts[ec] except KeyError: w_dict = self.create_new_dict(ec) + self.last_ec = ec + self.last_dict = w_dict return w_dict def descr_local__new__(space, w_subtype, __args__): @@ -91,3 +98,5 @@ local = wref() if local is not None: del local.dicts[ec] + local.last_dict = None + local.last_ec = None diff --git a/pypy/module/thread/test/test_local.py b/pypy/module/thread/test/test_local.py --- a/pypy/module/thread/test/test_local.py +++ b/pypy/module/thread/test/test_local.py @@ -108,3 +108,54 @@ gc.collect() assert done == ['ok', 'del'] done.append('shutdown') + +def test_local_caching(): + from pypy.module.thread.os_local import Local + class FakeSpace: + def getexecutioncontext(self): + return self.ec + + def getattr(*args): + pass + def call_obj_args(*args): + pass + def newdict(*args, **kwargs): + return {} + def wrap(self, obj): + return obj + def type(self, obj): + return type(obj) + class config: + class translation: + rweakref = True + + class FakeEC: + def __init__(self, space): + self.space = space + self._thread_local_objs = None + space = FakeSpace() + ec1 = FakeEC(space) + space.ec = ec1 + + l = Local(space, None) + assert l.last_dict is l.dicts[ec1] + assert l.last_ec is ec1 + d1 = l.getdict(space) + assert d1 is l.last_dict + + ec2 = space.ec = FakeEC(space) + d2 = l.getdict(space) + assert l.last_dict is d2 + assert d2 is l.dicts[ec2] + assert l.last_ec is ec2 + dicts = l.dicts + l.dicts = "nope" + assert l.getdict(space) is d2 + l.dicts = dicts + + space.ec = ec1 + assert l.getdict(space) is d1 + l.dicts = "nope" + assert l.getdict(space) is d1 + l.dicts = dicts + diff --git a/pypy/module/thread/test/test_thread.py b/pypy/module/thread/test/test_thread.py --- a/pypy/module/thread/test/test_thread.py +++ b/pypy/module/thread/test/test_thread.py @@ -220,16 +220,24 @@ import signal def f(): - time.sleep(0.5) - thread.interrupt_main() + for x in range(50): + if waiting: + thread.interrupt_main() + return + print 'tock...', x # <-force the GIL to be released, as + time.sleep(0.01) # time.sleep doesn't do non-translated def busy_wait(): - for x in range(1000): + waiting.append(None) + for x in range(100): print 'tick...', x # <-force the GIL to be released, as time.sleep(0.01) # time.sleep doesn't do non-translated + waiting.pop() # This is normally called by app_main.py signal.signal(signal.SIGINT, signal.default_int_handler) - thread.start_new_thread(f, ()) - raises(KeyboardInterrupt, busy_wait) + for i in range(100): + waiting = [] + thread.start_new_thread(f, ()) + raises(KeyboardInterrupt, busy_wait) diff --git a/pypy/module/unicodedata/interp_ucd.py b/pypy/module/unicodedata/interp_ucd.py --- a/pypy/module/unicodedata/interp_ucd.py +++ b/pypy/module/unicodedata/interp_ucd.py @@ -9,7 +9,7 @@ from rpython.rlib.objectmodel import we_are_translated from rpython.rlib.runicode import MAXUNICODE from rpython.rlib.unicodedata import unicodedb_5_2_0, unicodedb_3_2_0 -from rpython.rlib.runicode import code_to_unichr, ORD +from rpython.rlib.runicode import code_to_unichr, ord_accepts_surrogate import sys @@ -28,8 +28,6 @@ # handling: on narrow unicode builds, a surrogate pair is considered as one # unicode code point. -# The functions below are subtly different from the ones in runicode.py. -# When PyPy implements Python 3 they should be merged. if MAXUNICODE > 0xFFFF: # Target is wide build @@ -41,7 +39,7 @@ if not we_are_translated() and sys.maxunicode == 0xFFFF: # Host CPython is narrow build, accept surrogates try: - return ORD(space.unicode_w(w_unichr)) + return ord_accepts_surrogate(space.unicode_w(w_unichr)) except ValueError: raise OperationError(space.w_TypeError, space.wrap( 'need a single Unicode character as parameter')) @@ -68,7 +66,7 @@ else: # Accept surrogates try: - return ORD(space.unicode_w(w_unichr)) + return ord_accepts_surrogate(space.unicode_w(w_unichr)) except ValueError: raise OperationError(space.w_TypeError, space.wrap( 'need a single Unicode character as parameter')) diff --git a/pypy/module/unicodedata/test/test_unicodedata.py b/pypy/module/unicodedata/test/test_unicodedata.py --- a/pypy/module/unicodedata/test/test_unicodedata.py +++ b/pypy/module/unicodedata/test/test_unicodedata.py @@ -82,8 +82,7 @@ import unicodedata raises(TypeError, unicodedata.normalize, 'x') - @py.test.mark.skipif("sys.maxunicode < 0x10ffff", - reason="requires a 'wide' python build.") + @py.test.mark.skipif("sys.maxunicode < 0x10ffff") def test_normalize_wide(self): import unicodedata assert unicodedata.normalize('NFC', u'\U000110a5\U000110ba') == u'\U000110ab' diff --git a/pypy/objspace/std/bytearraytype.py b/pypy/objspace/std/bytearraytype.py --- a/pypy/objspace/std/bytearraytype.py +++ b/pypy/objspace/std/bytearraytype.py @@ -15,6 +15,7 @@ str_splitlines, str_translate) from pypy.objspace.std.listtype import ( list_append, list_extend) +from rpython.rlib.objectmodel import newlist_hint, resizelist_hint bytearray_insert = SMM('insert', 3, @@ -87,8 +88,10 @@ return [c for c in string] # sequence of bytes - data = [] w_iter = space.iter(w_source) + length_hint = space.length_hint(w_source, 0) + data = newlist_hint(length_hint) + extended = 0 while True: try: w_item = space.next(w_iter) @@ -98,6 +101,9 @@ break value = getbytevalue(space, w_item) data.append(value) + extended += 1 + if extended < length_hint: + resizelist_hint(data, extended) return data def descr_bytearray__reduce__(space, w_self): diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py --- a/pypy/objspace/std/dictmultiobject.py +++ b/pypy/objspace/std/dictmultiobject.py @@ -54,6 +54,9 @@ # every module needs its own strategy, because the strategy stores # the version tag strategy = ModuleDictStrategy(space) + elif space.config.objspace.std.withmapdict and instance: + from pypy.objspace.std.mapdict import MapDictStrategy + strategy = space.fromcache(MapDictStrategy) elif instance or strdict or module: assert w_type is None @@ -349,7 +352,7 @@ self.pos = 0 def length(self): - if self.dictimplementation is not None: + if self.dictimplementation is not None and self.len != -1: return self.len - self.pos return 0 diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py --- a/pypy/objspace/std/mapdict.py +++ b/pypy/objspace/std/mapdict.py @@ -593,6 +593,9 @@ # ____________________________________________________________ # dict implementation +def get_terminator_for_dicts(space): + return DictTerminator(space, None) + class MapDictStrategy(DictStrategy): erase, unerase = rerased.new_erasing_pair("map") @@ -602,13 +605,19 @@ def __init__(self, space): self.space = space + def get_empty_storage(self): + w_result = Object() + terminator = self.space.fromcache(get_terminator_for_dicts) + w_result._init_empty(terminator) + return self.erase(w_result) + def switch_to_object_strategy(self, w_dict): w_obj = self.unerase(w_dict.dstorage) strategy = self.space.fromcache(ObjectDictStrategy) dict_w = strategy.unerase(strategy.get_empty_storage()) w_dict.strategy = strategy w_dict.dstorage = strategy.erase(dict_w) - assert w_obj.getdict(self.space) is w_dict + assert w_obj.getdict(self.space) is w_dict or w_obj._get_mapdict_map().terminator.w_cls is None materialize_r_dict(self.space, w_obj, dict_w) def getitem(self, w_dict, w_key): diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py --- a/pypy/objspace/std/setobject.py +++ b/pypy/objspace/std/setobject.py @@ -898,7 +898,7 @@ raise NotImplementedError def length(self): - if self.setimplementation is not None: + if self.setimplementation is not None and self.len != -1: return self.len - self.pos return 0 diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py --- a/pypy/objspace/std/test/test_dictmultiobject.py +++ b/pypy/objspace/std/test/test_dictmultiobject.py @@ -954,6 +954,7 @@ withcelldict = False withmethodcache = False withidentitydict = False + withmapdict = False FakeSpace.config = Config() diff --git a/pypy/objspace/std/test/test_lengthhint.py b/pypy/objspace/std/test/test_lengthhint.py --- a/pypy/objspace/std/test/test_lengthhint.py +++ b/pypy/objspace/std/test/test_lengthhint.py @@ -6,7 +6,7 @@ SIZE = 4 ITEMS = range(SIZE) - def _test_length_hint(self, w_obj): + def _test_length_hint(self, w_obj, w_mutate=None): space = self.space assert space.length_hint(w_obj, 8) == self.SIZE @@ -18,6 +18,13 @@ space.next(w_iter) assert space.length_hint(w_iter, 8) == self.SIZE - 1 + if w_mutate is not None: + # Test handling of collections that enforce length + # immutability during iteration + space.call_function(w_mutate) + space.raises_w(space.w_RuntimeError, space.next, w_iter) + assert space.length_hint(w_iter, 8) == 0 + def test_bytearray(self): space = self.space w_bytearray = space.call_function(space.w_bytearray, @@ -31,16 +38,20 @@ self._test_length_hint(w_dict) def test_dict_iterkeys(self): - w_iterkeys = self.space.appexec([], """(): - return dict.fromkeys(%r).iterkeys() - """ % self.ITEMS) - self._test_length_hint(w_iterkeys) + space = self.space + w_iterkeys, w_mutate = space.fixedview(space.appexec([], """(): + d = dict.fromkeys(%r) + return d.iterkeys(), d.popitem + """ % self.ITEMS), 2) + self._test_length_hint(w_iterkeys, w_mutate) def test_dict_values(self): - w_itervalues = self.space.appexec([], """(): - return dict.fromkeys(%r).itervalues() - """ % self.ITEMS) - self._test_length_hint(w_itervalues) + space = self.space + w_itervalues, w_mutate = space.fixedview(space.appexec([], """(): + d = dict.fromkeys(%r) + return d.itervalues(), d.popitem + """ % self.ITEMS), 2) + self._test_length_hint(w_itervalues, w_mutate) def test_frozenset(self): space = self.space @@ -50,7 +61,8 @@ def test_set(self): space = self.space w_set = space.call_function(space.w_set, space.wrap(self.ITEMS)) - self._test_length_hint(w_set) + w_mutate = space.getattr(w_set, space.wrap('pop')) + self._test_length_hint(w_set, w_mutate) def test_list(self): self._test_length_hint(self.space.wrap(self.ITEMS)) @@ -101,12 +113,19 @@ self._test_length_hint(W_Repeat(space, space.wrap(22), space.wrap(self.SIZE))) - def test_collections_deque(self): + def _setup_deque(self): space = self.space w_deque = W_Deque(space) space.call_method(w_deque, '__init__', space.wrap(self.ITEMS)) - self._test_length_hint(w_deque) - self._test_length_hint(w_deque.reviter()) + w_mutate = space.getattr(w_deque, space.wrap('pop')) + return w_deque, w_mutate + + def test_collections_deque(self): + self._test_length_hint(*self._setup_deque()) + + def test_collections_deque_rev(self): + w_deque, w_mutate = self._setup_deque() + self._test_length_hint(w_deque.reviter(), w_mutate) def test_default(self): space = self.space diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py --- a/pypy/objspace/std/test/test_mapdict.py +++ b/pypy/objspace/std/test/test_mapdict.py @@ -1,7 +1,17 @@ from pypy.objspace.std.test.test_dictmultiobject import FakeSpace, W_DictMultiObject from pypy.objspace.std.mapdict import * +class Config: + class objspace: + class std: + withsmalldicts = False + withcelldict = False + withmethodcache = False + withidentitydict = False + withmapdict = True + space = FakeSpace() +space.config = Config class Class(object): def __init__(self, hasdict=True): @@ -1061,3 +1071,11 @@ return A() """) assert w_dict.user_overridden_class + +def test_newdict_instance(): + w_dict = space.newdict(instance=True) + assert type(w_dict.strategy) is MapDictStrategy + +class TestMapDictImplementationUsingnewdict(BaseTestRDictImplementation): + StrategyClass = MapDictStrategy + # NB: the get_impl method is not overwritten here, as opposed to above diff --git a/pypy/tool/jitlogparser/parser.py b/pypy/tool/jitlogparser/parser.py --- a/pypy/tool/jitlogparser/parser.py +++ b/pypy/tool/jitlogparser/parser.py @@ -215,6 +215,7 @@ _lineset = None is_bytecode = False inline_level = None + bytecode_name = None # factory method TraceForOpcode = TraceForOpcode @@ -244,23 +245,29 @@ return ",".join([str(len(v)) for v in stack]) def append_to_res(bc): - if not stack: - stack.append([]) - else: - if bc.inline_level is not None and bc.inline_level + 1 != len(stack): - if bc.inline_level < len(stack): + if bc.inline_level is not None: + if bc.inline_level == len(stack) - 1: + pass + elif bc.inline_level > len(stack) - 1: + stack.append([]) + else: + while bc.inline_level + 1 < len(stack): last = stack.pop() stack[-1].append(cls(last, getpath(stack), storage)) - else: - stack.append([]) stack[-1].append(bc) so_far = [] stack = [] + nothing_yet = True for op in operations: if op.name == 'debug_merge_point': if so_far: - append_to_res(cls.TraceForOpcode(so_far, storage, loopname)) + opc = cls.TraceForOpcode(so_far, storage, loopname) + if nothing_yet: + nothing_yet = False + for i in xrange(opc.inline_level + 1): + stack.append([]) + append_to_res(opc) if limit: break so_far = [] diff --git a/pypy/tool/jitlogparser/test/test_parser.py b/pypy/tool/jitlogparser/test/test_parser.py --- a/pypy/tool/jitlogparser/test/test_parser.py +++ b/pypy/tool/jitlogparser/test/test_parser.py @@ -354,3 +354,25 @@ f = Function.from_operations(loop.operations, LoopStorage()) assert f.chunks[-1].filename == 'x.py' assert f.filename is None + +def test_parse_2_levels_up(): + loop = parse(""" + [] + debug_merge_point(0, 0, 'one') + debug_merge_point(1, 0, 'two') + debug_merge_point(2, 0, 'three') + debug_merge_point(0, 0, 'one') + """) + f = Function.from_operations(loop.operations, LoopStorage()) + assert len(f.chunks) == 3 + +def test_parse_from_inside(): + loop = parse(""" + [] + debug_merge_point(1, 0, 'two') + debug_merge_point(2, 0, 'three') + debug_merge_point(0, 0, 'one') + """) + f = Function.from_operations(loop.operations, LoopStorage()) + assert len(f.chunks) == 2 + diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -1,6 +1,6 @@ #!/usr/bin/env python """ A sample script that packages PyPy, provided that it's already built. -It uses 'pypy/translator/goal/pypy-c' and parts of the rest of the working +It uses 'pypy/goal/pypy-c' and parts of the rest of the working copy. Usage: package.py root-pypy-dir [name-of-archive] [name-of-pypy-c] [destination-for-tarball] [pypy-c-path] @@ -13,7 +13,7 @@ import sys import os #Add toplevel repository dir to sys.path -sys.path.insert(0,os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))) +sys.path.insert(0,os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))) import py import fnmatch from rpython.tool.udir import udir diff --git a/pypy/tool/release/test/test_package.py b/pypy/tool/release/test/test_package.py --- a/pypy/tool/release/test/test_package.py +++ b/pypy/tool/release/test/test_package.py @@ -15,7 +15,7 @@ basename = 'pypy-c' rename_pypy_c = 'pypy' exe_name_in_archive = 'bin/pypy' - pypy_c = py.path.local(pypydir).join('..', basename) + pypy_c = py.path.local(pypydir).join('goal', basename) if not pypy_c.check(): os.system("echo faked_pypy_c> %s" % (pypy_c,)) pypy_c.chmod(0755) diff --git a/pypy/tool/rundictbenchmarks.py b/pypy/tool/rundictbenchmarks.py --- a/pypy/tool/rundictbenchmarks.py +++ b/pypy/tool/rundictbenchmarks.py @@ -3,7 +3,7 @@ # this file runs some benchmarks with a pypy-c that is assumed to be # built using the MeasuringDictImplementation. -# it should be run with pypy/translator/goal as the cwd, and you'll +# it should be run with pypy/goal as the cwd, and you'll # need to hack a copy of rst2html for yourself (svn docutils # required). diff --git a/rpython/annotator/builtin.py b/rpython/annotator/builtin.py --- a/rpython/annotator/builtin.py +++ b/rpython/annotator/builtin.py @@ -79,7 +79,7 @@ s_item = s_ImpossibleValue else: nonneg = False # so far - if step > 0: + if step > 0 or s_step.nonneg: nonneg = s_start.nonneg elif step < 0: nonneg = s_stop.nonneg or (s_stop.is_constant() and diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py --- a/rpython/annotator/test/test_annrpython.py +++ b/rpython/annotator/test/test_annrpython.py @@ -2694,6 +2694,23 @@ assert isinstance(s, annmodel.SomeInteger) assert s.nonneg + def test_range_nonneg_variablestep(self): + def get_step(n): + if n == 1: + return 2 + else: + return 3 + def fun(n, k): + step = get_step(n) + for i in range(0, n, step): + if k == 17: + return i + return 0 + a = self.RPythonAnnotator() + s = a.build_types(fun, [int, int]) + assert isinstance(s, annmodel.SomeInteger) + assert s.nonneg + def test_reverse_range_nonneg(self): def fun(n, k): for i in range(n-1, -1, -1): @@ -3254,6 +3271,7 @@ a = self.RPythonAnnotator() s = a.build_types(f, [str]) assert isinstance(s, annmodel.SomeString) + assert s.no_nul def f(x): return u'a'.replace(x, u'b') diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py --- a/rpython/annotator/unaryop.py +++ b/rpython/annotator/unaryop.py @@ -509,7 +509,7 @@ return getbookkeeper().newlist(s_item) def method_replace(str, s1, s2): - return str.basestringclass() + return str.basestringclass(no_nul=str.no_nul and s2.no_nul) def getslice(str, s_start, s_stop): check_negative_slice(s_start, s_stop) diff --git a/rpython/jit/backend/llgraph/llimpl.py b/rpython/jit/backend/llgraph/llimpl.py deleted file mode 100644 --- a/rpython/jit/backend/llgraph/llimpl.py +++ /dev/null @@ -1,1968 +0,0 @@ -""" -The non-RPythonic part of the llgraph backend. -This contains all the code that is directly run -when executing on top of the llinterpreter. -""" - -import weakref -from rpython.flowspace.model import Variable, Constant -from rpython.annotator import model as annmodel -from rpython.jit.metainterp.history import REF, INT, FLOAT -from rpython.jit.metainterp import history -from rpython.jit.codewriter import heaptracker -from rpython.rtyper.lltypesystem import lltype, llmemory, rclass, rstr, rffi -from rpython.rtyper.ootypesystem import ootype -from rpython.rtyper.module.support import LLSupport, OOSupport -from rpython.rtyper.llinterp import LLException -from rpython.rtyper.extregistry import ExtRegistryEntry - -from rpython.jit.metainterp import resoperation -from rpython.jit.metainterp.resoperation import rop -from rpython.jit.backend.llgraph import symbolic -from rpython.jit.codewriter import longlong -from rpython.jit.codewriter.effectinfo import EffectInfo - -from rpython.rlib.objectmodel import ComputedIntSymbolic, we_are_translated -from rpython.rlib.rarithmetic import ovfcheck -from rpython.rlib.rarithmetic import r_longlong, r_ulonglong, r_uint -from rpython.rlib.rtimer import read_timestamp - -import py -from rpython.tool.ansi_print import ansi_log -log = py.log.Producer('runner') -py.log.setconsumer('runner', ansi_log) - -IS_32_BIT = r_ulonglong is not r_uint - - -def _from_opaque(opq): - return opq._obj.externalobj - -_TO_OPAQUE = {} - -def _to_opaque(value): - try: - return value._the_opaque_pointer - except AttributeError: - op = lltype.opaqueptr(_TO_OPAQUE[value.__class__], 'opaque', - externalobj=value) - value._the_opaque_pointer = op - return op - -def _normalize(value): - if isinstance(value, lltype._ptr): - value = lltype.top_container(value._obj) - return value - -def from_opaque_string(s): - if isinstance(s, str): - return s - elif isinstance(s, ootype._string): - return OOSupport.from_rstr(s) - else: - return LLSupport.from_rstr(s) - -FLOAT_ARRAY_TP = lltype.Ptr(lltype.Array(lltype.Float, hints={"nolength": True})) -def maybe_uncast(TP, array): - if array._TYPE.TO.OF != lltype.Float: - # array._TYPE.TO._hints.get("uncast_on_llgraph"): - array = rffi.cast(TP, array) - return array - -# a list of argtypes of all operations - couldn't find any and it's -# very useful. Note however that the table is half-broken here and -# there, in ways that are sometimes a bit hard to fix; that's why -# it is not "official". -TYPES = { - 'int_add' : (('int', 'int'), 'int'), - 'int_sub' : (('int', 'int'), 'int'), - 'int_mul' : (('int', 'int'), 'int'), - 'int_floordiv' : (('int', 'int'), 'int'), - 'int_mod' : (('int', 'int'), 'int'), - 'int_and' : (('int', 'int'), 'int'), - 'int_or' : (('int', 'int'), 'int'), - 'int_xor' : (('int', 'int'), 'int'), - 'int_lshift' : (('int', 'int'), 'int'), - 'int_rshift' : (('int', 'int'), 'int'), - 'int_lt' : (('int', 'int'), 'bool'), - 'int_gt' : (('int', 'int'), 'bool'), - 'int_ge' : (('int', 'int'), 'bool'), - 'int_le' : (('int', 'int'), 'bool'), - 'int_eq' : (('int', 'int'), 'bool'), - 'int_ne' : (('int', 'int'), 'bool'), - 'int_is_true' : (('int',), 'bool'), - 'int_is_zero' : (('int',), 'bool'), - 'int_neg' : (('int',), 'int'), - 'int_invert' : (('int',), 'int'), - 'int_add_ovf' : (('int', 'int'), 'int'), - 'int_sub_ovf' : (('int', 'int'), 'int'), - 'int_mul_ovf' : (('int', 'int'), 'int'), - 'int_force_ge_zero':(('int',), 'int'), - 'uint_add' : (('int', 'int'), 'int'), - 'uint_sub' : (('int', 'int'), 'int'), - 'uint_mul' : (('int', 'int'), 'int'), - 'uint_lt' : (('int', 'int'), 'bool'), - 'uint_le' : (('int', 'int'), 'bool'), - 'uint_eq' : (('int', 'int'), 'bool'), - 'uint_ne' : (('int', 'int'), 'bool'), - 'uint_gt' : (('int', 'int'), 'bool'), - 'uint_ge' : (('int', 'int'), 'bool'), - 'uint_xor' : (('int', 'int'), 'int'), - 'uint_rshift' : (('int', 'int'), 'int'), - 'uint_floordiv' : (('int', 'int'), 'int'), - 'float_add' : (('float', 'float'), 'float'), - 'float_sub' : (('float', 'float'), 'float'), - 'float_mul' : (('float', 'float'), 'float'), - 'float_truediv' : (('float', 'float'), 'float'), - 'float_lt' : (('float', 'float'), 'bool'), - 'float_le' : (('float', 'float'), 'bool'), - 'float_eq' : (('float', 'float'), 'bool'), - 'float_ne' : (('float', 'float'), 'bool'), - 'float_gt' : (('float', 'float'), 'bool'), - 'float_ge' : (('float', 'float'), 'bool'), - 'float_neg' : (('float',), 'float'), - 'float_abs' : (('float',), 'float'), - 'cast_float_to_int':(('float',), 'int'), - 'cast_int_to_float':(('int',), 'float'), - 'same_as' : (('int',), 'int'), # could also be ptr=>ptr - 'new_with_vtable' : (('ref',), 'ref'), - 'new' : ((), 'ref'), - 'new_array' : (('int',), 'ref'), - 'oois' : (('ref', 'ref'), 'bool'), - 'ooisnot' : (('ref', 'ref'), 'bool'), - 'instanceof' : (('ref',), 'bool'), - 'subclassof' : (('ref', 'ref'), 'bool'), - 'runtimenew' : (('ref',), 'ref'), - 'setfield_gc' : (('ref', 'intorptr'), None), - 'getfield_gc' : (('ref',), 'intorptr'), - 'getfield_gc_pure': (('ref',), 'intorptr'), - 'setfield_raw' : (('ref', 'intorptr'), None), - 'getfield_raw' : (('ref',), 'intorptr'), - 'getfield_raw_pure': (('ref',), 'intorptr'), - 'setarrayitem_gc' : (('ref', 'int', 'intorptr'), None), - 'getarrayitem_gc' : (('ref', 'int'), 'intorptr'), - 'getarrayitem_gc_pure' : (('ref', 'int'), 'intorptr'), - 'setarrayitem_raw' : (('ref', 'int', 'intorptr'), None), - 'getarrayitem_raw' : (('ref', 'int'), 'intorptr'), - 'getarrayitem_raw_pure' : (('ref', 'int'), 'intorptr'), - 'arraylen_gc' : (('ref',), 'int'), - 'call' : (('ref', 'varargs'), 'intorptr'), - 'call_assembler' : (('varargs',), 'intorptr'), - 'cond_call_gc_wb' : (('ptr', 'ptr'), None), - 'cond_call_gc_wb_array': (('ptr', 'int', 'ptr'), None), - 'oosend' : (('varargs',), 'intorptr'), - 'oosend_pure' : (('varargs',), 'intorptr'), - 'guard_true' : (('bool',), None), - 'guard_false' : (('bool',), None), - 'guard_value' : (('int', 'int'), None), - 'guard_class' : (('ref', 'ref'), None), - 'guard_no_exception' : ((), None), - 'guard_exception' : (('ref',), 'ref'), - 'guard_no_overflow' : ((), None), - 'guard_overflow' : ((), None), - 'guard_nonnull' : (('ref',), None), - 'guard_isnull' : (('ref',), None), - 'guard_nonnull_class' : (('ref', 'ref'), None), - 'newstr' : (('int',), 'ref'), - 'strlen' : (('ref',), 'int'), - 'strgetitem' : (('ref', 'int'), 'int'), - 'strsetitem' : (('ref', 'int', 'int'), None), - 'newunicode' : (('int',), 'ref'), - 'unicodelen' : (('ref',), 'int'), - 'unicodegetitem' : (('ref', 'int'), 'int'), - 'unicodesetitem' : (('ref', 'int', 'int'), 'int'), - 'cast_ptr_to_int' : (('ref',), 'int'), - 'cast_int_to_ptr' : (('int',), 'ref'), - 'debug_merge_point': (('ref', 'int', 'int'), None), - 'force_token' : ((), 'int'), - 'call_may_force' : (('int', 'varargs'), 'intorptr'), - 'guard_not_forced': ((), None), -} - -# ____________________________________________________________ - -class CompiledLoop(object): - has_been_freed = False - invalid = False - - def __init__(self): - self.inputargs = [] - self.operations = [] - - def getargtypes(self): - return [v.concretetype for v in self.inputargs] - - def __repr__(self): - lines = [] - self.as_text(lines, 1) - return 'CompiledLoop %s:\n%s' % (self.inputargs, '\n'.join(lines)) - - def as_text(self, lines, indent): - for op in self.operations: - lines.append('\t'*indent + repr(op)) - -class Operation(object): - result = None - descr = None - jump_target = None - fail_args = None - - def __init__(self, opnum): - self.opnum = opnum - self.args = [] - - def __repr__(self): - if self.result is not None: - sres = repr0(self.result) + ' = ' - else: - sres = '' - return '{%s%s(%s)}' % (sres, self.getopname(), - ', '.join(map(repr0, self.args))) - - def getopname(self): - try: - return resoperation.opname[self.opnum] - except KeyError: - return '<%d>' % self.opnum - - def is_guard(self): - return rop._GUARD_FIRST <= self.opnum <= rop._GUARD_LAST - - def is_final(self): - return rop._FINAL_FIRST <= self.opnum <= rop._FINAL_LAST - -def repr0(x): - if isinstance(x, list): - return '[' + ', '.join(repr0(y) for y in x) + ']' - elif isinstance(x, Constant): - return '(' + repr0(x.value) + ')' - elif isinstance(x, lltype._ptr): - x = llmemory.cast_ptr_to_adr(x) - if x.ptr: - try: - container = x.ptr._obj._normalizedcontainer() - return '* %s' % (container._TYPE._short_name(),) - except AttributeError: - return repr(x) - else: - return 'NULL' - else: - return repr(x) - -def repr_list(lst, types): - res_l = [] - if types and types[-1] == 'varargs': - types = types[:-1] + ('int',) * (len(lst) - len(types) + 1) - assert len(types) == len(lst) - for elem, tp in zip(lst, types): - if isinstance(elem, Constant): - res_l.append('(%s)' % repr1(elem, tp)) - else: - res_l.append(repr1(elem, tp)) - return '[%s]' % (', '.join(res_l)) - -def repr1(x, tp): - if tp == "intorptr": - TYPE = lltype.typeOf(x) - if isinstance(TYPE, lltype.Ptr) and TYPE.TO._gckind == 'gc': - tp = "ref" - else: - tp = "int" - if tp == 'int': - return str(x) - elif tp == 'void': - return '---' - elif tp == 'ref': - if not x: - return '(* None)' - if isinstance(x, int): - # XXX normalize? - ptr = str(llmemory.cast_int_to_adr(x)) - elif isinstance(ootype.typeOf(x), ootype.OOType): - return repr(x) - else: - if getattr(x, '_fake', None): - return repr(x) - if lltype.typeOf(x) == llmemory.GCREF: - TP = lltype.Ptr(lltype.typeOf(x._obj.container)) - ptr = lltype.cast_opaque_ptr(TP, x) - else: - ptr = x - try: - container = ptr._obj._normalizedcontainer() - return '(* %s)' % (container._TYPE._short_name(),) - except AttributeError: - return '(%r)' % (ptr,) - elif tp == 'bool': - assert x == 0 or x == 1 - return str(bool(x)) - #elif tp == 'fieldname': - # return str(symbolic.TokenToField[x...][1]) - elif tp == 'float': - return str(x) - else: - raise NotImplementedError("tp = %s" % tp) - -_variables = [] - -def compile_start(): - del _variables[:] - return _to_opaque(CompiledLoop()) - -def mark_as_free(loop): - loop = _from_opaque(loop) - assert not loop.has_been_freed - loop.has_been_freed = True - -def compile_start_int_var(loop): - return compile_start_ref_var(loop, lltype.Signed) - -def compile_start_float_var(loop): - return compile_start_ref_var(loop, longlong.FLOATSTORAGE) - -def compile_start_ref_var(loop, TYPE): - loop = _from_opaque(loop) - assert not loop.operations - v = Variable() - v.concretetype = TYPE - loop.inputargs.append(v) - r = len(_variables) - _variables.append(v) - return r - -def compile_started_vars(clt): - if not hasattr(clt, '_debug_argtypes'): # only when compiling the loop - argtypes = [v.concretetype for v in _variables] - try: - clt._debug_argtypes = argtypes - except AttributeError: # when 'clt' is actually a translated - pass # GcStruct - -def compile_add(loop, opnum): - loop = _from_opaque(loop) - loop.operations.append(Operation(opnum)) - -def compile_add_descr(loop, ofs, type, arg_types, extrainfo, width): - from rpython.jit.backend.llgraph.runner import Descr - loop = _from_opaque(loop) - op = loop.operations[-1] - assert isinstance(type, str) and len(type) == 1 - op.descr = Descr(ofs, type, arg_types=arg_types, extrainfo=extrainfo, width=width) - -def compile_add_descr_arg(loop, ofs, type, arg_types): - from rpython.jit.backend.llgraph.runner import Descr - loop = _from_opaque(loop) - op = loop.operations[-1] - assert isinstance(type, str) and len(type) == 1 - op.args.append(Descr(ofs, type, arg_types=arg_types)) - -def compile_add_loop_token(loop, descr): - if we_are_translated(): - raise ValueError("CALL_ASSEMBLER not supported") - loop = _from_opaque(loop) - op = loop.operations[-1] - op.descr = weakref.ref(descr) - -TARGET_TOKENS = weakref.WeakKeyDictionary() - -def compile_add_target_token(loop, descr, clt): - # here, 'clt' is the compiled_loop_token of the original loop that - # we are compiling - loop = _from_opaque(loop) - op = loop.operations[-1] - descrobj = _normalize(descr) - TARGET_TOKENS[descrobj] = loop, len(loop.operations), op.args, clt - -def compile_add_var(loop, intvar): - loop = _from_opaque(loop) - op = loop.operations[-1] - op.args.append(_variables[intvar]) - -def compile_add_int_const(loop, value): - compile_add_ref_const(loop, value, lltype.Signed) - -def compile_add_float_const(loop, value): - compile_add_ref_const(loop, value, longlong.FLOATSTORAGE) - -def compile_add_ref_const(loop, value, TYPE): - loop = _from_opaque(loop) - const = Constant(value) - const.concretetype = TYPE - op = loop.operations[-1] - op.args.append(const) - -def compile_add_int_result(loop): - return compile_add_ref_result(loop, lltype.Signed) - -def compile_add_float_result(loop): - return compile_add_ref_result(loop, longlong.FLOATSTORAGE) - -def compile_add_ref_result(loop, TYPE): - loop = _from_opaque(loop) - v = Variable() - v.concretetype = TYPE - op = loop.operations[-1] - op.result = v - r = len(_variables) - _variables.append(v) - return r - -def compile_add_jump_target(loop, targettoken, source_clt): - loop = _from_opaque(loop) - descrobj = _normalize(targettoken) - (loop_target, target_opindex, target_inputargs, target_clt - ) = TARGET_TOKENS[descrobj] - # - try: - assert source_clt._debug_argtypes == target_clt._debug_argtypes - except AttributeError: # when translated - pass - # - op = loop.operations[-1] - op.jump_target = loop_target - op.jump_target_opindex = target_opindex - op.jump_target_inputargs = target_inputargs - assert op.opnum == rop.JUMP - assert [v.concretetype for v in op.args] == ( - [v.concretetype for v in target_inputargs]) - # - if loop_target == loop: - log.info("compiling new loop") - else: - log.info("compiling new bridge") - -def compile_add_guard_jump_target(loop, loop_target): - loop = _from_opaque(loop) - loop_target = _from_opaque(loop_target) - op = loop.operations[-1] - assert op.is_guard() - op.jump_target = loop_target - -def compile_add_fail(loop, fail_index): - loop = _from_opaque(loop) - index = len(loop.operations)-1 - op = loop.operations[index] - op.fail_index = fail_index - return index - -def compile_add_fail_arg(loop, intvar): - loop = _from_opaque(loop) - op = loop.operations[-1] - if op.fail_args is None: - op.fail_args = [] - if intvar == -1: - op.fail_args.append(None) - else: - op.fail_args.append(_variables[intvar]) - -def compile_redirect_fail(old_loop, old_index, new_loop): - old_loop = _from_opaque(old_loop) - new_loop = _from_opaque(new_loop) - guard_op = old_loop.operations[old_index] - assert guard_op.is_guard() - guard_op.jump_target = new_loop - # check that the bridge's inputargs are of the correct number and - # kind for the guard - if guard_op.fail_args is not None: - argkinds = [v.concretetype for v in guard_op.fail_args if v] - else: - argkinds = [] - assert argkinds == [v.concretetype for v in new_loop.inputargs] - -# ------------------------------ - -class Frame(object): - OPHANDLERS = [None] * (rop._LAST+1) - - def __init__(self, cpu): - self.verbose = False - self.cpu = cpu - self.opindex = 1 - self._forced = False - self._may_force = -1 - - def getenv(self, v): - from rpython.jit.backend.llgraph.runner import Descr - if isinstance(v, Constant): - return v.value - elif isinstance(v, Descr): - return v - else: - return self.env[v] - - def _populate_fail_args(self, op, skip=None): - fail_args = [] - if op.fail_args: - for fail_arg in op.fail_args: - if fail_arg is None: - fail_args.append(None) - elif fail_arg is skip: - fail_args.append(fail_arg.concretetype._defl()) - else: - fail_args.append(self.getenv(fail_arg)) - self.fail_args = fail_args - self.fail_index = op.fail_index - - def execute(self): - """Execute all operations in a loop, - possibly following to other loops as well. - """ - global _last_exception - assert _last_exception is None, "exception left behind" - verbose = True - self.opindex = 0 - while True: - assert not self.loop.has_been_freed - op = self.loop.operations[self.opindex] - args = [self.getenv(v) for v in op.args] - if not op.is_final(): - try: - result = self.execute_operation(op.opnum, args, op.descr, - verbose) - except GuardFailed: - assert op.is_guard() - _stats.exec_conditional_jumps += 1 - if op.jump_target is not None: - # a patched guard, pointing to further code - if op.fail_args: - args = [self.getenv(v) for v in op.fail_args if v] - else: - args = [] - assert len(op.jump_target.inputargs) == len(args) - self.env = dict(zip(op.jump_target.inputargs, args)) - self.loop = op.jump_target - self.opindex = 0 - continue - else: - self._populate_fail_args(op) - # a non-patched guard - if self.verbose: - log.trace('failed: %s' % ( - ', '.join(map(str, fail_args)),)) - return op.fail_index - #verbose = self.verbose - assert (result is None) == (op.result is None) - if op.result is not None: - RESTYPE = op.result.concretetype - if RESTYPE is lltype.Signed: - x = self.as_int(result) - elif RESTYPE is llmemory.GCREF: - x = self.as_ptr(result) - elif RESTYPE is ootype.Object: - x = self.as_object(result) - elif RESTYPE is longlong.FLOATSTORAGE: - x = self.as_floatstorage(result) - else: - raise Exception("op.result.concretetype is %r" - % (RESTYPE,)) - self.env[op.result] = x - self.opindex += 1 - continue - if op.opnum == rop.JUMP: - inputargs = op.jump_target_inputargs - assert len(inputargs) == len(args) - self.env = dict(zip(inputargs, args)) - self.loop = op.jump_target - self.opindex = op.jump_target_opindex - _stats.exec_jumps += 1 - elif op.opnum == rop.FINISH: - if self.verbose: - log.trace('finished: %s' % ( - ', '.join(map(str, args)),)) - self.fail_args = args - return op.fail_index - - else: - assert 0, "unknown final operation %d" % (op.opnum,) - - def execute_operation(self, opnum, values, descr, verbose): - """Execute a single operation. - """ - ophandler = self.OPHANDLERS[opnum] - if ophandler is None: - self._define_impl(opnum) - ophandler = self.OPHANDLERS[opnum] - assert ophandler is not None, "missing impl for op %d" % opnum - opname = resoperation.opname[opnum].lower() - exec_counters = _stats.exec_counters - exec_counters[opname] = exec_counters.get(opname, 0) + 1 - for i in range(len(values)): - if isinstance(values[i], ComputedIntSymbolic): - values[i] = values[i].compute_fn() - res = NotImplemented - try: - res = ophandler(self, descr, *values) - finally: - if 0: # if verbose: - argtypes, restype = TYPES[opname] - if res is None: - resdata = '' - elif res is NotImplemented: - resdata = '*fail*' - else: - resdata = '-> ' + repr1(res, restype) - # fish the types - log.cpu('\t%s %s %s' % (opname, repr_list(values, argtypes), - resdata)) - return res - - def as_int(self, x): - return cast_to_int(x) - - def as_ptr(self, x): - return cast_to_ptr(x) - - def as_object(self, x): - return ootype.cast_to_object(x) - - def as_floatstorage(self, x): - return cast_to_floatstorage(x) - - def log_progress(self): - count = sum(_stats.exec_counters.values()) - count_jumps = _stats.exec_jumps - log.trace('ran %d operations, %d jumps' % (count, count_jumps)) - - # ---------- - - @classmethod - def _define_impl(cls, opnum): - opname = resoperation.opname[opnum] - try: - op = getattr(cls, 'op_' + opname.lower()) # op_guard_true etc. - except AttributeError: - try: - impl = globals()['do_' + opname.lower()] # do_arraylen_gc etc. - def op(self, descr, *args): - if descr is None: - return impl(*args) - else: - return impl(descr, *args) - except KeyError: - op = cls._make_impl_from_blackhole_interp(opname) - cls.OPHANDLERS[opnum] = op - - @classmethod - def _make_impl_from_blackhole_interp(cls, opname): - from rpython.jit.metainterp.blackhole import BlackholeInterpreter - name = 'bhimpl_' + opname.lower() - func = BlackholeInterpreter.__dict__[name] - for argtype in func.argtypes: - assert argtype in ('i', 'r', 'f') - # - def _op_default_implementation(self, descr, *args): - # for all operations implemented in the blackhole interpreter - return func(*args) - # - return _op_default_implementation - - def op_label(self, _, *args): - op = self.loop.operations[self.opindex] - assert op.opnum == rop.LABEL - assert len(op.args) == len(args) - newenv = {} - for v, value in zip(op.args, args): - newenv[v] = value - self.env = newenv - - def op_debug_merge_point(self, _, *args): - from rpython.jit.metainterp.warmspot import get_stats - try: - stats = get_stats() - except AttributeError: - pass - else: - stats.add_merge_point_location(args[1:]) - pass - - def op_guard_true(self, _, value): - if not value: - raise GuardFailed - - def op_guard_false(self, _, value): - if value: - raise GuardFailed - - op_guard_nonnull = op_guard_true - op_guard_isnull = op_guard_false - - def op_guard_class(self, _, value, expected_class): - value = lltype.cast_opaque_ptr(rclass.OBJECTPTR, value) - expected_class = llmemory.cast_adr_to_ptr( - llmemory.cast_int_to_adr(expected_class), - rclass.CLASSTYPE) - if value.typeptr != expected_class: - raise GuardFailed - - def op_guard_nonnull_class(self, _, value, expected_class): - if not value: - raise GuardFailed - self.op_guard_class(_, value, expected_class) - - def op_guard_value(self, _, value, expected_value): - if value != expected_value: - raise GuardFailed - - def op_guard_no_exception(self, _): - if _last_exception: - raise GuardFailed - - def _check_exception(self, expected_exception): - global _last_exception - expected_exception = self._cast_exception(expected_exception) - assert expected_exception - exc = _last_exception - if exc: - got = exc.args[0] - # exact match! - if got != expected_exception: - return False - return True - else: - return False - - def _cast_exception(self, exception): - return llmemory.cast_adr_to_ptr( - llmemory.cast_int_to_adr(exception), - rclass.CLASSTYPE) - - def _issubclass(self, cls1, cls2): - return rclass.ll_issubclass(cls1, cls2) - - def op_guard_exception(self, _, expected_exception): - global _last_exception - if not self._check_exception(expected_exception): - raise GuardFailed - res = _last_exception[1] - _last_exception = None - return res - - def op_guard_no_overflow(self, _): - flag = self.overflow_flag - del self.overflow_flag - if flag: - raise GuardFailed - - def op_guard_overflow(self, _): - flag = self.overflow_flag - del self.overflow_flag - if not flag: - raise GuardFailed - - def op_int_add_ovf(self, _, x, y): - try: - z = ovfcheck(x + y) - except OverflowError: - ovf = True - z = 0 - else: - ovf = False - self.overflow_flag = ovf - return z - - def op_int_sub_ovf(self, _, x, y): - try: - z = ovfcheck(x - y) - except OverflowError: - ovf = True - z = 0 - else: - ovf = False - self.overflow_flag = ovf - return z - - def op_int_mul_ovf(self, _, x, y): - try: - z = ovfcheck(x * y) - except OverflowError: - ovf = True - z = 0 - else: - ovf = False - self.overflow_flag = ovf - return z - - def op_keepalive(self, _, x): - pass - - # ---------- - # delegating to the builtins do_xxx() (done automatically for simple cases) - - def op_getarrayitem_gc(self, arraydescr, array, index): - if arraydescr.typeinfo == REF: - return do_getarrayitem_gc_ptr(array, index) - elif arraydescr.typeinfo == INT: - return do_getarrayitem_gc_int(array, index) - elif arraydescr.typeinfo == FLOAT: - return do_getarrayitem_gc_float(array, index) - else: - raise NotImplementedError - - op_getarrayitem_gc_pure = op_getarrayitem_gc - - def op_getarrayitem_raw(self, arraydescr, array, index): - if arraydescr.typeinfo == REF: - raise NotImplementedError("getarrayitem_raw -> gcref") - elif arraydescr.typeinfo == INT: - return do_getarrayitem_raw_int(array, index, arraydescr.ofs) - elif arraydescr.typeinfo == FLOAT: - return do_getarrayitem_raw_float(array, index) - else: - raise NotImplementedError - - op_getarrayitem_raw_pure = op_getarrayitem_raw - - def op_getfield_gc(self, fielddescr, struct): - if fielddescr.typeinfo == REF: - return do_getfield_gc_ptr(struct, fielddescr.ofs) - elif fielddescr.typeinfo == INT: - return do_getfield_gc_int(struct, fielddescr.ofs) - elif fielddescr.typeinfo == FLOAT: - return do_getfield_gc_float(struct, fielddescr.ofs) - else: - raise NotImplementedError - - op_getfield_gc_pure = op_getfield_gc - - def op_getfield_raw(self, fielddescr, struct): - if fielddescr.typeinfo == REF: - return do_getfield_raw_ptr(struct, fielddescr.ofs) - elif fielddescr.typeinfo == INT: - return do_getfield_raw_int(struct, fielddescr.ofs) - elif fielddescr.typeinfo == FLOAT: - return do_getfield_raw_float(struct, fielddescr.ofs) - else: - raise NotImplementedError - - op_getfield_raw_pure = op_getfield_raw - - def op_raw_store(self, arraydescr, addr, offset, value): - if arraydescr.typeinfo == REF: - raise AssertionError("cannot store GC pointer in raw storage") - elif arraydescr.typeinfo == INT: - do_raw_store_int(addr, offset, arraydescr.ofs, value) - elif arraydescr.typeinfo == FLOAT: - do_raw_store_float(addr, offset, value) - else: - raise NotImplementedError - - def op_raw_load(self, arraydescr, addr, offset): - if arraydescr.typeinfo == REF: - raise AssertionError("cannot store GC pointer in raw storage") - elif arraydescr.typeinfo == INT: - return do_raw_load_int(addr, offset, arraydescr.ofs) - elif arraydescr.typeinfo == FLOAT: - return do_raw_load_float(addr, offset) - else: - raise NotImplementedError - - def op_new(self, size): - return do_new(size.ofs) - - def op_new_with_vtable(self, descr, vtable): - assert descr is None - descr = heaptracker.vtable2descr(self.cpu, vtable) - result = do_new(descr.ofs) - value = lltype.cast_opaque_ptr(rclass.OBJECTPTR, result) - value.typeptr = cast_from_int(rclass.CLASSTYPE, vtable) - return result - - def op_setarrayitem_gc(self, arraydescr, array, index, newvalue): - if arraydescr.typeinfo == REF: - do_setarrayitem_gc_ptr(array, index, newvalue) - elif arraydescr.typeinfo == INT: - do_setarrayitem_gc_int(array, index, newvalue) - elif arraydescr.typeinfo == FLOAT: - do_setarrayitem_gc_float(array, index, newvalue) - else: - raise NotImplementedError - - def op_setarrayitem_raw(self, arraydescr, array, index, newvalue): - if arraydescr.typeinfo == REF: - raise NotImplementedError("setarrayitem_raw <- gcref") - elif arraydescr.typeinfo == INT: - do_setarrayitem_raw_int(array, index, newvalue, arraydescr.ofs) - elif arraydescr.typeinfo == FLOAT: - do_setarrayitem_raw_float(array, index, newvalue) - else: - raise NotImplementedError - - def op_getinteriorfield_gc(self, descr, array, index): - if descr.typeinfo == REF: - return do_getinteriorfield_gc_ptr(array, index, descr.ofs) - elif descr.typeinfo == INT: - return do_getinteriorfield_gc_int(array, index, descr.ofs) - elif descr.typeinfo == FLOAT: - return do_getinteriorfield_gc_float(array, index, descr.ofs) - else: - raise NotImplementedError - - def op_getinteriorfield_raw(self, descr, array, index): - if descr.typeinfo == REF: - return do_getinteriorfield_raw_ptr(array, index, descr.width, descr.ofs) - elif descr.typeinfo == INT: - return do_getinteriorfield_raw_int(array, index, descr.width, descr.ofs) - elif descr.typeinfo == FLOAT: - return do_getinteriorfield_raw_float(array, index, descr.width, descr.ofs) - else: - raise NotImplementedError - - def op_setinteriorfield_gc(self, descr, array, index, newvalue): - if descr.typeinfo == REF: - return do_setinteriorfield_gc_ptr(array, index, descr.ofs, - newvalue) - elif descr.typeinfo == INT: - return do_setinteriorfield_gc_int(array, index, descr.ofs, - newvalue) - elif descr.typeinfo == FLOAT: - return do_setinteriorfield_gc_float(array, index, descr.ofs, - newvalue) - else: - raise NotImplementedError - - def op_setinteriorfield_raw(self, descr, array, index, newvalue): - if descr.typeinfo == REF: - return do_setinteriorfield_raw_ptr(array, index, newvalue, descr.width, descr.ofs) - elif descr.typeinfo == INT: - return do_setinteriorfield_raw_int(array, index, newvalue, descr.width, descr.ofs) - elif descr.typeinfo == FLOAT: - return do_setinteriorfield_raw_float(array, index, newvalue, descr.width, descr.ofs) - else: - raise NotImplementedError - - def op_setfield_gc(self, fielddescr, struct, newvalue): - if fielddescr.typeinfo == REF: - do_setfield_gc_ptr(struct, fielddescr.ofs, newvalue) - elif fielddescr.typeinfo == INT: - do_setfield_gc_int(struct, fielddescr.ofs, newvalue) - elif fielddescr.typeinfo == FLOAT: - do_setfield_gc_float(struct, fielddescr.ofs, newvalue) - else: - raise NotImplementedError - - def op_setfield_raw(self, fielddescr, struct, newvalue): - if fielddescr.typeinfo == REF: - do_setfield_raw_ptr(struct, fielddescr.ofs, newvalue) - elif fielddescr.typeinfo == INT: - do_setfield_raw_int(struct, fielddescr.ofs, newvalue) - elif fielddescr.typeinfo == FLOAT: - do_setfield_raw_float(struct, fielddescr.ofs, newvalue) - else: - raise NotImplementedError - - def op_call(self, calldescr, func, *args): - effectinfo = calldescr.get_extra_info() - if effectinfo is not None and hasattr(effectinfo, 'oopspecindex'): - oopspecindex = effectinfo.oopspecindex - if oopspecindex == EffectInfo.OS_MATH_SQRT: - return do_math_sqrt(args[0]) - return self._do_call(calldescr, func, args, call_with_llptr=False) - - def op_call_release_gil(self, calldescr, func, *args): - return self._do_call(calldescr, func, args, call_with_llptr=True) - - def _do_call(self, calldescr, func, args, call_with_llptr): - global _last_exception - assert _last_exception is None, "exception left behind" - assert _call_args_i == _call_args_r == _call_args_f == [] - args_in_order = [] - for x in args: - T = lltype.typeOf(x) - if T is lltype.Signed: - args_in_order.append('i') - _call_args_i.append(x) - elif T == llmemory.GCREF: - args_in_order.append('r') - _call_args_r.append(x) - elif T is longlong.FLOATSTORAGE: - args_in_order.append('f') - _call_args_f.append(x) - else: - raise TypeError(x) - try: - return _do_call_common(func, args_in_order, calldescr, - call_with_llptr) - except LLException, lle: - _last_exception = lle - d = {'v': None, - REF: lltype.nullptr(llmemory.GCREF.TO), - INT: 0, - FLOAT: 0.0} - return d[calldescr.typeinfo] - - def op_cond_call_gc_wb(self, descr, a, b): - py.test.skip("cond_call_gc_wb not supported") - - def op_cond_call_gc_wb_array(self, descr, a, b, c): - py.test.skip("cond_call_gc_wb_array not supported") - - def op_oosend(self, descr, obj, *args): - raise NotImplementedError("oosend for lltype backend??") - - op_oosend_pure = op_oosend - - def op_new_array(self, arraydescr, count): - return do_new_array(arraydescr.ofs, count) - - def op_force_token(self, descr): - opaque_frame = _to_opaque(self) - return llmemory.cast_ptr_to_adr(opaque_frame) - - def op_read_timestamp(self, descr): - return read_timestamp() - - def op_call_may_force(self, calldescr, func, *args): - assert not self._forced - self._may_force = self.opindex - try: - return self.op_call(calldescr, func, *args) - finally: - self._may_force = -1 - - def op_call_assembler(self, wref_loop_token, *args): - if we_are_translated(): - raise ValueError("CALL_ASSEMBLER not supported") - return self._do_call_assembler(wref_loop_token, *args) - - def _do_call_assembler(self, wref_loop_token, *args): - global _last_exception - loop_token = wref_loop_token() - assert loop_token, "CALL_ASSEMBLER to a target that already died" - ctl = loop_token.compiled_loop_token - if hasattr(ctl, 'redirected'): - return self._do_call_assembler(ctl.redirected, *args) - assert not self._forced - self._may_force = self.opindex - try: - inpargs = _from_opaque(ctl.compiled_version).inputargs - assert len(inpargs) == len(args) - for i, inparg in enumerate(inpargs): - TYPE = inparg.concretetype - if TYPE is lltype.Signed: - set_future_value_int(i, args[i]) - elif isinstance(TYPE, lltype.Ptr): - set_future_value_ref(i, args[i]) - elif TYPE is longlong.FLOATSTORAGE: - set_future_value_float(i, args[i]) - else: - raise Exception("Nonsense type %s" % TYPE) - - failindex = self.cpu._execute_token(loop_token) - jd = loop_token.outermost_jitdriver_sd - assert jd is not None, ("call_assembler(): the loop_token needs " - "to have 'outermost_jitdriver_sd'") - if jd.index_of_virtualizable != -1: - vable = args[jd.index_of_virtualizable] - else: - vable = lltype.nullptr(llmemory.GCREF.TO) - # - # Emulate the fast path - if failindex == self.cpu.done_with_this_frame_int_v: - reset_vable(jd, vable) - return self.cpu.get_latest_value_int(0) - if failindex == self.cpu.done_with_this_frame_ref_v: - reset_vable(jd, vable) - return self.cpu.get_latest_value_ref(0) - if failindex == self.cpu.done_with_this_frame_float_v: - reset_vable(jd, vable) - return self.cpu.get_latest_value_float(0) - if failindex == self.cpu.done_with_this_frame_void_v: - reset_vable(jd, vable) - return None - # - assembler_helper_ptr = jd.assembler_helper_adr.ptr # fish - try: - return assembler_helper_ptr(failindex, vable) - except LLException, lle: - assert _last_exception is None, "exception left behind" - _last_exception = lle - # fish op - op = self.loop.operations[self.opindex] - if op.result is not None: - return 0 - finally: - self._may_force = -1 - - def op_guard_not_forced(self, descr): - forced = self._forced - self._forced = False - if forced: - raise GuardFailed - - def op_guard_not_invalidated(self, descr): - if self.loop.invalid: - raise GuardFailed - -class OOFrame(Frame): - - OPHANDLERS = [None] * (rop._LAST+1) - - def op_new_with_vtable(self, descr, vtable): - assert descr is None - typedescr = get_class_size(self.memocast, vtable) - return ootype.cast_to_object(ootype.new(typedescr.TYPE)) - - def op_new_array(self, typedescr, count): - res = ootype.oonewarray(typedescr.ARRAY, count) - return ootype.cast_to_object(res) - - def op_getfield_gc(self, fielddescr, obj): - TYPE = fielddescr.TYPE - fieldname = fielddescr.fieldname - _, T = TYPE._lookup_field(fieldname) - obj = ootype.cast_from_object(TYPE, obj) - res = getattr(obj, fieldname) - if isinstance(T, ootype.OOType): - return ootype.cast_to_object(res) - return res - - op_getfield_gc_pure = op_getfield_gc - - def op_setfield_gc(self, fielddescr, obj, newvalue): - TYPE = fielddescr.TYPE - fieldname = fielddescr.fieldname - _, T = TYPE._lookup_field(fieldname) - obj = ootype.cast_from_object(TYPE, obj) - if isinstance(ootype.typeOf(newvalue), ootype.OOType): - newvalue = ootype.cast_from_object(T, newvalue) - elif isinstance(T, lltype.Primitive): - newvalue = lltype.cast_primitive(T, newvalue) - setattr(obj, fieldname, newvalue) - - def op_getarrayitem_gc(self, typedescr, obj, index): - array = ootype.cast_from_object(typedescr.ARRAY, obj) - res = array.ll_getitem_fast(index) - if isinstance(typedescr.TYPE, ootype.OOType): - return ootype.cast_to_object(res) - return res - - op_getarrayitem_gc_pure = op_getarrayitem_gc - - def op_setarrayitem_gc(self, typedescr, obj, index, objnewvalue): - array = ootype.cast_from_object(typedescr.ARRAY, obj) - if ootype.typeOf(objnewvalue) == ootype.Object: - newvalue = ootype.cast_from_object(typedescr.TYPE, objnewvalue) - else: - newvalue = objnewvalue - array.ll_setitem_fast(index, newvalue) - - def op_arraylen_gc(self, typedescr, obj): - array = ootype.cast_from_object(typedescr.ARRAY, obj) From noreply at buildbot.pypy.org Sun Jan 27 21:34:45 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 27 Jan 2013 21:34:45 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: infinite loops aren't cool Message-ID: <20130127203445.993F31C039A@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60545:3196abf1af75 Date: 2013-01-27 22:34 +0200 http://bitbucket.org/pypy/pypy/changeset/3196abf1af75/ Log: infinite loops aren't cool diff --git a/rpython/jit/backend/x86/test/test_zrpy_gc.py b/rpython/jit/backend/x86/test/test_zrpy_gc.py --- a/rpython/jit/backend/x86/test/test_zrpy_gc.py +++ b/rpython/jit/backend/x86/test/test_zrpy_gc.py @@ -803,6 +803,7 @@ s = lltype.malloc(S) s.s = prev_s prev_s = s + i += 1 return n - 1, x, x0, x1, x2, x3, x4, x5, x6, x7, l, s0 return None, f, None From noreply at buildbot.pypy.org Sun Jan 27 21:37:47 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Sun, 27 Jan 2013 21:37:47 +0100 (CET) Subject: [pypy-commit] pypy fix-version-tool: make rpython/tool/version.py usable generically for any repository root Message-ID: <20130127203747.F1F6B1C039A@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: fix-version-tool Changeset: r60546:305ed2a35119 Date: 2013-01-27 11:27 -0500 http://bitbucket.org/pypy/pypy/changeset/305ed2a35119/ Log: make rpython/tool/version.py usable generically for any repository root diff --git a/rpython/tool/test/test_version.py b/rpython/tool/test/test_version.py --- a/rpython/tool/test/test_version.py +++ b/rpython/tool/test/test_version.py @@ -11,11 +11,11 @@ assert version_for('release', tag='release-123', node='000', - ) == ('RPython', 'release-123', '000') + ) == ('release-123', '000') assert version_for('somebranch', node='000', branch='something', - ) == ('RPython', 'something', '000') + ) == ('something', '000') def test_get_repo_version_info(): diff --git a/rpython/tool/version.py b/rpython/tool/version.py --- a/rpython/tool/version.py +++ b/rpython/tool/version.py @@ -4,7 +4,7 @@ import rpython rpythondir = os.path.dirname(os.path.abspath(rpython.__file__)) rpythonroot = os.path.dirname(rpythondir) -default_retval = 'RPython', '?', '?' +default_retval = '?', '?' def maywarn(err, repo_type='Mercurial'): if not err: @@ -15,31 +15,36 @@ py.log.setconsumer("version", ansi_log) log.WARNING('Errors getting %s information: %s' % (repo_type, err)) -def get_repo_version_info(hgexe=None): +def get_repo_version_info(hgexe=None, root=rpythonroot): '''Obtain version information by invoking the 'hg' or 'git' commands.''' + if root == rpythonroot: + project = ("RPython",) + else: + project = ('?',) + # Try to see if we can get info from Git if hgexe is not specified. if not hgexe: - if os.path.isdir(os.path.join(rpythonroot, '.git')): - return _get_git_version() + if os.path.isdir(os.path.join(root, '.git')): + return project + _get_git_version(root) # Fallback to trying Mercurial. if hgexe is None: hgexe = py.path.local.sysfind('hg') - if os.path.isfile(os.path.join(rpythonroot, '.hg_archival.txt')): - return _get_hg_archive_version(os.path.join(rpythonroot, '.hg_archival.txt')) - elif not os.path.isdir(os.path.join(rpythonroot, '.hg')): + if os.path.isfile(os.path.join(root, '.hg_archival.txt')): + return project + _get_hg_archive_version(os.path.join(root, '.hg_archival.txt')) + elif not os.path.isdir(os.path.join(root, '.hg')): maywarn('Not running from a Mercurial repository!') - return default_retval + return project + default_retval elif not hgexe: maywarn('Cannot find Mercurial command!') - return default_retval + return project + default_retval else: - return _get_hg_version(hgexe) + return project + _get_hg_version(hgexe, root) -def _get_hg_version(hgexe): +def _get_hg_version(hgexe, root): env = dict(os.environ) # get Mercurial into scripting mode env['HGPLAIN'] = '1' @@ -57,14 +62,14 @@ maywarn('command does not identify itself as Mercurial') return default_retval - p = Popen([str(hgexe), 'id', '-i', rpythonroot], + p = Popen([str(hgexe), 'id', '-i', root], stdout=PIPE, stderr=PIPE, env=env) hgid = p.stdout.read().strip() maywarn(p.stderr.read()) if p.wait() != 0: hgid = '?' - p = Popen([str(hgexe), 'id', '-t', rpythonroot], + p = Popen([str(hgexe), 'id', '-t', root], stdout=PIPE, stderr=PIPE, env=env) hgtags = [t for t in p.stdout.read().strip().split() if t != 'tip'] maywarn(p.stderr.read()) @@ -72,15 +77,15 @@ hgtags = ['?'] if hgtags: - return 'RPython', hgtags[0], hgid + return hgtags[0], hgid else: # use the branch instead - p = Popen([str(hgexe), 'id', '-b', rpythonroot], + p = Popen([str(hgexe), 'id', '-b', root], stdout=PIPE, stderr=PIPE, env=env) hgbranch = p.stdout.read().strip() maywarn(p.stderr.read()) - return 'RPython', hgbranch, hgid + return hgbranch, hgid def _get_hg_archive_version(path): @@ -90,12 +95,12 @@ finally: fp.close() if 'tag' in data: - return 'RPython', data['tag'], data['node'] + return data['tag'], data['node'] else: - return 'RPython', data['branch'], data['node'] + return data['branch'], data['node'] -def _get_git_version(): +def _get_git_version(root): #XXX: this function is a untested hack, # so the git mirror tav made will work gitexe = py.path.local.sysfind('git') @@ -105,7 +110,7 @@ try: p = Popen( [str(gitexe), 'rev-parse', 'HEAD'], - stdout=PIPE, stderr=PIPE, cwd=rpythonroot + stdout=PIPE, stderr=PIPE, cwd=root ) except OSError, e: maywarn(e, 'Git') @@ -116,16 +121,16 @@ revision_id = p.stdout.read().strip()[:12] p = Popen( [str(gitexe), 'describe', '--tags', '--exact-match'], - stdout=PIPE, stderr=PIPE, cwd=rpythonroot + stdout=PIPE, stderr=PIPE, cwd=root ) if p.wait() != 0: p = Popen( [str(gitexe), 'branch'], stdout=PIPE, stderr=PIPE, - cwd=rpythonroot + cwd=root ) if p.wait() != 0: maywarn(p.stderr.read(), 'Git') - return 'RPython', '?', revision_id + return '?', revision_id branch = '?' for line in p.stdout.read().strip().split('\n'): if line.startswith('* '): @@ -133,8 +138,8 @@ if branch == '(no branch)': branch = '?' break - return 'RPython', branch, revision_id - return 'RPython', p.stdout.read().strip(), revision_id + return branch, revision_id + return p.stdout.read().strip(), revision_id if __name__ == '__main__': From noreply at buildbot.pypy.org Sun Jan 27 21:37:49 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Sun, 27 Jan 2013 21:37:49 +0100 (CET) Subject: [pypy-commit] pypy fix-version-tool: have pypy get_repo_version_info on its own root (for the later split) Message-ID: <20130127203749.3CBD91C039A@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: fix-version-tool Changeset: r60547:dea6b7a85a2a Date: 2013-01-27 11:28 -0500 http://bitbucket.org/pypy/pypy/changeset/dea6b7a85a2a/ Log: have pypy get_repo_version_info on its own root (for the later split) diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -28,6 +28,7 @@ import pypy pypydir = os.path.dirname(os.path.abspath(pypy.__file__)) +pypyroot = os.path.dirname(os.path.dirname(os.path.abspath(pypy.__file__))) del pypy from rpython.tool.version import get_repo_version_info @@ -68,7 +69,7 @@ CPYTHON_VERSION[0], CPYTHON_VERSION[1], CPYTHON_VERSION[2], - get_repo_version_info()[2], + get_repo_version_info(root=pypyroot)[2], date, time, ver, @@ -91,10 +92,10 @@ return space.wrap(('PyPy', '', '')) def get_repo_info(space): - info = get_repo_version_info() + info = get_repo_version_info(root=pypyroot) if info: project, repo_tag, repo_version = info - return space.newtuple([space.wrap(project), + return space.newtuple([space.wrap('PyPy'), space.wrap(repo_tag), space.wrap(repo_version)]) else: From noreply at buildbot.pypy.org Sun Jan 27 21:37:50 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Sun, 27 Jan 2013 21:37:50 +0100 (CET) Subject: [pypy-commit] pypy fix-version-tool: oops, this was the intent Message-ID: <20130127203750.7834F1C039A@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: fix-version-tool Changeset: r60548:ae5db54ff288 Date: 2013-01-27 11:44 -0500 http://bitbucket.org/pypy/pypy/changeset/ae5db54ff288/ Log: oops, this was the intent diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -28,7 +28,7 @@ import pypy pypydir = os.path.dirname(os.path.abspath(pypy.__file__)) -pypyroot = os.path.dirname(os.path.dirname(os.path.abspath(pypy.__file__))) +pypyroot = os.path.dirname(pypydir) del pypy from rpython.tool.version import get_repo_version_info From noreply at buildbot.pypy.org Sun Jan 27 21:37:51 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Sun, 27 Jan 2013 21:37:51 +0100 (CET) Subject: [pypy-commit] pypy fix-version-tool: unused import Message-ID: <20130127203751.A8A8C1C039A@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: fix-version-tool Changeset: r60549:5480de09f372 Date: 2013-01-27 11:56 -0500 http://bitbucket.org/pypy/pypy/changeset/5480de09f372/ Log: unused import diff --git a/rpython/tool/test/test_version.py b/rpython/tool/test/test_version.py --- a/rpython/tool/test/test_version.py +++ b/rpython/tool/test/test_version.py @@ -1,5 +1,4 @@ import os, sys -import py from rpython.tool.version import get_repo_version_info, _get_hg_archive_version def test_hg_archival_version(tmpdir): From noreply at buildbot.pypy.org Sun Jan 27 21:37:52 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 27 Jan 2013 21:37:52 +0100 (CET) Subject: [pypy-commit] pypy default: Merged in bdkearns/pypy/fix-version-tool (pull request #111) Message-ID: <20130127203752.CF36A1C039A@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: Changeset: r60550:7b8a30192de5 Date: 2013-01-27 22:37 +0200 http://bitbucket.org/pypy/pypy/changeset/7b8a30192de5/ Log: Merged in bdkearns/pypy/fix-version-tool (pull request #111) make get_repo_version_info() usable generically for any repository root diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -28,6 +28,7 @@ import pypy pypydir = os.path.dirname(os.path.abspath(pypy.__file__)) +pypyroot = os.path.dirname(pypydir) del pypy from rpython.tool.version import get_repo_version_info @@ -68,7 +69,7 @@ CPYTHON_VERSION[0], CPYTHON_VERSION[1], CPYTHON_VERSION[2], - get_repo_version_info()[2], + get_repo_version_info(root=pypyroot)[2], date, time, ver, @@ -91,10 +92,10 @@ return space.wrap(('PyPy', '', '')) def get_repo_info(space): - info = get_repo_version_info() + info = get_repo_version_info(root=pypyroot) if info: project, repo_tag, repo_version = info - return space.newtuple([space.wrap(project), + return space.newtuple([space.wrap('PyPy'), space.wrap(repo_tag), space.wrap(repo_version)]) else: diff --git a/rpython/tool/test/test_version.py b/rpython/tool/test/test_version.py --- a/rpython/tool/test/test_version.py +++ b/rpython/tool/test/test_version.py @@ -1,5 +1,4 @@ import os, sys -import py from rpython.tool.version import get_repo_version_info, _get_hg_archive_version def test_hg_archival_version(tmpdir): @@ -11,11 +10,11 @@ assert version_for('release', tag='release-123', node='000', - ) == ('RPython', 'release-123', '000') + ) == ('release-123', '000') assert version_for('somebranch', node='000', branch='something', - ) == ('RPython', 'something', '000') + ) == ('something', '000') def test_get_repo_version_info(): diff --git a/rpython/tool/version.py b/rpython/tool/version.py --- a/rpython/tool/version.py +++ b/rpython/tool/version.py @@ -4,7 +4,7 @@ import rpython rpythondir = os.path.dirname(os.path.abspath(rpython.__file__)) rpythonroot = os.path.dirname(rpythondir) -default_retval = 'RPython', '?', '?' +default_retval = '?', '?' def maywarn(err, repo_type='Mercurial'): if not err: @@ -15,31 +15,36 @@ py.log.setconsumer("version", ansi_log) log.WARNING('Errors getting %s information: %s' % (repo_type, err)) -def get_repo_version_info(hgexe=None): +def get_repo_version_info(hgexe=None, root=rpythonroot): '''Obtain version information by invoking the 'hg' or 'git' commands.''' + if root == rpythonroot: + project = ("RPython",) + else: + project = ('?',) + # Try to see if we can get info from Git if hgexe is not specified. if not hgexe: - if os.path.isdir(os.path.join(rpythonroot, '.git')): - return _get_git_version() + if os.path.isdir(os.path.join(root, '.git')): + return project + _get_git_version(root) # Fallback to trying Mercurial. if hgexe is None: hgexe = py.path.local.sysfind('hg') - if os.path.isfile(os.path.join(rpythonroot, '.hg_archival.txt')): - return _get_hg_archive_version(os.path.join(rpythonroot, '.hg_archival.txt')) - elif not os.path.isdir(os.path.join(rpythonroot, '.hg')): + if os.path.isfile(os.path.join(root, '.hg_archival.txt')): + return project + _get_hg_archive_version(os.path.join(root, '.hg_archival.txt')) + elif not os.path.isdir(os.path.join(root, '.hg')): maywarn('Not running from a Mercurial repository!') - return default_retval + return project + default_retval elif not hgexe: maywarn('Cannot find Mercurial command!') - return default_retval + return project + default_retval else: - return _get_hg_version(hgexe) + return project + _get_hg_version(hgexe, root) -def _get_hg_version(hgexe): +def _get_hg_version(hgexe, root): env = dict(os.environ) # get Mercurial into scripting mode env['HGPLAIN'] = '1' @@ -57,14 +62,14 @@ maywarn('command does not identify itself as Mercurial') return default_retval - p = Popen([str(hgexe), 'id', '-i', rpythonroot], + p = Popen([str(hgexe), 'id', '-i', root], stdout=PIPE, stderr=PIPE, env=env) hgid = p.stdout.read().strip() maywarn(p.stderr.read()) if p.wait() != 0: hgid = '?' - p = Popen([str(hgexe), 'id', '-t', rpythonroot], + p = Popen([str(hgexe), 'id', '-t', root], stdout=PIPE, stderr=PIPE, env=env) hgtags = [t for t in p.stdout.read().strip().split() if t != 'tip'] maywarn(p.stderr.read()) @@ -72,15 +77,15 @@ hgtags = ['?'] if hgtags: - return 'RPython', hgtags[0], hgid + return hgtags[0], hgid else: # use the branch instead - p = Popen([str(hgexe), 'id', '-b', rpythonroot], + p = Popen([str(hgexe), 'id', '-b', root], stdout=PIPE, stderr=PIPE, env=env) hgbranch = p.stdout.read().strip() maywarn(p.stderr.read()) - return 'RPython', hgbranch, hgid + return hgbranch, hgid def _get_hg_archive_version(path): @@ -90,12 +95,12 @@ finally: fp.close() if 'tag' in data: - return 'RPython', data['tag'], data['node'] + return data['tag'], data['node'] else: - return 'RPython', data['branch'], data['node'] + return data['branch'], data['node'] -def _get_git_version(): +def _get_git_version(root): #XXX: this function is a untested hack, # so the git mirror tav made will work gitexe = py.path.local.sysfind('git') @@ -105,7 +110,7 @@ try: p = Popen( [str(gitexe), 'rev-parse', 'HEAD'], - stdout=PIPE, stderr=PIPE, cwd=rpythonroot + stdout=PIPE, stderr=PIPE, cwd=root ) except OSError, e: maywarn(e, 'Git') @@ -116,16 +121,16 @@ revision_id = p.stdout.read().strip()[:12] p = Popen( [str(gitexe), 'describe', '--tags', '--exact-match'], - stdout=PIPE, stderr=PIPE, cwd=rpythonroot + stdout=PIPE, stderr=PIPE, cwd=root ) if p.wait() != 0: p = Popen( [str(gitexe), 'branch'], stdout=PIPE, stderr=PIPE, - cwd=rpythonroot + cwd=root ) if p.wait() != 0: maywarn(p.stderr.read(), 'Git') - return 'RPython', '?', revision_id + return '?', revision_id branch = '?' for line in p.stdout.read().strip().split('\n'): if line.startswith('* '): @@ -133,8 +138,8 @@ if branch == '(no branch)': branch = '?' break - return 'RPython', branch, revision_id - return 'RPython', p.stdout.read().strip(), revision_id + return branch, revision_id + return p.stdout.read().strip(), revision_id if __name__ == '__main__': From noreply at buildbot.pypy.org Sun Jan 27 21:59:19 2013 From: noreply at buildbot.pypy.org (rlamy) Date: Sun, 27 Jan 2013 21:59:19 +0100 (CET) Subject: [pypy-commit] pypy kill-flowobjspace: CallSpec: inline a bit Message-ID: <20130127205919.495221C125A@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: kill-flowobjspace Changeset: r60551:a0ef0f4b0dbc Date: 2013-01-27 18:47 +0000 http://bitbucket.org/pypy/pypy/changeset/a0ef0f4b0dbc/ Log: CallSpec: inline a bit diff --git a/rpython/flowspace/argument.py b/rpython/flowspace/argument.py --- a/rpython/flowspace/argument.py +++ b/rpython/flowspace/argument.py @@ -397,14 +397,11 @@ def combine_if_necessary(self): if self.combine_has_happened: return - self._combine_wrapped(self.w_stararg) + if self.w_stararg is not None: + args_w = self.space.unpackiterable(self.w_stararg) + self.arguments_w = self.arguments_w + args_w self.combine_has_happened = True - def _combine_wrapped(self, w_stararg): - "unpack the *arg and **kwd into arguments_w and keywords_w" - if w_stararg is not None: - self._combine_starargs_wrapped(w_stararg) - def _rawshape(self, nextra=0): assert not self.combine_has_happened shape_cnt = len(self.arguments_w)+nextra # Number of positional args From noreply at buildbot.pypy.org Sun Jan 27 21:59:20 2013 From: noreply at buildbot.pypy.org (rlamy) Date: Sun, 27 Jan 2013 21:59:20 +0100 (CET) Subject: [pypy-commit] pypy kill-flowobjspace: make CallSpec.unpack() non-destructive Message-ID: <20130127205920.6E7761C125A@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: kill-flowobjspace Changeset: r60552:eda004a832ff Date: 2013-01-27 20:51 +0000 http://bitbucket.org/pypy/pypy/changeset/eda004a832ff/ Log: make CallSpec.unpack() non-destructive diff --git a/rpython/flowspace/argument.py b/rpython/flowspace/argument.py --- a/rpython/flowspace/argument.py +++ b/rpython/flowspace/argument.py @@ -386,21 +386,26 @@ self.space = space assert isinstance(args_w, list) self.arguments_w = args_w - self.keywords = keywords - self.keywords_w = keywords_w + self.keywords = keywords or [] + self.keywords_w = keywords_w or [] self.keyword_names_w = None def copy(self): return CallSpec(self.space, self.arguments_w, self.keywords, self.keywords_w, self.w_stararg) + def unpack(self): + "Return a ([w1,w2...], {'kw':w3...}) pair." + if self.w_stararg is not None: + stargs_w = self.space.unpackiterable(self.w_stararg) + args_w = self.arguments_w + stargs_w + else: + args_w = self.arguments_w + kwds_w = dict(zip(self.keywords, self.keywords_w)) + return args_w, kwds_w + def combine_if_necessary(self): - if self.combine_has_happened: - return - if self.w_stararg is not None: - args_w = self.space.unpackiterable(self.w_stararg) - self.arguments_w = self.arguments_w + args_w - self.combine_has_happened = True + raise NotImplementedError def _rawshape(self, nextra=0): assert not self.combine_has_happened diff --git a/rpython/flowspace/objspace.py b/rpython/flowspace/objspace.py --- a/rpython/flowspace/objspace.py +++ b/rpython/flowspace/objspace.py @@ -417,7 +417,7 @@ return sc(self, fn, args) try: - args_w, kwds_w = args.copy().unpack() + args_w, kwds_w = args.unpack() except UnwrapException: args_w, kwds_w = '?', '?' # NOTE: annrpython needs to know about the following two operations! From noreply at buildbot.pypy.org Sun Jan 27 22:05:27 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 27 Jan 2013 22:05:27 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: give up Message-ID: <20130127210527.417C11C12E1@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60553:1079691f3cf0 Date: 2013-01-27 23:04 +0200 http://bitbucket.org/pypy/pypy/changeset/1079691f3cf0/ Log: give up diff --git a/rpython/jit/backend/x86/test/test_zrpy_gc.py b/rpython/jit/backend/x86/test/test_zrpy_gc.py --- a/rpython/jit/backend/x86/test/test_zrpy_gc.py +++ b/rpython/jit/backend/x86/test/test_zrpy_gc.py @@ -793,7 +793,7 @@ def define_compile_framework_call_assembler(self): S = lltype.GcForwardReference() S.become(lltype.GcStruct('S', ('s', lltype.Ptr(S)))) - driver = JitDriver(greens = [], reds = ['x', 'x0', 'x2']) + driver = JitDriver(greens = [], reds = 'auto') def f(n, x, x0, x1, x2, x3, x4, x5, x6, x7, l, s0): driver.jit_merge_point(x=x, x0=x0, x2=x2) From noreply at buildbot.pypy.org Sun Jan 27 22:06:19 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 27 Jan 2013 22:06:19 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: give up even more Message-ID: <20130127210619.996401C12E1@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60554:406487744603 Date: 2013-01-27 23:05 +0200 http://bitbucket.org/pypy/pypy/changeset/406487744603/ Log: give up even more diff --git a/rpython/jit/backend/x86/test/test_zrpy_gc.py b/rpython/jit/backend/x86/test/test_zrpy_gc.py --- a/rpython/jit/backend/x86/test/test_zrpy_gc.py +++ b/rpython/jit/backend/x86/test/test_zrpy_gc.py @@ -796,7 +796,7 @@ driver = JitDriver(greens = [], reds = 'auto') def f(n, x, x0, x1, x2, x3, x4, x5, x6, x7, l, s0): - driver.jit_merge_point(x=x, x0=x0, x2=x2) + driver.jit_merge_point() i = 0 prev_s = lltype.nullptr(S) while i < 100: From noreply at buildbot.pypy.org Sun Jan 27 22:10:51 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 27 Jan 2013 22:10:51 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: ll2ctypes drives me nuts Message-ID: <20130127211051.8CC041C12E1@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60555:7d06cb9306a1 Date: 2013-01-27 22:14 +0100 http://bitbucket.org/pypy/pypy/changeset/7d06cb9306a1/ Log: ll2ctypes drives me nuts diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py --- a/rpython/jit/backend/test/runner_test.py +++ b/rpython/jit/backend/test/runner_test.py @@ -2062,7 +2062,7 @@ def test_cond_call_gc_wb(self): def func_void(a): - record.append(a) + record.append(rffi.cast(lltype.Signed, a)) record = [] # S = lltype.GcStruct('S', ('tid', lltype.Signed)) @@ -2092,13 +2092,13 @@ [BoxPtr(sgcref), ConstPtr(tgcref)], 'void', descr=WriteBarrierDescr()) if cond: - assert record == [s] + assert record == [rffi.cast(lltype.Signed, sgcref)] else: assert record == [] def test_cond_call_gc_wb_array(self): def func_void(a): - record.append(a) + record.append(rffi.cast(lltype.Signed, a)) record = [] # S = lltype.GcStruct('S', ('tid', lltype.Signed)) @@ -2127,13 +2127,13 @@ [BoxPtr(sgcref), ConstInt(123), BoxPtr(sgcref)], 'void', descr=WriteBarrierDescr()) if cond: - assert record == [s] + assert record == [rffi.cast(lltype.Signed, sgcref)] else: assert record == [] def test_cond_call_gc_wb_array_card_marking_fast_path(self): def func_void(a): - record.append(a) + record.append(rffi.cast(lltype.Signed, a)) if cond == 1: # the write barrier sets the flag s.data.tid |= 32768 record = [] @@ -2192,7 +2192,7 @@ [BoxPtr(sgcref), box_index, BoxPtr(sgcref)], 'void', descr=WriteBarrierDescr()) if cond in [0, 1]: - assert record == [s.data] + assert record == [rffi.cast(lltype.Signed, s.data)] else: assert record == [] if cond in [1, 2]: From noreply at buildbot.pypy.org Sun Jan 27 22:19:15 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 27 Jan 2013 22:19:15 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: UH, that's what I get for naming variables wrongly Message-ID: <20130127211915.C629A1C12F9@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60556:32f451f7cffb Date: 2013-01-27 23:16 +0200 http://bitbucket.org/pypy/pypy/changeset/32f451f7cffb/ Log: UH, that's what I get for naming variables wrongly diff --git a/rpython/jit/backend/llsupport/jitframe.py b/rpython/jit/backend/llsupport/jitframe.py --- a/rpython/jit/backend/llsupport/jitframe.py +++ b/rpython/jit/backend/llsupport/jitframe.py @@ -14,9 +14,9 @@ GCMAP = lltype.GcArray(lltype.Unsigned) NULLGCMAP = lltype.nullptr(GCMAP) -def jitframeinfo_set_depth(jfi, new_depth): +def jitframeinfo_set_depth(jfi, base_ofs, new_depth): jfi.jfi_frame_depth = new_depth - jfi.jfi_frame_size = STATICSIZE + new_depth * SIZEOFSIGNED + jfi.jfi_frame_size = base_ofs + new_depth * SIZEOFSIGNED JITFRAMEINFO = lltype.GcStruct( 'JITFRAMEINFO', diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -507,7 +507,7 @@ clt = CompiledLoopToken(self.cpu, looptoken.number) clt.frame_info = lltype.malloc(jitframe.JITFRAMEINFO) clt.allgcrefs = [] - clt.frame_info.set_frame_depth(0) # for now + clt.frame_info.set_frame_depth(0, 0) # for now looptoken.compiled_loop_token = clt clt._debug_nbargs = len(inputargs) if not we_are_translated(): @@ -666,12 +666,13 @@ mc.copy_to_raw_memory(rawstart + pos_after_jz - 4) def update_frame_depth(self, frame_depth): - self.current_clt.frame_info.set_frame_depth(frame_depth) + baseofs = self.cpu.get_baseofs_of_frame_field() + self.current_clt.frame_info.set_frame_depth(baseofs, frame_depth) new_jumping_to = [] for wref in self.current_clt.jumping_to: clt = wref() if clt is not None: - clt.frame_info.set_frame_depth(max(frame_depth, + clt.frame_info.set_frame_depth(baseofs, max(frame_depth, clt.frame_info.jfi_frame_depth)) new_jumping_to.append(weakref.ref(clt)) self.current_clt.jumping_to = new_jumping_to @@ -898,7 +899,8 @@ # copy frame-info data old_fi = oldlooptoken.compiled_loop_token.frame_info new_fi = newlooptoken.compiled_loop_token.frame_info - old_fi.set_frame_depth(new_fi.jfi_frame_depth) + baseofs = self.cpu.get_baseofs_of_frame_field() + old_fi.set_frame_depth(baseofs, new_fi.jfi_frame_depth) mc = codebuf.MachineCodeBlockWrapper() mc.JMP(imm(target)) if WORD == 4: # keep in sync with prepare_loop() From noreply at buildbot.pypy.org Sun Jan 27 22:19:17 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 27 Jan 2013 22:19:17 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: actually just kill this misleading var Message-ID: <20130127211917.120C11C12F9@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60557:a06c4c0a4854 Date: 2013-01-27 23:17 +0200 http://bitbucket.org/pypy/pypy/changeset/a06c4c0a4854/ Log: actually just kill this misleading var diff --git a/rpython/jit/backend/llsupport/jitframe.py b/rpython/jit/backend/llsupport/jitframe.py --- a/rpython/jit/backend/llsupport/jitframe.py +++ b/rpython/jit/backend/llsupport/jitframe.py @@ -3,7 +3,6 @@ from rpython.rlib.objectmodel import specialize from rpython.rlib.debug import ll_assert -STATICSIZE = 0 # patch from the assembler backend SIZEOFSIGNED = rffi.sizeof(lltype.Signed) IS_32BIT = (SIZEOFSIGNED == 2 ** 31 - 1) diff --git a/rpython/jit/backend/llsupport/test/test_gc.py b/rpython/jit/backend/llsupport/test/test_gc.py --- a/rpython/jit/backend/llsupport/test/test_gc.py +++ b/rpython/jit/backend/llsupport/test/test_gc.py @@ -271,42 +271,37 @@ return (frame_adr + jitframe.getofs('jf_frame') + jitframe.BASEITEMOFS + jitframe.SIGN_SIZE * no) - PREV_STATICSIZE = jitframe.STATICSIZE - try: - jitframe.STATICSIZE = 3 - frame_info = lltype.malloc(jitframe.JITFRAMEINFO, zero=True) - frame = lltype.malloc(jitframe.JITFRAME, 15, zero=True) - frame.jf_frame_info = frame_info - frame.jf_gcmap = lltype.malloc(jitframe.GCMAP, 2) - if sys.maxint == 2**31 - 1: - max = r_uint(2 ** 31) - else: - max = r_uint(2 ** 63) - frame.jf_gcmap[0] = r_uint(1 | 2 | 8 | 32 | 128) | max - frame.jf_gcmap[1] = r_uint(2 | 16 | 32 | 128) - frame_adr = llmemory.cast_ptr_to_adr(frame) - all_addrs = [] - next = jitframe.jitframe_trace(frame_adr, llmemory.NULL) - while next: - all_addrs.append(next) - next = jitframe.jitframe_trace(frame_adr, next) - counter = 0 - for name in jitframe.JITFRAME._names: - TP = getattr(jitframe.JITFRAME, name) - if isinstance(TP, lltype.Ptr): # only GC pointers - assert all_addrs[counter] == frame_adr + jitframe.getofs(name) - counter += 1 - # gcpattern - assert all_addrs[6] == indexof(0) - assert all_addrs[7] == indexof(1) - assert all_addrs[8] == indexof(3) - assert all_addrs[9] == indexof(5) - assert all_addrs[10] == indexof(7) - assert all_addrs[11] == indexof(63) - # XXX 32bit - assert all_addrs[12] == indexof(65) + frame_info = lltype.malloc(jitframe.JITFRAMEINFO, zero=True) + frame = lltype.malloc(jitframe.JITFRAME, 15, zero=True) + frame.jf_frame_info = frame_info + frame.jf_gcmap = lltype.malloc(jitframe.GCMAP, 2) + if sys.maxint == 2**31 - 1: + max = r_uint(2 ** 31) + else: + max = r_uint(2 ** 63) + frame.jf_gcmap[0] = r_uint(1 | 2 | 8 | 32 | 128) | max + frame.jf_gcmap[1] = r_uint(2 | 16 | 32 | 128) + frame_adr = llmemory.cast_ptr_to_adr(frame) + all_addrs = [] + next = jitframe.jitframe_trace(frame_adr, llmemory.NULL) + while next: + all_addrs.append(next) + next = jitframe.jitframe_trace(frame_adr, next) + counter = 0 + for name in jitframe.JITFRAME._names: + TP = getattr(jitframe.JITFRAME, name) + if isinstance(TP, lltype.Ptr): # only GC pointers + assert all_addrs[counter] == frame_adr + jitframe.getofs(name) + counter += 1 + # gcpattern + assert all_addrs[6] == indexof(0) + assert all_addrs[7] == indexof(1) + assert all_addrs[8] == indexof(3) + assert all_addrs[9] == indexof(5) + assert all_addrs[10] == indexof(7) + assert all_addrs[11] == indexof(63) + # XXX 32bit + assert all_addrs[12] == indexof(65) - assert len(all_addrs) == 6 + 6 + 4 - # 6 static fields, 4 addresses from gcmap, 2 from gcpattern - finally: - jitframe.STATICSIZE = PREV_STATICSIZE + assert len(all_addrs) == 6 + 6 + 4 + # 6 static fields, 4 addresses from gcmap, 2 from gcpattern diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py --- a/rpython/jit/backend/x86/runner.py +++ b/rpython/jit/backend/x86/runner.py @@ -15,8 +15,6 @@ from rpython.jit.backend.llsupport.descr import ArrayDescr, FLAG_POINTER,\ FLAG_FLOAT -jitframe.STATICSIZE = JITFRAME_FIXED_SIZE - import sys from rpython.tool.ansi_print import ansi_log From noreply at buildbot.pypy.org Sun Jan 27 22:19:18 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 27 Jan 2013 22:19:18 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: merge Message-ID: <20130127211918.675A31C12F9@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60558:103c9c68341d Date: 2013-01-27 23:18 +0200 http://bitbucket.org/pypy/pypy/changeset/103c9c68341d/ Log: merge diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py --- a/rpython/jit/backend/test/runner_test.py +++ b/rpython/jit/backend/test/runner_test.py @@ -2062,7 +2062,7 @@ def test_cond_call_gc_wb(self): def func_void(a): - record.append(a) + record.append(rffi.cast(lltype.Signed, a)) record = [] # S = lltype.GcStruct('S', ('tid', lltype.Signed)) @@ -2092,13 +2092,13 @@ [BoxPtr(sgcref), ConstPtr(tgcref)], 'void', descr=WriteBarrierDescr()) if cond: - assert record == [s] + assert record == [rffi.cast(lltype.Signed, sgcref)] else: assert record == [] def test_cond_call_gc_wb_array(self): def func_void(a): - record.append(a) + record.append(rffi.cast(lltype.Signed, a)) record = [] # S = lltype.GcStruct('S', ('tid', lltype.Signed)) @@ -2127,13 +2127,13 @@ [BoxPtr(sgcref), ConstInt(123), BoxPtr(sgcref)], 'void', descr=WriteBarrierDescr()) if cond: - assert record == [s] + assert record == [rffi.cast(lltype.Signed, sgcref)] else: assert record == [] def test_cond_call_gc_wb_array_card_marking_fast_path(self): def func_void(a): - record.append(a) + record.append(rffi.cast(lltype.Signed, a)) if cond == 1: # the write barrier sets the flag s.data.tid |= 32768 record = [] @@ -2192,7 +2192,7 @@ [BoxPtr(sgcref), box_index, BoxPtr(sgcref)], 'void', descr=WriteBarrierDescr()) if cond in [0, 1]: - assert record == [s.data] + assert record == [rffi.cast(lltype.Signed, s.data)] else: assert record == [] if cond in [1, 2]: From noreply at buildbot.pypy.org Sun Jan 27 22:19:22 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Sun, 27 Jan 2013 22:19:22 +0100 (CET) Subject: [pypy-commit] pypy default: fix exe_name in gen_makefile when shared=True Message-ID: <20130127211922.A59331C12F9@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60559:060ce43207d0 Date: 2013-01-27 16:18 -0500 http://bitbucket.org/pypy/pypy/changeset/060ce43207d0/ Log: fix exe_name in gen_makefile when shared=True diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py --- a/rpython/translator/platform/posix.py +++ b/rpython/translator/platform/posix.py @@ -115,7 +115,7 @@ cflags = self.cflags + self.standalone_only m = GnuMakefile(path) - m.exe_name = path.join(target_name) + m.exe_name = path.join(exe_name.basename) m.eci = eci def rpyrel(fpath): diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -269,7 +269,7 @@ target_name = exe_name.basename m = NMakefile(path) - m.exe_name = path.join(target_name) + m.exe_name = path.join(exe_name.basename) m.eci = eci linkflags = list(self.link_flags) From noreply at buildbot.pypy.org Sun Jan 27 22:19:24 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Sun, 27 Jan 2013 22:19:24 +0100 (CET) Subject: [pypy-commit] pypy default: fix whatsnew Message-ID: <20130127211924.17A7D1C12F9@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60560:02e6ddae5c1d Date: 2013-01-27 16:19 -0500 http://bitbucket.org/pypy/pypy/changeset/02e6ddae5c1d/ Log: fix whatsnew diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -36,6 +36,7 @@ .. branch: task-decorator .. branch: fix-e4fa0b2 .. branch: win32-fixes +.. branch: fix-version-tool .. branch: release-2.0-beta1 From noreply at buildbot.pypy.org Sun Jan 27 22:38:19 2013 From: noreply at buildbot.pypy.org (fijal) Date: Sun, 27 Jan 2013 22:38:19 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: this is respectively uninteresting and done Message-ID: <20130127213819.C48A41C1306@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60561:4e549c373b0b Date: 2013-01-27 23:37 +0200 http://bitbucket.org/pypy/pypy/changeset/4e549c373b0b/ Log: this is respectively uninteresting and done diff --git a/pypy/TODO b/pypy/TODO --- a/pypy/TODO +++ b/pypy/TODO @@ -5,6 +5,3 @@ * shadowstack details - slowpath of stack check * kill jit2gc on translator * fix test_singlefloats in test_calling_conventions -* slowpaths can have more variants depending on how many registers we're using - (like floats vs non-floats for failures) -* fix jit hooks \ No newline at end of file From noreply at buildbot.pypy.org Sun Jan 27 22:53:33 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Sun, 27 Jan 2013 22:53:33 +0100 (CET) Subject: [pypy-commit] pypy fix-version-tool: merge default Message-ID: <20130127215333.D7ADB1C067C@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: fix-version-tool Changeset: r60562:a0295c6b1209 Date: 2013-01-27 16:41 -0500 http://bitbucket.org/pypy/pypy/changeset/a0295c6b1209/ Log: merge default diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -34,6 +34,9 @@ .. branch: kill-faking .. branch: improved_ebnfparse_error .. branch: task-decorator +.. branch: fix-e4fa0b2 +.. branch: win32-fixes +.. branch: fix-version-tool .. branch: release-2.0-beta1 diff --git a/pypy/module/unicodedata/interp_ucd.py b/pypy/module/unicodedata/interp_ucd.py --- a/pypy/module/unicodedata/interp_ucd.py +++ b/pypy/module/unicodedata/interp_ucd.py @@ -9,7 +9,7 @@ from rpython.rlib.objectmodel import we_are_translated from rpython.rlib.runicode import MAXUNICODE from rpython.rlib.unicodedata import unicodedb_5_2_0, unicodedb_3_2_0 -from rpython.rlib.runicode import code_to_unichr, ORD +from rpython.rlib.runicode import code_to_unichr, ord_accepts_surrogate import sys @@ -28,8 +28,6 @@ # handling: on narrow unicode builds, a surrogate pair is considered as one # unicode code point. -# The functions below are subtly different from the ones in runicode.py. -# When PyPy implements Python 3 they should be merged. if MAXUNICODE > 0xFFFF: # Target is wide build @@ -41,7 +39,7 @@ if not we_are_translated() and sys.maxunicode == 0xFFFF: # Host CPython is narrow build, accept surrogates try: - return ORD(space.unicode_w(w_unichr)) + return ord_accepts_surrogate(space.unicode_w(w_unichr)) except ValueError: raise OperationError(space.w_TypeError, space.wrap( 'need a single Unicode character as parameter')) @@ -68,7 +66,7 @@ else: # Accept surrogates try: - return ORD(space.unicode_w(w_unichr)) + return ord_accepts_surrogate(space.unicode_w(w_unichr)) except ValueError: raise OperationError(space.w_TypeError, space.wrap( 'need a single Unicode character as parameter')) diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py --- a/rpython/rlib/_rsocket_rffi.py +++ b/rpython/rlib/_rsocket_rffi.py @@ -344,7 +344,7 @@ ]) CConfig.WSAPROTOCOL_INFO = platform.Struct( - 'struct WSAPROTOCOL_INFO', + 'WSAPROTOCOL_INFO', []) # Struct is just passed between functions CConfig.FROM_PROTOCOL_INFO = platform.DefinedConstantInteger( 'FROM_PROTOCOL_INFO') @@ -617,7 +617,7 @@ WSASocket = external('WSASocket', [rffi.INT, rffi.INT, rffi.INT, lltype.Ptr(WSAPROTOCOL_INFO), - rffi.DWORD, rffi.DWORD], + rwin32.DWORD, rwin32.DWORD], socketfd_type) if WIN32: diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py --- a/rpython/rlib/runicode.py +++ b/rpython/rlib/runicode.py @@ -13,6 +13,27 @@ BYTEORDER = sys.byteorder +# python 2.7 has a preview of py3k behavior, so those functions +# are used either when we're testing wide pypy on narrow cpython +# or in unicodedata in pypy + +def unichr_returns_surrogate(c): + if c <= sys.maxunicode or c > MAXUNICODE: + return unichr(c) + else: + c -= 0x10000 + return (unichr(0xD800 + (c >> 10)) + + unichr(0xDC00 + (c & 0x03FF))) + +def ord_accepts_surrogate(u): + if isinstance(u, unicode) and len(u) == 2: + ch1 = ord(u[0]) + ch2 = ord(u[1]) + if 0xD800 <= ch1 <= 0xDBFF and 0xDC00 <= ch2 <= 0xDFFF: + return (((ch1 - 0xD800) << 10) | (ch2 - 0xDC00)) + 0x10000 + assert len(u) == 1 + return ord(u[0]) + if MAXUNICODE > sys.maxunicode: # A version of unichr which allows codes outside the BMP # even on narrow unicode builds. @@ -21,12 +42,7 @@ # Note that Python3 uses a similar implementation. def UNICHR(c): assert not we_are_translated() - if c <= sys.maxunicode or c > MAXUNICODE: - return unichr(c) - else: - c -= 0x10000 - return (unichr(0xD800 + (c >> 10)) + - unichr(0xDC00 + (c & 0x03FF))) + return unichr_returns_surrogate(c) UNICHR._flowspace_rewrite_directly_as_ = unichr # ^^^ NB.: for translation, it's essential to use this hack instead # of calling unichr() from UNICHR(), because unichr() detects if there @@ -34,12 +50,7 @@ def ORD(u): assert not we_are_translated() - if isinstance(u, unicode) and len(u) == 2: - ch1 = ord(u[0]) - ch2 = ord(u[1]) - if 0xD800 <= ch1 <= 0xDBFF and 0xDC00 <= ch2 <= 0xDFFF: - return (((ch1 - 0xD800) << 10) | (ch2 - 0xDC00)) + 0x10000 - return ord(u) + return ord_accepts_surrogate(u) ORD._flowspace_rewrite_directly_as_ = ord else: @@ -50,13 +61,13 @@ def code_to_unichr(code): if not we_are_translated() and sys.maxunicode == 0xFFFF: # Host CPython is narrow build, generate surrogates - return UNICHR(code) + return unichr_returns_surrogate(code) else: return unichr(code) else: def code_to_unichr(code): # generate surrogates for large codes - return UNICHR(code) + return unichr_returns_surrogate(code) def _STORECHAR(result, CH, byteorder): hi = chr(((CH) >> 8) & 0xff) diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py --- a/rpython/translator/platform/posix.py +++ b/rpython/translator/platform/posix.py @@ -115,7 +115,7 @@ cflags = self.cflags + self.standalone_only m = GnuMakefile(path) - m.exe_name = path.join(target_name) + m.exe_name = path.join(exe_name.basename) m.eci = eci def rpyrel(fpath): diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py --- a/rpython/translator/platform/windows.py +++ b/rpython/translator/platform/windows.py @@ -269,7 +269,7 @@ target_name = exe_name.basename m = NMakefile(path) - m.exe_name = path.join(target_name) + m.exe_name = path.join(exe_name.basename) m.eci = eci linkflags = list(self.link_flags) From noreply at buildbot.pypy.org Sun Jan 27 22:53:35 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Sun, 27 Jan 2013 22:53:35 +0100 (CET) Subject: [pypy-commit] pypy fix-version-tool: remove hardcoded project from get_repo_version_info() Message-ID: <20130127215335.24C931C067C@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: fix-version-tool Changeset: r60563:d28498b4b051 Date: 2013-01-27 16:53 -0500 http://bitbucket.org/pypy/pypy/changeset/d28498b4b051/ Log: remove hardcoded project from get_repo_version_info() diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -69,7 +69,7 @@ CPYTHON_VERSION[0], CPYTHON_VERSION[1], CPYTHON_VERSION[2], - get_repo_version_info(root=pypyroot)[2], + get_repo_version_info(root=pypyroot)[1], date, time, ver, @@ -94,7 +94,7 @@ def get_repo_info(space): info = get_repo_version_info(root=pypyroot) if info: - project, repo_tag, repo_version = info + repo_tag, repo_version = info return space.newtuple([space.wrap('PyPy'), space.wrap(repo_tag), space.wrap(repo_version)]) diff --git a/rpython/tool/test/test_version.py b/rpython/tool/test/test_version.py --- a/rpython/tool/test/test_version.py +++ b/rpython/tool/test/test_version.py @@ -19,5 +19,5 @@ def test_get_repo_version_info(): assert get_repo_version_info(None) - assert get_repo_version_info(os.devnull) == ('RPython', '?', '?') - assert get_repo_version_info(sys.executable) == ('RPython', '?', '?') + assert get_repo_version_info(os.devnull) == ('?', '?') + assert get_repo_version_info(sys.executable) == ('?', '?') diff --git a/rpython/tool/udir.py b/rpython/tool/udir.py --- a/rpython/tool/udir.py +++ b/rpython/tool/udir.py @@ -31,7 +31,7 @@ if basename is None: info = get_repo_version_info() if info: - project, hgtag, hgid = info + hgtag, hgid = info basename = hgtag if basename == '?': basename = 'unknown' # directories with ? are not fun diff --git a/rpython/tool/version.py b/rpython/tool/version.py --- a/rpython/tool/version.py +++ b/rpython/tool/version.py @@ -18,30 +18,25 @@ def get_repo_version_info(hgexe=None, root=rpythonroot): '''Obtain version information by invoking the 'hg' or 'git' commands.''' - if root == rpythonroot: - project = ("RPython",) - else: - project = ('?',) - # Try to see if we can get info from Git if hgexe is not specified. if not hgexe: if os.path.isdir(os.path.join(root, '.git')): - return project + _get_git_version(root) + return _get_git_version(root) # Fallback to trying Mercurial. if hgexe is None: hgexe = py.path.local.sysfind('hg') if os.path.isfile(os.path.join(root, '.hg_archival.txt')): - return project + _get_hg_archive_version(os.path.join(root, '.hg_archival.txt')) + return _get_hg_archive_version(os.path.join(root, '.hg_archival.txt')) elif not os.path.isdir(os.path.join(root, '.hg')): maywarn('Not running from a Mercurial repository!') - return project + default_retval + return default_retval elif not hgexe: maywarn('Cannot find Mercurial command!') - return project + default_retval + return default_retval else: - return project + _get_hg_version(hgexe, root) + return _get_hg_version(hgexe, root) def _get_hg_version(hgexe, root): From noreply at buildbot.pypy.org Sun Jan 27 23:46:57 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Sun, 27 Jan 2013 23:46:57 +0100 (CET) Subject: [pypy-commit] pypy default: clarify comment Message-ID: <20130127224657.ABF881C125A@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60566:022a44a9b630 Date: 2013-01-27 17:46 -0500 http://bitbucket.org/pypy/pypy/changeset/022a44a9b630/ Log: clarify comment diff --git a/pypy/interpreter/test2/test_app_main.py b/pypy/interpreter/test2/test_app_main.py --- a/pypy/interpreter/test2/test_app_main.py +++ b/pypy/interpreter/test2/test_app_main.py @@ -774,7 +774,7 @@ assert data == p + os.sep + '\n' def test_getfilesystemencoding(self): - py.test.skip("this has been failing since forever, but it's not tested nightly because buildbot uses python2.6 :-(") + py.test.skip("encoding is only set if stdout.isatty(), test is flawed") if sys.version_info < (2, 7): skip("test requires Python >= 2.7") p = getscript_in_dir(""" From noreply at buildbot.pypy.org Sun Jan 27 23:56:21 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Sun, 27 Jan 2013 23:56:21 +0100 (CET) Subject: [pypy-commit] pypy default: cleanups Message-ID: <20130127225621.8C3CE1C12E1@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60567:44a14b38dc4c Date: 2013-01-27 17:56 -0500 http://bitbucket.org/pypy/pypy/changeset/44a14b38dc4c/ Log: cleanups diff --git a/pypy/interpreter/test2/test_app_main.py b/pypy/interpreter/test2/test_app_main.py --- a/pypy/interpreter/test2/test_app_main.py +++ b/pypy/interpreter/test2/test_app_main.py @@ -881,7 +881,6 @@ def test_setup_bootstrap_path(self): import sys - import os old_sys_path = sys.path[:] sys.path.append(self.goal_dir) try: diff --git a/pypy/interpreter/test2/test_targetpypy.py b/pypy/interpreter/test2/test_targetpypy.py --- a/pypy/interpreter/test2/test_targetpypy.py +++ b/pypy/interpreter/test2/test_targetpypy.py @@ -1,5 +1,3 @@ - -import py from pypy.goal.targetpypystandalone import get_entry_point from pypy.config.pypyoption import get_pypy_config @@ -7,6 +5,4 @@ def test_run(self): config = get_pypy_config(translating=False) entry_point = get_entry_point(config)[0] - space = self.space - py.test.skip("not working so far") entry_point(['pypy-c' , '-S', '-c', 'print 3']) From noreply at buildbot.pypy.org Mon Jan 28 00:24:29 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 28 Jan 2013 00:24:29 +0100 (CET) Subject: [pypy-commit] pypy default: fix unnecessary skip Message-ID: <20130127232429.AE04B1C067C@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60568:af9676802136 Date: 2013-01-27 18:24 -0500 http://bitbucket.org/pypy/pypy/changeset/af9676802136/ Log: fix unnecessary skip diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -2,7 +2,7 @@ import os class AppTestFileIO: - spaceconfig = dict(usemodules=['_io']) + spaceconfig = dict(usemodules=['_io'] + (['fcntl'] if os.name != 'nt' else [])) def setup_class(cls): tmpfile = udir.join('tmpfile') From noreply at buildbot.pypy.org Mon Jan 28 00:58:29 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 28 Jan 2013 00:58:29 +0100 (CET) Subject: [pypy-commit] pypy default: fix some tests on darwin Message-ID: <20130127235829.DD9701C1055@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60569:b59c8e831df0 Date: 2013-01-27 18:58 -0500 http://bitbucket.org/pypy/pypy/changeset/b59c8e831df0/ Log: fix some tests on darwin diff --git a/pypy/module/_ffi/test/test_funcptr.py b/pypy/module/_ffi/test/test_funcptr.py --- a/pypy/module/_ffi/test/test_funcptr.py +++ b/pypy/module/_ffi/test/test_funcptr.py @@ -96,7 +96,7 @@ libm = CDLL(self.libm_name) pow_addr = libm.getaddressindll('pow') fff = sys.maxint*2-1 - if sys.platform == 'win32': + if sys.platform == 'win32' or sys.platform == 'darwin': fff = sys.maxint*2+1 assert pow_addr == self.pow_addr & fff diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py --- a/pypy/module/rctime/test/test_rctime.py +++ b/pypy/module/rctime/test/test_rctime.py @@ -213,7 +213,7 @@ def test_strftime(self): import time as rctime - import os + import os, sys t = rctime.time() tt = rctime.gmtime(t) @@ -234,6 +234,10 @@ # input to [w]strftime is not kosher. if os.name == 'nt': raises(ValueError, rctime.strftime, '%f') + elif sys.platform == 'darwin': + # darwin strips % of unknown format codes + # http://bugs.python.org/issue9811 + assert rctime.strftime('%f') == 'f' else: assert rctime.strftime('%f') == '%f' diff --git a/pypy/module/signal/test/test_signal.py b/pypy/module/signal/test/test_signal.py --- a/pypy/module/signal/test/test_signal.py +++ b/pypy/module/signal/test/test_signal.py @@ -157,6 +157,8 @@ if sys.platform == 'win32': raises(ValueError, signal, 42, lambda *args: None) raises(ValueError, signal, 7, lambda *args: None) + elif sys.platform == 'darwin': + raises(ValueError, signal, 42, lambda *args: None) else: signal(42, lambda *args: None) signal(42, SIG_DFL) From noreply at buildbot.pypy.org Mon Jan 28 02:17:06 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 28 Jan 2013 02:17:06 +0100 (CET) Subject: [pypy-commit] pypy fix-e4fa0b2: close merged branch Message-ID: <20130128011706.B24E81C039A@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: fix-e4fa0b2 Changeset: r60570:e5da67dc7430 Date: 2013-01-27 20:15 -0500 http://bitbucket.org/pypy/pypy/changeset/e5da67dc7430/ Log: close merged branch From noreply at buildbot.pypy.org Mon Jan 28 04:09:58 2013 From: noreply at buildbot.pypy.org (rlamy) Date: Mon, 28 Jan 2013 04:09:58 +0100 (CET) Subject: [pypy-commit] pypy kill-flowobjspace: flowspace: test kwarg call Message-ID: <20130128030958.31EF01C039A@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: kill-flowobjspace Changeset: r60571:448681fc6a2e Date: 2013-01-27 22:25 +0000 http://bitbucket.org/pypy/pypy/changeset/448681fc6a2e/ Log: flowspace: test kwarg call diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py --- a/rpython/flowspace/test/test_objspace.py +++ b/rpython/flowspace/test/test_objspace.py @@ -716,6 +716,18 @@ graph = self.codetest(f2) assert 'Dict-unpacking' in str(excinfo.value) + def test_kwarg_call(self): + def g(x): + return x + def f(): + return g(x=2) + graph = self.codetest(f) + for block in graph.iterblocks(): + for op in block.operations: + assert op.opname == "call_args" + assert op.args == map(Constant, + [g, (0, ('x',), False, False), 2]) + def test_catch_importerror_1(self): def f(): try: From noreply at buildbot.pypy.org Mon Jan 28 04:09:59 2013 From: noreply at buildbot.pypy.org (rlamy) Date: Mon, 28 Jan 2013 04:09:59 +0100 (CET) Subject: [pypy-commit] pypy kill-flowobjspace: Store CallSpec kwargs in a dict Message-ID: <20130128030959.7944C1C039A@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: kill-flowobjspace Changeset: r60572:4affc42b8dba Date: 2013-01-27 22:56 +0000 http://bitbucket.org/pypy/pypy/changeset/4affc42b8dba/ Log: Store CallSpec kwargs in a dict diff --git a/rpython/flowspace/argument.py b/rpython/flowspace/argument.py --- a/rpython/flowspace/argument.py +++ b/rpython/flowspace/argument.py @@ -378,21 +378,18 @@ """Represents the arguments passed into a function call, i.e. the `a, b, *c, **d` part in `return func(a, b, *c, **d)`. """ - def __init__(self, space, args_w, keywords=None, keywords_w=None, - w_stararg=None, w_starstararg=None): + def __init__(self, space, args_w, keywords=None, w_stararg=None, + w_starstararg=None): self.w_stararg = w_stararg assert w_starstararg is None, "No **-unpacking in RPython" self.combine_has_happened = False self.space = space assert isinstance(args_w, list) self.arguments_w = args_w - self.keywords = keywords or [] - self.keywords_w = keywords_w or [] - self.keyword_names_w = None + self.keywords = keywords or {} def copy(self): - return CallSpec(self.space, self.arguments_w, - self.keywords, self.keywords_w, self.w_stararg) + return self def unpack(self): "Return a ([w1,w2...], {'kw':w3...}) pair." @@ -401,23 +398,21 @@ args_w = self.arguments_w + stargs_w else: args_w = self.arguments_w - kwds_w = dict(zip(self.keywords, self.keywords_w)) - return args_w, kwds_w + return args_w, self.keywords def combine_if_necessary(self): raise NotImplementedError - def _rawshape(self, nextra=0): - assert not self.combine_has_happened - shape_cnt = len(self.arguments_w)+nextra # Number of positional args - if self.keywords: - shape_keys = self.keywords[:] # List of keywords (strings) - shape_keys.sort() - else: - shape_keys = [] + def flatten(self): + """ Argument <-> list of w_objects together with "shape" information """ + shape_cnt = len(self.arguments_w) # Number of positional args + shape_keys = tuple(sorted(self.keywords)) shape_star = self.w_stararg is not None # Flag: presence of *arg shape_stst = False # Flag: presence of **kwds - return shape_cnt, tuple(shape_keys), shape_star, shape_stst # shape_keys are sorted + data_w = self.arguments_w + [self.keywords[key] for key in shape_keys] + if shape_star: + data_w.append(self.w_stararg) + return (shape_cnt, shape_keys, shape_star, shape_stst), data_w # diff --git a/rpython/flowspace/flowcontext.py b/rpython/flowspace/flowcontext.py --- a/rpython/flowspace/flowcontext.py +++ b/rpython/flowspace/flowcontext.py @@ -975,24 +975,14 @@ raise FlowingError(self, "Dict-unpacking is not RPython") n_arguments = oparg & 0xff n_keywords = (oparg>>8) & 0xff - if n_keywords: - keywords = [None] * n_keywords - keywords_w = [None] * n_keywords - while True: - n_keywords -= 1 - if n_keywords < 0: - break - w_value = self.popvalue() - w_key = self.popvalue() - key = self.space.str_w(w_key) - keywords[n_keywords] = key - keywords_w[n_keywords] = w_value - else: - keywords = None - keywords_w = None + keywords = {} + for _ in range(n_keywords): + w_value = self.popvalue() + w_key = self.popvalue() + key = self.space.str_w(w_key) + keywords[key] = w_value arguments = self.popvalues(n_arguments) - args = CallSpec(self.space, arguments, keywords, - keywords_w, w_star, w_starstar) + args = CallSpec(self.space, arguments, keywords, w_star, w_starstar) w_function = self.popvalue() w_result = self.space.call_args(w_function, args) self.pushvalue(w_result) From noreply at buildbot.pypy.org Mon Jan 28 04:10:00 2013 From: noreply at buildbot.pypy.org (rlamy) Date: Mon, 28 Jan 2013 04:10:00 +0100 (CET) Subject: [pypy-commit] pypy kill-flowobjspace: make CallSpec independent of ArgumentsForTranslation Message-ID: <20130128031000.BC52D1C039A@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: kill-flowobjspace Changeset: r60573:5398a4125bb7 Date: 2013-01-28 03:09 +0000 http://bitbucket.org/pypy/pypy/changeset/5398a4125bb7/ Log: make CallSpec independent of ArgumentsForTranslation diff --git a/rpython/flowspace/argument.py b/rpython/flowspace/argument.py --- a/rpython/flowspace/argument.py +++ b/rpython/flowspace/argument.py @@ -374,7 +374,7 @@ return args._rawshape(nextra) -class CallSpec(ArgumentsForTranslation): +class CallSpec(object): """Represents the arguments passed into a function call, i.e. the `a, b, *c, **d` part in `return func(a, b, *c, **d)`. """ @@ -388,9 +388,6 @@ self.arguments_w = args_w self.keywords = keywords or {} - def copy(self): - return self - def unpack(self): "Return a ([w1,w2...], {'kw':w3...}) pair." if self.w_stararg is not None: @@ -400,9 +397,6 @@ args_w = self.arguments_w return args_w, self.keywords - def combine_if_necessary(self): - raise NotImplementedError - def flatten(self): """ Argument <-> list of w_objects together with "shape" information """ shape_cnt = len(self.arguments_w) # Number of positional args From noreply at buildbot.pypy.org Mon Jan 28 05:21:43 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 28 Jan 2013 05:21:43 +0100 (CET) Subject: [pypy-commit] pypy default: more darwin test fixes Message-ID: <20130128042143.90E151C039A@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60574:f35e9b243a35 Date: 2013-01-27 20:21 -0800 http://bitbucket.org/pypy/pypy/changeset/f35e9b243a35/ Log: more darwin test fixes diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -49,6 +49,8 @@ kwds["libraries"] = [api_library] # '%s' undefined; assuming extern returning int kwds["compile_extra"] = ["/we4013"] + elif sys.platform == 'darwin': + kwds["link_files"] = [str(api_library + '.dylib')] else: kwds["link_files"] = [str(api_library + '.so')] if sys.platform.startswith('linux'): diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py --- a/rpython/rlib/test/test_rposix.py +++ b/rpython/rlib/test/test_rposix.py @@ -117,7 +117,7 @@ curdir = os.getcwd() try: interpret(f, []) - assert os.getcwdu() == self.ufilename + assert os.getcwdu() == os.path.realpath(self.ufilename) finally: os.chdir(curdir) diff --git a/rpython/translator/c/test/test_extfunc.py b/rpython/translator/c/test/test_extfunc.py --- a/rpython/translator/c/test/test_extfunc.py +++ b/rpython/translator/c/test/test_extfunc.py @@ -316,7 +316,7 @@ return os.getcwd() f1 = compile(does_stuff, [str]) # different on windows please - assert f1('/tmp') == '/tmp' + assert f1('/tmp') == os.path.realpath('/tmp') def test_mkdir_rmdir(): def does_stuff(path, delete): diff --git a/rpython/translator/platform/test/test_posix.py b/rpython/translator/platform/test/test_posix.py --- a/rpython/translator/platform/test/test_posix.py +++ b/rpython/translator/platform/test/test_posix.py @@ -41,7 +41,8 @@ if self.strict_on_stderr: assert res.err == '' assert res.returncode == 0 - assert '-lrt' in tmpdir.join("Makefile").read() + if sys.platform.startswith('linux'): + assert '-lrt' in tmpdir.join("Makefile").read() def test_link_files(self): tmpdir = udir.join('link_files' + self.__class__.__name__).ensure(dir=1) From noreply at buildbot.pypy.org Mon Jan 28 09:14:14 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 28 Jan 2013 09:14:14 +0100 (CET) Subject: [pypy-commit] pypy default: add watchdog_nt.py lost in split-rpython Message-ID: <20130128081414.DDE9C1C12FA@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60575:c73d9156092a Date: 2013-01-28 03:08 -0500 http://bitbucket.org/pypy/pypy/changeset/c73d9156092a/ Log: add watchdog_nt.py lost in split-rpython diff --git a/rpython/tool/watchdog_nt.py b/rpython/tool/watchdog_nt.py new file mode 100644 --- /dev/null +++ b/rpython/tool/watchdog_nt.py @@ -0,0 +1,33 @@ +import sys, os +import threading +import ctypes + +def childkill(pid): + global timedout + timedout = True + sys.stderr.write("==== test running for %d seconds ====\n" % timeout) + sys.stderr.write("="*26 + "timedout" + "="*26 + "\n") + ctypes.windll.kernel32.TerminateProcess(pid, 1) + +if __name__ == '__main__': + PROCESS_TERMINATE = 0x1 + + timeout = float(sys.argv[1]) + timedout = False + + pid = os.spawnv(os.P_NOWAIT, sys.argv[2], sys.argv[2:]) + + t = threading.Timer(timeout, childkill, (pid,)) + t.start() + while True: + try: + pid, status = os.waitpid(pid, 0) + except KeyboardInterrupt: + continue + else: + t.cancel() + break + + #print 'status ', status >> 8 + sys.exit(status >> 8) + From noreply at buildbot.pypy.org Mon Jan 28 09:18:14 2013 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 28 Jan 2013 09:18:14 +0100 (CET) Subject: [pypy-commit] pypy default: Port @with_unicode_literals from the py3k branch Message-ID: <20130128081814.4500E1C12FA@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: Changeset: r60576:ef2d35b92279 Date: 2013-01-27 20:50 +0100 http://bitbucket.org/pypy/pypy/changeset/ef2d35b92279/ Log: Port @with_unicode_literals from the py3k branch diff --git a/rpython/tool/sourcetools.py b/rpython/tool/sourcetools.py --- a/rpython/tool/sourcetools.py +++ b/rpython/tool/sourcetools.py @@ -6,6 +6,7 @@ # XXX We should try to generalize and single out one approach to dynamic # XXX code compilation. +import types import sys, os, inspect, new import py @@ -295,3 +296,40 @@ result.func_defaults = f.func_defaults result.func_dict.update(f.func_dict) return result + + +def _convert_const_maybe(x, encoding): + if isinstance(x, str): + return x.decode(encoding) + elif isinstance(x, tuple): + items = [_convert_const_maybe(item, encoding) for item in x] + return tuple(items) + return x + +def with_unicode_literals(fn=None, **kwds): + """Decorator that replace all string literals with unicode literals. + Similar to 'from __future__ import string literals' at function level. + Useful to limit changes in the py3k branch. + """ + encoding = kwds.pop('encoding', 'ascii') + if kwds: + raise TypeError("Unexpected keyword argument(s): %s" % ', '.join(kwds.keys())) + def decorator(fn): + co = fn.func_code + new_consts = [] + for const in co.co_consts: + new_consts.append(_convert_const_maybe(const, encoding)) + new_consts = tuple(new_consts) + new_code = types.CodeType(co.co_argcount, co.co_nlocals, co.co_stacksize, + co.co_flags, co.co_code, new_consts, co.co_names, + co.co_varnames, co.co_filename, co.co_name, + co.co_firstlineno, co.co_lnotab) + fn.func_code = new_code + return fn + # + # support the usage of @with_unicode_literals instead of @with_unicode_literals() + if fn is not None: + assert type(fn) is types.FunctionType + return decorator(fn) + else: + return decorator diff --git a/rpython/tool/test/test_sourcetools.py b/rpython/tool/test/test_sourcetools.py --- a/rpython/tool/test/test_sourcetools.py +++ b/rpython/tool/test/test_sourcetools.py @@ -1,4 +1,7 @@ -from rpython.tool.sourcetools import func_with_new_name, func_renamer, rpython_wrapper +# -*- encoding: utf-8 -*- +import py +from rpython.tool.sourcetools import ( + func_with_new_name, func_renamer, rpython_wrapper, with_unicode_literals) def test_rename(): def f(x, y=5): @@ -56,3 +59,28 @@ ] +def test_with_unicode_literals(): + @with_unicode_literals() + def foo(): + return 'hello' + assert type(foo()) is unicode + # + @with_unicode_literals + def foo(): + return 'hello' + assert type(foo()) is unicode + # + def foo(): + return 'hello àèì' + py.test.raises(UnicodeDecodeError, "with_unicode_literals(foo)") + # + @with_unicode_literals(encoding='utf-8') + def foo(): + return 'hello àèì' + assert foo() == u'hello àèì' + # + @with_unicode_literals + def foo(): + return ('a', 'b') + assert type(foo()[0]) is unicode + From noreply at buildbot.pypy.org Mon Jan 28 09:18:29 2013 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 28 Jan 2013 09:18:29 +0100 (CET) Subject: [pypy-commit] pypy default: Sort trie substrings, to ensure repeatability across Python versions. Message-ID: <20130128081829.3AF1F1C12FA@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: Changeset: r60577:22dbfd77aec7 Date: 2013-01-27 22:40 +0100 http://bitbucket.org/pypy/pypy/changeset/22dbfd77aec7/ Log: Sort trie substrings, to ensure repeatability across Python versions. diff too long, truncating to 2000 out of 123496 lines diff --git a/rpython/rlib/unicodedata/generate_unicodedb.py b/rpython/rlib/unicodedata/generate_unicodedb.py --- a/rpython/rlib/unicodedata/generate_unicodedb.py +++ b/rpython/rlib/unicodedata/generate_unicodedb.py @@ -1,5 +1,8 @@ #!/usr/bin/env python +import sys, os +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))) + MAXUNICODE = 0x10FFFF # the value of sys.maxunicode of wide Python builds MANDATORY_LINE_BREAKS = ["BK", "CR", "LF", "NL"] # line break categories diff --git a/rpython/rlib/unicodedata/triegenerator.py b/rpython/rlib/unicodedata/triegenerator.py --- a/rpython/rlib/unicodedata/triegenerator.py +++ b/rpython/rlib/unicodedata/triegenerator.py @@ -174,7 +174,7 @@ print >> outfile, "_stringtable = (" stringidx = {} stridx = 0 - for string in rootnode.allstrings: + for string in sorted(rootnode.allstrings): strlen = len(string) assert strlen < 256, "Substring too long, > 255 chars" print >> outfile, "%r" % (chr(strlen) + string) diff --git a/rpython/rlib/unicodedata/unicodedb_3_2_0.py b/rpython/rlib/unicodedata/unicodedb_3_2_0.py --- a/rpython/rlib/unicodedata/unicodedb_3_2_0.py +++ b/rpython/rlib/unicodedata/unicodedb_3_2_0.py @@ -15974,9 +15974,7 @@ if not ('0' <= c <= '9' or 'A' <= c <= 'F'): raise KeyError code = int(cjk_code, 16) - if (0x3400 <= code <= 0x4DB5 or - 0x4E00 <= code <= 0x9FA5 or - 0x20000 <= code <= 0x2A6D6): + if (0x3400 <= code <= 0x4DB5 or 0x4E00 <= code <= 0x9FA5 or 0x20000 <= code <= 0x2A6D6): return code raise KeyError @@ -15998,9 +15996,7 @@ raise def name(code): - if (0x3400 <= code <= 0x4DB5 or - 0x4E00 <= code <= 0x9FA5 or - 0x20000 <= code <= 0x2A6D6): + if (0x3400 <= code <= 0x4DB5 or 0x4E00 <= code <= 0x9FA5 or 0x20000 <= code <= 0x2A6D6): return "CJK UNIFIED IDEOGRAPH-" + hex(code)[2:].upper() if 0xAC00 <= code <= 0xD7A3: # vl_code, t_code = divmod(code - 0xAC00, len(_hangul_T)) @@ -16011,7 +16007,7 @@ v_code = vl_code % len(_hangul_V) return ("HANGUL SYLLABLE " + _hangul_L[l_code] + _hangul_V[v_code] + _hangul_T[t_code]) - + if not base_mod: return lookup_charcode(code) else: diff --git a/rpython/rlib/unicodedata/unicodedb_5_2_0.py b/rpython/rlib/unicodedata/unicodedb_5_2_0.py --- a/rpython/rlib/unicodedata/unicodedb_5_2_0.py +++ b/rpython/rlib/unicodedata/unicodedb_5_2_0.py @@ -74,89651 +74,89651 @@ _stringtable = ( -'\x11TEN THOUSAND SIGN' -'\x06ESHTIN' -'\x04KAD3' -'\x03IX ' -'\x06JOINED' -'\x04LASH' -'\x06JOINER' -'\tUPTSTIMME' -'\x0bOPPOSING EN' -'\x07MICOLON' -'\x0eLESS-THAN NOR ' -'\x03IXT' -'\x07LEUT KA' -'\rYPOGEGRAMMENI' -'\x1dUP HEAVY AND RIGHT DOWN LIGHT' -'\nBLACK STAR' -'\x05RUHUA' -'\x05WISAD' -'\x15OPPOSING AN PLUS NAGA' -'\nMONOFONIAS' +'\x01 ' +'\x02 A' +'\x06 ABOVE' +'\x14 ABOVE LEFT TRIANGLE' +'\x14 ABOVE SLANTED EQUAL' +'\x05 ADEG' +'\x06 AGUNG' +'\x04 ALL' +'\x05 ALLO' +'\x0f ALTERNATE FORM' +'\x11 ALTERNATION MARK' +'\x05 AMPS' +'\x03 AN' +'\x05 AND ' +'\n AND ACUTE' +'\n AND BREVE' +'\x0e AND DIAERESIS' +'\x14 AND DIAGONAL STROKE' +'\x0e AND DOT ABOVE' +'\x10 AND HEAVY RIGHT' +'\t AND HOOK' +'\x10 AND LIGHT RIGHT' +'\x0f AND LOWER LEFT' +'\x10 AND LOWER RIGHT' +'\x0b AND MACRON' +'\x11 AND MIDDLE TILDE' +'\x13 AND PROSGEGRAMMENI' +'\x06 AND S' +'\x15 AND SLANTED PARALLEL' +'\x0e AND SMALL TAH' +'\x12 AND SMASH PRODUCT' +'\t AND TAIL' +'\x15 AND THREE DOTS ABOVE' +'\x13 AND TWO DOTS ABOVE' +'\x12 AND YPOGEGRAMMENI' +'\x07 APLOUN' +'\t APODEXIA' +'\t ARCHAION' +'\x06 ARROW' +'\r B BAR SYMBOL' +'\x05 BACK' +'\n BACKSLASH' +'\x04 BAR' +'\r BEHIND CLOUD' +'\x06 BELOW' +'\x1c BETWEEN TWO HORIZONTAL BARS' +'\r BLACK SQUARE' +'\x06 BLOCK' +'\r BZHI MIG CAN' +'\x04 C D' +'\x05 CAPO' +'\x06 CARET' +'\x08 CEDILLA' +'\x07 CER-WA' +'\x07 CHIKI ' +'\x07 CIRCLE' +'\x05 CLEF' +'\x0f CLOUD AND RAIN' +'\x0f CORNER BRACKET' +'\x1f CROSSING ASH OVER ASH OVER ASH' +'\x0e CROSSING GABA' +'\x0c CROSSING GU' +'\r CROSSING KA2' +'\x08 CURRENT' +'\x05 DEKA' +'\x0f DEYTEROU ICHOU' +'\r DIFFERENTIAL' +'\x0c DIMINUTION-' +'\r DIMINUTION-1' +'\x06 DIPLI' +'\x1b DIVIDED BY HORIZONTAL RULE' +'\x05 DOTS' +'\t DRACHMAS' +'\n ENCLOSURE' +'\x06 ENTRY' +'\t EQUAL TO' +'\x07 EQUALS' +'\x06 EQUID' +'\x03 FA' +'\x05 FACE' +'\x07 FLEXUS' +'\t FORM TWO' +'\t FROM BAR' +'\n FROM WALL' +'\x1a GAD OVER GAD GAR OVER GAR' +'\r GAR OVER GAR' +'\t GARSHUNI' +'\x05 GORA' +'\x07 GROUND' +'\x05 GUNU' +'\x05 HAA ' +'\x08 HANDLES' +'\x06 HANDS' +'\x0e HASER FOR VAV' +'\x03 HE' +'\r HEADED ARROW' +'\x03 HU' +'\x08 HUNDRED' +'\x17 HUNDRED TWENTY-EIGHTH ' +'\x07 HYPHEN' +'\x05 ILUT' +'\x03 IN' +'\x13 IN A RECTANGLE BOX' +'\x0c IN TRIANGLE' +'\x10 INSERTION POINT' +'\x0e ISOLATED FORM' +'\x05 JERA' +'\x02 K' +'\n KABA TENU' +'\x06 KAPAL' +'\x13 KASKAL U GUNU DISH' +'\x05 KAWI' +'\x07 KEFULA' +'\x08 KEMBANG' +'\x0e KISIM5 TIMES ' +'\x05 LACA' +'\x11 LAGAB TIMES ASH2' +'\x11 LAGAR OVER LAGAR' +'\x04 LAI' +'\n LATE FORM' +'\x05 LEFT' +'\x06 LELET' +'\x06 LENGA' +'\x1a LESS THAN THE DENOMINATOR' +'\x05 LINE' +'\x0c LINE SYMBOL' +'\x06 LONGA' +'\x07 LONSUM' +'\x02 M' +'\n MAHAPRANA' +'\x05 MARK' +'\x05 MEGA' +'\x06 MELIK' +'\x07 MENDUT' +'\x07 MUQDAM' +'\x06 MURDA' +'\x1a NEGATED WITH VERTICAL BAR' +'\x04 NEO' +'\x04 NOT' +'\n NOTATION ' +'\n NOTEHEAD ' +'\x08 NUTILLU' +'\x03 OF' +'\x04 OF ' +'\n OF DHARMA' +'\x04 OHM' +'\x10 ON BLACK SQUARE' +'\x0c ON PEDESTAL' +'\t OPENING ' +'\t OPERATOR' +'\x0f OPERATOR WITH ' +'\r OPPOSING KUR' +'\x04 OR ' +'\x1a OR APPROXIMATELY EQUAL TO' +'\t OR EQUAL' +'\x0c OR EQUAL TO' +'\x10 OR THE IMAGE OF' +'\x08 OTTAVA ' +'\t OVER BAL' +'\x0b OVER BULUG' +'\x16 OVER GAD GAR OVER GAR' +') OVER HI TIMES ASH2 KU OVER HI TIMES ASH2' +'\x0b OVER IDIM ' +'\x12 OVER INVERTED SHU' +'\n OVER KAD5' +'\x08 OVER KG' +'\x0c OVER KISIM5' +'\x14 OVER LAGAR GUNU SHE' +'\t OVER LUM' +'\x07 OVER M' +'\x08 OVER MU' +'\t OVER NUN' +'\x19 OVER NUN LAGAR TIMES SAL' +'\x1d OVER RIGHTWARDS ARROW TO BAR' +'\x07 OVER S' +'\x0f OVER SIG4 SHU2' +'\x08 OVER SU' +'\x18 OVER TOP SQUARE BRACKET' +'\x14 OVER TUR ZA OVER ZA' +'\x0c OVER TWO PI' +'\x08 OVER ZI' +'\x02 P' +'\x0f PARESTIGMENON ' +'\x07 PLOPHU' +'\x06 PLUS ' +'\x07 PLUS A' +'\t PLUS ASH' +'\x08 PLUS DI' +'\x08 PLUS DU' +'\x08 PLUS EN' +'\t PLUS GUD' +'\x0f PLUS HI PLUS A' +'\t PLUS KU3' +'\x08 PLUS LA' +'\t PLUS LAL' +'\x08 PLUS LU' +'\x12 PLUS LU PLUS ESH2' +'\n PLUS MASH' +'\x0b PLUS MASH2' +'\x0b PLUS NUNUZ' +'\x08 PLUS RA' +'\x08 PLUS RU' +'\x11 PLUS SHA3 PLUS A' +'\n PLUS SHU2' +'\t PLUS SUM' +'\t PLUS TUR' +'\x0f POINTING INDEX' +'\x0b PROPORTION' +'\x11 PUNCTUATION MARK' +'\x06 QATAN' +'\x0f QUOTATION MARK' +'\x07 RAMBAT' +'\x08 RASWADI' +'\x05 REPA' +'\x05 REST' +'\n RESUPINUS' +'\n RGYA GRAM' +'\x04 RHO' +'\x06 RIGHT' +'\x08 ROTUNDA' +'\x03 SA' +'\x05 SARI' +'\n SCHROEDER' +'\n SEPARATOR' +'\x05 SHAD' +'\x0b SHOE STILE' +'\x05 SIGN' +'\x06 SIGN ' +'\x0c SKEWED LEFT' +'\r SKEWED RIGHT' +'\x08 SLIDING' +'\x07 SLOWLY' +'\x06 SMALL' +'\x10 SOLIDUS OVERLAY' +'\x08 SQUARED' +'\x08 STATERS' +'\x07 STRIDE' +'\x03 SU' +'\x07 SYMBOL' +'\x05 TACK' +'\x0b TACK BELOW' +'\x05 TAIL' +'\x08 TALENTS' +'\x07 TEDUNG' +'\x05 TELU' +'\x05 TENU' +'\t THOUSAND' +'\x0f THROUGH CIRCLE' +'\x12 THROUGH DESCENDER' +'\x07 TIMES ' +'\x08 TIMES A' +'\x10 TIMES A PLUS NA' +'\n TIMES BAD' +'\x08 TIMES E' +'\x0b TIMES ESH2' +'\t TIMES GA' +'\x10 TIMES GAN2 TENU' +'\x0f TIMES IGI GUNU' +'\r TIMES KASKAL' +'\n TIMES KUR' +'\n TIMES LAL' +'\n TIMES SHE' +'\x08 TIMES U' +'\x11 TO BLACK DIAMOND' +'\x06 TONE-' +'\t TROMIKON' +'\x08 TTUDDAG' +'" TWO DOTS OVER ONE DOT PUNCTUATION' +'\x02 U' +'\t UNDERBAR' +'\x03 UP' +'\x0b UPPER HALF' '\r VARIANT FORM' -'\x04NGUE' -'\x0cMADDA ABOVE ' -'\x0eCORNER BRACKET' -'\x03BEH' +'\x05 WITH' +'\x06 WITH ' +'\n WITH BASE' +'\x0b WITH CARON' +'\x0c WITH CIRCLE' +'\x12 WITH CIRCLE ABOVE' +'\x12 WITH CIRCLE BELOW' +'\x17 WITH CIRCUMFLEX ACCENT' +'\x0b WITH COMMA' +'\x12 WITH CROSSED-TAIL' +'\n WITH CURL' +'\x07 WITH D' +'\x0c WITH DAGESH' +'\x0b WITH DASIA' +'\x0f WITH DESCENDER' +'\x0f WITH DIAERESIS' +'\t WITH DOT' +'\n WITH DOT ' +'\x0f WITH DOT ABOVE' +'\x10 WITH DOT INSIDE' +'\n WITH DOTS' +'\x19 WITH DOUBLE GRAVE ACCENT' +'\x14 WITH FATHATAN ABOVE' +'\x0e WITH FISHHOOK' +'\n WITH HOOK' +'\x14 WITH HORIZONTAL BAR' +'$ WITH HORIZONTAL MIDDLE BLACK STRIPE' +'\r WITH JEGOGAN' +'\x0e WITH LONG LEG' +'\x0b WITH MAPIQ' +'\r WITH OVERBAR' +'\x12 WITH PALATAL HOOK' +'\n WITH PLUS' +'\x14 WITH RETROFLEX HOOK' +'\x1d WITH REVERSED NEGATION SLASH' +'\x07 WITH S' +'\x13 WITH STRIKETHROUGH' +'\x0c WITH STROKE' +'\n WITH TAIL' +'\r WITH TEE TOP' +'\n WITH TICK' +'\x11 WITH TILDE ABOVE' +'\x0b WITH TITLO' +'\x0e WITH UNDERBAR' +'\r WITHOUT SNOW' +'\x07 WOLOSO' +'\x01-' +'\x02-0' +'\x02-2' +'\x11-9 QUOTATION MARK' +'\x02-A' +'\x05-ALAF' +'\x05-ARY ' +'\x05-BEAM' +'\x0c-BREAK SPACE' +'\x10-BREAKING HYPHEN' +'\x0f-CARRIER LETTER' +'\x06-CREE ' +'\x07-CREE R' +'\x08-CREE TH' +'\x0e-CURRENCY SIGN' +'\x0c-DZUD RTAGS ' +'\r-ESASA DOTTED' +'\x0b-FEATHERED ' +'\x1b-FEATHERED RIGHTWARDS ARROW' +'\x11-GAAHLAA TTUDDAAG' +'\x08-HEADED ' +'\x1f-HEADED ARROW FROM SMALL CIRCLE' +'\x18-HEADED RIGHTWARDS ARROW' +'\x06-HIDET' +'\x06-HIEUH' +'\n-HIRAGANA ' +'\x1e-HIRAGANA PROLONGED SOUND MARK' +'\x02-I' +'\x06-IEUNG' +'\x07-INDIC ' +'\x08-KHIEUKH' +'\t-KHYUD PA' +'\x07-KIYEOK' +'\x0b-LINE STAFF' +'\n-LUE KARAN' +'\x07-MACRON' +'\x06-MINUS' +'\x0b-MINUS SIGN' +'\x02-O' +'\x04-ONE' +'\r-OR-PLUS SIGN' +'\r-PER-EM SPACE' +'\x08-PHIEUPH' +'\x06-PIEUP' +'\n-POINTING ' +'\x10-POINTING ANGLE ' +' -POINTING ANGLE BRACKET ORNAMENT' +'\x18-POINTING TRIANGLE WITH ' +'\x15-ROUND NOTEHEAD DOWN ' +'\x15-SHAPED BAG DELIMITER' +'\n-SHAPED HA' +'\x0c-SIMPLIFIED ' +'\x05-SIOS' +'\t-STACCATO' +'\x0e-STEM TONE BAR' +'\x05-TAIL' +'\x03-TE' +'\x07-TIKEUT' +'\x02-U' +'\x04-UM ' +'\x02-W' +'\x010' +'\x080 BRONZE' +'\x050 DWO' +'\x0b0 FOOTSTOOL' +'\x040 KO' +'\x040 MA' +'\x050 MAN' +'\x040 NI' +'\x050 OIL' +'\x040 PU' +'\x040 RA' +'\x070 SPEAR' +'\x030 U' +'\x070 WHEAT' +'\x110 WHEELED CHARIOT' +'\x040 WI' +'\x040 ZO' +'\x030-0' +'\x0207' +'\x0209' +'\x030B9' +'\x011' +'\x071 ARROW' +'\x081 BARLEY' +'\t1 CHARIOT' +'\x041 DA' +'\x041 DU' +'\x051 DWE' +'\x061 GOLD' +'\x081 HELMET' +'\x061 HORN' +'\x041 KU' +'\x031 O' +'\n1 PLASTICS' +'\x041 PO' +'\x041 QI' +'\x041 SA' +'\x041 SI' +'\x051 TWO' +'\x061 WINE' +'\x031-0' +'\x0211' +'\x03121' +'\x03155' +'\x0216' +'\x0218' +'\x0318D' +'\x021A' +'\x021D' +'\x012' +'\x0f2 CHARIOT FRAME' +'\t2 GARMENT' +'\x042 NO' +'\x072 OLIVE' +'\x042 PE' +'\n2 PLASTICS' +'\n2 PLUS ASH' +'\n2 PLUS GAL' +'\x0b2 PLUS GIR2' +'\x052 PTE' +'\x042 QO' +'\x042 RO' +'\x042 SO' +'\x042 WO' +'\x072 WOMAN' +'\x032-0' +'\x0220' +'\x0221' +'\x0222' +'\t247 DIPTE' +'\x03253' +'\x0229' +'\x022D' +'\x032DD' +'\x022F' +'\x013' +'\x043 A3' +'\x073 AREPA' +'\x083 ARMOUR' +'\x043 ME' +'\x043 MI' +'\x073 MONTH' +'\x043 MU' +'\x043 PA' +'\n3 PLASTICS' +'\x053 RA3' +'\x043 RI' +'\x073 SPICE' +'\x073 SWORD' +'\x073 WHEEL' +'\x033-0' +'\x03305' +'\x0234' +'\x0239' +'\x014' +'\x064 DART' +'\x064 DEER' +'\x044 DO' +'\x044 KE' +'\x044 NE' +'\n4 PLASTICS' +'\x044 TE' +'\x044 WA' +'\x044 ZE' +'\x034-0' +'\x0240' +'\x03439' +'\x024B' +'\x024D' +'\x024E' +'\x015' +'\x045 A2' +'\x045 AU' +'\t5 BATHTUB' +'\t5 CYPERUS' +'\x045 DE' +'\x045 JU' +'\x065 MERI' +'\x045 MO' +'\x045 NU' +'\n5 PLASTICS' +'\x045 TO' +'\x045 WE' +'\x065 WOOL' +'\x035-0' +'\x0250' +'\x0351F' +'\x0253' +'\x0255' +'\x03557' +'\x0256' +'\x0856 TURO2' +'\x016' +'\x046 JE' +'\x046 JO' +'\x046 NA' +'\n6 PLASTICS' +'\x046 QA' +'\x056 RA2' +'\x046 RU' +'\x056 TA2' +'\x066 TREE' +'\x036-0' +'\x03620' +'\x036D7' +'\x017' +'\x047 DI' +'\x047 JA' +'\x047 KA' +'\x067 KAPO' +'\x047 KI' +'\n7 PLASTICS' +'\x047 RE' +'\x047 TI' +'\x057 TWE' +'\x047 ZA' +'\x0270' +'\x0372C' +'\x0279' +'\x018' +'\x038 A' +'\x038 E' +'\x038 I' +'\x088 KANAKO' +'\x058 NWA' +'\x048 QE' +'\x058 RO2' +'\x048 SU' +'\x028C' +'\x038F0' +'\x019' +'\x079 CLOTH' +'\x049 PI' +'\x059 PU2' +'\x049 SE' +'\x049 TA' +'\x049 TU' +'\x04904A' +'\x0295' +'\x0299' +'\x039E3' +'\x01A' +'\x02A ' +'\x07A -PHRU' +'\x06A HAAM' +'\x04A IE' +'\x07A NAME ' +'\x11A PLUS HA PLUS DA' +'\nA PLUS IGI' +'\nA PLUS KUR' +'\nA TANG LAI' +'\x05A UNA' +'\x14A- SHOG GI MGO RGYAN' '\x04A-HA' -'\x03BEN' -'\x03BEE' -'\tCENT SIGN' -'\x03BET' -'\x18UPRIGHT RECTANGULAR ZERO' -'\x04BIRD' -'\x0fMARRYING MAIDEN' -'\x0bMED RGYINGS' -'\x05SHIMA' -'\x04EIRT' -'\nMINUS SIGN' -'\x08MPERSAND' -'\x03NOO' -'\x03TJE' -'\x04IGER' -'\x05HIEUH' -'\x04RITU' -'\x18DIAGONAL CROSSHATCH FILL' -'\nRVICE MARK' -'\x03NOW' -'\tLAMATION ' -'\x03ED ' -'\x041 DU' -'\x041 DA' -'\x07R ABOVE' -'\rOBLIQUE LINE ' -'\x0b TIMES ESH2' -'\x07AM ALEF' -'\x0bILCROW SIGN' -'\x06MEMBER' -'\x11NASALIZATION MARK' -'\x11BOTTOM HALF BLACK' -'\x04ARTH' -'\x08F SYMBOL' -'\x08TRICOLON' -'\x06INSHIP' -'\x05UNITY' -'\x08 KEMBANG' -'\x0295' -'\x15HORIZONTAL TABULATION' -'\tTWO DOTS ' -'\x06 AND S' -'\x0cCSIMILE SIGN' -'\x14HORIZONTAL BAR WITH ' -'\x06KHAH I' -'\x06TICAL ' -'\x04YEH ' -'\x06ENIKI ' -'\nUPPER HALF' -'\x0eLOWER DIAGONAL' -'\x04OWER' -'\x08VOCALIC ' -'\x04PEEP' -'\x02LF' -'\x02LD' -'\x02LE' -'\x02LC' -'\x02LA' -'\x02LO' -'\x02LL' -'\x02LM' -'\x16LONG HORIZONTAL STROKE' -'\x02LK' -'\x02LH' -'\x02LI' -'\x02LT' -'\x02LU' -'\x02LY' -'\x01A' -'\x04HEAD' -'\x02L ' -'\x1cBESIDE AND JOINED WITH UNION' -'\x16RECTILINEAR BLACK STAR' -'\x04HEAT' -'\x02L2' -'\x02L0' -'\x0fOUTH EAST ARROW' -'\x16DOMAIN ANTIRESTRICTION' -'\x1cRECT CURRENT SYMBOL FORM TWO' -'\x06LASTON' -'\x0bQUIQUADRATE' -'\x15DOWN HORIZONTAL HEAVY' -'\rSMALL V BELOW' -'\x0eINVERTED DAMMA' -'\x1bRIPLE VERTICAL BAR OPERATOR' -'\x0fRIGHT HALF RING' -'\x08REVERSE ' -'\x05BEITH' -'\x0bORCE SYMBOL' -'\x04SA-I' -'\x04NPEA' -'\x04OXIA' -'\rANGLE BRACKET' -'\x06M BOAR' -'\x08REVERSED' -'\x05URAMU' -'\nTEMPLATION' -'\x07DOTTED ' -'\x0bDOUBLE SHAD' -'\x05SUKUN' -'\x03APH' -'\x04HAM ' -'\x03APA' -'\x13DIAGONAL HALF BLACK' -'\n5 PLASTICS' -'\x11 AND MIDDLE TILDE' -'\rIMAGE BRACKET' -'\x05NINTH' -'\x08MARITAN ' -'\x05MROCK' -'\x14EVERSED ROTATED RANA' -'\x07UP MARK' -'\x06SPIRAL' -'\x04ODO ' -'\x07ARRED B' -'\x0bPUNCTUATION' -'\x07ARRED O' -'\x13PLACEMENT CHARACTER' -'\x043 RI' -'\x05MUOY ' -'\x05HEAD ' -'\x08AVY BAND' -'\x07R TSHES' -'\x12KBAR ISOLATED FORM' -'\x07KYLISMA' -'\rCROSSING MUSH' -'\x05CLICK' -'\x05FSAAQ' -'\x03YAT' -'\x06VIYANI' -'\x0bUNJO WYNN W' -'\x03YAA' -'\nUR CORNERS' -'\x05 REST' -'\tY-FOURTH ' -'\x10YATHOS BASE SIGN' -'\x06PENCIL' -'\nPLICATION ' -'\x07IVISION' -'\tFULL STOP' -'\x0cSUR OVER SUR' -'\tMMER AND ' -'\x1bHITE DOWN-POINTING TRIANGLE' -'\x11VOICED SOUND MARK' -'\x04GGWS' +'\x05A-ROL' +'\x02A0' +'\x02A2' +'\x02A4' +'\x02AA' +'\x05AADHU' +'\x03AAF' +'\x04AAFU' +'\x05AAMAE' +'\x04AAMU' +'\x03AAN' +'\x02AB' +'\nAB2 TIMES ' +'\x08ABAAFILI' +'\x11ABBREVIATION MARK' +'\x11ABBREVIATION SIGN' +'\x04ABIC' +'\x04ABLE' +'\tABLED CAR' +'\x08ABOAFILI' +'\x04ABOR' +'\x05ABOVE' +'\x06ABOVE ' +'\x11ABOVE EQUALS SIGN' +'\x12ABOVE GREATER-THAN' +'&ABOVE LEFTWARDS HARPOON WITH BARB DOWN' +'\x0fABOVE LESS-THAN' +"'ABOVE RIGHTWARDS HARPOON WITH BARB DOWN" +'\x15ABOVE SHORT DOWN TACK' +'\tABOVE TO ' +'\x02AC' +'\x05ACCAT' +'\x06ACCENT' +'\x07ACCENT ' +'\x03ACE' +'\tACE NOTE ' +'\nACE SYMBOL' +'\x03ACH' +'\x0fACHES THE LIMIT' +'\x03ACK' +'\nACK CIRCLE' +'\x0bACKNOWLEDGE' +'\x08ACKSPACE' +'\x05ACRON' +'\x03ACT' +'\x07ACTION ' +'\x05ACUTE' +'\x0cACUTE ACCENT' +'\x15ACUTE AND HOOK SYMBOL' +'\x0bACUTE-GRAVE' +'\x02AD' +'\x03AD3' +'\x03ADA' +'\nADAK BINDI' +'\x04ADDA' +'\nADDA WITH ' +'\x05ADDAK' +'\x03ADE' +'\tADEG ADEG' +'\x04ADHE' +'\x04ADHU' +'\x10ADIAN SYLLABICS ' +'\tADMA GDAN' +'\x03ADU' +'\x02AE' +'\nAEDA-PILLA' +'\x03AEF' +'\nAELA-PILLA' +'\x07AELAENG' +'\x03AEM' +'\x0bAESCULAPIUS' +'\x06AESURA' +'\x11AEUM ONE PLETHRON' +'\x02AF' +'\x08AF WITH ' +'\x02AG' +'\x03AG-' +'\x04AGAB' +'\x0bAGAZ DAEG D' +'\x06AGE OF' +'\x05AGGER' +'\x1eAGGRAVATED INDEPENDENT SVARITA' +'\x03AGL' +'\x05AGMA ' +'\tAGOLITIC ' +'\x04AGON' +'\x06AGRANT' +'\x02AH' +'\x03AH ' +'\x04AH I' +'\x08AH WITH ' +'\x04AHAD' +'\x07AHAPAKH' +'\x08AHAPRANA' +'\x02AI' +'\x0eAI LENGTH MARK' +'\tAIGHTNESS' +'\x03AIL' +'\nAILED BIRD' +'\x0eAILLE PATTERN ' +'\x0bAILLESS PHI' +'\x04AILM' +'\x06AILURE' +'\x03AIN' +'\x05AINS ' '\x06AISED ' -'\x1aDROP-SHADOWED WHITE SQUARE' -'\x0cRNAM BCAD MA' -'\x14DOWN ARROWHEAD BELOW' -'\n PLUS MASH' -'\x17 HUNDRED TWENTY-EIGHTH ' -'\x11INEAR ANNOTATION ' -'\x06ARTER ' -'\nCOMBINING ' -'\x0cLTA EQUAL TO' -'\x07JIBWAY ' -'\nOSSED SHEI' -'\x0bFGHANI SIGN' -'\x06 LONGA' -'\x06MILAR ' -'\tXHAUSTION' -'\x07OTATED ' -'\x07AXIMATA' -'\x0cNATURAL SIGN' -'\x07A NAME ' -'!LESS-THAN ABOVE DOUBLE-LINE EQUAL' -'\x08VINATION' -'\x1eSUPERSCRIPT ALEF ISOLATED FORM' -'\x05XTILE' -'\x01X' -'\rGHT HALF RING' -'\x04NYIS' -'\nALENT SIGN' -'\x03DMA' -'\tAP FILLER' -'\x12OW TONE APOSTROPHE' -'\tABLED CAR' -'\x10KA- SHOG YIG MGO' -'\x0fXCLAMATION MARK' -'\x0bOF MASHFAAT' -'\x06 BELOW' -'\x04GIDA' -'\x05GRAPH' -'\x02A2' -'\nMONOGRAM B' -'\x13TO LOWER RIGHT FILL' -'\x12NINETEEN FULL STOP' -'\x1bEAVY WHITE RIGHTWARDS ARROW' -'\nDOWN ARROW' -'\x0bPHNAEK MUAN' -'\x04KURO' -'\x0cNUITY SYMBOL' -'\tRAL HEART' -'\x06 LENGA' -'\x04NGSI' -'\x18GLOTTAL STOP WITH STROKE' -'\x07SVARITA' -'\x05EVEN ' -'\x08EGORIAN ' -'\x02A ' -'\x03253' -'\x06WIDTH ' -'\x0cN ELEMENT OF' -'\x0fDAGESH OR MAPIQ' -'\x08CANDICUS' -'\x0cTHODOX CROSS' -'\x10AM ISOLATED FORM' -'\x08ULL STOP' -'\x0eDEYTEROS ICHOS' -'\x04ZERO' -'!ALL BUT UPPER LEFT QUADRANT BLACK' -'\x05OSS O' -'\x04ERET' -'\x10VERTICALLY ABOVE' -'\x03NMA' -'\tHIYYAALAA' -'\x06LEMENT' -'\x05EVENT' -'\x13WITH YEH FINAL FORM' -'\x03LON' -'\x03LOO' -'\x03LOR' -'\x03LOS' -'\x18UPWARD POINTING TRIANGLE' -'\x03LOW' -'\x0bUZEIRO SIGN' -'\x05OKEN ' -'\x0bRDHAVISARGA' -'\x10NORTH WEST ARROW' -'\x0bGRAMMA SIGN' -'\x05EGIN ' -'\tEMBEDDING' -'\x13LEFT TO LOWER RIGHT' -'\x08EAR SIGN' -'\x05POLI ' -'\tAS MEMBER' -'\nFEATHERED ' -'\x08EBEEFILI' -'\tDOWNWARDS' -'\x08IS-PILLA' -'\x04COND' -'\x16C GREATER-THAN BRACKET' -'\x0bREAKTHROUGH' -'\x07THESEOS' -'\tISEN-ISEN' -'\tEDESTRIAN' -'\x11TRIPLE DASH ARROW' -'\x07TAYANNA' -'\nARANI SIGN' -'\x1eGREATER-THAN ABOVE EQUALS SIGN' -'\x0bAESCULAPIUS' -'\x04IFAT' -'\x17OVERLAPPING LOGICAL AND' -'\x0cROWNING FACE' -'\x05VARIA' -'\x08OT BELOW' -'\x08LI GALI ' -'\x05HANNA' -'\x08ISH TAIL' -'\x04DAGS' -'\x0bEPIDAUREAN ' -'\x0b WITH CARON' -'\x16I YFESIS TETARTIMORION' -'\x0bME LONG CAN' -'\n WITH TICK' -'\x07STATERS' -'\x02OU' -'\x10 TIMES A PLUS NA' -'\x07DIAMOND' -'\x03MPL' -' NORMAL FACTOR SEMIDIRECT PRODUCT' -'\x03MPI' -'\x03AVE' -'\x08 HANDLES' -'\x05LAYAR' -'\x046 QA' -'\x04ADHU' -'\x04ADHE' -'\x0cVIAN LETTER ' -'\x0fSPOKED ASTERISK' -'\x04UHUR' -'\x04BOVE' -'\tIDEWAYS U' -'\x04SUNG' -'\x05HOLE ' -'\x08NUS SIGN' -'\x04NAL ' -'\x0cCIRCLED PLUS' -'\x16INDIRECT QUESTION MARK' -'\x04ISI ' -'\x05EISMA' -'\x06N SIGN' -'\x043 PA' -'\x14 ABOVE LEFT TRIANGLE' -'\x06NOWMAN' -'\x03155' -'\tIALYTIKA ' -'\x05PRINT' -'\x07NANCIAL' -'\x08-PHIEUPH' -'\x05FLUTE' -'\x07HYAAUSH' -'\x0bONE HUNDRED' -'\x0bUNIT DIGIT ' -'\x01E' -'\x16WITH VOICED SOUND MARK' -'\nKFONITIKON' -'\x0bIN EQUAL TO' -'\x0bNYIS -KHYIL' -'\x05DALET' -'\x0cPAP PLUS PAP' -'\x0299' -'(VERTICAL BAR AT END OF HORIZONTAL STROKE' -'\x05AR AE' -'\x1aDOTTED SUBSTITUTION MARKER' -'\x06TAIKHU' -'\x04EFT ' -'\x0bDYO CHRONON' -'\x19GAD OVER GAD GAR OVER GAR' -'\x0fMALAKON CHROMA ' -'\x08TONE BAR' -'\x05OTHAL' -'\x1aCLOCKWISE ARROW WITH MINUS' -'\x07MBOL B0' -'\x07CK MARK' -'\nRENCY SIGN' -'\x04WEEN' -'\n1 PLASTICS' -'\x07HASANTA' -'\x14ENARMONIOS ANTIFONIA' -'\x10TODO SOFT HYPHEN' -'\tUPPER DOT' -'\x0fALAYHE ASSALLAM' -'\t OVER BAL' -'\x04RAM ' -'\x04WEET' -'\n AND ACUTE' -'\x06CREASE' -'\x11RYBLION BASE SIGN' -'\nOMMA BELOW' -'\x06ESTION' -'\x07MINGKAL' -'\x038F0' -'\x06HIEUCH' -'\x0fRMAN PENNY SIGN' -'\x02M ' -'\x07 APLOUN' -'\tLOWER DOT' -'\x02M0' -'\x07KAYANNA' -'\x04OUND' -'\x04JEEM' -'\x07TCHFORK' -'\rEVERSED DAMMA' -'\x05UCEUS' -'\x05UNION' -'\x05ZYGOS' -'\x08RYNGEAL ' -'\x14INVERTED CANDRABINDU' -'\rRRIAGE RETURN' -'\x08ANUDATTA' -'\x07APEZIUM' -'\x02ME' -'\x02MA' -'\x02MB' -'\x02MM' -'\x02MO' -'\x02MI' -'\x02MU' -'\x06EMOLO-' -'\x0bRYVNIA SIGN' -'\x06EDIUM ' -'\x02MY' -'\x02MX' -'\x04ILDE' -'\x03DON' -'\tNDRABINDU' -'\x18SMALL ARABIC LETTER TAH ' -'\x0fHORIZONTAL DASH' -'\x07PRENKHA' -'\n WITH DOTS' -'\x06YFESIS' -'\x04REAT' -'\x06OFOUND' -'\x03DOT' -'\x13UP HORIZONTAL LIGHT' -'\x0eRFACE INTEGRAL' -'\nLITTLE YUS' -'\x08ER TRUTH' -'\x0bOPPOSING IM' -'\x07ARDNESS' -'\x04VINE' -'\x07VAYANNA' -'\x08LESS SHA' -'\nHEAVY AND ' -'\n WITH DOT ' -'\x04INUS' -'\x03DO ' -'\x03BAU' -'\x03BAR' -'\x0bMUUSIKATOAN' -'\r-PER-EM SPACE' -'\x03BAD' -'\x03BAA' -'\x03BAN' -'\x04NEO ' -'\x1dEFTWARDS HARPOON WITH BARB UP' -'\x05CHULA' -'\nTERNATIVE ' -'\tTIMES BAD' -'\x06I MAIM' -'\x056 RA2' -'\x03STA' -'\x05ETORU' -'\x04SADI' -'\x03STU' -'\x04MNAS' -'\x04ZETA' -'\rLVE FULL STOP' -'\x05HAALU' -'\nF SHE-GOAT' -'\x10OREAN CHARACTER ' -'\x1cRECTANGULAR PATH AROUND POLE' -'\x0bREE HUNDRED' -'\x071 ARROW' -'\x07ALESMA ' -'\x05 AND ' -'\x0c OVER TWO PI' -'\x08AMS HORN' -'\x03ACK' -'\x06 ABOVE' -'\x04 NOT' -'\x0cVAKRAHASANYA' -'\x0eIWAZ TIR TYR T' -'\x0bTHIRDS SIGN' -' WO DOTS OVER ONE DOT PUNCTUATION' -'\x05FTEEN' -'\x04BASA' -'\x06ITABLE' -'\x04BASH' -'\x0fLMOST EQUAL TO ' -'\x03KAB' -'\x03KAF' -'\x03KAK' -'\x03KAL' -'\x13HORIZONTAL ELLIPSIS' -'\x03RHO' -'\tUMED HEAD' -'\x03KAY' -'\tINVERTED ' -'\rSIGN PAMUDPOD' -'\nRACTER TIE' -'\x04PEAT' -'\x044 ZE' -'\x07VAV YOD' -'\x05OWEL ' -'\x0cHASIS SYMBOL' -'\nGBAKURUNEN' -'\x08UATRILLO' -"'ABOVE RIGHTWARDS HARPOON WITH BARB DOWN" -'\x0fUPERSCRIPT ALEF' -'\nRHO SYMBOL' -'\x08OVER GI4' -'\t OVER NUN' -'\x1bTALIC LATIN CAPITAL LETTER ' -'\x1cEXTENDED ARABIC-INDIC DIGIT ' -'\x07DIGRAPH' -'\x05LAGAB' -'\x12TROFLEX HOOK BELOW' -'\x05HITE ' -'\x11HORIZONTAL STROKE' -'\x0eIVE OBOLS SIGN' -'\x03ATE' -'\x12ABOVE GREATER-THAN' -'\x03ATH' -'\x03ATI' -'\x0bYEORINHIEUH' -'\x05ERET ' -'\x07NSULAR ' -'\x04TTER' -'\x045 AU' -'\x05 BACK' -'\x04AAMU' -'\x012' -'\x06SHORT ' -'\x06SPATHI' -'\x045 A2' -'\x14HIGH TONE APOSTROPHE' -'\x05TILDE' -'\x04CART' -'\x08IAERESIS' -'\x07UKKAKHA' -'\x0fINUS SIGN BELOW' -'\x10DOUBLE DOT ABOVE' -'\x04RUTU' -'\x03AT ' -'\tLLABLE OM' -'\x11JEEM INITIAL FORM' -'\x08BEVERAGE' -'\x07ITALIC ' -'\x04MER ' -'\x06OYANNA' -'\tLOCKWISE ' -'\x10EPSILON UNDERBAR' -'\tCCER BALL' -'\x05XAGON' -'\x05NUKTA' -'\x04NANA' -'\x04LO L' -'\x1aALEF MAKSURA ISOLATED FORM' -'\x05IAUDA' -'\x04IK H' -'\x0bOLDING BACK' -'\x0bLENGTH MARK' -'\x05EAGLE' -'\x05TIMES' -'\x05BREW ' -'\x04AMMA' -'\x0eWITH DIAERESIS' -'\x06OR EQU' -'\x08CRO SIGN' -'\x1eMAKSURA WITH SUPERSCRIPT ALEF ' -'\x07SEXTILE' -'\x18BLACK LENTICULAR BRACKET' -'\x07 MUQDAM' -'\x0fPERTHO PEORTH P' -'\nHAAPRAANA ' -'\x12DOWN-OUTPUT SYMBOL' -'6UIGHUR KIRGHIZ YEH WITH HAMZA ABOVE WITH ALEF MAKSURA ' -'\x17ELVE POINTED BLACK STAR' -'\x08TAALUJA ' -'\x05JUEUI' -'\x08YPORROON' -'\tRIZONTAL ' -'\nENTHUSIASM' -'\x04IOTA' -'\x03XIA' -'\x03YEH' -'\tLESS-THAN' -'\x03YEE' -'\nTHMIKON N ' -'\tJERUSALEM' -'\x06IRCLE ' -'\x06M BULL' -'\x080 BRONZE' -'\x05LIDUS' -'\x0bMBA BAYANNA' -'\tZ DIGRAPH' -'\x06APYRUS' -'\x19PERFECTUM CUM PROLATIONE ' -'\x07OX LINE' -'\rCROSSING GAN2' -'\x07SYNAGMA' -'\x06EMKATH' -'\x13LUNATE SIGMA SYMBOL' -'\tFEH WITH ' -'\x06DIVIDE' -'\x18P DIRECTIONAL FORMATTING' -'\x08ONG DASH' -'\x0eAPITAL LETTER ' -'\x10SMALL CAPITAL EL' -'\x05 TACK' -'\x05RITSI' -'\x05ERIN2' -'\x05WEST ' -'\x04DIM2' -'\x07CULAR O' -'\rFINAL LETTER ' -'\x05NCHU ' -'\x0cIDENTICAL TO' -'\x16PERSET OF NOR EQUAL TO' -'\x04MTI ' -'\n3 PLASTICS' -'\rTIRTA TUMETES' -'\x059 PU2' -'\x06IX OF ' -'\x03RUE' -'\x11 LAGAR OVER LAGAR' -'\x06C CLEF' -'\x03JHA' -'\x03DIM' -'\x03DIN' -'\x0bNUN HAFUKHA' -'\x03DIB' -'\x0eMENSION ORIGIN' -'\x06UNTAIN' -'\x06PERSET' -'\x07COPTIC ' -'\x11DENTAL PERCUSSIVE' -'\x11ESTERN PWO KAREN ' -'\x08OR EQUAL' -'\x1bIMPERFECTUM CUM PROLATIONE ' -'\x03UTE' -'\x01I' -'\x03IRT' -'\x03IRU' -'\tNTERPRISE' -'\x03IRI' -'\x1bKATHAKA INDEPENDENT SVARITA' -'\x05TYR T' -'\x03IRO' -'\tNOT EQUAL' -'\x07STIMATE' -'\x03IRC' -'\x03IRE' -'\nNTITY MARK' -'\rWO DOTS BELOW' -'\x0bBUT NOT EQU' -'\x03PWA' -'\x03BOX' -'\x03NAH' -'\x05NATE ' -'\x06EIGHTY' -'\x03NAR' -'\x06VIRIAM' -'\rU LENGTH MARK' -'\x06EIGHTH' -'\x0cOPPOSING LU2' -'\nR2 PLUS SU' -'\nTIMES SIGN' -'\x14RIGHT QUADRANT BLACK' -'\x07VE DOT ' -'\x0e WITH UNDERBAR' -'\x05LAMDA' -'\nWHITE STAR' -'\x11 ALTERNATION MARK' -'\x07OX BACK' -'\rTROFLEX CLICK' +'\tAISED DOT' +'\x16AISED OMISSION BRACKET' +'\x06AITING' +'\x08AIYANNOI' +'\tAJANYALAN' +'\x02AK' +'\x06AKABAT' +'\x03AKE' +'\x15AKIA TELOUS ICHIMATOS' '\nAKKHANGYAO' -'\x06EIGHT ' -'\rEMPHATIC TONE' -'\x11OUBLE MONOCULAR O' -'\x07OMERANG' -'\rUSICAL LEIMMA' -'\x0fRISING DIAGONAL' -'\x05REVIA' -'\x0bBETA SYMBOL' -'\x06ISSHAR' -'\x06SQUARE' -'\tMHANCHOLL' -'\x05OPBAR' -'\x05ASKAL' -'\x08 ROTUNDA' -'\x08VE SASAK' -'\x03LK ' -'\tTHOUSAND ' -'\x0cATTOOED HEAD' -'\x07JECTIVE' -'\x0cEIGHTH BLOCK' -'\x1cASTERISKS ALIGNED VERTICALLY' -'\x07ELLIPSE' -'\nOON LILITH' -'\nLOGICAL OR' -'\x08 TIMES E' -'\rMILLIONS SIGN' -'\x044 DO' -'\rONGRATULATION' -'\x06PIRIT ' -'\x02NN' -'\x02NO' -'\x02NA' -'\x02ND' -'\x02NE' -'\x02NG' -'\x02NY' -'\tIRCUMFLEX' -'\x02NT' -'\x02NU' -'\x02NW' -'\x0fFOUR DOTS ABOVE' -'\x02N ' -'\x06IASMA ' -'\x04RTAR' -'\x07S ABOVE' -'\x06BALUDA' -'\x05PONSE' -'\x03OZE' -'\x06YMBOL ' -'\x0bUTH CORNERS' -'\x05MALL ' -'\x0eERIC INDICATOR' -'\r DIFFERENTIAL' -'\x07VILIK B' -'\x06YLISMA' -'\x07OVER AN' -'\x07VERLINE' -'\x03AZU' -'\tSIXTEENTH' -'\tCOMPLETED' -'\x04RUNA' -'\x07F DAVID' -'\x04JONG' -'\x04RUNG' -'\x08OVERRIDE' -'\x10DOWN RIGHT BARB ' -'\x05ENANO' -'\x0eOVER GUD LUGAL' -'\x042 SO' -'\rVERSE DIVIDER' -'\x12UTLINED BLACK STAR' -'\x05KASRA' -'\x0cMBELLISHMENT' -'\tNCLOSING ' -'\x07MINIMA ' -'\x07SHORT A' -'\rEAVENLY EARTH' -'\x0cEMISOFT SIGN' -'\x043 MI' -'\nAILED BIRD' -'\x10INES CONVERGING ' -'\x08WASH KAF' -'\x17VERTICAL BISECTING LINE' -'\x0cREPONDERANCE' -'\x05EGL H' -'\x06INESE ' -'\x06TIKEUT' -'\x07BINING ' -'\nIVALENT TO' -'\x06OICING' -'\x04QEF ' -'\x06RIPLE ' -'\x07MARCATO' -'\nSE-CREE SK' -'\rBY DEFINITION' -'\x04PACE' -'\x070 WHEAT' -'\x04LOOR' -'\x04LOOP' -'\x0cLUS NOTEHEAD' -'\x05UTIES' -'\x05BLANK' -'\x05MURDA' -'\x18BERKANAN BEORC BJARKAN B' -'\x04SHAR' -'\x04APON' -'\x07UM IYEK' -'\x04SHAD' -'\x04SHAN' -'\x04SHA3' -'\nKAI SYMBOL' -'\x0eQUARTERS BLOCK' -'\x13STRATIAN FIFTY MNAS' -'\tACE NOTE ' -'\x07RAYANNA' -'\x14UBLE VERTICAL STROKE' -'\x08I SHAKTI' -'#TWO HORIZONTAL STROKES TO THE RIGHT' -'\rPAIRED ARROWS' -'\x0bDONG TSHUGS' -'\x07 HYPHEN' -'\x10PRECHGESANG STEM' -'\x06WN BOW' -'\x03VE ' -'\x016' -'\x06KIYEOK' -'\x07ESCENDO' -'\x05KAPPA' -'\x03VEE' -'\x07UP AND ' -'\x03VEN' -'\x0cCHAIR SYMBOL' -'\x0c OR EQUAL TO' -'\x0fBINARY RELATION' -'\x06DOKMAI' -'\x03Q00' -'\x04AILM' -'\x0bCARET TILDE' -'\x050 MAN' -'\x041 QI' -'\x04OUBT' -'\tSSANGSIOS' -'\tHARACTERS' -'\x06PARKLE' -'\tHARACTER ' -'\x04SKAL' -'\x05F SOW' -'\x07BAIRKAN' -'\x08OSTROPHE' -'\x07BARLINE' -'\x06ORETTE' -'\x03IP ' -'\x05VER S' -'\x04IXTY' -'\x04AH I' -'\x1dDOTS OVER ONE DOT PUNCTUATION' -'\x04IXTH' -'\x03UR-' -'\x08RDEL NAG' -'\x0bBLACK ARROW' -'\x073 AREPA' -'\x03URE' -'\x03URA' -'\x03URL' -'\x03URN' -'\x1cMONOGRAMMOS TESSERA DODEKATA' -'\x03URT' -'\x03URU' -'\rSTERTIUS SIGN' -'\x0e KISIM5 TIMES ' -'\x0eUNDER RELATION' -'\x03NSU' -'\x04REE ' -'\n-LUE KARAN' -'\x0bENTHESIZED ' -'\x03NGA' -'\x04IGMA' -'\x08LA LENGA' -'\x18MALL CIRCLE TO THE RIGHT' -'\x12VARIANT FORM LIMMU' -'\x03SH2' -'\x06DOTTED' -'\x088 KANAKO' -'\nTOM SYMBOL' -'\x03SH ' -'\x08SPERSION' -'\x04 ALL' -'\x05THIRD' -'\x04GIBA' -'\nLET SYMBOL' -'\rFULL SURROUND' -'\n TIMES LAL' -'\x18RIGHTWARDS THEN CURVING ' -'\x03SHA' -'\x0cBLACK SQUARE' -'\x08PRECEDES' -'\n KABA TENU' -'\x14JUDEO-SPANISH VARIKA' -'\x03NG ' -'\x11MALL CIRCLE ABOVE' -'\x03 AN' -'\x03LUS' -'\x03LUB' -'\nHEAVY BEAT' -'\x07CISSORS' -'\x04YIS ' -'\x03LUL' -'\x03LUM' -'\x07PTHAHA ' -'\x16UBSCRIPT SMALL LETTER ' -'\x05SASAK' -'\x06MALL A' -'\x06MALL F' -'\x05WIANG' -'\x06VIRAMA' -'\rARGOSYNTHETON' -'\x05HISTI' -'\x0bPARENTHESIS' -'\x04GEDE' -'\nCISIVENESS' -'\x0e WITH FISHHOOK' -'\x04OTE ' -'\x03YA ' -'\nRIGINAL OF' -'\x07NIKAHIT' -'\x03KET' -'\x0eWITH DOT BELOW' -'\x051 DWE' -'\x03OT ' -'\nGANDA MARK' -'\x04ABIC' -'\x05CECEK' -'\x07OTLESS ' -'\x05-ARY ' -'\x07ALLOT X' -'\x0bLEFT SINGLE' -'\x04TRIA' -'\x13ASOUL ISOLATED FORM' -'\x01M' -'\x06HESHIG' -'\x0eWORD SEPARATOR' -'\x03OTH' -'\x03OTO' -'\x03OTA' -'\x07THIRDS ' -'\x06 SMALL' -'\rSMALL V ABOVE' -'\x05TUUMU' -'\x05EPEAT' -'\x0cLACK DIAMOND' -'\x04IBE ' -'\rPREPONDERANCE' -'\x08 PLUS EN' -'\x0fLEVEN FULL STOP' -'\x06ERMATA' -'\x04KEMP' -'\x0bPARALLEL TO' -'\x12LENTICULAR BRAKCET' -'\x13 AND TWO DOTS ABOVE' -'\x1c BETWEEN TWO HORIZONTAL BARS' -'\x042 QO' -'\x08ION SIGN' -'\x08SSESSION' -'\x15LATIN CAPITAL LETTER ' -'\x03MZA' -'\x06CIRCLE' -'\x05TTED ' -'\x11OINTED BLACK STAR' -'\x13EFT-POINTING ANGLE ' -'\x0bSSANGKIYEOK' -'\tXIMATELY ' -'\x12WESTERN PWO KAREN ' -'\x13 KASKAL U GUNU DISH' -'\x08PALOCHKA' -'\x14 ABOVE SLANTED EQUAL' -'\x05TTORU' -'\x1eOPEN CENTRE EIGHT POINTED STAR' -'\x12 WITH CIRCLE BELOW' -'\x0bRANSMISSION' -'\x06NACLES' -'\x11VEN EIGHTHS BLOCK' -'\x03ZOO' -'\t AND HOOK' -'\x08ALE SIGN' -'\x04WAW ' -'\x15VERTICAL LINE OVERLAY' -'\nGREE SLASH' -'\x0eED PARAGRAPHOS' -'\x07DLE DOT' -'\nEONGCHIEUM' -'\x06UZEIRO' -'\t-KHYUD PA' -'\n-HIRAGANA ' -'\x05AADHU' -'\x0bETEI MAYEK ' -'\x05TACK ' -'\x06BELOW ' -'\x04GOAL' -'\r OPPOSING KUR' -'\x05NUENG' -'\x05SIGN ' -'\x0bNITIAL FORM' -'\x06LAMEDH' -'\x08MON TIME' -'\x0cVERTICAL BAR' -'\tING ABOVE' -'\x08OBOOFILI' -'\rIVE SLOW SIGN' -'\x05XTRA-' -'\t PLUS KU3' -'\x07LETTER ' -'\x04UND ' -'\tINE BELOW' -'\x13NGLE QUOTATION MARK' -'\x07NUTILLU' -'\x06OCENCE' -'\x05QUILL' -'\x15LESS-THAN OR EQUAL TO' -'\x05INIMA' -'\x03FWA' -'\x02O ' -'\x041 SI' -'\x04IFTY' -'\x041 SA' -'\x07OUR OF ' -'\tVER LUGAL' -'\x02O0' -'\x07L SEGNO' -'\x06 SIGN ' -'\x05GMENT' -'\x06YBOARD' -'\x06BEFILI' -'\x13ND SOUTH WEST ARROW' -'\tIN SQUARE' -'\nOUTER JOIN' -'\x0eGYPTOLOGICAL A' -'\x04GHWA' -'\x15ALATALIZED HOOK BELOW' -'\x13TRICTED LEFT ENTRY-' -'\x070 SPEAR' -'\x16HEXIFORM LONG ANUSVARA' -'\x02OO' -'\x02ON' -'\x02OM' -'\x02OL' -'\x02OK' -'\x13NAXIAN FIVE HUNDRED' -'\x02OH' -'\x02OG' -'\x02OF' -'\x02OE' -'\x04ALES' -'\n RGYA GRAM' -'\x02OY' -'\x03RQA' -'\x02OW' -'\x04ALEF' -'\x02OT' -'\x02OS' -'\x02OR' -'\x02OQ' -'\x02OP' -'\tNIGGAHITA' -'\x04INOR' -'\rGTER TSHEG MA' -'\x04KANG' -'\x03DES' -'\x05GHULU' -'\x08REE MARK' -'\x0cNCK CONSTANT' -'\x08VOLUTION' -'\x03DEK' -'\x06ILLAGE' -'\x03DEE' -'\x19LEFTWARDS ARROW WITH HOOK' -'\x08UP HEAVY' -'\x0eDASIA PNEUMATA' -'\x15RISING DIAGONAL SLASH' -'\x03UP ' -'!WARDS HARPOON WITH BARB DOWNWARDS' -'\tDEOGRAPH ' -'\x0fPERIAL ARAMAIC ' -'\x06LEADER' -'\x04HETH' -'\x04HETA' -'\x0cOID NOTEHEAD' -'\x0fBELGTHOR SYMBOL' -'\x03IVE' -'\x12UNATE SIGMA SYMBOL' -'\x15QUADRANT CIRCULAR ARC' -'\x03DE6' -'\x1cUPWARDS AND RIGHTWARDS ARROW' -'\x19FFICULTY AT THE BEGINNING' -'\x03NEO' -'\x1cUP HEAVY AND LEFT DOWN LIGHT' -'\x03ACE' -'\x053 RA3' -'\x03ACH' -'\x0fQUADRUPLE DASH ' -'\nLGIZ EOLHX' -'\x06INKING' -'\x03ACT' -'\x0cRIANGLE DOWN' -'\x04RING' -'\x03NER' -'\x11ABBREVIATION SIGN' -' ETALLED BLACK AND WHITE FLORETTE' -'\tMALL RING' -'\x0fTILDE DIAERESIS' -'\x03K00' -'\x07PENGKAL' -'\x04QUAD' -'\x05VZMET' -'\x19RIGHT DIAGONAL HALF BLACK' -'\x03SJE' -'\x17HANKED RIGHTWARDS ARROW' -'\x04HIRD' -'\x07RTHIAN ' -'\x03MEM' -'\x06WITH T' -'\x03MEN' -'\nINAL SIGMA' -'\x03NE ' -'\x04RIN2' -'\x04HIRT' -'\x064 DEER' -'\x0eMOTHETIC ABOVE' -'\x05ANGLE' -'\x0cRCHAIC SAMPI' -'\x07EL PUMP' -'\x05ENENG' -'\x06TIKENO' -'\x07ZQAPHA ' -'\tYEH ABOVE' -'\x08OT ABOVE' -'\x06APISMA' -'\tLINE FEED' -'\x043 MU' -'\x04HSDA' -'\x043 ME' -'\x03EZH' -'\x03EZI' -'\x0bSHEQEL SIGN' -'\x1cEFT ARC GREATER-THAN BRACKET' -'\nENETRATION' -'\x05ORNER' -'\n WITH HOOK' -'\x15INTERSECTION OPERATOR' -'\x0bAGAZ DAEG D' -'\x04CWAA' -'\x0eVARIANT FORM A' -'\tLEFT HOOK' -'\x04MONO' -'\x04ABOR' -'\x0bROKUTASTI A' -'\x04ARB ' -'\x0cUP ARROWHEAD' -'\x06RGLASS' -'\x05CREEN' -'\x04MON ' -'\x08NEIFORM ' -'\x05LEFT ' -'\x05A-ROL' -'\x0fQUADRUPLE ARROW' -'\x07PHUTHAO' -'\x08NG RTAGS' -'\x07 TEDUNG' -'\x0cNUMERAL SIGN' -'\x03OVE' -'\x06TO BAR' -'\x11TAN ISOLATED FORM' -'\x03D70' -'\x07BOTTOM ' -'\x05VRIDO' -'\x06MEDIUM' -'\x07APYEOUN' -'\x04 OF ' -'\x04QOPH' -'\x0fOGOGRAM KHAMTI ' -'\rW RING INSIDE' -'\x06EQUAL ' -'\x07IRGHIZ ' -'\x06NOR BU' -'\nMONOSPACE ' -'\x06EQUALS' -'\x14DOTLESS HEAD OF KHAH' -'\x042 WO' -'\tYRANISMA ' -'\x08ATAKANA ' -'\x0cYOUTHFULNESS' -'\nLOWER HOOK' -'\nCROPHONIC ' -'\x05HEART' -'\x03DUG' -'\x05POINT' -'\x06KASKAL' -'\x02YA' -'\x04NCEL' -'\x04AMLA' -'\x10RST QUARTER MOON' -'\rGYA GRAM SHAD' -'\x07RESILLO' -'\x10FALLING DIAGONAL' -'\x12 THROUGH DESCENDER' -'\x08THAMASAT' -'\x14SINGLE DOT TONE MARK' -'\x08 STATERS' -'\x05LABAT' -'\x08LUB SUIT' -'\x06VILIAN' -'\x06DERMA ' -'\x05BASA ' -'\x10PLE MEASURE REST' -'\x02YR' -'\x13ANTICLOCKWISE ARROW' -'\x0fNOON FINAL FORM' -'\x0fWITH RAIN DROPS' -'\x19STROKE AND TWO DOTS ABOVE' -'\x05 CLEF' -'\tAND ACUTE' -'\x04OMBU' -'\x01Q' -'\x0fR WITH FISHHOOK' -'\x11AEUM ONE PLETHRON' -'\x04USA ' -'\x06SWORDS' -'\x10TELOUS ICHIMATOS' -'\rUSHING UPWARD' -'\nEARLY FORM' -'\x06HIEUKH' -'\x12MINAL DIGIT SHAPES' -'\t ARCHAION' -'\x04MMA ' -'\rARLAUG SYMBOL' -'\x08TISMOS E' -'\x04SHE3' -'\x13TART OF RUB EL HIZB' -'\x17ININE ORDINAL INDICATOR' -'\x08GRAPH UK' -'\x15 AND SLANTED PARALLEL' -'\x04SHEI' -'\x11Y ON BLACK SQUARE' -'\x0cMALL LETTER ' -'\x03VIN' -'\x03VIR' -'\x03VIS' -'\x08ART SUIT' -'\x04STAR' -'\rUPONDIUS SIGN' -'\x0fHALF FILL SPACE' -'\x04FOUR' -'\x0fPLUS SIGN BELOW' -'+UPPER RIGHT-SHADOWED WHITE RIGHTWARDS ARROW' -'$LEFT TORTOISE SHELL BRACKET ORNAMENT' -'\x06ROBANG' -'\x05UDAAT' -'\nAELA-PILLA' -'\x0bCROSSING EN' -'\x05AGGER' -'\x17BALLOON-SPOKED ASTERISK' -'\x0fRNAMENT STROKE-' -'\x0fLIGHT AND LEFT ' -'\tSPIRITUS ' -'\nETTA-PILLA' -'\x08TAI LUE ' -'\x06FF OF ' -'\x05TIGMA' -'\x03IT ' -'\x04KAPA' -'\x10ENTRE WHITE STAR' -'\x04KAPH' -'\x08SIFISTON' -'\x05ZENGE' -'\x0fRIANGULAR COLON' -'\tBACKSLASH' -'\x0eTWO DOT LEADER' -'\x06CHAMKO' -'\x06HE MGO' -'\x08 OTTAVA ' -'\x1aRIGHT PARENTHESIS ORNAMENT' -'\x03UNT' -'\x06EDIAL ' -'\x03UNG' -'\x0bTHREE TIMES' -'\x04HERU' -'\x0bPARAKALESMA' -'\x0fAND FEMALE SIGN' -'\x03ITE' -'\x03ITA' -'\x03OMU' -'\x03ITY' -'\x03ITU' -'\x16USTOMER ACCOUNT NUMBER' -'\x03AAF' From noreply at buildbot.pypy.org Mon Jan 28 09:18:31 2013 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 28 Jan 2013 09:18:31 +0100 (CET) Subject: [pypy-commit] pypy default: Add .isprintable(), .isxidstart() and .xisxidcontinue() to rlib.unicodedata. Message-ID: <20130128081831.271831C12FA@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: Changeset: r60578:af806070c203 Date: 2013-01-27 23:42 +0100 http://bitbucket.org/pypy/pypy/changeset/af806070c203/ Log: Add .isprintable(), .isxidstart() and .xisxidcontinue() to rlib.unicodedata. These functions are used by Python3. diff too long, truncating to 2000 out of 16450 lines diff --git a/rpython/rlib/unicodedata/DerivedCoreProperties-3.2.0.txt b/rpython/rlib/unicodedata/DerivedCoreProperties-3.2.0.txt new file mode 100644 --- /dev/null +++ b/rpython/rlib/unicodedata/DerivedCoreProperties-3.2.0.txt @@ -0,0 +1,4218 @@ +# DerivedCoreProperties-3.2.0.txt +# Date: 2002-03-19,23:30:42 GMT [MD] +# +# Unicode Character Database: Derived Property Data +# Generated algorithmically from the Unicode Character Database +# For documentation, see DerivedProperties.html +# Note: Unassigned and Noncharacter codepoints are omitted, +# except when listing Noncharacter or Cn. +# ================================================ + +# ================================================ + +# Derived Property: Math +# Generated from: Sm + Other_Math + +0028 ; Math # Ps LEFT PARENTHESIS +0029 ; Math # Pe RIGHT PARENTHESIS +002A ; Math # Po ASTERISK +002B ; Math # Sm PLUS SIGN +002D ; Math # Pd HYPHEN-MINUS +002F ; Math # Po SOLIDUS +003C..003E ; Math # Sm [3] LESS-THAN SIGN..GREATER-THAN SIGN +005B ; Math # Ps LEFT SQUARE BRACKET +005C ; Math # Po REVERSE SOLIDUS +005D ; Math # Pe RIGHT SQUARE BRACKET +005E ; Math # Sk CIRCUMFLEX ACCENT +007B ; Math # Ps LEFT CURLY BRACKET +007C ; Math # Sm VERTICAL LINE +007D ; Math # Pe RIGHT CURLY BRACKET +007E ; Math # Sm TILDE +00AC ; Math # Sm NOT SIGN +00B1 ; Math # Sm PLUS-MINUS SIGN +00D7 ; Math # Sm MULTIPLICATION SIGN +00F7 ; Math # Sm DIVISION SIGN +03F6 ; Math # Sm GREEK REVERSED LUNATE EPSILON SYMBOL +2016 ; Math # Po DOUBLE VERTICAL LINE +2032..2034 ; Math # Po [3] PRIME..TRIPLE PRIME +2044 ; Math # Sm FRACTION SLASH +2052 ; Math # Sm COMMERCIAL MINUS SIGN +207A..207C ; Math # Sm [3] SUPERSCRIPT PLUS SIGN..SUPERSCRIPT EQUALS SIGN +207D ; Math # Ps SUPERSCRIPT LEFT PARENTHESIS +207E ; Math # Pe SUPERSCRIPT RIGHT PARENTHESIS +208A..208C ; Math # Sm [3] SUBSCRIPT PLUS SIGN..SUBSCRIPT EQUALS SIGN +208D ; Math # Ps SUBSCRIPT LEFT PARENTHESIS +208E ; Math # Pe SUBSCRIPT RIGHT PARENTHESIS +20D0..20DC ; Math # Mn [13] COMBINING LEFT HARPOON ABOVE..COMBINING FOUR DOTS ABOVE +20E1 ; Math # Mn COMBINING LEFT RIGHT ARROW ABOVE +2102 ; Math # L& DOUBLE-STRUCK CAPITAL C +210A..2113 ; Math # L& [10] SCRIPT SMALL G..SCRIPT SMALL L +2115 ; Math # L& DOUBLE-STRUCK CAPITAL N +2119..211D ; Math # L& [5] DOUBLE-STRUCK CAPITAL P..DOUBLE-STRUCK CAPITAL R +2124 ; Math # L& DOUBLE-STRUCK CAPITAL Z +2128 ; Math # L& BLACK-LETTER CAPITAL Z +212C..212D ; Math # L& [2] SCRIPT CAPITAL B..BLACK-LETTER CAPITAL C +212F..2131 ; Math # L& [3] SCRIPT SMALL E..SCRIPT CAPITAL F +2133..2134 ; Math # L& [2] SCRIPT CAPITAL M..SCRIPT SMALL O +2135..2138 ; Math # Lo [4] ALEF SYMBOL..DALET SYMBOL +2140..2144 ; Math # Sm [5] DOUBLE-STRUCK N-ARY SUMMATION..TURNED SANS-SERIF CAPITAL Y +214B ; Math # Sm TURNED AMPERSAND +2190..2194 ; Math # Sm [5] LEFTWARDS ARROW..LEFT RIGHT ARROW +219A..219B ; Math # Sm [2] LEFTWARDS ARROW WITH STROKE..RIGHTWARDS ARROW WITH STROKE +21A0 ; Math # Sm RIGHTWARDS TWO HEADED ARROW +21A3 ; Math # Sm RIGHTWARDS ARROW WITH TAIL +21A6 ; Math # Sm RIGHTWARDS ARROW FROM BAR +21AE ; Math # Sm LEFT RIGHT ARROW WITH STROKE +21CE..21CF ; Math # Sm [2] LEFT RIGHT DOUBLE ARROW WITH STROKE..RIGHTWARDS DOUBLE ARROW WITH STROKE +21D2 ; Math # Sm RIGHTWARDS DOUBLE ARROW +21D4 ; Math # Sm LEFT RIGHT DOUBLE ARROW +21F4..22FF ; Math # Sm [268] RIGHT ARROW WITH SMALL CIRCLE..Z NOTATION BAG MEMBERSHIP +2308..230B ; Math # Sm [4] LEFT CEILING..RIGHT FLOOR +2320..2321 ; Math # Sm [2] TOP HALF INTEGRAL..BOTTOM HALF INTEGRAL +237C ; Math # Sm RIGHT ANGLE WITH DOWNWARDS ZIGZAG ARROW +239B..23B3 ; Math # Sm [25] LEFT PARENTHESIS UPPER HOOK..SUMMATION BOTTOM +25B7 ; Math # Sm WHITE RIGHT-POINTING TRIANGLE +25C1 ; Math # Sm WHITE LEFT-POINTING TRIANGLE +25F8..25FF ; Math # Sm [8] UPPER LEFT TRIANGLE..LOWER RIGHT TRIANGLE +266F ; Math # Sm MUSIC SHARP SIGN +27D0..27E5 ; Math # Sm [22] WHITE DIAMOND WITH CENTRED DOT..WHITE SQUARE WITH RIGHTWARDS TICK +27F0..27FF ; Math # Sm [16] UPWARDS QUADRUPLE ARROW..LONG RIGHTWARDS SQUIGGLE ARROW +2900..2982 ; Math # Sm [131] RIGHTWARDS TWO-HEADED ARROW WITH VERTICAL STROKE..Z NOTATION TYPE COLON +2999..29D7 ; Math # Sm [63] DOTTED FENCE..BLACK HOURGLASS +29DC..29FB ; Math # Sm [32] INCOMPLETE INFINITY..TRIPLE PLUS +29FE..2AFF ; Math # Sm [258] TINY..N-ARY WHITE VERTICAL BAR +FB29 ; Math # Sm HEBREW LETTER ALTERNATIVE PLUS SIGN +FE35 ; Math # Ps PRESENTATION FORM FOR VERTICAL LEFT PARENTHESIS +FE36 ; Math # Pe PRESENTATION FORM FOR VERTICAL RIGHT PARENTHESIS +FE37 ; Math # Ps PRESENTATION FORM FOR VERTICAL LEFT CURLY BRACKET +FE38 ; Math # Pe PRESENTATION FORM FOR VERTICAL RIGHT CURLY BRACKET +FE59 ; Math # Ps SMALL LEFT PARENTHESIS +FE5A ; Math # Pe SMALL RIGHT PARENTHESIS +FE5B ; Math # Ps SMALL LEFT CURLY BRACKET +FE5C ; Math # Pe SMALL RIGHT CURLY BRACKET +FE61 ; Math # Po SMALL ASTERISK +FE62 ; Math # Sm SMALL PLUS SIGN +FE63 ; Math # Pd SMALL HYPHEN-MINUS +FE64..FE66 ; Math # Sm [3] SMALL LESS-THAN SIGN..SMALL EQUALS SIGN +FE68 ; Math # Po SMALL REVERSE SOLIDUS +FF08 ; Math # Ps FULLWIDTH LEFT PARENTHESIS +FF09 ; Math # Pe FULLWIDTH RIGHT PARENTHESIS +FF0A ; Math # Po FULLWIDTH ASTERISK +FF0B ; Math # Sm FULLWIDTH PLUS SIGN +FF0D ; Math # Pd FULLWIDTH HYPHEN-MINUS +FF0F ; Math # Po FULLWIDTH SOLIDUS +FF1C..FF1E ; Math # Sm [3] FULLWIDTH LESS-THAN SIGN..FULLWIDTH GREATER-THAN SIGN +FF3B ; Math # Ps FULLWIDTH LEFT SQUARE BRACKET +FF3C ; Math # Po FULLWIDTH REVERSE SOLIDUS +FF3D ; Math # Pe FULLWIDTH RIGHT SQUARE BRACKET +FF3E ; Math # Sk FULLWIDTH CIRCUMFLEX ACCENT +FF5B ; Math # Ps FULLWIDTH LEFT CURLY BRACKET +FF5C ; Math # Sm FULLWIDTH VERTICAL LINE +FF5D ; Math # Pe FULLWIDTH RIGHT CURLY BRACKET +FF5E ; Math # Sm FULLWIDTH TILDE +FFE2 ; Math # Sm FULLWIDTH NOT SIGN +FFE9..FFEC ; Math # Sm [4] HALFWIDTH LEFTWARDS ARROW..HALFWIDTH DOWNWARDS ARROW +1D400..1D454 ; Math # L& [85] MATHEMATICAL BOLD CAPITAL A..MATHEMATICAL ITALIC SMALL G +1D456..1D49C ; Math # L& [71] MATHEMATICAL ITALIC SMALL I..MATHEMATICAL SCRIPT CAPITAL A +1D49E..1D49F ; Math # L& [2] MATHEMATICAL SCRIPT CAPITAL C..MATHEMATICAL SCRIPT CAPITAL D +1D4A2 ; Math # L& MATHEMATICAL SCRIPT CAPITAL G +1D4A5..1D4A6 ; Math # L& [2] MATHEMATICAL SCRIPT CAPITAL J..MATHEMATICAL SCRIPT CAPITAL K +1D4A9..1D4AC ; Math # L& [4] MATHEMATICAL SCRIPT CAPITAL N..MATHEMATICAL SCRIPT CAPITAL Q +1D4AE..1D4B9 ; Math # L& [12] MATHEMATICAL SCRIPT CAPITAL S..MATHEMATICAL SCRIPT SMALL D +1D4BB ; Math # L& MATHEMATICAL SCRIPT SMALL F +1D4BD..1D4C0 ; Math # L& [4] MATHEMATICAL SCRIPT SMALL H..MATHEMATICAL SCRIPT SMALL K +1D4C2..1D4C3 ; Math # L& [2] MATHEMATICAL SCRIPT SMALL M..MATHEMATICAL SCRIPT SMALL N +1D4C5..1D505 ; Math # L& [65] MATHEMATICAL SCRIPT SMALL P..MATHEMATICAL FRAKTUR CAPITAL B +1D507..1D50A ; Math # L& [4] MATHEMATICAL FRAKTUR CAPITAL D..MATHEMATICAL FRAKTUR CAPITAL G +1D50D..1D514 ; Math # L& [8] MATHEMATICAL FRAKTUR CAPITAL J..MATHEMATICAL FRAKTUR CAPITAL Q +1D516..1D51C ; Math # L& [7] MATHEMATICAL FRAKTUR CAPITAL S..MATHEMATICAL FRAKTUR CAPITAL Y +1D51E..1D539 ; Math # L& [28] MATHEMATICAL FRAKTUR SMALL A..MATHEMATICAL DOUBLE-STRUCK CAPITAL B +1D53B..1D53E ; Math # L& [4] MATHEMATICAL DOUBLE-STRUCK CAPITAL D..MATHEMATICAL DOUBLE-STRUCK CAPITAL G +1D540..1D544 ; Math # L& [5] MATHEMATICAL DOUBLE-STRUCK CAPITAL I..MATHEMATICAL DOUBLE-STRUCK CAPITAL M +1D546 ; Math # L& MATHEMATICAL DOUBLE-STRUCK CAPITAL O +1D54A..1D550 ; Math # L& [7] MATHEMATICAL DOUBLE-STRUCK CAPITAL S..MATHEMATICAL DOUBLE-STRUCK CAPITAL Y +1D552..1D6A3 ; Math # L& [338] MATHEMATICAL DOUBLE-STRUCK SMALL A..MATHEMATICAL MONOSPACE SMALL Z +1D6A8..1D6C0 ; Math # L& [25] MATHEMATICAL BOLD CAPITAL ALPHA..MATHEMATICAL BOLD CAPITAL OMEGA +1D6C1 ; Math # Sm MATHEMATICAL BOLD NABLA +1D6C2..1D6DA ; Math # L& [25] MATHEMATICAL BOLD SMALL ALPHA..MATHEMATICAL BOLD SMALL OMEGA +1D6DB ; Math # Sm MATHEMATICAL BOLD PARTIAL DIFFERENTIAL +1D6DC..1D6FA ; Math # L& [31] MATHEMATICAL BOLD EPSILON SYMBOL..MATHEMATICAL ITALIC CAPITAL OMEGA +1D6FB ; Math # Sm MATHEMATICAL ITALIC NABLA +1D6FC..1D714 ; Math # L& [25] MATHEMATICAL ITALIC SMALL ALPHA..MATHEMATICAL ITALIC SMALL OMEGA +1D715 ; Math # Sm MATHEMATICAL ITALIC PARTIAL DIFFERENTIAL +1D716..1D734 ; Math # L& [31] MATHEMATICAL ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD ITALIC CAPITAL OMEGA +1D735 ; Math # Sm MATHEMATICAL BOLD ITALIC NABLA +1D736..1D74E ; Math # L& [25] MATHEMATICAL BOLD ITALIC SMALL ALPHA..MATHEMATICAL BOLD ITALIC SMALL OMEGA +1D74F ; Math # Sm MATHEMATICAL BOLD ITALIC PARTIAL DIFFERENTIAL +1D750..1D76E ; Math # L& [31] MATHEMATICAL BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD CAPITAL OMEGA +1D76F ; Math # Sm MATHEMATICAL SANS-SERIF BOLD NABLA +1D770..1D788 ; Math # L& [25] MATHEMATICAL SANS-SERIF BOLD SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD SMALL OMEGA +1D789 ; Math # Sm MATHEMATICAL SANS-SERIF BOLD PARTIAL DIFFERENTIAL +1D78A..1D7A8 ; Math # L& [31] MATHEMATICAL SANS-SERIF BOLD EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL OMEGA +1D7A9 ; Math # Sm MATHEMATICAL SANS-SERIF BOLD ITALIC NABLA +1D7AA..1D7C2 ; Math # L& [25] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL OMEGA +1D7C3 ; Math # Sm MATHEMATICAL SANS-SERIF BOLD ITALIC PARTIAL DIFFERENTIAL +1D7C4..1D7C9 ; Math # L& [6] MATHEMATICAL SANS-SERIF BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC PI SYMBOL +1D7CE..1D7FF ; Math # Nd [50] MATHEMATICAL BOLD DIGIT ZERO..MATHEMATICAL MONOSPACE DIGIT NINE + +# Total code points: 1965 + +# ================================================ + +# Derived Property: Alphabetic +# Generated from: Lu+Ll+Lt+Lm+Lo+Nl + Other_Alphabetic + +0041..005A ; Alphabetic # L& [26] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER Z +0061..007A ; Alphabetic # L& [26] LATIN SMALL LETTER A..LATIN SMALL LETTER Z +00AA ; Alphabetic # L& FEMININE ORDINAL INDICATOR +00B5 ; Alphabetic # L& MICRO SIGN +00BA ; Alphabetic # L& MASCULINE ORDINAL INDICATOR +00C0..00D6 ; Alphabetic # L& [23] LATIN CAPITAL LETTER A WITH GRAVE..LATIN CAPITAL LETTER O WITH DIAERESIS +00D8..00F6 ; Alphabetic # L& [31] LATIN CAPITAL LETTER O WITH STROKE..LATIN SMALL LETTER O WITH DIAERESIS +00F8..01BA ; Alphabetic # L& [195] LATIN SMALL LETTER O WITH STROKE..LATIN SMALL LETTER EZH WITH TAIL +01BB ; Alphabetic # Lo LATIN LETTER TWO WITH STROKE +01BC..01BF ; Alphabetic # L& [4] LATIN CAPITAL LETTER TONE FIVE..LATIN LETTER WYNN +01C0..01C3 ; Alphabetic # Lo [4] LATIN LETTER DENTAL CLICK..LATIN LETTER RETROFLEX CLICK +01C4..0220 ; Alphabetic # L& [93] LATIN CAPITAL LETTER DZ WITH CARON..LATIN CAPITAL LETTER N WITH LONG RIGHT LEG +0222..0233 ; Alphabetic # L& [18] LATIN CAPITAL LETTER OU..LATIN SMALL LETTER Y WITH MACRON +0250..02AD ; Alphabetic # L& [94] LATIN SMALL LETTER TURNED A..LATIN LETTER BIDENTAL PERCUSSIVE +02B0..02B8 ; Alphabetic # Lm [9] MODIFIER LETTER SMALL H..MODIFIER LETTER SMALL Y +02BB..02C1 ; Alphabetic # Lm [7] MODIFIER LETTER TURNED COMMA..MODIFIER LETTER REVERSED GLOTTAL STOP +02D0..02D1 ; Alphabetic # Lm [2] MODIFIER LETTER TRIANGULAR COLON..MODIFIER LETTER HALF TRIANGULAR COLON +02E0..02E4 ; Alphabetic # Lm [5] MODIFIER LETTER SMALL GAMMA..MODIFIER LETTER SMALL REVERSED GLOTTAL STOP +02EE ; Alphabetic # Lm MODIFIER LETTER DOUBLE APOSTROPHE +0345 ; Alphabetic # Mn COMBINING GREEK YPOGEGRAMMENI +037A ; Alphabetic # Lm GREEK YPOGEGRAMMENI +0386 ; Alphabetic # L& GREEK CAPITAL LETTER ALPHA WITH TONOS +0388..038A ; Alphabetic # L& [3] GREEK CAPITAL LETTER EPSILON WITH TONOS..GREEK CAPITAL LETTER IOTA WITH TONOS +038C ; Alphabetic # L& GREEK CAPITAL LETTER OMICRON WITH TONOS +038E..03A1 ; Alphabetic # L& [20] GREEK CAPITAL LETTER UPSILON WITH TONOS..GREEK CAPITAL LETTER RHO +03A3..03CE ; Alphabetic # L& [44] GREEK CAPITAL LETTER SIGMA..GREEK SMALL LETTER OMEGA WITH TONOS +03D0..03F5 ; Alphabetic # L& [38] GREEK BETA SYMBOL..GREEK LUNATE EPSILON SYMBOL +0400..0481 ; Alphabetic # L& [130] CYRILLIC CAPITAL LETTER IE WITH GRAVE..CYRILLIC SMALL LETTER KOPPA +048A..04CE ; Alphabetic # L& [69] CYRILLIC CAPITAL LETTER SHORT I WITH TAIL..CYRILLIC SMALL LETTER EM WITH TAIL +04D0..04F5 ; Alphabetic # L& [38] CYRILLIC CAPITAL LETTER A WITH BREVE..CYRILLIC SMALL LETTER CHE WITH DIAERESIS +04F8..04F9 ; Alphabetic # L& [2] CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS..CYRILLIC SMALL LETTER YERU WITH DIAERESIS +0500..050F ; Alphabetic # L& [16] CYRILLIC CAPITAL LETTER KOMI DE..CYRILLIC SMALL LETTER KOMI TJE +0531..0556 ; Alphabetic # L& [38] ARMENIAN CAPITAL LETTER AYB..ARMENIAN CAPITAL LETTER FEH +0559 ; Alphabetic # Lm ARMENIAN MODIFIER LETTER LEFT HALF RING +0561..0587 ; Alphabetic # L& [39] ARMENIAN SMALL LETTER AYB..ARMENIAN SMALL LIGATURE ECH YIWN +05B0..05B9 ; Alphabetic # Mn [10] HEBREW POINT SHEVA..HEBREW POINT HOLAM +05BB..05BD ; Alphabetic # Mn [3] HEBREW POINT QUBUTS..HEBREW POINT METEG +05BF ; Alphabetic # Mn HEBREW POINT RAFE +05C1..05C2 ; Alphabetic # Mn [2] HEBREW POINT SHIN DOT..HEBREW POINT SIN DOT +05C4 ; Alphabetic # Mn HEBREW MARK UPPER DOT +05D0..05EA ; Alphabetic # Lo [27] HEBREW LETTER ALEF..HEBREW LETTER TAV +05F0..05F2 ; Alphabetic # Lo [3] HEBREW LIGATURE YIDDISH DOUBLE VAV..HEBREW LIGATURE YIDDISH DOUBLE YOD +0621..063A ; Alphabetic # Lo [26] ARABIC LETTER HAMZA..ARABIC LETTER GHAIN +0640 ; Alphabetic # Lm ARABIC TATWEEL +0641..064A ; Alphabetic # Lo [10] ARABIC LETTER FEH..ARABIC LETTER YEH +064B..0655 ; Alphabetic # Mn [11] ARABIC FATHATAN..ARABIC HAMZA BELOW +066E..066F ; Alphabetic # Lo [2] ARABIC LETTER DOTLESS BEH..ARABIC LETTER DOTLESS QAF +0670 ; Alphabetic # Mn ARABIC LETTER SUPERSCRIPT ALEF +0671..06D3 ; Alphabetic # Lo [99] ARABIC LETTER ALEF WASLA..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE +06D5 ; Alphabetic # Lo ARABIC LETTER AE +06D6..06DC ; Alphabetic # Mn [7] ARABIC SMALL HIGH LIGATURE SAD WITH LAM WITH ALEF MAKSURA..ARABIC SMALL HIGH SEEN +06E1..06E4 ; Alphabetic # Mn [4] ARABIC SMALL HIGH DOTLESS HEAD OF KHAH..ARABIC SMALL HIGH MADDA +06E5..06E6 ; Alphabetic # Lm [2] ARABIC SMALL WAW..ARABIC SMALL YEH +06E7..06E8 ; Alphabetic # Mn [2] ARABIC SMALL HIGH YEH..ARABIC SMALL HIGH NOON +06ED ; Alphabetic # Mn ARABIC SMALL LOW MEEM +06FA..06FC ; Alphabetic # Lo [3] ARABIC LETTER SHEEN WITH DOT BELOW..ARABIC LETTER GHAIN WITH DOT BELOW +0710 ; Alphabetic # Lo SYRIAC LETTER ALAPH +0711 ; Alphabetic # Mn SYRIAC LETTER SUPERSCRIPT ALAPH +0712..072C ; Alphabetic # Lo [27] SYRIAC LETTER BETH..SYRIAC LETTER TAW +0730..073F ; Alphabetic # Mn [16] SYRIAC PTHAHA ABOVE..SYRIAC RWAHA +0780..07A5 ; Alphabetic # Lo [38] THAANA LETTER HAA..THAANA LETTER WAAVU +07A6..07B0 ; Alphabetic # Mn [11] THAANA ABAFILI..THAANA SUKUN +07B1 ; Alphabetic # Lo THAANA LETTER NAA +0901..0902 ; Alphabetic # Mn [2] DEVANAGARI SIGN CANDRABINDU..DEVANAGARI SIGN ANUSVARA +0903 ; Alphabetic # Mc DEVANAGARI SIGN VISARGA +0905..0939 ; Alphabetic # Lo [53] DEVANAGARI LETTER A..DEVANAGARI LETTER HA +093D ; Alphabetic # Lo DEVANAGARI SIGN AVAGRAHA +093E..0940 ; Alphabetic # Mc [3] DEVANAGARI VOWEL SIGN AA..DEVANAGARI VOWEL SIGN II +0941..0948 ; Alphabetic # Mn [8] DEVANAGARI VOWEL SIGN U..DEVANAGARI VOWEL SIGN AI +0949..094C ; Alphabetic # Mc [4] DEVANAGARI VOWEL SIGN CANDRA O..DEVANAGARI VOWEL SIGN AU +0950 ; Alphabetic # Lo DEVANAGARI OM +0958..0961 ; Alphabetic # Lo [10] DEVANAGARI LETTER QA..DEVANAGARI LETTER VOCALIC LL +0962..0963 ; Alphabetic # Mn [2] DEVANAGARI VOWEL SIGN VOCALIC L..DEVANAGARI VOWEL SIGN VOCALIC LL +0981 ; Alphabetic # Mn BENGALI SIGN CANDRABINDU +0982..0983 ; Alphabetic # Mc [2] BENGALI SIGN ANUSVARA..BENGALI SIGN VISARGA +0985..098C ; Alphabetic # Lo [8] BENGALI LETTER A..BENGALI LETTER VOCALIC L +098F..0990 ; Alphabetic # Lo [2] BENGALI LETTER E..BENGALI LETTER AI +0993..09A8 ; Alphabetic # Lo [22] BENGALI LETTER O..BENGALI LETTER NA +09AA..09B0 ; Alphabetic # Lo [7] BENGALI LETTER PA..BENGALI LETTER RA +09B2 ; Alphabetic # Lo BENGALI LETTER LA +09B6..09B9 ; Alphabetic # Lo [4] BENGALI LETTER SHA..BENGALI LETTER HA +09BE..09C0 ; Alphabetic # Mc [3] BENGALI VOWEL SIGN AA..BENGALI VOWEL SIGN II +09C1..09C4 ; Alphabetic # Mn [4] BENGALI VOWEL SIGN U..BENGALI VOWEL SIGN VOCALIC RR +09C7..09C8 ; Alphabetic # Mc [2] BENGALI VOWEL SIGN E..BENGALI VOWEL SIGN AI +09CB..09CC ; Alphabetic # Mc [2] BENGALI VOWEL SIGN O..BENGALI VOWEL SIGN AU +09D7 ; Alphabetic # Mc BENGALI AU LENGTH MARK +09DC..09DD ; Alphabetic # Lo [2] BENGALI LETTER RRA..BENGALI LETTER RHA +09DF..09E1 ; Alphabetic # Lo [3] BENGALI LETTER YYA..BENGALI LETTER VOCALIC LL +09E2..09E3 ; Alphabetic # Mn [2] BENGALI VOWEL SIGN VOCALIC L..BENGALI VOWEL SIGN VOCALIC LL +09F0..09F1 ; Alphabetic # Lo [2] BENGALI LETTER RA WITH MIDDLE DIAGONAL..BENGALI LETTER RA WITH LOWER DIAGONAL +0A02 ; Alphabetic # Mn GURMUKHI SIGN BINDI +0A05..0A0A ; Alphabetic # Lo [6] GURMUKHI LETTER A..GURMUKHI LETTER UU +0A0F..0A10 ; Alphabetic # Lo [2] GURMUKHI LETTER EE..GURMUKHI LETTER AI +0A13..0A28 ; Alphabetic # Lo [22] GURMUKHI LETTER OO..GURMUKHI LETTER NA +0A2A..0A30 ; Alphabetic # Lo [7] GURMUKHI LETTER PA..GURMUKHI LETTER RA +0A32..0A33 ; Alphabetic # Lo [2] GURMUKHI LETTER LA..GURMUKHI LETTER LLA +0A35..0A36 ; Alphabetic # Lo [2] GURMUKHI LETTER VA..GURMUKHI LETTER SHA +0A38..0A39 ; Alphabetic # Lo [2] GURMUKHI LETTER SA..GURMUKHI LETTER HA +0A3E..0A40 ; Alphabetic # Mc [3] GURMUKHI VOWEL SIGN AA..GURMUKHI VOWEL SIGN II +0A41..0A42 ; Alphabetic # Mn [2] GURMUKHI VOWEL SIGN U..GURMUKHI VOWEL SIGN UU +0A47..0A48 ; Alphabetic # Mn [2] GURMUKHI VOWEL SIGN EE..GURMUKHI VOWEL SIGN AI +0A4B..0A4C ; Alphabetic # Mn [2] GURMUKHI VOWEL SIGN OO..GURMUKHI VOWEL SIGN AU +0A59..0A5C ; Alphabetic # Lo [4] GURMUKHI LETTER KHHA..GURMUKHI LETTER RRA +0A5E ; Alphabetic # Lo GURMUKHI LETTER FA +0A70..0A71 ; Alphabetic # Mn [2] GURMUKHI TIPPI..GURMUKHI ADDAK +0A72..0A74 ; Alphabetic # Lo [3] GURMUKHI IRI..GURMUKHI EK ONKAR +0A81..0A82 ; Alphabetic # Mn [2] GUJARATI SIGN CANDRABINDU..GUJARATI SIGN ANUSVARA +0A83 ; Alphabetic # Mc GUJARATI SIGN VISARGA +0A85..0A8B ; Alphabetic # Lo [7] GUJARATI LETTER A..GUJARATI LETTER VOCALIC R +0A8D ; Alphabetic # Lo GUJARATI VOWEL CANDRA E +0A8F..0A91 ; Alphabetic # Lo [3] GUJARATI LETTER E..GUJARATI VOWEL CANDRA O +0A93..0AA8 ; Alphabetic # Lo [22] GUJARATI LETTER O..GUJARATI LETTER NA +0AAA..0AB0 ; Alphabetic # Lo [7] GUJARATI LETTER PA..GUJARATI LETTER RA +0AB2..0AB3 ; Alphabetic # Lo [2] GUJARATI LETTER LA..GUJARATI LETTER LLA +0AB5..0AB9 ; Alphabetic # Lo [5] GUJARATI LETTER VA..GUJARATI LETTER HA +0ABD ; Alphabetic # Lo GUJARATI SIGN AVAGRAHA +0ABE..0AC0 ; Alphabetic # Mc [3] GUJARATI VOWEL SIGN AA..GUJARATI VOWEL SIGN II +0AC1..0AC5 ; Alphabetic # Mn [5] GUJARATI VOWEL SIGN U..GUJARATI VOWEL SIGN CANDRA E +0AC7..0AC8 ; Alphabetic # Mn [2] GUJARATI VOWEL SIGN E..GUJARATI VOWEL SIGN AI +0AC9 ; Alphabetic # Mc GUJARATI VOWEL SIGN CANDRA O +0ACB..0ACC ; Alphabetic # Mc [2] GUJARATI VOWEL SIGN O..GUJARATI VOWEL SIGN AU +0AD0 ; Alphabetic # Lo GUJARATI OM +0AE0 ; Alphabetic # Lo GUJARATI LETTER VOCALIC RR +0B01 ; Alphabetic # Mn ORIYA SIGN CANDRABINDU +0B02..0B03 ; Alphabetic # Mc [2] ORIYA SIGN ANUSVARA..ORIYA SIGN VISARGA +0B05..0B0C ; Alphabetic # Lo [8] ORIYA LETTER A..ORIYA LETTER VOCALIC L +0B0F..0B10 ; Alphabetic # Lo [2] ORIYA LETTER E..ORIYA LETTER AI +0B13..0B28 ; Alphabetic # Lo [22] ORIYA LETTER O..ORIYA LETTER NA +0B2A..0B30 ; Alphabetic # Lo [7] ORIYA LETTER PA..ORIYA LETTER RA +0B32..0B33 ; Alphabetic # Lo [2] ORIYA LETTER LA..ORIYA LETTER LLA +0B36..0B39 ; Alphabetic # Lo [4] ORIYA LETTER SHA..ORIYA LETTER HA +0B3D ; Alphabetic # Lo ORIYA SIGN AVAGRAHA +0B3E ; Alphabetic # Mc ORIYA VOWEL SIGN AA +0B3F ; Alphabetic # Mn ORIYA VOWEL SIGN I +0B40 ; Alphabetic # Mc ORIYA VOWEL SIGN II +0B41..0B43 ; Alphabetic # Mn [3] ORIYA VOWEL SIGN U..ORIYA VOWEL SIGN VOCALIC R +0B47..0B48 ; Alphabetic # Mc [2] ORIYA VOWEL SIGN E..ORIYA VOWEL SIGN AI +0B4B..0B4C ; Alphabetic # Mc [2] ORIYA VOWEL SIGN O..ORIYA VOWEL SIGN AU +0B56 ; Alphabetic # Mn ORIYA AI LENGTH MARK +0B57 ; Alphabetic # Mc ORIYA AU LENGTH MARK +0B5C..0B5D ; Alphabetic # Lo [2] ORIYA LETTER RRA..ORIYA LETTER RHA +0B5F..0B61 ; Alphabetic # Lo [3] ORIYA LETTER YYA..ORIYA LETTER VOCALIC LL +0B82 ; Alphabetic # Mn TAMIL SIGN ANUSVARA +0B83 ; Alphabetic # Lo TAMIL SIGN VISARGA +0B85..0B8A ; Alphabetic # Lo [6] TAMIL LETTER A..TAMIL LETTER UU +0B8E..0B90 ; Alphabetic # Lo [3] TAMIL LETTER E..TAMIL LETTER AI +0B92..0B95 ; Alphabetic # Lo [4] TAMIL LETTER O..TAMIL LETTER KA +0B99..0B9A ; Alphabetic # Lo [2] TAMIL LETTER NGA..TAMIL LETTER CA +0B9C ; Alphabetic # Lo TAMIL LETTER JA +0B9E..0B9F ; Alphabetic # Lo [2] TAMIL LETTER NYA..TAMIL LETTER TTA +0BA3..0BA4 ; Alphabetic # Lo [2] TAMIL LETTER NNA..TAMIL LETTER TA +0BA8..0BAA ; Alphabetic # Lo [3] TAMIL LETTER NA..TAMIL LETTER PA +0BAE..0BB5 ; Alphabetic # Lo [8] TAMIL LETTER MA..TAMIL LETTER VA +0BB7..0BB9 ; Alphabetic # Lo [3] TAMIL LETTER SSA..TAMIL LETTER HA +0BBE..0BBF ; Alphabetic # Mc [2] TAMIL VOWEL SIGN AA..TAMIL VOWEL SIGN I +0BC0 ; Alphabetic # Mn TAMIL VOWEL SIGN II +0BC1..0BC2 ; Alphabetic # Mc [2] TAMIL VOWEL SIGN U..TAMIL VOWEL SIGN UU +0BC6..0BC8 ; Alphabetic # Mc [3] TAMIL VOWEL SIGN E..TAMIL VOWEL SIGN AI +0BCA..0BCC ; Alphabetic # Mc [3] TAMIL VOWEL SIGN O..TAMIL VOWEL SIGN AU +0BD7 ; Alphabetic # Mc TAMIL AU LENGTH MARK +0C01..0C03 ; Alphabetic # Mc [3] TELUGU SIGN CANDRABINDU..TELUGU SIGN VISARGA +0C05..0C0C ; Alphabetic # Lo [8] TELUGU LETTER A..TELUGU LETTER VOCALIC L +0C0E..0C10 ; Alphabetic # Lo [3] TELUGU LETTER E..TELUGU LETTER AI +0C12..0C28 ; Alphabetic # Lo [23] TELUGU LETTER O..TELUGU LETTER NA +0C2A..0C33 ; Alphabetic # Lo [10] TELUGU LETTER PA..TELUGU LETTER LLA +0C35..0C39 ; Alphabetic # Lo [5] TELUGU LETTER VA..TELUGU LETTER HA +0C3E..0C40 ; Alphabetic # Mn [3] TELUGU VOWEL SIGN AA..TELUGU VOWEL SIGN II +0C41..0C44 ; Alphabetic # Mc [4] TELUGU VOWEL SIGN U..TELUGU VOWEL SIGN VOCALIC RR +0C46..0C48 ; Alphabetic # Mn [3] TELUGU VOWEL SIGN E..TELUGU VOWEL SIGN AI +0C4A..0C4C ; Alphabetic # Mn [3] TELUGU VOWEL SIGN O..TELUGU VOWEL SIGN AU +0C55..0C56 ; Alphabetic # Mn [2] TELUGU LENGTH MARK..TELUGU AI LENGTH MARK +0C60..0C61 ; Alphabetic # Lo [2] TELUGU LETTER VOCALIC RR..TELUGU LETTER VOCALIC LL +0C82..0C83 ; Alphabetic # Mc [2] KANNADA SIGN ANUSVARA..KANNADA SIGN VISARGA +0C85..0C8C ; Alphabetic # Lo [8] KANNADA LETTER A..KANNADA LETTER VOCALIC L +0C8E..0C90 ; Alphabetic # Lo [3] KANNADA LETTER E..KANNADA LETTER AI +0C92..0CA8 ; Alphabetic # Lo [23] KANNADA LETTER O..KANNADA LETTER NA +0CAA..0CB3 ; Alphabetic # Lo [10] KANNADA LETTER PA..KANNADA LETTER LLA +0CB5..0CB9 ; Alphabetic # Lo [5] KANNADA LETTER VA..KANNADA LETTER HA +0CBE ; Alphabetic # Mc KANNADA VOWEL SIGN AA +0CBF ; Alphabetic # Mn KANNADA VOWEL SIGN I +0CC0..0CC4 ; Alphabetic # Mc [5] KANNADA VOWEL SIGN II..KANNADA VOWEL SIGN VOCALIC RR +0CC6 ; Alphabetic # Mn KANNADA VOWEL SIGN E +0CC7..0CC8 ; Alphabetic # Mc [2] KANNADA VOWEL SIGN EE..KANNADA VOWEL SIGN AI +0CCA..0CCB ; Alphabetic # Mc [2] KANNADA VOWEL SIGN O..KANNADA VOWEL SIGN OO +0CCC ; Alphabetic # Mn KANNADA VOWEL SIGN AU +0CD5..0CD6 ; Alphabetic # Mc [2] KANNADA LENGTH MARK..KANNADA AI LENGTH MARK +0CDE ; Alphabetic # Lo KANNADA LETTER FA +0CE0..0CE1 ; Alphabetic # Lo [2] KANNADA LETTER VOCALIC RR..KANNADA LETTER VOCALIC LL +0D02..0D03 ; Alphabetic # Mc [2] MALAYALAM SIGN ANUSVARA..MALAYALAM SIGN VISARGA +0D05..0D0C ; Alphabetic # Lo [8] MALAYALAM LETTER A..MALAYALAM LETTER VOCALIC L +0D0E..0D10 ; Alphabetic # Lo [3] MALAYALAM LETTER E..MALAYALAM LETTER AI +0D12..0D28 ; Alphabetic # Lo [23] MALAYALAM LETTER O..MALAYALAM LETTER NA +0D2A..0D39 ; Alphabetic # Lo [16] MALAYALAM LETTER PA..MALAYALAM LETTER HA +0D3E..0D40 ; Alphabetic # Mc [3] MALAYALAM VOWEL SIGN AA..MALAYALAM VOWEL SIGN II +0D41..0D43 ; Alphabetic # Mn [3] MALAYALAM VOWEL SIGN U..MALAYALAM VOWEL SIGN VOCALIC R +0D46..0D48 ; Alphabetic # Mc [3] MALAYALAM VOWEL SIGN E..MALAYALAM VOWEL SIGN AI +0D4A..0D4C ; Alphabetic # Mc [3] MALAYALAM VOWEL SIGN O..MALAYALAM VOWEL SIGN AU +0D57 ; Alphabetic # Mc MALAYALAM AU LENGTH MARK +0D60..0D61 ; Alphabetic # Lo [2] MALAYALAM LETTER VOCALIC RR..MALAYALAM LETTER VOCALIC LL +0D82..0D83 ; Alphabetic # Mc [2] SINHALA SIGN ANUSVARAYA..SINHALA SIGN VISARGAYA +0D85..0D96 ; Alphabetic # Lo [18] SINHALA LETTER AYANNA..SINHALA LETTER AUYANNA +0D9A..0DB1 ; Alphabetic # Lo [24] SINHALA LETTER ALPAPRAANA KAYANNA..SINHALA LETTER DANTAJA NAYANNA +0DB3..0DBB ; Alphabetic # Lo [9] SINHALA LETTER SANYAKA DAYANNA..SINHALA LETTER RAYANNA +0DBD ; Alphabetic # Lo SINHALA LETTER DANTAJA LAYANNA +0DC0..0DC6 ; Alphabetic # Lo [7] SINHALA LETTER VAYANNA..SINHALA LETTER FAYANNA +0DCF..0DD1 ; Alphabetic # Mc [3] SINHALA VOWEL SIGN AELA-PILLA..SINHALA VOWEL SIGN DIGA AEDA-PILLA +0DD2..0DD4 ; Alphabetic # Mn [3] SINHALA VOWEL SIGN KETTI IS-PILLA..SINHALA VOWEL SIGN KETTI PAA-PILLA +0DD6 ; Alphabetic # Mn SINHALA VOWEL SIGN DIGA PAA-PILLA +0DD8..0DDF ; Alphabetic # Mc [8] SINHALA VOWEL SIGN GAETTA-PILLA..SINHALA VOWEL SIGN GAYANUKITTA +0DF2..0DF3 ; Alphabetic # Mc [2] SINHALA VOWEL SIGN DIGA GAETTA-PILLA..SINHALA VOWEL SIGN DIGA GAYANUKITTA +0E01..0E30 ; Alphabetic # Lo [48] THAI CHARACTER KO KAI..THAI CHARACTER SARA A +0E31 ; Alphabetic # Mn THAI CHARACTER MAI HAN-AKAT +0E32..0E33 ; Alphabetic # Lo [2] THAI CHARACTER SARA AA..THAI CHARACTER SARA AM +0E34..0E3A ; Alphabetic # Mn [7] THAI CHARACTER SARA I..THAI CHARACTER PHINTHU +0E40..0E45 ; Alphabetic # Lo [6] THAI CHARACTER SARA E..THAI CHARACTER LAKKHANGYAO +0E46 ; Alphabetic # Lm THAI CHARACTER MAIYAMOK +0E4D ; Alphabetic # Mn THAI CHARACTER NIKHAHIT +0E81..0E82 ; Alphabetic # Lo [2] LAO LETTER KO..LAO LETTER KHO SUNG +0E84 ; Alphabetic # Lo LAO LETTER KHO TAM +0E87..0E88 ; Alphabetic # Lo [2] LAO LETTER NGO..LAO LETTER CO +0E8A ; Alphabetic # Lo LAO LETTER SO TAM +0E8D ; Alphabetic # Lo LAO LETTER NYO +0E94..0E97 ; Alphabetic # Lo [4] LAO LETTER DO..LAO LETTER THO TAM +0E99..0E9F ; Alphabetic # Lo [7] LAO LETTER NO..LAO LETTER FO SUNG +0EA1..0EA3 ; Alphabetic # Lo [3] LAO LETTER MO..LAO LETTER LO LING +0EA5 ; Alphabetic # Lo LAO LETTER LO LOOT +0EA7 ; Alphabetic # Lo LAO LETTER WO +0EAA..0EAB ; Alphabetic # Lo [2] LAO LETTER SO SUNG..LAO LETTER HO SUNG +0EAD..0EB0 ; Alphabetic # Lo [4] LAO LETTER O..LAO VOWEL SIGN A +0EB1 ; Alphabetic # Mn LAO VOWEL SIGN MAI KAN +0EB2..0EB3 ; Alphabetic # Lo [2] LAO VOWEL SIGN AA..LAO VOWEL SIGN AM +0EB4..0EB9 ; Alphabetic # Mn [6] LAO VOWEL SIGN I..LAO VOWEL SIGN UU +0EBB..0EBC ; Alphabetic # Mn [2] LAO VOWEL SIGN MAI KON..LAO SEMIVOWEL SIGN LO +0EBD ; Alphabetic # Lo LAO SEMIVOWEL SIGN NYO +0EC0..0EC4 ; Alphabetic # Lo [5] LAO VOWEL SIGN E..LAO VOWEL SIGN AI +0EC6 ; Alphabetic # Lm LAO KO LA +0ECD ; Alphabetic # Mn LAO NIGGAHITA +0EDC..0EDD ; Alphabetic # Lo [2] LAO HO NO..LAO HO MO +0F00 ; Alphabetic # Lo TIBETAN SYLLABLE OM +0F40..0F47 ; Alphabetic # Lo [8] TIBETAN LETTER KA..TIBETAN LETTER JA +0F49..0F6A ; Alphabetic # Lo [34] TIBETAN LETTER NYA..TIBETAN LETTER FIXED-FORM RA +0F71..0F7E ; Alphabetic # Mn [14] TIBETAN VOWEL SIGN AA..TIBETAN SIGN RJES SU NGA RO +0F7F ; Alphabetic # Mc TIBETAN SIGN RNAM BCAD +0F80..0F81 ; Alphabetic # Mn [2] TIBETAN VOWEL SIGN REVERSED I..TIBETAN VOWEL SIGN REVERSED II +0F88..0F8B ; Alphabetic # Lo [4] TIBETAN SIGN LCE TSA CAN..TIBETAN SIGN GRU MED RGYINGS +0F90..0F97 ; Alphabetic # Mn [8] TIBETAN SUBJOINED LETTER KA..TIBETAN SUBJOINED LETTER JA +0F99..0FBC ; Alphabetic # Mn [36] TIBETAN SUBJOINED LETTER NYA..TIBETAN SUBJOINED LETTER FIXED-FORM RA +1000..1021 ; Alphabetic # Lo [34] MYANMAR LETTER KA..MYANMAR LETTER A +1023..1027 ; Alphabetic # Lo [5] MYANMAR LETTER I..MYANMAR LETTER E +1029..102A ; Alphabetic # Lo [2] MYANMAR LETTER O..MYANMAR LETTER AU +102C ; Alphabetic # Mc MYANMAR VOWEL SIGN AA +102D..1030 ; Alphabetic # Mn [4] MYANMAR VOWEL SIGN I..MYANMAR VOWEL SIGN UU +1031 ; Alphabetic # Mc MYANMAR VOWEL SIGN E +1032 ; Alphabetic # Mn MYANMAR VOWEL SIGN AI +1036 ; Alphabetic # Mn MYANMAR SIGN ANUSVARA +1038 ; Alphabetic # Mc MYANMAR SIGN VISARGA +1050..1055 ; Alphabetic # Lo [6] MYANMAR LETTER SHA..MYANMAR LETTER VOCALIC LL +1056..1057 ; Alphabetic # Mc [2] MYANMAR VOWEL SIGN VOCALIC R..MYANMAR VOWEL SIGN VOCALIC RR +1058..1059 ; Alphabetic # Mn [2] MYANMAR VOWEL SIGN VOCALIC L..MYANMAR VOWEL SIGN VOCALIC LL +10A0..10C5 ; Alphabetic # L& [38] GEORGIAN CAPITAL LETTER AN..GEORGIAN CAPITAL LETTER HOE +10D0..10F8 ; Alphabetic # Lo [41] GEORGIAN LETTER AN..GEORGIAN LETTER ELIFI +1100..1159 ; Alphabetic # Lo [90] HANGUL CHOSEONG KIYEOK..HANGUL CHOSEONG YEORINHIEUH +115F..11A2 ; Alphabetic # Lo [68] HANGUL CHOSEONG FILLER..HANGUL JUNGSEONG SSANGARAEA +11A8..11F9 ; Alphabetic # Lo [82] HANGUL JONGSEONG KIYEOK..HANGUL JONGSEONG YEORINHIEUH +1200..1206 ; Alphabetic # Lo [7] ETHIOPIC SYLLABLE HA..ETHIOPIC SYLLABLE HO +1208..1246 ; Alphabetic # Lo [63] ETHIOPIC SYLLABLE LA..ETHIOPIC SYLLABLE QO +1248 ; Alphabetic # Lo ETHIOPIC SYLLABLE QWA +124A..124D ; Alphabetic # Lo [4] ETHIOPIC SYLLABLE QWI..ETHIOPIC SYLLABLE QWE +1250..1256 ; Alphabetic # Lo [7] ETHIOPIC SYLLABLE QHA..ETHIOPIC SYLLABLE QHO +1258 ; Alphabetic # Lo ETHIOPIC SYLLABLE QHWA +125A..125D ; Alphabetic # Lo [4] ETHIOPIC SYLLABLE QHWI..ETHIOPIC SYLLABLE QHWE +1260..1286 ; Alphabetic # Lo [39] ETHIOPIC SYLLABLE BA..ETHIOPIC SYLLABLE XO +1288 ; Alphabetic # Lo ETHIOPIC SYLLABLE XWA +128A..128D ; Alphabetic # Lo [4] ETHIOPIC SYLLABLE XWI..ETHIOPIC SYLLABLE XWE +1290..12AE ; Alphabetic # Lo [31] ETHIOPIC SYLLABLE NA..ETHIOPIC SYLLABLE KO +12B0 ; Alphabetic # Lo ETHIOPIC SYLLABLE KWA +12B2..12B5 ; Alphabetic # Lo [4] ETHIOPIC SYLLABLE KWI..ETHIOPIC SYLLABLE KWE +12B8..12BE ; Alphabetic # Lo [7] ETHIOPIC SYLLABLE KXA..ETHIOPIC SYLLABLE KXO +12C0 ; Alphabetic # Lo ETHIOPIC SYLLABLE KXWA +12C2..12C5 ; Alphabetic # Lo [4] ETHIOPIC SYLLABLE KXWI..ETHIOPIC SYLLABLE KXWE +12C8..12CE ; Alphabetic # Lo [7] ETHIOPIC SYLLABLE WA..ETHIOPIC SYLLABLE WO +12D0..12D6 ; Alphabetic # Lo [7] ETHIOPIC SYLLABLE PHARYNGEAL A..ETHIOPIC SYLLABLE PHARYNGEAL O +12D8..12EE ; Alphabetic # Lo [23] ETHIOPIC SYLLABLE ZA..ETHIOPIC SYLLABLE YO +12F0..130E ; Alphabetic # Lo [31] ETHIOPIC SYLLABLE DA..ETHIOPIC SYLLABLE GO +1310 ; Alphabetic # Lo ETHIOPIC SYLLABLE GWA +1312..1315 ; Alphabetic # Lo [4] ETHIOPIC SYLLABLE GWI..ETHIOPIC SYLLABLE GWE +1318..131E ; Alphabetic # Lo [7] ETHIOPIC SYLLABLE GGA..ETHIOPIC SYLLABLE GGO +1320..1346 ; Alphabetic # Lo [39] ETHIOPIC SYLLABLE THA..ETHIOPIC SYLLABLE TZO +1348..135A ; Alphabetic # Lo [19] ETHIOPIC SYLLABLE FA..ETHIOPIC SYLLABLE FYA +13A0..13F4 ; Alphabetic # Lo [85] CHEROKEE LETTER A..CHEROKEE LETTER YV +1401..166C ; Alphabetic # Lo [620] CANADIAN SYLLABICS E..CANADIAN SYLLABICS CARRIER TTSA +166F..1676 ; Alphabetic # Lo [8] CANADIAN SYLLABICS QAI..CANADIAN SYLLABICS NNGAA +1681..169A ; Alphabetic # Lo [26] OGHAM LETTER BEITH..OGHAM LETTER PEITH +16A0..16EA ; Alphabetic # Lo [75] RUNIC LETTER FEHU FEOH FE F..RUNIC LETTER X +16EE..16F0 ; Alphabetic # Nl [3] RUNIC ARLAUG SYMBOL..RUNIC BELGTHOR SYMBOL +1700..170C ; Alphabetic # Lo [13] TAGALOG LETTER A..TAGALOG LETTER YA +170E..1711 ; Alphabetic # Lo [4] TAGALOG LETTER LA..TAGALOG LETTER HA +1712..1713 ; Alphabetic # Mn [2] TAGALOG VOWEL SIGN I..TAGALOG VOWEL SIGN U +1720..1731 ; Alphabetic # Lo [18] HANUNOO LETTER A..HANUNOO LETTER HA +1732..1733 ; Alphabetic # Mn [2] HANUNOO VOWEL SIGN I..HANUNOO VOWEL SIGN U +1740..1751 ; Alphabetic # Lo [18] BUHID LETTER A..BUHID LETTER HA +1752..1753 ; Alphabetic # Mn [2] BUHID VOWEL SIGN I..BUHID VOWEL SIGN U +1760..176C ; Alphabetic # Lo [13] TAGBANWA LETTER A..TAGBANWA LETTER YA +176E..1770 ; Alphabetic # Lo [3] TAGBANWA LETTER LA..TAGBANWA LETTER SA +1772..1773 ; Alphabetic # Mn [2] TAGBANWA VOWEL SIGN I..TAGBANWA VOWEL SIGN U +1780..17B3 ; Alphabetic # Lo [52] KHMER LETTER KA..KHMER INDEPENDENT VOWEL QAU +17B4..17B6 ; Alphabetic # Mc [3] KHMER VOWEL INHERENT AQ..KHMER VOWEL SIGN AA +17B7..17BD ; Alphabetic # Mn [7] KHMER VOWEL SIGN I..KHMER VOWEL SIGN UA +17BE..17C5 ; Alphabetic # Mc [8] KHMER VOWEL SIGN OE..KHMER VOWEL SIGN AU +17C6 ; Alphabetic # Mn KHMER SIGN NIKAHIT +17C7..17C8 ; Alphabetic # Mc [2] KHMER SIGN REAHMUK..KHMER SIGN YUUKALEAPINTU +17D7 ; Alphabetic # Lm KHMER SIGN LEK TOO +17DC ; Alphabetic # Lo KHMER SIGN AVAKRAHASANYA +1820..1842 ; Alphabetic # Lo [35] MONGOLIAN LETTER A..MONGOLIAN LETTER CHI +1843 ; Alphabetic # Lm MONGOLIAN LETTER TODO LONG VOWEL SIGN +1844..1877 ; Alphabetic # Lo [52] MONGOLIAN LETTER TODO E..MONGOLIAN LETTER MANCHU ZHA +1880..18A8 ; Alphabetic # Lo [41] MONGOLIAN LETTER ALI GALI ANUSVARA ONE..MONGOLIAN LETTER MANCHU ALI GALI BHA +18A9 ; Alphabetic # Mn MONGOLIAN LETTER ALI GALI DAGALGA +1E00..1E9B ; Alphabetic # L& [156] LATIN CAPITAL LETTER A WITH RING BELOW..LATIN SMALL LETTER LONG S WITH DOT ABOVE +1EA0..1EF9 ; Alphabetic # L& [90] LATIN CAPITAL LETTER A WITH DOT BELOW..LATIN SMALL LETTER Y WITH TILDE +1F00..1F15 ; Alphabetic # L& [22] GREEK SMALL LETTER ALPHA WITH PSILI..GREEK SMALL LETTER EPSILON WITH DASIA AND OXIA +1F18..1F1D ; Alphabetic # L& [6] GREEK CAPITAL LETTER EPSILON WITH PSILI..GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA +1F20..1F45 ; Alphabetic # L& [38] GREEK SMALL LETTER ETA WITH PSILI..GREEK SMALL LETTER OMICRON WITH DASIA AND OXIA +1F48..1F4D ; Alphabetic # L& [6] GREEK CAPITAL LETTER OMICRON WITH PSILI..GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA +1F50..1F57 ; Alphabetic # L& [8] GREEK SMALL LETTER UPSILON WITH PSILI..GREEK SMALL LETTER UPSILON WITH DASIA AND PERISPOMENI +1F59 ; Alphabetic # L& GREEK CAPITAL LETTER UPSILON WITH DASIA +1F5B ; Alphabetic # L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA +1F5D ; Alphabetic # L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA +1F5F..1F7D ; Alphabetic # L& [31] GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI..GREEK SMALL LETTER OMEGA WITH OXIA +1F80..1FB4 ; Alphabetic # L& [53] GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI..GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI +1FB6..1FBC ; Alphabetic # L& [7] GREEK SMALL LETTER ALPHA WITH PERISPOMENI..GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI +1FBE ; Alphabetic # L& GREEK PROSGEGRAMMENI +1FC2..1FC4 ; Alphabetic # L& [3] GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI +1FC6..1FCC ; Alphabetic # L& [7] GREEK SMALL LETTER ETA WITH PERISPOMENI..GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI +1FD0..1FD3 ; Alphabetic # L& [4] GREEK SMALL LETTER IOTA WITH VRACHY..GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA +1FD6..1FDB ; Alphabetic # L& [6] GREEK SMALL LETTER IOTA WITH PERISPOMENI..GREEK CAPITAL LETTER IOTA WITH OXIA +1FE0..1FEC ; Alphabetic # L& [13] GREEK SMALL LETTER UPSILON WITH VRACHY..GREEK CAPITAL LETTER RHO WITH DASIA +1FF2..1FF4 ; Alphabetic # L& [3] GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI +1FF6..1FFC ; Alphabetic # L& [7] GREEK SMALL LETTER OMEGA WITH PERISPOMENI..GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI +2071 ; Alphabetic # L& SUPERSCRIPT LATIN SMALL LETTER I +207F ; Alphabetic # L& SUPERSCRIPT LATIN SMALL LETTER N +2102 ; Alphabetic # L& DOUBLE-STRUCK CAPITAL C +2107 ; Alphabetic # L& EULER CONSTANT +210A..2113 ; Alphabetic # L& [10] SCRIPT SMALL G..SCRIPT SMALL L +2115 ; Alphabetic # L& DOUBLE-STRUCK CAPITAL N +2119..211D ; Alphabetic # L& [5] DOUBLE-STRUCK CAPITAL P..DOUBLE-STRUCK CAPITAL R +2124 ; Alphabetic # L& DOUBLE-STRUCK CAPITAL Z +2126 ; Alphabetic # L& OHM SIGN +2128 ; Alphabetic # L& BLACK-LETTER CAPITAL Z +212A..212D ; Alphabetic # L& [4] KELVIN SIGN..BLACK-LETTER CAPITAL C +212F..2131 ; Alphabetic # L& [3] SCRIPT SMALL E..SCRIPT CAPITAL F +2133..2134 ; Alphabetic # L& [2] SCRIPT CAPITAL M..SCRIPT SMALL O +2135..2138 ; Alphabetic # Lo [4] ALEF SYMBOL..DALET SYMBOL +2139 ; Alphabetic # L& INFORMATION SOURCE +213D..213F ; Alphabetic # L& [3] DOUBLE-STRUCK SMALL GAMMA..DOUBLE-STRUCK CAPITAL PI +2145..2149 ; Alphabetic # L& [5] DOUBLE-STRUCK ITALIC CAPITAL D..DOUBLE-STRUCK ITALIC SMALL J +2160..2183 ; Alphabetic # Nl [36] ROMAN NUMERAL ONE..ROMAN NUMERAL REVERSED ONE HUNDRED +3005 ; Alphabetic # Lm IDEOGRAPHIC ITERATION MARK +3006 ; Alphabetic # Lo IDEOGRAPHIC CLOSING MARK +3007 ; Alphabetic # Nl IDEOGRAPHIC NUMBER ZERO +3021..3029 ; Alphabetic # Nl [9] HANGZHOU NUMERAL ONE..HANGZHOU NUMERAL NINE +3031..3035 ; Alphabetic # Lm [5] VERTICAL KANA REPEAT MARK..VERTICAL KANA REPEAT MARK LOWER HALF +3038..303A ; Alphabetic # Nl [3] HANGZHOU NUMERAL TEN..HANGZHOU NUMERAL THIRTY +303B ; Alphabetic # Lm VERTICAL IDEOGRAPHIC ITERATION MARK +303C ; Alphabetic # Lo MASU MARK +3041..3096 ; Alphabetic # Lo [86] HIRAGANA LETTER SMALL A..HIRAGANA LETTER SMALL KE +309D..309E ; Alphabetic # Lm [2] HIRAGANA ITERATION MARK..HIRAGANA VOICED ITERATION MARK +309F ; Alphabetic # Lo HIRAGANA DIGRAPH YORI +30A1..30FA ; Alphabetic # Lo [90] KATAKANA LETTER SMALL A..KATAKANA LETTER VO +30FC..30FE ; Alphabetic # Lm [3] KATAKANA-HIRAGANA PROLONGED SOUND MARK..KATAKANA VOICED ITERATION MARK +30FF ; Alphabetic # Lo KATAKANA DIGRAPH KOTO +3105..312C ; Alphabetic # Lo [40] BOPOMOFO LETTER B..BOPOMOFO LETTER GN +3131..318E ; Alphabetic # Lo [94] HANGUL LETTER KIYEOK..HANGUL LETTER ARAEAE +31A0..31B7 ; Alphabetic # Lo [24] BOPOMOFO LETTER BU..BOPOMOFO FINAL LETTER H +31F0..31FF ; Alphabetic # Lo [16] KATAKANA LETTER SMALL KU..KATAKANA LETTER SMALL RO +3400..4DB5 ; Alphabetic # Lo [6582] CJK UNIFIED IDEOGRAPH-3400..CJK UNIFIED IDEOGRAPH-4DB5 +4E00..9FA5 ; Alphabetic # Lo [20902] CJK UNIFIED IDEOGRAPH-4E00..CJK UNIFIED IDEOGRAPH-9FA5 +A000..A48C ; Alphabetic # Lo [1165] YI SYLLABLE IT..YI SYLLABLE YYR +AC00..D7A3 ; Alphabetic # Lo [11172] HANGUL SYLLABLE GA..HANGUL SYLLABLE HIH +F900..FA2D ; Alphabetic # Lo [302] CJK COMPATIBILITY IDEOGRAPH-F900..CJK COMPATIBILITY IDEOGRAPH-FA2D +FA30..FA6A ; Alphabetic # Lo [59] CJK COMPATIBILITY IDEOGRAPH-FA30..CJK COMPATIBILITY IDEOGRAPH-FA6A +FB00..FB06 ; Alphabetic # L& [7] LATIN SMALL LIGATURE FF..LATIN SMALL LIGATURE ST +FB13..FB17 ; Alphabetic # L& [5] ARMENIAN SMALL LIGATURE MEN NOW..ARMENIAN SMALL LIGATURE MEN XEH +FB1D ; Alphabetic # Lo HEBREW LETTER YOD WITH HIRIQ +FB1E ; Alphabetic # Mn HEBREW POINT JUDEO-SPANISH VARIKA +FB1F..FB28 ; Alphabetic # Lo [10] HEBREW LIGATURE YIDDISH YOD YOD PATAH..HEBREW LETTER WIDE TAV +FB2A..FB36 ; Alphabetic # Lo [13] HEBREW LETTER SHIN WITH SHIN DOT..HEBREW LETTER ZAYIN WITH DAGESH +FB38..FB3C ; Alphabetic # Lo [5] HEBREW LETTER TET WITH DAGESH..HEBREW LETTER LAMED WITH DAGESH +FB3E ; Alphabetic # Lo HEBREW LETTER MEM WITH DAGESH +FB40..FB41 ; Alphabetic # Lo [2] HEBREW LETTER NUN WITH DAGESH..HEBREW LETTER SAMEKH WITH DAGESH +FB43..FB44 ; Alphabetic # Lo [2] HEBREW LETTER FINAL PE WITH DAGESH..HEBREW LETTER PE WITH DAGESH +FB46..FBB1 ; Alphabetic # Lo [108] HEBREW LETTER TSADI WITH DAGESH..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE FINAL FORM +FBD3..FD3D ; Alphabetic # Lo [363] ARABIC LETTER NG ISOLATED FORM..ARABIC LIGATURE ALEF WITH FATHATAN ISOLATED FORM +FD50..FD8F ; Alphabetic # Lo [64] ARABIC LIGATURE TEH WITH JEEM WITH MEEM INITIAL FORM..ARABIC LIGATURE MEEM WITH KHAH WITH MEEM INITIAL FORM +FD92..FDC7 ; Alphabetic # Lo [54] ARABIC LIGATURE MEEM WITH JEEM WITH KHAH INITIAL FORM..ARABIC LIGATURE NOON WITH JEEM WITH YEH FINAL FORM +FDF0..FDFB ; Alphabetic # Lo [12] ARABIC LIGATURE SALLA USED AS KORANIC STOP SIGN ISOLATED FORM..ARABIC LIGATURE JALLAJALALOUHOU +FE70..FE74 ; Alphabetic # Lo [5] ARABIC FATHATAN ISOLATED FORM..ARABIC KASRATAN ISOLATED FORM +FE76..FEFC ; Alphabetic # Lo [135] ARABIC FATHA ISOLATED FORM..ARABIC LIGATURE LAM WITH ALEF FINAL FORM +FF21..FF3A ; Alphabetic # L& [26] FULLWIDTH LATIN CAPITAL LETTER A..FULLWIDTH LATIN CAPITAL LETTER Z +FF41..FF5A ; Alphabetic # L& [26] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH LATIN SMALL LETTER Z +FF66..FF6F ; Alphabetic # Lo [10] HALFWIDTH KATAKANA LETTER WO..HALFWIDTH KATAKANA LETTER SMALL TU +FF70 ; Alphabetic # Lm HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK +FF71..FF9D ; Alphabetic # Lo [45] HALFWIDTH KATAKANA LETTER A..HALFWIDTH KATAKANA LETTER N +FF9E..FF9F ; Alphabetic # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK +FFA0..FFBE ; Alphabetic # Lo [31] HALFWIDTH HANGUL FILLER..HALFWIDTH HANGUL LETTER HIEUH +FFC2..FFC7 ; Alphabetic # Lo [6] HALFWIDTH HANGUL LETTER A..HALFWIDTH HANGUL LETTER E +FFCA..FFCF ; Alphabetic # Lo [6] HALFWIDTH HANGUL LETTER YEO..HALFWIDTH HANGUL LETTER OE +FFD2..FFD7 ; Alphabetic # Lo [6] HALFWIDTH HANGUL LETTER YO..HALFWIDTH HANGUL LETTER YU +FFDA..FFDC ; Alphabetic # Lo [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL LETTER I +10300..1031E ; Alphabetic # Lo [31] OLD ITALIC LETTER A..OLD ITALIC LETTER UU +10330..10349 ; Alphabetic # Lo [26] GOTHIC LETTER AHSA..GOTHIC LETTER OTHAL +1034A ; Alphabetic # Nl GOTHIC LETTER NINE HUNDRED +10400..10425 ; Alphabetic # L& [38] DESERET CAPITAL LETTER LONG I..DESERET CAPITAL LETTER ENG +10428..1044D ; Alphabetic # L& [38] DESERET SMALL LETTER LONG I..DESERET SMALL LETTER ENG +1D400..1D454 ; Alphabetic # L& [85] MATHEMATICAL BOLD CAPITAL A..MATHEMATICAL ITALIC SMALL G +1D456..1D49C ; Alphabetic # L& [71] MATHEMATICAL ITALIC SMALL I..MATHEMATICAL SCRIPT CAPITAL A +1D49E..1D49F ; Alphabetic # L& [2] MATHEMATICAL SCRIPT CAPITAL C..MATHEMATICAL SCRIPT CAPITAL D +1D4A2 ; Alphabetic # L& MATHEMATICAL SCRIPT CAPITAL G +1D4A5..1D4A6 ; Alphabetic # L& [2] MATHEMATICAL SCRIPT CAPITAL J..MATHEMATICAL SCRIPT CAPITAL K +1D4A9..1D4AC ; Alphabetic # L& [4] MATHEMATICAL SCRIPT CAPITAL N..MATHEMATICAL SCRIPT CAPITAL Q +1D4AE..1D4B9 ; Alphabetic # L& [12] MATHEMATICAL SCRIPT CAPITAL S..MATHEMATICAL SCRIPT SMALL D +1D4BB ; Alphabetic # L& MATHEMATICAL SCRIPT SMALL F +1D4BD..1D4C0 ; Alphabetic # L& [4] MATHEMATICAL SCRIPT SMALL H..MATHEMATICAL SCRIPT SMALL K +1D4C2..1D4C3 ; Alphabetic # L& [2] MATHEMATICAL SCRIPT SMALL M..MATHEMATICAL SCRIPT SMALL N +1D4C5..1D505 ; Alphabetic # L& [65] MATHEMATICAL SCRIPT SMALL P..MATHEMATICAL FRAKTUR CAPITAL B +1D507..1D50A ; Alphabetic # L& [4] MATHEMATICAL FRAKTUR CAPITAL D..MATHEMATICAL FRAKTUR CAPITAL G +1D50D..1D514 ; Alphabetic # L& [8] MATHEMATICAL FRAKTUR CAPITAL J..MATHEMATICAL FRAKTUR CAPITAL Q +1D516..1D51C ; Alphabetic # L& [7] MATHEMATICAL FRAKTUR CAPITAL S..MATHEMATICAL FRAKTUR CAPITAL Y +1D51E..1D539 ; Alphabetic # L& [28] MATHEMATICAL FRAKTUR SMALL A..MATHEMATICAL DOUBLE-STRUCK CAPITAL B +1D53B..1D53E ; Alphabetic # L& [4] MATHEMATICAL DOUBLE-STRUCK CAPITAL D..MATHEMATICAL DOUBLE-STRUCK CAPITAL G +1D540..1D544 ; Alphabetic # L& [5] MATHEMATICAL DOUBLE-STRUCK CAPITAL I..MATHEMATICAL DOUBLE-STRUCK CAPITAL M +1D546 ; Alphabetic # L& MATHEMATICAL DOUBLE-STRUCK CAPITAL O +1D54A..1D550 ; Alphabetic # L& [7] MATHEMATICAL DOUBLE-STRUCK CAPITAL S..MATHEMATICAL DOUBLE-STRUCK CAPITAL Y +1D552..1D6A3 ; Alphabetic # L& [338] MATHEMATICAL DOUBLE-STRUCK SMALL A..MATHEMATICAL MONOSPACE SMALL Z +1D6A8..1D6C0 ; Alphabetic # L& [25] MATHEMATICAL BOLD CAPITAL ALPHA..MATHEMATICAL BOLD CAPITAL OMEGA +1D6C2..1D6DA ; Alphabetic # L& [25] MATHEMATICAL BOLD SMALL ALPHA..MATHEMATICAL BOLD SMALL OMEGA +1D6DC..1D6FA ; Alphabetic # L& [31] MATHEMATICAL BOLD EPSILON SYMBOL..MATHEMATICAL ITALIC CAPITAL OMEGA +1D6FC..1D714 ; Alphabetic # L& [25] MATHEMATICAL ITALIC SMALL ALPHA..MATHEMATICAL ITALIC SMALL OMEGA +1D716..1D734 ; Alphabetic # L& [31] MATHEMATICAL ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD ITALIC CAPITAL OMEGA +1D736..1D74E ; Alphabetic # L& [25] MATHEMATICAL BOLD ITALIC SMALL ALPHA..MATHEMATICAL BOLD ITALIC SMALL OMEGA +1D750..1D76E ; Alphabetic # L& [31] MATHEMATICAL BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD CAPITAL OMEGA +1D770..1D788 ; Alphabetic # L& [25] MATHEMATICAL SANS-SERIF BOLD SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD SMALL OMEGA +1D78A..1D7A8 ; Alphabetic # L& [31] MATHEMATICAL SANS-SERIF BOLD EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL OMEGA +1D7AA..1D7C2 ; Alphabetic # L& [25] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL OMEGA +1D7C4..1D7C9 ; Alphabetic # L& [6] MATHEMATICAL SANS-SERIF BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC PI SYMBOL +20000..2A6D6 ; Alphabetic # Lo [42711] CJK UNIFIED IDEOGRAPH-20000..CJK UNIFIED IDEOGRAPH-2A6D6 +2F800..2FA1D ; Alphabetic # Lo [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D + +# Total code points: 90368 + +# ================================================ + +# Derived Property: Lowercase +# Generated from: Ll + Other_Lowercase + +0061..007A ; Lowercase # L& [26] LATIN SMALL LETTER A..LATIN SMALL LETTER Z +00AA ; Lowercase # L& FEMININE ORDINAL INDICATOR +00B5 ; Lowercase # L& MICRO SIGN +00BA ; Lowercase # L& MASCULINE ORDINAL INDICATOR +00DF..00F6 ; Lowercase # L& [24] LATIN SMALL LETTER SHARP S..LATIN SMALL LETTER O WITH DIAERESIS +00F8..00FF ; Lowercase # L& [8] LATIN SMALL LETTER O WITH STROKE..LATIN SMALL LETTER Y WITH DIAERESIS +0101 ; Lowercase # L& LATIN SMALL LETTER A WITH MACRON +0103 ; Lowercase # L& LATIN SMALL LETTER A WITH BREVE +0105 ; Lowercase # L& LATIN SMALL LETTER A WITH OGONEK +0107 ; Lowercase # L& LATIN SMALL LETTER C WITH ACUTE +0109 ; Lowercase # L& LATIN SMALL LETTER C WITH CIRCUMFLEX +010B ; Lowercase # L& LATIN SMALL LETTER C WITH DOT ABOVE +010D ; Lowercase # L& LATIN SMALL LETTER C WITH CARON +010F ; Lowercase # L& LATIN SMALL LETTER D WITH CARON +0111 ; Lowercase # L& LATIN SMALL LETTER D WITH STROKE +0113 ; Lowercase # L& LATIN SMALL LETTER E WITH MACRON +0115 ; Lowercase # L& LATIN SMALL LETTER E WITH BREVE +0117 ; Lowercase # L& LATIN SMALL LETTER E WITH DOT ABOVE +0119 ; Lowercase # L& LATIN SMALL LETTER E WITH OGONEK +011B ; Lowercase # L& LATIN SMALL LETTER E WITH CARON +011D ; Lowercase # L& LATIN SMALL LETTER G WITH CIRCUMFLEX +011F ; Lowercase # L& LATIN SMALL LETTER G WITH BREVE +0121 ; Lowercase # L& LATIN SMALL LETTER G WITH DOT ABOVE +0123 ; Lowercase # L& LATIN SMALL LETTER G WITH CEDILLA +0125 ; Lowercase # L& LATIN SMALL LETTER H WITH CIRCUMFLEX +0127 ; Lowercase # L& LATIN SMALL LETTER H WITH STROKE +0129 ; Lowercase # L& LATIN SMALL LETTER I WITH TILDE +012B ; Lowercase # L& LATIN SMALL LETTER I WITH MACRON +012D ; Lowercase # L& LATIN SMALL LETTER I WITH BREVE +012F ; Lowercase # L& LATIN SMALL LETTER I WITH OGONEK +0131 ; Lowercase # L& LATIN SMALL LETTER DOTLESS I +0133 ; Lowercase # L& LATIN SMALL LIGATURE IJ +0135 ; Lowercase # L& LATIN SMALL LETTER J WITH CIRCUMFLEX +0137..0138 ; Lowercase # L& [2] LATIN SMALL LETTER K WITH CEDILLA..LATIN SMALL LETTER KRA +013A ; Lowercase # L& LATIN SMALL LETTER L WITH ACUTE +013C ; Lowercase # L& LATIN SMALL LETTER L WITH CEDILLA +013E ; Lowercase # L& LATIN SMALL LETTER L WITH CARON +0140 ; Lowercase # L& LATIN SMALL LETTER L WITH MIDDLE DOT +0142 ; Lowercase # L& LATIN SMALL LETTER L WITH STROKE +0144 ; Lowercase # L& LATIN SMALL LETTER N WITH ACUTE +0146 ; Lowercase # L& LATIN SMALL LETTER N WITH CEDILLA +0148..0149 ; Lowercase # L& [2] LATIN SMALL LETTER N WITH CARON..LATIN SMALL LETTER N PRECEDED BY APOSTROPHE +014B ; Lowercase # L& LATIN SMALL LETTER ENG +014D ; Lowercase # L& LATIN SMALL LETTER O WITH MACRON +014F ; Lowercase # L& LATIN SMALL LETTER O WITH BREVE +0151 ; Lowercase # L& LATIN SMALL LETTER O WITH DOUBLE ACUTE +0153 ; Lowercase # L& LATIN SMALL LIGATURE OE +0155 ; Lowercase # L& LATIN SMALL LETTER R WITH ACUTE +0157 ; Lowercase # L& LATIN SMALL LETTER R WITH CEDILLA +0159 ; Lowercase # L& LATIN SMALL LETTER R WITH CARON +015B ; Lowercase # L& LATIN SMALL LETTER S WITH ACUTE +015D ; Lowercase # L& LATIN SMALL LETTER S WITH CIRCUMFLEX +015F ; Lowercase # L& LATIN SMALL LETTER S WITH CEDILLA +0161 ; Lowercase # L& LATIN SMALL LETTER S WITH CARON +0163 ; Lowercase # L& LATIN SMALL LETTER T WITH CEDILLA +0165 ; Lowercase # L& LATIN SMALL LETTER T WITH CARON +0167 ; Lowercase # L& LATIN SMALL LETTER T WITH STROKE +0169 ; Lowercase # L& LATIN SMALL LETTER U WITH TILDE +016B ; Lowercase # L& LATIN SMALL LETTER U WITH MACRON +016D ; Lowercase # L& LATIN SMALL LETTER U WITH BREVE +016F ; Lowercase # L& LATIN SMALL LETTER U WITH RING ABOVE +0171 ; Lowercase # L& LATIN SMALL LETTER U WITH DOUBLE ACUTE +0173 ; Lowercase # L& LATIN SMALL LETTER U WITH OGONEK +0175 ; Lowercase # L& LATIN SMALL LETTER W WITH CIRCUMFLEX +0177 ; Lowercase # L& LATIN SMALL LETTER Y WITH CIRCUMFLEX +017A ; Lowercase # L& LATIN SMALL LETTER Z WITH ACUTE +017C ; Lowercase # L& LATIN SMALL LETTER Z WITH DOT ABOVE +017E..0180 ; Lowercase # L& [3] LATIN SMALL LETTER Z WITH CARON..LATIN SMALL LETTER B WITH STROKE +0183 ; Lowercase # L& LATIN SMALL LETTER B WITH TOPBAR +0185 ; Lowercase # L& LATIN SMALL LETTER TONE SIX +0188 ; Lowercase # L& LATIN SMALL LETTER C WITH HOOK +018C..018D ; Lowercase # L& [2] LATIN SMALL LETTER D WITH TOPBAR..LATIN SMALL LETTER TURNED DELTA +0192 ; Lowercase # L& LATIN SMALL LETTER F WITH HOOK +0195 ; Lowercase # L& LATIN SMALL LETTER HV +0199..019B ; Lowercase # L& [3] LATIN SMALL LETTER K WITH HOOK..LATIN SMALL LETTER LAMBDA WITH STROKE +019E ; Lowercase # L& LATIN SMALL LETTER N WITH LONG RIGHT LEG +01A1 ; Lowercase # L& LATIN SMALL LETTER O WITH HORN +01A3 ; Lowercase # L& LATIN SMALL LETTER OI +01A5 ; Lowercase # L& LATIN SMALL LETTER P WITH HOOK +01A8 ; Lowercase # L& LATIN SMALL LETTER TONE TWO +01AA..01AB ; Lowercase # L& [2] LATIN LETTER REVERSED ESH LOOP..LATIN SMALL LETTER T WITH PALATAL HOOK +01AD ; Lowercase # L& LATIN SMALL LETTER T WITH HOOK +01B0 ; Lowercase # L& LATIN SMALL LETTER U WITH HORN +01B4 ; Lowercase # L& LATIN SMALL LETTER Y WITH HOOK +01B6 ; Lowercase # L& LATIN SMALL LETTER Z WITH STROKE +01B9..01BA ; Lowercase # L& [2] LATIN SMALL LETTER EZH REVERSED..LATIN SMALL LETTER EZH WITH TAIL +01BD..01BF ; Lowercase # L& [3] LATIN SMALL LETTER TONE FIVE..LATIN LETTER WYNN +01C6 ; Lowercase # L& LATIN SMALL LETTER DZ WITH CARON +01C9 ; Lowercase # L& LATIN SMALL LETTER LJ +01CC ; Lowercase # L& LATIN SMALL LETTER NJ +01CE ; Lowercase # L& LATIN SMALL LETTER A WITH CARON +01D0 ; Lowercase # L& LATIN SMALL LETTER I WITH CARON +01D2 ; Lowercase # L& LATIN SMALL LETTER O WITH CARON +01D4 ; Lowercase # L& LATIN SMALL LETTER U WITH CARON +01D6 ; Lowercase # L& LATIN SMALL LETTER U WITH DIAERESIS AND MACRON +01D8 ; Lowercase # L& LATIN SMALL LETTER U WITH DIAERESIS AND ACUTE +01DA ; Lowercase # L& LATIN SMALL LETTER U WITH DIAERESIS AND CARON +01DC..01DD ; Lowercase # L& [2] LATIN SMALL LETTER U WITH DIAERESIS AND GRAVE..LATIN SMALL LETTER TURNED E +01DF ; Lowercase # L& LATIN SMALL LETTER A WITH DIAERESIS AND MACRON +01E1 ; Lowercase # L& LATIN SMALL LETTER A WITH DOT ABOVE AND MACRON +01E3 ; Lowercase # L& LATIN SMALL LETTER AE WITH MACRON +01E5 ; Lowercase # L& LATIN SMALL LETTER G WITH STROKE +01E7 ; Lowercase # L& LATIN SMALL LETTER G WITH CARON +01E9 ; Lowercase # L& LATIN SMALL LETTER K WITH CARON +01EB ; Lowercase # L& LATIN SMALL LETTER O WITH OGONEK +01ED ; Lowercase # L& LATIN SMALL LETTER O WITH OGONEK AND MACRON +01EF..01F0 ; Lowercase # L& [2] LATIN SMALL LETTER EZH WITH CARON..LATIN SMALL LETTER J WITH CARON +01F3 ; Lowercase # L& LATIN SMALL LETTER DZ +01F5 ; Lowercase # L& LATIN SMALL LETTER G WITH ACUTE +01F9 ; Lowercase # L& LATIN SMALL LETTER N WITH GRAVE +01FB ; Lowercase # L& LATIN SMALL LETTER A WITH RING ABOVE AND ACUTE +01FD ; Lowercase # L& LATIN SMALL LETTER AE WITH ACUTE +01FF ; Lowercase # L& LATIN SMALL LETTER O WITH STROKE AND ACUTE +0201 ; Lowercase # L& LATIN SMALL LETTER A WITH DOUBLE GRAVE +0203 ; Lowercase # L& LATIN SMALL LETTER A WITH INVERTED BREVE +0205 ; Lowercase # L& LATIN SMALL LETTER E WITH DOUBLE GRAVE +0207 ; Lowercase # L& LATIN SMALL LETTER E WITH INVERTED BREVE +0209 ; Lowercase # L& LATIN SMALL LETTER I WITH DOUBLE GRAVE +020B ; Lowercase # L& LATIN SMALL LETTER I WITH INVERTED BREVE +020D ; Lowercase # L& LATIN SMALL LETTER O WITH DOUBLE GRAVE +020F ; Lowercase # L& LATIN SMALL LETTER O WITH INVERTED BREVE +0211 ; Lowercase # L& LATIN SMALL LETTER R WITH DOUBLE GRAVE +0213 ; Lowercase # L& LATIN SMALL LETTER R WITH INVERTED BREVE +0215 ; Lowercase # L& LATIN SMALL LETTER U WITH DOUBLE GRAVE +0217 ; Lowercase # L& LATIN SMALL LETTER U WITH INVERTED BREVE +0219 ; Lowercase # L& LATIN SMALL LETTER S WITH COMMA BELOW +021B ; Lowercase # L& LATIN SMALL LETTER T WITH COMMA BELOW +021D ; Lowercase # L& LATIN SMALL LETTER YOGH +021F ; Lowercase # L& LATIN SMALL LETTER H WITH CARON +0223 ; Lowercase # L& LATIN SMALL LETTER OU +0225 ; Lowercase # L& LATIN SMALL LETTER Z WITH HOOK +0227 ; Lowercase # L& LATIN SMALL LETTER A WITH DOT ABOVE +0229 ; Lowercase # L& LATIN SMALL LETTER E WITH CEDILLA +022B ; Lowercase # L& LATIN SMALL LETTER O WITH DIAERESIS AND MACRON +022D ; Lowercase # L& LATIN SMALL LETTER O WITH TILDE AND MACRON +022F ; Lowercase # L& LATIN SMALL LETTER O WITH DOT ABOVE +0231 ; Lowercase # L& LATIN SMALL LETTER O WITH DOT ABOVE AND MACRON +0233 ; Lowercase # L& LATIN SMALL LETTER Y WITH MACRON +0250..02AD ; Lowercase # L& [94] LATIN SMALL LETTER TURNED A..LATIN LETTER BIDENTAL PERCUSSIVE +02B0..02B8 ; Lowercase # Lm [9] MODIFIER LETTER SMALL H..MODIFIER LETTER SMALL Y +02C0..02C1 ; Lowercase # Lm [2] MODIFIER LETTER GLOTTAL STOP..MODIFIER LETTER REVERSED GLOTTAL STOP +02E0..02E4 ; Lowercase # Lm [5] MODIFIER LETTER SMALL GAMMA..MODIFIER LETTER SMALL REVERSED GLOTTAL STOP +0345 ; Lowercase # Mn COMBINING GREEK YPOGEGRAMMENI +037A ; Lowercase # Lm GREEK YPOGEGRAMMENI +0390 ; Lowercase # L& GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS +03AC..03CE ; Lowercase # L& [35] GREEK SMALL LETTER ALPHA WITH TONOS..GREEK SMALL LETTER OMEGA WITH TONOS +03D0..03D1 ; Lowercase # L& [2] GREEK BETA SYMBOL..GREEK THETA SYMBOL +03D5..03D7 ; Lowercase # L& [3] GREEK PHI SYMBOL..GREEK KAI SYMBOL +03D9 ; Lowercase # L& GREEK SMALL LETTER ARCHAIC KOPPA +03DB ; Lowercase # L& GREEK SMALL LETTER STIGMA +03DD ; Lowercase # L& GREEK SMALL LETTER DIGAMMA +03DF ; Lowercase # L& GREEK SMALL LETTER KOPPA +03E1 ; Lowercase # L& GREEK SMALL LETTER SAMPI +03E3 ; Lowercase # L& COPTIC SMALL LETTER SHEI +03E5 ; Lowercase # L& COPTIC SMALL LETTER FEI +03E7 ; Lowercase # L& COPTIC SMALL LETTER KHEI +03E9 ; Lowercase # L& COPTIC SMALL LETTER HORI +03EB ; Lowercase # L& COPTIC SMALL LETTER GANGIA +03ED ; Lowercase # L& COPTIC SMALL LETTER SHIMA +03EF..03F3 ; Lowercase # L& [5] COPTIC SMALL LETTER DEI..GREEK LETTER YOT +03F5 ; Lowercase # L& GREEK LUNATE EPSILON SYMBOL +0430..045F ; Lowercase # L& [48] CYRILLIC SMALL LETTER A..CYRILLIC SMALL LETTER DZHE +0461 ; Lowercase # L& CYRILLIC SMALL LETTER OMEGA +0463 ; Lowercase # L& CYRILLIC SMALL LETTER YAT +0465 ; Lowercase # L& CYRILLIC SMALL LETTER IOTIFIED E +0467 ; Lowercase # L& CYRILLIC SMALL LETTER LITTLE YUS +0469 ; Lowercase # L& CYRILLIC SMALL LETTER IOTIFIED LITTLE YUS +046B ; Lowercase # L& CYRILLIC SMALL LETTER BIG YUS +046D ; Lowercase # L& CYRILLIC SMALL LETTER IOTIFIED BIG YUS +046F ; Lowercase # L& CYRILLIC SMALL LETTER KSI +0471 ; Lowercase # L& CYRILLIC SMALL LETTER PSI +0473 ; Lowercase # L& CYRILLIC SMALL LETTER FITA +0475 ; Lowercase # L& CYRILLIC SMALL LETTER IZHITSA +0477 ; Lowercase # L& CYRILLIC SMALL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT +0479 ; Lowercase # L& CYRILLIC SMALL LETTER UK +047B ; Lowercase # L& CYRILLIC SMALL LETTER ROUND OMEGA +047D ; Lowercase # L& CYRILLIC SMALL LETTER OMEGA WITH TITLO +047F ; Lowercase # L& CYRILLIC SMALL LETTER OT +0481 ; Lowercase # L& CYRILLIC SMALL LETTER KOPPA +048B ; Lowercase # L& CYRILLIC SMALL LETTER SHORT I WITH TAIL +048D ; Lowercase # L& CYRILLIC SMALL LETTER SEMISOFT SIGN +048F ; Lowercase # L& CYRILLIC SMALL LETTER ER WITH TICK +0491 ; Lowercase # L& CYRILLIC SMALL LETTER GHE WITH UPTURN +0493 ; Lowercase # L& CYRILLIC SMALL LETTER GHE WITH STROKE +0495 ; Lowercase # L& CYRILLIC SMALL LETTER GHE WITH MIDDLE HOOK +0497 ; Lowercase # L& CYRILLIC SMALL LETTER ZHE WITH DESCENDER +0499 ; Lowercase # L& CYRILLIC SMALL LETTER ZE WITH DESCENDER +049B ; Lowercase # L& CYRILLIC SMALL LETTER KA WITH DESCENDER +049D ; Lowercase # L& CYRILLIC SMALL LETTER KA WITH VERTICAL STROKE +049F ; Lowercase # L& CYRILLIC SMALL LETTER KA WITH STROKE +04A1 ; Lowercase # L& CYRILLIC SMALL LETTER BASHKIR KA +04A3 ; Lowercase # L& CYRILLIC SMALL LETTER EN WITH DESCENDER +04A5 ; Lowercase # L& CYRILLIC SMALL LIGATURE EN GHE +04A7 ; Lowercase # L& CYRILLIC SMALL LETTER PE WITH MIDDLE HOOK +04A9 ; Lowercase # L& CYRILLIC SMALL LETTER ABKHASIAN HA +04AB ; Lowercase # L& CYRILLIC SMALL LETTER ES WITH DESCENDER +04AD ; Lowercase # L& CYRILLIC SMALL LETTER TE WITH DESCENDER +04AF ; Lowercase # L& CYRILLIC SMALL LETTER STRAIGHT U +04B1 ; Lowercase # L& CYRILLIC SMALL LETTER STRAIGHT U WITH STROKE +04B3 ; Lowercase # L& CYRILLIC SMALL LETTER HA WITH DESCENDER +04B5 ; Lowercase # L& CYRILLIC SMALL LIGATURE TE TSE +04B7 ; Lowercase # L& CYRILLIC SMALL LETTER CHE WITH DESCENDER +04B9 ; Lowercase # L& CYRILLIC SMALL LETTER CHE WITH VERTICAL STROKE +04BB ; Lowercase # L& CYRILLIC SMALL LETTER SHHA +04BD ; Lowercase # L& CYRILLIC SMALL LETTER ABKHASIAN CHE +04BF ; Lowercase # L& CYRILLIC SMALL LETTER ABKHASIAN CHE WITH DESCENDER +04C2 ; Lowercase # L& CYRILLIC SMALL LETTER ZHE WITH BREVE +04C4 ; Lowercase # L& CYRILLIC SMALL LETTER KA WITH HOOK +04C6 ; Lowercase # L& CYRILLIC SMALL LETTER EL WITH TAIL +04C8 ; Lowercase # L& CYRILLIC SMALL LETTER EN WITH HOOK +04CA ; Lowercase # L& CYRILLIC SMALL LETTER EN WITH TAIL +04CC ; Lowercase # L& CYRILLIC SMALL LETTER KHAKASSIAN CHE +04CE ; Lowercase # L& CYRILLIC SMALL LETTER EM WITH TAIL +04D1 ; Lowercase # L& CYRILLIC SMALL LETTER A WITH BREVE +04D3 ; Lowercase # L& CYRILLIC SMALL LETTER A WITH DIAERESIS +04D5 ; Lowercase # L& CYRILLIC SMALL LIGATURE A IE +04D7 ; Lowercase # L& CYRILLIC SMALL LETTER IE WITH BREVE +04D9 ; Lowercase # L& CYRILLIC SMALL LETTER SCHWA +04DB ; Lowercase # L& CYRILLIC SMALL LETTER SCHWA WITH DIAERESIS +04DD ; Lowercase # L& CYRILLIC SMALL LETTER ZHE WITH DIAERESIS +04DF ; Lowercase # L& CYRILLIC SMALL LETTER ZE WITH DIAERESIS +04E1 ; Lowercase # L& CYRILLIC SMALL LETTER ABKHASIAN DZE +04E3 ; Lowercase # L& CYRILLIC SMALL LETTER I WITH MACRON +04E5 ; Lowercase # L& CYRILLIC SMALL LETTER I WITH DIAERESIS +04E7 ; Lowercase # L& CYRILLIC SMALL LETTER O WITH DIAERESIS +04E9 ; Lowercase # L& CYRILLIC SMALL LETTER BARRED O +04EB ; Lowercase # L& CYRILLIC SMALL LETTER BARRED O WITH DIAERESIS +04ED ; Lowercase # L& CYRILLIC SMALL LETTER E WITH DIAERESIS +04EF ; Lowercase # L& CYRILLIC SMALL LETTER U WITH MACRON +04F1 ; Lowercase # L& CYRILLIC SMALL LETTER U WITH DIAERESIS +04F3 ; Lowercase # L& CYRILLIC SMALL LETTER U WITH DOUBLE ACUTE +04F5 ; Lowercase # L& CYRILLIC SMALL LETTER CHE WITH DIAERESIS +04F9 ; Lowercase # L& CYRILLIC SMALL LETTER YERU WITH DIAERESIS +0501 ; Lowercase # L& CYRILLIC SMALL LETTER KOMI DE +0503 ; Lowercase # L& CYRILLIC SMALL LETTER KOMI DJE +0505 ; Lowercase # L& CYRILLIC SMALL LETTER KOMI ZJE +0507 ; Lowercase # L& CYRILLIC SMALL LETTER KOMI DZJE +0509 ; Lowercase # L& CYRILLIC SMALL LETTER KOMI LJE +050B ; Lowercase # L& CYRILLIC SMALL LETTER KOMI NJE +050D ; Lowercase # L& CYRILLIC SMALL LETTER KOMI SJE +050F ; Lowercase # L& CYRILLIC SMALL LETTER KOMI TJE +0561..0587 ; Lowercase # L& [39] ARMENIAN SMALL LETTER AYB..ARMENIAN SMALL LIGATURE ECH YIWN +1E01 ; Lowercase # L& LATIN SMALL LETTER A WITH RING BELOW +1E03 ; Lowercase # L& LATIN SMALL LETTER B WITH DOT ABOVE +1E05 ; Lowercase # L& LATIN SMALL LETTER B WITH DOT BELOW +1E07 ; Lowercase # L& LATIN SMALL LETTER B WITH LINE BELOW +1E09 ; Lowercase # L& LATIN SMALL LETTER C WITH CEDILLA AND ACUTE +1E0B ; Lowercase # L& LATIN SMALL LETTER D WITH DOT ABOVE +1E0D ; Lowercase # L& LATIN SMALL LETTER D WITH DOT BELOW +1E0F ; Lowercase # L& LATIN SMALL LETTER D WITH LINE BELOW +1E11 ; Lowercase # L& LATIN SMALL LETTER D WITH CEDILLA +1E13 ; Lowercase # L& LATIN SMALL LETTER D WITH CIRCUMFLEX BELOW +1E15 ; Lowercase # L& LATIN SMALL LETTER E WITH MACRON AND GRAVE +1E17 ; Lowercase # L& LATIN SMALL LETTER E WITH MACRON AND ACUTE +1E19 ; Lowercase # L& LATIN SMALL LETTER E WITH CIRCUMFLEX BELOW +1E1B ; Lowercase # L& LATIN SMALL LETTER E WITH TILDE BELOW +1E1D ; Lowercase # L& LATIN SMALL LETTER E WITH CEDILLA AND BREVE +1E1F ; Lowercase # L& LATIN SMALL LETTER F WITH DOT ABOVE +1E21 ; Lowercase # L& LATIN SMALL LETTER G WITH MACRON +1E23 ; Lowercase # L& LATIN SMALL LETTER H WITH DOT ABOVE +1E25 ; Lowercase # L& LATIN SMALL LETTER H WITH DOT BELOW +1E27 ; Lowercase # L& LATIN SMALL LETTER H WITH DIAERESIS +1E29 ; Lowercase # L& LATIN SMALL LETTER H WITH CEDILLA +1E2B ; Lowercase # L& LATIN SMALL LETTER H WITH BREVE BELOW +1E2D ; Lowercase # L& LATIN SMALL LETTER I WITH TILDE BELOW +1E2F ; Lowercase # L& LATIN SMALL LETTER I WITH DIAERESIS AND ACUTE +1E31 ; Lowercase # L& LATIN SMALL LETTER K WITH ACUTE +1E33 ; Lowercase # L& LATIN SMALL LETTER K WITH DOT BELOW +1E35 ; Lowercase # L& LATIN SMALL LETTER K WITH LINE BELOW +1E37 ; Lowercase # L& LATIN SMALL LETTER L WITH DOT BELOW +1E39 ; Lowercase # L& LATIN SMALL LETTER L WITH DOT BELOW AND MACRON +1E3B ; Lowercase # L& LATIN SMALL LETTER L WITH LINE BELOW +1E3D ; Lowercase # L& LATIN SMALL LETTER L WITH CIRCUMFLEX BELOW +1E3F ; Lowercase # L& LATIN SMALL LETTER M WITH ACUTE +1E41 ; Lowercase # L& LATIN SMALL LETTER M WITH DOT ABOVE +1E43 ; Lowercase # L& LATIN SMALL LETTER M WITH DOT BELOW +1E45 ; Lowercase # L& LATIN SMALL LETTER N WITH DOT ABOVE +1E47 ; Lowercase # L& LATIN SMALL LETTER N WITH DOT BELOW +1E49 ; Lowercase # L& LATIN SMALL LETTER N WITH LINE BELOW +1E4B ; Lowercase # L& LATIN SMALL LETTER N WITH CIRCUMFLEX BELOW +1E4D ; Lowercase # L& LATIN SMALL LETTER O WITH TILDE AND ACUTE +1E4F ; Lowercase # L& LATIN SMALL LETTER O WITH TILDE AND DIAERESIS +1E51 ; Lowercase # L& LATIN SMALL LETTER O WITH MACRON AND GRAVE +1E53 ; Lowercase # L& LATIN SMALL LETTER O WITH MACRON AND ACUTE +1E55 ; Lowercase # L& LATIN SMALL LETTER P WITH ACUTE +1E57 ; Lowercase # L& LATIN SMALL LETTER P WITH DOT ABOVE +1E59 ; Lowercase # L& LATIN SMALL LETTER R WITH DOT ABOVE +1E5B ; Lowercase # L& LATIN SMALL LETTER R WITH DOT BELOW +1E5D ; Lowercase # L& LATIN SMALL LETTER R WITH DOT BELOW AND MACRON +1E5F ; Lowercase # L& LATIN SMALL LETTER R WITH LINE BELOW +1E61 ; Lowercase # L& LATIN SMALL LETTER S WITH DOT ABOVE +1E63 ; Lowercase # L& LATIN SMALL LETTER S WITH DOT BELOW +1E65 ; Lowercase # L& LATIN SMALL LETTER S WITH ACUTE AND DOT ABOVE +1E67 ; Lowercase # L& LATIN SMALL LETTER S WITH CARON AND DOT ABOVE +1E69 ; Lowercase # L& LATIN SMALL LETTER S WITH DOT BELOW AND DOT ABOVE +1E6B ; Lowercase # L& LATIN SMALL LETTER T WITH DOT ABOVE +1E6D ; Lowercase # L& LATIN SMALL LETTER T WITH DOT BELOW +1E6F ; Lowercase # L& LATIN SMALL LETTER T WITH LINE BELOW +1E71 ; Lowercase # L& LATIN SMALL LETTER T WITH CIRCUMFLEX BELOW +1E73 ; Lowercase # L& LATIN SMALL LETTER U WITH DIAERESIS BELOW +1E75 ; Lowercase # L& LATIN SMALL LETTER U WITH TILDE BELOW +1E77 ; Lowercase # L& LATIN SMALL LETTER U WITH CIRCUMFLEX BELOW +1E79 ; Lowercase # L& LATIN SMALL LETTER U WITH TILDE AND ACUTE +1E7B ; Lowercase # L& LATIN SMALL LETTER U WITH MACRON AND DIAERESIS +1E7D ; Lowercase # L& LATIN SMALL LETTER V WITH TILDE +1E7F ; Lowercase # L& LATIN SMALL LETTER V WITH DOT BELOW +1E81 ; Lowercase # L& LATIN SMALL LETTER W WITH GRAVE +1E83 ; Lowercase # L& LATIN SMALL LETTER W WITH ACUTE +1E85 ; Lowercase # L& LATIN SMALL LETTER W WITH DIAERESIS +1E87 ; Lowercase # L& LATIN SMALL LETTER W WITH DOT ABOVE +1E89 ; Lowercase # L& LATIN SMALL LETTER W WITH DOT BELOW +1E8B ; Lowercase # L& LATIN SMALL LETTER X WITH DOT ABOVE +1E8D ; Lowercase # L& LATIN SMALL LETTER X WITH DIAERESIS +1E8F ; Lowercase # L& LATIN SMALL LETTER Y WITH DOT ABOVE +1E91 ; Lowercase # L& LATIN SMALL LETTER Z WITH CIRCUMFLEX +1E93 ; Lowercase # L& LATIN SMALL LETTER Z WITH DOT BELOW +1E95..1E9B ; Lowercase # L& [7] LATIN SMALL LETTER Z WITH LINE BELOW..LATIN SMALL LETTER LONG S WITH DOT ABOVE +1EA1 ; Lowercase # L& LATIN SMALL LETTER A WITH DOT BELOW +1EA3 ; Lowercase # L& LATIN SMALL LETTER A WITH HOOK ABOVE +1EA5 ; Lowercase # L& LATIN SMALL LETTER A WITH CIRCUMFLEX AND ACUTE +1EA7 ; Lowercase # L& LATIN SMALL LETTER A WITH CIRCUMFLEX AND GRAVE +1EA9 ; Lowercase # L& LATIN SMALL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE +1EAB ; Lowercase # L& LATIN SMALL LETTER A WITH CIRCUMFLEX AND TILDE +1EAD ; Lowercase # L& LATIN SMALL LETTER A WITH CIRCUMFLEX AND DOT BELOW +1EAF ; Lowercase # L& LATIN SMALL LETTER A WITH BREVE AND ACUTE +1EB1 ; Lowercase # L& LATIN SMALL LETTER A WITH BREVE AND GRAVE +1EB3 ; Lowercase # L& LATIN SMALL LETTER A WITH BREVE AND HOOK ABOVE +1EB5 ; Lowercase # L& LATIN SMALL LETTER A WITH BREVE AND TILDE +1EB7 ; Lowercase # L& LATIN SMALL LETTER A WITH BREVE AND DOT BELOW +1EB9 ; Lowercase # L& LATIN SMALL LETTER E WITH DOT BELOW +1EBB ; Lowercase # L& LATIN SMALL LETTER E WITH HOOK ABOVE +1EBD ; Lowercase # L& LATIN SMALL LETTER E WITH TILDE +1EBF ; Lowercase # L& LATIN SMALL LETTER E WITH CIRCUMFLEX AND ACUTE +1EC1 ; Lowercase # L& LATIN SMALL LETTER E WITH CIRCUMFLEX AND GRAVE +1EC3 ; Lowercase # L& LATIN SMALL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE +1EC5 ; Lowercase # L& LATIN SMALL LETTER E WITH CIRCUMFLEX AND TILDE +1EC7 ; Lowercase # L& LATIN SMALL LETTER E WITH CIRCUMFLEX AND DOT BELOW +1EC9 ; Lowercase # L& LATIN SMALL LETTER I WITH HOOK ABOVE +1ECB ; Lowercase # L& LATIN SMALL LETTER I WITH DOT BELOW +1ECD ; Lowercase # L& LATIN SMALL LETTER O WITH DOT BELOW +1ECF ; Lowercase # L& LATIN SMALL LETTER O WITH HOOK ABOVE +1ED1 ; Lowercase # L& LATIN SMALL LETTER O WITH CIRCUMFLEX AND ACUTE +1ED3 ; Lowercase # L& LATIN SMALL LETTER O WITH CIRCUMFLEX AND GRAVE +1ED5 ; Lowercase # L& LATIN SMALL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE +1ED7 ; Lowercase # L& LATIN SMALL LETTER O WITH CIRCUMFLEX AND TILDE +1ED9 ; Lowercase # L& LATIN SMALL LETTER O WITH CIRCUMFLEX AND DOT BELOW +1EDB ; Lowercase # L& LATIN SMALL LETTER O WITH HORN AND ACUTE +1EDD ; Lowercase # L& LATIN SMALL LETTER O WITH HORN AND GRAVE +1EDF ; Lowercase # L& LATIN SMALL LETTER O WITH HORN AND HOOK ABOVE +1EE1 ; Lowercase # L& LATIN SMALL LETTER O WITH HORN AND TILDE +1EE3 ; Lowercase # L& LATIN SMALL LETTER O WITH HORN AND DOT BELOW +1EE5 ; Lowercase # L& LATIN SMALL LETTER U WITH DOT BELOW +1EE7 ; Lowercase # L& LATIN SMALL LETTER U WITH HOOK ABOVE +1EE9 ; Lowercase # L& LATIN SMALL LETTER U WITH HORN AND ACUTE +1EEB ; Lowercase # L& LATIN SMALL LETTER U WITH HORN AND GRAVE +1EED ; Lowercase # L& LATIN SMALL LETTER U WITH HORN AND HOOK ABOVE +1EEF ; Lowercase # L& LATIN SMALL LETTER U WITH HORN AND TILDE +1EF1 ; Lowercase # L& LATIN SMALL LETTER U WITH HORN AND DOT BELOW +1EF3 ; Lowercase # L& LATIN SMALL LETTER Y WITH GRAVE +1EF5 ; Lowercase # L& LATIN SMALL LETTER Y WITH DOT BELOW +1EF7 ; Lowercase # L& LATIN SMALL LETTER Y WITH HOOK ABOVE +1EF9 ; Lowercase # L& LATIN SMALL LETTER Y WITH TILDE +1F00..1F07 ; Lowercase # L& [8] GREEK SMALL LETTER ALPHA WITH PSILI..GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI +1F10..1F15 ; Lowercase # L& [6] GREEK SMALL LETTER EPSILON WITH PSILI..GREEK SMALL LETTER EPSILON WITH DASIA AND OXIA +1F20..1F27 ; Lowercase # L& [8] GREEK SMALL LETTER ETA WITH PSILI..GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI +1F30..1F37 ; Lowercase # L& [8] GREEK SMALL LETTER IOTA WITH PSILI..GREEK SMALL LETTER IOTA WITH DASIA AND PERISPOMENI +1F40..1F45 ; Lowercase # L& [6] GREEK SMALL LETTER OMICRON WITH PSILI..GREEK SMALL LETTER OMICRON WITH DASIA AND OXIA +1F50..1F57 ; Lowercase # L& [8] GREEK SMALL LETTER UPSILON WITH PSILI..GREEK SMALL LETTER UPSILON WITH DASIA AND PERISPOMENI +1F60..1F67 ; Lowercase # L& [8] GREEK SMALL LETTER OMEGA WITH PSILI..GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI +1F70..1F7D ; Lowercase # L& [14] GREEK SMALL LETTER ALPHA WITH VARIA..GREEK SMALL LETTER OMEGA WITH OXIA +1F80..1F87 ; Lowercase # L& [8] GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI..GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI +1F90..1F97 ; Lowercase # L& [8] GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI..GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI +1FA0..1FA7 ; Lowercase # L& [8] GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI..GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI +1FB0..1FB4 ; Lowercase # L& [5] GREEK SMALL LETTER ALPHA WITH VRACHY..GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI +1FB6..1FB7 ; Lowercase # L& [2] GREEK SMALL LETTER ALPHA WITH PERISPOMENI..GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI +1FBE ; Lowercase # L& GREEK PROSGEGRAMMENI +1FC2..1FC4 ; Lowercase # L& [3] GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI +1FC6..1FC7 ; Lowercase # L& [2] GREEK SMALL LETTER ETA WITH PERISPOMENI..GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI +1FD0..1FD3 ; Lowercase # L& [4] GREEK SMALL LETTER IOTA WITH VRACHY..GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA +1FD6..1FD7 ; Lowercase # L& [2] GREEK SMALL LETTER IOTA WITH PERISPOMENI..GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI +1FE0..1FE7 ; Lowercase # L& [8] GREEK SMALL LETTER UPSILON WITH VRACHY..GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI +1FF2..1FF4 ; Lowercase # L& [3] GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI +1FF6..1FF7 ; Lowercase # L& [2] GREEK SMALL LETTER OMEGA WITH PERISPOMENI..GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI +2071 ; Lowercase # L& SUPERSCRIPT LATIN SMALL LETTER I +207F ; Lowercase # L& SUPERSCRIPT LATIN SMALL LETTER N +210A ; Lowercase # L& SCRIPT SMALL G +210E..210F ; Lowercase # L& [2] PLANCK CONSTANT..PLANCK CONSTANT OVER TWO PI +2113 ; Lowercase # L& SCRIPT SMALL L +212F ; Lowercase # L& SCRIPT SMALL E +2134 ; Lowercase # L& SCRIPT SMALL O +2139 ; Lowercase # L& INFORMATION SOURCE +213D ; Lowercase # L& DOUBLE-STRUCK SMALL GAMMA +2146..2149 ; Lowercase # L& [4] DOUBLE-STRUCK ITALIC SMALL D..DOUBLE-STRUCK ITALIC SMALL J +2170..217F ; Lowercase # Nl [16] SMALL ROMAN NUMERAL ONE..SMALL ROMAN NUMERAL ONE THOUSAND +24D0..24E9 ; Lowercase # So [26] CIRCLED LATIN SMALL LETTER A..CIRCLED LATIN SMALL LETTER Z +FB00..FB06 ; Lowercase # L& [7] LATIN SMALL LIGATURE FF..LATIN SMALL LIGATURE ST +FB13..FB17 ; Lowercase # L& [5] ARMENIAN SMALL LIGATURE MEN NOW..ARMENIAN SMALL LIGATURE MEN XEH +FF41..FF5A ; Lowercase # L& [26] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH LATIN SMALL LETTER Z +10428..1044D ; Lowercase # L& [38] DESERET SMALL LETTER LONG I..DESERET SMALL LETTER ENG +1D41A..1D433 ; Lowercase # L& [26] MATHEMATICAL BOLD SMALL A..MATHEMATICAL BOLD SMALL Z +1D44E..1D454 ; Lowercase # L& [7] MATHEMATICAL ITALIC SMALL A..MATHEMATICAL ITALIC SMALL G +1D456..1D467 ; Lowercase # L& [18] MATHEMATICAL ITALIC SMALL I..MATHEMATICAL ITALIC SMALL Z +1D482..1D49B ; Lowercase # L& [26] MATHEMATICAL BOLD ITALIC SMALL A..MATHEMATICAL BOLD ITALIC SMALL Z +1D4B6..1D4B9 ; Lowercase # L& [4] MATHEMATICAL SCRIPT SMALL A..MATHEMATICAL SCRIPT SMALL D +1D4BB ; Lowercase # L& MATHEMATICAL SCRIPT SMALL F +1D4BD..1D4C0 ; Lowercase # L& [4] MATHEMATICAL SCRIPT SMALL H..MATHEMATICAL SCRIPT SMALL K +1D4C2..1D4C3 ; Lowercase # L& [2] MATHEMATICAL SCRIPT SMALL M..MATHEMATICAL SCRIPT SMALL N +1D4C5..1D4CF ; Lowercase # L& [11] MATHEMATICAL SCRIPT SMALL P..MATHEMATICAL SCRIPT SMALL Z +1D4EA..1D503 ; Lowercase # L& [26] MATHEMATICAL BOLD SCRIPT SMALL A..MATHEMATICAL BOLD SCRIPT SMALL Z +1D51E..1D537 ; Lowercase # L& [26] MATHEMATICAL FRAKTUR SMALL A..MATHEMATICAL FRAKTUR SMALL Z +1D552..1D56B ; Lowercase # L& [26] MATHEMATICAL DOUBLE-STRUCK SMALL A..MATHEMATICAL DOUBLE-STRUCK SMALL Z +1D586..1D59F ; Lowercase # L& [26] MATHEMATICAL BOLD FRAKTUR SMALL A..MATHEMATICAL BOLD FRAKTUR SMALL Z +1D5BA..1D5D3 ; Lowercase # L& [26] MATHEMATICAL SANS-SERIF SMALL A..MATHEMATICAL SANS-SERIF SMALL Z +1D5EE..1D607 ; Lowercase # L& [26] MATHEMATICAL SANS-SERIF BOLD SMALL A..MATHEMATICAL SANS-SERIF BOLD SMALL Z +1D622..1D63B ; Lowercase # L& [26] MATHEMATICAL SANS-SERIF ITALIC SMALL A..MATHEMATICAL SANS-SERIF ITALIC SMALL Z +1D656..1D66F ; Lowercase # L& [26] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL A..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL Z +1D68A..1D6A3 ; Lowercase # L& [26] MATHEMATICAL MONOSPACE SMALL A..MATHEMATICAL MONOSPACE SMALL Z +1D6C2..1D6DA ; Lowercase # L& [25] MATHEMATICAL BOLD SMALL ALPHA..MATHEMATICAL BOLD SMALL OMEGA +1D6DC..1D6E1 ; Lowercase # L& [6] MATHEMATICAL BOLD EPSILON SYMBOL..MATHEMATICAL BOLD PI SYMBOL +1D6FC..1D714 ; Lowercase # L& [25] MATHEMATICAL ITALIC SMALL ALPHA..MATHEMATICAL ITALIC SMALL OMEGA +1D716..1D71B ; Lowercase # L& [6] MATHEMATICAL ITALIC EPSILON SYMBOL..MATHEMATICAL ITALIC PI SYMBOL +1D736..1D74E ; Lowercase # L& [25] MATHEMATICAL BOLD ITALIC SMALL ALPHA..MATHEMATICAL BOLD ITALIC SMALL OMEGA +1D750..1D755 ; Lowercase # L& [6] MATHEMATICAL BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD ITALIC PI SYMBOL +1D770..1D788 ; Lowercase # L& [25] MATHEMATICAL SANS-SERIF BOLD SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD SMALL OMEGA +1D78A..1D78F ; Lowercase # L& [6] MATHEMATICAL SANS-SERIF BOLD EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD PI SYMBOL +1D7AA..1D7C2 ; Lowercase # L& [25] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL OMEGA +1D7C4..1D7C9 ; Lowercase # L& [6] MATHEMATICAL SANS-SERIF BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC PI SYMBOL + +# Total code points: 1410 + +# ================================================ + +# Derived Property: Uppercase +# Generated from: Lu + Other_Uppercase + +0041..005A ; Uppercase # L& [26] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER Z +00C0..00D6 ; Uppercase # L& [23] LATIN CAPITAL LETTER A WITH GRAVE..LATIN CAPITAL LETTER O WITH DIAERESIS +00D8..00DE ; Uppercase # L& [7] LATIN CAPITAL LETTER O WITH STROKE..LATIN CAPITAL LETTER THORN +0100 ; Uppercase # L& LATIN CAPITAL LETTER A WITH MACRON +0102 ; Uppercase # L& LATIN CAPITAL LETTER A WITH BREVE +0104 ; Uppercase # L& LATIN CAPITAL LETTER A WITH OGONEK +0106 ; Uppercase # L& LATIN CAPITAL LETTER C WITH ACUTE +0108 ; Uppercase # L& LATIN CAPITAL LETTER C WITH CIRCUMFLEX +010A ; Uppercase # L& LATIN CAPITAL LETTER C WITH DOT ABOVE +010C ; Uppercase # L& LATIN CAPITAL LETTER C WITH CARON +010E ; Uppercase # L& LATIN CAPITAL LETTER D WITH CARON +0110 ; Uppercase # L& LATIN CAPITAL LETTER D WITH STROKE +0112 ; Uppercase # L& LATIN CAPITAL LETTER E WITH MACRON +0114 ; Uppercase # L& LATIN CAPITAL LETTER E WITH BREVE +0116 ; Uppercase # L& LATIN CAPITAL LETTER E WITH DOT ABOVE +0118 ; Uppercase # L& LATIN CAPITAL LETTER E WITH OGONEK +011A ; Uppercase # L& LATIN CAPITAL LETTER E WITH CARON +011C ; Uppercase # L& LATIN CAPITAL LETTER G WITH CIRCUMFLEX +011E ; Uppercase # L& LATIN CAPITAL LETTER G WITH BREVE +0120 ; Uppercase # L& LATIN CAPITAL LETTER G WITH DOT ABOVE +0122 ; Uppercase # L& LATIN CAPITAL LETTER G WITH CEDILLA +0124 ; Uppercase # L& LATIN CAPITAL LETTER H WITH CIRCUMFLEX +0126 ; Uppercase # L& LATIN CAPITAL LETTER H WITH STROKE +0128 ; Uppercase # L& LATIN CAPITAL LETTER I WITH TILDE +012A ; Uppercase # L& LATIN CAPITAL LETTER I WITH MACRON +012C ; Uppercase # L& LATIN CAPITAL LETTER I WITH BREVE +012E ; Uppercase # L& LATIN CAPITAL LETTER I WITH OGONEK +0130 ; Uppercase # L& LATIN CAPITAL LETTER I WITH DOT ABOVE +0132 ; Uppercase # L& LATIN CAPITAL LIGATURE IJ +0134 ; Uppercase # L& LATIN CAPITAL LETTER J WITH CIRCUMFLEX +0136 ; Uppercase # L& LATIN CAPITAL LETTER K WITH CEDILLA +0139 ; Uppercase # L& LATIN CAPITAL LETTER L WITH ACUTE +013B ; Uppercase # L& LATIN CAPITAL LETTER L WITH CEDILLA +013D ; Uppercase # L& LATIN CAPITAL LETTER L WITH CARON +013F ; Uppercase # L& LATIN CAPITAL LETTER L WITH MIDDLE DOT +0141 ; Uppercase # L& LATIN CAPITAL LETTER L WITH STROKE +0143 ; Uppercase # L& LATIN CAPITAL LETTER N WITH ACUTE +0145 ; Uppercase # L& LATIN CAPITAL LETTER N WITH CEDILLA +0147 ; Uppercase # L& LATIN CAPITAL LETTER N WITH CARON +014A ; Uppercase # L& LATIN CAPITAL LETTER ENG +014C ; Uppercase # L& LATIN CAPITAL LETTER O WITH MACRON +014E ; Uppercase # L& LATIN CAPITAL LETTER O WITH BREVE +0150 ; Uppercase # L& LATIN CAPITAL LETTER O WITH DOUBLE ACUTE +0152 ; Uppercase # L& LATIN CAPITAL LIGATURE OE +0154 ; Uppercase # L& LATIN CAPITAL LETTER R WITH ACUTE +0156 ; Uppercase # L& LATIN CAPITAL LETTER R WITH CEDILLA +0158 ; Uppercase # L& LATIN CAPITAL LETTER R WITH CARON +015A ; Uppercase # L& LATIN CAPITAL LETTER S WITH ACUTE +015C ; Uppercase # L& LATIN CAPITAL LETTER S WITH CIRCUMFLEX +015E ; Uppercase # L& LATIN CAPITAL LETTER S WITH CEDILLA +0160 ; Uppercase # L& LATIN CAPITAL LETTER S WITH CARON +0162 ; Uppercase # L& LATIN CAPITAL LETTER T WITH CEDILLA +0164 ; Uppercase # L& LATIN CAPITAL LETTER T WITH CARON +0166 ; Uppercase # L& LATIN CAPITAL LETTER T WITH STROKE +0168 ; Uppercase # L& LATIN CAPITAL LETTER U WITH TILDE +016A ; Uppercase # L& LATIN CAPITAL LETTER U WITH MACRON +016C ; Uppercase # L& LATIN CAPITAL LETTER U WITH BREVE +016E ; Uppercase # L& LATIN CAPITAL LETTER U WITH RING ABOVE +0170 ; Uppercase # L& LATIN CAPITAL LETTER U WITH DOUBLE ACUTE +0172 ; Uppercase # L& LATIN CAPITAL LETTER U WITH OGONEK +0174 ; Uppercase # L& LATIN CAPITAL LETTER W WITH CIRCUMFLEX +0176 ; Uppercase # L& LATIN CAPITAL LETTER Y WITH CIRCUMFLEX +0178..0179 ; Uppercase # L& [2] LATIN CAPITAL LETTER Y WITH DIAERESIS..LATIN CAPITAL LETTER Z WITH ACUTE +017B ; Uppercase # L& LATIN CAPITAL LETTER Z WITH DOT ABOVE +017D ; Uppercase # L& LATIN CAPITAL LETTER Z WITH CARON +0181..0182 ; Uppercase # L& [2] LATIN CAPITAL LETTER B WITH HOOK..LATIN CAPITAL LETTER B WITH TOPBAR +0184 ; Uppercase # L& LATIN CAPITAL LETTER TONE SIX +0186..0187 ; Uppercase # L& [2] LATIN CAPITAL LETTER OPEN O..LATIN CAPITAL LETTER C WITH HOOK +0189..018B ; Uppercase # L& [3] LATIN CAPITAL LETTER AFRICAN D..LATIN CAPITAL LETTER D WITH TOPBAR +018E..0191 ; Uppercase # L& [4] LATIN CAPITAL LETTER REVERSED E..LATIN CAPITAL LETTER F WITH HOOK +0193..0194 ; Uppercase # L& [2] LATIN CAPITAL LETTER G WITH HOOK..LATIN CAPITAL LETTER GAMMA +0196..0198 ; Uppercase # L& [3] LATIN CAPITAL LETTER IOTA..LATIN CAPITAL LETTER K WITH HOOK +019C..019D ; Uppercase # L& [2] LATIN CAPITAL LETTER TURNED M..LATIN CAPITAL LETTER N WITH LEFT HOOK +019F..01A0 ; Uppercase # L& [2] LATIN CAPITAL LETTER O WITH MIDDLE TILDE..LATIN CAPITAL LETTER O WITH HORN +01A2 ; Uppercase # L& LATIN CAPITAL LETTER OI +01A4 ; Uppercase # L& LATIN CAPITAL LETTER P WITH HOOK +01A6..01A7 ; Uppercase # L& [2] LATIN LETTER YR..LATIN CAPITAL LETTER TONE TWO +01A9 ; Uppercase # L& LATIN CAPITAL LETTER ESH +01AC ; Uppercase # L& LATIN CAPITAL LETTER T WITH HOOK +01AE..01AF ; Uppercase # L& [2] LATIN CAPITAL LETTER T WITH RETROFLEX HOOK..LATIN CAPITAL LETTER U WITH HORN +01B1..01B3 ; Uppercase # L& [3] LATIN CAPITAL LETTER UPSILON..LATIN CAPITAL LETTER Y WITH HOOK +01B5 ; Uppercase # L& LATIN CAPITAL LETTER Z WITH STROKE +01B7..01B8 ; Uppercase # L& [2] LATIN CAPITAL LETTER EZH..LATIN CAPITAL LETTER EZH REVERSED +01BC ; Uppercase # L& LATIN CAPITAL LETTER TONE FIVE +01C4 ; Uppercase # L& LATIN CAPITAL LETTER DZ WITH CARON +01C7 ; Uppercase # L& LATIN CAPITAL LETTER LJ +01CA ; Uppercase # L& LATIN CAPITAL LETTER NJ +01CD ; Uppercase # L& LATIN CAPITAL LETTER A WITH CARON +01CF ; Uppercase # L& LATIN CAPITAL LETTER I WITH CARON +01D1 ; Uppercase # L& LATIN CAPITAL LETTER O WITH CARON +01D3 ; Uppercase # L& LATIN CAPITAL LETTER U WITH CARON +01D5 ; Uppercase # L& LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON +01D7 ; Uppercase # L& LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE +01D9 ; Uppercase # L& LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON +01DB ; Uppercase # L& LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE +01DE ; Uppercase # L& LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON +01E0 ; Uppercase # L& LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON +01E2 ; Uppercase # L& LATIN CAPITAL LETTER AE WITH MACRON +01E4 ; Uppercase # L& LATIN CAPITAL LETTER G WITH STROKE +01E6 ; Uppercase # L& LATIN CAPITAL LETTER G WITH CARON +01E8 ; Uppercase # L& LATIN CAPITAL LETTER K WITH CARON +01EA ; Uppercase # L& LATIN CAPITAL LETTER O WITH OGONEK +01EC ; Uppercase # L& LATIN CAPITAL LETTER O WITH OGONEK AND MACRON +01EE ; Uppercase # L& LATIN CAPITAL LETTER EZH WITH CARON +01F1 ; Uppercase # L& LATIN CAPITAL LETTER DZ +01F4 ; Uppercase # L& LATIN CAPITAL LETTER G WITH ACUTE +01F6..01F8 ; Uppercase # L& [3] LATIN CAPITAL LETTER HWAIR..LATIN CAPITAL LETTER N WITH GRAVE +01FA ; Uppercase # L& LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE +01FC ; Uppercase # L& LATIN CAPITAL LETTER AE WITH ACUTE +01FE ; Uppercase # L& LATIN CAPITAL LETTER O WITH STROKE AND ACUTE +0200 ; Uppercase # L& LATIN CAPITAL LETTER A WITH DOUBLE GRAVE +0202 ; Uppercase # L& LATIN CAPITAL LETTER A WITH INVERTED BREVE +0204 ; Uppercase # L& LATIN CAPITAL LETTER E WITH DOUBLE GRAVE +0206 ; Uppercase # L& LATIN CAPITAL LETTER E WITH INVERTED BREVE +0208 ; Uppercase # L& LATIN CAPITAL LETTER I WITH DOUBLE GRAVE +020A ; Uppercase # L& LATIN CAPITAL LETTER I WITH INVERTED BREVE +020C ; Uppercase # L& LATIN CAPITAL LETTER O WITH DOUBLE GRAVE +020E ; Uppercase # L& LATIN CAPITAL LETTER O WITH INVERTED BREVE +0210 ; Uppercase # L& LATIN CAPITAL LETTER R WITH DOUBLE GRAVE +0212 ; Uppercase # L& LATIN CAPITAL LETTER R WITH INVERTED BREVE +0214 ; Uppercase # L& LATIN CAPITAL LETTER U WITH DOUBLE GRAVE +0216 ; Uppercase # L& LATIN CAPITAL LETTER U WITH INVERTED BREVE +0218 ; Uppercase # L& LATIN CAPITAL LETTER S WITH COMMA BELOW +021A ; Uppercase # L& LATIN CAPITAL LETTER T WITH COMMA BELOW +021C ; Uppercase # L& LATIN CAPITAL LETTER YOGH +021E ; Uppercase # L& LATIN CAPITAL LETTER H WITH CARON +0220 ; Uppercase # L& LATIN CAPITAL LETTER N WITH LONG RIGHT LEG +0222 ; Uppercase # L& LATIN CAPITAL LETTER OU +0224 ; Uppercase # L& LATIN CAPITAL LETTER Z WITH HOOK +0226 ; Uppercase # L& LATIN CAPITAL LETTER A WITH DOT ABOVE +0228 ; Uppercase # L& LATIN CAPITAL LETTER E WITH CEDILLA +022A ; Uppercase # L& LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON +022C ; Uppercase # L& LATIN CAPITAL LETTER O WITH TILDE AND MACRON +022E ; Uppercase # L& LATIN CAPITAL LETTER O WITH DOT ABOVE +0230 ; Uppercase # L& LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON +0232 ; Uppercase # L& LATIN CAPITAL LETTER Y WITH MACRON +0386 ; Uppercase # L& GREEK CAPITAL LETTER ALPHA WITH TONOS +0388..038A ; Uppercase # L& [3] GREEK CAPITAL LETTER EPSILON WITH TONOS..GREEK CAPITAL LETTER IOTA WITH TONOS +038C ; Uppercase # L& GREEK CAPITAL LETTER OMICRON WITH TONOS +038E..038F ; Uppercase # L& [2] GREEK CAPITAL LETTER UPSILON WITH TONOS..GREEK CAPITAL LETTER OMEGA WITH TONOS +0391..03A1 ; Uppercase # L& [17] GREEK CAPITAL LETTER ALPHA..GREEK CAPITAL LETTER RHO +03A3..03AB ; Uppercase # L& [9] GREEK CAPITAL LETTER SIGMA..GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA +03D2..03D4 ; Uppercase # L& [3] GREEK UPSILON WITH HOOK SYMBOL..GREEK UPSILON WITH DIAERESIS AND HOOK SYMBOL +03D8 ; Uppercase # L& GREEK LETTER ARCHAIC KOPPA +03DA ; Uppercase # L& GREEK LETTER STIGMA +03DC ; Uppercase # L& GREEK LETTER DIGAMMA +03DE ; Uppercase # L& GREEK LETTER KOPPA +03E0 ; Uppercase # L& GREEK LETTER SAMPI +03E2 ; Uppercase # L& COPTIC CAPITAL LETTER SHEI +03E4 ; Uppercase # L& COPTIC CAPITAL LETTER FEI +03E6 ; Uppercase # L& COPTIC CAPITAL LETTER KHEI +03E8 ; Uppercase # L& COPTIC CAPITAL LETTER HORI +03EA ; Uppercase # L& COPTIC CAPITAL LETTER GANGIA +03EC ; Uppercase # L& COPTIC CAPITAL LETTER SHIMA +03EE ; Uppercase # L& COPTIC CAPITAL LETTER DEI +03F4 ; Uppercase # L& GREEK CAPITAL THETA SYMBOL +0400..042F ; Uppercase # L& [48] CYRILLIC CAPITAL LETTER IE WITH GRAVE..CYRILLIC CAPITAL LETTER YA +0460 ; Uppercase # L& CYRILLIC CAPITAL LETTER OMEGA +0462 ; Uppercase # L& CYRILLIC CAPITAL LETTER YAT +0464 ; Uppercase # L& CYRILLIC CAPITAL LETTER IOTIFIED E +0466 ; Uppercase # L& CYRILLIC CAPITAL LETTER LITTLE YUS +0468 ; Uppercase # L& CYRILLIC CAPITAL LETTER IOTIFIED LITTLE YUS +046A ; Uppercase # L& CYRILLIC CAPITAL LETTER BIG YUS +046C ; Uppercase # L& CYRILLIC CAPITAL LETTER IOTIFIED BIG YUS +046E ; Uppercase # L& CYRILLIC CAPITAL LETTER KSI +0470 ; Uppercase # L& CYRILLIC CAPITAL LETTER PSI +0472 ; Uppercase # L& CYRILLIC CAPITAL LETTER FITA +0474 ; Uppercase # L& CYRILLIC CAPITAL LETTER IZHITSA +0476 ; Uppercase # L& CYRILLIC CAPITAL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT +0478 ; Uppercase # L& CYRILLIC CAPITAL LETTER UK +047A ; Uppercase # L& CYRILLIC CAPITAL LETTER ROUND OMEGA +047C ; Uppercase # L& CYRILLIC CAPITAL LETTER OMEGA WITH TITLO +047E ; Uppercase # L& CYRILLIC CAPITAL LETTER OT +0480 ; Uppercase # L& CYRILLIC CAPITAL LETTER KOPPA +048A ; Uppercase # L& CYRILLIC CAPITAL LETTER SHORT I WITH TAIL +048C ; Uppercase # L& CYRILLIC CAPITAL LETTER SEMISOFT SIGN +048E ; Uppercase # L& CYRILLIC CAPITAL LETTER ER WITH TICK +0490 ; Uppercase # L& CYRILLIC CAPITAL LETTER GHE WITH UPTURN +0492 ; Uppercase # L& CYRILLIC CAPITAL LETTER GHE WITH STROKE +0494 ; Uppercase # L& CYRILLIC CAPITAL LETTER GHE WITH MIDDLE HOOK +0496 ; Uppercase # L& CYRILLIC CAPITAL LETTER ZHE WITH DESCENDER +0498 ; Uppercase # L& CYRILLIC CAPITAL LETTER ZE WITH DESCENDER +049A ; Uppercase # L& CYRILLIC CAPITAL LETTER KA WITH DESCENDER +049C ; Uppercase # L& CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE +049E ; Uppercase # L& CYRILLIC CAPITAL LETTER KA WITH STROKE +04A0 ; Uppercase # L& CYRILLIC CAPITAL LETTER BASHKIR KA +04A2 ; Uppercase # L& CYRILLIC CAPITAL LETTER EN WITH DESCENDER +04A4 ; Uppercase # L& CYRILLIC CAPITAL LIGATURE EN GHE +04A6 ; Uppercase # L& CYRILLIC CAPITAL LETTER PE WITH MIDDLE HOOK +04A8 ; Uppercase # L& CYRILLIC CAPITAL LETTER ABKHASIAN HA +04AA ; Uppercase # L& CYRILLIC CAPITAL LETTER ES WITH DESCENDER +04AC ; Uppercase # L& CYRILLIC CAPITAL LETTER TE WITH DESCENDER +04AE ; Uppercase # L& CYRILLIC CAPITAL LETTER STRAIGHT U +04B0 ; Uppercase # L& CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE +04B2 ; Uppercase # L& CYRILLIC CAPITAL LETTER HA WITH DESCENDER +04B4 ; Uppercase # L& CYRILLIC CAPITAL LIGATURE TE TSE +04B6 ; Uppercase # L& CYRILLIC CAPITAL LETTER CHE WITH DESCENDER +04B8 ; Uppercase # L& CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE +04BA ; Uppercase # L& CYRILLIC CAPITAL LETTER SHHA +04BC ; Uppercase # L& CYRILLIC CAPITAL LETTER ABKHASIAN CHE +04BE ; Uppercase # L& CYRILLIC CAPITAL LETTER ABKHASIAN CHE WITH DESCENDER +04C0..04C1 ; Uppercase # L& [2] CYRILLIC LETTER PALOCHKA..CYRILLIC CAPITAL LETTER ZHE WITH BREVE +04C3 ; Uppercase # L& CYRILLIC CAPITAL LETTER KA WITH HOOK +04C5 ; Uppercase # L& CYRILLIC CAPITAL LETTER EL WITH TAIL +04C7 ; Uppercase # L& CYRILLIC CAPITAL LETTER EN WITH HOOK +04C9 ; Uppercase # L& CYRILLIC CAPITAL LETTER EN WITH TAIL +04CB ; Uppercase # L& CYRILLIC CAPITAL LETTER KHAKASSIAN CHE +04CD ; Uppercase # L& CYRILLIC CAPITAL LETTER EM WITH TAIL +04D0 ; Uppercase # L& CYRILLIC CAPITAL LETTER A WITH BREVE +04D2 ; Uppercase # L& CYRILLIC CAPITAL LETTER A WITH DIAERESIS +04D4 ; Uppercase # L& CYRILLIC CAPITAL LIGATURE A IE +04D6 ; Uppercase # L& CYRILLIC CAPITAL LETTER IE WITH BREVE +04D8 ; Uppercase # L& CYRILLIC CAPITAL LETTER SCHWA +04DA ; Uppercase # L& CYRILLIC CAPITAL LETTER SCHWA WITH DIAERESIS +04DC ; Uppercase # L& CYRILLIC CAPITAL LETTER ZHE WITH DIAERESIS +04DE ; Uppercase # L& CYRILLIC CAPITAL LETTER ZE WITH DIAERESIS +04E0 ; Uppercase # L& CYRILLIC CAPITAL LETTER ABKHASIAN DZE +04E2 ; Uppercase # L& CYRILLIC CAPITAL LETTER I WITH MACRON +04E4 ; Uppercase # L& CYRILLIC CAPITAL LETTER I WITH DIAERESIS +04E6 ; Uppercase # L& CYRILLIC CAPITAL LETTER O WITH DIAERESIS +04E8 ; Uppercase # L& CYRILLIC CAPITAL LETTER BARRED O +04EA ; Uppercase # L& CYRILLIC CAPITAL LETTER BARRED O WITH DIAERESIS +04EC ; Uppercase # L& CYRILLIC CAPITAL LETTER E WITH DIAERESIS +04EE ; Uppercase # L& CYRILLIC CAPITAL LETTER U WITH MACRON +04F0 ; Uppercase # L& CYRILLIC CAPITAL LETTER U WITH DIAERESIS +04F2 ; Uppercase # L& CYRILLIC CAPITAL LETTER U WITH DOUBLE ACUTE +04F4 ; Uppercase # L& CYRILLIC CAPITAL LETTER CHE WITH DIAERESIS +04F8 ; Uppercase # L& CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS +0500 ; Uppercase # L& CYRILLIC CAPITAL LETTER KOMI DE +0502 ; Uppercase # L& CYRILLIC CAPITAL LETTER KOMI DJE +0504 ; Uppercase # L& CYRILLIC CAPITAL LETTER KOMI ZJE +0506 ; Uppercase # L& CYRILLIC CAPITAL LETTER KOMI DZJE +0508 ; Uppercase # L& CYRILLIC CAPITAL LETTER KOMI LJE +050A ; Uppercase # L& CYRILLIC CAPITAL LETTER KOMI NJE +050C ; Uppercase # L& CYRILLIC CAPITAL LETTER KOMI SJE +050E ; Uppercase # L& CYRILLIC CAPITAL LETTER KOMI TJE +0531..0556 ; Uppercase # L& [38] ARMENIAN CAPITAL LETTER AYB..ARMENIAN CAPITAL LETTER FEH +10A0..10C5 ; Uppercase # L& [38] GEORGIAN CAPITAL LETTER AN..GEORGIAN CAPITAL LETTER HOE +1E00 ; Uppercase # L& LATIN CAPITAL LETTER A WITH RING BELOW +1E02 ; Uppercase # L& LATIN CAPITAL LETTER B WITH DOT ABOVE +1E04 ; Uppercase # L& LATIN CAPITAL LETTER B WITH DOT BELOW +1E06 ; Uppercase # L& LATIN CAPITAL LETTER B WITH LINE BELOW +1E08 ; Uppercase # L& LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE +1E0A ; Uppercase # L& LATIN CAPITAL LETTER D WITH DOT ABOVE +1E0C ; Uppercase # L& LATIN CAPITAL LETTER D WITH DOT BELOW +1E0E ; Uppercase # L& LATIN CAPITAL LETTER D WITH LINE BELOW +1E10 ; Uppercase # L& LATIN CAPITAL LETTER D WITH CEDILLA +1E12 ; Uppercase # L& LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW +1E14 ; Uppercase # L& LATIN CAPITAL LETTER E WITH MACRON AND GRAVE +1E16 ; Uppercase # L& LATIN CAPITAL LETTER E WITH MACRON AND ACUTE +1E18 ; Uppercase # L& LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW +1E1A ; Uppercase # L& LATIN CAPITAL LETTER E WITH TILDE BELOW +1E1C ; Uppercase # L& LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE +1E1E ; Uppercase # L& LATIN CAPITAL LETTER F WITH DOT ABOVE +1E20 ; Uppercase # L& LATIN CAPITAL LETTER G WITH MACRON +1E22 ; Uppercase # L& LATIN CAPITAL LETTER H WITH DOT ABOVE +1E24 ; Uppercase # L& LATIN CAPITAL LETTER H WITH DOT BELOW +1E26 ; Uppercase # L& LATIN CAPITAL LETTER H WITH DIAERESIS +1E28 ; Uppercase # L& LATIN CAPITAL LETTER H WITH CEDILLA +1E2A ; Uppercase # L& LATIN CAPITAL LETTER H WITH BREVE BELOW +1E2C ; Uppercase # L& LATIN CAPITAL LETTER I WITH TILDE BELOW +1E2E ; Uppercase # L& LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE +1E30 ; Uppercase # L& LATIN CAPITAL LETTER K WITH ACUTE +1E32 ; Uppercase # L& LATIN CAPITAL LETTER K WITH DOT BELOW +1E34 ; Uppercase # L& LATIN CAPITAL LETTER K WITH LINE BELOW +1E36 ; Uppercase # L& LATIN CAPITAL LETTER L WITH DOT BELOW +1E38 ; Uppercase # L& LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON +1E3A ; Uppercase # L& LATIN CAPITAL LETTER L WITH LINE BELOW +1E3C ; Uppercase # L& LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW +1E3E ; Uppercase # L& LATIN CAPITAL LETTER M WITH ACUTE +1E40 ; Uppercase # L& LATIN CAPITAL LETTER M WITH DOT ABOVE +1E42 ; Uppercase # L& LATIN CAPITAL LETTER M WITH DOT BELOW +1E44 ; Uppercase # L& LATIN CAPITAL LETTER N WITH DOT ABOVE +1E46 ; Uppercase # L& LATIN CAPITAL LETTER N WITH DOT BELOW +1E48 ; Uppercase # L& LATIN CAPITAL LETTER N WITH LINE BELOW +1E4A ; Uppercase # L& LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW +1E4C ; Uppercase # L& LATIN CAPITAL LETTER O WITH TILDE AND ACUTE +1E4E ; Uppercase # L& LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS +1E50 ; Uppercase # L& LATIN CAPITAL LETTER O WITH MACRON AND GRAVE +1E52 ; Uppercase # L& LATIN CAPITAL LETTER O WITH MACRON AND ACUTE +1E54 ; Uppercase # L& LATIN CAPITAL LETTER P WITH ACUTE +1E56 ; Uppercase # L& LATIN CAPITAL LETTER P WITH DOT ABOVE +1E58 ; Uppercase # L& LATIN CAPITAL LETTER R WITH DOT ABOVE +1E5A ; Uppercase # L& LATIN CAPITAL LETTER R WITH DOT BELOW +1E5C ; Uppercase # L& LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON +1E5E ; Uppercase # L& LATIN CAPITAL LETTER R WITH LINE BELOW +1E60 ; Uppercase # L& LATIN CAPITAL LETTER S WITH DOT ABOVE +1E62 ; Uppercase # L& LATIN CAPITAL LETTER S WITH DOT BELOW +1E64 ; Uppercase # L& LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE +1E66 ; Uppercase # L& LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE +1E68 ; Uppercase # L& LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE +1E6A ; Uppercase # L& LATIN CAPITAL LETTER T WITH DOT ABOVE +1E6C ; Uppercase # L& LATIN CAPITAL LETTER T WITH DOT BELOW +1E6E ; Uppercase # L& LATIN CAPITAL LETTER T WITH LINE BELOW +1E70 ; Uppercase # L& LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW +1E72 ; Uppercase # L& LATIN CAPITAL LETTER U WITH DIAERESIS BELOW +1E74 ; Uppercase # L& LATIN CAPITAL LETTER U WITH TILDE BELOW +1E76 ; Uppercase # L& LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW +1E78 ; Uppercase # L& LATIN CAPITAL LETTER U WITH TILDE AND ACUTE +1E7A ; Uppercase # L& LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS +1E7C ; Uppercase # L& LATIN CAPITAL LETTER V WITH TILDE +1E7E ; Uppercase # L& LATIN CAPITAL LETTER V WITH DOT BELOW +1E80 ; Uppercase # L& LATIN CAPITAL LETTER W WITH GRAVE +1E82 ; Uppercase # L& LATIN CAPITAL LETTER W WITH ACUTE +1E84 ; Uppercase # L& LATIN CAPITAL LETTER W WITH DIAERESIS +1E86 ; Uppercase # L& LATIN CAPITAL LETTER W WITH DOT ABOVE +1E88 ; Uppercase # L& LATIN CAPITAL LETTER W WITH DOT BELOW +1E8A ; Uppercase # L& LATIN CAPITAL LETTER X WITH DOT ABOVE +1E8C ; Uppercase # L& LATIN CAPITAL LETTER X WITH DIAERESIS +1E8E ; Uppercase # L& LATIN CAPITAL LETTER Y WITH DOT ABOVE +1E90 ; Uppercase # L& LATIN CAPITAL LETTER Z WITH CIRCUMFLEX +1E92 ; Uppercase # L& LATIN CAPITAL LETTER Z WITH DOT BELOW +1E94 ; Uppercase # L& LATIN CAPITAL LETTER Z WITH LINE BELOW +1EA0 ; Uppercase # L& LATIN CAPITAL LETTER A WITH DOT BELOW +1EA2 ; Uppercase # L& LATIN CAPITAL LETTER A WITH HOOK ABOVE +1EA4 ; Uppercase # L& LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE +1EA6 ; Uppercase # L& LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE +1EA8 ; Uppercase # L& LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE +1EAA ; Uppercase # L& LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE +1EAC ; Uppercase # L& LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW +1EAE ; Uppercase # L& LATIN CAPITAL LETTER A WITH BREVE AND ACUTE +1EB0 ; Uppercase # L& LATIN CAPITAL LETTER A WITH BREVE AND GRAVE +1EB2 ; Uppercase # L& LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE +1EB4 ; Uppercase # L& LATIN CAPITAL LETTER A WITH BREVE AND TILDE +1EB6 ; Uppercase # L& LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW +1EB8 ; Uppercase # L& LATIN CAPITAL LETTER E WITH DOT BELOW +1EBA ; Uppercase # L& LATIN CAPITAL LETTER E WITH HOOK ABOVE +1EBC ; Uppercase # L& LATIN CAPITAL LETTER E WITH TILDE +1EBE ; Uppercase # L& LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE +1EC0 ; Uppercase # L& LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE +1EC2 ; Uppercase # L& LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE +1EC4 ; Uppercase # L& LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE +1EC6 ; Uppercase # L& LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW +1EC8 ; Uppercase # L& LATIN CAPITAL LETTER I WITH HOOK ABOVE +1ECA ; Uppercase # L& LATIN CAPITAL LETTER I WITH DOT BELOW +1ECC ; Uppercase # L& LATIN CAPITAL LETTER O WITH DOT BELOW +1ECE ; Uppercase # L& LATIN CAPITAL LETTER O WITH HOOK ABOVE +1ED0 ; Uppercase # L& LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE +1ED2 ; Uppercase # L& LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE +1ED4 ; Uppercase # L& LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE +1ED6 ; Uppercase # L& LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE +1ED8 ; Uppercase # L& LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW +1EDA ; Uppercase # L& LATIN CAPITAL LETTER O WITH HORN AND ACUTE +1EDC ; Uppercase # L& LATIN CAPITAL LETTER O WITH HORN AND GRAVE +1EDE ; Uppercase # L& LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE +1EE0 ; Uppercase # L& LATIN CAPITAL LETTER O WITH HORN AND TILDE +1EE2 ; Uppercase # L& LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW +1EE4 ; Uppercase # L& LATIN CAPITAL LETTER U WITH DOT BELOW +1EE6 ; Uppercase # L& LATIN CAPITAL LETTER U WITH HOOK ABOVE +1EE8 ; Uppercase # L& LATIN CAPITAL LETTER U WITH HORN AND ACUTE +1EEA ; Uppercase # L& LATIN CAPITAL LETTER U WITH HORN AND GRAVE +1EEC ; Uppercase # L& LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE +1EEE ; Uppercase # L& LATIN CAPITAL LETTER U WITH HORN AND TILDE +1EF0 ; Uppercase # L& LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW +1EF2 ; Uppercase # L& LATIN CAPITAL LETTER Y WITH GRAVE +1EF4 ; Uppercase # L& LATIN CAPITAL LETTER Y WITH DOT BELOW +1EF6 ; Uppercase # L& LATIN CAPITAL LETTER Y WITH HOOK ABOVE +1EF8 ; Uppercase # L& LATIN CAPITAL LETTER Y WITH TILDE +1F08..1F0F ; Uppercase # L& [8] GREEK CAPITAL LETTER ALPHA WITH PSILI..GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI +1F18..1F1D ; Uppercase # L& [6] GREEK CAPITAL LETTER EPSILON WITH PSILI..GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA +1F28..1F2F ; Uppercase # L& [8] GREEK CAPITAL LETTER ETA WITH PSILI..GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI +1F38..1F3F ; Uppercase # L& [8] GREEK CAPITAL LETTER IOTA WITH PSILI..GREEK CAPITAL LETTER IOTA WITH DASIA AND PERISPOMENI +1F48..1F4D ; Uppercase # L& [6] GREEK CAPITAL LETTER OMICRON WITH PSILI..GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA +1F59 ; Uppercase # L& GREEK CAPITAL LETTER UPSILON WITH DASIA +1F5B ; Uppercase # L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA +1F5D ; Uppercase # L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA +1F5F ; Uppercase # L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI +1F68..1F6F ; Uppercase # L& [8] GREEK CAPITAL LETTER OMEGA WITH PSILI..GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI +1FB8..1FBB ; Uppercase # L& [4] GREEK CAPITAL LETTER ALPHA WITH VRACHY..GREEK CAPITAL LETTER ALPHA WITH OXIA +1FC8..1FCB ; Uppercase # L& [4] GREEK CAPITAL LETTER EPSILON WITH VARIA..GREEK CAPITAL LETTER ETA WITH OXIA +1FD8..1FDB ; Uppercase # L& [4] GREEK CAPITAL LETTER IOTA WITH VRACHY..GREEK CAPITAL LETTER IOTA WITH OXIA +1FE8..1FEC ; Uppercase # L& [5] GREEK CAPITAL LETTER UPSILON WITH VRACHY..GREEK CAPITAL LETTER RHO WITH DASIA +1FF8..1FFB ; Uppercase # L& [4] GREEK CAPITAL LETTER OMICRON WITH VARIA..GREEK CAPITAL LETTER OMEGA WITH OXIA +2102 ; Uppercase # L& DOUBLE-STRUCK CAPITAL C +2107 ; Uppercase # L& EULER CONSTANT +210B..210D ; Uppercase # L& [3] SCRIPT CAPITAL H..DOUBLE-STRUCK CAPITAL H +2110..2112 ; Uppercase # L& [3] SCRIPT CAPITAL I..SCRIPT CAPITAL L +2115 ; Uppercase # L& DOUBLE-STRUCK CAPITAL N +2119..211D ; Uppercase # L& [5] DOUBLE-STRUCK CAPITAL P..DOUBLE-STRUCK CAPITAL R +2124 ; Uppercase # L& DOUBLE-STRUCK CAPITAL Z +2126 ; Uppercase # L& OHM SIGN +2128 ; Uppercase # L& BLACK-LETTER CAPITAL Z +212A..212D ; Uppercase # L& [4] KELVIN SIGN..BLACK-LETTER CAPITAL C +2130..2131 ; Uppercase # L& [2] SCRIPT CAPITAL E..SCRIPT CAPITAL F +2133 ; Uppercase # L& SCRIPT CAPITAL M +213E..213F ; Uppercase # L& [2] DOUBLE-STRUCK CAPITAL GAMMA..DOUBLE-STRUCK CAPITAL PI +2145 ; Uppercase # L& DOUBLE-STRUCK ITALIC CAPITAL D +2160..216F ; Uppercase # Nl [16] ROMAN NUMERAL ONE..ROMAN NUMERAL ONE THOUSAND +24B6..24CF ; Uppercase # So [26] CIRCLED LATIN CAPITAL LETTER A..CIRCLED LATIN CAPITAL LETTER Z +FF21..FF3A ; Uppercase # L& [26] FULLWIDTH LATIN CAPITAL LETTER A..FULLWIDTH LATIN CAPITAL LETTER Z +10400..10425 ; Uppercase # L& [38] DESERET CAPITAL LETTER LONG I..DESERET CAPITAL LETTER ENG +1D400..1D419 ; Uppercase # L& [26] MATHEMATICAL BOLD CAPITAL A..MATHEMATICAL BOLD CAPITAL Z +1D434..1D44D ; Uppercase # L& [26] MATHEMATICAL ITALIC CAPITAL A..MATHEMATICAL ITALIC CAPITAL Z +1D468..1D481 ; Uppercase # L& [26] MATHEMATICAL BOLD ITALIC CAPITAL A..MATHEMATICAL BOLD ITALIC CAPITAL Z +1D49C ; Uppercase # L& MATHEMATICAL SCRIPT CAPITAL A +1D49E..1D49F ; Uppercase # L& [2] MATHEMATICAL SCRIPT CAPITAL C..MATHEMATICAL SCRIPT CAPITAL D +1D4A2 ; Uppercase # L& MATHEMATICAL SCRIPT CAPITAL G +1D4A5..1D4A6 ; Uppercase # L& [2] MATHEMATICAL SCRIPT CAPITAL J..MATHEMATICAL SCRIPT CAPITAL K +1D4A9..1D4AC ; Uppercase # L& [4] MATHEMATICAL SCRIPT CAPITAL N..MATHEMATICAL SCRIPT CAPITAL Q +1D4AE..1D4B5 ; Uppercase # L& [8] MATHEMATICAL SCRIPT CAPITAL S..MATHEMATICAL SCRIPT CAPITAL Z +1D4D0..1D4E9 ; Uppercase # L& [26] MATHEMATICAL BOLD SCRIPT CAPITAL A..MATHEMATICAL BOLD SCRIPT CAPITAL Z +1D504..1D505 ; Uppercase # L& [2] MATHEMATICAL FRAKTUR CAPITAL A..MATHEMATICAL FRAKTUR CAPITAL B +1D507..1D50A ; Uppercase # L& [4] MATHEMATICAL FRAKTUR CAPITAL D..MATHEMATICAL FRAKTUR CAPITAL G +1D50D..1D514 ; Uppercase # L& [8] MATHEMATICAL FRAKTUR CAPITAL J..MATHEMATICAL FRAKTUR CAPITAL Q +1D516..1D51C ; Uppercase # L& [7] MATHEMATICAL FRAKTUR CAPITAL S..MATHEMATICAL FRAKTUR CAPITAL Y +1D538..1D539 ; Uppercase # L& [2] MATHEMATICAL DOUBLE-STRUCK CAPITAL A..MATHEMATICAL DOUBLE-STRUCK CAPITAL B +1D53B..1D53E ; Uppercase # L& [4] MATHEMATICAL DOUBLE-STRUCK CAPITAL D..MATHEMATICAL DOUBLE-STRUCK CAPITAL G +1D540..1D544 ; Uppercase # L& [5] MATHEMATICAL DOUBLE-STRUCK CAPITAL I..MATHEMATICAL DOUBLE-STRUCK CAPITAL M +1D546 ; Uppercase # L& MATHEMATICAL DOUBLE-STRUCK CAPITAL O +1D54A..1D550 ; Uppercase # L& [7] MATHEMATICAL DOUBLE-STRUCK CAPITAL S..MATHEMATICAL DOUBLE-STRUCK CAPITAL Y +1D56C..1D585 ; Uppercase # L& [26] MATHEMATICAL BOLD FRAKTUR CAPITAL A..MATHEMATICAL BOLD FRAKTUR CAPITAL Z +1D5A0..1D5B9 ; Uppercase # L& [26] MATHEMATICAL SANS-SERIF CAPITAL A..MATHEMATICAL SANS-SERIF CAPITAL Z +1D5D4..1D5ED ; Uppercase # L& [26] MATHEMATICAL SANS-SERIF BOLD CAPITAL A..MATHEMATICAL SANS-SERIF BOLD CAPITAL Z +1D608..1D621 ; Uppercase # L& [26] MATHEMATICAL SANS-SERIF ITALIC CAPITAL A..MATHEMATICAL SANS-SERIF ITALIC CAPITAL Z +1D63C..1D655 ; Uppercase # L& [26] MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL A..MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL Z +1D670..1D689 ; Uppercase # L& [26] MATHEMATICAL MONOSPACE CAPITAL A..MATHEMATICAL MONOSPACE CAPITAL Z +1D6A8..1D6C0 ; Uppercase # L& [25] MATHEMATICAL BOLD CAPITAL ALPHA..MATHEMATICAL BOLD CAPITAL OMEGA +1D6E2..1D6FA ; Uppercase # L& [25] MATHEMATICAL ITALIC CAPITAL ALPHA..MATHEMATICAL ITALIC CAPITAL OMEGA +1D71C..1D734 ; Uppercase # L& [25] MATHEMATICAL BOLD ITALIC CAPITAL ALPHA..MATHEMATICAL BOLD ITALIC CAPITAL OMEGA +1D756..1D76E ; Uppercase # L& [25] MATHEMATICAL SANS-SERIF BOLD CAPITAL ALPHA..MATHEMATICAL SANS-SERIF BOLD CAPITAL OMEGA +1D790..1D7A8 ; Uppercase # L& [25] MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL ALPHA..MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL OMEGA + +# Total code points: 1227 + +# ================================================ + +# Derived Property: ID_Start +# Characters that can start an identifier. +# Generated from Lu+Ll+Lt+Lm+Lo+Nl + +0041..005A ; ID_Start # L& [26] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER Z +0061..007A ; ID_Start # L& [26] LATIN SMALL LETTER A..LATIN SMALL LETTER Z +00AA ; ID_Start # L& FEMININE ORDINAL INDICATOR +00B5 ; ID_Start # L& MICRO SIGN +00BA ; ID_Start # L& MASCULINE ORDINAL INDICATOR +00C0..00D6 ; ID_Start # L& [23] LATIN CAPITAL LETTER A WITH GRAVE..LATIN CAPITAL LETTER O WITH DIAERESIS +00D8..00F6 ; ID_Start # L& [31] LATIN CAPITAL LETTER O WITH STROKE..LATIN SMALL LETTER O WITH DIAERESIS +00F8..01BA ; ID_Start # L& [195] LATIN SMALL LETTER O WITH STROKE..LATIN SMALL LETTER EZH WITH TAIL +01BB ; ID_Start # Lo LATIN LETTER TWO WITH STROKE +01BC..01BF ; ID_Start # L& [4] LATIN CAPITAL LETTER TONE FIVE..LATIN LETTER WYNN +01C0..01C3 ; ID_Start # Lo [4] LATIN LETTER DENTAL CLICK..LATIN LETTER RETROFLEX CLICK +01C4..0220 ; ID_Start # L& [93] LATIN CAPITAL LETTER DZ WITH CARON..LATIN CAPITAL LETTER N WITH LONG RIGHT LEG +0222..0233 ; ID_Start # L& [18] LATIN CAPITAL LETTER OU..LATIN SMALL LETTER Y WITH MACRON +0250..02AD ; ID_Start # L& [94] LATIN SMALL LETTER TURNED A..LATIN LETTER BIDENTAL PERCUSSIVE +02B0..02B8 ; ID_Start # Lm [9] MODIFIER LETTER SMALL H..MODIFIER LETTER SMALL Y +02BB..02C1 ; ID_Start # Lm [7] MODIFIER LETTER TURNED COMMA..MODIFIER LETTER REVERSED GLOTTAL STOP +02D0..02D1 ; ID_Start # Lm [2] MODIFIER LETTER TRIANGULAR COLON..MODIFIER LETTER HALF TRIANGULAR COLON +02E0..02E4 ; ID_Start # Lm [5] MODIFIER LETTER SMALL GAMMA..MODIFIER LETTER SMALL REVERSED GLOTTAL STOP +02EE ; ID_Start # Lm MODIFIER LETTER DOUBLE APOSTROPHE +037A ; ID_Start # Lm GREEK YPOGEGRAMMENI +0386 ; ID_Start # L& GREEK CAPITAL LETTER ALPHA WITH TONOS +0388..038A ; ID_Start # L& [3] GREEK CAPITAL LETTER EPSILON WITH TONOS..GREEK CAPITAL LETTER IOTA WITH TONOS +038C ; ID_Start # L& GREEK CAPITAL LETTER OMICRON WITH TONOS +038E..03A1 ; ID_Start # L& [20] GREEK CAPITAL LETTER UPSILON WITH TONOS..GREEK CAPITAL LETTER RHO +03A3..03CE ; ID_Start # L& [44] GREEK CAPITAL LETTER SIGMA..GREEK SMALL LETTER OMEGA WITH TONOS +03D0..03F5 ; ID_Start # L& [38] GREEK BETA SYMBOL..GREEK LUNATE EPSILON SYMBOL +0400..0481 ; ID_Start # L& [130] CYRILLIC CAPITAL LETTER IE WITH GRAVE..CYRILLIC SMALL LETTER KOPPA +048A..04CE ; ID_Start # L& [69] CYRILLIC CAPITAL LETTER SHORT I WITH TAIL..CYRILLIC SMALL LETTER EM WITH TAIL +04D0..04F5 ; ID_Start # L& [38] CYRILLIC CAPITAL LETTER A WITH BREVE..CYRILLIC SMALL LETTER CHE WITH DIAERESIS +04F8..04F9 ; ID_Start # L& [2] CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS..CYRILLIC SMALL LETTER YERU WITH DIAERESIS +0500..050F ; ID_Start # L& [16] CYRILLIC CAPITAL LETTER KOMI DE..CYRILLIC SMALL LETTER KOMI TJE +0531..0556 ; ID_Start # L& [38] ARMENIAN CAPITAL LETTER AYB..ARMENIAN CAPITAL LETTER FEH +0559 ; ID_Start # Lm ARMENIAN MODIFIER LETTER LEFT HALF RING +0561..0587 ; ID_Start # L& [39] ARMENIAN SMALL LETTER AYB..ARMENIAN SMALL LIGATURE ECH YIWN +05D0..05EA ; ID_Start # Lo [27] HEBREW LETTER ALEF..HEBREW LETTER TAV +05F0..05F2 ; ID_Start # Lo [3] HEBREW LIGATURE YIDDISH DOUBLE VAV..HEBREW LIGATURE YIDDISH DOUBLE YOD +0621..063A ; ID_Start # Lo [26] ARABIC LETTER HAMZA..ARABIC LETTER GHAIN +0640 ; ID_Start # Lm ARABIC TATWEEL +0641..064A ; ID_Start # Lo [10] ARABIC LETTER FEH..ARABIC LETTER YEH +066E..066F ; ID_Start # Lo [2] ARABIC LETTER DOTLESS BEH..ARABIC LETTER DOTLESS QAF +0671..06D3 ; ID_Start # Lo [99] ARABIC LETTER ALEF WASLA..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE +06D5 ; ID_Start # Lo ARABIC LETTER AE +06E5..06E6 ; ID_Start # Lm [2] ARABIC SMALL WAW..ARABIC SMALL YEH +06FA..06FC ; ID_Start # Lo [3] ARABIC LETTER SHEEN WITH DOT BELOW..ARABIC LETTER GHAIN WITH DOT BELOW +0710 ; ID_Start # Lo SYRIAC LETTER ALAPH +0712..072C ; ID_Start # Lo [27] SYRIAC LETTER BETH..SYRIAC LETTER TAW +0780..07A5 ; ID_Start # Lo [38] THAANA LETTER HAA..THAANA LETTER WAAVU +07B1 ; ID_Start # Lo THAANA LETTER NAA +0905..0939 ; ID_Start # Lo [53] DEVANAGARI LETTER A..DEVANAGARI LETTER HA +093D ; ID_Start # Lo DEVANAGARI SIGN AVAGRAHA +0950 ; ID_Start # Lo DEVANAGARI OM +0958..0961 ; ID_Start # Lo [10] DEVANAGARI LETTER QA..DEVANAGARI LETTER VOCALIC LL +0985..098C ; ID_Start # Lo [8] BENGALI LETTER A..BENGALI LETTER VOCALIC L +098F..0990 ; ID_Start # Lo [2] BENGALI LETTER E..BENGALI LETTER AI +0993..09A8 ; ID_Start # Lo [22] BENGALI LETTER O..BENGALI LETTER NA +09AA..09B0 ; ID_Start # Lo [7] BENGALI LETTER PA..BENGALI LETTER RA +09B2 ; ID_Start # Lo BENGALI LETTER LA +09B6..09B9 ; ID_Start # Lo [4] BENGALI LETTER SHA..BENGALI LETTER HA +09DC..09DD ; ID_Start # Lo [2] BENGALI LETTER RRA..BENGALI LETTER RHA +09DF..09E1 ; ID_Start # Lo [3] BENGALI LETTER YYA..BENGALI LETTER VOCALIC LL +09F0..09F1 ; ID_Start # Lo [2] BENGALI LETTER RA WITH MIDDLE DIAGONAL..BENGALI LETTER RA WITH LOWER DIAGONAL +0A05..0A0A ; ID_Start # Lo [6] GURMUKHI LETTER A..GURMUKHI LETTER UU +0A0F..0A10 ; ID_Start # Lo [2] GURMUKHI LETTER EE..GURMUKHI LETTER AI +0A13..0A28 ; ID_Start # Lo [22] GURMUKHI LETTER OO..GURMUKHI LETTER NA +0A2A..0A30 ; ID_Start # Lo [7] GURMUKHI LETTER PA..GURMUKHI LETTER RA +0A32..0A33 ; ID_Start # Lo [2] GURMUKHI LETTER LA..GURMUKHI LETTER LLA +0A35..0A36 ; ID_Start # Lo [2] GURMUKHI LETTER VA..GURMUKHI LETTER SHA +0A38..0A39 ; ID_Start # Lo [2] GURMUKHI LETTER SA..GURMUKHI LETTER HA +0A59..0A5C ; ID_Start # Lo [4] GURMUKHI LETTER KHHA..GURMUKHI LETTER RRA +0A5E ; ID_Start # Lo GURMUKHI LETTER FA +0A72..0A74 ; ID_Start # Lo [3] GURMUKHI IRI..GURMUKHI EK ONKAR +0A85..0A8B ; ID_Start # Lo [7] GUJARATI LETTER A..GUJARATI LETTER VOCALIC R +0A8D ; ID_Start # Lo GUJARATI VOWEL CANDRA E +0A8F..0A91 ; ID_Start # Lo [3] GUJARATI LETTER E..GUJARATI VOWEL CANDRA O +0A93..0AA8 ; ID_Start # Lo [22] GUJARATI LETTER O..GUJARATI LETTER NA +0AAA..0AB0 ; ID_Start # Lo [7] GUJARATI LETTER PA..GUJARATI LETTER RA +0AB2..0AB3 ; ID_Start # Lo [2] GUJARATI LETTER LA..GUJARATI LETTER LLA +0AB5..0AB9 ; ID_Start # Lo [5] GUJARATI LETTER VA..GUJARATI LETTER HA +0ABD ; ID_Start # Lo GUJARATI SIGN AVAGRAHA +0AD0 ; ID_Start # Lo GUJARATI OM +0AE0 ; ID_Start # Lo GUJARATI LETTER VOCALIC RR +0B05..0B0C ; ID_Start # Lo [8] ORIYA LETTER A..ORIYA LETTER VOCALIC L +0B0F..0B10 ; ID_Start # Lo [2] ORIYA LETTER E..ORIYA LETTER AI +0B13..0B28 ; ID_Start # Lo [22] ORIYA LETTER O..ORIYA LETTER NA +0B2A..0B30 ; ID_Start # Lo [7] ORIYA LETTER PA..ORIYA LETTER RA +0B32..0B33 ; ID_Start # Lo [2] ORIYA LETTER LA..ORIYA LETTER LLA +0B36..0B39 ; ID_Start # Lo [4] ORIYA LETTER SHA..ORIYA LETTER HA +0B3D ; ID_Start # Lo ORIYA SIGN AVAGRAHA +0B5C..0B5D ; ID_Start # Lo [2] ORIYA LETTER RRA..ORIYA LETTER RHA +0B5F..0B61 ; ID_Start # Lo [3] ORIYA LETTER YYA..ORIYA LETTER VOCALIC LL +0B83 ; ID_Start # Lo TAMIL SIGN VISARGA +0B85..0B8A ; ID_Start # Lo [6] TAMIL LETTER A..TAMIL LETTER UU +0B8E..0B90 ; ID_Start # Lo [3] TAMIL LETTER E..TAMIL LETTER AI +0B92..0B95 ; ID_Start # Lo [4] TAMIL LETTER O..TAMIL LETTER KA +0B99..0B9A ; ID_Start # Lo [2] TAMIL LETTER NGA..TAMIL LETTER CA +0B9C ; ID_Start # Lo TAMIL LETTER JA +0B9E..0B9F ; ID_Start # Lo [2] TAMIL LETTER NYA..TAMIL LETTER TTA +0BA3..0BA4 ; ID_Start # Lo [2] TAMIL LETTER NNA..TAMIL LETTER TA +0BA8..0BAA ; ID_Start # Lo [3] TAMIL LETTER NA..TAMIL LETTER PA +0BAE..0BB5 ; ID_Start # Lo [8] TAMIL LETTER MA..TAMIL LETTER VA +0BB7..0BB9 ; ID_Start # Lo [3] TAMIL LETTER SSA..TAMIL LETTER HA +0C05..0C0C ; ID_Start # Lo [8] TELUGU LETTER A..TELUGU LETTER VOCALIC L +0C0E..0C10 ; ID_Start # Lo [3] TELUGU LETTER E..TELUGU LETTER AI +0C12..0C28 ; ID_Start # Lo [23] TELUGU LETTER O..TELUGU LETTER NA +0C2A..0C33 ; ID_Start # Lo [10] TELUGU LETTER PA..TELUGU LETTER LLA +0C35..0C39 ; ID_Start # Lo [5] TELUGU LETTER VA..TELUGU LETTER HA +0C60..0C61 ; ID_Start # Lo [2] TELUGU LETTER VOCALIC RR..TELUGU LETTER VOCALIC LL +0C85..0C8C ; ID_Start # Lo [8] KANNADA LETTER A..KANNADA LETTER VOCALIC L +0C8E..0C90 ; ID_Start # Lo [3] KANNADA LETTER E..KANNADA LETTER AI +0C92..0CA8 ; ID_Start # Lo [23] KANNADA LETTER O..KANNADA LETTER NA +0CAA..0CB3 ; ID_Start # Lo [10] KANNADA LETTER PA..KANNADA LETTER LLA +0CB5..0CB9 ; ID_Start # Lo [5] KANNADA LETTER VA..KANNADA LETTER HA +0CDE ; ID_Start # Lo KANNADA LETTER FA +0CE0..0CE1 ; ID_Start # Lo [2] KANNADA LETTER VOCALIC RR..KANNADA LETTER VOCALIC LL +0D05..0D0C ; ID_Start # Lo [8] MALAYALAM LETTER A..MALAYALAM LETTER VOCALIC L +0D0E..0D10 ; ID_Start # Lo [3] MALAYALAM LETTER E..MALAYALAM LETTER AI +0D12..0D28 ; ID_Start # Lo [23] MALAYALAM LETTER O..MALAYALAM LETTER NA +0D2A..0D39 ; ID_Start # Lo [16] MALAYALAM LETTER PA..MALAYALAM LETTER HA +0D60..0D61 ; ID_Start # Lo [2] MALAYALAM LETTER VOCALIC RR..MALAYALAM LETTER VOCALIC LL +0D85..0D96 ; ID_Start # Lo [18] SINHALA LETTER AYANNA..SINHALA LETTER AUYANNA +0D9A..0DB1 ; ID_Start # Lo [24] SINHALA LETTER ALPAPRAANA KAYANNA..SINHALA LETTER DANTAJA NAYANNA +0DB3..0DBB ; ID_Start # Lo [9] SINHALA LETTER SANYAKA DAYANNA..SINHALA LETTER RAYANNA +0DBD ; ID_Start # Lo SINHALA LETTER DANTAJA LAYANNA +0DC0..0DC6 ; ID_Start # Lo [7] SINHALA LETTER VAYANNA..SINHALA LETTER FAYANNA +0E01..0E30 ; ID_Start # Lo [48] THAI CHARACTER KO KAI..THAI CHARACTER SARA A +0E32..0E33 ; ID_Start # Lo [2] THAI CHARACTER SARA AA..THAI CHARACTER SARA AM +0E40..0E45 ; ID_Start # Lo [6] THAI CHARACTER SARA E..THAI CHARACTER LAKKHANGYAO +0E46 ; ID_Start # Lm THAI CHARACTER MAIYAMOK +0E81..0E82 ; ID_Start # Lo [2] LAO LETTER KO..LAO LETTER KHO SUNG +0E84 ; ID_Start # Lo LAO LETTER KHO TAM +0E87..0E88 ; ID_Start # Lo [2] LAO LETTER NGO..LAO LETTER CO +0E8A ; ID_Start # Lo LAO LETTER SO TAM +0E8D ; ID_Start # Lo LAO LETTER NYO +0E94..0E97 ; ID_Start # Lo [4] LAO LETTER DO..LAO LETTER THO TAM +0E99..0E9F ; ID_Start # Lo [7] LAO LETTER NO..LAO LETTER FO SUNG +0EA1..0EA3 ; ID_Start # Lo [3] LAO LETTER MO..LAO LETTER LO LING +0EA5 ; ID_Start # Lo LAO LETTER LO LOOT +0EA7 ; ID_Start # Lo LAO LETTER WO +0EAA..0EAB ; ID_Start # Lo [2] LAO LETTER SO SUNG..LAO LETTER HO SUNG +0EAD..0EB0 ; ID_Start # Lo [4] LAO LETTER O..LAO VOWEL SIGN A +0EB2..0EB3 ; ID_Start # Lo [2] LAO VOWEL SIGN AA..LAO VOWEL SIGN AM +0EBD ; ID_Start # Lo LAO SEMIVOWEL SIGN NYO +0EC0..0EC4 ; ID_Start # Lo [5] LAO VOWEL SIGN E..LAO VOWEL SIGN AI +0EC6 ; ID_Start # Lm LAO KO LA +0EDC..0EDD ; ID_Start # Lo [2] LAO HO NO..LAO HO MO +0F00 ; ID_Start # Lo TIBETAN SYLLABLE OM +0F40..0F47 ; ID_Start # Lo [8] TIBETAN LETTER KA..TIBETAN LETTER JA +0F49..0F6A ; ID_Start # Lo [34] TIBETAN LETTER NYA..TIBETAN LETTER FIXED-FORM RA +0F88..0F8B ; ID_Start # Lo [4] TIBETAN SIGN LCE TSA CAN..TIBETAN SIGN GRU MED RGYINGS +1000..1021 ; ID_Start # Lo [34] MYANMAR LETTER KA..MYANMAR LETTER A +1023..1027 ; ID_Start # Lo [5] MYANMAR LETTER I..MYANMAR LETTER E +1029..102A ; ID_Start # Lo [2] MYANMAR LETTER O..MYANMAR LETTER AU +1050..1055 ; ID_Start # Lo [6] MYANMAR LETTER SHA..MYANMAR LETTER VOCALIC LL +10A0..10C5 ; ID_Start # L& [38] GEORGIAN CAPITAL LETTER AN..GEORGIAN CAPITAL LETTER HOE +10D0..10F8 ; ID_Start # Lo [41] GEORGIAN LETTER AN..GEORGIAN LETTER ELIFI +1100..1159 ; ID_Start # Lo [90] HANGUL CHOSEONG KIYEOK..HANGUL CHOSEONG YEORINHIEUH +115F..11A2 ; ID_Start # Lo [68] HANGUL CHOSEONG FILLER..HANGUL JUNGSEONG SSANGARAEA +11A8..11F9 ; ID_Start # Lo [82] HANGUL JONGSEONG KIYEOK..HANGUL JONGSEONG YEORINHIEUH +1200..1206 ; ID_Start # Lo [7] ETHIOPIC SYLLABLE HA..ETHIOPIC SYLLABLE HO +1208..1246 ; ID_Start # Lo [63] ETHIOPIC SYLLABLE LA..ETHIOPIC SYLLABLE QO +1248 ; ID_Start # Lo ETHIOPIC SYLLABLE QWA +124A..124D ; ID_Start # Lo [4] ETHIOPIC SYLLABLE QWI..ETHIOPIC SYLLABLE QWE +1250..1256 ; ID_Start # Lo [7] ETHIOPIC SYLLABLE QHA..ETHIOPIC SYLLABLE QHO +1258 ; ID_Start # Lo ETHIOPIC SYLLABLE QHWA +125A..125D ; ID_Start # Lo [4] ETHIOPIC SYLLABLE QHWI..ETHIOPIC SYLLABLE QHWE +1260..1286 ; ID_Start # Lo [39] ETHIOPIC SYLLABLE BA..ETHIOPIC SYLLABLE XO +1288 ; ID_Start # Lo ETHIOPIC SYLLABLE XWA +128A..128D ; ID_Start # Lo [4] ETHIOPIC SYLLABLE XWI..ETHIOPIC SYLLABLE XWE +1290..12AE ; ID_Start # Lo [31] ETHIOPIC SYLLABLE NA..ETHIOPIC SYLLABLE KO +12B0 ; ID_Start # Lo ETHIOPIC SYLLABLE KWA +12B2..12B5 ; ID_Start # Lo [4] ETHIOPIC SYLLABLE KWI..ETHIOPIC SYLLABLE KWE +12B8..12BE ; ID_Start # Lo [7] ETHIOPIC SYLLABLE KXA..ETHIOPIC SYLLABLE KXO +12C0 ; ID_Start # Lo ETHIOPIC SYLLABLE KXWA +12C2..12C5 ; ID_Start # Lo [4] ETHIOPIC SYLLABLE KXWI..ETHIOPIC SYLLABLE KXWE +12C8..12CE ; ID_Start # Lo [7] ETHIOPIC SYLLABLE WA..ETHIOPIC SYLLABLE WO +12D0..12D6 ; ID_Start # Lo [7] ETHIOPIC SYLLABLE PHARYNGEAL A..ETHIOPIC SYLLABLE PHARYNGEAL O +12D8..12EE ; ID_Start # Lo [23] ETHIOPIC SYLLABLE ZA..ETHIOPIC SYLLABLE YO +12F0..130E ; ID_Start # Lo [31] ETHIOPIC SYLLABLE DA..ETHIOPIC SYLLABLE GO +1310 ; ID_Start # Lo ETHIOPIC SYLLABLE GWA +1312..1315 ; ID_Start # Lo [4] ETHIOPIC SYLLABLE GWI..ETHIOPIC SYLLABLE GWE +1318..131E ; ID_Start # Lo [7] ETHIOPIC SYLLABLE GGA..ETHIOPIC SYLLABLE GGO +1320..1346 ; ID_Start # Lo [39] ETHIOPIC SYLLABLE THA..ETHIOPIC SYLLABLE TZO +1348..135A ; ID_Start # Lo [19] ETHIOPIC SYLLABLE FA..ETHIOPIC SYLLABLE FYA +13A0..13F4 ; ID_Start # Lo [85] CHEROKEE LETTER A..CHEROKEE LETTER YV +1401..166C ; ID_Start # Lo [620] CANADIAN SYLLABICS E..CANADIAN SYLLABICS CARRIER TTSA +166F..1676 ; ID_Start # Lo [8] CANADIAN SYLLABICS QAI..CANADIAN SYLLABICS NNGAA +1681..169A ; ID_Start # Lo [26] OGHAM LETTER BEITH..OGHAM LETTER PEITH +16A0..16EA ; ID_Start # Lo [75] RUNIC LETTER FEHU FEOH FE F..RUNIC LETTER X +16EE..16F0 ; ID_Start # Nl [3] RUNIC ARLAUG SYMBOL..RUNIC BELGTHOR SYMBOL +1700..170C ; ID_Start # Lo [13] TAGALOG LETTER A..TAGALOG LETTER YA +170E..1711 ; ID_Start # Lo [4] TAGALOG LETTER LA..TAGALOG LETTER HA +1720..1731 ; ID_Start # Lo [18] HANUNOO LETTER A..HANUNOO LETTER HA +1740..1751 ; ID_Start # Lo [18] BUHID LETTER A..BUHID LETTER HA +1760..176C ; ID_Start # Lo [13] TAGBANWA LETTER A..TAGBANWA LETTER YA +176E..1770 ; ID_Start # Lo [3] TAGBANWA LETTER LA..TAGBANWA LETTER SA +1780..17B3 ; ID_Start # Lo [52] KHMER LETTER KA..KHMER INDEPENDENT VOWEL QAU +17D7 ; ID_Start # Lm KHMER SIGN LEK TOO +17DC ; ID_Start # Lo KHMER SIGN AVAKRAHASANYA +1820..1842 ; ID_Start # Lo [35] MONGOLIAN LETTER A..MONGOLIAN LETTER CHI +1843 ; ID_Start # Lm MONGOLIAN LETTER TODO LONG VOWEL SIGN +1844..1877 ; ID_Start # Lo [52] MONGOLIAN LETTER TODO E..MONGOLIAN LETTER MANCHU ZHA +1880..18A8 ; ID_Start # Lo [41] MONGOLIAN LETTER ALI GALI ANUSVARA ONE..MONGOLIAN LETTER MANCHU ALI GALI BHA +1E00..1E9B ; ID_Start # L& [156] LATIN CAPITAL LETTER A WITH RING BELOW..LATIN SMALL LETTER LONG S WITH DOT ABOVE +1EA0..1EF9 ; ID_Start # L& [90] LATIN CAPITAL LETTER A WITH DOT BELOW..LATIN SMALL LETTER Y WITH TILDE +1F00..1F15 ; ID_Start # L& [22] GREEK SMALL LETTER ALPHA WITH PSILI..GREEK SMALL LETTER EPSILON WITH DASIA AND OXIA +1F18..1F1D ; ID_Start # L& [6] GREEK CAPITAL LETTER EPSILON WITH PSILI..GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA +1F20..1F45 ; ID_Start # L& [38] GREEK SMALL LETTER ETA WITH PSILI..GREEK SMALL LETTER OMICRON WITH DASIA AND OXIA +1F48..1F4D ; ID_Start # L& [6] GREEK CAPITAL LETTER OMICRON WITH PSILI..GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA +1F50..1F57 ; ID_Start # L& [8] GREEK SMALL LETTER UPSILON WITH PSILI..GREEK SMALL LETTER UPSILON WITH DASIA AND PERISPOMENI +1F59 ; ID_Start # L& GREEK CAPITAL LETTER UPSILON WITH DASIA +1F5B ; ID_Start # L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA +1F5D ; ID_Start # L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA +1F5F..1F7D ; ID_Start # L& [31] GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI..GREEK SMALL LETTER OMEGA WITH OXIA +1F80..1FB4 ; ID_Start # L& [53] GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI..GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI +1FB6..1FBC ; ID_Start # L& [7] GREEK SMALL LETTER ALPHA WITH PERISPOMENI..GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI +1FBE ; ID_Start # L& GREEK PROSGEGRAMMENI +1FC2..1FC4 ; ID_Start # L& [3] GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI +1FC6..1FCC ; ID_Start # L& [7] GREEK SMALL LETTER ETA WITH PERISPOMENI..GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI +1FD0..1FD3 ; ID_Start # L& [4] GREEK SMALL LETTER IOTA WITH VRACHY..GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA +1FD6..1FDB ; ID_Start # L& [6] GREEK SMALL LETTER IOTA WITH PERISPOMENI..GREEK CAPITAL LETTER IOTA WITH OXIA +1FE0..1FEC ; ID_Start # L& [13] GREEK SMALL LETTER UPSILON WITH VRACHY..GREEK CAPITAL LETTER RHO WITH DASIA +1FF2..1FF4 ; ID_Start # L& [3] GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI +1FF6..1FFC ; ID_Start # L& [7] GREEK SMALL LETTER OMEGA WITH PERISPOMENI..GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI +2071 ; ID_Start # L& SUPERSCRIPT LATIN SMALL LETTER I +207F ; ID_Start # L& SUPERSCRIPT LATIN SMALL LETTER N +2102 ; ID_Start # L& DOUBLE-STRUCK CAPITAL C +2107 ; ID_Start # L& EULER CONSTANT +210A..2113 ; ID_Start # L& [10] SCRIPT SMALL G..SCRIPT SMALL L +2115 ; ID_Start # L& DOUBLE-STRUCK CAPITAL N +2119..211D ; ID_Start # L& [5] DOUBLE-STRUCK CAPITAL P..DOUBLE-STRUCK CAPITAL R +2124 ; ID_Start # L& DOUBLE-STRUCK CAPITAL Z +2126 ; ID_Start # L& OHM SIGN +2128 ; ID_Start # L& BLACK-LETTER CAPITAL Z +212A..212D ; ID_Start # L& [4] KELVIN SIGN..BLACK-LETTER CAPITAL C +212F..2131 ; ID_Start # L& [3] SCRIPT SMALL E..SCRIPT CAPITAL F +2133..2134 ; ID_Start # L& [2] SCRIPT CAPITAL M..SCRIPT SMALL O +2135..2138 ; ID_Start # Lo [4] ALEF SYMBOL..DALET SYMBOL +2139 ; ID_Start # L& INFORMATION SOURCE +213D..213F ; ID_Start # L& [3] DOUBLE-STRUCK SMALL GAMMA..DOUBLE-STRUCK CAPITAL PI +2145..2149 ; ID_Start # L& [5] DOUBLE-STRUCK ITALIC CAPITAL D..DOUBLE-STRUCK ITALIC SMALL J +2160..2183 ; ID_Start # Nl [36] ROMAN NUMERAL ONE..ROMAN NUMERAL REVERSED ONE HUNDRED +3005 ; ID_Start # Lm IDEOGRAPHIC ITERATION MARK +3006 ; ID_Start # Lo IDEOGRAPHIC CLOSING MARK +3007 ; ID_Start # Nl IDEOGRAPHIC NUMBER ZERO +3021..3029 ; ID_Start # Nl [9] HANGZHOU NUMERAL ONE..HANGZHOU NUMERAL NINE +3031..3035 ; ID_Start # Lm [5] VERTICAL KANA REPEAT MARK..VERTICAL KANA REPEAT MARK LOWER HALF +3038..303A ; ID_Start # Nl [3] HANGZHOU NUMERAL TEN..HANGZHOU NUMERAL THIRTY +303B ; ID_Start # Lm VERTICAL IDEOGRAPHIC ITERATION MARK +303C ; ID_Start # Lo MASU MARK +3041..3096 ; ID_Start # Lo [86] HIRAGANA LETTER SMALL A..HIRAGANA LETTER SMALL KE +309D..309E ; ID_Start # Lm [2] HIRAGANA ITERATION MARK..HIRAGANA VOICED ITERATION MARK +309F ; ID_Start # Lo HIRAGANA DIGRAPH YORI +30A1..30FA ; ID_Start # Lo [90] KATAKANA LETTER SMALL A..KATAKANA LETTER VO +30FC..30FE ; ID_Start # Lm [3] KATAKANA-HIRAGANA PROLONGED SOUND MARK..KATAKANA VOICED ITERATION MARK +30FF ; ID_Start # Lo KATAKANA DIGRAPH KOTO +3105..312C ; ID_Start # Lo [40] BOPOMOFO LETTER B..BOPOMOFO LETTER GN +3131..318E ; ID_Start # Lo [94] HANGUL LETTER KIYEOK..HANGUL LETTER ARAEAE +31A0..31B7 ; ID_Start # Lo [24] BOPOMOFO LETTER BU..BOPOMOFO FINAL LETTER H +31F0..31FF ; ID_Start # Lo [16] KATAKANA LETTER SMALL KU..KATAKANA LETTER SMALL RO +3400..4DB5 ; ID_Start # Lo [6582] CJK UNIFIED IDEOGRAPH-3400..CJK UNIFIED IDEOGRAPH-4DB5 +4E00..9FA5 ; ID_Start # Lo [20902] CJK UNIFIED IDEOGRAPH-4E00..CJK UNIFIED IDEOGRAPH-9FA5 +A000..A48C ; ID_Start # Lo [1165] YI SYLLABLE IT..YI SYLLABLE YYR +AC00..D7A3 ; ID_Start # Lo [11172] HANGUL SYLLABLE GA..HANGUL SYLLABLE HIH +F900..FA2D ; ID_Start # Lo [302] CJK COMPATIBILITY IDEOGRAPH-F900..CJK COMPATIBILITY IDEOGRAPH-FA2D +FA30..FA6A ; ID_Start # Lo [59] CJK COMPATIBILITY IDEOGRAPH-FA30..CJK COMPATIBILITY IDEOGRAPH-FA6A +FB00..FB06 ; ID_Start # L& [7] LATIN SMALL LIGATURE FF..LATIN SMALL LIGATURE ST +FB13..FB17 ; ID_Start # L& [5] ARMENIAN SMALL LIGATURE MEN NOW..ARMENIAN SMALL LIGATURE MEN XEH +FB1D ; ID_Start # Lo HEBREW LETTER YOD WITH HIRIQ +FB1F..FB28 ; ID_Start # Lo [10] HEBREW LIGATURE YIDDISH YOD YOD PATAH..HEBREW LETTER WIDE TAV +FB2A..FB36 ; ID_Start # Lo [13] HEBREW LETTER SHIN WITH SHIN DOT..HEBREW LETTER ZAYIN WITH DAGESH +FB38..FB3C ; ID_Start # Lo [5] HEBREW LETTER TET WITH DAGESH..HEBREW LETTER LAMED WITH DAGESH +FB3E ; ID_Start # Lo HEBREW LETTER MEM WITH DAGESH +FB40..FB41 ; ID_Start # Lo [2] HEBREW LETTER NUN WITH DAGESH..HEBREW LETTER SAMEKH WITH DAGESH +FB43..FB44 ; ID_Start # Lo [2] HEBREW LETTER FINAL PE WITH DAGESH..HEBREW LETTER PE WITH DAGESH +FB46..FBB1 ; ID_Start # Lo [108] HEBREW LETTER TSADI WITH DAGESH..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE FINAL FORM +FBD3..FD3D ; ID_Start # Lo [363] ARABIC LETTER NG ISOLATED FORM..ARABIC LIGATURE ALEF WITH FATHATAN ISOLATED FORM +FD50..FD8F ; ID_Start # Lo [64] ARABIC LIGATURE TEH WITH JEEM WITH MEEM INITIAL FORM..ARABIC LIGATURE MEEM WITH KHAH WITH MEEM INITIAL FORM +FD92..FDC7 ; ID_Start # Lo [54] ARABIC LIGATURE MEEM WITH JEEM WITH KHAH INITIAL FORM..ARABIC LIGATURE NOON WITH JEEM WITH YEH FINAL FORM +FDF0..FDFB ; ID_Start # Lo [12] ARABIC LIGATURE SALLA USED AS KORANIC STOP SIGN ISOLATED FORM..ARABIC LIGATURE JALLAJALALOUHOU +FE70..FE74 ; ID_Start # Lo [5] ARABIC FATHATAN ISOLATED FORM..ARABIC KASRATAN ISOLATED FORM +FE76..FEFC ; ID_Start # Lo [135] ARABIC FATHA ISOLATED FORM..ARABIC LIGATURE LAM WITH ALEF FINAL FORM +FF21..FF3A ; ID_Start # L& [26] FULLWIDTH LATIN CAPITAL LETTER A..FULLWIDTH LATIN CAPITAL LETTER Z +FF41..FF5A ; ID_Start # L& [26] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH LATIN SMALL LETTER Z +FF66..FF6F ; ID_Start # Lo [10] HALFWIDTH KATAKANA LETTER WO..HALFWIDTH KATAKANA LETTER SMALL TU +FF70 ; ID_Start # Lm HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK +FF71..FF9D ; ID_Start # Lo [45] HALFWIDTH KATAKANA LETTER A..HALFWIDTH KATAKANA LETTER N +FF9E..FF9F ; ID_Start # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK +FFA0..FFBE ; ID_Start # Lo [31] HALFWIDTH HANGUL FILLER..HALFWIDTH HANGUL LETTER HIEUH +FFC2..FFC7 ; ID_Start # Lo [6] HALFWIDTH HANGUL LETTER A..HALFWIDTH HANGUL LETTER E +FFCA..FFCF ; ID_Start # Lo [6] HALFWIDTH HANGUL LETTER YEO..HALFWIDTH HANGUL LETTER OE +FFD2..FFD7 ; ID_Start # Lo [6] HALFWIDTH HANGUL LETTER YO..HALFWIDTH HANGUL LETTER YU +FFDA..FFDC ; ID_Start # Lo [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL LETTER I +10300..1031E ; ID_Start # Lo [31] OLD ITALIC LETTER A..OLD ITALIC LETTER UU +10330..10349 ; ID_Start # Lo [26] GOTHIC LETTER AHSA..GOTHIC LETTER OTHAL +1034A ; ID_Start # Nl GOTHIC LETTER NINE HUNDRED +10400..10425 ; ID_Start # L& [38] DESERET CAPITAL LETTER LONG I..DESERET CAPITAL LETTER ENG +10428..1044D ; ID_Start # L& [38] DESERET SMALL LETTER LONG I..DESERET SMALL LETTER ENG +1D400..1D454 ; ID_Start # L& [85] MATHEMATICAL BOLD CAPITAL A..MATHEMATICAL ITALIC SMALL G +1D456..1D49C ; ID_Start # L& [71] MATHEMATICAL ITALIC SMALL I..MATHEMATICAL SCRIPT CAPITAL A +1D49E..1D49F ; ID_Start # L& [2] MATHEMATICAL SCRIPT CAPITAL C..MATHEMATICAL SCRIPT CAPITAL D +1D4A2 ; ID_Start # L& MATHEMATICAL SCRIPT CAPITAL G +1D4A5..1D4A6 ; ID_Start # L& [2] MATHEMATICAL SCRIPT CAPITAL J..MATHEMATICAL SCRIPT CAPITAL K +1D4A9..1D4AC ; ID_Start # L& [4] MATHEMATICAL SCRIPT CAPITAL N..MATHEMATICAL SCRIPT CAPITAL Q +1D4AE..1D4B9 ; ID_Start # L& [12] MATHEMATICAL SCRIPT CAPITAL S..MATHEMATICAL SCRIPT SMALL D +1D4BB ; ID_Start # L& MATHEMATICAL SCRIPT SMALL F +1D4BD..1D4C0 ; ID_Start # L& [4] MATHEMATICAL SCRIPT SMALL H..MATHEMATICAL SCRIPT SMALL K +1D4C2..1D4C3 ; ID_Start # L& [2] MATHEMATICAL SCRIPT SMALL M..MATHEMATICAL SCRIPT SMALL N +1D4C5..1D505 ; ID_Start # L& [65] MATHEMATICAL SCRIPT SMALL P..MATHEMATICAL FRAKTUR CAPITAL B +1D507..1D50A ; ID_Start # L& [4] MATHEMATICAL FRAKTUR CAPITAL D..MATHEMATICAL FRAKTUR CAPITAL G +1D50D..1D514 ; ID_Start # L& [8] MATHEMATICAL FRAKTUR CAPITAL J..MATHEMATICAL FRAKTUR CAPITAL Q +1D516..1D51C ; ID_Start # L& [7] MATHEMATICAL FRAKTUR CAPITAL S..MATHEMATICAL FRAKTUR CAPITAL Y +1D51E..1D539 ; ID_Start # L& [28] MATHEMATICAL FRAKTUR SMALL A..MATHEMATICAL DOUBLE-STRUCK CAPITAL B +1D53B..1D53E ; ID_Start # L& [4] MATHEMATICAL DOUBLE-STRUCK CAPITAL D..MATHEMATICAL DOUBLE-STRUCK CAPITAL G +1D540..1D544 ; ID_Start # L& [5] MATHEMATICAL DOUBLE-STRUCK CAPITAL I..MATHEMATICAL DOUBLE-STRUCK CAPITAL M +1D546 ; ID_Start # L& MATHEMATICAL DOUBLE-STRUCK CAPITAL O +1D54A..1D550 ; ID_Start # L& [7] MATHEMATICAL DOUBLE-STRUCK CAPITAL S..MATHEMATICAL DOUBLE-STRUCK CAPITAL Y +1D552..1D6A3 ; ID_Start # L& [338] MATHEMATICAL DOUBLE-STRUCK SMALL A..MATHEMATICAL MONOSPACE SMALL Z +1D6A8..1D6C0 ; ID_Start # L& [25] MATHEMATICAL BOLD CAPITAL ALPHA..MATHEMATICAL BOLD CAPITAL OMEGA +1D6C2..1D6DA ; ID_Start # L& [25] MATHEMATICAL BOLD SMALL ALPHA..MATHEMATICAL BOLD SMALL OMEGA +1D6DC..1D6FA ; ID_Start # L& [31] MATHEMATICAL BOLD EPSILON SYMBOL..MATHEMATICAL ITALIC CAPITAL OMEGA +1D6FC..1D714 ; ID_Start # L& [25] MATHEMATICAL ITALIC SMALL ALPHA..MATHEMATICAL ITALIC SMALL OMEGA +1D716..1D734 ; ID_Start # L& [31] MATHEMATICAL ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD ITALIC CAPITAL OMEGA +1D736..1D74E ; ID_Start # L& [25] MATHEMATICAL BOLD ITALIC SMALL ALPHA..MATHEMATICAL BOLD ITALIC SMALL OMEGA +1D750..1D76E ; ID_Start # L& [31] MATHEMATICAL BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD CAPITAL OMEGA +1D770..1D788 ; ID_Start # L& [25] MATHEMATICAL SANS-SERIF BOLD SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD SMALL OMEGA +1D78A..1D7A8 ; ID_Start # L& [31] MATHEMATICAL SANS-SERIF BOLD EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL OMEGA +1D7AA..1D7C2 ; ID_Start # L& [25] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL OMEGA +1D7C4..1D7C9 ; ID_Start # L& [6] MATHEMATICAL SANS-SERIF BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC PI SYMBOL +20000..2A6D6 ; ID_Start # Lo [42711] CJK UNIFIED IDEOGRAPH-20000..CJK UNIFIED IDEOGRAPH-2A6D6 +2F800..2FA1D ; ID_Start # Lo [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D + +# Total code points: 90010 + +# ================================================ + +# Derived Property: ID_Continue +# Characters that can continue an identifier. +# Generated from: ID_Start + Mn+Mc+Nd+Pc +# NOTE: Cf characters should be filtered out. + +0030..0039 ; ID_Continue # Nd [10] DIGIT ZERO..DIGIT NINE +0041..005A ; ID_Continue # L& [26] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER Z +005F ; ID_Continue # Pc LOW LINE +0061..007A ; ID_Continue # L& [26] LATIN SMALL LETTER A..LATIN SMALL LETTER Z +00AA ; ID_Continue # L& FEMININE ORDINAL INDICATOR +00B5 ; ID_Continue # L& MICRO SIGN +00BA ; ID_Continue # L& MASCULINE ORDINAL INDICATOR +00C0..00D6 ; ID_Continue # L& [23] LATIN CAPITAL LETTER A WITH GRAVE..LATIN CAPITAL LETTER O WITH DIAERESIS +00D8..00F6 ; ID_Continue # L& [31] LATIN CAPITAL LETTER O WITH STROKE..LATIN SMALL LETTER O WITH DIAERESIS +00F8..01BA ; ID_Continue # L& [195] LATIN SMALL LETTER O WITH STROKE..LATIN SMALL LETTER EZH WITH TAIL +01BB ; ID_Continue # Lo LATIN LETTER TWO WITH STROKE +01BC..01BF ; ID_Continue # L& [4] LATIN CAPITAL LETTER TONE FIVE..LATIN LETTER WYNN +01C0..01C3 ; ID_Continue # Lo [4] LATIN LETTER DENTAL CLICK..LATIN LETTER RETROFLEX CLICK +01C4..0220 ; ID_Continue # L& [93] LATIN CAPITAL LETTER DZ WITH CARON..LATIN CAPITAL LETTER N WITH LONG RIGHT LEG +0222..0233 ; ID_Continue # L& [18] LATIN CAPITAL LETTER OU..LATIN SMALL LETTER Y WITH MACRON +0250..02AD ; ID_Continue # L& [94] LATIN SMALL LETTER TURNED A..LATIN LETTER BIDENTAL PERCUSSIVE +02B0..02B8 ; ID_Continue # Lm [9] MODIFIER LETTER SMALL H..MODIFIER LETTER SMALL Y +02BB..02C1 ; ID_Continue # Lm [7] MODIFIER LETTER TURNED COMMA..MODIFIER LETTER REVERSED GLOTTAL STOP +02D0..02D1 ; ID_Continue # Lm [2] MODIFIER LETTER TRIANGULAR COLON..MODIFIER LETTER HALF TRIANGULAR COLON +02E0..02E4 ; ID_Continue # Lm [5] MODIFIER LETTER SMALL GAMMA..MODIFIER LETTER SMALL REVERSED GLOTTAL STOP +02EE ; ID_Continue # Lm MODIFIER LETTER DOUBLE APOSTROPHE +0300..034F ; ID_Continue # Mn [80] COMBINING GRAVE ACCENT..COMBINING GRAPHEME JOINER +0360..036F ; ID_Continue # Mn [16] COMBINING DOUBLE TILDE..COMBINING LATIN SMALL LETTER X +037A ; ID_Continue # Lm GREEK YPOGEGRAMMENI +0386 ; ID_Continue # L& GREEK CAPITAL LETTER ALPHA WITH TONOS +0388..038A ; ID_Continue # L& [3] GREEK CAPITAL LETTER EPSILON WITH TONOS..GREEK CAPITAL LETTER IOTA WITH TONOS +038C ; ID_Continue # L& GREEK CAPITAL LETTER OMICRON WITH TONOS +038E..03A1 ; ID_Continue # L& [20] GREEK CAPITAL LETTER UPSILON WITH TONOS..GREEK CAPITAL LETTER RHO +03A3..03CE ; ID_Continue # L& [44] GREEK CAPITAL LETTER SIGMA..GREEK SMALL LETTER OMEGA WITH TONOS +03D0..03F5 ; ID_Continue # L& [38] GREEK BETA SYMBOL..GREEK LUNATE EPSILON SYMBOL +0400..0481 ; ID_Continue # L& [130] CYRILLIC CAPITAL LETTER IE WITH GRAVE..CYRILLIC SMALL LETTER KOPPA +0483..0486 ; ID_Continue # Mn [4] COMBINING CYRILLIC TITLO..COMBINING CYRILLIC PSILI PNEUMATA +048A..04CE ; ID_Continue # L& [69] CYRILLIC CAPITAL LETTER SHORT I WITH TAIL..CYRILLIC SMALL LETTER EM WITH TAIL +04D0..04F5 ; ID_Continue # L& [38] CYRILLIC CAPITAL LETTER A WITH BREVE..CYRILLIC SMALL LETTER CHE WITH DIAERESIS +04F8..04F9 ; ID_Continue # L& [2] CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS..CYRILLIC SMALL LETTER YERU WITH DIAERESIS +0500..050F ; ID_Continue # L& [16] CYRILLIC CAPITAL LETTER KOMI DE..CYRILLIC SMALL LETTER KOMI TJE +0531..0556 ; ID_Continue # L& [38] ARMENIAN CAPITAL LETTER AYB..ARMENIAN CAPITAL LETTER FEH +0559 ; ID_Continue # Lm ARMENIAN MODIFIER LETTER LEFT HALF RING +0561..0587 ; ID_Continue # L& [39] ARMENIAN SMALL LETTER AYB..ARMENIAN SMALL LIGATURE ECH YIWN +0591..05A1 ; ID_Continue # Mn [17] HEBREW ACCENT ETNAHTA..HEBREW ACCENT PAZER +05A3..05B9 ; ID_Continue # Mn [23] HEBREW ACCENT MUNAH..HEBREW POINT HOLAM +05BB..05BD ; ID_Continue # Mn [3] HEBREW POINT QUBUTS..HEBREW POINT METEG +05BF ; ID_Continue # Mn HEBREW POINT RAFE +05C1..05C2 ; ID_Continue # Mn [2] HEBREW POINT SHIN DOT..HEBREW POINT SIN DOT +05C4 ; ID_Continue # Mn HEBREW MARK UPPER DOT +05D0..05EA ; ID_Continue # Lo [27] HEBREW LETTER ALEF..HEBREW LETTER TAV +05F0..05F2 ; ID_Continue # Lo [3] HEBREW LIGATURE YIDDISH DOUBLE VAV..HEBREW LIGATURE YIDDISH DOUBLE YOD +0621..063A ; ID_Continue # Lo [26] ARABIC LETTER HAMZA..ARABIC LETTER GHAIN +0640 ; ID_Continue # Lm ARABIC TATWEEL +0641..064A ; ID_Continue # Lo [10] ARABIC LETTER FEH..ARABIC LETTER YEH +064B..0655 ; ID_Continue # Mn [11] ARABIC FATHATAN..ARABIC HAMZA BELOW +0660..0669 ; ID_Continue # Nd [10] ARABIC-INDIC DIGIT ZERO..ARABIC-INDIC DIGIT NINE +066E..066F ; ID_Continue # Lo [2] ARABIC LETTER DOTLESS BEH..ARABIC LETTER DOTLESS QAF +0670 ; ID_Continue # Mn ARABIC LETTER SUPERSCRIPT ALEF +0671..06D3 ; ID_Continue # Lo [99] ARABIC LETTER ALEF WASLA..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE +06D5 ; ID_Continue # Lo ARABIC LETTER AE +06D6..06DC ; ID_Continue # Mn [7] ARABIC SMALL HIGH LIGATURE SAD WITH LAM WITH ALEF MAKSURA..ARABIC SMALL HIGH SEEN +06DF..06E4 ; ID_Continue # Mn [6] ARABIC SMALL HIGH ROUNDED ZERO..ARABIC SMALL HIGH MADDA +06E5..06E6 ; ID_Continue # Lm [2] ARABIC SMALL WAW..ARABIC SMALL YEH +06E7..06E8 ; ID_Continue # Mn [2] ARABIC SMALL HIGH YEH..ARABIC SMALL HIGH NOON +06EA..06ED ; ID_Continue # Mn [4] ARABIC EMPTY CENTRE LOW STOP..ARABIC SMALL LOW MEEM +06F0..06F9 ; ID_Continue # Nd [10] EXTENDED ARABIC-INDIC DIGIT ZERO..EXTENDED ARABIC-INDIC DIGIT NINE +06FA..06FC ; ID_Continue # Lo [3] ARABIC LETTER SHEEN WITH DOT BELOW..ARABIC LETTER GHAIN WITH DOT BELOW +0710 ; ID_Continue # Lo SYRIAC LETTER ALAPH +0711 ; ID_Continue # Mn SYRIAC LETTER SUPERSCRIPT ALAPH +0712..072C ; ID_Continue # Lo [27] SYRIAC LETTER BETH..SYRIAC LETTER TAW +0730..074A ; ID_Continue # Mn [27] SYRIAC PTHAHA ABOVE..SYRIAC BARREKH +0780..07A5 ; ID_Continue # Lo [38] THAANA LETTER HAA..THAANA LETTER WAAVU +07A6..07B0 ; ID_Continue # Mn [11] THAANA ABAFILI..THAANA SUKUN +07B1 ; ID_Continue # Lo THAANA LETTER NAA +0901..0902 ; ID_Continue # Mn [2] DEVANAGARI SIGN CANDRABINDU..DEVANAGARI SIGN ANUSVARA +0903 ; ID_Continue # Mc DEVANAGARI SIGN VISARGA +0905..0939 ; ID_Continue # Lo [53] DEVANAGARI LETTER A..DEVANAGARI LETTER HA +093C ; ID_Continue # Mn DEVANAGARI SIGN NUKTA +093D ; ID_Continue # Lo DEVANAGARI SIGN AVAGRAHA +093E..0940 ; ID_Continue # Mc [3] DEVANAGARI VOWEL SIGN AA..DEVANAGARI VOWEL SIGN II +0941..0948 ; ID_Continue # Mn [8] DEVANAGARI VOWEL SIGN U..DEVANAGARI VOWEL SIGN AI +0949..094C ; ID_Continue # Mc [4] DEVANAGARI VOWEL SIGN CANDRA O..DEVANAGARI VOWEL SIGN AU +094D ; ID_Continue # Mn DEVANAGARI SIGN VIRAMA +0950 ; ID_Continue # Lo DEVANAGARI OM +0951..0954 ; ID_Continue # Mn [4] DEVANAGARI STRESS SIGN UDATTA..DEVANAGARI ACUTE ACCENT +0958..0961 ; ID_Continue # Lo [10] DEVANAGARI LETTER QA..DEVANAGARI LETTER VOCALIC LL +0962..0963 ; ID_Continue # Mn [2] DEVANAGARI VOWEL SIGN VOCALIC L..DEVANAGARI VOWEL SIGN VOCALIC LL +0966..096F ; ID_Continue # Nd [10] DEVANAGARI DIGIT ZERO..DEVANAGARI DIGIT NINE +0981 ; ID_Continue # Mn BENGALI SIGN CANDRABINDU +0982..0983 ; ID_Continue # Mc [2] BENGALI SIGN ANUSVARA..BENGALI SIGN VISARGA +0985..098C ; ID_Continue # Lo [8] BENGALI LETTER A..BENGALI LETTER VOCALIC L +098F..0990 ; ID_Continue # Lo [2] BENGALI LETTER E..BENGALI LETTER AI +0993..09A8 ; ID_Continue # Lo [22] BENGALI LETTER O..BENGALI LETTER NA +09AA..09B0 ; ID_Continue # Lo [7] BENGALI LETTER PA..BENGALI LETTER RA +09B2 ; ID_Continue # Lo BENGALI LETTER LA +09B6..09B9 ; ID_Continue # Lo [4] BENGALI LETTER SHA..BENGALI LETTER HA +09BC ; ID_Continue # Mn BENGALI SIGN NUKTA +09BE..09C0 ; ID_Continue # Mc [3] BENGALI VOWEL SIGN AA..BENGALI VOWEL SIGN II +09C1..09C4 ; ID_Continue # Mn [4] BENGALI VOWEL SIGN U..BENGALI VOWEL SIGN VOCALIC RR +09C7..09C8 ; ID_Continue # Mc [2] BENGALI VOWEL SIGN E..BENGALI VOWEL SIGN AI +09CB..09CC ; ID_Continue # Mc [2] BENGALI VOWEL SIGN O..BENGALI VOWEL SIGN AU +09CD ; ID_Continue # Mn BENGALI SIGN VIRAMA +09D7 ; ID_Continue # Mc BENGALI AU LENGTH MARK +09DC..09DD ; ID_Continue # Lo [2] BENGALI LETTER RRA..BENGALI LETTER RHA +09DF..09E1 ; ID_Continue # Lo [3] BENGALI LETTER YYA..BENGALI LETTER VOCALIC LL +09E2..09E3 ; ID_Continue # Mn [2] BENGALI VOWEL SIGN VOCALIC L..BENGALI VOWEL SIGN VOCALIC LL +09E6..09EF ; ID_Continue # Nd [10] BENGALI DIGIT ZERO..BENGALI DIGIT NINE +09F0..09F1 ; ID_Continue # Lo [2] BENGALI LETTER RA WITH MIDDLE DIAGONAL..BENGALI LETTER RA WITH LOWER DIAGONAL +0A02 ; ID_Continue # Mn GURMUKHI SIGN BINDI +0A05..0A0A ; ID_Continue # Lo [6] GURMUKHI LETTER A..GURMUKHI LETTER UU +0A0F..0A10 ; ID_Continue # Lo [2] GURMUKHI LETTER EE..GURMUKHI LETTER AI +0A13..0A28 ; ID_Continue # Lo [22] GURMUKHI LETTER OO..GURMUKHI LETTER NA +0A2A..0A30 ; ID_Continue # Lo [7] GURMUKHI LETTER PA..GURMUKHI LETTER RA +0A32..0A33 ; ID_Continue # Lo [2] GURMUKHI LETTER LA..GURMUKHI LETTER LLA +0A35..0A36 ; ID_Continue # Lo [2] GURMUKHI LETTER VA..GURMUKHI LETTER SHA +0A38..0A39 ; ID_Continue # Lo [2] GURMUKHI LETTER SA..GURMUKHI LETTER HA +0A3C ; ID_Continue # Mn GURMUKHI SIGN NUKTA +0A3E..0A40 ; ID_Continue # Mc [3] GURMUKHI VOWEL SIGN AA..GURMUKHI VOWEL SIGN II +0A41..0A42 ; ID_Continue # Mn [2] GURMUKHI VOWEL SIGN U..GURMUKHI VOWEL SIGN UU +0A47..0A48 ; ID_Continue # Mn [2] GURMUKHI VOWEL SIGN EE..GURMUKHI VOWEL SIGN AI +0A4B..0A4D ; ID_Continue # Mn [3] GURMUKHI VOWEL SIGN OO..GURMUKHI SIGN VIRAMA +0A59..0A5C ; ID_Continue # Lo [4] GURMUKHI LETTER KHHA..GURMUKHI LETTER RRA +0A5E ; ID_Continue # Lo GURMUKHI LETTER FA +0A66..0A6F ; ID_Continue # Nd [10] GURMUKHI DIGIT ZERO..GURMUKHI DIGIT NINE +0A70..0A71 ; ID_Continue # Mn [2] GURMUKHI TIPPI..GURMUKHI ADDAK +0A72..0A74 ; ID_Continue # Lo [3] GURMUKHI IRI..GURMUKHI EK ONKAR +0A81..0A82 ; ID_Continue # Mn [2] GUJARATI SIGN CANDRABINDU..GUJARATI SIGN ANUSVARA +0A83 ; ID_Continue # Mc GUJARATI SIGN VISARGA +0A85..0A8B ; ID_Continue # Lo [7] GUJARATI LETTER A..GUJARATI LETTER VOCALIC R +0A8D ; ID_Continue # Lo GUJARATI VOWEL CANDRA E +0A8F..0A91 ; ID_Continue # Lo [3] GUJARATI LETTER E..GUJARATI VOWEL CANDRA O +0A93..0AA8 ; ID_Continue # Lo [22] GUJARATI LETTER O..GUJARATI LETTER NA +0AAA..0AB0 ; ID_Continue # Lo [7] GUJARATI LETTER PA..GUJARATI LETTER RA +0AB2..0AB3 ; ID_Continue # Lo [2] GUJARATI LETTER LA..GUJARATI LETTER LLA +0AB5..0AB9 ; ID_Continue # Lo [5] GUJARATI LETTER VA..GUJARATI LETTER HA +0ABC ; ID_Continue # Mn GUJARATI SIGN NUKTA +0ABD ; ID_Continue # Lo GUJARATI SIGN AVAGRAHA +0ABE..0AC0 ; ID_Continue # Mc [3] GUJARATI VOWEL SIGN AA..GUJARATI VOWEL SIGN II +0AC1..0AC5 ; ID_Continue # Mn [5] GUJARATI VOWEL SIGN U..GUJARATI VOWEL SIGN CANDRA E +0AC7..0AC8 ; ID_Continue # Mn [2] GUJARATI VOWEL SIGN E..GUJARATI VOWEL SIGN AI +0AC9 ; ID_Continue # Mc GUJARATI VOWEL SIGN CANDRA O +0ACB..0ACC ; ID_Continue # Mc [2] GUJARATI VOWEL SIGN O..GUJARATI VOWEL SIGN AU +0ACD ; ID_Continue # Mn GUJARATI SIGN VIRAMA +0AD0 ; ID_Continue # Lo GUJARATI OM +0AE0 ; ID_Continue # Lo GUJARATI LETTER VOCALIC RR +0AE6..0AEF ; ID_Continue # Nd [10] GUJARATI DIGIT ZERO..GUJARATI DIGIT NINE +0B01 ; ID_Continue # Mn ORIYA SIGN CANDRABINDU +0B02..0B03 ; ID_Continue # Mc [2] ORIYA SIGN ANUSVARA..ORIYA SIGN VISARGA +0B05..0B0C ; ID_Continue # Lo [8] ORIYA LETTER A..ORIYA LETTER VOCALIC L +0B0F..0B10 ; ID_Continue # Lo [2] ORIYA LETTER E..ORIYA LETTER AI +0B13..0B28 ; ID_Continue # Lo [22] ORIYA LETTER O..ORIYA LETTER NA +0B2A..0B30 ; ID_Continue # Lo [7] ORIYA LETTER PA..ORIYA LETTER RA +0B32..0B33 ; ID_Continue # Lo [2] ORIYA LETTER LA..ORIYA LETTER LLA +0B36..0B39 ; ID_Continue # Lo [4] ORIYA LETTER SHA..ORIYA LETTER HA +0B3C ; ID_Continue # Mn ORIYA SIGN NUKTA +0B3D ; ID_Continue # Lo ORIYA SIGN AVAGRAHA +0B3E ; ID_Continue # Mc ORIYA VOWEL SIGN AA +0B3F ; ID_Continue # Mn ORIYA VOWEL SIGN I +0B40 ; ID_Continue # Mc ORIYA VOWEL SIGN II +0B41..0B43 ; ID_Continue # Mn [3] ORIYA VOWEL SIGN U..ORIYA VOWEL SIGN VOCALIC R +0B47..0B48 ; ID_Continue # Mc [2] ORIYA VOWEL SIGN E..ORIYA VOWEL SIGN AI +0B4B..0B4C ; ID_Continue # Mc [2] ORIYA VOWEL SIGN O..ORIYA VOWEL SIGN AU +0B4D ; ID_Continue # Mn ORIYA SIGN VIRAMA +0B56 ; ID_Continue # Mn ORIYA AI LENGTH MARK +0B57 ; ID_Continue # Mc ORIYA AU LENGTH MARK +0B5C..0B5D ; ID_Continue # Lo [2] ORIYA LETTER RRA..ORIYA LETTER RHA +0B5F..0B61 ; ID_Continue # Lo [3] ORIYA LETTER YYA..ORIYA LETTER VOCALIC LL +0B66..0B6F ; ID_Continue # Nd [10] ORIYA DIGIT ZERO..ORIYA DIGIT NINE +0B82 ; ID_Continue # Mn TAMIL SIGN ANUSVARA +0B83 ; ID_Continue # Lo TAMIL SIGN VISARGA +0B85..0B8A ; ID_Continue # Lo [6] TAMIL LETTER A..TAMIL LETTER UU +0B8E..0B90 ; ID_Continue # Lo [3] TAMIL LETTER E..TAMIL LETTER AI +0B92..0B95 ; ID_Continue # Lo [4] TAMIL LETTER O..TAMIL LETTER KA +0B99..0B9A ; ID_Continue # Lo [2] TAMIL LETTER NGA..TAMIL LETTER CA +0B9C ; ID_Continue # Lo TAMIL LETTER JA From noreply at buildbot.pypy.org Mon Jan 28 09:18:32 2013 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 28 Jan 2013 09:18:32 +0100 (CET) Subject: [pypy-commit] pypy default: Expose more of openssl, required by Python3 Message-ID: <20130128081832.4BE561C12FA@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: Changeset: r60579:bc46a72bce60 Date: 2013-01-27 23:47 +0100 http://bitbucket.org/pypy/pypy/changeset/bc46a72bce60/ Log: Expose more of openssl, required by Python3 diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py --- a/rpython/rlib/ropenssl.py +++ b/rpython/rlib/ropenssl.py @@ -2,6 +2,7 @@ from rpython.rtyper.tool import rffi_platform from rpython.translator.platform import platform from rpython.translator.tool.cbuild import ExternalCompilationInfo +from rpython.rlib.unroll import unrolling_iterable import sys, os @@ -92,8 +93,12 @@ OPENSSL_NO_SSL2 = rffi_platform.Defined("OPENSSL_NO_SSL2") SSL_FILETYPE_PEM = rffi_platform.ConstantInteger("SSL_FILETYPE_PEM") SSL_OP_ALL = rffi_platform.ConstantInteger("SSL_OP_ALL") + SSL_OP_NO_SSLv2 = rffi_platform.ConstantInteger("SSL_OP_NO_SSLv2") + SSL_OP_NO_SSLv3 = rffi_platform.ConstantInteger("SSL_OP_NO_SSLv3") + SSL_OP_NO_TLSv1 = rffi_platform.ConstantInteger("SSL_OP_NO_TLSv1") SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS = rffi_platform.ConstantInteger( "SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS") + HAS_SNI = rffi_platform.Defined("SSL_CTRL_SET_TLSEXT_HOSTNAME") SSL_VERIFY_NONE = rffi_platform.ConstantInteger("SSL_VERIFY_NONE") SSL_VERIFY_PEER = rffi_platform.ConstantInteger("SSL_VERIFY_PEER") SSL_VERIFY_FAIL_IF_NO_PEER_CERT = rffi_platform.ConstantInteger("SSL_VERIFY_FAIL_IF_NO_PEER_CERT") @@ -118,6 +123,9 @@ CRYPTO_LOCK = rffi_platform.ConstantInteger("CRYPTO_LOCK") + OBJ_NAME_TYPE_MD_METH = rffi_platform.ConstantInteger( + "OBJ_NAME_TYPE_MD_METH") + # Some structures, with only the fields used in the _ssl module X509_name_entry_st = rffi_platform.Struct('struct X509_name_entry_st', [('set', rffi.INT)]) @@ -146,6 +154,12 @@ OPENSSL_EXPORT_VAR_AS_FUNCTION = rffi_platform.Defined( "OPENSSL_EXPORT_VAR_AS_FUNCTION") + OBJ_NAME_st = rffi_platform.Struct( + 'OBJ_NAME', + [('alias', rffi.INT), + ('name', rffi.CCHARP), + ]) + for k, v in rffi_platform.configure(CConfig).items(): globals()[k] = v @@ -166,8 +180,10 @@ ASN1_INTEGER = rffi.COpaquePtr('ASN1_INTEGER') GENERAL_NAMES = rffi.COpaquePtr('GENERAL_NAMES') GENERAL_NAME = rffi.CArrayPtr(GENERAL_NAME_st) +OBJ_NAME = rffi.CArrayPtr(OBJ_NAME_st) HAVE_OPENSSL_RAND = OPENSSL_VERSION_NUMBER >= 0x0090500f +HAVE_SSL_CTX_CLEAR_OPTIONS = OPENSSL_VERSION_NUMBER >= 0x009080df def external(name, argtypes, restype, **kw): kw['compilation_info'] = eci @@ -203,12 +219,26 @@ ssl_external('SSLv23_method', [], SSL_METHOD) ssl_external('SSL_CTX_use_PrivateKey_file', [SSL_CTX, rffi.CCHARP, rffi.INT], rffi.INT) ssl_external('SSL_CTX_use_certificate_chain_file', [SSL_CTX, rffi.CCHARP], rffi.INT) +ssl_external('SSL_CTX_get_options', [SSL_CTX], rffi.INT, macro=True) ssl_external('SSL_CTX_set_options', [SSL_CTX, rffi.INT], rffi.INT, macro=True) +if HAVE_SSL_CTX_CLEAR_OPTIONS: + ssl_external('SSL_CTX_clear_options', [SSL_CTX, rffi.INT], rffi.INT, + macro=True) ssl_external('SSL_CTX_ctrl', [SSL_CTX, rffi.INT, rffi.INT, rffi.VOIDP], rffi.INT) ssl_external('SSL_CTX_set_verify', [SSL_CTX, rffi.INT, rffi.VOIDP], lltype.Void) ssl_external('SSL_CTX_get_verify_mode', [SSL_CTX], rffi.INT) +ssl_external('SSL_CTX_set_default_verify_paths', [SSL_CTX], rffi.INT) ssl_external('SSL_CTX_set_cipher_list', [SSL_CTX, rffi.CCHARP], rffi.INT) ssl_external('SSL_CTX_load_verify_locations', [SSL_CTX, rffi.CCHARP, rffi.CCHARP], rffi.INT) +ssl_external('SSL_CTX_check_private_key', [SSL_CTX], rffi.INT) +ssl_external('SSL_CTX_set_session_id_context', [SSL_CTX, rffi.CCHARP, rffi.UINT], rffi.INT) +SSL_CTX_STATS_NAMES = """ + number connect connect_good connect_renegotiate accept accept_good + accept_renegotiate hits misses timeouts cache_full""".split() +SSL_CTX_STATS = unrolling_iterable( + (name, external('SSL_CTX_sess_' + name, [SSL_CTX], rffi.LONG, macro=True)) + for name in SSL_CTX_STATS_NAMES) + ssl_external('SSL_new', [SSL_CTX], SSL) ssl_external('SSL_set_fd', [SSL, rffi.INT], rffi.INT) ssl_external('SSL_set_mode', [SSL, rffi.INT], rffi.INT, macro=True) @@ -224,6 +254,7 @@ ssl_external('SSL_get_error', [SSL, rffi.INT], rffi.INT) ssl_external('SSL_get_shutdown', [SSL], rffi.INT) ssl_external('SSL_set_read_ahead', [SSL, rffi.INT], lltype.Void) +ssl_external('SSL_set_tlsext_host_name', [SSL, rffi.CCHARP], rffi.INT, macro=True) ssl_external('SSL_get_peer_certificate', [SSL], X509) ssl_external('X509_get_subject_name', [X509], X509_NAME) @@ -234,7 +265,7 @@ ssl_external('X509_NAME_ENTRY_get_object', [X509_NAME_ENTRY], ASN1_OBJECT) ssl_external('X509_NAME_ENTRY_get_data', [X509_NAME_ENTRY], ASN1_STRING) ssl_external('i2d_X509', [X509, rffi.CCHARPP], rffi.INT) -ssl_external('X509_free', [X509], lltype.Void) +ssl_external('X509_free', [X509], lltype.Void, threadsafe=False) ssl_external('X509_get_notBefore', [X509], ASN1_TIME, macro=True) ssl_external('X509_get_notAfter', [X509], ASN1_TIME, macro=True) ssl_external('X509_get_serialNumber', [X509], ASN1_INTEGER) @@ -272,9 +303,12 @@ ssl_external('ERR_get_error', [], rffi.INT) ssl_external('ERR_peek_last_error', [], rffi.INT) ssl_external('ERR_error_string', [rffi.ULONG, rffi.CCHARP], rffi.CCHARP) +ssl_external('ERR_clear_error', [], lltype.Void) -ssl_external('SSL_free', [SSL], lltype.Void) -ssl_external('SSL_CTX_free', [SSL_CTX], lltype.Void) +# 'threadsafe=False' here indicates that this function will be called +# with the GIL held, and so is allowed to run in a RPython __del__ method. +ssl_external('SSL_free', [SSL], lltype.Void, threadsafe=False) +ssl_external('SSL_CTX_free', [SSL_CTX], lltype.Void, threadsafe=False) ssl_external('CRYPTO_free', [rffi.VOIDP], lltype.Void) libssl_OPENSSL_free = libssl_CRYPTO_free @@ -316,6 +350,11 @@ EVP_MD_CTX_cleanup = external( 'EVP_MD_CTX_cleanup', [EVP_MD_CTX], rffi.INT, threadsafe=False) +OBJ_NAME_CALLBACK = lltype.Ptr(lltype.FuncType( + [OBJ_NAME, rffi.VOIDP], lltype.Void)) +OBJ_NAME_do_all = external( + 'OBJ_NAME_do_all', [rffi.INT, OBJ_NAME_CALLBACK, rffi.VOIDP], lltype.Void) + # HASH_MALLOC_SIZE is the size of EVP_MD, EVP_MD_CTX plus their points # Used for adding memory pressure. Last number is an (under?)estimate of # EVP_PKEY_CTX's size. From noreply at buildbot.pypy.org Mon Jan 28 09:18:33 2013 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 28 Jan 2013 09:18:33 +0100 (CET) Subject: [pypy-commit] pypy default: Install the py3k version of C thread helpers. Message-ID: <20130128081833.7414F1C12FA@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: Changeset: r60580:053753bebf0c Date: 2013-01-28 09:14 +0100 http://bitbucket.org/pypy/pypy/changeset/053753bebf0c/ Log: Install the py3k version of C thread helpers. Also implements Rlock.acquire_timed() diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py --- a/rpython/rlib/rthread.py +++ b/rpython/rlib/rthread.py @@ -19,9 +19,9 @@ separate_module_files = [translator_c_dir / 'src' / 'thread.c'], include_dirs = [translator_c_dir], export_symbols = ['RPyThreadGetIdent', 'RPyThreadLockInit', - 'RPyThreadAcquireLock', 'RPyThreadReleaseLock', - 'RPyGilAllocate', 'RPyGilYieldThread', - 'RPyGilRelease', 'RPyGilAcquire', + 'RPyThreadAcquireLock', 'RPyThreadAcquireLockTimed', + 'RPyThreadReleaseLock', 'RPyGilAllocate', + 'RPyGilYieldThread', 'RPyGilRelease', 'RPyGilAcquire', 'RPyThreadGetStackSize', 'RPyThreadSetStackSize', 'RPyOpaqueDealloc_ThreadLock', 'RPyThreadAfterFork'] @@ -61,6 +61,10 @@ c_thread_acquirelock = llexternal('RPyThreadAcquireLock', [TLOCKP, rffi.INT], rffi.INT, threadsafe=True) # release the GIL +c_thread_acquirelock_timed = llexternal('RPyThreadAcquireLockTimed', + [TLOCKP, rffi.LONGLONG, rffi.INT], + rffi.INT, + threadsafe=True) # release the GIL c_thread_releaselock = llexternal('RPyThreadReleaseLock', [TLOCKP], lltype.Void, threadsafe=True) # release the GIL @@ -121,6 +125,12 @@ res = rffi.cast(lltype.Signed, res) return bool(res) + def acquire_timed(self, timeout): + "timeout is in microseconds." + res = c_thread_acquirelock_timed(self._lock, timeout, 1) + res = rffi.cast(lltype.Signed, res) + return bool(res) + def release(self): # Sanity check: the lock must be locked if self.acquire(False): diff --git a/rpython/rlib/test/test_rthread.py b/rpython/rlib/test/test_rthread.py --- a/rpython/rlib/test/test_rthread.py +++ b/rpython/rlib/test/test_rthread.py @@ -153,6 +153,23 @@ answers = fn() assert answers == expected + def test_acquire_timed(self): + import time + def f(): + l = allocate_lock() + l.acquire(True) + t1 = time.time() + ok = l.acquire_timed(1000000) + t2 = time.time() + delay = t2 - t1 + if ok: + return delay + else: + return -delay + fn = self.getcompiled(f, []) + res = fn() + assert res < -1.0 + #class TestRunDirectly(AbstractThreadTests): # def getcompiled(self, f, argtypes): # return f diff --git a/rpython/translator/c/src/thread.h b/rpython/translator/c/src/thread.h --- a/rpython/translator/c/src/thread.h +++ b/rpython/translator/c/src/thread.h @@ -2,6 +2,14 @@ #define __PYPY_THREAD_H #include +#define RPY_TIMEOUT_T long long + +typedef enum RPyLockStatus { + RPY_LOCK_FAILURE = 0, + RPY_LOCK_ACQUIRED = 1, + RPY_LOCK_INTR +} RPyLockStatus; + #ifdef _WIN32 #include "thread_nt.h" #else diff --git a/rpython/translator/c/src/thread_nt.c b/rpython/translator/c/src/thread_nt.c --- a/rpython/translator/c/src/thread_nt.c +++ b/rpython/translator/c/src/thread_nt.c @@ -102,47 +102,24 @@ BOOL InitializeNonRecursiveMutex(PNRMUTEX mutex) { - mutex->owned = -1 ; /* No threads have entered NonRecursiveMutex */ - mutex->thread_id = 0 ; - mutex->hevent = CreateEvent(NULL, FALSE, FALSE, NULL) ; - return mutex->hevent != NULL ; /* TRUE if the mutex is created */ + mutex->sem = CreateSemaphore(NULL, 1, 1, NULL); } VOID DeleteNonRecursiveMutex(PNRMUTEX mutex) { - /* No in-use check */ - CloseHandle(mutex->hevent) ; - mutex->hevent = NULL ; /* Just in case */ + /* No in-use check */ + CloseHandle(mutex->sem); + mutex->sem = NULL ; /* Just in case */ } DWORD EnterNonRecursiveMutex(PNRMUTEX mutex, BOOL wait) { - /* Assume that the thread waits successfully */ - DWORD ret ; - - /* InterlockedIncrement(&mutex->owned) == 0 means that no thread currently owns the mutex */ - if (!wait) - { - if (InterlockedCompareExchange(&mutex->owned, 0, -1) != -1) - return WAIT_TIMEOUT ; - ret = WAIT_OBJECT_0 ; - } - else - ret = InterlockedIncrement(&mutex->owned) ? - /* Some thread owns the mutex, let's wait... */ - WaitForSingleObject(mutex->hevent, INFINITE) : WAIT_OBJECT_0 ; - - mutex->thread_id = GetCurrentThreadId() ; /* We own it */ - return ret ; + return WaitForSingleObject(mutex->sem, milliseconds); } BOOL LeaveNonRecursiveMutex(PNRMUTEX mutex) { - /* We don't own the mutex */ - mutex->thread_id = 0 ; - return - InterlockedDecrement(&mutex->owned) < 0 || - SetEvent(mutex->hevent) ; /* Other threads are waiting, wake one on them up */ + return ReleaseSemaphore(mutex->sem, 1, NULL); } /************************************************************/ @@ -158,8 +135,8 @@ void RPyOpaqueDealloc_ThreadLock(struct RPyOpaque_ThreadLock *lock) { - if (lock->hevent != NULL) - DeleteNonRecursiveMutex(lock); + if (lock->sem != NULL) + DeleteNonRecursiveMutex(lock); } /* @@ -168,9 +145,40 @@ * and 0 if the lock was not acquired. This means a 0 is returned * if the lock has already been acquired by this thread! */ +RPyLockStatus +RPyThreadAcquireLockTimed(struct RPyOpaque_ThreadLock *lock, + RPY_TIMEOUT_T microseconds, int intr_flag) +{ + /* Fow now, intr_flag does nothing on Windows, and lock acquires are + * uninterruptible. */ + PyLockStatus success; + PY_TIMEOUT_T milliseconds; + + if (microseconds >= 0) { + milliseconds = microseconds / 1000; + if (microseconds % 1000 > 0) + ++milliseconds; + if ((DWORD) milliseconds != milliseconds) + Py_FatalError("Timeout too large for a DWORD, " + "please check PY_TIMEOUT_MAX"); + } + else + milliseconds = INFINITE; + + if (lock && EnterNonRecursiveMutex( + lock, (DWORD)milliseconds) == WAIT_OBJECT_0) { + success = PY_LOCK_ACQUIRED; + } + else { + success = PY_LOCK_FAILURE; + } + + return success; +} + int RPyThreadAcquireLock(struct RPyOpaque_ThreadLock *lock, int waitflag) { - return EnterNonRecursiveMutex(lock, (waitflag != 0 ? INFINITE : 0)) == WAIT_OBJECT_0; + return RPyThreadAcquireLockTimed(lock, waitflag ? -1 : 0, /*intr_flag=*/0); } void RPyThreadReleaseLock(struct RPyOpaque_ThreadLock *lock) diff --git a/rpython/translator/c/src/thread_nt.h b/rpython/translator/c/src/thread_nt.h --- a/rpython/translator/c/src/thread_nt.h +++ b/rpython/translator/c/src/thread_nt.h @@ -5,16 +5,16 @@ */ typedef struct RPyOpaque_ThreadLock { - LONG owned ; - DWORD thread_id ; - HANDLE hevent ; -}; + HANDLE sem; +} NRMUTEX, *PNRMUTEX; /* prototypes */ long RPyThreadStart(void (*func)(void)); int RPyThreadLockInit(struct RPyOpaque_ThreadLock *lock); void RPyOpaqueDealloc_ThreadLock(struct RPyOpaque_ThreadLock *lock); int RPyThreadAcquireLock(struct RPyOpaque_ThreadLock *lock, int waitflag); +RPyLockStatus RPyThreadAcquireLockTimed(struct RPyOpaque_ThreadLock *lock, + RPY_TIMEOUT_T timeout, int intr_flag); void RPyThreadReleaseLock(struct RPyOpaque_ThreadLock *lock); long RPyThreadGetStackSize(void); long RPyThreadSetStackSize(long); diff --git a/rpython/translator/c/src/thread_pthread.c b/rpython/translator/c/src/thread_pthread.c --- a/rpython/translator/c/src/thread_pthread.c +++ b/rpython/translator/c/src/thread_pthread.c @@ -8,6 +8,7 @@ #include #include #include +#include /* The following is hopefully equivalent to what CPython does (which is trying to compile a snippet of code using it) */ @@ -170,6 +171,29 @@ #endif } +#ifdef GETTIMEOFDAY_NO_TZ +#define RPY_GETTIMEOFDAY(ptv) gettimeofday(ptv) +#else +#define RPY_GETTIMEOFDAY(ptv) gettimeofday(ptv, (struct timezone *)NULL) +#endif + +#define RPY_MICROSECONDS_TO_TIMESPEC(microseconds, ts) \ +do { \ + struct timeval tv; \ + RPY_GETTIMEOFDAY(&tv); \ + tv.tv_usec += microseconds % 1000000; \ + tv.tv_sec += microseconds / 1000000; \ + tv.tv_sec += tv.tv_usec / 1000000; \ + tv.tv_usec %= 1000000; \ + ts.tv_sec = tv.tv_sec; \ + ts.tv_nsec = tv.tv_usec * 1000; \ +} while(0) + +int RPyThreadAcquireLock(struct RPyOpaque_ThreadLock *lock, int waitflag) +{ + return RPyThreadAcquireLockTimed(lock, waitflag ? -1 : 0, /*intr_flag=*/0); +} + /************************************************************/ #ifdef USE_SEMAPHORES /************************************************************/ @@ -215,26 +239,50 @@ return (status == -1) ? errno : status; } -int RPyThreadAcquireLock(struct RPyOpaque_ThreadLock *lock, int waitflag) +RPyLockStatus +RPyThreadAcquireLockTimed(struct RPyOpaque_ThreadLock *lock, + RPY_TIMEOUT_T microseconds, int intr_flag) { - int success; + RPyLockStatus success; sem_t *thelock = &lock->sem; int status, error = 0; + struct timespec ts; + if (microseconds > 0) + RPY_MICROSECONDS_TO_TIMESPEC(microseconds, ts); do { - if (waitflag) - status = rpythread_fix_status(sem_wait(thelock)); - else - status = rpythread_fix_status(sem_trywait(thelock)); - } while (status == EINTR); /* Retry if interrupted by a signal */ + if (microseconds > 0) + status = rpythread_fix_status(sem_timedwait(thelock, &ts)); + else if (microseconds == 0) + status = rpythread_fix_status(sem_trywait(thelock)); + else + status = rpythread_fix_status(sem_wait(thelock)); + /* Retry if interrupted by a signal, unless the caller wants to be + notified. */ + } while (!intr_flag && status == EINTR); - if (waitflag) { + /* Don't check the status if we're stopping because of an interrupt. */ + if (!(intr_flag && status == EINTR)) { + if (microseconds > 0) { + if (status != ETIMEDOUT) + CHECK_STATUS("sem_timedwait"); + } + else if (microseconds == 0) { + if (status != EAGAIN) + CHECK_STATUS("sem_trywait"); + } + else { CHECK_STATUS("sem_wait"); - } else if (status != EAGAIN) { - CHECK_STATUS("sem_trywait"); + } } - - success = (status == 0) ? 1 : 0; + + if (status == 0) { + success = RPY_LOCK_ACQUIRED; + } else if (intr_flag && status == EINTR) { + success = RPY_LOCK_INTR; + } else { + success = RPY_LOCK_FAILURE; + } return success; } @@ -326,32 +374,63 @@ } } -int RPyThreadAcquireLock(struct RPyOpaque_ThreadLock *lock, int waitflag) +RPyLockStatus +RPyThreadAcquireLockTimed(struct RPyOpaque_ThreadLock *lock, + RPY_TIMEOUT_T microseconds, int intr_flag) { - int success; + RPyLockStatus success; int status, error = 0; status = pthread_mutex_lock( &lock->mut ); CHECK_STATUS("pthread_mutex_lock[1]"); - success = lock->locked == 0; - if ( !success && waitflag ) { + if (lock->locked == 0) { + success = RPY_LOCK_ACQUIRED; + } else if (microseconds == 0) { + success = RPY_LOCK_FAILURE; + } else { + struct timespec ts; + if (microseconds > 0) + RPY_MICROSECONDS_TO_TIMESPEC(microseconds, ts); /* continue trying until we get the lock */ /* mut must be locked by me -- part of the condition * protocol */ - while ( lock->locked ) { - status = pthread_cond_wait(&lock->lock_released, - &lock->mut); + success = RPY_LOCK_FAILURE; + while (success == RPY_LOCK_FAILURE) { + if (microseconds > 0) { + status = pthread_cond_timedwait( + &lock->lock_released, + &lock->mut, &ts); + if (status == ETIMEDOUT) + break; + CHECK_STATUS("pthread_cond_timed_wait"); + } + else { + status = pthread_cond_wait( + &lock->lock_released, + &lock->mut); CHECK_STATUS("pthread_cond_wait"); + } + + if (intr_flag && status == 0 && lock->locked) { + /* We were woken up, but didn't get the lock. We probably received + * a signal. Return RPY_LOCK_INTR to allow the caller to handle + * it and retry. */ + success = RPY_LOCK_INTR; + break; + } else if (status == 0 && !lock->locked) { + success = RPY_LOCK_ACQUIRED; + } else { + success = RPY_LOCK_FAILURE; + } } - success = 1; } - if (success) lock->locked = 1; + if (success == RPY_LOCK_ACQUIRED) lock->locked = 1; status = pthread_mutex_unlock( &lock->mut ); CHECK_STATUS("pthread_mutex_unlock[1]"); - if (error) success = 0; + if (error) success = RPY_LOCK_FAILURE; return success; } diff --git a/rpython/translator/c/src/thread_pthread.h b/rpython/translator/c/src/thread_pthread.h --- a/rpython/translator/c/src/thread_pthread.h +++ b/rpython/translator/c/src/thread_pthread.h @@ -64,6 +64,8 @@ int RPyThreadLockInit(struct RPyOpaque_ThreadLock *lock); void RPyOpaqueDealloc_ThreadLock(struct RPyOpaque_ThreadLock *lock); int RPyThreadAcquireLock(struct RPyOpaque_ThreadLock *lock, int waitflag); +RPyLockStatus RPyThreadAcquireLockTimed(struct RPyOpaque_ThreadLock *lock, + RPY_TIMEOUT_T timeout, int intr_flag); void RPyThreadReleaseLock(struct RPyOpaque_ThreadLock *lock); long RPyThreadGetStackSize(void); long RPyThreadSetStackSize(long); From noreply at buildbot.pypy.org Mon Jan 28 09:31:35 2013 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 28 Jan 2013 09:31:35 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: Another TODO. Message-ID: <20130128083135.DD7971C1306@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r60581:19113876fc13 Date: 2013-01-28 08:26 +0100 http://bitbucket.org/pypy/pypy/changeset/19113876fc13/ Log: Another TODO. diff --git a/TODO b/TODO --- a/TODO +++ b/TODO @@ -42,3 +42,7 @@ ------------------------------------------------------------ implement thread-locals in RPython (for the executioncontext) + +------------------------------------------------------------ + +optimize the static placement of the STM_XxxBARRIERs From noreply at buildbot.pypy.org Mon Jan 28 09:31:37 2013 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 28 Jan 2013 09:31:37 +0100 (CET) Subject: [pypy-commit] pypy default: Replace two pages of code with only one: in the multithreaded case, keep Message-ID: <20130128083137.39FF21C1306@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r60582:5a1110abb728 Date: 2013-01-28 08:47 +0100 http://bitbucket.org/pypy/pypy/changeset/5a1110abb728/ Log: Replace two pages of code with only one: in the multithreaded case, keep the pending signals in the C-level data structure inspected by pypysig_poll(). This avoids yet another list of pending signals here. diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py --- a/pypy/module/signal/interp_signal.py +++ b/pypy/module/signal/interp_signal.py @@ -19,7 +19,8 @@ class SignalActionFlag(AbstractActionFlag): # This class uses the C-level pypysig_counter variable as the tick # counter. The C-level signal handler will reset it to -1 whenever - # a signal is received. + # a signal is received. This causes CheckSignalAction.perform() to + # be called. def get_ticker(self): p = pypysig_getaddr_occurred() @@ -47,45 +48,40 @@ def __init__(self, space): AsyncAction.__init__(self, space) self.handlers_w = {} - if space.config.objspace.usemodules.thread: - # need a helper action in case signals arrive in a non-main thread - self.pending_signals = {} - self.reissue_signal_action = ReissueSignalAction(space) - else: - self.reissue_signal_action = None + self.emulated_sigint = False @jit.dont_look_inside def perform(self, executioncontext, frame): - while True: - n = pypysig_poll() - if n < 0: - break - self.perform_signal(executioncontext, n) - - @jit.dont_look_inside - def perform_signal(self, executioncontext, n): - if self.reissue_signal_action is None: - # no threads: we can report the signal immediately - self.report_signal(n) + if self.space.config.objspace.usemodules.thread: + main_ec = self.space.threadlocals.getmainthreadvalue() + in_main = executioncontext is main_ec else: - main_ec = self.space.threadlocals.getmainthreadvalue() - if executioncontext is main_ec: - # running in the main thread: we can report the - # signal immediately - self.report_signal(n) - else: - # running in another thread: we need to hack a bit - self.pending_signals[n] = None - self.reissue_signal_action.fire_after_thread_switch() + in_main = True + # If we are in the main thread, poll and report the signals now. + if in_main: + if self.emulated_sigint: + self.emulated_sigint = False + self._report_signal(cpy_signal.SIGINT) + while True: + n = pypysig_poll() + if n < 0: + break + self._report_signal(n) + else: + # Otherwise, don't call pypysig_poll() at all. Instead, + # arrange for perform() to be called again after a thread + # switch. It might be called again and again, until we + # land in the main thread. + self.fire_after_thread_switch() @jit.dont_look_inside def set_interrupt(self): "Simulates the effect of a SIGINT signal arriving" ec = self.space.getexecutioncontext() - self.perform_signal(ec, cpy_signal.SIGINT) + self.emulated_sigint = True + self.perform(ec, None) - @jit.dont_look_inside - def report_signal(self, n): + def _report_signal(self, n): try: w_handler = self.handlers_w[n] except KeyError: @@ -100,39 +96,6 @@ w_frame = space.wrap(ec.gettopframe_nohidden()) space.call_function(w_handler, space.wrap(n), w_frame) - @jit.dont_look_inside - def report_pending_signals(self): - # XXX this logic isn't so complicated but I have no clue how - # to test it :-( - pending_signals = self.pending_signals.keys() - self.pending_signals.clear() - try: - while pending_signals: - self.report_signal(pending_signals.pop()) - finally: - # in case of exception, put the undelivered signals back - # into the dict instead of silently swallowing them - if pending_signals: - for n in pending_signals: - self.pending_signals[n] = None - self.reissue_signal_action.fire() - - -class ReissueSignalAction(AsyncAction): - """A special action to help deliver signals to the main thread. If - a non-main thread caught a signal, this action fires after every - thread switch until we land in the main thread. - """ - - def perform(self, executioncontext, frame): - main_ec = self.space.threadlocals.getmainthreadvalue() - if executioncontext is main_ec: - # now running in the main thread: we can really report the signals - self.space.check_signal_action.report_pending_signals() - else: - # still running in some other thread: try again later - self.fire_after_thread_switch() - @unwrap_spec(signum=int) def getsignal(space, signum): From noreply at buildbot.pypy.org Mon Jan 28 09:31:38 2013 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 28 Jan 2013 09:31:38 +0100 (CET) Subject: [pypy-commit] pypy default: Oups! Indentation mistake probably. Can't set a field on an object that is None... Message-ID: <20130128083138.6B43A1C1306@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r60583:abdf791cac36 Date: 2013-01-28 09:31 +0100 http://bitbucket.org/pypy/pypy/changeset/abdf791cac36/ Log: Oups! Indentation mistake probably. Can't set a field on an object that is None... diff --git a/pypy/module/thread/os_local.py b/pypy/module/thread/os_local.py --- a/pypy/module/thread/os_local.py +++ b/pypy/module/thread/os_local.py @@ -98,5 +98,5 @@ local = wref() if local is not None: del local.dicts[ec] - local.last_dict = None - local.last_ec = None + local.last_dict = None + local.last_ec = None From noreply at buildbot.pypy.org Mon Jan 28 09:31:53 2013 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 28 Jan 2013 09:31:53 +0100 (CET) Subject: [pypy-commit] pypy default: merge heads Message-ID: <20130128083153.AEDFD1C1306@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r60584:c85129d90a3c Date: 2013-01-28 09:31 +0100 http://bitbucket.org/pypy/pypy/changeset/c85129d90a3c/ Log: merge heads diff too long, truncating to 2000 out of 141039 lines diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -34,6 +34,9 @@ .. branch: kill-faking .. branch: improved_ebnfparse_error .. branch: task-decorator +.. branch: fix-e4fa0b2 +.. branch: win32-fixes +.. branch: fix-version-tool .. branch: release-2.0-beta1 diff --git a/pypy/interpreter/test2/test_app_main.py b/pypy/interpreter/test2/test_app_main.py --- a/pypy/interpreter/test2/test_app_main.py +++ b/pypy/interpreter/test2/test_app_main.py @@ -774,7 +774,7 @@ assert data == p + os.sep + '\n' def test_getfilesystemencoding(self): - py.test.skip("this has been failing since forever, but it's not tested nightly because buildbot uses python2.6 :-(") + py.test.skip("encoding is only set if stdout.isatty(), test is flawed") if sys.version_info < (2, 7): skip("test requires Python >= 2.7") p = getscript_in_dir(""" @@ -881,7 +881,6 @@ def test_setup_bootstrap_path(self): import sys - import os old_sys_path = sys.path[:] sys.path.append(self.goal_dir) try: diff --git a/pypy/interpreter/test2/test_targetpypy.py b/pypy/interpreter/test2/test_targetpypy.py --- a/pypy/interpreter/test2/test_targetpypy.py +++ b/pypy/interpreter/test2/test_targetpypy.py @@ -1,5 +1,3 @@ - -import py from pypy.goal.targetpypystandalone import get_entry_point from pypy.config.pypyoption import get_pypy_config @@ -7,6 +5,4 @@ def test_run(self): config = get_pypy_config(translating=False) entry_point = get_entry_point(config)[0] - space = self.space - py.test.skip("not working so far") entry_point(['pypy-c' , '-S', '-c', 'print 3']) diff --git a/pypy/module/_ffi/test/test_funcptr.py b/pypy/module/_ffi/test/test_funcptr.py --- a/pypy/module/_ffi/test/test_funcptr.py +++ b/pypy/module/_ffi/test/test_funcptr.py @@ -96,7 +96,7 @@ libm = CDLL(self.libm_name) pow_addr = libm.getaddressindll('pow') fff = sys.maxint*2-1 - if sys.platform == 'win32': + if sys.platform == 'win32' or sys.platform == 'darwin': fff = sys.maxint*2+1 assert pow_addr == self.pow_addr & fff diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -2,7 +2,7 @@ import os class AppTestFileIO: - spaceconfig = dict(usemodules=['_io']) + spaceconfig = dict(usemodules=['_io'] + (['fcntl'] if os.name != 'nt' else [])) def setup_class(cls): tmpfile = udir.join('tmpfile') diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -49,6 +49,8 @@ kwds["libraries"] = [api_library] # '%s' undefined; assuming extern returning int kwds["compile_extra"] = ["/we4013"] + elif sys.platform == 'darwin': + kwds["link_files"] = [str(api_library + '.dylib')] else: kwds["link_files"] = [str(api_library + '.so')] if sys.platform.startswith('linux'): diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py --- a/pypy/module/rctime/test/test_rctime.py +++ b/pypy/module/rctime/test/test_rctime.py @@ -213,7 +213,7 @@ def test_strftime(self): import time as rctime - import os + import os, sys t = rctime.time() tt = rctime.gmtime(t) @@ -234,6 +234,10 @@ # input to [w]strftime is not kosher. if os.name == 'nt': raises(ValueError, rctime.strftime, '%f') + elif sys.platform == 'darwin': + # darwin strips % of unknown format codes + # http://bugs.python.org/issue9811 + assert rctime.strftime('%f') == 'f' else: assert rctime.strftime('%f') == '%f' diff --git a/pypy/module/signal/test/test_signal.py b/pypy/module/signal/test/test_signal.py --- a/pypy/module/signal/test/test_signal.py +++ b/pypy/module/signal/test/test_signal.py @@ -157,6 +157,8 @@ if sys.platform == 'win32': raises(ValueError, signal, 42, lambda *args: None) raises(ValueError, signal, 7, lambda *args: None) + elif sys.platform == 'darwin': + raises(ValueError, signal, 42, lambda *args: None) else: signal(42, lambda *args: None) signal(42, SIG_DFL) diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -28,6 +28,7 @@ import pypy pypydir = os.path.dirname(os.path.abspath(pypy.__file__)) +pypyroot = os.path.dirname(pypydir) del pypy from rpython.tool.version import get_repo_version_info @@ -68,7 +69,7 @@ CPYTHON_VERSION[0], CPYTHON_VERSION[1], CPYTHON_VERSION[2], - get_repo_version_info()[2], + get_repo_version_info(root=pypyroot)[1], date, time, ver, @@ -91,10 +92,10 @@ return space.wrap(('PyPy', '', '')) def get_repo_info(space): - info = get_repo_version_info() + info = get_repo_version_info(root=pypyroot) if info: - project, repo_tag, repo_version = info - return space.newtuple([space.wrap(project), + repo_tag, repo_version = info + return space.newtuple([space.wrap('PyPy'), space.wrap(repo_tag), space.wrap(repo_version)]) else: diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py --- a/rpython/rlib/_rsocket_rffi.py +++ b/rpython/rlib/_rsocket_rffi.py @@ -344,7 +344,7 @@ ]) CConfig.WSAPROTOCOL_INFO = platform.Struct( - 'struct WSAPROTOCOL_INFO', + 'WSAPROTOCOL_INFO', []) # Struct is just passed between functions CConfig.FROM_PROTOCOL_INFO = platform.DefinedConstantInteger( 'FROM_PROTOCOL_INFO') @@ -617,7 +617,7 @@ WSASocket = external('WSASocket', [rffi.INT, rffi.INT, rffi.INT, lltype.Ptr(WSAPROTOCOL_INFO), - rffi.DWORD, rffi.DWORD], + rwin32.DWORD, rwin32.DWORD], socketfd_type) if WIN32: diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py --- a/rpython/rlib/ropenssl.py +++ b/rpython/rlib/ropenssl.py @@ -2,6 +2,7 @@ from rpython.rtyper.tool import rffi_platform from rpython.translator.platform import platform from rpython.translator.tool.cbuild import ExternalCompilationInfo +from rpython.rlib.unroll import unrolling_iterable import sys, os @@ -92,8 +93,12 @@ OPENSSL_NO_SSL2 = rffi_platform.Defined("OPENSSL_NO_SSL2") SSL_FILETYPE_PEM = rffi_platform.ConstantInteger("SSL_FILETYPE_PEM") SSL_OP_ALL = rffi_platform.ConstantInteger("SSL_OP_ALL") + SSL_OP_NO_SSLv2 = rffi_platform.ConstantInteger("SSL_OP_NO_SSLv2") + SSL_OP_NO_SSLv3 = rffi_platform.ConstantInteger("SSL_OP_NO_SSLv3") + SSL_OP_NO_TLSv1 = rffi_platform.ConstantInteger("SSL_OP_NO_TLSv1") SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS = rffi_platform.ConstantInteger( "SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS") + HAS_SNI = rffi_platform.Defined("SSL_CTRL_SET_TLSEXT_HOSTNAME") SSL_VERIFY_NONE = rffi_platform.ConstantInteger("SSL_VERIFY_NONE") SSL_VERIFY_PEER = rffi_platform.ConstantInteger("SSL_VERIFY_PEER") SSL_VERIFY_FAIL_IF_NO_PEER_CERT = rffi_platform.ConstantInteger("SSL_VERIFY_FAIL_IF_NO_PEER_CERT") @@ -118,6 +123,9 @@ CRYPTO_LOCK = rffi_platform.ConstantInteger("CRYPTO_LOCK") + OBJ_NAME_TYPE_MD_METH = rffi_platform.ConstantInteger( + "OBJ_NAME_TYPE_MD_METH") + # Some structures, with only the fields used in the _ssl module X509_name_entry_st = rffi_platform.Struct('struct X509_name_entry_st', [('set', rffi.INT)]) @@ -146,6 +154,12 @@ OPENSSL_EXPORT_VAR_AS_FUNCTION = rffi_platform.Defined( "OPENSSL_EXPORT_VAR_AS_FUNCTION") + OBJ_NAME_st = rffi_platform.Struct( + 'OBJ_NAME', + [('alias', rffi.INT), + ('name', rffi.CCHARP), + ]) + for k, v in rffi_platform.configure(CConfig).items(): globals()[k] = v @@ -166,8 +180,10 @@ ASN1_INTEGER = rffi.COpaquePtr('ASN1_INTEGER') GENERAL_NAMES = rffi.COpaquePtr('GENERAL_NAMES') GENERAL_NAME = rffi.CArrayPtr(GENERAL_NAME_st) +OBJ_NAME = rffi.CArrayPtr(OBJ_NAME_st) HAVE_OPENSSL_RAND = OPENSSL_VERSION_NUMBER >= 0x0090500f +HAVE_SSL_CTX_CLEAR_OPTIONS = OPENSSL_VERSION_NUMBER >= 0x009080df def external(name, argtypes, restype, **kw): kw['compilation_info'] = eci @@ -203,12 +219,26 @@ ssl_external('SSLv23_method', [], SSL_METHOD) ssl_external('SSL_CTX_use_PrivateKey_file', [SSL_CTX, rffi.CCHARP, rffi.INT], rffi.INT) ssl_external('SSL_CTX_use_certificate_chain_file', [SSL_CTX, rffi.CCHARP], rffi.INT) +ssl_external('SSL_CTX_get_options', [SSL_CTX], rffi.INT, macro=True) ssl_external('SSL_CTX_set_options', [SSL_CTX, rffi.INT], rffi.INT, macro=True) +if HAVE_SSL_CTX_CLEAR_OPTIONS: + ssl_external('SSL_CTX_clear_options', [SSL_CTX, rffi.INT], rffi.INT, + macro=True) ssl_external('SSL_CTX_ctrl', [SSL_CTX, rffi.INT, rffi.INT, rffi.VOIDP], rffi.INT) ssl_external('SSL_CTX_set_verify', [SSL_CTX, rffi.INT, rffi.VOIDP], lltype.Void) ssl_external('SSL_CTX_get_verify_mode', [SSL_CTX], rffi.INT) +ssl_external('SSL_CTX_set_default_verify_paths', [SSL_CTX], rffi.INT) ssl_external('SSL_CTX_set_cipher_list', [SSL_CTX, rffi.CCHARP], rffi.INT) ssl_external('SSL_CTX_load_verify_locations', [SSL_CTX, rffi.CCHARP, rffi.CCHARP], rffi.INT) +ssl_external('SSL_CTX_check_private_key', [SSL_CTX], rffi.INT) +ssl_external('SSL_CTX_set_session_id_context', [SSL_CTX, rffi.CCHARP, rffi.UINT], rffi.INT) +SSL_CTX_STATS_NAMES = """ + number connect connect_good connect_renegotiate accept accept_good + accept_renegotiate hits misses timeouts cache_full""".split() +SSL_CTX_STATS = unrolling_iterable( + (name, external('SSL_CTX_sess_' + name, [SSL_CTX], rffi.LONG, macro=True)) + for name in SSL_CTX_STATS_NAMES) + ssl_external('SSL_new', [SSL_CTX], SSL) ssl_external('SSL_set_fd', [SSL, rffi.INT], rffi.INT) ssl_external('SSL_set_mode', [SSL, rffi.INT], rffi.INT, macro=True) @@ -224,6 +254,7 @@ ssl_external('SSL_get_error', [SSL, rffi.INT], rffi.INT) ssl_external('SSL_get_shutdown', [SSL], rffi.INT) ssl_external('SSL_set_read_ahead', [SSL, rffi.INT], lltype.Void) +ssl_external('SSL_set_tlsext_host_name', [SSL, rffi.CCHARP], rffi.INT, macro=True) ssl_external('SSL_get_peer_certificate', [SSL], X509) ssl_external('X509_get_subject_name', [X509], X509_NAME) @@ -234,7 +265,7 @@ ssl_external('X509_NAME_ENTRY_get_object', [X509_NAME_ENTRY], ASN1_OBJECT) ssl_external('X509_NAME_ENTRY_get_data', [X509_NAME_ENTRY], ASN1_STRING) ssl_external('i2d_X509', [X509, rffi.CCHARPP], rffi.INT) -ssl_external('X509_free', [X509], lltype.Void) +ssl_external('X509_free', [X509], lltype.Void, threadsafe=False) ssl_external('X509_get_notBefore', [X509], ASN1_TIME, macro=True) ssl_external('X509_get_notAfter', [X509], ASN1_TIME, macro=True) ssl_external('X509_get_serialNumber', [X509], ASN1_INTEGER) @@ -272,9 +303,12 @@ ssl_external('ERR_get_error', [], rffi.INT) ssl_external('ERR_peek_last_error', [], rffi.INT) ssl_external('ERR_error_string', [rffi.ULONG, rffi.CCHARP], rffi.CCHARP) +ssl_external('ERR_clear_error', [], lltype.Void) -ssl_external('SSL_free', [SSL], lltype.Void) -ssl_external('SSL_CTX_free', [SSL_CTX], lltype.Void) +# 'threadsafe=False' here indicates that this function will be called +# with the GIL held, and so is allowed to run in a RPython __del__ method. +ssl_external('SSL_free', [SSL], lltype.Void, threadsafe=False) +ssl_external('SSL_CTX_free', [SSL_CTX], lltype.Void, threadsafe=False) ssl_external('CRYPTO_free', [rffi.VOIDP], lltype.Void) libssl_OPENSSL_free = libssl_CRYPTO_free @@ -316,6 +350,11 @@ EVP_MD_CTX_cleanup = external( 'EVP_MD_CTX_cleanup', [EVP_MD_CTX], rffi.INT, threadsafe=False) +OBJ_NAME_CALLBACK = lltype.Ptr(lltype.FuncType( + [OBJ_NAME, rffi.VOIDP], lltype.Void)) +OBJ_NAME_do_all = external( + 'OBJ_NAME_do_all', [rffi.INT, OBJ_NAME_CALLBACK, rffi.VOIDP], lltype.Void) + # HASH_MALLOC_SIZE is the size of EVP_MD, EVP_MD_CTX plus their points # Used for adding memory pressure. Last number is an (under?)estimate of # EVP_PKEY_CTX's size. diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py --- a/rpython/rlib/rthread.py +++ b/rpython/rlib/rthread.py @@ -19,9 +19,9 @@ separate_module_files = [translator_c_dir / 'src' / 'thread.c'], include_dirs = [translator_c_dir], export_symbols = ['RPyThreadGetIdent', 'RPyThreadLockInit', - 'RPyThreadAcquireLock', 'RPyThreadReleaseLock', - 'RPyGilAllocate', 'RPyGilYieldThread', - 'RPyGilRelease', 'RPyGilAcquire', + 'RPyThreadAcquireLock', 'RPyThreadAcquireLockTimed', + 'RPyThreadReleaseLock', 'RPyGilAllocate', + 'RPyGilYieldThread', 'RPyGilRelease', 'RPyGilAcquire', 'RPyThreadGetStackSize', 'RPyThreadSetStackSize', 'RPyOpaqueDealloc_ThreadLock', 'RPyThreadAfterFork'] @@ -61,6 +61,10 @@ c_thread_acquirelock = llexternal('RPyThreadAcquireLock', [TLOCKP, rffi.INT], rffi.INT, threadsafe=True) # release the GIL +c_thread_acquirelock_timed = llexternal('RPyThreadAcquireLockTimed', + [TLOCKP, rffi.LONGLONG, rffi.INT], + rffi.INT, + threadsafe=True) # release the GIL c_thread_releaselock = llexternal('RPyThreadReleaseLock', [TLOCKP], lltype.Void, threadsafe=True) # release the GIL @@ -121,6 +125,12 @@ res = rffi.cast(lltype.Signed, res) return bool(res) + def acquire_timed(self, timeout): + "timeout is in microseconds." + res = c_thread_acquirelock_timed(self._lock, timeout, 1) + res = rffi.cast(lltype.Signed, res) + return bool(res) + def release(self): # Sanity check: the lock must be locked if self.acquire(False): diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py --- a/rpython/rlib/runicode.py +++ b/rpython/rlib/runicode.py @@ -31,7 +31,8 @@ ch2 = ord(u[1]) if 0xD800 <= ch1 <= 0xDBFF and 0xDC00 <= ch2 <= 0xDFFF: return (((ch1 - 0xD800) << 10) | (ch2 - 0xDC00)) + 0x10000 - return ord(u) + assert len(u) == 1 + return ord(u[0]) if MAXUNICODE > sys.maxunicode: # A version of unichr which allows codes outside the BMP diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py --- a/rpython/rlib/test/test_rposix.py +++ b/rpython/rlib/test/test_rposix.py @@ -117,7 +117,7 @@ curdir = os.getcwd() try: interpret(f, []) - assert os.getcwdu() == self.ufilename + assert os.getcwdu() == os.path.realpath(self.ufilename) finally: os.chdir(curdir) diff --git a/rpython/rlib/test/test_rthread.py b/rpython/rlib/test/test_rthread.py --- a/rpython/rlib/test/test_rthread.py +++ b/rpython/rlib/test/test_rthread.py @@ -153,6 +153,23 @@ answers = fn() assert answers == expected + def test_acquire_timed(self): + import time + def f(): + l = allocate_lock() + l.acquire(True) + t1 = time.time() + ok = l.acquire_timed(1000000) + t2 = time.time() + delay = t2 - t1 + if ok: + return delay + else: + return -delay + fn = self.getcompiled(f, []) + res = fn() + assert res < -1.0 + #class TestRunDirectly(AbstractThreadTests): # def getcompiled(self, f, argtypes): # return f diff --git a/rpython/rlib/unicodedata/DerivedCoreProperties-3.2.0.txt b/rpython/rlib/unicodedata/DerivedCoreProperties-3.2.0.txt new file mode 100644 --- /dev/null +++ b/rpython/rlib/unicodedata/DerivedCoreProperties-3.2.0.txt @@ -0,0 +1,4218 @@ +# DerivedCoreProperties-3.2.0.txt +# Date: 2002-03-19,23:30:42 GMT [MD] +# +# Unicode Character Database: Derived Property Data +# Generated algorithmically from the Unicode Character Database +# For documentation, see DerivedProperties.html +# Note: Unassigned and Noncharacter codepoints are omitted, +# except when listing Noncharacter or Cn. +# ================================================ + +# ================================================ + +# Derived Property: Math +# Generated from: Sm + Other_Math + +0028 ; Math # Ps LEFT PARENTHESIS +0029 ; Math # Pe RIGHT PARENTHESIS +002A ; Math # Po ASTERISK +002B ; Math # Sm PLUS SIGN +002D ; Math # Pd HYPHEN-MINUS +002F ; Math # Po SOLIDUS +003C..003E ; Math # Sm [3] LESS-THAN SIGN..GREATER-THAN SIGN +005B ; Math # Ps LEFT SQUARE BRACKET +005C ; Math # Po REVERSE SOLIDUS +005D ; Math # Pe RIGHT SQUARE BRACKET +005E ; Math # Sk CIRCUMFLEX ACCENT +007B ; Math # Ps LEFT CURLY BRACKET +007C ; Math # Sm VERTICAL LINE +007D ; Math # Pe RIGHT CURLY BRACKET +007E ; Math # Sm TILDE +00AC ; Math # Sm NOT SIGN +00B1 ; Math # Sm PLUS-MINUS SIGN +00D7 ; Math # Sm MULTIPLICATION SIGN +00F7 ; Math # Sm DIVISION SIGN +03F6 ; Math # Sm GREEK REVERSED LUNATE EPSILON SYMBOL +2016 ; Math # Po DOUBLE VERTICAL LINE +2032..2034 ; Math # Po [3] PRIME..TRIPLE PRIME +2044 ; Math # Sm FRACTION SLASH +2052 ; Math # Sm COMMERCIAL MINUS SIGN +207A..207C ; Math # Sm [3] SUPERSCRIPT PLUS SIGN..SUPERSCRIPT EQUALS SIGN +207D ; Math # Ps SUPERSCRIPT LEFT PARENTHESIS +207E ; Math # Pe SUPERSCRIPT RIGHT PARENTHESIS +208A..208C ; Math # Sm [3] SUBSCRIPT PLUS SIGN..SUBSCRIPT EQUALS SIGN +208D ; Math # Ps SUBSCRIPT LEFT PARENTHESIS +208E ; Math # Pe SUBSCRIPT RIGHT PARENTHESIS +20D0..20DC ; Math # Mn [13] COMBINING LEFT HARPOON ABOVE..COMBINING FOUR DOTS ABOVE +20E1 ; Math # Mn COMBINING LEFT RIGHT ARROW ABOVE +2102 ; Math # L& DOUBLE-STRUCK CAPITAL C +210A..2113 ; Math # L& [10] SCRIPT SMALL G..SCRIPT SMALL L +2115 ; Math # L& DOUBLE-STRUCK CAPITAL N +2119..211D ; Math # L& [5] DOUBLE-STRUCK CAPITAL P..DOUBLE-STRUCK CAPITAL R +2124 ; Math # L& DOUBLE-STRUCK CAPITAL Z +2128 ; Math # L& BLACK-LETTER CAPITAL Z +212C..212D ; Math # L& [2] SCRIPT CAPITAL B..BLACK-LETTER CAPITAL C +212F..2131 ; Math # L& [3] SCRIPT SMALL E..SCRIPT CAPITAL F +2133..2134 ; Math # L& [2] SCRIPT CAPITAL M..SCRIPT SMALL O +2135..2138 ; Math # Lo [4] ALEF SYMBOL..DALET SYMBOL +2140..2144 ; Math # Sm [5] DOUBLE-STRUCK N-ARY SUMMATION..TURNED SANS-SERIF CAPITAL Y +214B ; Math # Sm TURNED AMPERSAND +2190..2194 ; Math # Sm [5] LEFTWARDS ARROW..LEFT RIGHT ARROW +219A..219B ; Math # Sm [2] LEFTWARDS ARROW WITH STROKE..RIGHTWARDS ARROW WITH STROKE +21A0 ; Math # Sm RIGHTWARDS TWO HEADED ARROW +21A3 ; Math # Sm RIGHTWARDS ARROW WITH TAIL +21A6 ; Math # Sm RIGHTWARDS ARROW FROM BAR +21AE ; Math # Sm LEFT RIGHT ARROW WITH STROKE +21CE..21CF ; Math # Sm [2] LEFT RIGHT DOUBLE ARROW WITH STROKE..RIGHTWARDS DOUBLE ARROW WITH STROKE +21D2 ; Math # Sm RIGHTWARDS DOUBLE ARROW +21D4 ; Math # Sm LEFT RIGHT DOUBLE ARROW +21F4..22FF ; Math # Sm [268] RIGHT ARROW WITH SMALL CIRCLE..Z NOTATION BAG MEMBERSHIP +2308..230B ; Math # Sm [4] LEFT CEILING..RIGHT FLOOR +2320..2321 ; Math # Sm [2] TOP HALF INTEGRAL..BOTTOM HALF INTEGRAL +237C ; Math # Sm RIGHT ANGLE WITH DOWNWARDS ZIGZAG ARROW +239B..23B3 ; Math # Sm [25] LEFT PARENTHESIS UPPER HOOK..SUMMATION BOTTOM +25B7 ; Math # Sm WHITE RIGHT-POINTING TRIANGLE +25C1 ; Math # Sm WHITE LEFT-POINTING TRIANGLE +25F8..25FF ; Math # Sm [8] UPPER LEFT TRIANGLE..LOWER RIGHT TRIANGLE +266F ; Math # Sm MUSIC SHARP SIGN +27D0..27E5 ; Math # Sm [22] WHITE DIAMOND WITH CENTRED DOT..WHITE SQUARE WITH RIGHTWARDS TICK +27F0..27FF ; Math # Sm [16] UPWARDS QUADRUPLE ARROW..LONG RIGHTWARDS SQUIGGLE ARROW +2900..2982 ; Math # Sm [131] RIGHTWARDS TWO-HEADED ARROW WITH VERTICAL STROKE..Z NOTATION TYPE COLON +2999..29D7 ; Math # Sm [63] DOTTED FENCE..BLACK HOURGLASS +29DC..29FB ; Math # Sm [32] INCOMPLETE INFINITY..TRIPLE PLUS +29FE..2AFF ; Math # Sm [258] TINY..N-ARY WHITE VERTICAL BAR +FB29 ; Math # Sm HEBREW LETTER ALTERNATIVE PLUS SIGN +FE35 ; Math # Ps PRESENTATION FORM FOR VERTICAL LEFT PARENTHESIS +FE36 ; Math # Pe PRESENTATION FORM FOR VERTICAL RIGHT PARENTHESIS +FE37 ; Math # Ps PRESENTATION FORM FOR VERTICAL LEFT CURLY BRACKET +FE38 ; Math # Pe PRESENTATION FORM FOR VERTICAL RIGHT CURLY BRACKET +FE59 ; Math # Ps SMALL LEFT PARENTHESIS +FE5A ; Math # Pe SMALL RIGHT PARENTHESIS +FE5B ; Math # Ps SMALL LEFT CURLY BRACKET +FE5C ; Math # Pe SMALL RIGHT CURLY BRACKET +FE61 ; Math # Po SMALL ASTERISK +FE62 ; Math # Sm SMALL PLUS SIGN +FE63 ; Math # Pd SMALL HYPHEN-MINUS +FE64..FE66 ; Math # Sm [3] SMALL LESS-THAN SIGN..SMALL EQUALS SIGN +FE68 ; Math # Po SMALL REVERSE SOLIDUS +FF08 ; Math # Ps FULLWIDTH LEFT PARENTHESIS +FF09 ; Math # Pe FULLWIDTH RIGHT PARENTHESIS +FF0A ; Math # Po FULLWIDTH ASTERISK +FF0B ; Math # Sm FULLWIDTH PLUS SIGN +FF0D ; Math # Pd FULLWIDTH HYPHEN-MINUS +FF0F ; Math # Po FULLWIDTH SOLIDUS +FF1C..FF1E ; Math # Sm [3] FULLWIDTH LESS-THAN SIGN..FULLWIDTH GREATER-THAN SIGN +FF3B ; Math # Ps FULLWIDTH LEFT SQUARE BRACKET +FF3C ; Math # Po FULLWIDTH REVERSE SOLIDUS +FF3D ; Math # Pe FULLWIDTH RIGHT SQUARE BRACKET +FF3E ; Math # Sk FULLWIDTH CIRCUMFLEX ACCENT +FF5B ; Math # Ps FULLWIDTH LEFT CURLY BRACKET +FF5C ; Math # Sm FULLWIDTH VERTICAL LINE +FF5D ; Math # Pe FULLWIDTH RIGHT CURLY BRACKET +FF5E ; Math # Sm FULLWIDTH TILDE +FFE2 ; Math # Sm FULLWIDTH NOT SIGN +FFE9..FFEC ; Math # Sm [4] HALFWIDTH LEFTWARDS ARROW..HALFWIDTH DOWNWARDS ARROW +1D400..1D454 ; Math # L& [85] MATHEMATICAL BOLD CAPITAL A..MATHEMATICAL ITALIC SMALL G +1D456..1D49C ; Math # L& [71] MATHEMATICAL ITALIC SMALL I..MATHEMATICAL SCRIPT CAPITAL A +1D49E..1D49F ; Math # L& [2] MATHEMATICAL SCRIPT CAPITAL C..MATHEMATICAL SCRIPT CAPITAL D +1D4A2 ; Math # L& MATHEMATICAL SCRIPT CAPITAL G +1D4A5..1D4A6 ; Math # L& [2] MATHEMATICAL SCRIPT CAPITAL J..MATHEMATICAL SCRIPT CAPITAL K +1D4A9..1D4AC ; Math # L& [4] MATHEMATICAL SCRIPT CAPITAL N..MATHEMATICAL SCRIPT CAPITAL Q +1D4AE..1D4B9 ; Math # L& [12] MATHEMATICAL SCRIPT CAPITAL S..MATHEMATICAL SCRIPT SMALL D +1D4BB ; Math # L& MATHEMATICAL SCRIPT SMALL F +1D4BD..1D4C0 ; Math # L& [4] MATHEMATICAL SCRIPT SMALL H..MATHEMATICAL SCRIPT SMALL K +1D4C2..1D4C3 ; Math # L& [2] MATHEMATICAL SCRIPT SMALL M..MATHEMATICAL SCRIPT SMALL N +1D4C5..1D505 ; Math # L& [65] MATHEMATICAL SCRIPT SMALL P..MATHEMATICAL FRAKTUR CAPITAL B +1D507..1D50A ; Math # L& [4] MATHEMATICAL FRAKTUR CAPITAL D..MATHEMATICAL FRAKTUR CAPITAL G +1D50D..1D514 ; Math # L& [8] MATHEMATICAL FRAKTUR CAPITAL J..MATHEMATICAL FRAKTUR CAPITAL Q +1D516..1D51C ; Math # L& [7] MATHEMATICAL FRAKTUR CAPITAL S..MATHEMATICAL FRAKTUR CAPITAL Y +1D51E..1D539 ; Math # L& [28] MATHEMATICAL FRAKTUR SMALL A..MATHEMATICAL DOUBLE-STRUCK CAPITAL B +1D53B..1D53E ; Math # L& [4] MATHEMATICAL DOUBLE-STRUCK CAPITAL D..MATHEMATICAL DOUBLE-STRUCK CAPITAL G +1D540..1D544 ; Math # L& [5] MATHEMATICAL DOUBLE-STRUCK CAPITAL I..MATHEMATICAL DOUBLE-STRUCK CAPITAL M +1D546 ; Math # L& MATHEMATICAL DOUBLE-STRUCK CAPITAL O +1D54A..1D550 ; Math # L& [7] MATHEMATICAL DOUBLE-STRUCK CAPITAL S..MATHEMATICAL DOUBLE-STRUCK CAPITAL Y +1D552..1D6A3 ; Math # L& [338] MATHEMATICAL DOUBLE-STRUCK SMALL A..MATHEMATICAL MONOSPACE SMALL Z +1D6A8..1D6C0 ; Math # L& [25] MATHEMATICAL BOLD CAPITAL ALPHA..MATHEMATICAL BOLD CAPITAL OMEGA +1D6C1 ; Math # Sm MATHEMATICAL BOLD NABLA +1D6C2..1D6DA ; Math # L& [25] MATHEMATICAL BOLD SMALL ALPHA..MATHEMATICAL BOLD SMALL OMEGA +1D6DB ; Math # Sm MATHEMATICAL BOLD PARTIAL DIFFERENTIAL +1D6DC..1D6FA ; Math # L& [31] MATHEMATICAL BOLD EPSILON SYMBOL..MATHEMATICAL ITALIC CAPITAL OMEGA +1D6FB ; Math # Sm MATHEMATICAL ITALIC NABLA +1D6FC..1D714 ; Math # L& [25] MATHEMATICAL ITALIC SMALL ALPHA..MATHEMATICAL ITALIC SMALL OMEGA +1D715 ; Math # Sm MATHEMATICAL ITALIC PARTIAL DIFFERENTIAL +1D716..1D734 ; Math # L& [31] MATHEMATICAL ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD ITALIC CAPITAL OMEGA +1D735 ; Math # Sm MATHEMATICAL BOLD ITALIC NABLA +1D736..1D74E ; Math # L& [25] MATHEMATICAL BOLD ITALIC SMALL ALPHA..MATHEMATICAL BOLD ITALIC SMALL OMEGA +1D74F ; Math # Sm MATHEMATICAL BOLD ITALIC PARTIAL DIFFERENTIAL +1D750..1D76E ; Math # L& [31] MATHEMATICAL BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD CAPITAL OMEGA +1D76F ; Math # Sm MATHEMATICAL SANS-SERIF BOLD NABLA +1D770..1D788 ; Math # L& [25] MATHEMATICAL SANS-SERIF BOLD SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD SMALL OMEGA +1D789 ; Math # Sm MATHEMATICAL SANS-SERIF BOLD PARTIAL DIFFERENTIAL +1D78A..1D7A8 ; Math # L& [31] MATHEMATICAL SANS-SERIF BOLD EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL OMEGA +1D7A9 ; Math # Sm MATHEMATICAL SANS-SERIF BOLD ITALIC NABLA +1D7AA..1D7C2 ; Math # L& [25] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL OMEGA +1D7C3 ; Math # Sm MATHEMATICAL SANS-SERIF BOLD ITALIC PARTIAL DIFFERENTIAL +1D7C4..1D7C9 ; Math # L& [6] MATHEMATICAL SANS-SERIF BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC PI SYMBOL +1D7CE..1D7FF ; Math # Nd [50] MATHEMATICAL BOLD DIGIT ZERO..MATHEMATICAL MONOSPACE DIGIT NINE + +# Total code points: 1965 + +# ================================================ + +# Derived Property: Alphabetic +# Generated from: Lu+Ll+Lt+Lm+Lo+Nl + Other_Alphabetic + +0041..005A ; Alphabetic # L& [26] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER Z +0061..007A ; Alphabetic # L& [26] LATIN SMALL LETTER A..LATIN SMALL LETTER Z +00AA ; Alphabetic # L& FEMININE ORDINAL INDICATOR +00B5 ; Alphabetic # L& MICRO SIGN +00BA ; Alphabetic # L& MASCULINE ORDINAL INDICATOR +00C0..00D6 ; Alphabetic # L& [23] LATIN CAPITAL LETTER A WITH GRAVE..LATIN CAPITAL LETTER O WITH DIAERESIS +00D8..00F6 ; Alphabetic # L& [31] LATIN CAPITAL LETTER O WITH STROKE..LATIN SMALL LETTER O WITH DIAERESIS +00F8..01BA ; Alphabetic # L& [195] LATIN SMALL LETTER O WITH STROKE..LATIN SMALL LETTER EZH WITH TAIL +01BB ; Alphabetic # Lo LATIN LETTER TWO WITH STROKE +01BC..01BF ; Alphabetic # L& [4] LATIN CAPITAL LETTER TONE FIVE..LATIN LETTER WYNN +01C0..01C3 ; Alphabetic # Lo [4] LATIN LETTER DENTAL CLICK..LATIN LETTER RETROFLEX CLICK +01C4..0220 ; Alphabetic # L& [93] LATIN CAPITAL LETTER DZ WITH CARON..LATIN CAPITAL LETTER N WITH LONG RIGHT LEG +0222..0233 ; Alphabetic # L& [18] LATIN CAPITAL LETTER OU..LATIN SMALL LETTER Y WITH MACRON +0250..02AD ; Alphabetic # L& [94] LATIN SMALL LETTER TURNED A..LATIN LETTER BIDENTAL PERCUSSIVE +02B0..02B8 ; Alphabetic # Lm [9] MODIFIER LETTER SMALL H..MODIFIER LETTER SMALL Y +02BB..02C1 ; Alphabetic # Lm [7] MODIFIER LETTER TURNED COMMA..MODIFIER LETTER REVERSED GLOTTAL STOP +02D0..02D1 ; Alphabetic # Lm [2] MODIFIER LETTER TRIANGULAR COLON..MODIFIER LETTER HALF TRIANGULAR COLON +02E0..02E4 ; Alphabetic # Lm [5] MODIFIER LETTER SMALL GAMMA..MODIFIER LETTER SMALL REVERSED GLOTTAL STOP +02EE ; Alphabetic # Lm MODIFIER LETTER DOUBLE APOSTROPHE +0345 ; Alphabetic # Mn COMBINING GREEK YPOGEGRAMMENI +037A ; Alphabetic # Lm GREEK YPOGEGRAMMENI +0386 ; Alphabetic # L& GREEK CAPITAL LETTER ALPHA WITH TONOS +0388..038A ; Alphabetic # L& [3] GREEK CAPITAL LETTER EPSILON WITH TONOS..GREEK CAPITAL LETTER IOTA WITH TONOS +038C ; Alphabetic # L& GREEK CAPITAL LETTER OMICRON WITH TONOS +038E..03A1 ; Alphabetic # L& [20] GREEK CAPITAL LETTER UPSILON WITH TONOS..GREEK CAPITAL LETTER RHO +03A3..03CE ; Alphabetic # L& [44] GREEK CAPITAL LETTER SIGMA..GREEK SMALL LETTER OMEGA WITH TONOS +03D0..03F5 ; Alphabetic # L& [38] GREEK BETA SYMBOL..GREEK LUNATE EPSILON SYMBOL +0400..0481 ; Alphabetic # L& [130] CYRILLIC CAPITAL LETTER IE WITH GRAVE..CYRILLIC SMALL LETTER KOPPA +048A..04CE ; Alphabetic # L& [69] CYRILLIC CAPITAL LETTER SHORT I WITH TAIL..CYRILLIC SMALL LETTER EM WITH TAIL +04D0..04F5 ; Alphabetic # L& [38] CYRILLIC CAPITAL LETTER A WITH BREVE..CYRILLIC SMALL LETTER CHE WITH DIAERESIS +04F8..04F9 ; Alphabetic # L& [2] CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS..CYRILLIC SMALL LETTER YERU WITH DIAERESIS +0500..050F ; Alphabetic # L& [16] CYRILLIC CAPITAL LETTER KOMI DE..CYRILLIC SMALL LETTER KOMI TJE +0531..0556 ; Alphabetic # L& [38] ARMENIAN CAPITAL LETTER AYB..ARMENIAN CAPITAL LETTER FEH +0559 ; Alphabetic # Lm ARMENIAN MODIFIER LETTER LEFT HALF RING +0561..0587 ; Alphabetic # L& [39] ARMENIAN SMALL LETTER AYB..ARMENIAN SMALL LIGATURE ECH YIWN +05B0..05B9 ; Alphabetic # Mn [10] HEBREW POINT SHEVA..HEBREW POINT HOLAM +05BB..05BD ; Alphabetic # Mn [3] HEBREW POINT QUBUTS..HEBREW POINT METEG +05BF ; Alphabetic # Mn HEBREW POINT RAFE +05C1..05C2 ; Alphabetic # Mn [2] HEBREW POINT SHIN DOT..HEBREW POINT SIN DOT +05C4 ; Alphabetic # Mn HEBREW MARK UPPER DOT +05D0..05EA ; Alphabetic # Lo [27] HEBREW LETTER ALEF..HEBREW LETTER TAV +05F0..05F2 ; Alphabetic # Lo [3] HEBREW LIGATURE YIDDISH DOUBLE VAV..HEBREW LIGATURE YIDDISH DOUBLE YOD +0621..063A ; Alphabetic # Lo [26] ARABIC LETTER HAMZA..ARABIC LETTER GHAIN +0640 ; Alphabetic # Lm ARABIC TATWEEL +0641..064A ; Alphabetic # Lo [10] ARABIC LETTER FEH..ARABIC LETTER YEH +064B..0655 ; Alphabetic # Mn [11] ARABIC FATHATAN..ARABIC HAMZA BELOW +066E..066F ; Alphabetic # Lo [2] ARABIC LETTER DOTLESS BEH..ARABIC LETTER DOTLESS QAF +0670 ; Alphabetic # Mn ARABIC LETTER SUPERSCRIPT ALEF +0671..06D3 ; Alphabetic # Lo [99] ARABIC LETTER ALEF WASLA..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE +06D5 ; Alphabetic # Lo ARABIC LETTER AE +06D6..06DC ; Alphabetic # Mn [7] ARABIC SMALL HIGH LIGATURE SAD WITH LAM WITH ALEF MAKSURA..ARABIC SMALL HIGH SEEN +06E1..06E4 ; Alphabetic # Mn [4] ARABIC SMALL HIGH DOTLESS HEAD OF KHAH..ARABIC SMALL HIGH MADDA +06E5..06E6 ; Alphabetic # Lm [2] ARABIC SMALL WAW..ARABIC SMALL YEH +06E7..06E8 ; Alphabetic # Mn [2] ARABIC SMALL HIGH YEH..ARABIC SMALL HIGH NOON +06ED ; Alphabetic # Mn ARABIC SMALL LOW MEEM +06FA..06FC ; Alphabetic # Lo [3] ARABIC LETTER SHEEN WITH DOT BELOW..ARABIC LETTER GHAIN WITH DOT BELOW +0710 ; Alphabetic # Lo SYRIAC LETTER ALAPH +0711 ; Alphabetic # Mn SYRIAC LETTER SUPERSCRIPT ALAPH +0712..072C ; Alphabetic # Lo [27] SYRIAC LETTER BETH..SYRIAC LETTER TAW +0730..073F ; Alphabetic # Mn [16] SYRIAC PTHAHA ABOVE..SYRIAC RWAHA +0780..07A5 ; Alphabetic # Lo [38] THAANA LETTER HAA..THAANA LETTER WAAVU +07A6..07B0 ; Alphabetic # Mn [11] THAANA ABAFILI..THAANA SUKUN +07B1 ; Alphabetic # Lo THAANA LETTER NAA +0901..0902 ; Alphabetic # Mn [2] DEVANAGARI SIGN CANDRABINDU..DEVANAGARI SIGN ANUSVARA +0903 ; Alphabetic # Mc DEVANAGARI SIGN VISARGA +0905..0939 ; Alphabetic # Lo [53] DEVANAGARI LETTER A..DEVANAGARI LETTER HA +093D ; Alphabetic # Lo DEVANAGARI SIGN AVAGRAHA +093E..0940 ; Alphabetic # Mc [3] DEVANAGARI VOWEL SIGN AA..DEVANAGARI VOWEL SIGN II +0941..0948 ; Alphabetic # Mn [8] DEVANAGARI VOWEL SIGN U..DEVANAGARI VOWEL SIGN AI +0949..094C ; Alphabetic # Mc [4] DEVANAGARI VOWEL SIGN CANDRA O..DEVANAGARI VOWEL SIGN AU +0950 ; Alphabetic # Lo DEVANAGARI OM +0958..0961 ; Alphabetic # Lo [10] DEVANAGARI LETTER QA..DEVANAGARI LETTER VOCALIC LL +0962..0963 ; Alphabetic # Mn [2] DEVANAGARI VOWEL SIGN VOCALIC L..DEVANAGARI VOWEL SIGN VOCALIC LL +0981 ; Alphabetic # Mn BENGALI SIGN CANDRABINDU +0982..0983 ; Alphabetic # Mc [2] BENGALI SIGN ANUSVARA..BENGALI SIGN VISARGA +0985..098C ; Alphabetic # Lo [8] BENGALI LETTER A..BENGALI LETTER VOCALIC L +098F..0990 ; Alphabetic # Lo [2] BENGALI LETTER E..BENGALI LETTER AI +0993..09A8 ; Alphabetic # Lo [22] BENGALI LETTER O..BENGALI LETTER NA +09AA..09B0 ; Alphabetic # Lo [7] BENGALI LETTER PA..BENGALI LETTER RA +09B2 ; Alphabetic # Lo BENGALI LETTER LA +09B6..09B9 ; Alphabetic # Lo [4] BENGALI LETTER SHA..BENGALI LETTER HA +09BE..09C0 ; Alphabetic # Mc [3] BENGALI VOWEL SIGN AA..BENGALI VOWEL SIGN II +09C1..09C4 ; Alphabetic # Mn [4] BENGALI VOWEL SIGN U..BENGALI VOWEL SIGN VOCALIC RR +09C7..09C8 ; Alphabetic # Mc [2] BENGALI VOWEL SIGN E..BENGALI VOWEL SIGN AI +09CB..09CC ; Alphabetic # Mc [2] BENGALI VOWEL SIGN O..BENGALI VOWEL SIGN AU +09D7 ; Alphabetic # Mc BENGALI AU LENGTH MARK +09DC..09DD ; Alphabetic # Lo [2] BENGALI LETTER RRA..BENGALI LETTER RHA +09DF..09E1 ; Alphabetic # Lo [3] BENGALI LETTER YYA..BENGALI LETTER VOCALIC LL +09E2..09E3 ; Alphabetic # Mn [2] BENGALI VOWEL SIGN VOCALIC L..BENGALI VOWEL SIGN VOCALIC LL +09F0..09F1 ; Alphabetic # Lo [2] BENGALI LETTER RA WITH MIDDLE DIAGONAL..BENGALI LETTER RA WITH LOWER DIAGONAL +0A02 ; Alphabetic # Mn GURMUKHI SIGN BINDI +0A05..0A0A ; Alphabetic # Lo [6] GURMUKHI LETTER A..GURMUKHI LETTER UU +0A0F..0A10 ; Alphabetic # Lo [2] GURMUKHI LETTER EE..GURMUKHI LETTER AI +0A13..0A28 ; Alphabetic # Lo [22] GURMUKHI LETTER OO..GURMUKHI LETTER NA +0A2A..0A30 ; Alphabetic # Lo [7] GURMUKHI LETTER PA..GURMUKHI LETTER RA +0A32..0A33 ; Alphabetic # Lo [2] GURMUKHI LETTER LA..GURMUKHI LETTER LLA +0A35..0A36 ; Alphabetic # Lo [2] GURMUKHI LETTER VA..GURMUKHI LETTER SHA +0A38..0A39 ; Alphabetic # Lo [2] GURMUKHI LETTER SA..GURMUKHI LETTER HA +0A3E..0A40 ; Alphabetic # Mc [3] GURMUKHI VOWEL SIGN AA..GURMUKHI VOWEL SIGN II +0A41..0A42 ; Alphabetic # Mn [2] GURMUKHI VOWEL SIGN U..GURMUKHI VOWEL SIGN UU +0A47..0A48 ; Alphabetic # Mn [2] GURMUKHI VOWEL SIGN EE..GURMUKHI VOWEL SIGN AI +0A4B..0A4C ; Alphabetic # Mn [2] GURMUKHI VOWEL SIGN OO..GURMUKHI VOWEL SIGN AU +0A59..0A5C ; Alphabetic # Lo [4] GURMUKHI LETTER KHHA..GURMUKHI LETTER RRA +0A5E ; Alphabetic # Lo GURMUKHI LETTER FA +0A70..0A71 ; Alphabetic # Mn [2] GURMUKHI TIPPI..GURMUKHI ADDAK +0A72..0A74 ; Alphabetic # Lo [3] GURMUKHI IRI..GURMUKHI EK ONKAR +0A81..0A82 ; Alphabetic # Mn [2] GUJARATI SIGN CANDRABINDU..GUJARATI SIGN ANUSVARA +0A83 ; Alphabetic # Mc GUJARATI SIGN VISARGA +0A85..0A8B ; Alphabetic # Lo [7] GUJARATI LETTER A..GUJARATI LETTER VOCALIC R +0A8D ; Alphabetic # Lo GUJARATI VOWEL CANDRA E +0A8F..0A91 ; Alphabetic # Lo [3] GUJARATI LETTER E..GUJARATI VOWEL CANDRA O +0A93..0AA8 ; Alphabetic # Lo [22] GUJARATI LETTER O..GUJARATI LETTER NA +0AAA..0AB0 ; Alphabetic # Lo [7] GUJARATI LETTER PA..GUJARATI LETTER RA +0AB2..0AB3 ; Alphabetic # Lo [2] GUJARATI LETTER LA..GUJARATI LETTER LLA +0AB5..0AB9 ; Alphabetic # Lo [5] GUJARATI LETTER VA..GUJARATI LETTER HA +0ABD ; Alphabetic # Lo GUJARATI SIGN AVAGRAHA +0ABE..0AC0 ; Alphabetic # Mc [3] GUJARATI VOWEL SIGN AA..GUJARATI VOWEL SIGN II +0AC1..0AC5 ; Alphabetic # Mn [5] GUJARATI VOWEL SIGN U..GUJARATI VOWEL SIGN CANDRA E +0AC7..0AC8 ; Alphabetic # Mn [2] GUJARATI VOWEL SIGN E..GUJARATI VOWEL SIGN AI +0AC9 ; Alphabetic # Mc GUJARATI VOWEL SIGN CANDRA O +0ACB..0ACC ; Alphabetic # Mc [2] GUJARATI VOWEL SIGN O..GUJARATI VOWEL SIGN AU +0AD0 ; Alphabetic # Lo GUJARATI OM +0AE0 ; Alphabetic # Lo GUJARATI LETTER VOCALIC RR +0B01 ; Alphabetic # Mn ORIYA SIGN CANDRABINDU +0B02..0B03 ; Alphabetic # Mc [2] ORIYA SIGN ANUSVARA..ORIYA SIGN VISARGA +0B05..0B0C ; Alphabetic # Lo [8] ORIYA LETTER A..ORIYA LETTER VOCALIC L +0B0F..0B10 ; Alphabetic # Lo [2] ORIYA LETTER E..ORIYA LETTER AI +0B13..0B28 ; Alphabetic # Lo [22] ORIYA LETTER O..ORIYA LETTER NA +0B2A..0B30 ; Alphabetic # Lo [7] ORIYA LETTER PA..ORIYA LETTER RA +0B32..0B33 ; Alphabetic # Lo [2] ORIYA LETTER LA..ORIYA LETTER LLA +0B36..0B39 ; Alphabetic # Lo [4] ORIYA LETTER SHA..ORIYA LETTER HA +0B3D ; Alphabetic # Lo ORIYA SIGN AVAGRAHA +0B3E ; Alphabetic # Mc ORIYA VOWEL SIGN AA +0B3F ; Alphabetic # Mn ORIYA VOWEL SIGN I +0B40 ; Alphabetic # Mc ORIYA VOWEL SIGN II +0B41..0B43 ; Alphabetic # Mn [3] ORIYA VOWEL SIGN U..ORIYA VOWEL SIGN VOCALIC R +0B47..0B48 ; Alphabetic # Mc [2] ORIYA VOWEL SIGN E..ORIYA VOWEL SIGN AI +0B4B..0B4C ; Alphabetic # Mc [2] ORIYA VOWEL SIGN O..ORIYA VOWEL SIGN AU +0B56 ; Alphabetic # Mn ORIYA AI LENGTH MARK +0B57 ; Alphabetic # Mc ORIYA AU LENGTH MARK +0B5C..0B5D ; Alphabetic # Lo [2] ORIYA LETTER RRA..ORIYA LETTER RHA +0B5F..0B61 ; Alphabetic # Lo [3] ORIYA LETTER YYA..ORIYA LETTER VOCALIC LL +0B82 ; Alphabetic # Mn TAMIL SIGN ANUSVARA +0B83 ; Alphabetic # Lo TAMIL SIGN VISARGA +0B85..0B8A ; Alphabetic # Lo [6] TAMIL LETTER A..TAMIL LETTER UU +0B8E..0B90 ; Alphabetic # Lo [3] TAMIL LETTER E..TAMIL LETTER AI +0B92..0B95 ; Alphabetic # Lo [4] TAMIL LETTER O..TAMIL LETTER KA +0B99..0B9A ; Alphabetic # Lo [2] TAMIL LETTER NGA..TAMIL LETTER CA +0B9C ; Alphabetic # Lo TAMIL LETTER JA +0B9E..0B9F ; Alphabetic # Lo [2] TAMIL LETTER NYA..TAMIL LETTER TTA +0BA3..0BA4 ; Alphabetic # Lo [2] TAMIL LETTER NNA..TAMIL LETTER TA +0BA8..0BAA ; Alphabetic # Lo [3] TAMIL LETTER NA..TAMIL LETTER PA +0BAE..0BB5 ; Alphabetic # Lo [8] TAMIL LETTER MA..TAMIL LETTER VA +0BB7..0BB9 ; Alphabetic # Lo [3] TAMIL LETTER SSA..TAMIL LETTER HA +0BBE..0BBF ; Alphabetic # Mc [2] TAMIL VOWEL SIGN AA..TAMIL VOWEL SIGN I +0BC0 ; Alphabetic # Mn TAMIL VOWEL SIGN II +0BC1..0BC2 ; Alphabetic # Mc [2] TAMIL VOWEL SIGN U..TAMIL VOWEL SIGN UU +0BC6..0BC8 ; Alphabetic # Mc [3] TAMIL VOWEL SIGN E..TAMIL VOWEL SIGN AI +0BCA..0BCC ; Alphabetic # Mc [3] TAMIL VOWEL SIGN O..TAMIL VOWEL SIGN AU +0BD7 ; Alphabetic # Mc TAMIL AU LENGTH MARK +0C01..0C03 ; Alphabetic # Mc [3] TELUGU SIGN CANDRABINDU..TELUGU SIGN VISARGA +0C05..0C0C ; Alphabetic # Lo [8] TELUGU LETTER A..TELUGU LETTER VOCALIC L +0C0E..0C10 ; Alphabetic # Lo [3] TELUGU LETTER E..TELUGU LETTER AI +0C12..0C28 ; Alphabetic # Lo [23] TELUGU LETTER O..TELUGU LETTER NA +0C2A..0C33 ; Alphabetic # Lo [10] TELUGU LETTER PA..TELUGU LETTER LLA +0C35..0C39 ; Alphabetic # Lo [5] TELUGU LETTER VA..TELUGU LETTER HA +0C3E..0C40 ; Alphabetic # Mn [3] TELUGU VOWEL SIGN AA..TELUGU VOWEL SIGN II +0C41..0C44 ; Alphabetic # Mc [4] TELUGU VOWEL SIGN U..TELUGU VOWEL SIGN VOCALIC RR +0C46..0C48 ; Alphabetic # Mn [3] TELUGU VOWEL SIGN E..TELUGU VOWEL SIGN AI +0C4A..0C4C ; Alphabetic # Mn [3] TELUGU VOWEL SIGN O..TELUGU VOWEL SIGN AU +0C55..0C56 ; Alphabetic # Mn [2] TELUGU LENGTH MARK..TELUGU AI LENGTH MARK +0C60..0C61 ; Alphabetic # Lo [2] TELUGU LETTER VOCALIC RR..TELUGU LETTER VOCALIC LL +0C82..0C83 ; Alphabetic # Mc [2] KANNADA SIGN ANUSVARA..KANNADA SIGN VISARGA +0C85..0C8C ; Alphabetic # Lo [8] KANNADA LETTER A..KANNADA LETTER VOCALIC L +0C8E..0C90 ; Alphabetic # Lo [3] KANNADA LETTER E..KANNADA LETTER AI +0C92..0CA8 ; Alphabetic # Lo [23] KANNADA LETTER O..KANNADA LETTER NA +0CAA..0CB3 ; Alphabetic # Lo [10] KANNADA LETTER PA..KANNADA LETTER LLA +0CB5..0CB9 ; Alphabetic # Lo [5] KANNADA LETTER VA..KANNADA LETTER HA +0CBE ; Alphabetic # Mc KANNADA VOWEL SIGN AA +0CBF ; Alphabetic # Mn KANNADA VOWEL SIGN I +0CC0..0CC4 ; Alphabetic # Mc [5] KANNADA VOWEL SIGN II..KANNADA VOWEL SIGN VOCALIC RR +0CC6 ; Alphabetic # Mn KANNADA VOWEL SIGN E +0CC7..0CC8 ; Alphabetic # Mc [2] KANNADA VOWEL SIGN EE..KANNADA VOWEL SIGN AI +0CCA..0CCB ; Alphabetic # Mc [2] KANNADA VOWEL SIGN O..KANNADA VOWEL SIGN OO +0CCC ; Alphabetic # Mn KANNADA VOWEL SIGN AU +0CD5..0CD6 ; Alphabetic # Mc [2] KANNADA LENGTH MARK..KANNADA AI LENGTH MARK +0CDE ; Alphabetic # Lo KANNADA LETTER FA +0CE0..0CE1 ; Alphabetic # Lo [2] KANNADA LETTER VOCALIC RR..KANNADA LETTER VOCALIC LL +0D02..0D03 ; Alphabetic # Mc [2] MALAYALAM SIGN ANUSVARA..MALAYALAM SIGN VISARGA +0D05..0D0C ; Alphabetic # Lo [8] MALAYALAM LETTER A..MALAYALAM LETTER VOCALIC L +0D0E..0D10 ; Alphabetic # Lo [3] MALAYALAM LETTER E..MALAYALAM LETTER AI +0D12..0D28 ; Alphabetic # Lo [23] MALAYALAM LETTER O..MALAYALAM LETTER NA +0D2A..0D39 ; Alphabetic # Lo [16] MALAYALAM LETTER PA..MALAYALAM LETTER HA +0D3E..0D40 ; Alphabetic # Mc [3] MALAYALAM VOWEL SIGN AA..MALAYALAM VOWEL SIGN II +0D41..0D43 ; Alphabetic # Mn [3] MALAYALAM VOWEL SIGN U..MALAYALAM VOWEL SIGN VOCALIC R +0D46..0D48 ; Alphabetic # Mc [3] MALAYALAM VOWEL SIGN E..MALAYALAM VOWEL SIGN AI +0D4A..0D4C ; Alphabetic # Mc [3] MALAYALAM VOWEL SIGN O..MALAYALAM VOWEL SIGN AU +0D57 ; Alphabetic # Mc MALAYALAM AU LENGTH MARK +0D60..0D61 ; Alphabetic # Lo [2] MALAYALAM LETTER VOCALIC RR..MALAYALAM LETTER VOCALIC LL +0D82..0D83 ; Alphabetic # Mc [2] SINHALA SIGN ANUSVARAYA..SINHALA SIGN VISARGAYA +0D85..0D96 ; Alphabetic # Lo [18] SINHALA LETTER AYANNA..SINHALA LETTER AUYANNA +0D9A..0DB1 ; Alphabetic # Lo [24] SINHALA LETTER ALPAPRAANA KAYANNA..SINHALA LETTER DANTAJA NAYANNA +0DB3..0DBB ; Alphabetic # Lo [9] SINHALA LETTER SANYAKA DAYANNA..SINHALA LETTER RAYANNA +0DBD ; Alphabetic # Lo SINHALA LETTER DANTAJA LAYANNA +0DC0..0DC6 ; Alphabetic # Lo [7] SINHALA LETTER VAYANNA..SINHALA LETTER FAYANNA +0DCF..0DD1 ; Alphabetic # Mc [3] SINHALA VOWEL SIGN AELA-PILLA..SINHALA VOWEL SIGN DIGA AEDA-PILLA +0DD2..0DD4 ; Alphabetic # Mn [3] SINHALA VOWEL SIGN KETTI IS-PILLA..SINHALA VOWEL SIGN KETTI PAA-PILLA +0DD6 ; Alphabetic # Mn SINHALA VOWEL SIGN DIGA PAA-PILLA +0DD8..0DDF ; Alphabetic # Mc [8] SINHALA VOWEL SIGN GAETTA-PILLA..SINHALA VOWEL SIGN GAYANUKITTA +0DF2..0DF3 ; Alphabetic # Mc [2] SINHALA VOWEL SIGN DIGA GAETTA-PILLA..SINHALA VOWEL SIGN DIGA GAYANUKITTA +0E01..0E30 ; Alphabetic # Lo [48] THAI CHARACTER KO KAI..THAI CHARACTER SARA A +0E31 ; Alphabetic # Mn THAI CHARACTER MAI HAN-AKAT +0E32..0E33 ; Alphabetic # Lo [2] THAI CHARACTER SARA AA..THAI CHARACTER SARA AM +0E34..0E3A ; Alphabetic # Mn [7] THAI CHARACTER SARA I..THAI CHARACTER PHINTHU +0E40..0E45 ; Alphabetic # Lo [6] THAI CHARACTER SARA E..THAI CHARACTER LAKKHANGYAO +0E46 ; Alphabetic # Lm THAI CHARACTER MAIYAMOK +0E4D ; Alphabetic # Mn THAI CHARACTER NIKHAHIT +0E81..0E82 ; Alphabetic # Lo [2] LAO LETTER KO..LAO LETTER KHO SUNG +0E84 ; Alphabetic # Lo LAO LETTER KHO TAM +0E87..0E88 ; Alphabetic # Lo [2] LAO LETTER NGO..LAO LETTER CO +0E8A ; Alphabetic # Lo LAO LETTER SO TAM +0E8D ; Alphabetic # Lo LAO LETTER NYO +0E94..0E97 ; Alphabetic # Lo [4] LAO LETTER DO..LAO LETTER THO TAM +0E99..0E9F ; Alphabetic # Lo [7] LAO LETTER NO..LAO LETTER FO SUNG +0EA1..0EA3 ; Alphabetic # Lo [3] LAO LETTER MO..LAO LETTER LO LING +0EA5 ; Alphabetic # Lo LAO LETTER LO LOOT +0EA7 ; Alphabetic # Lo LAO LETTER WO +0EAA..0EAB ; Alphabetic # Lo [2] LAO LETTER SO SUNG..LAO LETTER HO SUNG +0EAD..0EB0 ; Alphabetic # Lo [4] LAO LETTER O..LAO VOWEL SIGN A +0EB1 ; Alphabetic # Mn LAO VOWEL SIGN MAI KAN +0EB2..0EB3 ; Alphabetic # Lo [2] LAO VOWEL SIGN AA..LAO VOWEL SIGN AM +0EB4..0EB9 ; Alphabetic # Mn [6] LAO VOWEL SIGN I..LAO VOWEL SIGN UU +0EBB..0EBC ; Alphabetic # Mn [2] LAO VOWEL SIGN MAI KON..LAO SEMIVOWEL SIGN LO +0EBD ; Alphabetic # Lo LAO SEMIVOWEL SIGN NYO +0EC0..0EC4 ; Alphabetic # Lo [5] LAO VOWEL SIGN E..LAO VOWEL SIGN AI +0EC6 ; Alphabetic # Lm LAO KO LA +0ECD ; Alphabetic # Mn LAO NIGGAHITA +0EDC..0EDD ; Alphabetic # Lo [2] LAO HO NO..LAO HO MO +0F00 ; Alphabetic # Lo TIBETAN SYLLABLE OM +0F40..0F47 ; Alphabetic # Lo [8] TIBETAN LETTER KA..TIBETAN LETTER JA +0F49..0F6A ; Alphabetic # Lo [34] TIBETAN LETTER NYA..TIBETAN LETTER FIXED-FORM RA +0F71..0F7E ; Alphabetic # Mn [14] TIBETAN VOWEL SIGN AA..TIBETAN SIGN RJES SU NGA RO +0F7F ; Alphabetic # Mc TIBETAN SIGN RNAM BCAD +0F80..0F81 ; Alphabetic # Mn [2] TIBETAN VOWEL SIGN REVERSED I..TIBETAN VOWEL SIGN REVERSED II +0F88..0F8B ; Alphabetic # Lo [4] TIBETAN SIGN LCE TSA CAN..TIBETAN SIGN GRU MED RGYINGS +0F90..0F97 ; Alphabetic # Mn [8] TIBETAN SUBJOINED LETTER KA..TIBETAN SUBJOINED LETTER JA +0F99..0FBC ; Alphabetic # Mn [36] TIBETAN SUBJOINED LETTER NYA..TIBETAN SUBJOINED LETTER FIXED-FORM RA +1000..1021 ; Alphabetic # Lo [34] MYANMAR LETTER KA..MYANMAR LETTER A +1023..1027 ; Alphabetic # Lo [5] MYANMAR LETTER I..MYANMAR LETTER E +1029..102A ; Alphabetic # Lo [2] MYANMAR LETTER O..MYANMAR LETTER AU +102C ; Alphabetic # Mc MYANMAR VOWEL SIGN AA +102D..1030 ; Alphabetic # Mn [4] MYANMAR VOWEL SIGN I..MYANMAR VOWEL SIGN UU +1031 ; Alphabetic # Mc MYANMAR VOWEL SIGN E +1032 ; Alphabetic # Mn MYANMAR VOWEL SIGN AI +1036 ; Alphabetic # Mn MYANMAR SIGN ANUSVARA +1038 ; Alphabetic # Mc MYANMAR SIGN VISARGA +1050..1055 ; Alphabetic # Lo [6] MYANMAR LETTER SHA..MYANMAR LETTER VOCALIC LL +1056..1057 ; Alphabetic # Mc [2] MYANMAR VOWEL SIGN VOCALIC R..MYANMAR VOWEL SIGN VOCALIC RR +1058..1059 ; Alphabetic # Mn [2] MYANMAR VOWEL SIGN VOCALIC L..MYANMAR VOWEL SIGN VOCALIC LL +10A0..10C5 ; Alphabetic # L& [38] GEORGIAN CAPITAL LETTER AN..GEORGIAN CAPITAL LETTER HOE +10D0..10F8 ; Alphabetic # Lo [41] GEORGIAN LETTER AN..GEORGIAN LETTER ELIFI +1100..1159 ; Alphabetic # Lo [90] HANGUL CHOSEONG KIYEOK..HANGUL CHOSEONG YEORINHIEUH +115F..11A2 ; Alphabetic # Lo [68] HANGUL CHOSEONG FILLER..HANGUL JUNGSEONG SSANGARAEA +11A8..11F9 ; Alphabetic # Lo [82] HANGUL JONGSEONG KIYEOK..HANGUL JONGSEONG YEORINHIEUH +1200..1206 ; Alphabetic # Lo [7] ETHIOPIC SYLLABLE HA..ETHIOPIC SYLLABLE HO +1208..1246 ; Alphabetic # Lo [63] ETHIOPIC SYLLABLE LA..ETHIOPIC SYLLABLE QO +1248 ; Alphabetic # Lo ETHIOPIC SYLLABLE QWA +124A..124D ; Alphabetic # Lo [4] ETHIOPIC SYLLABLE QWI..ETHIOPIC SYLLABLE QWE +1250..1256 ; Alphabetic # Lo [7] ETHIOPIC SYLLABLE QHA..ETHIOPIC SYLLABLE QHO +1258 ; Alphabetic # Lo ETHIOPIC SYLLABLE QHWA +125A..125D ; Alphabetic # Lo [4] ETHIOPIC SYLLABLE QHWI..ETHIOPIC SYLLABLE QHWE +1260..1286 ; Alphabetic # Lo [39] ETHIOPIC SYLLABLE BA..ETHIOPIC SYLLABLE XO +1288 ; Alphabetic # Lo ETHIOPIC SYLLABLE XWA +128A..128D ; Alphabetic # Lo [4] ETHIOPIC SYLLABLE XWI..ETHIOPIC SYLLABLE XWE +1290..12AE ; Alphabetic # Lo [31] ETHIOPIC SYLLABLE NA..ETHIOPIC SYLLABLE KO +12B0 ; Alphabetic # Lo ETHIOPIC SYLLABLE KWA +12B2..12B5 ; Alphabetic # Lo [4] ETHIOPIC SYLLABLE KWI..ETHIOPIC SYLLABLE KWE +12B8..12BE ; Alphabetic # Lo [7] ETHIOPIC SYLLABLE KXA..ETHIOPIC SYLLABLE KXO +12C0 ; Alphabetic # Lo ETHIOPIC SYLLABLE KXWA +12C2..12C5 ; Alphabetic # Lo [4] ETHIOPIC SYLLABLE KXWI..ETHIOPIC SYLLABLE KXWE +12C8..12CE ; Alphabetic # Lo [7] ETHIOPIC SYLLABLE WA..ETHIOPIC SYLLABLE WO +12D0..12D6 ; Alphabetic # Lo [7] ETHIOPIC SYLLABLE PHARYNGEAL A..ETHIOPIC SYLLABLE PHARYNGEAL O +12D8..12EE ; Alphabetic # Lo [23] ETHIOPIC SYLLABLE ZA..ETHIOPIC SYLLABLE YO +12F0..130E ; Alphabetic # Lo [31] ETHIOPIC SYLLABLE DA..ETHIOPIC SYLLABLE GO +1310 ; Alphabetic # Lo ETHIOPIC SYLLABLE GWA +1312..1315 ; Alphabetic # Lo [4] ETHIOPIC SYLLABLE GWI..ETHIOPIC SYLLABLE GWE +1318..131E ; Alphabetic # Lo [7] ETHIOPIC SYLLABLE GGA..ETHIOPIC SYLLABLE GGO +1320..1346 ; Alphabetic # Lo [39] ETHIOPIC SYLLABLE THA..ETHIOPIC SYLLABLE TZO +1348..135A ; Alphabetic # Lo [19] ETHIOPIC SYLLABLE FA..ETHIOPIC SYLLABLE FYA +13A0..13F4 ; Alphabetic # Lo [85] CHEROKEE LETTER A..CHEROKEE LETTER YV +1401..166C ; Alphabetic # Lo [620] CANADIAN SYLLABICS E..CANADIAN SYLLABICS CARRIER TTSA +166F..1676 ; Alphabetic # Lo [8] CANADIAN SYLLABICS QAI..CANADIAN SYLLABICS NNGAA +1681..169A ; Alphabetic # Lo [26] OGHAM LETTER BEITH..OGHAM LETTER PEITH +16A0..16EA ; Alphabetic # Lo [75] RUNIC LETTER FEHU FEOH FE F..RUNIC LETTER X +16EE..16F0 ; Alphabetic # Nl [3] RUNIC ARLAUG SYMBOL..RUNIC BELGTHOR SYMBOL +1700..170C ; Alphabetic # Lo [13] TAGALOG LETTER A..TAGALOG LETTER YA +170E..1711 ; Alphabetic # Lo [4] TAGALOG LETTER LA..TAGALOG LETTER HA +1712..1713 ; Alphabetic # Mn [2] TAGALOG VOWEL SIGN I..TAGALOG VOWEL SIGN U +1720..1731 ; Alphabetic # Lo [18] HANUNOO LETTER A..HANUNOO LETTER HA +1732..1733 ; Alphabetic # Mn [2] HANUNOO VOWEL SIGN I..HANUNOO VOWEL SIGN U +1740..1751 ; Alphabetic # Lo [18] BUHID LETTER A..BUHID LETTER HA +1752..1753 ; Alphabetic # Mn [2] BUHID VOWEL SIGN I..BUHID VOWEL SIGN U +1760..176C ; Alphabetic # Lo [13] TAGBANWA LETTER A..TAGBANWA LETTER YA +176E..1770 ; Alphabetic # Lo [3] TAGBANWA LETTER LA..TAGBANWA LETTER SA +1772..1773 ; Alphabetic # Mn [2] TAGBANWA VOWEL SIGN I..TAGBANWA VOWEL SIGN U +1780..17B3 ; Alphabetic # Lo [52] KHMER LETTER KA..KHMER INDEPENDENT VOWEL QAU +17B4..17B6 ; Alphabetic # Mc [3] KHMER VOWEL INHERENT AQ..KHMER VOWEL SIGN AA +17B7..17BD ; Alphabetic # Mn [7] KHMER VOWEL SIGN I..KHMER VOWEL SIGN UA +17BE..17C5 ; Alphabetic # Mc [8] KHMER VOWEL SIGN OE..KHMER VOWEL SIGN AU +17C6 ; Alphabetic # Mn KHMER SIGN NIKAHIT +17C7..17C8 ; Alphabetic # Mc [2] KHMER SIGN REAHMUK..KHMER SIGN YUUKALEAPINTU +17D7 ; Alphabetic # Lm KHMER SIGN LEK TOO +17DC ; Alphabetic # Lo KHMER SIGN AVAKRAHASANYA +1820..1842 ; Alphabetic # Lo [35] MONGOLIAN LETTER A..MONGOLIAN LETTER CHI +1843 ; Alphabetic # Lm MONGOLIAN LETTER TODO LONG VOWEL SIGN +1844..1877 ; Alphabetic # Lo [52] MONGOLIAN LETTER TODO E..MONGOLIAN LETTER MANCHU ZHA +1880..18A8 ; Alphabetic # Lo [41] MONGOLIAN LETTER ALI GALI ANUSVARA ONE..MONGOLIAN LETTER MANCHU ALI GALI BHA +18A9 ; Alphabetic # Mn MONGOLIAN LETTER ALI GALI DAGALGA +1E00..1E9B ; Alphabetic # L& [156] LATIN CAPITAL LETTER A WITH RING BELOW..LATIN SMALL LETTER LONG S WITH DOT ABOVE +1EA0..1EF9 ; Alphabetic # L& [90] LATIN CAPITAL LETTER A WITH DOT BELOW..LATIN SMALL LETTER Y WITH TILDE +1F00..1F15 ; Alphabetic # L& [22] GREEK SMALL LETTER ALPHA WITH PSILI..GREEK SMALL LETTER EPSILON WITH DASIA AND OXIA +1F18..1F1D ; Alphabetic # L& [6] GREEK CAPITAL LETTER EPSILON WITH PSILI..GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA +1F20..1F45 ; Alphabetic # L& [38] GREEK SMALL LETTER ETA WITH PSILI..GREEK SMALL LETTER OMICRON WITH DASIA AND OXIA +1F48..1F4D ; Alphabetic # L& [6] GREEK CAPITAL LETTER OMICRON WITH PSILI..GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA +1F50..1F57 ; Alphabetic # L& [8] GREEK SMALL LETTER UPSILON WITH PSILI..GREEK SMALL LETTER UPSILON WITH DASIA AND PERISPOMENI +1F59 ; Alphabetic # L& GREEK CAPITAL LETTER UPSILON WITH DASIA +1F5B ; Alphabetic # L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA +1F5D ; Alphabetic # L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA +1F5F..1F7D ; Alphabetic # L& [31] GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI..GREEK SMALL LETTER OMEGA WITH OXIA +1F80..1FB4 ; Alphabetic # L& [53] GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI..GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI +1FB6..1FBC ; Alphabetic # L& [7] GREEK SMALL LETTER ALPHA WITH PERISPOMENI..GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI +1FBE ; Alphabetic # L& GREEK PROSGEGRAMMENI +1FC2..1FC4 ; Alphabetic # L& [3] GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI +1FC6..1FCC ; Alphabetic # L& [7] GREEK SMALL LETTER ETA WITH PERISPOMENI..GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI +1FD0..1FD3 ; Alphabetic # L& [4] GREEK SMALL LETTER IOTA WITH VRACHY..GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA +1FD6..1FDB ; Alphabetic # L& [6] GREEK SMALL LETTER IOTA WITH PERISPOMENI..GREEK CAPITAL LETTER IOTA WITH OXIA +1FE0..1FEC ; Alphabetic # L& [13] GREEK SMALL LETTER UPSILON WITH VRACHY..GREEK CAPITAL LETTER RHO WITH DASIA +1FF2..1FF4 ; Alphabetic # L& [3] GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI +1FF6..1FFC ; Alphabetic # L& [7] GREEK SMALL LETTER OMEGA WITH PERISPOMENI..GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI +2071 ; Alphabetic # L& SUPERSCRIPT LATIN SMALL LETTER I +207F ; Alphabetic # L& SUPERSCRIPT LATIN SMALL LETTER N +2102 ; Alphabetic # L& DOUBLE-STRUCK CAPITAL C +2107 ; Alphabetic # L& EULER CONSTANT +210A..2113 ; Alphabetic # L& [10] SCRIPT SMALL G..SCRIPT SMALL L +2115 ; Alphabetic # L& DOUBLE-STRUCK CAPITAL N +2119..211D ; Alphabetic # L& [5] DOUBLE-STRUCK CAPITAL P..DOUBLE-STRUCK CAPITAL R +2124 ; Alphabetic # L& DOUBLE-STRUCK CAPITAL Z +2126 ; Alphabetic # L& OHM SIGN +2128 ; Alphabetic # L& BLACK-LETTER CAPITAL Z +212A..212D ; Alphabetic # L& [4] KELVIN SIGN..BLACK-LETTER CAPITAL C +212F..2131 ; Alphabetic # L& [3] SCRIPT SMALL E..SCRIPT CAPITAL F +2133..2134 ; Alphabetic # L& [2] SCRIPT CAPITAL M..SCRIPT SMALL O +2135..2138 ; Alphabetic # Lo [4] ALEF SYMBOL..DALET SYMBOL +2139 ; Alphabetic # L& INFORMATION SOURCE +213D..213F ; Alphabetic # L& [3] DOUBLE-STRUCK SMALL GAMMA..DOUBLE-STRUCK CAPITAL PI +2145..2149 ; Alphabetic # L& [5] DOUBLE-STRUCK ITALIC CAPITAL D..DOUBLE-STRUCK ITALIC SMALL J +2160..2183 ; Alphabetic # Nl [36] ROMAN NUMERAL ONE..ROMAN NUMERAL REVERSED ONE HUNDRED +3005 ; Alphabetic # Lm IDEOGRAPHIC ITERATION MARK +3006 ; Alphabetic # Lo IDEOGRAPHIC CLOSING MARK +3007 ; Alphabetic # Nl IDEOGRAPHIC NUMBER ZERO +3021..3029 ; Alphabetic # Nl [9] HANGZHOU NUMERAL ONE..HANGZHOU NUMERAL NINE +3031..3035 ; Alphabetic # Lm [5] VERTICAL KANA REPEAT MARK..VERTICAL KANA REPEAT MARK LOWER HALF +3038..303A ; Alphabetic # Nl [3] HANGZHOU NUMERAL TEN..HANGZHOU NUMERAL THIRTY +303B ; Alphabetic # Lm VERTICAL IDEOGRAPHIC ITERATION MARK +303C ; Alphabetic # Lo MASU MARK +3041..3096 ; Alphabetic # Lo [86] HIRAGANA LETTER SMALL A..HIRAGANA LETTER SMALL KE +309D..309E ; Alphabetic # Lm [2] HIRAGANA ITERATION MARK..HIRAGANA VOICED ITERATION MARK +309F ; Alphabetic # Lo HIRAGANA DIGRAPH YORI +30A1..30FA ; Alphabetic # Lo [90] KATAKANA LETTER SMALL A..KATAKANA LETTER VO +30FC..30FE ; Alphabetic # Lm [3] KATAKANA-HIRAGANA PROLONGED SOUND MARK..KATAKANA VOICED ITERATION MARK +30FF ; Alphabetic # Lo KATAKANA DIGRAPH KOTO +3105..312C ; Alphabetic # Lo [40] BOPOMOFO LETTER B..BOPOMOFO LETTER GN +3131..318E ; Alphabetic # Lo [94] HANGUL LETTER KIYEOK..HANGUL LETTER ARAEAE +31A0..31B7 ; Alphabetic # Lo [24] BOPOMOFO LETTER BU..BOPOMOFO FINAL LETTER H +31F0..31FF ; Alphabetic # Lo [16] KATAKANA LETTER SMALL KU..KATAKANA LETTER SMALL RO +3400..4DB5 ; Alphabetic # Lo [6582] CJK UNIFIED IDEOGRAPH-3400..CJK UNIFIED IDEOGRAPH-4DB5 +4E00..9FA5 ; Alphabetic # Lo [20902] CJK UNIFIED IDEOGRAPH-4E00..CJK UNIFIED IDEOGRAPH-9FA5 +A000..A48C ; Alphabetic # Lo [1165] YI SYLLABLE IT..YI SYLLABLE YYR +AC00..D7A3 ; Alphabetic # Lo [11172] HANGUL SYLLABLE GA..HANGUL SYLLABLE HIH +F900..FA2D ; Alphabetic # Lo [302] CJK COMPATIBILITY IDEOGRAPH-F900..CJK COMPATIBILITY IDEOGRAPH-FA2D +FA30..FA6A ; Alphabetic # Lo [59] CJK COMPATIBILITY IDEOGRAPH-FA30..CJK COMPATIBILITY IDEOGRAPH-FA6A +FB00..FB06 ; Alphabetic # L& [7] LATIN SMALL LIGATURE FF..LATIN SMALL LIGATURE ST +FB13..FB17 ; Alphabetic # L& [5] ARMENIAN SMALL LIGATURE MEN NOW..ARMENIAN SMALL LIGATURE MEN XEH +FB1D ; Alphabetic # Lo HEBREW LETTER YOD WITH HIRIQ +FB1E ; Alphabetic # Mn HEBREW POINT JUDEO-SPANISH VARIKA +FB1F..FB28 ; Alphabetic # Lo [10] HEBREW LIGATURE YIDDISH YOD YOD PATAH..HEBREW LETTER WIDE TAV +FB2A..FB36 ; Alphabetic # Lo [13] HEBREW LETTER SHIN WITH SHIN DOT..HEBREW LETTER ZAYIN WITH DAGESH +FB38..FB3C ; Alphabetic # Lo [5] HEBREW LETTER TET WITH DAGESH..HEBREW LETTER LAMED WITH DAGESH +FB3E ; Alphabetic # Lo HEBREW LETTER MEM WITH DAGESH +FB40..FB41 ; Alphabetic # Lo [2] HEBREW LETTER NUN WITH DAGESH..HEBREW LETTER SAMEKH WITH DAGESH +FB43..FB44 ; Alphabetic # Lo [2] HEBREW LETTER FINAL PE WITH DAGESH..HEBREW LETTER PE WITH DAGESH +FB46..FBB1 ; Alphabetic # Lo [108] HEBREW LETTER TSADI WITH DAGESH..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE FINAL FORM +FBD3..FD3D ; Alphabetic # Lo [363] ARABIC LETTER NG ISOLATED FORM..ARABIC LIGATURE ALEF WITH FATHATAN ISOLATED FORM +FD50..FD8F ; Alphabetic # Lo [64] ARABIC LIGATURE TEH WITH JEEM WITH MEEM INITIAL FORM..ARABIC LIGATURE MEEM WITH KHAH WITH MEEM INITIAL FORM +FD92..FDC7 ; Alphabetic # Lo [54] ARABIC LIGATURE MEEM WITH JEEM WITH KHAH INITIAL FORM..ARABIC LIGATURE NOON WITH JEEM WITH YEH FINAL FORM +FDF0..FDFB ; Alphabetic # Lo [12] ARABIC LIGATURE SALLA USED AS KORANIC STOP SIGN ISOLATED FORM..ARABIC LIGATURE JALLAJALALOUHOU +FE70..FE74 ; Alphabetic # Lo [5] ARABIC FATHATAN ISOLATED FORM..ARABIC KASRATAN ISOLATED FORM +FE76..FEFC ; Alphabetic # Lo [135] ARABIC FATHA ISOLATED FORM..ARABIC LIGATURE LAM WITH ALEF FINAL FORM +FF21..FF3A ; Alphabetic # L& [26] FULLWIDTH LATIN CAPITAL LETTER A..FULLWIDTH LATIN CAPITAL LETTER Z +FF41..FF5A ; Alphabetic # L& [26] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH LATIN SMALL LETTER Z +FF66..FF6F ; Alphabetic # Lo [10] HALFWIDTH KATAKANA LETTER WO..HALFWIDTH KATAKANA LETTER SMALL TU +FF70 ; Alphabetic # Lm HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK +FF71..FF9D ; Alphabetic # Lo [45] HALFWIDTH KATAKANA LETTER A..HALFWIDTH KATAKANA LETTER N +FF9E..FF9F ; Alphabetic # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK +FFA0..FFBE ; Alphabetic # Lo [31] HALFWIDTH HANGUL FILLER..HALFWIDTH HANGUL LETTER HIEUH +FFC2..FFC7 ; Alphabetic # Lo [6] HALFWIDTH HANGUL LETTER A..HALFWIDTH HANGUL LETTER E +FFCA..FFCF ; Alphabetic # Lo [6] HALFWIDTH HANGUL LETTER YEO..HALFWIDTH HANGUL LETTER OE +FFD2..FFD7 ; Alphabetic # Lo [6] HALFWIDTH HANGUL LETTER YO..HALFWIDTH HANGUL LETTER YU +FFDA..FFDC ; Alphabetic # Lo [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL LETTER I +10300..1031E ; Alphabetic # Lo [31] OLD ITALIC LETTER A..OLD ITALIC LETTER UU +10330..10349 ; Alphabetic # Lo [26] GOTHIC LETTER AHSA..GOTHIC LETTER OTHAL +1034A ; Alphabetic # Nl GOTHIC LETTER NINE HUNDRED +10400..10425 ; Alphabetic # L& [38] DESERET CAPITAL LETTER LONG I..DESERET CAPITAL LETTER ENG +10428..1044D ; Alphabetic # L& [38] DESERET SMALL LETTER LONG I..DESERET SMALL LETTER ENG +1D400..1D454 ; Alphabetic # L& [85] MATHEMATICAL BOLD CAPITAL A..MATHEMATICAL ITALIC SMALL G +1D456..1D49C ; Alphabetic # L& [71] MATHEMATICAL ITALIC SMALL I..MATHEMATICAL SCRIPT CAPITAL A +1D49E..1D49F ; Alphabetic # L& [2] MATHEMATICAL SCRIPT CAPITAL C..MATHEMATICAL SCRIPT CAPITAL D +1D4A2 ; Alphabetic # L& MATHEMATICAL SCRIPT CAPITAL G +1D4A5..1D4A6 ; Alphabetic # L& [2] MATHEMATICAL SCRIPT CAPITAL J..MATHEMATICAL SCRIPT CAPITAL K +1D4A9..1D4AC ; Alphabetic # L& [4] MATHEMATICAL SCRIPT CAPITAL N..MATHEMATICAL SCRIPT CAPITAL Q +1D4AE..1D4B9 ; Alphabetic # L& [12] MATHEMATICAL SCRIPT CAPITAL S..MATHEMATICAL SCRIPT SMALL D +1D4BB ; Alphabetic # L& MATHEMATICAL SCRIPT SMALL F +1D4BD..1D4C0 ; Alphabetic # L& [4] MATHEMATICAL SCRIPT SMALL H..MATHEMATICAL SCRIPT SMALL K +1D4C2..1D4C3 ; Alphabetic # L& [2] MATHEMATICAL SCRIPT SMALL M..MATHEMATICAL SCRIPT SMALL N +1D4C5..1D505 ; Alphabetic # L& [65] MATHEMATICAL SCRIPT SMALL P..MATHEMATICAL FRAKTUR CAPITAL B +1D507..1D50A ; Alphabetic # L& [4] MATHEMATICAL FRAKTUR CAPITAL D..MATHEMATICAL FRAKTUR CAPITAL G +1D50D..1D514 ; Alphabetic # L& [8] MATHEMATICAL FRAKTUR CAPITAL J..MATHEMATICAL FRAKTUR CAPITAL Q +1D516..1D51C ; Alphabetic # L& [7] MATHEMATICAL FRAKTUR CAPITAL S..MATHEMATICAL FRAKTUR CAPITAL Y +1D51E..1D539 ; Alphabetic # L& [28] MATHEMATICAL FRAKTUR SMALL A..MATHEMATICAL DOUBLE-STRUCK CAPITAL B +1D53B..1D53E ; Alphabetic # L& [4] MATHEMATICAL DOUBLE-STRUCK CAPITAL D..MATHEMATICAL DOUBLE-STRUCK CAPITAL G +1D540..1D544 ; Alphabetic # L& [5] MATHEMATICAL DOUBLE-STRUCK CAPITAL I..MATHEMATICAL DOUBLE-STRUCK CAPITAL M +1D546 ; Alphabetic # L& MATHEMATICAL DOUBLE-STRUCK CAPITAL O +1D54A..1D550 ; Alphabetic # L& [7] MATHEMATICAL DOUBLE-STRUCK CAPITAL S..MATHEMATICAL DOUBLE-STRUCK CAPITAL Y +1D552..1D6A3 ; Alphabetic # L& [338] MATHEMATICAL DOUBLE-STRUCK SMALL A..MATHEMATICAL MONOSPACE SMALL Z +1D6A8..1D6C0 ; Alphabetic # L& [25] MATHEMATICAL BOLD CAPITAL ALPHA..MATHEMATICAL BOLD CAPITAL OMEGA +1D6C2..1D6DA ; Alphabetic # L& [25] MATHEMATICAL BOLD SMALL ALPHA..MATHEMATICAL BOLD SMALL OMEGA +1D6DC..1D6FA ; Alphabetic # L& [31] MATHEMATICAL BOLD EPSILON SYMBOL..MATHEMATICAL ITALIC CAPITAL OMEGA +1D6FC..1D714 ; Alphabetic # L& [25] MATHEMATICAL ITALIC SMALL ALPHA..MATHEMATICAL ITALIC SMALL OMEGA +1D716..1D734 ; Alphabetic # L& [31] MATHEMATICAL ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD ITALIC CAPITAL OMEGA +1D736..1D74E ; Alphabetic # L& [25] MATHEMATICAL BOLD ITALIC SMALL ALPHA..MATHEMATICAL BOLD ITALIC SMALL OMEGA +1D750..1D76E ; Alphabetic # L& [31] MATHEMATICAL BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD CAPITAL OMEGA +1D770..1D788 ; Alphabetic # L& [25] MATHEMATICAL SANS-SERIF BOLD SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD SMALL OMEGA +1D78A..1D7A8 ; Alphabetic # L& [31] MATHEMATICAL SANS-SERIF BOLD EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL OMEGA +1D7AA..1D7C2 ; Alphabetic # L& [25] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL OMEGA +1D7C4..1D7C9 ; Alphabetic # L& [6] MATHEMATICAL SANS-SERIF BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC PI SYMBOL +20000..2A6D6 ; Alphabetic # Lo [42711] CJK UNIFIED IDEOGRAPH-20000..CJK UNIFIED IDEOGRAPH-2A6D6 +2F800..2FA1D ; Alphabetic # Lo [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D + +# Total code points: 90368 + +# ================================================ + +# Derived Property: Lowercase +# Generated from: Ll + Other_Lowercase + +0061..007A ; Lowercase # L& [26] LATIN SMALL LETTER A..LATIN SMALL LETTER Z +00AA ; Lowercase # L& FEMININE ORDINAL INDICATOR +00B5 ; Lowercase # L& MICRO SIGN +00BA ; Lowercase # L& MASCULINE ORDINAL INDICATOR +00DF..00F6 ; Lowercase # L& [24] LATIN SMALL LETTER SHARP S..LATIN SMALL LETTER O WITH DIAERESIS +00F8..00FF ; Lowercase # L& [8] LATIN SMALL LETTER O WITH STROKE..LATIN SMALL LETTER Y WITH DIAERESIS +0101 ; Lowercase # L& LATIN SMALL LETTER A WITH MACRON +0103 ; Lowercase # L& LATIN SMALL LETTER A WITH BREVE +0105 ; Lowercase # L& LATIN SMALL LETTER A WITH OGONEK +0107 ; Lowercase # L& LATIN SMALL LETTER C WITH ACUTE +0109 ; Lowercase # L& LATIN SMALL LETTER C WITH CIRCUMFLEX +010B ; Lowercase # L& LATIN SMALL LETTER C WITH DOT ABOVE +010D ; Lowercase # L& LATIN SMALL LETTER C WITH CARON +010F ; Lowercase # L& LATIN SMALL LETTER D WITH CARON +0111 ; Lowercase # L& LATIN SMALL LETTER D WITH STROKE +0113 ; Lowercase # L& LATIN SMALL LETTER E WITH MACRON +0115 ; Lowercase # L& LATIN SMALL LETTER E WITH BREVE +0117 ; Lowercase # L& LATIN SMALL LETTER E WITH DOT ABOVE +0119 ; Lowercase # L& LATIN SMALL LETTER E WITH OGONEK +011B ; Lowercase # L& LATIN SMALL LETTER E WITH CARON +011D ; Lowercase # L& LATIN SMALL LETTER G WITH CIRCUMFLEX +011F ; Lowercase # L& LATIN SMALL LETTER G WITH BREVE +0121 ; Lowercase # L& LATIN SMALL LETTER G WITH DOT ABOVE +0123 ; Lowercase # L& LATIN SMALL LETTER G WITH CEDILLA +0125 ; Lowercase # L& LATIN SMALL LETTER H WITH CIRCUMFLEX +0127 ; Lowercase # L& LATIN SMALL LETTER H WITH STROKE +0129 ; Lowercase # L& LATIN SMALL LETTER I WITH TILDE +012B ; Lowercase # L& LATIN SMALL LETTER I WITH MACRON +012D ; Lowercase # L& LATIN SMALL LETTER I WITH BREVE +012F ; Lowercase # L& LATIN SMALL LETTER I WITH OGONEK +0131 ; Lowercase # L& LATIN SMALL LETTER DOTLESS I +0133 ; Lowercase # L& LATIN SMALL LIGATURE IJ +0135 ; Lowercase # L& LATIN SMALL LETTER J WITH CIRCUMFLEX +0137..0138 ; Lowercase # L& [2] LATIN SMALL LETTER K WITH CEDILLA..LATIN SMALL LETTER KRA +013A ; Lowercase # L& LATIN SMALL LETTER L WITH ACUTE +013C ; Lowercase # L& LATIN SMALL LETTER L WITH CEDILLA +013E ; Lowercase # L& LATIN SMALL LETTER L WITH CARON +0140 ; Lowercase # L& LATIN SMALL LETTER L WITH MIDDLE DOT +0142 ; Lowercase # L& LATIN SMALL LETTER L WITH STROKE +0144 ; Lowercase # L& LATIN SMALL LETTER N WITH ACUTE +0146 ; Lowercase # L& LATIN SMALL LETTER N WITH CEDILLA +0148..0149 ; Lowercase # L& [2] LATIN SMALL LETTER N WITH CARON..LATIN SMALL LETTER N PRECEDED BY APOSTROPHE +014B ; Lowercase # L& LATIN SMALL LETTER ENG +014D ; Lowercase # L& LATIN SMALL LETTER O WITH MACRON +014F ; Lowercase # L& LATIN SMALL LETTER O WITH BREVE +0151 ; Lowercase # L& LATIN SMALL LETTER O WITH DOUBLE ACUTE +0153 ; Lowercase # L& LATIN SMALL LIGATURE OE +0155 ; Lowercase # L& LATIN SMALL LETTER R WITH ACUTE +0157 ; Lowercase # L& LATIN SMALL LETTER R WITH CEDILLA +0159 ; Lowercase # L& LATIN SMALL LETTER R WITH CARON +015B ; Lowercase # L& LATIN SMALL LETTER S WITH ACUTE +015D ; Lowercase # L& LATIN SMALL LETTER S WITH CIRCUMFLEX +015F ; Lowercase # L& LATIN SMALL LETTER S WITH CEDILLA +0161 ; Lowercase # L& LATIN SMALL LETTER S WITH CARON +0163 ; Lowercase # L& LATIN SMALL LETTER T WITH CEDILLA +0165 ; Lowercase # L& LATIN SMALL LETTER T WITH CARON +0167 ; Lowercase # L& LATIN SMALL LETTER T WITH STROKE +0169 ; Lowercase # L& LATIN SMALL LETTER U WITH TILDE +016B ; Lowercase # L& LATIN SMALL LETTER U WITH MACRON +016D ; Lowercase # L& LATIN SMALL LETTER U WITH BREVE +016F ; Lowercase # L& LATIN SMALL LETTER U WITH RING ABOVE +0171 ; Lowercase # L& LATIN SMALL LETTER U WITH DOUBLE ACUTE +0173 ; Lowercase # L& LATIN SMALL LETTER U WITH OGONEK +0175 ; Lowercase # L& LATIN SMALL LETTER W WITH CIRCUMFLEX +0177 ; Lowercase # L& LATIN SMALL LETTER Y WITH CIRCUMFLEX +017A ; Lowercase # L& LATIN SMALL LETTER Z WITH ACUTE +017C ; Lowercase # L& LATIN SMALL LETTER Z WITH DOT ABOVE +017E..0180 ; Lowercase # L& [3] LATIN SMALL LETTER Z WITH CARON..LATIN SMALL LETTER B WITH STROKE +0183 ; Lowercase # L& LATIN SMALL LETTER B WITH TOPBAR +0185 ; Lowercase # L& LATIN SMALL LETTER TONE SIX +0188 ; Lowercase # L& LATIN SMALL LETTER C WITH HOOK +018C..018D ; Lowercase # L& [2] LATIN SMALL LETTER D WITH TOPBAR..LATIN SMALL LETTER TURNED DELTA +0192 ; Lowercase # L& LATIN SMALL LETTER F WITH HOOK +0195 ; Lowercase # L& LATIN SMALL LETTER HV +0199..019B ; Lowercase # L& [3] LATIN SMALL LETTER K WITH HOOK..LATIN SMALL LETTER LAMBDA WITH STROKE +019E ; Lowercase # L& LATIN SMALL LETTER N WITH LONG RIGHT LEG +01A1 ; Lowercase # L& LATIN SMALL LETTER O WITH HORN +01A3 ; Lowercase # L& LATIN SMALL LETTER OI +01A5 ; Lowercase # L& LATIN SMALL LETTER P WITH HOOK +01A8 ; Lowercase # L& LATIN SMALL LETTER TONE TWO +01AA..01AB ; Lowercase # L& [2] LATIN LETTER REVERSED ESH LOOP..LATIN SMALL LETTER T WITH PALATAL HOOK +01AD ; Lowercase # L& LATIN SMALL LETTER T WITH HOOK +01B0 ; Lowercase # L& LATIN SMALL LETTER U WITH HORN +01B4 ; Lowercase # L& LATIN SMALL LETTER Y WITH HOOK +01B6 ; Lowercase # L& LATIN SMALL LETTER Z WITH STROKE +01B9..01BA ; Lowercase # L& [2] LATIN SMALL LETTER EZH REVERSED..LATIN SMALL LETTER EZH WITH TAIL +01BD..01BF ; Lowercase # L& [3] LATIN SMALL LETTER TONE FIVE..LATIN LETTER WYNN +01C6 ; Lowercase # L& LATIN SMALL LETTER DZ WITH CARON +01C9 ; Lowercase # L& LATIN SMALL LETTER LJ +01CC ; Lowercase # L& LATIN SMALL LETTER NJ +01CE ; Lowercase # L& LATIN SMALL LETTER A WITH CARON +01D0 ; Lowercase # L& LATIN SMALL LETTER I WITH CARON +01D2 ; Lowercase # L& LATIN SMALL LETTER O WITH CARON +01D4 ; Lowercase # L& LATIN SMALL LETTER U WITH CARON +01D6 ; Lowercase # L& LATIN SMALL LETTER U WITH DIAERESIS AND MACRON +01D8 ; Lowercase # L& LATIN SMALL LETTER U WITH DIAERESIS AND ACUTE +01DA ; Lowercase # L& LATIN SMALL LETTER U WITH DIAERESIS AND CARON +01DC..01DD ; Lowercase # L& [2] LATIN SMALL LETTER U WITH DIAERESIS AND GRAVE..LATIN SMALL LETTER TURNED E +01DF ; Lowercase # L& LATIN SMALL LETTER A WITH DIAERESIS AND MACRON +01E1 ; Lowercase # L& LATIN SMALL LETTER A WITH DOT ABOVE AND MACRON +01E3 ; Lowercase # L& LATIN SMALL LETTER AE WITH MACRON +01E5 ; Lowercase # L& LATIN SMALL LETTER G WITH STROKE +01E7 ; Lowercase # L& LATIN SMALL LETTER G WITH CARON +01E9 ; Lowercase # L& LATIN SMALL LETTER K WITH CARON +01EB ; Lowercase # L& LATIN SMALL LETTER O WITH OGONEK +01ED ; Lowercase # L& LATIN SMALL LETTER O WITH OGONEK AND MACRON +01EF..01F0 ; Lowercase # L& [2] LATIN SMALL LETTER EZH WITH CARON..LATIN SMALL LETTER J WITH CARON +01F3 ; Lowercase # L& LATIN SMALL LETTER DZ +01F5 ; Lowercase # L& LATIN SMALL LETTER G WITH ACUTE +01F9 ; Lowercase # L& LATIN SMALL LETTER N WITH GRAVE +01FB ; Lowercase # L& LATIN SMALL LETTER A WITH RING ABOVE AND ACUTE +01FD ; Lowercase # L& LATIN SMALL LETTER AE WITH ACUTE +01FF ; Lowercase # L& LATIN SMALL LETTER O WITH STROKE AND ACUTE +0201 ; Lowercase # L& LATIN SMALL LETTER A WITH DOUBLE GRAVE +0203 ; Lowercase # L& LATIN SMALL LETTER A WITH INVERTED BREVE +0205 ; Lowercase # L& LATIN SMALL LETTER E WITH DOUBLE GRAVE +0207 ; Lowercase # L& LATIN SMALL LETTER E WITH INVERTED BREVE +0209 ; Lowercase # L& LATIN SMALL LETTER I WITH DOUBLE GRAVE +020B ; Lowercase # L& LATIN SMALL LETTER I WITH INVERTED BREVE +020D ; Lowercase # L& LATIN SMALL LETTER O WITH DOUBLE GRAVE +020F ; Lowercase # L& LATIN SMALL LETTER O WITH INVERTED BREVE +0211 ; Lowercase # L& LATIN SMALL LETTER R WITH DOUBLE GRAVE +0213 ; Lowercase # L& LATIN SMALL LETTER R WITH INVERTED BREVE +0215 ; Lowercase # L& LATIN SMALL LETTER U WITH DOUBLE GRAVE +0217 ; Lowercase # L& LATIN SMALL LETTER U WITH INVERTED BREVE +0219 ; Lowercase # L& LATIN SMALL LETTER S WITH COMMA BELOW +021B ; Lowercase # L& LATIN SMALL LETTER T WITH COMMA BELOW +021D ; Lowercase # L& LATIN SMALL LETTER YOGH +021F ; Lowercase # L& LATIN SMALL LETTER H WITH CARON +0223 ; Lowercase # L& LATIN SMALL LETTER OU +0225 ; Lowercase # L& LATIN SMALL LETTER Z WITH HOOK +0227 ; Lowercase # L& LATIN SMALL LETTER A WITH DOT ABOVE +0229 ; Lowercase # L& LATIN SMALL LETTER E WITH CEDILLA +022B ; Lowercase # L& LATIN SMALL LETTER O WITH DIAERESIS AND MACRON +022D ; Lowercase # L& LATIN SMALL LETTER O WITH TILDE AND MACRON +022F ; Lowercase # L& LATIN SMALL LETTER O WITH DOT ABOVE +0231 ; Lowercase # L& LATIN SMALL LETTER O WITH DOT ABOVE AND MACRON +0233 ; Lowercase # L& LATIN SMALL LETTER Y WITH MACRON +0250..02AD ; Lowercase # L& [94] LATIN SMALL LETTER TURNED A..LATIN LETTER BIDENTAL PERCUSSIVE +02B0..02B8 ; Lowercase # Lm [9] MODIFIER LETTER SMALL H..MODIFIER LETTER SMALL Y +02C0..02C1 ; Lowercase # Lm [2] MODIFIER LETTER GLOTTAL STOP..MODIFIER LETTER REVERSED GLOTTAL STOP +02E0..02E4 ; Lowercase # Lm [5] MODIFIER LETTER SMALL GAMMA..MODIFIER LETTER SMALL REVERSED GLOTTAL STOP +0345 ; Lowercase # Mn COMBINING GREEK YPOGEGRAMMENI +037A ; Lowercase # Lm GREEK YPOGEGRAMMENI +0390 ; Lowercase # L& GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS +03AC..03CE ; Lowercase # L& [35] GREEK SMALL LETTER ALPHA WITH TONOS..GREEK SMALL LETTER OMEGA WITH TONOS +03D0..03D1 ; Lowercase # L& [2] GREEK BETA SYMBOL..GREEK THETA SYMBOL +03D5..03D7 ; Lowercase # L& [3] GREEK PHI SYMBOL..GREEK KAI SYMBOL +03D9 ; Lowercase # L& GREEK SMALL LETTER ARCHAIC KOPPA +03DB ; Lowercase # L& GREEK SMALL LETTER STIGMA +03DD ; Lowercase # L& GREEK SMALL LETTER DIGAMMA +03DF ; Lowercase # L& GREEK SMALL LETTER KOPPA +03E1 ; Lowercase # L& GREEK SMALL LETTER SAMPI +03E3 ; Lowercase # L& COPTIC SMALL LETTER SHEI +03E5 ; Lowercase # L& COPTIC SMALL LETTER FEI +03E7 ; Lowercase # L& COPTIC SMALL LETTER KHEI +03E9 ; Lowercase # L& COPTIC SMALL LETTER HORI +03EB ; Lowercase # L& COPTIC SMALL LETTER GANGIA +03ED ; Lowercase # L& COPTIC SMALL LETTER SHIMA +03EF..03F3 ; Lowercase # L& [5] COPTIC SMALL LETTER DEI..GREEK LETTER YOT +03F5 ; Lowercase # L& GREEK LUNATE EPSILON SYMBOL +0430..045F ; Lowercase # L& [48] CYRILLIC SMALL LETTER A..CYRILLIC SMALL LETTER DZHE +0461 ; Lowercase # L& CYRILLIC SMALL LETTER OMEGA +0463 ; Lowercase # L& CYRILLIC SMALL LETTER YAT +0465 ; Lowercase # L& CYRILLIC SMALL LETTER IOTIFIED E +0467 ; Lowercase # L& CYRILLIC SMALL LETTER LITTLE YUS +0469 ; Lowercase # L& CYRILLIC SMALL LETTER IOTIFIED LITTLE YUS +046B ; Lowercase # L& CYRILLIC SMALL LETTER BIG YUS +046D ; Lowercase # L& CYRILLIC SMALL LETTER IOTIFIED BIG YUS +046F ; Lowercase # L& CYRILLIC SMALL LETTER KSI +0471 ; Lowercase # L& CYRILLIC SMALL LETTER PSI +0473 ; Lowercase # L& CYRILLIC SMALL LETTER FITA +0475 ; Lowercase # L& CYRILLIC SMALL LETTER IZHITSA +0477 ; Lowercase # L& CYRILLIC SMALL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT +0479 ; Lowercase # L& CYRILLIC SMALL LETTER UK +047B ; Lowercase # L& CYRILLIC SMALL LETTER ROUND OMEGA +047D ; Lowercase # L& CYRILLIC SMALL LETTER OMEGA WITH TITLO +047F ; Lowercase # L& CYRILLIC SMALL LETTER OT +0481 ; Lowercase # L& CYRILLIC SMALL LETTER KOPPA +048B ; Lowercase # L& CYRILLIC SMALL LETTER SHORT I WITH TAIL +048D ; Lowercase # L& CYRILLIC SMALL LETTER SEMISOFT SIGN +048F ; Lowercase # L& CYRILLIC SMALL LETTER ER WITH TICK +0491 ; Lowercase # L& CYRILLIC SMALL LETTER GHE WITH UPTURN +0493 ; Lowercase # L& CYRILLIC SMALL LETTER GHE WITH STROKE +0495 ; Lowercase # L& CYRILLIC SMALL LETTER GHE WITH MIDDLE HOOK +0497 ; Lowercase # L& CYRILLIC SMALL LETTER ZHE WITH DESCENDER +0499 ; Lowercase # L& CYRILLIC SMALL LETTER ZE WITH DESCENDER +049B ; Lowercase # L& CYRILLIC SMALL LETTER KA WITH DESCENDER +049D ; Lowercase # L& CYRILLIC SMALL LETTER KA WITH VERTICAL STROKE +049F ; Lowercase # L& CYRILLIC SMALL LETTER KA WITH STROKE +04A1 ; Lowercase # L& CYRILLIC SMALL LETTER BASHKIR KA +04A3 ; Lowercase # L& CYRILLIC SMALL LETTER EN WITH DESCENDER +04A5 ; Lowercase # L& CYRILLIC SMALL LIGATURE EN GHE +04A7 ; Lowercase # L& CYRILLIC SMALL LETTER PE WITH MIDDLE HOOK +04A9 ; Lowercase # L& CYRILLIC SMALL LETTER ABKHASIAN HA +04AB ; Lowercase # L& CYRILLIC SMALL LETTER ES WITH DESCENDER +04AD ; Lowercase # L& CYRILLIC SMALL LETTER TE WITH DESCENDER +04AF ; Lowercase # L& CYRILLIC SMALL LETTER STRAIGHT U +04B1 ; Lowercase # L& CYRILLIC SMALL LETTER STRAIGHT U WITH STROKE +04B3 ; Lowercase # L& CYRILLIC SMALL LETTER HA WITH DESCENDER +04B5 ; Lowercase # L& CYRILLIC SMALL LIGATURE TE TSE +04B7 ; Lowercase # L& CYRILLIC SMALL LETTER CHE WITH DESCENDER +04B9 ; Lowercase # L& CYRILLIC SMALL LETTER CHE WITH VERTICAL STROKE +04BB ; Lowercase # L& CYRILLIC SMALL LETTER SHHA +04BD ; Lowercase # L& CYRILLIC SMALL LETTER ABKHASIAN CHE +04BF ; Lowercase # L& CYRILLIC SMALL LETTER ABKHASIAN CHE WITH DESCENDER +04C2 ; Lowercase # L& CYRILLIC SMALL LETTER ZHE WITH BREVE +04C4 ; Lowercase # L& CYRILLIC SMALL LETTER KA WITH HOOK +04C6 ; Lowercase # L& CYRILLIC SMALL LETTER EL WITH TAIL +04C8 ; Lowercase # L& CYRILLIC SMALL LETTER EN WITH HOOK +04CA ; Lowercase # L& CYRILLIC SMALL LETTER EN WITH TAIL +04CC ; Lowercase # L& CYRILLIC SMALL LETTER KHAKASSIAN CHE +04CE ; Lowercase # L& CYRILLIC SMALL LETTER EM WITH TAIL +04D1 ; Lowercase # L& CYRILLIC SMALL LETTER A WITH BREVE +04D3 ; Lowercase # L& CYRILLIC SMALL LETTER A WITH DIAERESIS +04D5 ; Lowercase # L& CYRILLIC SMALL LIGATURE A IE +04D7 ; Lowercase # L& CYRILLIC SMALL LETTER IE WITH BREVE +04D9 ; Lowercase # L& CYRILLIC SMALL LETTER SCHWA +04DB ; Lowercase # L& CYRILLIC SMALL LETTER SCHWA WITH DIAERESIS +04DD ; Lowercase # L& CYRILLIC SMALL LETTER ZHE WITH DIAERESIS +04DF ; Lowercase # L& CYRILLIC SMALL LETTER ZE WITH DIAERESIS +04E1 ; Lowercase # L& CYRILLIC SMALL LETTER ABKHASIAN DZE +04E3 ; Lowercase # L& CYRILLIC SMALL LETTER I WITH MACRON +04E5 ; Lowercase # L& CYRILLIC SMALL LETTER I WITH DIAERESIS +04E7 ; Lowercase # L& CYRILLIC SMALL LETTER O WITH DIAERESIS +04E9 ; Lowercase # L& CYRILLIC SMALL LETTER BARRED O +04EB ; Lowercase # L& CYRILLIC SMALL LETTER BARRED O WITH DIAERESIS +04ED ; Lowercase # L& CYRILLIC SMALL LETTER E WITH DIAERESIS +04EF ; Lowercase # L& CYRILLIC SMALL LETTER U WITH MACRON +04F1 ; Lowercase # L& CYRILLIC SMALL LETTER U WITH DIAERESIS +04F3 ; Lowercase # L& CYRILLIC SMALL LETTER U WITH DOUBLE ACUTE +04F5 ; Lowercase # L& CYRILLIC SMALL LETTER CHE WITH DIAERESIS +04F9 ; Lowercase # L& CYRILLIC SMALL LETTER YERU WITH DIAERESIS +0501 ; Lowercase # L& CYRILLIC SMALL LETTER KOMI DE +0503 ; Lowercase # L& CYRILLIC SMALL LETTER KOMI DJE +0505 ; Lowercase # L& CYRILLIC SMALL LETTER KOMI ZJE +0507 ; Lowercase # L& CYRILLIC SMALL LETTER KOMI DZJE +0509 ; Lowercase # L& CYRILLIC SMALL LETTER KOMI LJE +050B ; Lowercase # L& CYRILLIC SMALL LETTER KOMI NJE +050D ; Lowercase # L& CYRILLIC SMALL LETTER KOMI SJE +050F ; Lowercase # L& CYRILLIC SMALL LETTER KOMI TJE +0561..0587 ; Lowercase # L& [39] ARMENIAN SMALL LETTER AYB..ARMENIAN SMALL LIGATURE ECH YIWN +1E01 ; Lowercase # L& LATIN SMALL LETTER A WITH RING BELOW +1E03 ; Lowercase # L& LATIN SMALL LETTER B WITH DOT ABOVE +1E05 ; Lowercase # L& LATIN SMALL LETTER B WITH DOT BELOW +1E07 ; Lowercase # L& LATIN SMALL LETTER B WITH LINE BELOW +1E09 ; Lowercase # L& LATIN SMALL LETTER C WITH CEDILLA AND ACUTE +1E0B ; Lowercase # L& LATIN SMALL LETTER D WITH DOT ABOVE +1E0D ; Lowercase # L& LATIN SMALL LETTER D WITH DOT BELOW +1E0F ; Lowercase # L& LATIN SMALL LETTER D WITH LINE BELOW +1E11 ; Lowercase # L& LATIN SMALL LETTER D WITH CEDILLA +1E13 ; Lowercase # L& LATIN SMALL LETTER D WITH CIRCUMFLEX BELOW +1E15 ; Lowercase # L& LATIN SMALL LETTER E WITH MACRON AND GRAVE +1E17 ; Lowercase # L& LATIN SMALL LETTER E WITH MACRON AND ACUTE +1E19 ; Lowercase # L& LATIN SMALL LETTER E WITH CIRCUMFLEX BELOW +1E1B ; Lowercase # L& LATIN SMALL LETTER E WITH TILDE BELOW +1E1D ; Lowercase # L& LATIN SMALL LETTER E WITH CEDILLA AND BREVE +1E1F ; Lowercase # L& LATIN SMALL LETTER F WITH DOT ABOVE +1E21 ; Lowercase # L& LATIN SMALL LETTER G WITH MACRON +1E23 ; Lowercase # L& LATIN SMALL LETTER H WITH DOT ABOVE +1E25 ; Lowercase # L& LATIN SMALL LETTER H WITH DOT BELOW +1E27 ; Lowercase # L& LATIN SMALL LETTER H WITH DIAERESIS +1E29 ; Lowercase # L& LATIN SMALL LETTER H WITH CEDILLA +1E2B ; Lowercase # L& LATIN SMALL LETTER H WITH BREVE BELOW +1E2D ; Lowercase # L& LATIN SMALL LETTER I WITH TILDE BELOW +1E2F ; Lowercase # L& LATIN SMALL LETTER I WITH DIAERESIS AND ACUTE +1E31 ; Lowercase # L& LATIN SMALL LETTER K WITH ACUTE +1E33 ; Lowercase # L& LATIN SMALL LETTER K WITH DOT BELOW +1E35 ; Lowercase # L& LATIN SMALL LETTER K WITH LINE BELOW +1E37 ; Lowercase # L& LATIN SMALL LETTER L WITH DOT BELOW +1E39 ; Lowercase # L& LATIN SMALL LETTER L WITH DOT BELOW AND MACRON +1E3B ; Lowercase # L& LATIN SMALL LETTER L WITH LINE BELOW +1E3D ; Lowercase # L& LATIN SMALL LETTER L WITH CIRCUMFLEX BELOW +1E3F ; Lowercase # L& LATIN SMALL LETTER M WITH ACUTE +1E41 ; Lowercase # L& LATIN SMALL LETTER M WITH DOT ABOVE +1E43 ; Lowercase # L& LATIN SMALL LETTER M WITH DOT BELOW +1E45 ; Lowercase # L& LATIN SMALL LETTER N WITH DOT ABOVE +1E47 ; Lowercase # L& LATIN SMALL LETTER N WITH DOT BELOW +1E49 ; Lowercase # L& LATIN SMALL LETTER N WITH LINE BELOW +1E4B ; Lowercase # L& LATIN SMALL LETTER N WITH CIRCUMFLEX BELOW +1E4D ; Lowercase # L& LATIN SMALL LETTER O WITH TILDE AND ACUTE +1E4F ; Lowercase # L& LATIN SMALL LETTER O WITH TILDE AND DIAERESIS +1E51 ; Lowercase # L& LATIN SMALL LETTER O WITH MACRON AND GRAVE +1E53 ; Lowercase # L& LATIN SMALL LETTER O WITH MACRON AND ACUTE +1E55 ; Lowercase # L& LATIN SMALL LETTER P WITH ACUTE +1E57 ; Lowercase # L& LATIN SMALL LETTER P WITH DOT ABOVE +1E59 ; Lowercase # L& LATIN SMALL LETTER R WITH DOT ABOVE +1E5B ; Lowercase # L& LATIN SMALL LETTER R WITH DOT BELOW +1E5D ; Lowercase # L& LATIN SMALL LETTER R WITH DOT BELOW AND MACRON +1E5F ; Lowercase # L& LATIN SMALL LETTER R WITH LINE BELOW +1E61 ; Lowercase # L& LATIN SMALL LETTER S WITH DOT ABOVE +1E63 ; Lowercase # L& LATIN SMALL LETTER S WITH DOT BELOW +1E65 ; Lowercase # L& LATIN SMALL LETTER S WITH ACUTE AND DOT ABOVE +1E67 ; Lowercase # L& LATIN SMALL LETTER S WITH CARON AND DOT ABOVE +1E69 ; Lowercase # L& LATIN SMALL LETTER S WITH DOT BELOW AND DOT ABOVE +1E6B ; Lowercase # L& LATIN SMALL LETTER T WITH DOT ABOVE +1E6D ; Lowercase # L& LATIN SMALL LETTER T WITH DOT BELOW +1E6F ; Lowercase # L& LATIN SMALL LETTER T WITH LINE BELOW +1E71 ; Lowercase # L& LATIN SMALL LETTER T WITH CIRCUMFLEX BELOW +1E73 ; Lowercase # L& LATIN SMALL LETTER U WITH DIAERESIS BELOW +1E75 ; Lowercase # L& LATIN SMALL LETTER U WITH TILDE BELOW +1E77 ; Lowercase # L& LATIN SMALL LETTER U WITH CIRCUMFLEX BELOW +1E79 ; Lowercase # L& LATIN SMALL LETTER U WITH TILDE AND ACUTE +1E7B ; Lowercase # L& LATIN SMALL LETTER U WITH MACRON AND DIAERESIS +1E7D ; Lowercase # L& LATIN SMALL LETTER V WITH TILDE +1E7F ; Lowercase # L& LATIN SMALL LETTER V WITH DOT BELOW +1E81 ; Lowercase # L& LATIN SMALL LETTER W WITH GRAVE +1E83 ; Lowercase # L& LATIN SMALL LETTER W WITH ACUTE +1E85 ; Lowercase # L& LATIN SMALL LETTER W WITH DIAERESIS +1E87 ; Lowercase # L& LATIN SMALL LETTER W WITH DOT ABOVE +1E89 ; Lowercase # L& LATIN SMALL LETTER W WITH DOT BELOW +1E8B ; Lowercase # L& LATIN SMALL LETTER X WITH DOT ABOVE +1E8D ; Lowercase # L& LATIN SMALL LETTER X WITH DIAERESIS +1E8F ; Lowercase # L& LATIN SMALL LETTER Y WITH DOT ABOVE +1E91 ; Lowercase # L& LATIN SMALL LETTER Z WITH CIRCUMFLEX +1E93 ; Lowercase # L& LATIN SMALL LETTER Z WITH DOT BELOW +1E95..1E9B ; Lowercase # L& [7] LATIN SMALL LETTER Z WITH LINE BELOW..LATIN SMALL LETTER LONG S WITH DOT ABOVE +1EA1 ; Lowercase # L& LATIN SMALL LETTER A WITH DOT BELOW +1EA3 ; Lowercase # L& LATIN SMALL LETTER A WITH HOOK ABOVE +1EA5 ; Lowercase # L& LATIN SMALL LETTER A WITH CIRCUMFLEX AND ACUTE +1EA7 ; Lowercase # L& LATIN SMALL LETTER A WITH CIRCUMFLEX AND GRAVE +1EA9 ; Lowercase # L& LATIN SMALL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE +1EAB ; Lowercase # L& LATIN SMALL LETTER A WITH CIRCUMFLEX AND TILDE +1EAD ; Lowercase # L& LATIN SMALL LETTER A WITH CIRCUMFLEX AND DOT BELOW +1EAF ; Lowercase # L& LATIN SMALL LETTER A WITH BREVE AND ACUTE +1EB1 ; Lowercase # L& LATIN SMALL LETTER A WITH BREVE AND GRAVE +1EB3 ; Lowercase # L& LATIN SMALL LETTER A WITH BREVE AND HOOK ABOVE +1EB5 ; Lowercase # L& LATIN SMALL LETTER A WITH BREVE AND TILDE +1EB7 ; Lowercase # L& LATIN SMALL LETTER A WITH BREVE AND DOT BELOW +1EB9 ; Lowercase # L& LATIN SMALL LETTER E WITH DOT BELOW +1EBB ; Lowercase # L& LATIN SMALL LETTER E WITH HOOK ABOVE +1EBD ; Lowercase # L& LATIN SMALL LETTER E WITH TILDE +1EBF ; Lowercase # L& LATIN SMALL LETTER E WITH CIRCUMFLEX AND ACUTE +1EC1 ; Lowercase # L& LATIN SMALL LETTER E WITH CIRCUMFLEX AND GRAVE +1EC3 ; Lowercase # L& LATIN SMALL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE +1EC5 ; Lowercase # L& LATIN SMALL LETTER E WITH CIRCUMFLEX AND TILDE +1EC7 ; Lowercase # L& LATIN SMALL LETTER E WITH CIRCUMFLEX AND DOT BELOW +1EC9 ; Lowercase # L& LATIN SMALL LETTER I WITH HOOK ABOVE +1ECB ; Lowercase # L& LATIN SMALL LETTER I WITH DOT BELOW +1ECD ; Lowercase # L& LATIN SMALL LETTER O WITH DOT BELOW +1ECF ; Lowercase # L& LATIN SMALL LETTER O WITH HOOK ABOVE +1ED1 ; Lowercase # L& LATIN SMALL LETTER O WITH CIRCUMFLEX AND ACUTE +1ED3 ; Lowercase # L& LATIN SMALL LETTER O WITH CIRCUMFLEX AND GRAVE +1ED5 ; Lowercase # L& LATIN SMALL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE +1ED7 ; Lowercase # L& LATIN SMALL LETTER O WITH CIRCUMFLEX AND TILDE +1ED9 ; Lowercase # L& LATIN SMALL LETTER O WITH CIRCUMFLEX AND DOT BELOW +1EDB ; Lowercase # L& LATIN SMALL LETTER O WITH HORN AND ACUTE +1EDD ; Lowercase # L& LATIN SMALL LETTER O WITH HORN AND GRAVE +1EDF ; Lowercase # L& LATIN SMALL LETTER O WITH HORN AND HOOK ABOVE +1EE1 ; Lowercase # L& LATIN SMALL LETTER O WITH HORN AND TILDE +1EE3 ; Lowercase # L& LATIN SMALL LETTER O WITH HORN AND DOT BELOW +1EE5 ; Lowercase # L& LATIN SMALL LETTER U WITH DOT BELOW +1EE7 ; Lowercase # L& LATIN SMALL LETTER U WITH HOOK ABOVE +1EE9 ; Lowercase # L& LATIN SMALL LETTER U WITH HORN AND ACUTE +1EEB ; Lowercase # L& LATIN SMALL LETTER U WITH HORN AND GRAVE +1EED ; Lowercase # L& LATIN SMALL LETTER U WITH HORN AND HOOK ABOVE +1EEF ; Lowercase # L& LATIN SMALL LETTER U WITH HORN AND TILDE +1EF1 ; Lowercase # L& LATIN SMALL LETTER U WITH HORN AND DOT BELOW +1EF3 ; Lowercase # L& LATIN SMALL LETTER Y WITH GRAVE +1EF5 ; Lowercase # L& LATIN SMALL LETTER Y WITH DOT BELOW +1EF7 ; Lowercase # L& LATIN SMALL LETTER Y WITH HOOK ABOVE +1EF9 ; Lowercase # L& LATIN SMALL LETTER Y WITH TILDE +1F00..1F07 ; Lowercase # L& [8] GREEK SMALL LETTER ALPHA WITH PSILI..GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI +1F10..1F15 ; Lowercase # L& [6] GREEK SMALL LETTER EPSILON WITH PSILI..GREEK SMALL LETTER EPSILON WITH DASIA AND OXIA +1F20..1F27 ; Lowercase # L& [8] GREEK SMALL LETTER ETA WITH PSILI..GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI +1F30..1F37 ; Lowercase # L& [8] GREEK SMALL LETTER IOTA WITH PSILI..GREEK SMALL LETTER IOTA WITH DASIA AND PERISPOMENI +1F40..1F45 ; Lowercase # L& [6] GREEK SMALL LETTER OMICRON WITH PSILI..GREEK SMALL LETTER OMICRON WITH DASIA AND OXIA +1F50..1F57 ; Lowercase # L& [8] GREEK SMALL LETTER UPSILON WITH PSILI..GREEK SMALL LETTER UPSILON WITH DASIA AND PERISPOMENI +1F60..1F67 ; Lowercase # L& [8] GREEK SMALL LETTER OMEGA WITH PSILI..GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI +1F70..1F7D ; Lowercase # L& [14] GREEK SMALL LETTER ALPHA WITH VARIA..GREEK SMALL LETTER OMEGA WITH OXIA +1F80..1F87 ; Lowercase # L& [8] GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI..GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI +1F90..1F97 ; Lowercase # L& [8] GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI..GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI +1FA0..1FA7 ; Lowercase # L& [8] GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI..GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI +1FB0..1FB4 ; Lowercase # L& [5] GREEK SMALL LETTER ALPHA WITH VRACHY..GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI +1FB6..1FB7 ; Lowercase # L& [2] GREEK SMALL LETTER ALPHA WITH PERISPOMENI..GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI +1FBE ; Lowercase # L& GREEK PROSGEGRAMMENI +1FC2..1FC4 ; Lowercase # L& [3] GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI +1FC6..1FC7 ; Lowercase # L& [2] GREEK SMALL LETTER ETA WITH PERISPOMENI..GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI +1FD0..1FD3 ; Lowercase # L& [4] GREEK SMALL LETTER IOTA WITH VRACHY..GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA +1FD6..1FD7 ; Lowercase # L& [2] GREEK SMALL LETTER IOTA WITH PERISPOMENI..GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI +1FE0..1FE7 ; Lowercase # L& [8] GREEK SMALL LETTER UPSILON WITH VRACHY..GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI +1FF2..1FF4 ; Lowercase # L& [3] GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI +1FF6..1FF7 ; Lowercase # L& [2] GREEK SMALL LETTER OMEGA WITH PERISPOMENI..GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI +2071 ; Lowercase # L& SUPERSCRIPT LATIN SMALL LETTER I +207F ; Lowercase # L& SUPERSCRIPT LATIN SMALL LETTER N +210A ; Lowercase # L& SCRIPT SMALL G +210E..210F ; Lowercase # L& [2] PLANCK CONSTANT..PLANCK CONSTANT OVER TWO PI +2113 ; Lowercase # L& SCRIPT SMALL L +212F ; Lowercase # L& SCRIPT SMALL E +2134 ; Lowercase # L& SCRIPT SMALL O +2139 ; Lowercase # L& INFORMATION SOURCE +213D ; Lowercase # L& DOUBLE-STRUCK SMALL GAMMA +2146..2149 ; Lowercase # L& [4] DOUBLE-STRUCK ITALIC SMALL D..DOUBLE-STRUCK ITALIC SMALL J +2170..217F ; Lowercase # Nl [16] SMALL ROMAN NUMERAL ONE..SMALL ROMAN NUMERAL ONE THOUSAND +24D0..24E9 ; Lowercase # So [26] CIRCLED LATIN SMALL LETTER A..CIRCLED LATIN SMALL LETTER Z +FB00..FB06 ; Lowercase # L& [7] LATIN SMALL LIGATURE FF..LATIN SMALL LIGATURE ST +FB13..FB17 ; Lowercase # L& [5] ARMENIAN SMALL LIGATURE MEN NOW..ARMENIAN SMALL LIGATURE MEN XEH +FF41..FF5A ; Lowercase # L& [26] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH LATIN SMALL LETTER Z +10428..1044D ; Lowercase # L& [38] DESERET SMALL LETTER LONG I..DESERET SMALL LETTER ENG +1D41A..1D433 ; Lowercase # L& [26] MATHEMATICAL BOLD SMALL A..MATHEMATICAL BOLD SMALL Z +1D44E..1D454 ; Lowercase # L& [7] MATHEMATICAL ITALIC SMALL A..MATHEMATICAL ITALIC SMALL G +1D456..1D467 ; Lowercase # L& [18] MATHEMATICAL ITALIC SMALL I..MATHEMATICAL ITALIC SMALL Z +1D482..1D49B ; Lowercase # L& [26] MATHEMATICAL BOLD ITALIC SMALL A..MATHEMATICAL BOLD ITALIC SMALL Z +1D4B6..1D4B9 ; Lowercase # L& [4] MATHEMATICAL SCRIPT SMALL A..MATHEMATICAL SCRIPT SMALL D +1D4BB ; Lowercase # L& MATHEMATICAL SCRIPT SMALL F +1D4BD..1D4C0 ; Lowercase # L& [4] MATHEMATICAL SCRIPT SMALL H..MATHEMATICAL SCRIPT SMALL K +1D4C2..1D4C3 ; Lowercase # L& [2] MATHEMATICAL SCRIPT SMALL M..MATHEMATICAL SCRIPT SMALL N +1D4C5..1D4CF ; Lowercase # L& [11] MATHEMATICAL SCRIPT SMALL P..MATHEMATICAL SCRIPT SMALL Z +1D4EA..1D503 ; Lowercase # L& [26] MATHEMATICAL BOLD SCRIPT SMALL A..MATHEMATICAL BOLD SCRIPT SMALL Z +1D51E..1D537 ; Lowercase # L& [26] MATHEMATICAL FRAKTUR SMALL A..MATHEMATICAL FRAKTUR SMALL Z +1D552..1D56B ; Lowercase # L& [26] MATHEMATICAL DOUBLE-STRUCK SMALL A..MATHEMATICAL DOUBLE-STRUCK SMALL Z +1D586..1D59F ; Lowercase # L& [26] MATHEMATICAL BOLD FRAKTUR SMALL A..MATHEMATICAL BOLD FRAKTUR SMALL Z +1D5BA..1D5D3 ; Lowercase # L& [26] MATHEMATICAL SANS-SERIF SMALL A..MATHEMATICAL SANS-SERIF SMALL Z +1D5EE..1D607 ; Lowercase # L& [26] MATHEMATICAL SANS-SERIF BOLD SMALL A..MATHEMATICAL SANS-SERIF BOLD SMALL Z +1D622..1D63B ; Lowercase # L& [26] MATHEMATICAL SANS-SERIF ITALIC SMALL A..MATHEMATICAL SANS-SERIF ITALIC SMALL Z +1D656..1D66F ; Lowercase # L& [26] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL A..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL Z +1D68A..1D6A3 ; Lowercase # L& [26] MATHEMATICAL MONOSPACE SMALL A..MATHEMATICAL MONOSPACE SMALL Z +1D6C2..1D6DA ; Lowercase # L& [25] MATHEMATICAL BOLD SMALL ALPHA..MATHEMATICAL BOLD SMALL OMEGA +1D6DC..1D6E1 ; Lowercase # L& [6] MATHEMATICAL BOLD EPSILON SYMBOL..MATHEMATICAL BOLD PI SYMBOL +1D6FC..1D714 ; Lowercase # L& [25] MATHEMATICAL ITALIC SMALL ALPHA..MATHEMATICAL ITALIC SMALL OMEGA +1D716..1D71B ; Lowercase # L& [6] MATHEMATICAL ITALIC EPSILON SYMBOL..MATHEMATICAL ITALIC PI SYMBOL +1D736..1D74E ; Lowercase # L& [25] MATHEMATICAL BOLD ITALIC SMALL ALPHA..MATHEMATICAL BOLD ITALIC SMALL OMEGA +1D750..1D755 ; Lowercase # L& [6] MATHEMATICAL BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD ITALIC PI SYMBOL +1D770..1D788 ; Lowercase # L& [25] MATHEMATICAL SANS-SERIF BOLD SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD SMALL OMEGA +1D78A..1D78F ; Lowercase # L& [6] MATHEMATICAL SANS-SERIF BOLD EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD PI SYMBOL +1D7AA..1D7C2 ; Lowercase # L& [25] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL OMEGA +1D7C4..1D7C9 ; Lowercase # L& [6] MATHEMATICAL SANS-SERIF BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC PI SYMBOL + +# Total code points: 1410 + +# ================================================ + +# Derived Property: Uppercase +# Generated from: Lu + Other_Uppercase + +0041..005A ; Uppercase # L& [26] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER Z +00C0..00D6 ; Uppercase # L& [23] LATIN CAPITAL LETTER A WITH GRAVE..LATIN CAPITAL LETTER O WITH DIAERESIS +00D8..00DE ; Uppercase # L& [7] LATIN CAPITAL LETTER O WITH STROKE..LATIN CAPITAL LETTER THORN +0100 ; Uppercase # L& LATIN CAPITAL LETTER A WITH MACRON +0102 ; Uppercase # L& LATIN CAPITAL LETTER A WITH BREVE +0104 ; Uppercase # L& LATIN CAPITAL LETTER A WITH OGONEK +0106 ; Uppercase # L& LATIN CAPITAL LETTER C WITH ACUTE +0108 ; Uppercase # L& LATIN CAPITAL LETTER C WITH CIRCUMFLEX +010A ; Uppercase # L& LATIN CAPITAL LETTER C WITH DOT ABOVE +010C ; Uppercase # L& LATIN CAPITAL LETTER C WITH CARON +010E ; Uppercase # L& LATIN CAPITAL LETTER D WITH CARON +0110 ; Uppercase # L& LATIN CAPITAL LETTER D WITH STROKE +0112 ; Uppercase # L& LATIN CAPITAL LETTER E WITH MACRON +0114 ; Uppercase # L& LATIN CAPITAL LETTER E WITH BREVE +0116 ; Uppercase # L& LATIN CAPITAL LETTER E WITH DOT ABOVE +0118 ; Uppercase # L& LATIN CAPITAL LETTER E WITH OGONEK +011A ; Uppercase # L& LATIN CAPITAL LETTER E WITH CARON +011C ; Uppercase # L& LATIN CAPITAL LETTER G WITH CIRCUMFLEX +011E ; Uppercase # L& LATIN CAPITAL LETTER G WITH BREVE +0120 ; Uppercase # L& LATIN CAPITAL LETTER G WITH DOT ABOVE +0122 ; Uppercase # L& LATIN CAPITAL LETTER G WITH CEDILLA +0124 ; Uppercase # L& LATIN CAPITAL LETTER H WITH CIRCUMFLEX +0126 ; Uppercase # L& LATIN CAPITAL LETTER H WITH STROKE +0128 ; Uppercase # L& LATIN CAPITAL LETTER I WITH TILDE +012A ; Uppercase # L& LATIN CAPITAL LETTER I WITH MACRON +012C ; Uppercase # L& LATIN CAPITAL LETTER I WITH BREVE +012E ; Uppercase # L& LATIN CAPITAL LETTER I WITH OGONEK +0130 ; Uppercase # L& LATIN CAPITAL LETTER I WITH DOT ABOVE +0132 ; Uppercase # L& LATIN CAPITAL LIGATURE IJ +0134 ; Uppercase # L& LATIN CAPITAL LETTER J WITH CIRCUMFLEX +0136 ; Uppercase # L& LATIN CAPITAL LETTER K WITH CEDILLA +0139 ; Uppercase # L& LATIN CAPITAL LETTER L WITH ACUTE +013B ; Uppercase # L& LATIN CAPITAL LETTER L WITH CEDILLA +013D ; Uppercase # L& LATIN CAPITAL LETTER L WITH CARON +013F ; Uppercase # L& LATIN CAPITAL LETTER L WITH MIDDLE DOT +0141 ; Uppercase # L& LATIN CAPITAL LETTER L WITH STROKE +0143 ; Uppercase # L& LATIN CAPITAL LETTER N WITH ACUTE +0145 ; Uppercase # L& LATIN CAPITAL LETTER N WITH CEDILLA +0147 ; Uppercase # L& LATIN CAPITAL LETTER N WITH CARON +014A ; Uppercase # L& LATIN CAPITAL LETTER ENG +014C ; Uppercase # L& LATIN CAPITAL LETTER O WITH MACRON +014E ; Uppercase # L& LATIN CAPITAL LETTER O WITH BREVE +0150 ; Uppercase # L& LATIN CAPITAL LETTER O WITH DOUBLE ACUTE +0152 ; Uppercase # L& LATIN CAPITAL LIGATURE OE +0154 ; Uppercase # L& LATIN CAPITAL LETTER R WITH ACUTE +0156 ; Uppercase # L& LATIN CAPITAL LETTER R WITH CEDILLA +0158 ; Uppercase # L& LATIN CAPITAL LETTER R WITH CARON +015A ; Uppercase # L& LATIN CAPITAL LETTER S WITH ACUTE +015C ; Uppercase # L& LATIN CAPITAL LETTER S WITH CIRCUMFLEX +015E ; Uppercase # L& LATIN CAPITAL LETTER S WITH CEDILLA +0160 ; Uppercase # L& LATIN CAPITAL LETTER S WITH CARON +0162 ; Uppercase # L& LATIN CAPITAL LETTER T WITH CEDILLA +0164 ; Uppercase # L& LATIN CAPITAL LETTER T WITH CARON +0166 ; Uppercase # L& LATIN CAPITAL LETTER T WITH STROKE +0168 ; Uppercase # L& LATIN CAPITAL LETTER U WITH TILDE +016A ; Uppercase # L& LATIN CAPITAL LETTER U WITH MACRON +016C ; Uppercase # L& LATIN CAPITAL LETTER U WITH BREVE +016E ; Uppercase # L& LATIN CAPITAL LETTER U WITH RING ABOVE +0170 ; Uppercase # L& LATIN CAPITAL LETTER U WITH DOUBLE ACUTE +0172 ; Uppercase # L& LATIN CAPITAL LETTER U WITH OGONEK +0174 ; Uppercase # L& LATIN CAPITAL LETTER W WITH CIRCUMFLEX +0176 ; Uppercase # L& LATIN CAPITAL LETTER Y WITH CIRCUMFLEX +0178..0179 ; Uppercase # L& [2] LATIN CAPITAL LETTER Y WITH DIAERESIS..LATIN CAPITAL LETTER Z WITH ACUTE +017B ; Uppercase # L& LATIN CAPITAL LETTER Z WITH DOT ABOVE +017D ; Uppercase # L& LATIN CAPITAL LETTER Z WITH CARON +0181..0182 ; Uppercase # L& [2] LATIN CAPITAL LETTER B WITH HOOK..LATIN CAPITAL LETTER B WITH TOPBAR +0184 ; Uppercase # L& LATIN CAPITAL LETTER TONE SIX +0186..0187 ; Uppercase # L& [2] LATIN CAPITAL LETTER OPEN O..LATIN CAPITAL LETTER C WITH HOOK +0189..018B ; Uppercase # L& [3] LATIN CAPITAL LETTER AFRICAN D..LATIN CAPITAL LETTER D WITH TOPBAR +018E..0191 ; Uppercase # L& [4] LATIN CAPITAL LETTER REVERSED E..LATIN CAPITAL LETTER F WITH HOOK +0193..0194 ; Uppercase # L& [2] LATIN CAPITAL LETTER G WITH HOOK..LATIN CAPITAL LETTER GAMMA +0196..0198 ; Uppercase # L& [3] LATIN CAPITAL LETTER IOTA..LATIN CAPITAL LETTER K WITH HOOK +019C..019D ; Uppercase # L& [2] LATIN CAPITAL LETTER TURNED M..LATIN CAPITAL LETTER N WITH LEFT HOOK +019F..01A0 ; Uppercase # L& [2] LATIN CAPITAL LETTER O WITH MIDDLE TILDE..LATIN CAPITAL LETTER O WITH HORN +01A2 ; Uppercase # L& LATIN CAPITAL LETTER OI +01A4 ; Uppercase # L& LATIN CAPITAL LETTER P WITH HOOK +01A6..01A7 ; Uppercase # L& [2] LATIN LETTER YR..LATIN CAPITAL LETTER TONE TWO +01A9 ; Uppercase # L& LATIN CAPITAL LETTER ESH +01AC ; Uppercase # L& LATIN CAPITAL LETTER T WITH HOOK +01AE..01AF ; Uppercase # L& [2] LATIN CAPITAL LETTER T WITH RETROFLEX HOOK..LATIN CAPITAL LETTER U WITH HORN +01B1..01B3 ; Uppercase # L& [3] LATIN CAPITAL LETTER UPSILON..LATIN CAPITAL LETTER Y WITH HOOK +01B5 ; Uppercase # L& LATIN CAPITAL LETTER Z WITH STROKE +01B7..01B8 ; Uppercase # L& [2] LATIN CAPITAL LETTER EZH..LATIN CAPITAL LETTER EZH REVERSED +01BC ; Uppercase # L& LATIN CAPITAL LETTER TONE FIVE +01C4 ; Uppercase # L& LATIN CAPITAL LETTER DZ WITH CARON +01C7 ; Uppercase # L& LATIN CAPITAL LETTER LJ +01CA ; Uppercase # L& LATIN CAPITAL LETTER NJ +01CD ; Uppercase # L& LATIN CAPITAL LETTER A WITH CARON +01CF ; Uppercase # L& LATIN CAPITAL LETTER I WITH CARON +01D1 ; Uppercase # L& LATIN CAPITAL LETTER O WITH CARON +01D3 ; Uppercase # L& LATIN CAPITAL LETTER U WITH CARON +01D5 ; Uppercase # L& LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON +01D7 ; Uppercase # L& LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE +01D9 ; Uppercase # L& LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON +01DB ; Uppercase # L& LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE +01DE ; Uppercase # L& LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON +01E0 ; Uppercase # L& LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON +01E2 ; Uppercase # L& LATIN CAPITAL LETTER AE WITH MACRON +01E4 ; Uppercase # L& LATIN CAPITAL LETTER G WITH STROKE +01E6 ; Uppercase # L& LATIN CAPITAL LETTER G WITH CARON +01E8 ; Uppercase # L& LATIN CAPITAL LETTER K WITH CARON +01EA ; Uppercase # L& LATIN CAPITAL LETTER O WITH OGONEK +01EC ; Uppercase # L& LATIN CAPITAL LETTER O WITH OGONEK AND MACRON +01EE ; Uppercase # L& LATIN CAPITAL LETTER EZH WITH CARON +01F1 ; Uppercase # L& LATIN CAPITAL LETTER DZ +01F4 ; Uppercase # L& LATIN CAPITAL LETTER G WITH ACUTE +01F6..01F8 ; Uppercase # L& [3] LATIN CAPITAL LETTER HWAIR..LATIN CAPITAL LETTER N WITH GRAVE +01FA ; Uppercase # L& LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE +01FC ; Uppercase # L& LATIN CAPITAL LETTER AE WITH ACUTE +01FE ; Uppercase # L& LATIN CAPITAL LETTER O WITH STROKE AND ACUTE +0200 ; Uppercase # L& LATIN CAPITAL LETTER A WITH DOUBLE GRAVE +0202 ; Uppercase # L& LATIN CAPITAL LETTER A WITH INVERTED BREVE +0204 ; Uppercase # L& LATIN CAPITAL LETTER E WITH DOUBLE GRAVE +0206 ; Uppercase # L& LATIN CAPITAL LETTER E WITH INVERTED BREVE +0208 ; Uppercase # L& LATIN CAPITAL LETTER I WITH DOUBLE GRAVE +020A ; Uppercase # L& LATIN CAPITAL LETTER I WITH INVERTED BREVE +020C ; Uppercase # L& LATIN CAPITAL LETTER O WITH DOUBLE GRAVE +020E ; Uppercase # L& LATIN CAPITAL LETTER O WITH INVERTED BREVE +0210 ; Uppercase # L& LATIN CAPITAL LETTER R WITH DOUBLE GRAVE +0212 ; Uppercase # L& LATIN CAPITAL LETTER R WITH INVERTED BREVE +0214 ; Uppercase # L& LATIN CAPITAL LETTER U WITH DOUBLE GRAVE +0216 ; Uppercase # L& LATIN CAPITAL LETTER U WITH INVERTED BREVE +0218 ; Uppercase # L& LATIN CAPITAL LETTER S WITH COMMA BELOW +021A ; Uppercase # L& LATIN CAPITAL LETTER T WITH COMMA BELOW +021C ; Uppercase # L& LATIN CAPITAL LETTER YOGH +021E ; Uppercase # L& LATIN CAPITAL LETTER H WITH CARON +0220 ; Uppercase # L& LATIN CAPITAL LETTER N WITH LONG RIGHT LEG +0222 ; Uppercase # L& LATIN CAPITAL LETTER OU +0224 ; Uppercase # L& LATIN CAPITAL LETTER Z WITH HOOK +0226 ; Uppercase # L& LATIN CAPITAL LETTER A WITH DOT ABOVE +0228 ; Uppercase # L& LATIN CAPITAL LETTER E WITH CEDILLA +022A ; Uppercase # L& LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON +022C ; Uppercase # L& LATIN CAPITAL LETTER O WITH TILDE AND MACRON +022E ; Uppercase # L& LATIN CAPITAL LETTER O WITH DOT ABOVE +0230 ; Uppercase # L& LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON +0232 ; Uppercase # L& LATIN CAPITAL LETTER Y WITH MACRON +0386 ; Uppercase # L& GREEK CAPITAL LETTER ALPHA WITH TONOS +0388..038A ; Uppercase # L& [3] GREEK CAPITAL LETTER EPSILON WITH TONOS..GREEK CAPITAL LETTER IOTA WITH TONOS +038C ; Uppercase # L& GREEK CAPITAL LETTER OMICRON WITH TONOS +038E..038F ; Uppercase # L& [2] GREEK CAPITAL LETTER UPSILON WITH TONOS..GREEK CAPITAL LETTER OMEGA WITH TONOS +0391..03A1 ; Uppercase # L& [17] GREEK CAPITAL LETTER ALPHA..GREEK CAPITAL LETTER RHO +03A3..03AB ; Uppercase # L& [9] GREEK CAPITAL LETTER SIGMA..GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA +03D2..03D4 ; Uppercase # L& [3] GREEK UPSILON WITH HOOK SYMBOL..GREEK UPSILON WITH DIAERESIS AND HOOK SYMBOL +03D8 ; Uppercase # L& GREEK LETTER ARCHAIC KOPPA +03DA ; Uppercase # L& GREEK LETTER STIGMA +03DC ; Uppercase # L& GREEK LETTER DIGAMMA +03DE ; Uppercase # L& GREEK LETTER KOPPA +03E0 ; Uppercase # L& GREEK LETTER SAMPI +03E2 ; Uppercase # L& COPTIC CAPITAL LETTER SHEI +03E4 ; Uppercase # L& COPTIC CAPITAL LETTER FEI +03E6 ; Uppercase # L& COPTIC CAPITAL LETTER KHEI +03E8 ; Uppercase # L& COPTIC CAPITAL LETTER HORI +03EA ; Uppercase # L& COPTIC CAPITAL LETTER GANGIA +03EC ; Uppercase # L& COPTIC CAPITAL LETTER SHIMA +03EE ; Uppercase # L& COPTIC CAPITAL LETTER DEI +03F4 ; Uppercase # L& GREEK CAPITAL THETA SYMBOL +0400..042F ; Uppercase # L& [48] CYRILLIC CAPITAL LETTER IE WITH GRAVE..CYRILLIC CAPITAL LETTER YA +0460 ; Uppercase # L& CYRILLIC CAPITAL LETTER OMEGA +0462 ; Uppercase # L& CYRILLIC CAPITAL LETTER YAT +0464 ; Uppercase # L& CYRILLIC CAPITAL LETTER IOTIFIED E +0466 ; Uppercase # L& CYRILLIC CAPITAL LETTER LITTLE YUS +0468 ; Uppercase # L& CYRILLIC CAPITAL LETTER IOTIFIED LITTLE YUS +046A ; Uppercase # L& CYRILLIC CAPITAL LETTER BIG YUS +046C ; Uppercase # L& CYRILLIC CAPITAL LETTER IOTIFIED BIG YUS +046E ; Uppercase # L& CYRILLIC CAPITAL LETTER KSI +0470 ; Uppercase # L& CYRILLIC CAPITAL LETTER PSI +0472 ; Uppercase # L& CYRILLIC CAPITAL LETTER FITA +0474 ; Uppercase # L& CYRILLIC CAPITAL LETTER IZHITSA +0476 ; Uppercase # L& CYRILLIC CAPITAL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT +0478 ; Uppercase # L& CYRILLIC CAPITAL LETTER UK +047A ; Uppercase # L& CYRILLIC CAPITAL LETTER ROUND OMEGA +047C ; Uppercase # L& CYRILLIC CAPITAL LETTER OMEGA WITH TITLO +047E ; Uppercase # L& CYRILLIC CAPITAL LETTER OT +0480 ; Uppercase # L& CYRILLIC CAPITAL LETTER KOPPA +048A ; Uppercase # L& CYRILLIC CAPITAL LETTER SHORT I WITH TAIL +048C ; Uppercase # L& CYRILLIC CAPITAL LETTER SEMISOFT SIGN +048E ; Uppercase # L& CYRILLIC CAPITAL LETTER ER WITH TICK +0490 ; Uppercase # L& CYRILLIC CAPITAL LETTER GHE WITH UPTURN +0492 ; Uppercase # L& CYRILLIC CAPITAL LETTER GHE WITH STROKE +0494 ; Uppercase # L& CYRILLIC CAPITAL LETTER GHE WITH MIDDLE HOOK +0496 ; Uppercase # L& CYRILLIC CAPITAL LETTER ZHE WITH DESCENDER +0498 ; Uppercase # L& CYRILLIC CAPITAL LETTER ZE WITH DESCENDER +049A ; Uppercase # L& CYRILLIC CAPITAL LETTER KA WITH DESCENDER +049C ; Uppercase # L& CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE +049E ; Uppercase # L& CYRILLIC CAPITAL LETTER KA WITH STROKE +04A0 ; Uppercase # L& CYRILLIC CAPITAL LETTER BASHKIR KA +04A2 ; Uppercase # L& CYRILLIC CAPITAL LETTER EN WITH DESCENDER +04A4 ; Uppercase # L& CYRILLIC CAPITAL LIGATURE EN GHE +04A6 ; Uppercase # L& CYRILLIC CAPITAL LETTER PE WITH MIDDLE HOOK +04A8 ; Uppercase # L& CYRILLIC CAPITAL LETTER ABKHASIAN HA +04AA ; Uppercase # L& CYRILLIC CAPITAL LETTER ES WITH DESCENDER +04AC ; Uppercase # L& CYRILLIC CAPITAL LETTER TE WITH DESCENDER +04AE ; Uppercase # L& CYRILLIC CAPITAL LETTER STRAIGHT U +04B0 ; Uppercase # L& CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE +04B2 ; Uppercase # L& CYRILLIC CAPITAL LETTER HA WITH DESCENDER +04B4 ; Uppercase # L& CYRILLIC CAPITAL LIGATURE TE TSE +04B6 ; Uppercase # L& CYRILLIC CAPITAL LETTER CHE WITH DESCENDER +04B8 ; Uppercase # L& CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE +04BA ; Uppercase # L& CYRILLIC CAPITAL LETTER SHHA +04BC ; Uppercase # L& CYRILLIC CAPITAL LETTER ABKHASIAN CHE +04BE ; Uppercase # L& CYRILLIC CAPITAL LETTER ABKHASIAN CHE WITH DESCENDER +04C0..04C1 ; Uppercase # L& [2] CYRILLIC LETTER PALOCHKA..CYRILLIC CAPITAL LETTER ZHE WITH BREVE +04C3 ; Uppercase # L& CYRILLIC CAPITAL LETTER KA WITH HOOK +04C5 ; Uppercase # L& CYRILLIC CAPITAL LETTER EL WITH TAIL +04C7 ; Uppercase # L& CYRILLIC CAPITAL LETTER EN WITH HOOK +04C9 ; Uppercase # L& CYRILLIC CAPITAL LETTER EN WITH TAIL +04CB ; Uppercase # L& CYRILLIC CAPITAL LETTER KHAKASSIAN CHE +04CD ; Uppercase # L& CYRILLIC CAPITAL LETTER EM WITH TAIL +04D0 ; Uppercase # L& CYRILLIC CAPITAL LETTER A WITH BREVE +04D2 ; Uppercase # L& CYRILLIC CAPITAL LETTER A WITH DIAERESIS +04D4 ; Uppercase # L& CYRILLIC CAPITAL LIGATURE A IE +04D6 ; Uppercase # L& CYRILLIC CAPITAL LETTER IE WITH BREVE +04D8 ; Uppercase # L& CYRILLIC CAPITAL LETTER SCHWA +04DA ; Uppercase # L& CYRILLIC CAPITAL LETTER SCHWA WITH DIAERESIS +04DC ; Uppercase # L& CYRILLIC CAPITAL LETTER ZHE WITH DIAERESIS +04DE ; Uppercase # L& CYRILLIC CAPITAL LETTER ZE WITH DIAERESIS +04E0 ; Uppercase # L& CYRILLIC CAPITAL LETTER ABKHASIAN DZE +04E2 ; Uppercase # L& CYRILLIC CAPITAL LETTER I WITH MACRON +04E4 ; Uppercase # L& CYRILLIC CAPITAL LETTER I WITH DIAERESIS +04E6 ; Uppercase # L& CYRILLIC CAPITAL LETTER O WITH DIAERESIS +04E8 ; Uppercase # L& CYRILLIC CAPITAL LETTER BARRED O +04EA ; Uppercase # L& CYRILLIC CAPITAL LETTER BARRED O WITH DIAERESIS +04EC ; Uppercase # L& CYRILLIC CAPITAL LETTER E WITH DIAERESIS +04EE ; Uppercase # L& CYRILLIC CAPITAL LETTER U WITH MACRON +04F0 ; Uppercase # L& CYRILLIC CAPITAL LETTER U WITH DIAERESIS +04F2 ; Uppercase # L& CYRILLIC CAPITAL LETTER U WITH DOUBLE ACUTE +04F4 ; Uppercase # L& CYRILLIC CAPITAL LETTER CHE WITH DIAERESIS +04F8 ; Uppercase # L& CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS +0500 ; Uppercase # L& CYRILLIC CAPITAL LETTER KOMI DE +0502 ; Uppercase # L& CYRILLIC CAPITAL LETTER KOMI DJE +0504 ; Uppercase # L& CYRILLIC CAPITAL LETTER KOMI ZJE +0506 ; Uppercase # L& CYRILLIC CAPITAL LETTER KOMI DZJE +0508 ; Uppercase # L& CYRILLIC CAPITAL LETTER KOMI LJE +050A ; Uppercase # L& CYRILLIC CAPITAL LETTER KOMI NJE +050C ; Uppercase # L& CYRILLIC CAPITAL LETTER KOMI SJE +050E ; Uppercase # L& CYRILLIC CAPITAL LETTER KOMI TJE +0531..0556 ; Uppercase # L& [38] ARMENIAN CAPITAL LETTER AYB..ARMENIAN CAPITAL LETTER FEH +10A0..10C5 ; Uppercase # L& [38] GEORGIAN CAPITAL LETTER AN..GEORGIAN CAPITAL LETTER HOE +1E00 ; Uppercase # L& LATIN CAPITAL LETTER A WITH RING BELOW +1E02 ; Uppercase # L& LATIN CAPITAL LETTER B WITH DOT ABOVE +1E04 ; Uppercase # L& LATIN CAPITAL LETTER B WITH DOT BELOW +1E06 ; Uppercase # L& LATIN CAPITAL LETTER B WITH LINE BELOW +1E08 ; Uppercase # L& LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE +1E0A ; Uppercase # L& LATIN CAPITAL LETTER D WITH DOT ABOVE +1E0C ; Uppercase # L& LATIN CAPITAL LETTER D WITH DOT BELOW +1E0E ; Uppercase # L& LATIN CAPITAL LETTER D WITH LINE BELOW +1E10 ; Uppercase # L& LATIN CAPITAL LETTER D WITH CEDILLA +1E12 ; Uppercase # L& LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW +1E14 ; Uppercase # L& LATIN CAPITAL LETTER E WITH MACRON AND GRAVE +1E16 ; Uppercase # L& LATIN CAPITAL LETTER E WITH MACRON AND ACUTE +1E18 ; Uppercase # L& LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW +1E1A ; Uppercase # L& LATIN CAPITAL LETTER E WITH TILDE BELOW +1E1C ; Uppercase # L& LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE +1E1E ; Uppercase # L& LATIN CAPITAL LETTER F WITH DOT ABOVE +1E20 ; Uppercase # L& LATIN CAPITAL LETTER G WITH MACRON +1E22 ; Uppercase # L& LATIN CAPITAL LETTER H WITH DOT ABOVE +1E24 ; Uppercase # L& LATIN CAPITAL LETTER H WITH DOT BELOW +1E26 ; Uppercase # L& LATIN CAPITAL LETTER H WITH DIAERESIS +1E28 ; Uppercase # L& LATIN CAPITAL LETTER H WITH CEDILLA +1E2A ; Uppercase # L& LATIN CAPITAL LETTER H WITH BREVE BELOW +1E2C ; Uppercase # L& LATIN CAPITAL LETTER I WITH TILDE BELOW +1E2E ; Uppercase # L& LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE +1E30 ; Uppercase # L& LATIN CAPITAL LETTER K WITH ACUTE +1E32 ; Uppercase # L& LATIN CAPITAL LETTER K WITH DOT BELOW +1E34 ; Uppercase # L& LATIN CAPITAL LETTER K WITH LINE BELOW +1E36 ; Uppercase # L& LATIN CAPITAL LETTER L WITH DOT BELOW +1E38 ; Uppercase # L& LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON +1E3A ; Uppercase # L& LATIN CAPITAL LETTER L WITH LINE BELOW +1E3C ; Uppercase # L& LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW +1E3E ; Uppercase # L& LATIN CAPITAL LETTER M WITH ACUTE +1E40 ; Uppercase # L& LATIN CAPITAL LETTER M WITH DOT ABOVE +1E42 ; Uppercase # L& LATIN CAPITAL LETTER M WITH DOT BELOW +1E44 ; Uppercase # L& LATIN CAPITAL LETTER N WITH DOT ABOVE +1E46 ; Uppercase # L& LATIN CAPITAL LETTER N WITH DOT BELOW +1E48 ; Uppercase # L& LATIN CAPITAL LETTER N WITH LINE BELOW +1E4A ; Uppercase # L& LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW +1E4C ; Uppercase # L& LATIN CAPITAL LETTER O WITH TILDE AND ACUTE +1E4E ; Uppercase # L& LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS +1E50 ; Uppercase # L& LATIN CAPITAL LETTER O WITH MACRON AND GRAVE +1E52 ; Uppercase # L& LATIN CAPITAL LETTER O WITH MACRON AND ACUTE +1E54 ; Uppercase # L& LATIN CAPITAL LETTER P WITH ACUTE +1E56 ; Uppercase # L& LATIN CAPITAL LETTER P WITH DOT ABOVE +1E58 ; Uppercase # L& LATIN CAPITAL LETTER R WITH DOT ABOVE +1E5A ; Uppercase # L& LATIN CAPITAL LETTER R WITH DOT BELOW +1E5C ; Uppercase # L& LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON +1E5E ; Uppercase # L& LATIN CAPITAL LETTER R WITH LINE BELOW +1E60 ; Uppercase # L& LATIN CAPITAL LETTER S WITH DOT ABOVE +1E62 ; Uppercase # L& LATIN CAPITAL LETTER S WITH DOT BELOW +1E64 ; Uppercase # L& LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE +1E66 ; Uppercase # L& LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE +1E68 ; Uppercase # L& LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE +1E6A ; Uppercase # L& LATIN CAPITAL LETTER T WITH DOT ABOVE +1E6C ; Uppercase # L& LATIN CAPITAL LETTER T WITH DOT BELOW +1E6E ; Uppercase # L& LATIN CAPITAL LETTER T WITH LINE BELOW +1E70 ; Uppercase # L& LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW +1E72 ; Uppercase # L& LATIN CAPITAL LETTER U WITH DIAERESIS BELOW +1E74 ; Uppercase # L& LATIN CAPITAL LETTER U WITH TILDE BELOW +1E76 ; Uppercase # L& LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW +1E78 ; Uppercase # L& LATIN CAPITAL LETTER U WITH TILDE AND ACUTE +1E7A ; Uppercase # L& LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS +1E7C ; Uppercase # L& LATIN CAPITAL LETTER V WITH TILDE +1E7E ; Uppercase # L& LATIN CAPITAL LETTER V WITH DOT BELOW +1E80 ; Uppercase # L& LATIN CAPITAL LETTER W WITH GRAVE +1E82 ; Uppercase # L& LATIN CAPITAL LETTER W WITH ACUTE +1E84 ; Uppercase # L& LATIN CAPITAL LETTER W WITH DIAERESIS +1E86 ; Uppercase # L& LATIN CAPITAL LETTER W WITH DOT ABOVE +1E88 ; Uppercase # L& LATIN CAPITAL LETTER W WITH DOT BELOW +1E8A ; Uppercase # L& LATIN CAPITAL LETTER X WITH DOT ABOVE +1E8C ; Uppercase # L& LATIN CAPITAL LETTER X WITH DIAERESIS +1E8E ; Uppercase # L& LATIN CAPITAL LETTER Y WITH DOT ABOVE +1E90 ; Uppercase # L& LATIN CAPITAL LETTER Z WITH CIRCUMFLEX +1E92 ; Uppercase # L& LATIN CAPITAL LETTER Z WITH DOT BELOW +1E94 ; Uppercase # L& LATIN CAPITAL LETTER Z WITH LINE BELOW +1EA0 ; Uppercase # L& LATIN CAPITAL LETTER A WITH DOT BELOW +1EA2 ; Uppercase # L& LATIN CAPITAL LETTER A WITH HOOK ABOVE +1EA4 ; Uppercase # L& LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE +1EA6 ; Uppercase # L& LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE +1EA8 ; Uppercase # L& LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE +1EAA ; Uppercase # L& LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE +1EAC ; Uppercase # L& LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW +1EAE ; Uppercase # L& LATIN CAPITAL LETTER A WITH BREVE AND ACUTE +1EB0 ; Uppercase # L& LATIN CAPITAL LETTER A WITH BREVE AND GRAVE +1EB2 ; Uppercase # L& LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE +1EB4 ; Uppercase # L& LATIN CAPITAL LETTER A WITH BREVE AND TILDE +1EB6 ; Uppercase # L& LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW +1EB8 ; Uppercase # L& LATIN CAPITAL LETTER E WITH DOT BELOW +1EBA ; Uppercase # L& LATIN CAPITAL LETTER E WITH HOOK ABOVE +1EBC ; Uppercase # L& LATIN CAPITAL LETTER E WITH TILDE +1EBE ; Uppercase # L& LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE +1EC0 ; Uppercase # L& LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE +1EC2 ; Uppercase # L& LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE +1EC4 ; Uppercase # L& LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE +1EC6 ; Uppercase # L& LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW +1EC8 ; Uppercase # L& LATIN CAPITAL LETTER I WITH HOOK ABOVE +1ECA ; Uppercase # L& LATIN CAPITAL LETTER I WITH DOT BELOW +1ECC ; Uppercase # L& LATIN CAPITAL LETTER O WITH DOT BELOW +1ECE ; Uppercase # L& LATIN CAPITAL LETTER O WITH HOOK ABOVE +1ED0 ; Uppercase # L& LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE +1ED2 ; Uppercase # L& LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE +1ED4 ; Uppercase # L& LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE +1ED6 ; Uppercase # L& LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE +1ED8 ; Uppercase # L& LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW +1EDA ; Uppercase # L& LATIN CAPITAL LETTER O WITH HORN AND ACUTE +1EDC ; Uppercase # L& LATIN CAPITAL LETTER O WITH HORN AND GRAVE +1EDE ; Uppercase # L& LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE +1EE0 ; Uppercase # L& LATIN CAPITAL LETTER O WITH HORN AND TILDE +1EE2 ; Uppercase # L& LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW +1EE4 ; Uppercase # L& LATIN CAPITAL LETTER U WITH DOT BELOW +1EE6 ; Uppercase # L& LATIN CAPITAL LETTER U WITH HOOK ABOVE +1EE8 ; Uppercase # L& LATIN CAPITAL LETTER U WITH HORN AND ACUTE +1EEA ; Uppercase # L& LATIN CAPITAL LETTER U WITH HORN AND GRAVE +1EEC ; Uppercase # L& LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE +1EEE ; Uppercase # L& LATIN CAPITAL LETTER U WITH HORN AND TILDE +1EF0 ; Uppercase # L& LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW +1EF2 ; Uppercase # L& LATIN CAPITAL LETTER Y WITH GRAVE +1EF4 ; Uppercase # L& LATIN CAPITAL LETTER Y WITH DOT BELOW +1EF6 ; Uppercase # L& LATIN CAPITAL LETTER Y WITH HOOK ABOVE +1EF8 ; Uppercase # L& LATIN CAPITAL LETTER Y WITH TILDE +1F08..1F0F ; Uppercase # L& [8] GREEK CAPITAL LETTER ALPHA WITH PSILI..GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI +1F18..1F1D ; Uppercase # L& [6] GREEK CAPITAL LETTER EPSILON WITH PSILI..GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA +1F28..1F2F ; Uppercase # L& [8] GREEK CAPITAL LETTER ETA WITH PSILI..GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI +1F38..1F3F ; Uppercase # L& [8] GREEK CAPITAL LETTER IOTA WITH PSILI..GREEK CAPITAL LETTER IOTA WITH DASIA AND PERISPOMENI +1F48..1F4D ; Uppercase # L& [6] GREEK CAPITAL LETTER OMICRON WITH PSILI..GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA +1F59 ; Uppercase # L& GREEK CAPITAL LETTER UPSILON WITH DASIA +1F5B ; Uppercase # L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA +1F5D ; Uppercase # L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA +1F5F ; Uppercase # L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI +1F68..1F6F ; Uppercase # L& [8] GREEK CAPITAL LETTER OMEGA WITH PSILI..GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI +1FB8..1FBB ; Uppercase # L& [4] GREEK CAPITAL LETTER ALPHA WITH VRACHY..GREEK CAPITAL LETTER ALPHA WITH OXIA +1FC8..1FCB ; Uppercase # L& [4] GREEK CAPITAL LETTER EPSILON WITH VARIA..GREEK CAPITAL LETTER ETA WITH OXIA +1FD8..1FDB ; Uppercase # L& [4] GREEK CAPITAL LETTER IOTA WITH VRACHY..GREEK CAPITAL LETTER IOTA WITH OXIA +1FE8..1FEC ; Uppercase # L& [5] GREEK CAPITAL LETTER UPSILON WITH VRACHY..GREEK CAPITAL LETTER RHO WITH DASIA +1FF8..1FFB ; Uppercase # L& [4] GREEK CAPITAL LETTER OMICRON WITH VARIA..GREEK CAPITAL LETTER OMEGA WITH OXIA +2102 ; Uppercase # L& DOUBLE-STRUCK CAPITAL C +2107 ; Uppercase # L& EULER CONSTANT +210B..210D ; Uppercase # L& [3] SCRIPT CAPITAL H..DOUBLE-STRUCK CAPITAL H +2110..2112 ; Uppercase # L& [3] SCRIPT CAPITAL I..SCRIPT CAPITAL L +2115 ; Uppercase # L& DOUBLE-STRUCK CAPITAL N +2119..211D ; Uppercase # L& [5] DOUBLE-STRUCK CAPITAL P..DOUBLE-STRUCK CAPITAL R +2124 ; Uppercase # L& DOUBLE-STRUCK CAPITAL Z +2126 ; Uppercase # L& OHM SIGN +2128 ; Uppercase # L& BLACK-LETTER CAPITAL Z +212A..212D ; Uppercase # L& [4] KELVIN SIGN..BLACK-LETTER CAPITAL C +2130..2131 ; Uppercase # L& [2] SCRIPT CAPITAL E..SCRIPT CAPITAL F +2133 ; Uppercase # L& SCRIPT CAPITAL M +213E..213F ; Uppercase # L& [2] DOUBLE-STRUCK CAPITAL GAMMA..DOUBLE-STRUCK CAPITAL PI +2145 ; Uppercase # L& DOUBLE-STRUCK ITALIC CAPITAL D +2160..216F ; Uppercase # Nl [16] ROMAN NUMERAL ONE..ROMAN NUMERAL ONE THOUSAND +24B6..24CF ; Uppercase # So [26] CIRCLED LATIN CAPITAL LETTER A..CIRCLED LATIN CAPITAL LETTER Z +FF21..FF3A ; Uppercase # L& [26] FULLWIDTH LATIN CAPITAL LETTER A..FULLWIDTH LATIN CAPITAL LETTER Z +10400..10425 ; Uppercase # L& [38] DESERET CAPITAL LETTER LONG I..DESERET CAPITAL LETTER ENG +1D400..1D419 ; Uppercase # L& [26] MATHEMATICAL BOLD CAPITAL A..MATHEMATICAL BOLD CAPITAL Z +1D434..1D44D ; Uppercase # L& [26] MATHEMATICAL ITALIC CAPITAL A..MATHEMATICAL ITALIC CAPITAL Z +1D468..1D481 ; Uppercase # L& [26] MATHEMATICAL BOLD ITALIC CAPITAL A..MATHEMATICAL BOLD ITALIC CAPITAL Z +1D49C ; Uppercase # L& MATHEMATICAL SCRIPT CAPITAL A +1D49E..1D49F ; Uppercase # L& [2] MATHEMATICAL SCRIPT CAPITAL C..MATHEMATICAL SCRIPT CAPITAL D +1D4A2 ; Uppercase # L& MATHEMATICAL SCRIPT CAPITAL G +1D4A5..1D4A6 ; Uppercase # L& [2] MATHEMATICAL SCRIPT CAPITAL J..MATHEMATICAL SCRIPT CAPITAL K +1D4A9..1D4AC ; Uppercase # L& [4] MATHEMATICAL SCRIPT CAPITAL N..MATHEMATICAL SCRIPT CAPITAL Q +1D4AE..1D4B5 ; Uppercase # L& [8] MATHEMATICAL SCRIPT CAPITAL S..MATHEMATICAL SCRIPT CAPITAL Z +1D4D0..1D4E9 ; Uppercase # L& [26] MATHEMATICAL BOLD SCRIPT CAPITAL A..MATHEMATICAL BOLD SCRIPT CAPITAL Z +1D504..1D505 ; Uppercase # L& [2] MATHEMATICAL FRAKTUR CAPITAL A..MATHEMATICAL FRAKTUR CAPITAL B +1D507..1D50A ; Uppercase # L& [4] MATHEMATICAL FRAKTUR CAPITAL D..MATHEMATICAL FRAKTUR CAPITAL G +1D50D..1D514 ; Uppercase # L& [8] MATHEMATICAL FRAKTUR CAPITAL J..MATHEMATICAL FRAKTUR CAPITAL Q +1D516..1D51C ; Uppercase # L& [7] MATHEMATICAL FRAKTUR CAPITAL S..MATHEMATICAL FRAKTUR CAPITAL Y +1D538..1D539 ; Uppercase # L& [2] MATHEMATICAL DOUBLE-STRUCK CAPITAL A..MATHEMATICAL DOUBLE-STRUCK CAPITAL B +1D53B..1D53E ; Uppercase # L& [4] MATHEMATICAL DOUBLE-STRUCK CAPITAL D..MATHEMATICAL DOUBLE-STRUCK CAPITAL G +1D540..1D544 ; Uppercase # L& [5] MATHEMATICAL DOUBLE-STRUCK CAPITAL I..MATHEMATICAL DOUBLE-STRUCK CAPITAL M +1D546 ; Uppercase # L& MATHEMATICAL DOUBLE-STRUCK CAPITAL O +1D54A..1D550 ; Uppercase # L& [7] MATHEMATICAL DOUBLE-STRUCK CAPITAL S..MATHEMATICAL DOUBLE-STRUCK CAPITAL Y +1D56C..1D585 ; Uppercase # L& [26] MATHEMATICAL BOLD FRAKTUR CAPITAL A..MATHEMATICAL BOLD FRAKTUR CAPITAL Z +1D5A0..1D5B9 ; Uppercase # L& [26] MATHEMATICAL SANS-SERIF CAPITAL A..MATHEMATICAL SANS-SERIF CAPITAL Z +1D5D4..1D5ED ; Uppercase # L& [26] MATHEMATICAL SANS-SERIF BOLD CAPITAL A..MATHEMATICAL SANS-SERIF BOLD CAPITAL Z +1D608..1D621 ; Uppercase # L& [26] MATHEMATICAL SANS-SERIF ITALIC CAPITAL A..MATHEMATICAL SANS-SERIF ITALIC CAPITAL Z +1D63C..1D655 ; Uppercase # L& [26] MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL A..MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL Z +1D670..1D689 ; Uppercase # L& [26] MATHEMATICAL MONOSPACE CAPITAL A..MATHEMATICAL MONOSPACE CAPITAL Z +1D6A8..1D6C0 ; Uppercase # L& [25] MATHEMATICAL BOLD CAPITAL ALPHA..MATHEMATICAL BOLD CAPITAL OMEGA +1D6E2..1D6FA ; Uppercase # L& [25] MATHEMATICAL ITALIC CAPITAL ALPHA..MATHEMATICAL ITALIC CAPITAL OMEGA +1D71C..1D734 ; Uppercase # L& [25] MATHEMATICAL BOLD ITALIC CAPITAL ALPHA..MATHEMATICAL BOLD ITALIC CAPITAL OMEGA +1D756..1D76E ; Uppercase # L& [25] MATHEMATICAL SANS-SERIF BOLD CAPITAL ALPHA..MATHEMATICAL SANS-SERIF BOLD CAPITAL OMEGA +1D790..1D7A8 ; Uppercase # L& [25] MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL ALPHA..MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL OMEGA + +# Total code points: 1227 + +# ================================================ + +# Derived Property: ID_Start +# Characters that can start an identifier. +# Generated from Lu+Ll+Lt+Lm+Lo+Nl + +0041..005A ; ID_Start # L& [26] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER Z +0061..007A ; ID_Start # L& [26] LATIN SMALL LETTER A..LATIN SMALL LETTER Z +00AA ; ID_Start # L& FEMININE ORDINAL INDICATOR +00B5 ; ID_Start # L& MICRO SIGN +00BA ; ID_Start # L& MASCULINE ORDINAL INDICATOR +00C0..00D6 ; ID_Start # L& [23] LATIN CAPITAL LETTER A WITH GRAVE..LATIN CAPITAL LETTER O WITH DIAERESIS +00D8..00F6 ; ID_Start # L& [31] LATIN CAPITAL LETTER O WITH STROKE..LATIN SMALL LETTER O WITH DIAERESIS +00F8..01BA ; ID_Start # L& [195] LATIN SMALL LETTER O WITH STROKE..LATIN SMALL LETTER EZH WITH TAIL +01BB ; ID_Start # Lo LATIN LETTER TWO WITH STROKE +01BC..01BF ; ID_Start # L& [4] LATIN CAPITAL LETTER TONE FIVE..LATIN LETTER WYNN +01C0..01C3 ; ID_Start # Lo [4] LATIN LETTER DENTAL CLICK..LATIN LETTER RETROFLEX CLICK +01C4..0220 ; ID_Start # L& [93] LATIN CAPITAL LETTER DZ WITH CARON..LATIN CAPITAL LETTER N WITH LONG RIGHT LEG +0222..0233 ; ID_Start # L& [18] LATIN CAPITAL LETTER OU..LATIN SMALL LETTER Y WITH MACRON +0250..02AD ; ID_Start # L& [94] LATIN SMALL LETTER TURNED A..LATIN LETTER BIDENTAL PERCUSSIVE +02B0..02B8 ; ID_Start # Lm [9] MODIFIER LETTER SMALL H..MODIFIER LETTER SMALL Y +02BB..02C1 ; ID_Start # Lm [7] MODIFIER LETTER TURNED COMMA..MODIFIER LETTER REVERSED GLOTTAL STOP +02D0..02D1 ; ID_Start # Lm [2] MODIFIER LETTER TRIANGULAR COLON..MODIFIER LETTER HALF TRIANGULAR COLON +02E0..02E4 ; ID_Start # Lm [5] MODIFIER LETTER SMALL GAMMA..MODIFIER LETTER SMALL REVERSED GLOTTAL STOP +02EE ; ID_Start # Lm MODIFIER LETTER DOUBLE APOSTROPHE +037A ; ID_Start # Lm GREEK YPOGEGRAMMENI +0386 ; ID_Start # L& GREEK CAPITAL LETTER ALPHA WITH TONOS +0388..038A ; ID_Start # L& [3] GREEK CAPITAL LETTER EPSILON WITH TONOS..GREEK CAPITAL LETTER IOTA WITH TONOS +038C ; ID_Start # L& GREEK CAPITAL LETTER OMICRON WITH TONOS +038E..03A1 ; ID_Start # L& [20] GREEK CAPITAL LETTER UPSILON WITH TONOS..GREEK CAPITAL LETTER RHO +03A3..03CE ; ID_Start # L& [44] GREEK CAPITAL LETTER SIGMA..GREEK SMALL LETTER OMEGA WITH TONOS +03D0..03F5 ; ID_Start # L& [38] GREEK BETA SYMBOL..GREEK LUNATE EPSILON SYMBOL +0400..0481 ; ID_Start # L& [130] CYRILLIC CAPITAL LETTER IE WITH GRAVE..CYRILLIC SMALL LETTER KOPPA +048A..04CE ; ID_Start # L& [69] CYRILLIC CAPITAL LETTER SHORT I WITH TAIL..CYRILLIC SMALL LETTER EM WITH TAIL +04D0..04F5 ; ID_Start # L& [38] CYRILLIC CAPITAL LETTER A WITH BREVE..CYRILLIC SMALL LETTER CHE WITH DIAERESIS +04F8..04F9 ; ID_Start # L& [2] CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS..CYRILLIC SMALL LETTER YERU WITH DIAERESIS +0500..050F ; ID_Start # L& [16] CYRILLIC CAPITAL LETTER KOMI DE..CYRILLIC SMALL LETTER KOMI TJE +0531..0556 ; ID_Start # L& [38] ARMENIAN CAPITAL LETTER AYB..ARMENIAN CAPITAL LETTER FEH +0559 ; ID_Start # Lm ARMENIAN MODIFIER LETTER LEFT HALF RING +0561..0587 ; ID_Start # L& [39] ARMENIAN SMALL LETTER AYB..ARMENIAN SMALL LIGATURE ECH YIWN +05D0..05EA ; ID_Start # Lo [27] HEBREW LETTER ALEF..HEBREW LETTER TAV +05F0..05F2 ; ID_Start # Lo [3] HEBREW LIGATURE YIDDISH DOUBLE VAV..HEBREW LIGATURE YIDDISH DOUBLE YOD +0621..063A ; ID_Start # Lo [26] ARABIC LETTER HAMZA..ARABIC LETTER GHAIN +0640 ; ID_Start # Lm ARABIC TATWEEL +0641..064A ; ID_Start # Lo [10] ARABIC LETTER FEH..ARABIC LETTER YEH +066E..066F ; ID_Start # Lo [2] ARABIC LETTER DOTLESS BEH..ARABIC LETTER DOTLESS QAF +0671..06D3 ; ID_Start # Lo [99] ARABIC LETTER ALEF WASLA..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE +06D5 ; ID_Start # Lo ARABIC LETTER AE +06E5..06E6 ; ID_Start # Lm [2] ARABIC SMALL WAW..ARABIC SMALL YEH +06FA..06FC ; ID_Start # Lo [3] ARABIC LETTER SHEEN WITH DOT BELOW..ARABIC LETTER GHAIN WITH DOT BELOW +0710 ; ID_Start # Lo SYRIAC LETTER ALAPH +0712..072C ; ID_Start # Lo [27] SYRIAC LETTER BETH..SYRIAC LETTER TAW +0780..07A5 ; ID_Start # Lo [38] THAANA LETTER HAA..THAANA LETTER WAAVU +07B1 ; ID_Start # Lo THAANA LETTER NAA +0905..0939 ; ID_Start # Lo [53] DEVANAGARI LETTER A..DEVANAGARI LETTER HA +093D ; ID_Start # Lo DEVANAGARI SIGN AVAGRAHA +0950 ; ID_Start # Lo DEVANAGARI OM +0958..0961 ; ID_Start # Lo [10] DEVANAGARI LETTER QA..DEVANAGARI LETTER VOCALIC LL +0985..098C ; ID_Start # Lo [8] BENGALI LETTER A..BENGALI LETTER VOCALIC L +098F..0990 ; ID_Start # Lo [2] BENGALI LETTER E..BENGALI LETTER AI +0993..09A8 ; ID_Start # Lo [22] BENGALI LETTER O..BENGALI LETTER NA +09AA..09B0 ; ID_Start # Lo [7] BENGALI LETTER PA..BENGALI LETTER RA +09B2 ; ID_Start # Lo BENGALI LETTER LA +09B6..09B9 ; ID_Start # Lo [4] BENGALI LETTER SHA..BENGALI LETTER HA +09DC..09DD ; ID_Start # Lo [2] BENGALI LETTER RRA..BENGALI LETTER RHA +09DF..09E1 ; ID_Start # Lo [3] BENGALI LETTER YYA..BENGALI LETTER VOCALIC LL +09F0..09F1 ; ID_Start # Lo [2] BENGALI LETTER RA WITH MIDDLE DIAGONAL..BENGALI LETTER RA WITH LOWER DIAGONAL +0A05..0A0A ; ID_Start # Lo [6] GURMUKHI LETTER A..GURMUKHI LETTER UU +0A0F..0A10 ; ID_Start # Lo [2] GURMUKHI LETTER EE..GURMUKHI LETTER AI +0A13..0A28 ; ID_Start # Lo [22] GURMUKHI LETTER OO..GURMUKHI LETTER NA +0A2A..0A30 ; ID_Start # Lo [7] GURMUKHI LETTER PA..GURMUKHI LETTER RA +0A32..0A33 ; ID_Start # Lo [2] GURMUKHI LETTER LA..GURMUKHI LETTER LLA +0A35..0A36 ; ID_Start # Lo [2] GURMUKHI LETTER VA..GURMUKHI LETTER SHA +0A38..0A39 ; ID_Start # Lo [2] GURMUKHI LETTER SA..GURMUKHI LETTER HA +0A59..0A5C ; ID_Start # Lo [4] GURMUKHI LETTER KHHA..GURMUKHI LETTER RRA +0A5E ; ID_Start # Lo GURMUKHI LETTER FA +0A72..0A74 ; ID_Start # Lo [3] GURMUKHI IRI..GURMUKHI EK ONKAR +0A85..0A8B ; ID_Start # Lo [7] GUJARATI LETTER A..GUJARATI LETTER VOCALIC R +0A8D ; ID_Start # Lo GUJARATI VOWEL CANDRA E +0A8F..0A91 ; ID_Start # Lo [3] GUJARATI LETTER E..GUJARATI VOWEL CANDRA O +0A93..0AA8 ; ID_Start # Lo [22] GUJARATI LETTER O..GUJARATI LETTER NA +0AAA..0AB0 ; ID_Start # Lo [7] GUJARATI LETTER PA..GUJARATI LETTER RA +0AB2..0AB3 ; ID_Start # Lo [2] GUJARATI LETTER LA..GUJARATI LETTER LLA +0AB5..0AB9 ; ID_Start # Lo [5] GUJARATI LETTER VA..GUJARATI LETTER HA +0ABD ; ID_Start # Lo GUJARATI SIGN AVAGRAHA +0AD0 ; ID_Start # Lo GUJARATI OM +0AE0 ; ID_Start # Lo GUJARATI LETTER VOCALIC RR +0B05..0B0C ; ID_Start # Lo [8] ORIYA LETTER A..ORIYA LETTER VOCALIC L +0B0F..0B10 ; ID_Start # Lo [2] ORIYA LETTER E..ORIYA LETTER AI +0B13..0B28 ; ID_Start # Lo [22] ORIYA LETTER O..ORIYA LETTER NA +0B2A..0B30 ; ID_Start # Lo [7] ORIYA LETTER PA..ORIYA LETTER RA +0B32..0B33 ; ID_Start # Lo [2] ORIYA LETTER LA..ORIYA LETTER LLA +0B36..0B39 ; ID_Start # Lo [4] ORIYA LETTER SHA..ORIYA LETTER HA +0B3D ; ID_Start # Lo ORIYA SIGN AVAGRAHA +0B5C..0B5D ; ID_Start # Lo [2] ORIYA LETTER RRA..ORIYA LETTER RHA +0B5F..0B61 ; ID_Start # Lo [3] ORIYA LETTER YYA..ORIYA LETTER VOCALIC LL +0B83 ; ID_Start # Lo TAMIL SIGN VISARGA +0B85..0B8A ; ID_Start # Lo [6] TAMIL LETTER A..TAMIL LETTER UU +0B8E..0B90 ; ID_Start # Lo [3] TAMIL LETTER E..TAMIL LETTER AI +0B92..0B95 ; ID_Start # Lo [4] TAMIL LETTER O..TAMIL LETTER KA +0B99..0B9A ; ID_Start # Lo [2] TAMIL LETTER NGA..TAMIL LETTER CA +0B9C ; ID_Start # Lo TAMIL LETTER JA +0B9E..0B9F ; ID_Start # Lo [2] TAMIL LETTER NYA..TAMIL LETTER TTA +0BA3..0BA4 ; ID_Start # Lo [2] TAMIL LETTER NNA..TAMIL LETTER TA +0BA8..0BAA ; ID_Start # Lo [3] TAMIL LETTER NA..TAMIL LETTER PA +0BAE..0BB5 ; ID_Start # Lo [8] TAMIL LETTER MA..TAMIL LETTER VA +0BB7..0BB9 ; ID_Start # Lo [3] TAMIL LETTER SSA..TAMIL LETTER HA +0C05..0C0C ; ID_Start # Lo [8] TELUGU LETTER A..TELUGU LETTER VOCALIC L +0C0E..0C10 ; ID_Start # Lo [3] TELUGU LETTER E..TELUGU LETTER AI +0C12..0C28 ; ID_Start # Lo [23] TELUGU LETTER O..TELUGU LETTER NA +0C2A..0C33 ; ID_Start # Lo [10] TELUGU LETTER PA..TELUGU LETTER LLA +0C35..0C39 ; ID_Start # Lo [5] TELUGU LETTER VA..TELUGU LETTER HA +0C60..0C61 ; ID_Start # Lo [2] TELUGU LETTER VOCALIC RR..TELUGU LETTER VOCALIC LL +0C85..0C8C ; ID_Start # Lo [8] KANNADA LETTER A..KANNADA LETTER VOCALIC L +0C8E..0C90 ; ID_Start # Lo [3] KANNADA LETTER E..KANNADA LETTER AI +0C92..0CA8 ; ID_Start # Lo [23] KANNADA LETTER O..KANNADA LETTER NA +0CAA..0CB3 ; ID_Start # Lo [10] KANNADA LETTER PA..KANNADA LETTER LLA +0CB5..0CB9 ; ID_Start # Lo [5] KANNADA LETTER VA..KANNADA LETTER HA +0CDE ; ID_Start # Lo KANNADA LETTER FA +0CE0..0CE1 ; ID_Start # Lo [2] KANNADA LETTER VOCALIC RR..KANNADA LETTER VOCALIC LL +0D05..0D0C ; ID_Start # Lo [8] MALAYALAM LETTER A..MALAYALAM LETTER VOCALIC L From noreply at buildbot.pypy.org Mon Jan 28 09:53:47 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 28 Jan 2013 09:53:47 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: some extra security Message-ID: <20130128085347.3D4A01C0543@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60585:540d7e169bac Date: 2013-01-28 10:48 +0200 http://bitbucket.org/pypy/pypy/changeset/540d7e169bac/ Log: some extra security diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -2271,7 +2271,9 @@ else: raise AssertionError(kind) - value = rffi.cast(lltype.Signed, cast_instance_to_gcref(value)) + gcref = cast_instance_to_gcref(value) + rgc._make_sure_does_not_move(gcref) + value = rffi.cast(lltype.Signed, gcref) base_ofs = self.cpu.get_baseofs_of_frame_field() ofs = self.cpu.get_ofs_of_frame_field('jf_descr') self.mc.CMP_mi((eax.value, base_ofs + ofs), value) From noreply at buildbot.pypy.org Mon Jan 28 09:53:48 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 28 Jan 2013 09:53:48 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: Make sure we call make_done_loop_tokens only once Message-ID: <20130128085348.813861C067C@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60586:169ce263f181 Date: 2013-01-28 10:53 +0200 http://bitbucket.org/pypy/pypy/changeset/169ce263f181/ Log: Make sure we call make_done_loop_tokens only once diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py --- a/rpython/jit/backend/model.py +++ b/rpython/jit/backend/model.py @@ -1,5 +1,4 @@ from rpython.rlib.debug import debug_start, debug_print, debug_stop -from rpython.jit.metainterp import compile from rpython.rtyper.lltypesystem import lltype class CPUTotalTracker(object): @@ -23,7 +22,6 @@ propagate_exception_descr = None def __init__(self): - self.__dict__.update(compile.make_done_loop_tokens()) self.tracker = CPUTotalTracker() def _freeze_(self): diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py --- a/rpython/jit/backend/test/runner_test.py +++ b/rpython/jit/backend/test/runner_test.py @@ -2787,6 +2787,7 @@ looptoken = JitCellToken() looptoken.outermost_jitdriver_sd = FakeJitDriverSD() finish_descr = loop.operations[-1].getdescr() + self.cpu.done_with_this_frame_descr_int = BasicFinalDescr() self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) ARGS = [lltype.Signed] * 10 RES = lltype.Signed @@ -2858,6 +2859,7 @@ finish_descr = loop.operations[-1].getdescr() looptoken = JitCellToken() looptoken.outermost_jitdriver_sd = FakeJitDriverSD() + self.cpu.done_with_this_frame_descr_float = BasicFinalDescr() self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) args = [longlong.getfloatstorage(1.2), longlong.getfloatstorage(2.3)] @@ -2950,6 +2952,7 @@ loop = parse(ops) looptoken = JitCellToken() looptoken.outermost_jitdriver_sd = FakeJitDriverSD() + self.cpu.done_with_this_frame_descr_float = BasicFinalDescr() self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) finish_descr = loop.operations[-1].getdescr() args = [longlong.getfloatstorage(1.25), diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -1420,6 +1420,11 @@ self._addr2name_values = [] self.__dict__.update(compile.make_done_loop_tokens()) + for val in ['int', 'float', 'ref', 'void']: + fullname = 'done_with_this_frame_descr_' + val + setattr(self.cpu, fullname, getattr(self, fullname)) + d = self.exit_frame_with_exception_descr_ref + self.cpu.exit_frame_with_exception_descr_ref = d def _freeze_(self): return True From noreply at buildbot.pypy.org Mon Jan 28 09:54:14 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 28 Jan 2013 09:54:14 +0100 (CET) Subject: [pypy-commit] pypy default: fix some tests for win32 Message-ID: <20130128085414.0C8311C0543@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60587:d236d76a099a Date: 2013-01-28 03:53 -0500 http://bitbucket.org/pypy/pypy/changeset/d236d76a099a/ Log: fix some tests for win32 diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py --- a/pypy/module/_file/test/test_file.py +++ b/pypy/module/_file/test/test_file.py @@ -430,7 +430,7 @@ assert f.subclass_closed def test_readline_unbuffered_should_read_one_line_only(self): - import posix + import os with self.file(self.temppath, 'w') as f: f.write('foo\nbar\n') @@ -438,7 +438,7 @@ with self.file(self.temppath, 'r', 0) as f: s = f.readline() assert s == 'foo\n' - s = posix.read(f.fileno(), 10) + s = os.read(f.fileno(), 10) assert s == 'bar\n' def test_flush_at_exit(): diff --git a/pypy/module/_file/test/test_file_extra.py b/pypy/module/_file/test/test_file_extra.py --- a/pypy/module/_file/test/test_file_extra.py +++ b/pypy/module/_file/test/test_file_extra.py @@ -612,6 +612,7 @@ repr(unicode(self.temptestfile))) f.close() + @py.test.mark.skipif("os.name != 'posix'") def test_EAGAIN(self): import _socket, posix s1, s2 = _socket.socketpair() diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py --- a/pypy/module/rctime/test/test_rctime.py +++ b/pypy/module/rctime/test/test_rctime.py @@ -114,7 +114,10 @@ assert rctime.mktime(rctime.localtime(-1)) == -1 res = rctime.mktime((2000, 1, 1, 0, 0, 0, -1, -1, -1)) - assert rctime.ctime(res) == 'Sat Jan 1 00:00:00 2000' + if os.name == 'nt': + assert rctime.ctime(res) == 'Sat Jan 01 00:00:00 2000' + else: + assert rctime.ctime(res) == 'Sat Jan 1 00:00:00 2000' def test_asctime(self): import time as rctime From noreply at buildbot.pypy.org Mon Jan 28 09:54:15 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 28 Jan 2013 09:54:15 +0100 (CET) Subject: [pypy-commit] pypy default: merge heads Message-ID: <20130128085415.5DD5D1C0543@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60588:d1767635d7c4 Date: 2013-01-28 03:54 -0500 http://bitbucket.org/pypy/pypy/changeset/d1767635d7c4/ Log: merge heads diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py --- a/pypy/module/signal/interp_signal.py +++ b/pypy/module/signal/interp_signal.py @@ -19,7 +19,8 @@ class SignalActionFlag(AbstractActionFlag): # This class uses the C-level pypysig_counter variable as the tick # counter. The C-level signal handler will reset it to -1 whenever - # a signal is received. + # a signal is received. This causes CheckSignalAction.perform() to + # be called. def get_ticker(self): p = pypysig_getaddr_occurred() @@ -47,45 +48,40 @@ def __init__(self, space): AsyncAction.__init__(self, space) self.handlers_w = {} - if space.config.objspace.usemodules.thread: - # need a helper action in case signals arrive in a non-main thread - self.pending_signals = {} - self.reissue_signal_action = ReissueSignalAction(space) - else: - self.reissue_signal_action = None + self.emulated_sigint = False @jit.dont_look_inside def perform(self, executioncontext, frame): - while True: - n = pypysig_poll() - if n < 0: - break - self.perform_signal(executioncontext, n) - - @jit.dont_look_inside - def perform_signal(self, executioncontext, n): - if self.reissue_signal_action is None: - # no threads: we can report the signal immediately - self.report_signal(n) + if self.space.config.objspace.usemodules.thread: + main_ec = self.space.threadlocals.getmainthreadvalue() + in_main = executioncontext is main_ec else: - main_ec = self.space.threadlocals.getmainthreadvalue() - if executioncontext is main_ec: - # running in the main thread: we can report the - # signal immediately - self.report_signal(n) - else: - # running in another thread: we need to hack a bit - self.pending_signals[n] = None - self.reissue_signal_action.fire_after_thread_switch() + in_main = True + # If we are in the main thread, poll and report the signals now. + if in_main: + if self.emulated_sigint: + self.emulated_sigint = False + self._report_signal(cpy_signal.SIGINT) + while True: + n = pypysig_poll() + if n < 0: + break + self._report_signal(n) + else: + # Otherwise, don't call pypysig_poll() at all. Instead, + # arrange for perform() to be called again after a thread + # switch. It might be called again and again, until we + # land in the main thread. + self.fire_after_thread_switch() @jit.dont_look_inside def set_interrupt(self): "Simulates the effect of a SIGINT signal arriving" ec = self.space.getexecutioncontext() - self.perform_signal(ec, cpy_signal.SIGINT) + self.emulated_sigint = True + self.perform(ec, None) - @jit.dont_look_inside - def report_signal(self, n): + def _report_signal(self, n): try: w_handler = self.handlers_w[n] except KeyError: @@ -100,39 +96,6 @@ w_frame = space.wrap(ec.gettopframe_nohidden()) space.call_function(w_handler, space.wrap(n), w_frame) - @jit.dont_look_inside - def report_pending_signals(self): - # XXX this logic isn't so complicated but I have no clue how - # to test it :-( - pending_signals = self.pending_signals.keys() - self.pending_signals.clear() - try: - while pending_signals: - self.report_signal(pending_signals.pop()) - finally: - # in case of exception, put the undelivered signals back - # into the dict instead of silently swallowing them - if pending_signals: - for n in pending_signals: - self.pending_signals[n] = None - self.reissue_signal_action.fire() - - -class ReissueSignalAction(AsyncAction): - """A special action to help deliver signals to the main thread. If - a non-main thread caught a signal, this action fires after every - thread switch until we land in the main thread. - """ - - def perform(self, executioncontext, frame): - main_ec = self.space.threadlocals.getmainthreadvalue() - if executioncontext is main_ec: - # now running in the main thread: we can really report the signals - self.space.check_signal_action.report_pending_signals() - else: - # still running in some other thread: try again later - self.fire_after_thread_switch() - @unwrap_spec(signum=int) def getsignal(space, signum): diff --git a/pypy/module/thread/os_local.py b/pypy/module/thread/os_local.py --- a/pypy/module/thread/os_local.py +++ b/pypy/module/thread/os_local.py @@ -98,5 +98,5 @@ local = wref() if local is not None: del local.dicts[ec] - local.last_dict = None - local.last_ec = None + local.last_dict = None + local.last_ec = None From noreply at buildbot.pypy.org Mon Jan 28 10:04:42 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Mon, 28 Jan 2013 10:04:42 +0100 (CET) Subject: [pypy-commit] pypy default: Some small style cleanups to the signal module. Message-ID: <20130128090442.344C51C12E1@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: Changeset: r60589:558a57a27b19 Date: 2013-01-28 01:03 -0800 http://bitbucket.org/pypy/pypy/changeset/558a57a27b19/ Log: Some small style cleanups to the signal module. diff --git a/pypy/module/signal/__init__.py b/pypy/module/signal/__init__.py --- a/pypy/module/signal/__init__.py +++ b/pypy/module/signal/__init__.py @@ -30,9 +30,10 @@ } def buildloaders(cls): - from pypy.module.signal import interp_signal - for name in interp_signal.signal_names: - signum = getattr(interp_signal, name) + from rpython.rlib import rsignal + + for name in rsignal.signal_names: + signum = getattr(rsignal, name) if signum is not None: Module.interpleveldefs[name] = 'space.wrap(%d)' % (signum,) super(Module, cls).buildloaders() diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py --- a/pypy/module/signal/interp_signal.py +++ b/pypy/module/signal/interp_signal.py @@ -1,21 +1,26 @@ from __future__ import with_statement + +import signal as cpy_signal +import sys + from pypy.interpreter.error import OperationError, exception_from_errno -from pypy.interpreter.executioncontext import AsyncAction, AbstractActionFlag -from pypy.interpreter.executioncontext import PeriodicAsyncAction +from pypy.interpreter.executioncontext import (AsyncAction, AbstractActionFlag, + PeriodicAsyncAction) from pypy.interpreter.gateway import unwrap_spec -import signal as cpy_signal -from rpython.rtyper.lltypesystem import lltype, rffi -from rpython.rtyper.tool import rffi_platform -from rpython.translator.tool.cbuild import ExternalCompilationInfo -from rpython.conftest import cdir -import py -import sys + from rpython.rlib import jit, rposix from rpython.rlib.rarithmetic import intmask -from rpython.rlib.rsignal import * +from rpython.rlib.rsignal import (pypysig_getaddr_occurred, pypysig_setflag, + pypysig_poll, pypysig_reinstall, pypysig_ignore, pypysig_default, + pypysig_set_wakeup_fd, c_alarm, c_pause, c_getitimer, c_setitimer, + c_siginterrupt, itimervalP, NSIG, SIG_DFL, SIG_IGN, ITIMER_REAL, + ITIMER_PROF, ITIMER_VIRTUAL, signal_values) +from rpython.rtyper.lltypesystem import lltype, rffi + WIN32 = sys.platform == 'win32' + class SignalActionFlag(AbstractActionFlag): # This class uses the C-level pypysig_counter variable as the tick # counter. The C-level signal handler will reset it to -1 whenever @@ -117,6 +122,7 @@ return action.handlers_w[signum] return space.wrap(SIG_DFL) + def default_int_handler(space, w_signum, w_frame): """ default_int_handler(...) @@ -127,22 +133,26 @@ raise OperationError(space.w_KeyboardInterrupt, space.w_None) + @jit.dont_look_inside @unwrap_spec(timeout=int) def alarm(space, timeout): return space.wrap(c_alarm(timeout)) + @jit.dont_look_inside def pause(space): c_pause() return space.w_None + def check_signum_exists(space, signum): if signum in signal_values: return raise OperationError(space.w_ValueError, space.wrap("invalid signal value")) + def check_signum_in_range(space, signum): if 1 <= signum < NSIG: return @@ -164,7 +174,7 @@ A signal handler function is called with two arguments: the first is the signal number, the second is the interrupted stack frame. """ - ec = space.getexecutioncontext() + ec = space.getexecutioncontext() main_ec = space.threadlocals.getmainthreadvalue() old_handler = getsignal(space, signum) @@ -187,13 +197,14 @@ action.handlers_w[signum] = w_handler return old_handler + @jit.dont_look_inside @unwrap_spec(fd=int) def set_wakeup_fd(space, fd): """Sets the fd to be written to (with '\0') when a signal comes in. Returns the old fd. A library can use this to wakeup select or poll. The previous fd is returned. - + The fd must be non-blocking. """ if space.config.objspace.usemodules.thread: @@ -206,6 +217,7 @@ old_fd = pypysig_set_wakeup_fd(fd) return space.wrap(intmask(old_fd)) + @jit.dont_look_inside @unwrap_spec(signum=int, flag=int) def siginterrupt(space, signum, flag): @@ -221,33 +233,38 @@ rffi.setintfield(timeval, 'c_tv_sec', int(d)) rffi.setintfield(timeval, 'c_tv_usec', int((d - int(d)) * 1000000)) + def double_from_timeval(tv): return rffi.getintfield(tv, 'c_tv_sec') + ( rffi.getintfield(tv, 'c_tv_usec') / 1000000.0) + def itimer_retval(space, val): w_value = space.wrap(double_from_timeval(val.c_it_value)) w_interval = space.wrap(double_from_timeval(val.c_it_interval)) return space.newtuple([w_value, w_interval]) + class Cache: def __init__(self, space): self.w_itimererror = space.new_exception_class("signal.ItimerError", space.w_IOError) + def get_itimer_error(space): return space.fromcache(Cache).w_itimererror + @jit.dont_look_inside @unwrap_spec(which=int, first=float, interval=float) def setitimer(space, which, first, interval=0): """setitimer(which, seconds[, interval]) - Sets given itimer (one of ITIMER_REAL, ITIMER_VIRTUAL + or ITIMER_PROF) to fire after value seconds and after that every interval seconds. The itimer can be cleared by setting seconds to zero. - + Returns old values as a tuple: (delay, interval). """ with lltype.scoped_alloc(itimervalP.TO, 1) as new: @@ -261,14 +278,14 @@ if ret != 0: raise exception_from_errno(space, get_itimer_error(space)) + return itimer_retval(space, old[0]) - return itimer_retval(space, old[0]) @jit.dont_look_inside @unwrap_spec(which=int) def getitimer(space, which): """getitimer(which) - + Returns current value of given itimer. """ with lltype.scoped_alloc(itimervalP.TO, 1) as old: From noreply at buildbot.pypy.org Mon Jan 28 10:04:43 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Mon, 28 Jan 2013 10:04:43 +0100 (CET) Subject: [pypy-commit] pypy default: merged upstream Message-ID: <20130128090443.82C301C12E1@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: Changeset: r60590:39bc105d07db Date: 2013-01-28 01:04 -0800 http://bitbucket.org/pypy/pypy/changeset/39bc105d07db/ Log: merged upstream diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py --- a/pypy/module/_file/test/test_file.py +++ b/pypy/module/_file/test/test_file.py @@ -430,7 +430,7 @@ assert f.subclass_closed def test_readline_unbuffered_should_read_one_line_only(self): - import posix + import os with self.file(self.temppath, 'w') as f: f.write('foo\nbar\n') @@ -438,7 +438,7 @@ with self.file(self.temppath, 'r', 0) as f: s = f.readline() assert s == 'foo\n' - s = posix.read(f.fileno(), 10) + s = os.read(f.fileno(), 10) assert s == 'bar\n' def test_flush_at_exit(): diff --git a/pypy/module/_file/test/test_file_extra.py b/pypy/module/_file/test/test_file_extra.py --- a/pypy/module/_file/test/test_file_extra.py +++ b/pypy/module/_file/test/test_file_extra.py @@ -612,6 +612,7 @@ repr(unicode(self.temptestfile))) f.close() + @py.test.mark.skipif("os.name != 'posix'") def test_EAGAIN(self): import _socket, posix s1, s2 = _socket.socketpair() diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py --- a/pypy/module/rctime/test/test_rctime.py +++ b/pypy/module/rctime/test/test_rctime.py @@ -114,7 +114,10 @@ assert rctime.mktime(rctime.localtime(-1)) == -1 res = rctime.mktime((2000, 1, 1, 0, 0, 0, -1, -1, -1)) - assert rctime.ctime(res) == 'Sat Jan 1 00:00:00 2000' + if os.name == 'nt': + assert rctime.ctime(res) == 'Sat Jan 01 00:00:00 2000' + else: + assert rctime.ctime(res) == 'Sat Jan 1 00:00:00 2000' def test_asctime(self): import time as rctime From noreply at buildbot.pypy.org Mon Jan 28 10:07:34 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 28 Jan 2013 10:07:34 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fix never-executed-so-far path Message-ID: <20130128090734.37CE41C12E1@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60591:540f951e15e4 Date: 2013-01-28 11:07 +0200 http://bitbucket.org/pypy/pypy/changeset/540f951e15e4/ Log: fix never-executed-so-far path diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -2304,7 +2304,7 @@ fielddescr = jd.vable_token_descr assert isinstance(fielddescr, FieldDescr) vtoken_ofs = fielddescr.offset - vable_ofs = (jd.index_of_virtualizable + JITFRAME_FIXED_SIZE) * WORD + vable_ofs = base_ofs + (jd.index_of_virtualizable + JITFRAME_FIXED_SIZE) * WORD self.mc.MOV_rm(edx.value, (eax.value, vable_ofs)) self.mc.MOV_mi((edx.value, vtoken_ofs), 0) # in the line above, TOKEN_NONE = 0 From noreply at buildbot.pypy.org Mon Jan 28 10:17:29 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 28 Jan 2013 10:17:29 +0100 (CET) Subject: [pypy-commit] pypy default: fix the backported py3k thread_nt.c Message-ID: <20130128091729.969071C12E1@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60592:6e3f502b3be7 Date: 2013-01-28 04:16 -0500 http://bitbucket.org/pypy/pypy/changeset/6e3f502b3be7/ Log: fix the backported py3k thread_nt.c diff --git a/rpython/translator/c/src/thread_nt.c b/rpython/translator/c/src/thread_nt.c --- a/rpython/translator/c/src/thread_nt.c +++ b/rpython/translator/c/src/thread_nt.c @@ -112,7 +112,7 @@ mutex->sem = NULL ; /* Just in case */ } -DWORD EnterNonRecursiveMutex(PNRMUTEX mutex, BOOL wait) +DWORD EnterNonRecursiveMutex(PNRMUTEX mutex, DWORD milliseconds) { return WaitForSingleObject(mutex->sem, milliseconds); } @@ -151,8 +151,8 @@ { /* Fow now, intr_flag does nothing on Windows, and lock acquires are * uninterruptible. */ - PyLockStatus success; - PY_TIMEOUT_T milliseconds; + RPyLockStatus success; + RPY_TIMEOUT_T milliseconds; if (microseconds >= 0) { milliseconds = microseconds / 1000; @@ -167,10 +167,10 @@ if (lock && EnterNonRecursiveMutex( lock, (DWORD)milliseconds) == WAIT_OBJECT_0) { - success = PY_LOCK_ACQUIRED; + success = RPY_LOCK_ACQUIRED; } else { - success = PY_LOCK_FAILURE; + success = RPY_LOCK_FAILURE; } return success; From noreply at buildbot.pypy.org Mon Jan 28 10:17:30 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 28 Jan 2013 10:17:30 +0100 (CET) Subject: [pypy-commit] pypy default: merge heads Message-ID: <20130128091730.DD5FC1C12E1@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60593:8276dd6cc7cd Date: 2013-01-28 04:17 -0500 http://bitbucket.org/pypy/pypy/changeset/8276dd6cc7cd/ Log: merge heads diff --git a/pypy/module/signal/__init__.py b/pypy/module/signal/__init__.py --- a/pypy/module/signal/__init__.py +++ b/pypy/module/signal/__init__.py @@ -30,9 +30,10 @@ } def buildloaders(cls): - from pypy.module.signal import interp_signal - for name in interp_signal.signal_names: - signum = getattr(interp_signal, name) + from rpython.rlib import rsignal + + for name in rsignal.signal_names: + signum = getattr(rsignal, name) if signum is not None: Module.interpleveldefs[name] = 'space.wrap(%d)' % (signum,) super(Module, cls).buildloaders() diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py --- a/pypy/module/signal/interp_signal.py +++ b/pypy/module/signal/interp_signal.py @@ -1,21 +1,26 @@ from __future__ import with_statement + +import signal as cpy_signal +import sys + from pypy.interpreter.error import OperationError, exception_from_errno -from pypy.interpreter.executioncontext import AsyncAction, AbstractActionFlag -from pypy.interpreter.executioncontext import PeriodicAsyncAction +from pypy.interpreter.executioncontext import (AsyncAction, AbstractActionFlag, + PeriodicAsyncAction) from pypy.interpreter.gateway import unwrap_spec -import signal as cpy_signal -from rpython.rtyper.lltypesystem import lltype, rffi -from rpython.rtyper.tool import rffi_platform -from rpython.translator.tool.cbuild import ExternalCompilationInfo -from rpython.conftest import cdir -import py -import sys + from rpython.rlib import jit, rposix from rpython.rlib.rarithmetic import intmask -from rpython.rlib.rsignal import * +from rpython.rlib.rsignal import (pypysig_getaddr_occurred, pypysig_setflag, + pypysig_poll, pypysig_reinstall, pypysig_ignore, pypysig_default, + pypysig_set_wakeup_fd, c_alarm, c_pause, c_getitimer, c_setitimer, + c_siginterrupt, itimervalP, NSIG, SIG_DFL, SIG_IGN, ITIMER_REAL, + ITIMER_PROF, ITIMER_VIRTUAL, signal_values) +from rpython.rtyper.lltypesystem import lltype, rffi + WIN32 = sys.platform == 'win32' + class SignalActionFlag(AbstractActionFlag): # This class uses the C-level pypysig_counter variable as the tick # counter. The C-level signal handler will reset it to -1 whenever @@ -117,6 +122,7 @@ return action.handlers_w[signum] return space.wrap(SIG_DFL) + def default_int_handler(space, w_signum, w_frame): """ default_int_handler(...) @@ -127,22 +133,26 @@ raise OperationError(space.w_KeyboardInterrupt, space.w_None) + @jit.dont_look_inside @unwrap_spec(timeout=int) def alarm(space, timeout): return space.wrap(c_alarm(timeout)) + @jit.dont_look_inside def pause(space): c_pause() return space.w_None + def check_signum_exists(space, signum): if signum in signal_values: return raise OperationError(space.w_ValueError, space.wrap("invalid signal value")) + def check_signum_in_range(space, signum): if 1 <= signum < NSIG: return @@ -164,7 +174,7 @@ A signal handler function is called with two arguments: the first is the signal number, the second is the interrupted stack frame. """ - ec = space.getexecutioncontext() + ec = space.getexecutioncontext() main_ec = space.threadlocals.getmainthreadvalue() old_handler = getsignal(space, signum) @@ -187,13 +197,14 @@ action.handlers_w[signum] = w_handler return old_handler + @jit.dont_look_inside @unwrap_spec(fd=int) def set_wakeup_fd(space, fd): """Sets the fd to be written to (with '\0') when a signal comes in. Returns the old fd. A library can use this to wakeup select or poll. The previous fd is returned. - + The fd must be non-blocking. """ if space.config.objspace.usemodules.thread: @@ -206,6 +217,7 @@ old_fd = pypysig_set_wakeup_fd(fd) return space.wrap(intmask(old_fd)) + @jit.dont_look_inside @unwrap_spec(signum=int, flag=int) def siginterrupt(space, signum, flag): @@ -221,33 +233,38 @@ rffi.setintfield(timeval, 'c_tv_sec', int(d)) rffi.setintfield(timeval, 'c_tv_usec', int((d - int(d)) * 1000000)) + def double_from_timeval(tv): return rffi.getintfield(tv, 'c_tv_sec') + ( rffi.getintfield(tv, 'c_tv_usec') / 1000000.0) + def itimer_retval(space, val): w_value = space.wrap(double_from_timeval(val.c_it_value)) w_interval = space.wrap(double_from_timeval(val.c_it_interval)) return space.newtuple([w_value, w_interval]) + class Cache: def __init__(self, space): self.w_itimererror = space.new_exception_class("signal.ItimerError", space.w_IOError) + def get_itimer_error(space): return space.fromcache(Cache).w_itimererror + @jit.dont_look_inside @unwrap_spec(which=int, first=float, interval=float) def setitimer(space, which, first, interval=0): """setitimer(which, seconds[, interval]) - Sets given itimer (one of ITIMER_REAL, ITIMER_VIRTUAL + or ITIMER_PROF) to fire after value seconds and after that every interval seconds. The itimer can be cleared by setting seconds to zero. - + Returns old values as a tuple: (delay, interval). """ with lltype.scoped_alloc(itimervalP.TO, 1) as new: @@ -261,14 +278,14 @@ if ret != 0: raise exception_from_errno(space, get_itimer_error(space)) + return itimer_retval(space, old[0]) - return itimer_retval(space, old[0]) @jit.dont_look_inside @unwrap_spec(which=int) def getitimer(space, which): """getitimer(which) - + Returns current value of given itimer. """ with lltype.scoped_alloc(itimervalP.TO, 1) as old: From noreply at buildbot.pypy.org Mon Jan 28 10:23:11 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 28 Jan 2013 10:23:11 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: argh, the same problem again Message-ID: <20130128092311.9CA981C12EA@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60594:30497ebd7b72 Date: 2013-01-28 11:22 +0200 http://bitbucket.org/pypy/pypy/changeset/30497ebd7b72/ Log: argh, the same problem again diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -2304,8 +2304,7 @@ fielddescr = jd.vable_token_descr assert isinstance(fielddescr, FieldDescr) vtoken_ofs = fielddescr.offset - vable_ofs = base_ofs + (jd.index_of_virtualizable + JITFRAME_FIXED_SIZE) * WORD - self.mc.MOV_rm(edx.value, (eax.value, vable_ofs)) + self.mc.MOV(edx, vloc) # we know vloc is on the current frame self.mc.MOV_mi((edx.value, vtoken_ofs), 0) # in the line above, TOKEN_NONE = 0 # From noreply at buildbot.pypy.org Mon Jan 28 10:29:05 2013 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 28 Jan 2013 10:29:05 +0100 (CET) Subject: [pypy-commit] pypy default: Fix test_jit_ffi_vref, actually by improving the code produced. Message-ID: <20130128092905.13BAA1C1306@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r60595:37b71225d44c Date: 2013-01-28 10:28 +0100 http://bitbucket.org/pypy/pypy/changeset/37b71225d44c/ Log: Fix test_jit_ffi_vref, actually by improving the code produced. diff --git a/rpython/jit/backend/llsupport/ffisupport.py b/rpython/jit/backend/llsupport/ffisupport.py --- a/rpython/jit/backend/llsupport/ffisupport.py +++ b/rpython/jit/backend/llsupport/ffisupport.py @@ -42,11 +42,14 @@ @specialize.memo() def _get_ffi2descr_dict(cpu): - d = {('v', 0): ('v', None)} + def entry(letter, TYPE): + return (letter, cpu.arraydescrof(rffi.CArray(TYPE)), rffi.sizeof(TYPE)) + # + d = {('v', 0): ('v', None, 1)} if cpu.supports_floats: - d[('f', 0)] = ('f', cpu.arraydescrof(rffi.CArray(lltype.Float))) + d[('f', 0)] = entry('f', lltype.Float) if cpu.supports_singlefloats: - d[('S', 0)] = ('i', cpu.arraydescrof(rffi.CArray(lltype.SingleFloat))) + d[('S', 0)] = entry('i', lltype.SingleFloat) for SIGNED_TYPE in [rffi.SIGNEDCHAR, rffi.SHORT, rffi.INT, @@ -59,7 +62,7 @@ continue key = ('L', 0) kind = 'f' - d[key] = (kind, cpu.arraydescrof(rffi.CArray(SIGNED_TYPE))) + d[key] = entry(kind, SIGNED_TYPE) for UNSIGNED_TYPE in [rffi.UCHAR, rffi.USHORT, rffi.UINT, @@ -68,7 +71,7 @@ key = ('u', rffi.sizeof(UNSIGNED_TYPE)) if key[1] > rffi.sizeof(lltype.Signed): continue - d[key] = ('i', cpu.arraydescrof(rffi.CArray(UNSIGNED_TYPE))) + d[key] = entry('i', UNSIGNED_TYPE) return d def get_arg_descr(cpu, ffi_type): diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -2567,7 +2567,8 @@ self.history.operations.pop() arg_boxes = [] for i in range(cif_description.nargs): - kind, descr = get_arg_descr(self.cpu, cif_description.atypes[i]) + kind, descr, itemsize = get_arg_descr(self.cpu, + cif_description.atypes[i]) if kind == 'i': box_arg = history.BoxInt() elif kind == 'f': @@ -2576,16 +2577,14 @@ assert kind == 'v' continue ofs = cif_description.exchange_args[i] - box_argpos = history.BoxInt() - self.history.record(rop.INT_ADD, - [box_exchange_buffer, ConstInt(ofs)], - box_argpos) + assert ofs % itemsize == 0 # alignment check self.history.record(rop.GETARRAYITEM_RAW, - [box_argpos, ConstInt(0)], + [box_exchange_buffer, + ConstInt(ofs // itemsize)], box_arg, descr) arg_boxes.append(box_arg) # - kind, descr = get_arg_descr(self.cpu, cif_description.rtype) + kind, descr, itemsize = get_arg_descr(self.cpu, cif_description.rtype) if kind == 'i': box_result = history.BoxInt() elif kind == 'f': @@ -2601,12 +2600,10 @@ # if box_result is not None: ofs = cif_description.exchange_result - box_resultpos = history.BoxInt() - self.history.record(rop.INT_ADD, - [box_exchange_buffer, ConstInt(ofs)], - box_resultpos) + assert ofs % itemsize == 0 # alignment check (result) self.history.record(rop.SETARRAYITEM_RAW, - [box_resultpos, ConstInt(0), box_result], + [box_exchange_buffer, + ConstInt(ofs // itemsize), box_result], None, descr) def direct_call_release_gil(self): From noreply at buildbot.pypy.org Mon Jan 28 10:41:28 2013 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 28 Jan 2013 10:41:28 +0100 (CET) Subject: [pypy-commit] pypy default: Skip the @inline tests and disable it. Keep it around for later, maybe. Message-ID: <20130128094128.2AA311C039A@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r60596:102a9b7e5720 Date: 2013-01-28 10:41 +0100 http://bitbucket.org/pypy/pypy/changeset/102a9b7e5720/ Log: Skip the @inline tests and disable it. Keep it around for later, maybe. diff --git a/rpython/jit/metainterp/test/test_warmspot.py b/rpython/jit/metainterp/test/test_warmspot.py --- a/rpython/jit/metainterp/test/test_warmspot.py +++ b/rpython/jit/metainterp/test/test_warmspot.py @@ -401,6 +401,7 @@ assert len(oplabel.getarglist()) == 2 # 'n', 'res' in some order def test_inline_jit_merge_point(self): + py.test.skip("fix the test if you want to re-enable this") # test that the machinery to inline jit_merge_points in callers # works. The final user does not need to mess manually with the # _inline_jit_merge_point_ attribute and similar, it is all nicely @@ -430,6 +431,7 @@ self.check_resops(int_add=4) def test_jitdriver_inline(self): + py.test.skip("fix the test if you want to re-enable this") myjitdriver = JitDriver(greens = [], reds = 'auto') class MyRange(object): def __init__(self, n): @@ -462,6 +464,7 @@ self.check_trace_count(1) def test_jitdriver_inline_twice(self): + py.test.skip("fix the test if you want to re-enable this") myjitdriver = JitDriver(greens = [], reds = 'auto') def jit_merge_point(a, b): @@ -492,6 +495,7 @@ self.check_trace_count(2) def test_jitdriver_inline_exception(self): + py.test.skip("fix the test if you want to re-enable this") # this simulates what happens in a real case scenario: inside the next # we have a call which we cannot inline (e.g. space.next in the case # of W_InterpIterable), but we need to put it in a try/except block. diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -578,6 +578,8 @@ pass def inline(self, call_jit_merge_point): + assert False, "@inline off: see skipped failures in test_warmspot." + # assert self.autoreds, "@inline works only with reds='auto'" self.inline_jit_merge_point = True def decorate(func): From noreply at buildbot.pypy.org Mon Jan 28 10:48:21 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 28 Jan 2013 10:48:21 +0100 (CET) Subject: [pypy-commit] pypy default: another fix for thread_nt.c Message-ID: <20130128094821.D31A41C0543@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60597:1495b422b4ba Date: 2013-01-28 04:47 -0500 http://bitbucket.org/pypy/pypy/changeset/1495b422b4ba/ Log: another fix for thread_nt.c diff --git a/rpython/translator/c/src/thread_nt.c b/rpython/translator/c/src/thread_nt.c --- a/rpython/translator/c/src/thread_nt.c +++ b/rpython/translator/c/src/thread_nt.c @@ -158,9 +158,11 @@ milliseconds = microseconds / 1000; if (microseconds % 1000 > 0) ++milliseconds; - if ((DWORD) milliseconds != milliseconds) - Py_FatalError("Timeout too large for a DWORD, " - "please check PY_TIMEOUT_MAX"); + if ((DWORD) milliseconds != milliseconds) { + fprintf(stderr, "Timeout too large for a DWORD, " + "please check RPY_TIMEOUT_MAX"); + abort(); + } } else milliseconds = INFINITE; From noreply at buildbot.pypy.org Mon Jan 28 10:48:23 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 28 Jan 2013 10:48:23 +0100 (CET) Subject: [pypy-commit] pypy default: merge heads Message-ID: <20130128094823.123421C0543@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60598:c4d480f3bce6 Date: 2013-01-28 04:48 -0500 http://bitbucket.org/pypy/pypy/changeset/c4d480f3bce6/ Log: merge heads diff --git a/rpython/jit/backend/llsupport/ffisupport.py b/rpython/jit/backend/llsupport/ffisupport.py --- a/rpython/jit/backend/llsupport/ffisupport.py +++ b/rpython/jit/backend/llsupport/ffisupport.py @@ -42,11 +42,14 @@ @specialize.memo() def _get_ffi2descr_dict(cpu): - d = {('v', 0): ('v', None)} + def entry(letter, TYPE): + return (letter, cpu.arraydescrof(rffi.CArray(TYPE)), rffi.sizeof(TYPE)) + # + d = {('v', 0): ('v', None, 1)} if cpu.supports_floats: - d[('f', 0)] = ('f', cpu.arraydescrof(rffi.CArray(lltype.Float))) + d[('f', 0)] = entry('f', lltype.Float) if cpu.supports_singlefloats: - d[('S', 0)] = ('i', cpu.arraydescrof(rffi.CArray(lltype.SingleFloat))) + d[('S', 0)] = entry('i', lltype.SingleFloat) for SIGNED_TYPE in [rffi.SIGNEDCHAR, rffi.SHORT, rffi.INT, @@ -59,7 +62,7 @@ continue key = ('L', 0) kind = 'f' - d[key] = (kind, cpu.arraydescrof(rffi.CArray(SIGNED_TYPE))) + d[key] = entry(kind, SIGNED_TYPE) for UNSIGNED_TYPE in [rffi.UCHAR, rffi.USHORT, rffi.UINT, @@ -68,7 +71,7 @@ key = ('u', rffi.sizeof(UNSIGNED_TYPE)) if key[1] > rffi.sizeof(lltype.Signed): continue - d[key] = ('i', cpu.arraydescrof(rffi.CArray(UNSIGNED_TYPE))) + d[key] = entry('i', UNSIGNED_TYPE) return d def get_arg_descr(cpu, ffi_type): diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -2567,7 +2567,8 @@ self.history.operations.pop() arg_boxes = [] for i in range(cif_description.nargs): - kind, descr = get_arg_descr(self.cpu, cif_description.atypes[i]) + kind, descr, itemsize = get_arg_descr(self.cpu, + cif_description.atypes[i]) if kind == 'i': box_arg = history.BoxInt() elif kind == 'f': @@ -2576,16 +2577,14 @@ assert kind == 'v' continue ofs = cif_description.exchange_args[i] - box_argpos = history.BoxInt() - self.history.record(rop.INT_ADD, - [box_exchange_buffer, ConstInt(ofs)], - box_argpos) + assert ofs % itemsize == 0 # alignment check self.history.record(rop.GETARRAYITEM_RAW, - [box_argpos, ConstInt(0)], + [box_exchange_buffer, + ConstInt(ofs // itemsize)], box_arg, descr) arg_boxes.append(box_arg) # - kind, descr = get_arg_descr(self.cpu, cif_description.rtype) + kind, descr, itemsize = get_arg_descr(self.cpu, cif_description.rtype) if kind == 'i': box_result = history.BoxInt() elif kind == 'f': @@ -2601,12 +2600,10 @@ # if box_result is not None: ofs = cif_description.exchange_result - box_resultpos = history.BoxInt() - self.history.record(rop.INT_ADD, - [box_exchange_buffer, ConstInt(ofs)], - box_resultpos) + assert ofs % itemsize == 0 # alignment check (result) self.history.record(rop.SETARRAYITEM_RAW, - [box_resultpos, ConstInt(0), box_result], + [box_exchange_buffer, + ConstInt(ofs // itemsize), box_result], None, descr) def direct_call_release_gil(self): diff --git a/rpython/jit/metainterp/test/test_warmspot.py b/rpython/jit/metainterp/test/test_warmspot.py --- a/rpython/jit/metainterp/test/test_warmspot.py +++ b/rpython/jit/metainterp/test/test_warmspot.py @@ -401,6 +401,7 @@ assert len(oplabel.getarglist()) == 2 # 'n', 'res' in some order def test_inline_jit_merge_point(self): + py.test.skip("fix the test if you want to re-enable this") # test that the machinery to inline jit_merge_points in callers # works. The final user does not need to mess manually with the # _inline_jit_merge_point_ attribute and similar, it is all nicely @@ -430,6 +431,7 @@ self.check_resops(int_add=4) def test_jitdriver_inline(self): + py.test.skip("fix the test if you want to re-enable this") myjitdriver = JitDriver(greens = [], reds = 'auto') class MyRange(object): def __init__(self, n): @@ -462,6 +464,7 @@ self.check_trace_count(1) def test_jitdriver_inline_twice(self): + py.test.skip("fix the test if you want to re-enable this") myjitdriver = JitDriver(greens = [], reds = 'auto') def jit_merge_point(a, b): @@ -492,6 +495,7 @@ self.check_trace_count(2) def test_jitdriver_inline_exception(self): + py.test.skip("fix the test if you want to re-enable this") # this simulates what happens in a real case scenario: inside the next # we have a call which we cannot inline (e.g. space.next in the case # of W_InterpIterable), but we need to put it in a try/except block. diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -578,6 +578,8 @@ pass def inline(self, call_jit_merge_point): + assert False, "@inline off: see skipped failures in test_warmspot." + # assert self.autoreds, "@inline works only with reds='auto'" self.inline_jit_merge_point = True def decorate(func): From noreply at buildbot.pypy.org Mon Jan 28 10:51:23 2013 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 28 Jan 2013 10:51:23 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: Copy the same simplification as in trunk. Message-ID: <20130128095123.3A7F81C067C@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r60599:ab8f8aab5c76 Date: 2013-01-28 10:51 +0100 http://bitbucket.org/pypy/pypy/changeset/ab8f8aab5c76/ Log: Copy the same simplification as in trunk. diff --git a/pypy/module/signal/__init__.py b/pypy/module/signal/__init__.py --- a/pypy/module/signal/__init__.py +++ b/pypy/module/signal/__init__.py @@ -40,11 +40,11 @@ def __init__(self, space, *args): "NOT_RPYTHON" - from pypy.module.signal import sigaction + from pypy.module.signal import interp_signal MixedModule.__init__(self, space, *args) # add the signal-checking callback as an action on the space - space.check_signal_action = sigaction.CheckSignalAction(space) + space.check_signal_action = interp_signal.CheckSignalAction(space) space.actionflag.register_periodic_action(space.check_signal_action, use_bytecode_counter=False) - space.actionflag.__class__ = sigaction.SignalActionFlag + space.actionflag.__class__ = interp_signal.SignalActionFlag # xxx yes I know the previous line is a hack diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py --- a/pypy/module/signal/interp_signal.py +++ b/pypy/module/signal/interp_signal.py @@ -1,10 +1,101 @@ from __future__ import with_statement + +import signal as cpy_signal + from pypy.interpreter.error import OperationError, exception_from_errno +from pypy.interpreter.executioncontext import (AsyncAction, AbstractActionFlag, + PeriodicAsyncAction) from pypy.interpreter.gateway import unwrap_spec -from pypy.rpython.lltypesystem import lltype, rffi + from pypy.rlib import jit, rposix from pypy.rlib.rarithmetic import intmask -from pypy.rlib.rsignal import * +from pypy.rlib.rsignal import (pypysig_getaddr_occurred, pypysig_setflag, + pypysig_poll, pypysig_reinstall, pypysig_ignore, pypysig_default, + pypysig_set_wakeup_fd, c_alarm, c_pause, c_getitimer, c_setitimer, + c_siginterrupt, itimervalP, NSIG, SIG_DFL, SIG_IGN, ITIMER_REAL, + ITIMER_PROF, ITIMER_VIRTUAL, signal_values) +from pypy.rpython.lltypesystem import lltype, rffi + + +class SignalActionFlag(AbstractActionFlag): + # This class uses the C-level pypysig_counter variable as the tick + # counter. The C-level signal handler will reset it to -1 whenever + # a signal is received. This causes CheckSignalAction.perform() to + # be called. + + def get_ticker(self): + p = pypysig_getaddr_occurred() + return p.c_value + + def reset_ticker(self, value): + p = pypysig_getaddr_occurred() + p.c_value = value + + def decrement_ticker(self, by): + p = pypysig_getaddr_occurred() + value = p.c_value + if self.has_bytecode_counter: # this 'if' is constant-folded + if jit.isconstant(by) and by == 0: + pass # normally constant-folded too + else: + value -= by + p.c_value = value + return value + + +class CheckSignalAction(PeriodicAsyncAction): + """An action that is automatically invoked when a signal is received.""" + + def __init__(self, space): + AsyncAction.__init__(self, space) + self.handlers_w = {} + self.emulated_sigint = False + + @jit.dont_look_inside + def perform(self, executioncontext, frame): + if self.space.config.objspace.usemodules.thread: + main_ec = self.space.threadlocals.getmainthreadvalue() + in_main = executioncontext is main_ec + else: + in_main = True + # If we are in the main thread, poll and report the signals now. + if in_main: + if self.emulated_sigint: + self.emulated_sigint = False + self._report_signal(cpy_signal.SIGINT) + while True: + n = pypysig_poll() + if n < 0: + break + self._report_signal(n) + else: + # Otherwise, don't call pypysig_poll() at all. Instead, + # arrange for perform() to be called again after a thread + # switch. It might be called again and again, until we + # land in the main thread. + self.fire_after_thread_switch() + + @jit.dont_look_inside + def set_interrupt(self): + "Simulates the effect of a SIGINT signal arriving" + ec = self.space.getexecutioncontext() + self.emulated_sigint = True + self.perform(ec, None) + + def _report_signal(self, n): + try: + w_handler = self.handlers_w[n] + except KeyError: + return # no handler, ignore signal + space = self.space + if not space.is_true(space.callable(w_handler)): + return # w_handler is SIG_IGN or SIG_DFL? + # re-install signal handler, for OSes that clear it + pypysig_reinstall(n) + # invoke the app-level handler + ec = space.getexecutioncontext() + w_frame = space.wrap(ec.gettopframe_nohidden()) + space.call_function(w_handler, space.wrap(n), w_frame) @unwrap_spec(signum=int) diff --git a/pypy/module/signal/sigaction.py b/pypy/module/signal/sigaction.py deleted file mode 100644 --- a/pypy/module/signal/sigaction.py +++ /dev/null @@ -1,123 +0,0 @@ -from pypy.interpreter.executioncontext import AsyncAction, AbstractActionFlag -from pypy.interpreter.executioncontext import PeriodicAsyncAction -from pypy.rlib import jit -from pypy.rlib.rsignal import pypysig_getaddr_occurred -from pypy.rlib.rsignal import pypysig_poll, pypysig_reinstall - - -class SignalActionFlag(AbstractActionFlag): - # This class uses the C-level pypysig_counter variable as the tick - # counter. The C-level signal handler will reset it to -1 whenever - # a signal is received. - - def get_ticker(self): - p = pypysig_getaddr_occurred() - return p.c_value - - def reset_ticker(self, value): - p = pypysig_getaddr_occurred() - p.c_value = value - - def decrement_ticker(self, by): - p = pypysig_getaddr_occurred() - value = p.c_value - if self.has_bytecode_counter: # this 'if' is constant-folded - if jit.isconstant(by) and by == 0: - pass # normally constant-folded too - else: - value -= by - p.c_value = value - return value - - -class CheckSignalAction(PeriodicAsyncAction): - """An action that is automatically invoked when a signal is received.""" - - def __init__(self, space): - AsyncAction.__init__(self, space) - self.handlers_w = {} - if space.config.objspace.usemodules.thread: - # need a helper action in case signals arrive in a non-main thread - self.pending_signals = {} - self.reissue_signal_action = ReissueSignalAction(space) - else: - self.reissue_signal_action = None - - @jit.dont_look_inside - def perform(self, executioncontext, frame): - while True: - n = pypysig_poll() - if n < 0: - break - self.perform_signal(executioncontext, n) - - @jit.dont_look_inside - def perform_signal(self, executioncontext, n): - if self.reissue_signal_action is None: - # no threads: we can report the signal immediately - self.report_signal(n) - else: - main_ec = self.space.threadlocals.getmainthreadvalue() - if executioncontext is main_ec: - # running in the main thread: we can report the - # signal immediately - self.report_signal(n) - else: - # running in another thread: we need to hack a bit - self.pending_signals[n] = None - self.reissue_signal_action.fire_after_thread_switch() - - @jit.dont_look_inside - def set_interrupt(self): - "Simulates the effect of a SIGINT signal arriving" - ec = self.space.getexecutioncontext() - self.perform_signal(ec, cpy_signal.SIGINT) - - @jit.dont_look_inside - def report_signal(self, n): - try: - w_handler = self.handlers_w[n] - except KeyError: - return # no handler, ignore signal - space = self.space - if not space.is_true(space.callable(w_handler)): - return # w_handler is SIG_IGN or SIG_DFL? - # re-install signal handler, for OSes that clear it - pypysig_reinstall(n) - # invoke the app-level handler - ec = space.getexecutioncontext() - w_frame = space.wrap(ec.gettopframe_nohidden()) - space.call_function(w_handler, space.wrap(n), w_frame) - - @jit.dont_look_inside - def report_pending_signals(self): - # XXX this logic isn't so complicated but I have no clue how - # to test it :-( - pending_signals = self.pending_signals.keys() - self.pending_signals.clear() - try: - while pending_signals: - self.report_signal(pending_signals.pop()) - finally: - # in case of exception, put the undelivered signals back - # into the dict instead of silently swallowing them - if pending_signals: - for n in pending_signals: - self.pending_signals[n] = None - self.reissue_signal_action.fire() - - -class ReissueSignalAction(AsyncAction): - """A special action to help deliver signals to the main thread. If - a non-main thread caught a signal, this action fires after every - thread switch until we land in the main thread. - """ - - def perform(self, executioncontext, frame): - main_ec = self.space.threadlocals.getmainthreadvalue() - if executioncontext is main_ec: - # now running in the main thread: we can really report the signals - self.space.check_signal_action.report_pending_signals() - else: - # still running in some other thread: try again later - self.fire_after_thread_switch() From noreply at buildbot.pypy.org Mon Jan 28 10:53:18 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 28 Jan 2013 10:53:18 +0100 (CET) Subject: [pypy-commit] pypy default: argh Message-ID: <20130128095318.427031C13A0@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60600:8164c439d288 Date: 2013-01-28 04:53 -0500 http://bitbucket.org/pypy/pypy/changeset/8164c439d288/ Log: argh diff --git a/rpython/translator/c/src/thread_nt.c b/rpython/translator/c/src/thread_nt.c --- a/rpython/translator/c/src/thread_nt.c +++ b/rpython/translator/c/src/thread_nt.c @@ -5,6 +5,7 @@ /* Eliminated some memory leaks, gsw at agere.com */ #include +#include #include #include From noreply at buildbot.pypy.org Mon Jan 28 11:04:16 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 28 Jan 2013 11:04:16 +0100 (CET) Subject: [pypy-commit] pypy default: this style cleanup broke translation on win32, reverting until can be styled properly Message-ID: <20130128100416.A357E1C13A0@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60601:05361d0bffe5 Date: 2013-01-28 05:04 -0500 http://bitbucket.org/pypy/pypy/changeset/05361d0bffe5/ Log: this style cleanup broke translation on win32, reverting until can be styled properly diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py --- a/pypy/module/signal/interp_signal.py +++ b/pypy/module/signal/interp_signal.py @@ -10,11 +10,7 @@ from rpython.rlib import jit, rposix from rpython.rlib.rarithmetic import intmask -from rpython.rlib.rsignal import (pypysig_getaddr_occurred, pypysig_setflag, - pypysig_poll, pypysig_reinstall, pypysig_ignore, pypysig_default, - pypysig_set_wakeup_fd, c_alarm, c_pause, c_getitimer, c_setitimer, - c_siginterrupt, itimervalP, NSIG, SIG_DFL, SIG_IGN, ITIMER_REAL, - ITIMER_PROF, ITIMER_VIRTUAL, signal_values) +from rpython.rlib.rsignal import * from rpython.rtyper.lltypesystem import lltype, rffi From noreply at buildbot.pypy.org Mon Jan 28 11:17:53 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 28 Jan 2013 11:17:53 +0100 (CET) Subject: [pypy-commit] pypy default: another test fix for win32 Message-ID: <20130128101753.C15A01C039A@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60602:3d0d298b83de Date: 2013-01-28 05:17 -0500 http://bitbucket.org/pypy/pypy/changeset/3d0d298b83de/ Log: another test fix for win32 diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py --- a/pypy/module/_file/test/test_file.py +++ b/pypy/module/_file/test/test_file.py @@ -432,10 +432,10 @@ def test_readline_unbuffered_should_read_one_line_only(self): import os - with self.file(self.temppath, 'w') as f: + with self.file(self.temppath, 'wb') as f: f.write('foo\nbar\n') - with self.file(self.temppath, 'r', 0) as f: + with self.file(self.temppath, 'rb', 0) as f: s = f.readline() assert s == 'foo\n' s = os.read(f.fileno(), 10) From noreply at buildbot.pypy.org Mon Jan 28 11:52:26 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 28 Jan 2013 11:52:26 +0100 (CET) Subject: [pypy-commit] pypy default: disable @inline's test Message-ID: <20130128105226.E28B21C0543@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60603:15a90151b233 Date: 2013-01-28 05:52 -0500 http://bitbucket.org/pypy/pypy/changeset/15a90151b233/ Log: disable @inline's test diff --git a/rpython/rlib/test/test_jit.py b/rpython/rlib/test/test_jit.py --- a/rpython/rlib/test/test_jit.py +++ b/rpython/rlib/test/test_jit.py @@ -35,6 +35,7 @@ assert driver.reds == ['a', 'b'] assert driver.numreds == 2 + at py.test.mark.xfail(reason="@inline off: see skipped failures in test_warmspot.") def test_jitdriver_inline(): driver = JitDriver(greens=[], reds='auto') calls = [] From noreply at buildbot.pypy.org Mon Jan 28 12:16:09 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 28 Jan 2013 12:16:09 +0100 (CET) Subject: [pypy-commit] pypy default: another @inline test to skip Message-ID: <20130128111609.6E1561C0264@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60604:829c136ca2df Date: 2013-01-28 06:15 -0500 http://bitbucket.org/pypy/pypy/changeset/829c136ca2df/ Log: another @inline test to skip diff --git a/rpython/rlib/test/test_jit.py b/rpython/rlib/test/test_jit.py --- a/rpython/rlib/test/test_jit.py +++ b/rpython/rlib/test/test_jit.py @@ -55,6 +55,7 @@ ('bar', 40, 2), ] + at py.test.mark.xfail(reason="@inline off: see skipped failures in test_warmspot.") def test_jitdriver_clone(): def bar(): pass def foo(): pass From noreply at buildbot.pypy.org Mon Jan 28 12:56:31 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 28 Jan 2013 12:56:31 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: enable JIT for stackless Message-ID: <20130128115631.57C891C12FC@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60605:f4a2add377b0 Date: 2013-01-28 13:55 +0200 http://bitbucket.org/pypy/pypy/changeset/f4a2add377b0/ Log: enable JIT for stackless diff --git a/pypy/module/_continuation/interp_continuation.py b/pypy/module/_continuation/interp_continuation.py --- a/pypy/module/_continuation/interp_continuation.py +++ b/pypy/module/_continuation/interp_continuation.py @@ -34,7 +34,7 @@ if self.sthread is not None: raise geterror(self.space, "continulet already __init__ialized") sthread = build_sthread(self.space) - workaround_disable_jit(sthread) + #workaround_disable_jit(sthread) # # hackish: build the frame "by hand", passing it the correct arguments space = self.space @@ -77,7 +77,7 @@ global_state.clear() raise geterror(self.space, "continulet already finished") self.check_sthread() - workaround_disable_jit(self.sthread) + #workaround_disable_jit(self.sthread) # global_state.origin = self if to is None: From noreply at buildbot.pypy.org Mon Jan 28 14:30:09 2013 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 28 Jan 2013 14:30:09 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: Trying out signals Message-ID: <20130128133009.E8DF31C12E1@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r60606:f04476ddc1d9 Date: 2013-01-28 10:56 +0100 http://bitbucket.org/pypy/pypy/changeset/f04476ddc1d9/ Log: Trying out signals diff --git a/pypy/rlib/rsignal.py b/pypy/rlib/rsignal.py --- a/pypy/rlib/rsignal.py +++ b/pypy/rlib/rsignal.py @@ -89,6 +89,7 @@ pypysig_getaddr_occurred = external('pypysig_getaddr_occurred', [], lltype.Ptr(LONG_STRUCT), _nowrapper=True, + transactionsafe=True, elidable_function=True) c_alarm = external('alarm', [rffi.INT], rffi.INT) c_pause = external('pause', [], rffi.INT, threadsafe=True) diff --git a/pypy/translator/goal/targetpypystandalone.py b/pypy/translator/goal/targetpypystandalone.py --- a/pypy/translator/goal/targetpypystandalone.py +++ b/pypy/translator/goal/targetpypystandalone.py @@ -144,7 +144,6 @@ global space, entry_point if config.translation.stm: - config.objspace.usemodules.signal = False # XXX! FIXME config.translation.thread = True if config.objspace.allworkingmodules: From noreply at buildbot.pypy.org Mon Jan 28 14:30:11 2013 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 28 Jan 2013 14:30:11 +0100 (CET) Subject: [pypy-commit] pypy default: Replace the fire_after_thread_switch() logic with different logic, Message-ID: <20130128133011.2AE861C12E1@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r60607:05672556b297 Date: 2013-01-28 14:28 +0100 http://bitbucket.org/pypy/pypy/changeset/05672556b297/ Log: Replace the fire_after_thread_switch() logic with different logic, which is just as subtle as the original one but maybe more correct :-/ diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -343,9 +343,13 @@ signal, the tick counter is set to -1 by C code in signals.h. """ assert isinstance(action, PeriodicAsyncAction) - self._periodic_actions.append(action) + # hack to put the release-the-GIL one at the end of the list, + # and the report-the-signals one at the start of the list. if use_bytecode_counter: + self._periodic_actions.append(action) self.has_bytecode_counter = True + else: + self._periodic_actions.insert(0, action) self._rebuild_action_dispatcher() def getcheckinterval(self): @@ -419,15 +423,6 @@ The action must have been registered at space initalization time.""" self.space.actionflag.fire(self) - def fire_after_thread_switch(self): - """Bit of a hack: fire() the action but only the next time the GIL - is released and re-acquired (i.e. after a potential thread switch). - Don't call this if threads are not enabled. Currently limited to - one action (i.e. reserved for CheckSignalAction from module/signal). - """ - from pypy.module.thread.gil import spacestate - spacestate.action_after_thread_switch = self - def perform(self, executioncontext, frame): """To be overridden.""" diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py --- a/pypy/module/signal/interp_signal.py +++ b/pypy/module/signal/interp_signal.py @@ -8,7 +8,8 @@ PeriodicAsyncAction) from pypy.interpreter.gateway import unwrap_spec -from rpython.rlib import jit, rposix +from rpython.rlib import jit, rposix, rgc +from rpython.rlib.objectmodel import we_are_translated from rpython.rlib.rarithmetic import intmask from rpython.rlib.rsignal import * from rpython.rtyper.lltypesystem import lltype, rffi @@ -31,6 +32,11 @@ p = pypysig_getaddr_occurred() p.c_value = value + @staticmethod + def rearm_ticker(): + p = pypysig_getaddr_occurred() + p.c_value = -1 + def decrement_ticker(self, by): p = pypysig_getaddr_occurred() value = p.c_value @@ -46,41 +52,62 @@ class CheckSignalAction(PeriodicAsyncAction): """An action that is automatically invoked when a signal is received.""" + # Note that this is a PeriodicAsyncAction: it means more precisely + # that it is called whenever the C-level ticker becomes < 0. + # Without threads, it is only ever set to -1 when we receive a + # signal. With threads, it also decrements steadily (but slowly). + def __init__(self, space): + "NOT_RPYTHON" AsyncAction.__init__(self, space) self.handlers_w = {} - self.emulated_sigint = False + self.pending_signal = -1 + self.fire_in_main_thread = False + if self.space.config.objspace.usemodules.thread: + from pypy.module.thread import gil + gil.after_thread_switch = self._after_thread_switch + + @rgc.no_collect + def _after_thread_switch(self): + if self.fire_in_main_thread: + if self.space.threadlocals.ismainthread(): + self.fire_in_main_thread = False + SignalActionFlag.rearm_ticker() + # this occurs when we just switched to the main thread + # and there is a signal pending: we force the ticker to + # -1, which should ensure perform() is called quickly. @jit.dont_look_inside def perform(self, executioncontext, frame): - if self.space.config.objspace.usemodules.thread: - main_ec = self.space.threadlocals.getmainthreadvalue() - in_main = executioncontext is main_ec - else: - in_main = True - # If we are in the main thread, poll and report the signals now. - if in_main: - if self.emulated_sigint: - self.emulated_sigint = False - self._report_signal(cpy_signal.SIGINT) - while True: - n = pypysig_poll() - if n < 0: - break + # Poll for the next signal, if any + n = self.pending_signal + if n < 0: n = pypysig_poll() + while n >= 0: + if self.space.config.objspace.usemodules.thread: + in_main = self.space.threadlocals.ismainthread() + else: + in_main = True + if in_main: + # If we are in the main thread, report the signal now, + # and poll more + self.pending_signal = -1 self._report_signal(n) - else: - # Otherwise, don't call pypysig_poll() at all. Instead, - # arrange for perform() to be called again after a thread - # switch. It might be called again and again, until we - # land in the main thread. - self.fire_after_thread_switch() + n = self.pending_signal + if n < 0: n = pypysig_poll() + else: + # Otherwise, arrange for perform() to be called again + # after we switch to the main thread. + self.fire_in_main_thread = True + break - @jit.dont_look_inside def set_interrupt(self): "Simulates the effect of a SIGINT signal arriving" - ec = self.space.getexecutioncontext() - self.emulated_sigint = True - self.perform(ec, None) + if not we_are_translated(): + self.pending_signal = cpy_signal.SIGINT + # ^^^ may override another signal, but it's just for testing + else: + pypysig_pushback(cpy_signal.SIGINT) + self.fire_in_main_thread = True def _report_signal(self, n): try: diff --git a/pypy/module/thread/gil.py b/pypy/module/thread/gil.py --- a/pypy/module/thread/gil.py +++ b/pypy/module/thread/gil.py @@ -62,22 +62,7 @@ do_yield_thread() -class SpaceState: - - def _cleanup_(self): - self.action_after_thread_switch = None - # ^^^ set by AsyncAction.fire_after_thread_switch() - - def after_thread_switch(self): - # this is support logic for the signal module, to help it deliver - # signals to the main thread. - action = self.action_after_thread_switch - if action is not None: - self.action_after_thread_switch = None - action.fire() - -spacestate = SpaceState() -spacestate._cleanup_() +after_thread_switch = lambda: None # hook for signal.py # Fragile code below. We have to preserve the C-level errno manually... @@ -94,7 +79,7 @@ e = get_errno() thread.gil_acquire() thread.gc_thread_run() - spacestate.after_thread_switch() + after_thread_switch() set_errno(e) after_external_call._gctransformer_hint_cannot_collect_ = True after_external_call._dont_reach_me_in_del_ = True @@ -112,7 +97,7 @@ # the same thread). if thread.gil_yield_thread(): thread.gc_thread_run() - spacestate.after_thread_switch() + after_thread_switch() do_yield_thread._gctransformer_hint_close_stack_ = True do_yield_thread._dont_reach_me_in_del_ = True do_yield_thread._dont_inline_ = True diff --git a/pypy/module/thread/threadlocals.py b/pypy/module/thread/threadlocals.py --- a/pypy/module/thread/threadlocals.py +++ b/pypy/module/thread/threadlocals.py @@ -48,6 +48,9 @@ ident = self._mainthreadident return self._valuedict.get(ident, None) + def ismainthread(self): + return thread.get_ident() == self._mainthreadident + def getallvalues(self): return self._valuedict diff --git a/rpython/rlib/rsignal.py b/rpython/rlib/rsignal.py --- a/rpython/rlib/rsignal.py +++ b/rpython/rlib/rsignal.py @@ -80,6 +80,8 @@ pypysig_poll = external('pypysig_poll', [], rffi.INT, threadsafe=False) # don't bother releasing the GIL around a call to pypysig_poll: it's # pointless and a performance issue +pypysig_pushback = external('pypysig_pushback', [rffi.INT], lltype.Void, + threadsafe=False) # don't use rffi.LONGP because the JIT doesn't support raw arrays so far struct_name = 'pypysig_long_struct' diff --git a/rpython/translator/c/src/signals.c b/rpython/translator/c/src/signals.c --- a/rpython/translator/c/src/signals.c +++ b/rpython/translator/c/src/signals.c @@ -71,7 +71,7 @@ #endif } -static void signal_setflag_handler(int signum) +static void pypysig_pushback(int signum) { if (0 <= signum && signum < NSIG) { @@ -79,6 +79,11 @@ pypysig_occurred = 1; pypysig_counter.value = -1; } +} + +static void signal_setflag_handler(int signum) +{ + pypysig_pushback(signum); if (wakeup_fd != -1) { diff --git a/rpython/translator/c/src/signals.h b/rpython/translator/c/src/signals.h --- a/rpython/translator/c/src/signals.h +++ b/rpython/translator/c/src/signals.h @@ -11,6 +11,7 @@ /* utility to poll for signals that arrived */ int pypysig_poll(void); /* => signum or -1 */ +void pypysig_pushback(int signum); /* When a signal is received, pypysig_counter is set to -1. */ /* This is a struct for the JIT. See rsignal.py. */ From noreply at buildbot.pypy.org Mon Jan 28 14:30:12 2013 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 28 Jan 2013 14:30:12 +0100 (CET) Subject: [pypy-commit] pypy default: merge heads Message-ID: <20130128133012.5E0E81C12E1@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r60608:7640994f3c35 Date: 2013-01-28 14:29 +0100 http://bitbucket.org/pypy/pypy/changeset/7640994f3c35/ Log: merge heads diff --git a/rpython/rlib/test/test_jit.py b/rpython/rlib/test/test_jit.py --- a/rpython/rlib/test/test_jit.py +++ b/rpython/rlib/test/test_jit.py @@ -55,6 +55,7 @@ ('bar', 40, 2), ] + at py.test.mark.xfail(reason="@inline off: see skipped failures in test_warmspot.") def test_jitdriver_clone(): def bar(): pass def foo(): pass From noreply at buildbot.pypy.org Mon Jan 28 14:32:33 2013 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 28 Jan 2013 14:32:33 +0100 (CET) Subject: [pypy-commit] pypy default: Fix Message-ID: <20130128133233.B6C9B1C12E1@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r60609:569b1e7cfd82 Date: 2013-01-28 14:36 +0100 http://bitbucket.org/pypy/pypy/changeset/569b1e7cfd82/ Log: Fix diff --git a/rpython/translator/c/src/signals.c b/rpython/translator/c/src/signals.c --- a/rpython/translator/c/src/signals.c +++ b/rpython/translator/c/src/signals.c @@ -71,7 +71,7 @@ #endif } -static void pypysig_pushback(int signum) +void pypysig_pushback(int signum) { if (0 <= signum && signum < NSIG) { From noreply at buildbot.pypy.org Mon Jan 28 14:40:38 2013 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 28 Jan 2013 14:40:38 +0100 (CET) Subject: [pypy-commit] pypy default: Missing line Message-ID: <20130128134038.99D2D1C0264@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r60610:3716d32168fd Date: 2013-01-28 14:40 +0100 http://bitbucket.org/pypy/pypy/changeset/3716d32168fd/ Log: Missing line diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py --- a/pypy/module/signal/interp_signal.py +++ b/pypy/module/signal/interp_signal.py @@ -97,6 +97,7 @@ else: # Otherwise, arrange for perform() to be called again # after we switch to the main thread. + self.pending_signal = n self.fire_in_main_thread = True break From noreply at buildbot.pypy.org Mon Jan 28 15:52:24 2013 From: noreply at buildbot.pypy.org (arigo) Date: Mon, 28 Jan 2013 15:52:24 +0100 (CET) Subject: [pypy-commit] pypy default: Sprinkle some RLocks here and there randomly, trying to avoid the Message-ID: <20130128145224.193CE1C12E1@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r60611:2bb610b6bf93 Date: 2013-01-28 15:52 +0100 http://bitbucket.org/pypy/pypy/changeset/2bb610b6bf93/ Log: Sprinkle some RLocks here and there randomly, trying to avoid the occasional crashes in module/thread's test_interrupt_main. diff --git a/rpython/rtyper/lltypesystem/ll2ctypes.py b/rpython/rtyper/lltypesystem/ll2ctypes.py --- a/rpython/rtyper/lltypesystem/ll2ctypes.py +++ b/rpython/rtyper/lltypesystem/ll2ctypes.py @@ -32,6 +32,15 @@ except ImportError: class tlsobject(object): pass +try: + from threading import RLock +except ImportError: + class RLock(object): + def __enter__(self): + pass + def __exit__(self, *args): + pass +rlock = RLock() _POSIX = os.name == "posix" _MS_WINDOWS = os.name == "nt" @@ -694,10 +703,11 @@ return None def lltype2ctypes(llobj, normalize=True): - """Convert the lltype object 'llobj' to its ctypes equivalent. - 'normalize' should only be False in tests, where we want to - inspect the resulting ctypes object manually. - """ + """Convert the lltype object 'llobj' to its ctypes equivalent. + 'normalize' should only be False in tests, where we want to + inspect the resulting ctypes object manually. + """ + with rlock: if isinstance(llobj, lltype._uninitialized): return uninitialized2ctypes(llobj.TYPE) if isinstance(llobj, llmemory.AddressAsInt): @@ -875,9 +885,10 @@ return llobj def ctypes2lltype(T, cobj): - """Convert the ctypes object 'cobj' to its lltype equivalent. - 'T' is the expected lltype type. - """ + """Convert the ctypes object 'cobj' to its lltype equivalent. + 'T' is the expected lltype type. + """ + with rlock: if T is lltype.Void: return None if isinstance(T, lltype.Typedef): @@ -1176,10 +1187,11 @@ #self.funcptr = ... set later def __call__(self, *argvalues): - if self.trampoline is None: - # lazily build the corresponding ctypes function object - cfunc = get_ctypes_callable(self.funcptr, self.calling_conv) - self.trampoline = get_ctypes_trampoline(self.FUNCTYPE, cfunc) + with rlock: + if self.trampoline is None: + # lazily build the corresponding ctypes function object + cfunc = get_ctypes_callable(self.funcptr, self.calling_conv) + self.trampoline = get_ctypes_trampoline(self.FUNCTYPE, cfunc) # perform the call return self.trampoline(*argvalues) @@ -1215,8 +1227,9 @@ return ctypes2lltype(RESULT, cres) return invoke_via_ctypes + def force_cast(RESTYPE, value): - """Cast a value to a result type, trying to use the same rules as C.""" + with rlock: if not isinstance(RESTYPE, lltype.LowLevelType): raise TypeError("rffi.cast() first arg should be a TYPE") if isinstance(value, llmemory.AddressAsInt): From noreply at buildbot.pypy.org Mon Jan 28 16:07:13 2013 From: noreply at buildbot.pypy.org (bivab) Date: Mon, 28 Jan 2013 16:07:13 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: move helper functions to arm/support.py Message-ID: <20130128150713.7DF891C087E@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: jitframe-on-heap Changeset: r60612:a921fbd70801 Date: 2013-01-24 15:17 +0100 http://bitbucket.org/pypy/pypy/changeset/a921fbd70801/ Log: move helper functions to arm/support.py diff --git a/rpython/jit/backend/arm/arch.py b/rpython/jit/backend/arm/arch.py --- a/rpython/jit/backend/arm/arch.py +++ b/rpython/jit/backend/arm/arch.py @@ -1,7 +1,3 @@ -from rpython.rtyper.lltypesystem import lltype, rffi -from rpython.rlib.rarithmetic import r_uint - - FUNC_ALIGN = 8 WORD = 4 DOUBLE_WORD = 8 @@ -14,54 +10,8 @@ PC_OFFSET = 8 FORCE_INDEX_OFS = 0 -from rpython.translator.tool.cbuild import ExternalCompilationInfo -eci = ExternalCompilationInfo(post_include_bits=[""" -static int pypy__arm_int_div(int a, int b) { - return a/b; -} -static unsigned int pypy__arm_uint_div(unsigned int a, unsigned int b) { - return a/b; -} -static int pypy__arm_int_mod(int a, int b) { - return a % b; -} -"""]) -def arm_int_div_emulator(a, b): - return int(a / float(b)) -arm_int_div_sign = lltype.Ptr( - lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed)) -arm_int_div = rffi.llexternal( - "pypy__arm_int_div", [lltype.Signed, lltype.Signed], lltype.Signed, - _callable=arm_int_div_emulator, - compilation_info=eci, - _nowrapper=True, elidable_function=True) -def arm_uint_div_emulator(a, b): - return r_uint(a) / r_uint(b) -arm_uint_div_sign = lltype.Ptr( - lltype.FuncType([lltype.Unsigned, lltype.Unsigned], lltype.Unsigned)) -arm_uint_div = rffi.llexternal( - "pypy__arm_uint_div", [lltype.Unsigned, lltype.Unsigned], lltype.Unsigned, - _callable=arm_uint_div_emulator, - compilation_info=eci, - _nowrapper=True, elidable_function=True) - -def arm_int_mod_emulator(a, b): - sign = 1 - if a < 0: - a = -1 * a - sign = -1 - if b < 0: - b = -1 * b - res = a % b - return sign * res -arm_int_mod_sign = arm_int_div_sign -arm_int_mod = rffi.llexternal( - "pypy__arm_int_mod", [lltype.Signed, lltype.Signed], lltype.Signed, - _callable=arm_int_mod_emulator, - compilation_info=eci, - _nowrapper=True, elidable_function=True) diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py --- a/rpython/jit/backend/arm/assembler.py +++ b/rpython/jit/backend/arm/assembler.py @@ -30,8 +30,7 @@ from rpython.rlib.jit import AsmInfo from rpython.rlib.objectmodel import compute_unique_id -# XXX Move to llsupport -from rpython.jit.backend.x86.support import memcpy_fn +from rpython.jit.backend.arm.support import memcpy_fn DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', ('i', lltype.Signed), ('type', lltype.Char), # 'b'ridge, 'l'abel or diff --git a/rpython/jit/backend/arm/codebuilder.py b/rpython/jit/backend/arm/codebuilder.py --- a/rpython/jit/backend/arm/codebuilder.py +++ b/rpython/jit/backend/arm/codebuilder.py @@ -1,6 +1,6 @@ -from rpython.jit.backend.arm import arch from rpython.jit.backend.arm import conditions as cond from rpython.jit.backend.arm import registers as reg +from rpython.jit.backend.arm import support from rpython.jit.backend.arm.arch import (WORD, FUNC_ALIGN) from rpython.jit.backend.arm.instruction_builder import define_instructions from rpython.jit.backend.llsupport.asmmemmgr import BlockBuilderMixin @@ -17,7 +17,7 @@ def binary_helper_call(name): - function = getattr(arch, 'arm_%s' % name) + function = getattr(support, 'arm_%s' % name) def f(self, c=cond.AL): """Generates a call to a helper function, takes its diff --git a/rpython/jit/backend/arm/support.py b/rpython/jit/backend/arm/support.py new file mode 100644 --- /dev/null +++ b/rpython/jit/backend/arm/support.py @@ -0,0 +1,61 @@ +from rpython.rtyper.lltypesystem import lltype, rffi, llmemory +from rpython.rlib.rarithmetic import r_uint +from rpython.translator.tool.cbuild import ExternalCompilationInfo + +eci = ExternalCompilationInfo(post_include_bits=[""" +static int pypy__arm_int_div(int a, int b) { + return a/b; +} +static unsigned int pypy__arm_uint_div(unsigned int a, unsigned int b) { + return a/b; +} +static int pypy__arm_int_mod(int a, int b) { + return a % b; +} +"""]) + + +def arm_int_div_emulator(a, b): + return int(a / float(b)) +arm_int_div_sign = lltype.Ptr( + lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed)) +arm_int_div = rffi.llexternal( + "pypy__arm_int_div", [lltype.Signed, lltype.Signed], lltype.Signed, + _callable=arm_int_div_emulator, + compilation_info=eci, + _nowrapper=True, elidable_function=True) + + +def arm_uint_div_emulator(a, b): + return r_uint(a) / r_uint(b) +arm_uint_div_sign = lltype.Ptr( + lltype.FuncType([lltype.Unsigned, lltype.Unsigned], lltype.Unsigned)) +arm_uint_div = rffi.llexternal( + "pypy__arm_uint_div", [lltype.Unsigned, lltype.Unsigned], lltype.Unsigned, + _callable=arm_uint_div_emulator, + compilation_info=eci, + _nowrapper=True, elidable_function=True) + + +def arm_int_mod_emulator(a, b): + sign = 1 + if a < 0: + a = -1 * a + sign = -1 + if b < 0: + b = -1 * b + res = a % b + return sign * res +arm_int_mod_sign = arm_int_div_sign +arm_int_mod = rffi.llexternal( + "pypy__arm_int_mod", [lltype.Signed, lltype.Signed], lltype.Signed, + _callable=arm_int_mod_emulator, + compilation_info=eci, + _nowrapper=True, elidable_function=True) +# ____________________________________________________________ + +memcpy_fn = rffi.llexternal('memcpy', [llmemory.Address, llmemory.Address, + rffi.SIZE_T], lltype.Void, + sandboxsafe=True, _nowrapper=True) + +# ____________________________________________________________ From noreply at buildbot.pypy.org Mon Jan 28 16:07:14 2013 From: noreply at buildbot.pypy.org (bivab) Date: Mon, 28 Jan 2013 16:07:14 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fix import Message-ID: <20130128150714.B49781C087E@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: jitframe-on-heap Changeset: r60613:463eea155509 Date: 2013-01-25 14:59 +0100 http://bitbucket.org/pypy/pypy/changeset/463eea155509/ Log: fix import diff --git a/rpython/jit/backend/arm/test/test_assembler.py b/rpython/jit/backend/arm/test/test_assembler.py --- a/rpython/jit/backend/arm/test/test_assembler.py +++ b/rpython/jit/backend/arm/test/test_assembler.py @@ -1,6 +1,6 @@ from rpython.jit.backend.arm import conditions as c from rpython.jit.backend.arm import registers as r -from rpython.jit.backend.arm.arch import arm_int_div +from rpython.jit.backend.arm.support import arm_int_div from rpython.jit.backend.arm.assembler import AssemblerARM from rpython.jit.backend.arm.locations import imm from rpython.jit.backend.arm.test.support import run_asm From noreply at buildbot.pypy.org Mon Jan 28 16:07:15 2013 From: noreply at buildbot.pypy.org (bivab) Date: Mon, 28 Jan 2013 16:07:15 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: move methods to llsupport Message-ID: <20130128150715.DE6111C087E@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: jitframe-on-heap Changeset: r60614:75d5ea715262 Date: 2013-01-28 14:24 +0100 http://bitbucket.org/pypy/pypy/changeset/75d5ea715262/ Log: move methods to llsupport diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py --- a/rpython/jit/backend/llsupport/llmodel.py +++ b/rpython/jit/backend/llsupport/llmodel.py @@ -335,6 +335,40 @@ # ____________________________________________________________ + def set_int_value(self, newframe, index, value): + """ Note that we keep index multiplied by WORD here mostly + for completeness with get_int_value and friends + """ + descr = self.gc_ll_descr.getframedescrs(self).arraydescr + ofs = self.unpack_arraydescr(descr) + self.write_int_at_mem(newframe, ofs + index, WORD, 1, value) + + def set_ref_value(self, newframe, index, value): + descr = self.gc_ll_descr.getframedescrs(self).arraydescr + ofs = self.unpack_arraydescr(descr) + self.write_ref_at_mem(newframe, ofs + index, value) + + def set_float_value(self, newframe, index, value): + descr = self.gc_ll_descr.getframedescrs(self).arraydescr + ofs = self.unpack_arraydescr(descr) + self.write_float_at_mem(newframe, ofs + index, value) + + @specialize.arg(1) + def get_ofs_of_frame_field(self, name): + descrs = self.gc_ll_descr.getframedescrs(self) + if name.startswith('jfi_'): + base_ofs = 0 # not relative to frame + else: + base_ofs = self.unpack_arraydescr(descrs.arraydescr) + ofs = self.unpack_fielddescr(getattr(descrs, name)) + return ofs - base_ofs + + def get_baseofs_of_frame_field(self): + descrs = self.gc_ll_descr.getframedescrs(self) + base_ofs = self.unpack_arraydescr(descrs.arraydescr) + return base_ofs + # ____________________________________________________________ + def bh_arraylen_gc(self, array, arraydescr): assert isinstance(arraydescr, ArrayDescr) diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py --- a/rpython/jit/backend/x86/runner.py +++ b/rpython/jit/backend/x86/runner.py @@ -109,7 +109,7 @@ lst = [(i, history.getkind(ARG)[0]) for i, ARG in enumerate(ARGS)] kinds = unrolling_iterable(lst) - + def execute_token(executable_token, *args): clt = executable_token.compiled_loop_token assert len(args) == clt._debug_nbargs @@ -164,7 +164,7 @@ def invalidate_loop(self, looptoken): from rpython.jit.backend.x86 import codebuf - + for addr, tgt in looptoken.compiled_loop_token.invalidate_positions: mc = codebuf.MachineCodeBlockWrapper() mc.JMP_l(tgt) @@ -182,38 +182,6 @@ l[i].counter = ll_s.i return l - def set_int_value(self, newframe, index, value): - """ Note that we keep index multiplied by WORD here mostly - for completeness with get_int_value and friends - """ - descr = self.gc_ll_descr.getframedescrs(self).arraydescr - ofs = self.unpack_arraydescr(descr) - self.write_int_at_mem(newframe, ofs + index, WORD, 1, value) - - def set_ref_value(self, newframe, index, value): - descr = self.gc_ll_descr.getframedescrs(self).arraydescr - ofs = self.unpack_arraydescr(descr) - self.write_ref_at_mem(newframe, ofs + index, value) - - def set_float_value(self, newframe, index, value): - descr = self.gc_ll_descr.getframedescrs(self).arraydescr - ofs = self.unpack_arraydescr(descr) - self.write_float_at_mem(newframe, ofs + index, value) - - @specialize.arg(1) - def get_ofs_of_frame_field(self, name): - descrs = self.gc_ll_descr.getframedescrs(self) - if name.startswith('jfi_'): - base_ofs = 0 # not relative to frame - else: - base_ofs = self.unpack_arraydescr(descrs.arraydescr) - ofs = self.unpack_fielddescr(getattr(descrs, name)) - return ofs - base_ofs - - def get_baseofs_of_frame_field(self): - descrs = self.gc_ll_descr.getframedescrs(self) - base_ofs = self.unpack_arraydescr(descrs.arraydescr) - return base_ofs class CPU386(AbstractX86CPU): backend_name = 'x86' From noreply at buildbot.pypy.org Mon Jan 28 16:07:17 2013 From: noreply at buildbot.pypy.org (bivab) Date: Mon, 28 Jan 2013 16:07:17 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: skip test if floats are not supported on cpu Message-ID: <20130128150717.173C71C087E@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: jitframe-on-heap Changeset: r60615:c94896974746 Date: 2013-01-28 15:53 +0100 http://bitbucket.org/pypy/pypy/changeset/c94896974746/ Log: skip test if floats are not supported on cpu diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py --- a/rpython/jit/backend/test/runner_test.py +++ b/rpython/jit/backend/test/runner_test.py @@ -129,6 +129,8 @@ assert fail.identifier == 1 def test_compile_linear_float_loop(self): + if not self.cpu.supports_floats: + py.test.skip("requires floats") i0 = BoxFloat() i1 = BoxFloat() operations = [ From noreply at buildbot.pypy.org Mon Jan 28 16:07:18 2013 From: noreply at buildbot.pypy.org (bivab) Date: Mon, 28 Jan 2013 16:07:18 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: in progress, first test passes. Changes a bit everywhere to support the jitframe Message-ID: <20130128150718.405ED1C087E@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: jitframe-on-heap Changeset: r60616:1ced49baa459 Date: 2013-01-28 15:56 +0100 http://bitbucket.org/pypy/pypy/changeset/1ced49baa459/ Log: in progress, first test passes. Changes a bit everywhere to support the jitframe diff --git a/rpython/jit/backend/arm/arch.py b/rpython/jit/backend/arm/arch.py --- a/rpython/jit/backend/arm/arch.py +++ b/rpython/jit/backend/arm/arch.py @@ -10,8 +10,13 @@ PC_OFFSET = 8 FORCE_INDEX_OFS = 0 +# The stack contains the force_index and the, callee saved registers and +# ABI required information +# All the rest of the data is in a GC-managed variable-size "frame". +# This jitframe object's address is always stored in the register FP +# A jitframe is a jit.backend.llsupport.llmodel.JITFRAME = GcArray(Signed). +# Stack frame fixed area +# Currently only the force_index +FRAME_FIXED_SIZE = 1 +JITFRAME_FIXED_SIZE = 16 + 16 * 2 # 16 GPR + 16 VFP Regs (64bit) - - - - diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py --- a/rpython/jit/backend/arm/assembler.py +++ b/rpython/jit/backend/arm/assembler.py @@ -5,7 +5,8 @@ from rpython.jit.backend.arm import conditions as c from rpython.jit.backend.arm import registers as r from rpython.jit.backend.arm.arch import WORD, DOUBLE_WORD, FUNC_ALIGN, \ - N_REGISTERS_SAVED_BY_MALLOC + N_REGISTERS_SAVED_BY_MALLOC, \ + JITFRAME_FIXED_SIZE, FRAME_FIXED_SIZE from rpython.jit.backend.arm.codebuilder import ARMv7Builder, OverwritingBuilder from rpython.jit.backend.arm.locations import get_fp_offset from rpython.jit.backend.arm.regalloc import (Regalloc, ARMFrameManager, @@ -21,7 +22,7 @@ from rpython.jit.metainterp.resoperation import rop, ResOperation from rpython.rlib import rgc from rpython.rlib.objectmodel import we_are_translated, specialize -from rpython.rtyper.annlowlevel import llhelper +from rpython.rtyper.annlowlevel import llhelper, cast_instance_to_gcref from rpython.rtyper.lltypesystem import lltype, rffi, llmemory from rpython.rtyper.lltypesystem.lloperation import llop from rpython.jit.backend.arm.opassembler import ResOpAssembler @@ -29,6 +30,7 @@ have_debug_prints, fatalerror) from rpython.rlib.jit import AsmInfo from rpython.rlib.objectmodel import compute_unique_id +from rpython.rlib.rarithmetic import intmask, r_uint from rpython.jit.backend.arm.support import memcpy_fn @@ -40,8 +42,6 @@ class AssemblerARM(ResOpAssembler): - STACK_FIXED_AREA = -1 - debug = True def __init__(self, cpu, translate_support_code=False): @@ -58,33 +58,19 @@ self.datablockwrapper = None self.propagate_exception_path = 0 self.stack_check_slowpath = 0 - self._compute_stack_size() self._debug = False self.loop_run_counters = [] self.debug_counter_descr = cpu.fielddescrof(DEBUG_COUNTER, 'i') - self.force_token_to_dead_frame = {} # XXX temporary hack + self.gcrootmap_retaddr_forced = 0 def set_debug(self, v): r = self._debug self._debug = v return r - def _compute_stack_size(self): - self.STACK_FIXED_AREA = len(r.callee_saved_registers) * WORD - self.STACK_FIXED_AREA += WORD # FORCE_TOKEN - self.STACK_FIXED_AREA += N_REGISTERS_SAVED_BY_MALLOC * WORD - if self.cpu.supports_floats: - self.STACK_FIXED_AREA += (len(r.callee_saved_vfp_registers) - * DOUBLE_WORD) - if self.STACK_FIXED_AREA % 8 != 0: - self.STACK_FIXED_AREA += WORD # Stack alignment - assert self.STACK_FIXED_AREA % 8 == 0 - - def setup(self, looptoken, operations): + def setup(self, looptoken): + assert self.memcpy_addr != 0, 'setup_once() not called?' self.current_clt = looptoken.compiled_loop_token - operations = self.cpu.gc_ll_descr.rewrite_assembler(self.cpu, - operations, self.current_clt.allgcrefs) - assert self.memcpy_addr != 0, 'setup_once() not called?' self.mc = ARMv7Builder() self.pending_guards = [] assert self.datablockwrapper is None @@ -92,7 +78,6 @@ self.datablockwrapper = MachineDataBlockWrapper(self.cpu.asmmemmgr, allblocks) self.target_tokens_currently_compiling = {} - return operations def teardown(self): self.current_clt = None @@ -105,10 +90,11 @@ # Addresses of functions called by new_xxx operations gc_ll_descr = self.cpu.gc_ll_descr gc_ll_descr.initialize() + self.memcpy_addr = self.cpu.cast_ptr_to_int(memcpy_fn) + self._build_failure_recovery(exc=True, withfloats=False) + self._build_failure_recovery(exc=False, withfloats=False) self._build_wb_slowpath(False) self._build_wb_slowpath(True) - self._build_failure_recovery(exc=True, withfloats=False) - self._build_failure_recovery(exc=False, withfloats=False) if self.cpu.supports_floats: self._build_wb_slowpath(False, withfloats=True) self._build_wb_slowpath(True, withfloats=True) @@ -120,7 +106,6 @@ self._build_stack_check_slowpath() if gc_ll_descr.gcrootmap and gc_ll_descr.gcrootmap.is_shadow_stack: self._build_release_gil(gc_ll_descr.gcrootmap) - self.memcpy_addr = self.cpu.cast_ptr_to_int(memcpy_fn) if not self._debug: # if self._debug is already set it means that someone called @@ -129,6 +114,9 @@ debug_start('jit-backend-counts') self.set_debug(have_debug_prints()) debug_stop('jit-backend-counts') + # when finishing, we only have one value at [0], the rest dies + self.gcmap_for_finish = lltype.malloc(jitframe.GCMAP, 1, zero=True) + self.gcmap_for_finish[0] = r_uint(1) def finish_once(self): if self._debug: @@ -217,18 +205,50 @@ self.reacqgil_addr = rffi.cast(lltype.Signed, reacqgil_func) def _build_propagate_exception_path(self): - if self.cpu.propagate_exception_v < 0: + if not self.cpu.propagate_exception_descr: return # not supported (for tests, or non-translated) # mc = ARMv7Builder() # - # Call the helper, which will return a dead frame object with - # the correct exception set, or MemoryError by default - # XXX make sure we return the correct value here + # read and reset the current exception addr = rffi.cast(lltype.Signed, self.cpu.get_propagate_exception()) mc.BL(addr) self.gen_func_epilog(mc=mc) self.propagate_exception_path = mc.materialize(self.cpu.asmmemmgr, []) + # + self._store_and_reset_exception(r.r0) + ofs = self.cpu.get_ofs_of_frame_field('jf_guard_exc') + # make sure ofs fits into a register + assert check_imm_arg(ofs) + self.mc.STR_ri(r.r0.value, r.fp.value, imm=ofs) + propagate_exception_descr = rffi.cast(lltype.Signed, + cast_instance_to_gcref(self.cpu.propagate_exception_descr)) + ofs = self.cpu.get_ofs_of_frame_field('jf_descr') + # make sure ofs fits into a register + assert check_imm_arg(ofs) + self.mc.BKPT() + #base_ofs = self.cpu.get_baseofs_of_frame_field() + #self.mc.MOV_bi(ofs, propagate_exception_descr) + #self.mc.LEA_rb(eax.value, -base_ofs) + # + self._call_footer() + rawstart = self.mc.materialize(self.cpu.asmmemmgr, []) + self.propagate_exception_path = rawstart + self.mc = None + + def _store_and_reset_exception(self, resloc=None): + assert resloc is not r.ip + if resloc is not None: + self.mc.gen_load_int(resloc.value, self.cpu.pos_exc_value()) + self.mc.LDR_ri(resloc.value, resloc.value) + self.mc.MOV(resloc, heap(self.cpu.pos_exc_value())) + + with saved_registers(self.mc, [r.r0]): + self.mc.gen_load_int(r.r0.value, self.cpu.pos_exc_value()) + self.mc.gen_load_int(r.ip.value, 0) + self.mc.STR_ri(r.ip.value, r.r0.value) + self.mc.gen_load_int(r.r0.value, self.cpu.pos_exception()) + self.mc.STR_ri(r.ip.value, r.r0.value) def _build_stack_check_slowpath(self): _, _, slowpathaddr = self.cpu.insert_stack_check() @@ -557,7 +577,7 @@ # We might have an exception pending. Load it into r4 # (this is a register saved across calls) mc.gen_load_int(r.r5.value, self.cpu.pos_exc_value()) - mc.LDR_ri(r.r4.value, self.cpu.pos_exc_value()) + mc.LDR_ri(r.r4.value, r.r5.value) # clear the exc flags mc.gen_load_int(r.r6.value, 0) mc.STR_ri(r.r6.value, r.r5.value) @@ -660,37 +680,35 @@ self.mc.writechar(chr(0)) def gen_func_epilog(self, mc=None, cond=c.AL): - stack_size = self.STACK_FIXED_AREA - stack_size -= len(r.callee_saved_registers) * WORD - if self.cpu.supports_floats: - stack_size -= len(r.callee_saved_vfp_registers) * 2 * WORD - gcrootmap = self.cpu.gc_ll_descr.gcrootmap if mc is None: mc = self.mc if gcrootmap and gcrootmap.is_shadow_stack: self.gen_footer_shadowstack(gcrootmap, mc) - mc.MOV_rr(r.sp.value, r.fp.value, cond=cond) - mc.ADD_ri(r.sp.value, r.sp.value, stack_size, cond=cond) + mc.ADD_ri(r.sp.value, r.sp.value, WORD, cond=cond) # for the force index if self.cpu.supports_floats: mc.VPOP([reg.value for reg in r.callee_saved_vfp_registers], cond=cond) mc.POP([reg.value for reg in r.callee_restored_registers], cond=cond) + mc.BKPT() def gen_func_prolog(self): - stack_size = self.STACK_FIXED_AREA - stack_size -= len(r.callee_saved_registers) * WORD + stack_size = FRAME_FIXED_SIZE * WORD + stack_size += len(r.callee_saved_registers) * WORD if self.cpu.supports_floats: - stack_size -= len(r.callee_saved_vfp_registers) * 2 * WORD + stack_size += len(r.callee_saved_vfp_registers) * 2 * WORD self.mc.PUSH([reg.value for reg in r.callee_saved_registers]) if self.cpu.supports_floats: self.mc.VPUSH([reg.value for reg in r.callee_saved_vfp_registers]) - # here we modify the stack pointer to leave room for the 9 registers - # that are going to be saved here around malloc calls and one word to - # store the force index - self.mc.SUB_ri(r.sp.value, r.sp.value, stack_size) - self.mc.MOV_rr(r.fp.value, r.sp.value) + self.mc.SUB_ri(r.sp.value, r.sp.value, WORD) # for the force index + assert stack_size % 8 == 0 # ensure we keep alignment + + # set fp to point to the JITFRAME + ofs + ofs = self.cpu.get_baseofs_of_frame_field() + assert check_imm_arg(ofs) + self.mc.ADD_ri(r.fp.value, r.r0.value, imm=ofs) + # gcrootmap = self.cpu.gc_ll_descr.gcrootmap if gcrootmap and gcrootmap.is_shadow_stack: self.gen_shadowstack_header(gcrootmap) @@ -753,7 +771,9 @@ # cpu interface def assemble_loop(self, loopname, inputargs, operations, looptoken, log): clt = CompiledLoopToken(self.cpu, looptoken.number) + clt.frame_info = lltype.malloc(jitframe.JITFRAMEINFO) clt.allgcrefs = [] + clt.frame_info.jfi_frame_depth = 0 # for now looptoken.compiled_loop_token = clt clt._debug_nbargs = len(inputargs) @@ -761,38 +781,40 @@ # Arguments should be unique assert len(set(inputargs)) == len(inputargs) - operations = self.setup(looptoken, operations) - if log: + self.setup(looptoken) + if False and log: operations = self._inject_debugging_code(looptoken, operations, 'e', looptoken.number) self._call_header_with_stack_check() - sp_patch_location = self._prepare_sp_patch_position() + #sp_patch_location = self._prepare_sp_patch_position() - regalloc = Regalloc(assembler=self, frame_manager=ARMFrameManager()) - regalloc.prepare_loop(inputargs, operations) + regalloc = Regalloc(assembler=self) + operations = regalloc.prepare_loop(inputargs, operations, looptoken, + clt.allgcrefs) + rgc._make_sure_does_not_move(lltype.cast_opaque_ptr(llmemory.GCREF, + clt.frame_info)) loop_head = self.mc.get_relative_pos() looptoken._arm_loop_code = loop_head # - clt.frame_depth = -1 - frame_depth = self._assemble(operations, regalloc) - clt.frame_depth = frame_depth + frame_depth = self._assemble(regalloc, inputargs, operations) + self.update_frame_depth(frame_depth + JITFRAME_FIXED_SIZE) # size_excluding_failure_stuff = self.mc.get_relative_pos() - self._patch_sp_offset(sp_patch_location, frame_depth) + #self._patch_sp_offset(sp_patch_location, frame_depth) self.write_pending_failure_recoveries() rawstart = self.materialize_loop(looptoken) - looptoken._arm_func_addr = rawstart + looptoken._function_addr = looptoken._arm_func_addr = rawstart self.process_pending_guards(rawstart) self.fixup_target_tokens(rawstart) if log and not we_are_translated(): self.mc._dump_trace(rawstart, - 'loop_%s.asm' % self.cpu.total_compiled_loops) + 'loop.asm') ops_offset = self.mc.ops_offset self.teardown() @@ -808,18 +830,20 @@ return AsmInfo(ops_offset, rawstart + loop_head, size_excluding_failure_stuff - loop_head) - def _assemble(self, operations, regalloc): + def _assemble(self, regalloc, inputargs, operations): regalloc.compute_hint_frame_locations(operations) - self._walk_operations(operations, regalloc) - frame_depth = regalloc.frame_manager.get_frame_depth() + self._walk_operations(inputargs, operations, regalloc) + frame_depth = regalloc.get_final_frame_depth() jump_target_descr = regalloc.jump_target_descr if jump_target_descr is not None: - frame_depth = max(frame_depth, - jump_target_descr._arm_clt.frame_depth) + tgt_depth = jump_target_descr._arm_clt.frame_info.jfi_frame_depth + target_frame_depth = tgt_depth - JITFRAME_FIXED_SIZE + frame_depth = max(frame_depth, target_frame_depth) return frame_depth def assemble_bridge(self, faildescr, inputargs, operations, original_loop_token, log): + assert 0 operations = self.setup(original_loop_token, operations) descr_number = self.cpu.get_fail_descr_number(faildescr) if log: @@ -898,6 +922,17 @@ return self.mc.materialize(self.cpu.asmmemmgr, allblocks, self.cpu.gc_ll_descr.gcrootmap) + def update_frame_depth(self, frame_depth): + self.current_clt.frame_info.jfi_frame_depth = frame_depth + new_jumping_to = [] + for wref in self.current_clt.jumping_to: + clt = wref() + if clt is not None: + clt.frame_info.jfi_frame_depth = max(frame_depth, + clt.frame_info.jfi_frame_depth) + new_jumping_to.append(weakref.ref(clt)) + self.current_clt.jumping_to = new_jumping_to + def write_pending_failure_recoveries(self): for tok in self.pending_guards: #generate the exit stub and the encoded representation @@ -971,7 +1006,7 @@ else: cb.SUB_rr(r.sp.value, base_reg.value, r.ip.value, cond=fcond) - def _walk_operations(self, operations, regalloc): + def _walk_operations(self, inputargs, operations, regalloc): fcond = c.AL self._regalloc = regalloc while regalloc.position() < len(operations) - 1: @@ -1140,10 +1175,10 @@ if not check_imm_arg(offset, size=0xFFF): self.mc.PUSH([r.lr.value], cond=cond) pushed = True - self.mc.gen_load_int(r.lr.value, -offset, cond=cond) + self.mc.gen_load_int(r.lr.value, offset, cond=cond) self.mc.LDR_rr(loc.value, r.fp.value, r.lr.value, cond=cond) else: - self.mc.LDR_ri(loc.value, r.fp.value, imm=-offset, cond=cond) + self.mc.LDR_ri(loc.value, r.fp.value, imm=offset, cond=cond) if pushed: self.mc.POP([r.lr.value], cond=cond) elif loc.is_vfp_reg(): @@ -1363,6 +1398,26 @@ else: return 0 + def push_gcmap(self, mc, gcmap, push=False, mov=False, store=False): + gcmapref = lltype.cast_opaque_ptr(llmemory.GCREF, gcmap) + # keep the ref alive + self.current_clt.allgcrefs.append(gcmapref) + rgc._make_sure_does_not_move(gcmapref) + pass + #if push: + # mc.PUSH(imm(rffi.cast(lltype.Signed, gcmapref))) + #elif mov: + # mc.MOV(RawEspLoc(0, REF), + # imm(rffi.cast(lltype.Signed, gcmapref))) + #else: + # assert store + # ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') + # mc.MOV(raw_stack(ofs), imm(rffi.cast(lltype.Signed, gcmapref))) + + def pop_gcmap(self, mc): + ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') + mc.MOV_bi(ofs, 0) + def not_implemented(msg): os.write(2, '[ARM/asm] %s\n' % msg) diff --git a/rpython/jit/backend/arm/locations.py b/rpython/jit/backend/arm/locations.py --- a/rpython/jit/backend/arm/locations.py +++ b/rpython/jit/backend/arm/locations.py @@ -1,5 +1,5 @@ from rpython.jit.metainterp.history import INT, FLOAT -from rpython.jit.backend.arm.arch import WORD, DOUBLE_WORD +from rpython.jit.backend.arm.arch import WORD, DOUBLE_WORD, JITFRAME_FIXED_SIZE class AssemblerLocation(object): @@ -136,9 +136,5 @@ return ImmLocation(i) -def get_fp_offset(i): - if i >= 0: - # Take the FORCE_TOKEN into account - return (1 + i) * WORD - else: - return i * WORD +def get_fp_offset(position): + return WORD * (position + JITFRAME_FIXED_SIZE) diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -317,9 +317,27 @@ return fcond def emit_op_finish(self, op, arglocs, regalloc, fcond): - [argloc] = arglocs - if argloc is not r.r0: #XXX verify this - self.mov_loc_loc(argloc, r.r0, fcond) + base_ofs = self.cpu.get_baseofs_of_frame_field() - WORD + if len(arglocs) == 2: + [return_val, fail_descr_loc] = arglocs + if op.getarg(0).type == FLOAT and not IS_X86_64: + XXX + size = WORD * 2 + else: + size = WORD + self.mc.STR_ri(return_val.value, r.fp.value)#, imm=-base_ofs) + #self.save_into_mem(raw_stack(0), return_val, imm(size)) + else: + [fail_descr_loc] = arglocs + ofs = self.cpu.get_ofs_of_frame_field('jf_descr') + base_ofs = self.cpu.get_baseofs_of_frame_field() + + self.mc.gen_load_int(r.ip.value, fail_descr_loc.value) + # XXX self.mov(fail_descr_loc, RawStackLoc(ofs)) + self.mc.STR_ri(r.ip.value, r.fp.value, imm=ofs) + gcmap = self.gcmap_for_finish + self.push_gcmap(self.mc, gcmap, store=True) + self.mc.SUB_ri(r.r0.value, r.fp.value, base_ofs) # exit function self.gen_func_epilog() return fcond diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py --- a/rpython/jit/backend/arm/regalloc.py +++ b/rpython/jit/backend/arm/regalloc.py @@ -1,3 +1,5 @@ +from rpython.rtyper.annlowlevel import llhelper, cast_instance_to_gcref +from rpython.rlib import rgc from rpython.jit.backend.llsupport.regalloc import FrameManager, \ RegisterManager, TempBox, compute_vars_longevity from rpython.jit.backend.arm import registers as r @@ -180,10 +182,10 @@ class Regalloc(object): - def __init__(self, frame_manager=None, assembler=None): + def __init__(self, assembler=None): self.cpu = assembler.cpu self.assembler = assembler - self.frame_manager = frame_manager + self.frame_manager = None self.jump_target_descr = None self.final_jump_op = None @@ -282,7 +284,12 @@ assert isinstance(value, ConstFloat) return self.vfprm.convert_to_imm(value) - def _prepare(self, inputargs, operations): + def _prepare(self, inputargs, operations, allgcrefs): + self.frame_manager = self.fm = ARMFrameManager() + cpu = self.assembler.cpu + operations = cpu.gc_ll_descr.rewrite_assembler(cpu, operations, + allgcrefs) + # compute longevity of variables longevity, last_real_usage = compute_vars_longevity( inputargs, operations) self.longevity = longevity @@ -291,92 +298,27 @@ asm = self.assembler self.vfprm = VFPRegisterManager(longevity, fm, asm) self.rm = CoreRegisterManager(longevity, fm, asm) + return operations - def prepare_loop(self, inputargs, operations): - self._prepare(inputargs, operations) + def prepare_loop(self, inputargs, operations, looptoken, allgcrefs): + operations = self._prepare(inputargs, operations, allgcrefs) self._set_initial_bindings(inputargs) - self.possibly_free_vars(inputargs) + self.possibly_free_vars(list(inputargs)) + return operations def prepare_bridge(self, inputargs, arglocs, ops): self._prepare(inputargs, ops) self._update_bindings(arglocs, inputargs) + def get_final_frame_depth(self): + return self.frame_manager.get_frame_depth() + def _set_initial_bindings(self, inputargs): - # The first inputargs are passed in registers r0-r3 - # we relly on the soft-float calling convention so we need to move - # float params to the coprocessor. - if self.cpu.use_hf_abi: - self._set_initial_bindings_hf(inputargs) - else: - self._set_initial_bindings_sf(inputargs) - - def _set_initial_bindings_sf(self, inputargs): - - arg_index = 0 - count = 0 - n_register_args = len(r.argument_regs) - cur_frame_pos = 1 - (self.assembler.STACK_FIXED_AREA // WORD) + # the input args are passed in the jitframe for box in inputargs: assert isinstance(box, Box) - # handle inputargs in argument registers - if box.type == FLOAT and arg_index % 2 != 0: - arg_index += 1 # align argument index for float passed - # in register - if arg_index < n_register_args: - if box.type == FLOAT: - loc = r.argument_regs[arg_index] - loc2 = r.argument_regs[arg_index + 1] - vfpreg = self.try_allocate_reg(box) - # move soft-float argument to vfp - self.assembler.mov_to_vfp_loc(loc, loc2, vfpreg) - arg_index += 2 # this argument used two argument registers - else: - loc = r.argument_regs[arg_index] - self.try_allocate_reg(box, selected_reg=loc) - arg_index += 1 - else: - # treat stack args as stack locations with a negative offset - if box.type == FLOAT: - cur_frame_pos -= 2 - if count % 2 != 0: # Stack argument alignment - cur_frame_pos -= 1 - count = 0 - else: - cur_frame_pos -= 1 - count += 1 - loc = self.frame_manager.frame_pos(cur_frame_pos, box.type) - self.frame_manager.set_binding(box, loc) - - def _set_initial_bindings_hf(self, inputargs): - - arg_index = vfp_arg_index = 0 - count = 0 - n_reg_args = len(r.argument_regs) - n_vfp_reg_args = len(r.vfp_argument_regs) - cur_frame_pos = 1 - (self.assembler.STACK_FIXED_AREA // WORD) - for box in inputargs: - assert isinstance(box, Box) - # handle inputargs in argument registers - if box.type != FLOAT and arg_index < n_reg_args: - reg = r.argument_regs[arg_index] - self.try_allocate_reg(box, selected_reg=reg) - arg_index += 1 - elif box.type == FLOAT and vfp_arg_index < n_vfp_reg_args: - reg = r.vfp_argument_regs[vfp_arg_index] - self.try_allocate_reg(box, selected_reg=reg) - vfp_arg_index += 1 - else: - # treat stack args as stack locations with a negative offset - if box.type == FLOAT: - cur_frame_pos -= 2 - if count % 2 != 0: # Stack argument alignment - cur_frame_pos -= 1 - count = 0 - else: - cur_frame_pos -= 1 - count += 1 - loc = self.frame_manager.frame_pos(cur_frame_pos, box.type) - self.frame_manager.set_binding(box, loc) + assert box.type != FLOAT + self.fm.get_new_loc(box) def _update_bindings(self, locs, inputargs): used = {} @@ -644,9 +586,19 @@ return args def prepare_op_finish(self, op, fcond): - loc = self.loc(op.getarg(0)) - self.possibly_free_var(op.getarg(0)) - return [loc] + # the frame is in fp, but we have to point where in the frame is + # the potential argument to FINISH + descr = op.getdescr() + fail_descr = cast_instance_to_gcref(descr) + # we know it does not move, but well + rgc._make_sure_does_not_move(fail_descr) + fail_descr = rffi.cast(lltype.Signed, fail_descr) + if op.numargs() == 1: + loc = self.make_sure_var_in_reg(op.getarg(0)) + locs = [loc, imm(fail_descr)] + else: + locs = [imm(fail_descr)] + return locs def prepare_op_guard_true(self, op, fcond): l0 = self.make_sure_var_in_reg(op.getarg(0)) diff --git a/rpython/jit/backend/arm/runner.py b/rpython/jit/backend/arm/runner.py --- a/rpython/jit/backend/arm/runner.py +++ b/rpython/jit/backend/arm/runner.py @@ -1,12 +1,18 @@ +from rpython.jit.backend.arm.arch import JITFRAME_FIXED_SIZE from rpython.jit.backend.arm.assembler import AssemblerARM from rpython.jit.backend.arm.registers import all_regs, all_vfp_regs +from rpython.jit.backend.llsupport import jitframe +from rpython.jit.backend.llsupport.symbolic import WORD from rpython.jit.backend.llsupport.llmodel import AbstractLLCPU +from rpython.jit.metainterp import history +from rpython.rlib.jit_hooks import LOOP_RUN_CONTAINER +from rpython.rlib.unroll import unrolling_iterable from rpython.rtyper.llinterp import LLInterpreter from rpython.rtyper.lltypesystem import lltype, rffi, llmemory -from rpython.rlib.jit_hooks import LOOP_RUN_CONTAINER -from rpython.jit.backend.arm.arch import FORCE_INDEX_OFS +jitframe.STATICSIZE = JITFRAME_FIXED_SIZE + class AbstractARMCPU(AbstractLLCPU): supports_floats = True @@ -18,14 +24,9 @@ def __init__(self, rtyper, stats, opts=None, translate_support_code=False, gcdescr=None): - if gcdescr is not None: - gcdescr.force_index_ofs = FORCE_INDEX_OFS AbstractLLCPU.__init__(self, rtyper, stats, opts, translate_support_code, gcdescr) - from rpython.jit.backend.llsupport import jitframe - self.deadframe_size_max = llmemory.sizeof(jitframe.DEADFRAME, - self.get_failargs_limit()) def set_debug(self, flag): return self.assembler.set_debug(flag) @@ -64,7 +65,11 @@ setitem(index, null) def make_execute_token(self, *ARGS): - FUNCPTR = lltype.Ptr(lltype.FuncType(ARGS, llmemory.GCREF)) + FUNCPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF], + llmemory.GCREF)) + + lst = [(i, history.getkind(ARG)[0]) for i, ARG in enumerate(ARGS)] + kinds = unrolling_iterable(lst) def execute_token(executable_token, *args): clt = executable_token.compiled_loop_token @@ -74,18 +79,32 @@ assert addr % 8 == 0 func = rffi.cast(FUNCPTR, addr) #llop.debug_print(lltype.Void, ">>>> Entering", addr) + frame_info = clt.frame_info + frame = self.gc_ll_descr.malloc_jitframe(frame_info) + ll_frame = lltype.cast_opaque_ptr(llmemory.GCREF, frame) prev_interpreter = None # help flow space if not self.translate_support_code: prev_interpreter = LLInterpreter.current_interpreter LLInterpreter.current_interpreter = self.debug_ll_interpreter try: - deadframe = func(*args) + num = JITFRAME_FIXED_SIZE * WORD + for i, kind in kinds: + arg = args[i] + if kind == history.INT: + self.set_int_value(ll_frame, num, arg) + elif kind == history.FLOAT: + self.set_float_value(ll_frame, num, arg) + num += WORD # on ARM(32 bit) a FLOAT needs two words + else: + assert kind == history.REF + self.set_ref_value(ll_frame, num, arg) + num += WORD + ll_frame = func(ll_frame) finally: if not self.translate_support_code: LLInterpreter.current_interpreter = prev_interpreter #llop.debug_print(lltype.Void, "<<<< Back") - self.gc_set_extra_threshold() - return deadframe + return ll_frame return execute_token def cast_ptr_to_int(x): From noreply at buildbot.pypy.org Mon Jan 28 16:07:21 2013 From: noreply at buildbot.pypy.org (bivab) Date: Mon, 28 Jan 2013 16:07:21 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: merge upstream Message-ID: <20130128150721.2F74D1C087E@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: jitframe-on-heap Changeset: r60617:af98de29c35f Date: 2013-01-28 16:03 +0100 http://bitbucket.org/pypy/pypy/changeset/af98de29c35f/ Log: merge upstream diff too long, truncating to 2000 out of 4779 lines diff --git a/lib-python/2.7/test/test_iterlen.py b/lib-python/2.7/test/test_iterlen.py --- a/lib-python/2.7/test/test_iterlen.py +++ b/lib-python/2.7/test/test_iterlen.py @@ -94,7 +94,11 @@ def test_no_len_for_infinite_repeat(self): # The repeat() object can also be infinite - self.assertRaises(TypeError, len, repeat(None)) + if test_support.check_impl_detail(pypy=True): + # 3.4 (PEP 424) behavior + self.assertEqual(len(repeat(None)), NotImplemented) + else: + self.assertRaises(TypeError, len, repeat(None)) class TestXrange(TestInvariantWithoutMutations): @@ -230,6 +234,7 @@ self.assertRaises(RuntimeError, b.extend, BadLen()) self.assertRaises(RuntimeError, b.extend, BadLengthHint()) + @test_support.impl_detail("PEP 424 disallows None results", pypy=False) def test_invalid_hint(self): # Make sure an invalid result doesn't muck-up the works self.assertEqual(list(NoneLengthHint()), list(range(10))) diff --git a/lib-python/conftest.py b/lib-python/conftest.py --- a/lib-python/conftest.py +++ b/lib-python/conftest.py @@ -17,8 +17,8 @@ from pypy.conftest import option as pypy_option from pypy.tool.pytest import appsupport -from pypy.tool.pytest.confpath import pypydir, testdir, testresultdir -from pypy.config.parse import parse_info +from pypy.tool.pytest.confpath import pypydir, rpythondir, testdir, testresultdir +from rpython.config.parse import parse_info pytest_plugins = "resultlog", rsyncdirs = ['.', '../pypy/'] @@ -40,7 +40,7 @@ dest="unittest_filter", help="Similar to -k, XXX") def gettimeout(timeout): - from test import pystone + from rpython.translator.test import rpystone if timeout.endswith('mp'): megapystone = float(timeout[:-2]) t, stone = pystone.Proc0(10000) @@ -281,7 +281,7 @@ RegrTest('test_ioctl.py'), RegrTest('test_isinstance.py', core=True), RegrTest('test_iter.py', core=True), - RegrTest('test_iterlen.py', skip="undocumented internal API behavior __length_hint__"), + RegrTest('test_iterlen.py', core=True, usemodules="_collections itertools"), RegrTest('test_itertools.py', core=True, usemodules="itertools struct"), RegrTest('test_json.py'), RegrTest('test_kqueue.py'), @@ -584,7 +584,7 @@ watchdog_name = 'watchdog_nt.py' else: watchdog_name = 'watchdog.py' - watchdog_script = pypydir.join('tool', watchdog_name) + watchdog_script = rpythondir.join('tool', watchdog_name) regr_script = pypydir.join('tool', 'pytest', 'run-script', 'regrverbose.py') diff --git a/py/_code/source.py b/py/_code/source.py --- a/py/_code/source.py +++ b/py/_code/source.py @@ -141,6 +141,8 @@ trysource = self[start:end] if trysource.isparseable(): return start, end + if end == start + 100: # XXX otherwise, it takes forever + break # XXX if end is None: raise IndexError("no valid source range around line %d " % (lineno,)) return start, end diff --git a/pypy/TODO b/pypy/TODO --- a/pypy/TODO +++ b/pypy/TODO @@ -1,8 +1,7 @@ * 32bit x86 * ARM -* shadowstack + asmgcc (shadowstack needs reloading of ebp after frame - got potentially moved, which is after each potentially collecting call - or slowpath malloc) +* asmgcc +* shadowstack details - slowpath of stack check * kill jit2gc on translator -* fix test_singlefloats in test_calling_conventions \ No newline at end of file +* fix test_singlefloats in test_calling_conventions diff --git a/pypy/doc/arm.rst b/pypy/doc/arm.rst --- a/pypy/doc/arm.rst +++ b/pypy/doc/arm.rst @@ -130,15 +130,7 @@ export SB2=/srv/chroot/precise_arm export SB2OPT='-t ARM' -Once this is set, you can call the translator - -:: - - pypy ~/path_to_pypy_checkout/pypy/translator/goal/translate.py -O1 --platform=arm target.py - -If everything worked correctly this should yield an ARM binary. Running this binary in the ARM chroot or on an ARM device should produce the output ``"Hello World"``. - -.. _`this`: +Once this is set, you can call the translator. For example save this file :: @@ -148,3 +140,22 @@ def target(*args): return main, None + +and call the translator + +:: + + pypy ~/path_to_pypy_checkout/pypy/translator/goal/translate.py -O1 --platform=arm target.py + +If everything worked correctly this should yield an ARM binary. Running this binary in the ARM chroot or on an ARM device should produce the output ``"Hello World"``. + +To translate the full python pypy interpreter with a jit, you can cd into pypy/goal and call + +:: + + pypy /rpython/translator/goal/translate.py -Ojit --platform=arm --gcrootfinder=shadowstack --jit-backend=arm targetpypystandalone.py + +The gcrootfinder option is needed to work around `issue 1377`_ and the jit-backend works around `issue 1376`_ + +.. _`issue 1377`: https://bugs.pypy.org/issue1377 +.. _`issue 1376`: https://bugs.pypy.org/issue1376 diff --git a/pypy/doc/getting-started-python.rst b/pypy/doc/getting-started-python.rst --- a/pypy/doc/getting-started-python.rst +++ b/pypy/doc/getting-started-python.rst @@ -86,8 +86,8 @@ 4. Run:: - cd pypy/translator/goal - python translate.py --opt=jit targetpypystandalone.py + cd pypy/goal + python ../../rpython/bin/rpython --opt=jit targetpypystandalone.py possibly replacing ``--opt=jit`` with another `optimization level`_ of your choice like ``--opt=2`` if you do not want to include the JIT diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -5,6 +5,9 @@ .. this is a revision shortly after release-2.0-beta1 .. startrev: 0e6161a009c6 +.. branch: split-rpython +Split rpython and pypy into seperate directories + .. branch: callback-jit Callbacks from C are now better JITted @@ -40,3 +43,7 @@ .. branch: fix-lookinside-iff-oopspec Fixed the interaction between two internal tools for controlling the JIT. + +.. branch: inline-virtualref-2 +Better optimized certain types of frame accesses in the JIT, particularly +around exceptions that escape the function they were raised in. diff --git a/pypy/goal/__init__.py b/pypy/goal/__init__.py new file mode 100644 --- /dev/null +++ b/pypy/goal/__init__.py @@ -0,0 +1,1 @@ +#empty diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py --- a/pypy/interpreter/astcompiler/codegen.py +++ b/pypy/interpreter/astcompiler/codegen.py @@ -904,8 +904,9 @@ def visit_Set(self, s): self.update_position(s.lineno) + elt_count = len(s.elts) if s.elts is not None else 0 self.visit_sequence(s.elts) - self.emit_op_arg(ops.BUILD_SET, len(s.elts)) + self.emit_op_arg(ops.BUILD_SET, elt_count) def visit_Name(self, name): self.update_position(name.lineno) diff --git a/pypy/interpreter/test2/test_app_main.py b/pypy/interpreter/test2/test_app_main.py --- a/pypy/interpreter/test2/test_app_main.py +++ b/pypy/interpreter/test2/test_app_main.py @@ -84,7 +84,7 @@ def check(self, argv, **expected): import StringIO - from rpython.translator.goal import app_main + from pypy.interpreter import app_main saved_sys_argv = sys.argv[:] saved_sys_stdout = sys.stdout saved_sys_stderr = sys.stdout @@ -825,7 +825,7 @@ class TestAppMain: def test_print_info(self): - from rpython.translator.goal import app_main + from pypy.interpreter import app_main import sys, cStringIO prev_so = sys.stdout prev_ti = getattr(sys, 'pypy_translation_info', 'missing') diff --git a/pypy/interpreter/test2/test_targetpypy.py b/pypy/interpreter/test2/test_targetpypy.py --- a/pypy/interpreter/test2/test_targetpypy.py +++ b/pypy/interpreter/test2/test_targetpypy.py @@ -1,6 +1,6 @@ import py -from goal.targetpypystandalone import get_entry_point +from pypy.goal.targetpypystandalone import get_entry_point from pypy.config.pypyoption import get_pypy_config class TestTargetPyPy(object): diff --git a/pypy/module/_ast/test/test_ast.py b/pypy/module/_ast/test/test_ast.py --- a/pypy/module/_ast/test/test_ast.py +++ b/pypy/module/_ast/test/test_ast.py @@ -290,6 +290,12 @@ ]) exec compile(body, '', 'exec') + def test_empty_set(self): + import ast + m = ast.Module(body=[ast.Expr(value=ast.Set(elts=[]))]) + ast.fix_missing_locations(m) + compile(m, "", "exec") + def test_invalid_sum(self): import _ast as ast pos = dict(lineno=2, col_offset=3) diff --git a/pypy/module/_collections/interp_deque.py b/pypy/module/_collections/interp_deque.py --- a/pypy/module/_collections/interp_deque.py +++ b/pypy/module/_collections/interp_deque.py @@ -521,7 +521,11 @@ return self.space.wrap(self.counter) def next(self): - self.deque.checklock(self.lock) + if self.lock is not self.deque.lock: + self.counter = 0 + raise OperationError( + self.space.w_RuntimeError, + self.space.wrap("deque mutated during iteration")) if self.counter == 0: raise OperationError(self.space.w_StopIteration, self.space.w_None) self.counter -= 1 @@ -560,7 +564,11 @@ return self.space.wrap(self.counter) def next(self): - self.deque.checklock(self.lock) + if self.lock is not self.deque.lock: + self.counter = 0 + raise OperationError( + self.space.w_RuntimeError, + self.space.wrap("deque mutated during iteration")) if self.counter == 0: raise OperationError(self.space.w_StopIteration, self.space.w_None) self.counter -= 1 diff --git a/pypy/module/_continuation/interp_continuation.py b/pypy/module/_continuation/interp_continuation.py --- a/pypy/module/_continuation/interp_continuation.py +++ b/pypy/module/_continuation/interp_continuation.py @@ -34,7 +34,7 @@ if self.sthread is not None: raise geterror(self.space, "continulet already __init__ialized") sthread = build_sthread(self.space) - workaround_disable_jit(sthread) + #workaround_disable_jit(sthread) # # hackish: build the frame "by hand", passing it the correct arguments space = self.space @@ -77,7 +77,7 @@ global_state.clear() raise geterror(self.space, "continulet already finished") self.check_sthread() - workaround_disable_jit(self.sthread) + #workaround_disable_jit(self.sthread) # global_state.origin = self if to is None: diff --git a/pypy/module/cpyext/classobject.py b/pypy/module/cpyext/classobject.py --- a/pypy/module/cpyext/classobject.py +++ b/pypy/module/cpyext/classobject.py @@ -22,6 +22,12 @@ w_result.setdict(space, w_dict) return w_result + at cpython_api([PyObject, PyObject, PyObject], PyObject) +def PyInstance_New(space, w_cls, w_arg, w_kw): + """Create a new instance of a specific class. The parameters arg and kw are + used as the positional and keyword parameters to the object's constructor.""" + return space.call(w_cls, w_arg, w_kw) + @cpython_api([PyObject, PyObject], PyObject, error=CANNOT_FAIL) def _PyInstance_Lookup(space, w_instance, w_name): name = space.str_w(w_name) diff --git a/pypy/module/cpyext/include/ceval.h b/pypy/module/cpyext/include/ceval.h new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/include/ceval.h @@ -0,0 +1,1 @@ +/* empty */ diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py --- a/pypy/module/cpyext/stubs.py +++ b/pypy/module/cpyext/stubs.py @@ -176,12 +176,6 @@ """Return true if klass is a subclass of base. Return false in all other cases.""" raise NotImplementedError - at cpython_api([PyObject, PyObject, PyObject], PyObject) -def PyInstance_New(space, cls, arg, kw): - """Create a new instance of a specific class. The parameters arg and kw are - used as the positional and keyword parameters to the object's constructor.""" - raise NotImplementedError - @cpython_api([PyObject], rffi.INT_real, error=-1) def PyCodec_Register(space, search_function): """Register a new codec search function. diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py --- a/pypy/module/cpyext/test/conftest.py +++ b/pypy/module/cpyext/test/conftest.py @@ -1,6 +1,15 @@ import py import pytest +def pytest_configure(config): + from pypy.tool.pytest.objspace import gettestobjspace + # For some reason (probably a ll2ctypes cache issue on linux64) + # it's necessary to run "import time" at least once before any + # other cpyext test, otherwise the same statement will fail in + # test_datetime.py. + space = gettestobjspace(usemodules=['rctime']) + space.getbuiltinmodule("time") + def pytest_ignore_collect(path, config): if config.option.runappdirect: return True # "cannot be run by py.test -A" diff --git a/pypy/module/cpyext/test/test_classobject.py b/pypy/module/cpyext/test/test_classobject.py --- a/pypy/module/cpyext/test/test_classobject.py +++ b/pypy/module/cpyext/test/test_classobject.py @@ -7,8 +7,10 @@ w_class = space.appexec([], """(): class C: x = None - def __init__(self): + def __init__(self, *args, **kwargs): self.x = 1 + self.args = args + self.__dict__.update(kwargs) return C """) @@ -22,6 +24,13 @@ assert space.getattr(w_instance, space.wrap('x')) is space.w_None assert space.unwrap(space.getattr(w_instance, space.wrap('a'))) == 3 + w_instance = api.PyInstance_New(w_class, + space.wrap((3,)), space.wrap(dict(y=2))) + assert space.unwrap(space.getattr(w_instance, space.wrap('x'))) == 1 + assert space.unwrap(space.getattr(w_instance, space.wrap('y'))) == 2 + assert space.unwrap(space.getattr(w_instance, space.wrap('args'))) == (3,) + + def test_lookup(self, space, api): w_instance = space.appexec([], """(): class C: diff --git a/pypy/module/micronumpy/arrayimpl/scalar.py b/pypy/module/micronumpy/arrayimpl/scalar.py --- a/pypy/module/micronumpy/arrayimpl/scalar.py +++ b/pypy/module/micronumpy/arrayimpl/scalar.py @@ -34,6 +34,9 @@ def get_shape(self): return [] + def get_strides(self): + return [] + def create_iter(self, shape=None): return ScalarIterator(self.value) diff --git a/pypy/module/micronumpy/interp_numarray.py b/pypy/module/micronumpy/interp_numarray.py --- a/pypy/module/micronumpy/interp_numarray.py +++ b/pypy/module/micronumpy/interp_numarray.py @@ -40,6 +40,10 @@ self.implementation = self.implementation.set_shape(space, get_shape_from_iterable(space, self.get_size(), w_new_shape)) + def descr_get_strides(self, space): + strides = self.implementation.get_strides() + return space.newtuple([space.wrap(i) for i in strides]) + def get_dtype(self): return self.implementation.dtype @@ -645,6 +649,7 @@ dtype = GetSetProperty(W_NDimArray.descr_get_dtype), shape = GetSetProperty(W_NDimArray.descr_get_shape, W_NDimArray.descr_set_shape), + strides = GetSetProperty(W_NDimArray.descr_get_strides), ndim = GetSetProperty(W_NDimArray.descr_get_ndim), size = GetSetProperty(W_NDimArray.descr_get_size), itemsize = GetSetProperty(W_NDimArray.descr_get_itemsize), diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -684,6 +684,18 @@ assert a.reshape([1]).shape == (1,) raises(ValueError, "a.reshape(3)") + def test_strides(self): + from _numpypy import array + a = array([[1.0, 2.0], + [3.0, 4.0]]) + assert a.strides == (16, 8) + assert a[1:].strides == (16, 8) + + def test_strides_scalar(self): + from _numpypy import array + a = array(42) + assert a.strides == () + def test_add(self): from _numpypy import array a = array(range(5)) diff --git a/pypy/module/pypyjit/interp_resop.py b/pypy/module/pypyjit/interp_resop.py --- a/pypy/module/pypyjit/interp_resop.py +++ b/pypy/module/pypyjit/interp_resop.py @@ -5,14 +5,14 @@ from pypy.interpreter.gateway import unwrap_spec, interp2app from pypy.interpreter.pycode import PyCode from pypy.interpreter.error import OperationError -from rpython.rtyper.lltypesystem import lltype, llmemory +from rpython.rtyper.lltypesystem import lltype from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance, hlstr from rpython.rtyper.lltypesystem.rclass import OBJECT -from rpython.jit.metainterp.resoperation import rop, AbstractResOp +from rpython.jit.metainterp.resoperation import rop from rpython.rlib.nonconst import NonConstant from rpython.rlib import jit_hooks from rpython.rlib.jit import Counters -from rpython.rlib.rarithmetic import r_uint +from rpython.rlib.objectmodel import compute_unique_id from pypy.module.pypyjit.interp_jit import pypyjitdriver class Cache(object): @@ -270,7 +270,8 @@ self.jd_name = debug_info.get_jitdriver().name self.type = debug_info.type if is_bridge: - self.bridge_no = debug_info.fail_descr_no + self.bridge_no = compute_unique_id(debug_info.fail_descr) + #self.bridge_no = debug_info.fail_descr_no self.w_green_key = space.w_None else: self.w_green_key = wrap_greenkey(space, diff --git a/pypy/module/pypyjit/test/test_jit_hook.py b/pypy/module/pypyjit/test/test_jit_hook.py --- a/pypy/module/pypyjit/test/test_jit_hook.py +++ b/pypy/module/pypyjit/test/test_jit_hook.py @@ -2,7 +2,8 @@ import py from pypy.interpreter.gateway import interp2app from pypy.interpreter.pycode import PyCode -from rpython.jit.metainterp.history import JitCellToken, ConstInt, ConstPtr +from rpython.jit.metainterp.history import JitCellToken, ConstInt, ConstPtr,\ + BasicFailDescr from rpython.jit.metainterp.resoperation import rop from rpython.jit.metainterp.logger import Logger from rpython.rtyper.annlowlevel import (cast_instance_to_base_ptr, @@ -69,7 +70,7 @@ oplist, 'loop', greenkey) di_loop.asminfo = AsmInfo(offset, 0, 0) di_bridge = JitDebugInfo(MockJitDriverSD, logger, JitCellToken(), - oplist, 'bridge', fail_descr_no=0) + oplist, 'bridge', fail_descr=BasicFailDescr()) di_bridge.asminfo = AsmInfo(offset, 0, 0) def interp_on_compile(): diff --git a/pypy/module/pypyjit/test_pypy_c/test_thread.py b/pypy/module/pypyjit/test_pypy_c/test_thread.py --- a/pypy/module/pypyjit/test_pypy_c/test_thread.py +++ b/pypy/module/pypyjit/test_pypy_c/test_thread.py @@ -26,3 +26,25 @@ --THREAD-TICK-- jump(..., descr=...) """) + + def test_tls(self): + def main(n): + import thread + local = thread._local() + local.x = 1 + i = 0 + while i < n: + i += local.x + return 0 + log = self.run(main, [500]) + assert round(log.result, 6) == round(main(500), 6) + loop, = log.loops_by_filename(self.filepath) + assert loop.match(""" + i53 = int_lt(i48, i27) + guard_true(i53, descr=...) + i54 = int_add_ovf(i48, i47) + guard_no_overflow(descr=...) + --TICK-- + i58 = arraylen_gc(p43, descr=...) + jump(p0, p1, p3, p5, p10, p12, p14, i54, i27, i47, p45, p43, descr=...) + """) diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py --- a/pypy/module/rctime/interp_time.py +++ b/pypy/module/rctime/interp_time.py @@ -448,7 +448,9 @@ raise OperationError(space.w_ValueError, space.wrap("year out of range")) - if rffi.getintfield(glob_buf, 'c_tm_wday') < 0: + # tm_wday does not need checking of its upper-bound since taking "% + # 7" in gettmarg() automatically restricts the range. + if rffi.getintfield(glob_buf, 'c_tm_wday') < -1: raise OperationError(space.w_ValueError, space.wrap("day of week out of range")) diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py --- a/pypy/module/rctime/test/test_rctime.py +++ b/pypy/module/rctime/test/test_rctime.py @@ -113,6 +113,9 @@ if os.name != 'nt': assert rctime.mktime(rctime.localtime(-1)) == -1 + res = rctime.mktime((2000, 1, 1, 0, 0, 0, -1, -1, -1)) + assert rctime.ctime(res) == 'Sat Jan 1 00:00:00 2000' + def test_asctime(self): import time as rctime rctime.asctime() diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py --- a/pypy/module/sys/vm.py +++ b/pypy/module/sys/vm.py @@ -95,7 +95,7 @@ def exc_info_with_tb(space): operror = space.getexecutioncontext().sys_exc_info() if operror is None: - return space.newtuple([space.w_None,space.w_None,space.w_None]) + return space.newtuple([space.w_None, space.w_None, space.w_None]) else: return space.newtuple([operror.w_type, operror.get_w_value(space), space.wrap(operror.get_traceback())]) diff --git a/pypy/module/thread/os_local.py b/pypy/module/thread/os_local.py --- a/pypy/module/thread/os_local.py +++ b/pypy/module/thread/os_local.py @@ -30,6 +30,9 @@ w_dict = space.newdict(instance=True) self.dicts[ec] = w_dict self._register_in_ec(ec) + # cache the last seen dict, works because we are protected by the GIL + self.last_dict = w_dict + self.last_ec = ec def _register_in_ec(self, ec): if not ec.space.config.translation.rweakref: @@ -60,10 +63,14 @@ def getdict(self, space): ec = space.getexecutioncontext() + if ec is self.last_ec: + return self.last_dict try: w_dict = self.dicts[ec] except KeyError: w_dict = self.create_new_dict(ec) + self.last_ec = ec + self.last_dict = w_dict return w_dict def descr_local__new__(space, w_subtype, __args__): @@ -91,3 +98,5 @@ local = wref() if local is not None: del local.dicts[ec] + local.last_dict = None + local.last_ec = None diff --git a/pypy/module/thread/test/test_local.py b/pypy/module/thread/test/test_local.py --- a/pypy/module/thread/test/test_local.py +++ b/pypy/module/thread/test/test_local.py @@ -108,3 +108,54 @@ gc.collect() assert done == ['ok', 'del'] done.append('shutdown') + +def test_local_caching(): + from pypy.module.thread.os_local import Local + class FakeSpace: + def getexecutioncontext(self): + return self.ec + + def getattr(*args): + pass + def call_obj_args(*args): + pass + def newdict(*args, **kwargs): + return {} + def wrap(self, obj): + return obj + def type(self, obj): + return type(obj) + class config: + class translation: + rweakref = True + + class FakeEC: + def __init__(self, space): + self.space = space + self._thread_local_objs = None + space = FakeSpace() + ec1 = FakeEC(space) + space.ec = ec1 + + l = Local(space, None) + assert l.last_dict is l.dicts[ec1] + assert l.last_ec is ec1 + d1 = l.getdict(space) + assert d1 is l.last_dict + + ec2 = space.ec = FakeEC(space) + d2 = l.getdict(space) + assert l.last_dict is d2 + assert d2 is l.dicts[ec2] + assert l.last_ec is ec2 + dicts = l.dicts + l.dicts = "nope" + assert l.getdict(space) is d2 + l.dicts = dicts + + space.ec = ec1 + assert l.getdict(space) is d1 + l.dicts = "nope" + assert l.getdict(space) is d1 + l.dicts = dicts + diff --git a/pypy/module/unicodedata/test/test_unicodedata.py b/pypy/module/unicodedata/test/test_unicodedata.py new file mode 100644 --- /dev/null +++ b/pypy/module/unicodedata/test/test_unicodedata.py @@ -0,0 +1,104 @@ +import py +import sys + +class AppTestUnicodeData: + spaceconfig = dict(usemodules=('unicodedata',)) + + def test_hangul_syllables(self): + import unicodedata + # Test all leading, vowel and trailing jamo + # but not every combination of them. + for code, name in ((0xAC00, 'HANGUL SYLLABLE GA'), + (0xAE69, 'HANGUL SYLLABLE GGAEG'), + (0xB0D2, 'HANGUL SYLLABLE NYAGG'), + (0xB33B, 'HANGUL SYLLABLE DYAEGS'), + (0xB5A4, 'HANGUL SYLLABLE DDEON'), + (0xB80D, 'HANGUL SYLLABLE RENJ'), + (0xBA76, 'HANGUL SYLLABLE MYEONH'), + (0xBCDF, 'HANGUL SYLLABLE BYED'), + (0xBF48, 'HANGUL SYLLABLE BBOL'), + (0xC1B1, 'HANGUL SYLLABLE SWALG'), + (0xC41A, 'HANGUL SYLLABLE SSWAELM'), + (0xC683, 'HANGUL SYLLABLE OELB'), + (0xC8EC, 'HANGUL SYLLABLE JYOLS'), + (0xCB55, 'HANGUL SYLLABLE JJULT'), + (0xCDBE, 'HANGUL SYLLABLE CWEOLP'), + (0xD027, 'HANGUL SYLLABLE KWELH'), + (0xD290, 'HANGUL SYLLABLE TWIM'), + (0xD4F9, 'HANGUL SYLLABLE PYUB'), + (0xD762, 'HANGUL SYLLABLE HEUBS'), + (0xAE27, 'HANGUL SYLLABLE GYIS'), + (0xB090, 'HANGUL SYLLABLE GGISS'), + (0xB0AD, 'HANGUL SYLLABLE NANG'), + (0xB316, 'HANGUL SYLLABLE DAEJ'), + (0xB57F, 'HANGUL SYLLABLE DDYAC'), + (0xB7E8, 'HANGUL SYLLABLE RYAEK'), + (0xBA51, 'HANGUL SYLLABLE MEOT'), + (0xBCBA, 'HANGUL SYLLABLE BEP'), + (0xBF23, 'HANGUL SYLLABLE BBYEOH'), + (0xD7A3, 'HANGUL SYLLABLE HIH')): + assert unicodedata.name(unichr(code)) == name + assert unicodedata.lookup(name) == unichr(code) + # Test outside the range + raises(ValueError, unicodedata.name, unichr(0xAC00 - 1)) + raises(ValueError, unicodedata.name, unichr(0xD7A3 + 1)) + + def test_cjk(self): + import sys + import unicodedata + cases = ((0x3400, 0x4DB5), + (0x4E00, 0x9FA5)) + if unicodedata.unidata_version >= "5": # don't know the exact limit + cases = ((0x3400, 0x4DB5), + (0x4E00, 0x9FCB), + (0x20000, 0x2A6D6), + (0x2A700, 0x2B734)) + elif unicodedata.unidata_version >= "4.1": + cases = ((0x3400, 0x4DB5), + (0x4E00, 0x9FBB), + (0x20000, 0x2A6D6)) + for first, last in cases: + # Test at and inside the boundary + for i in (first, first + 1, last - 1, last): + charname = 'CJK UNIFIED IDEOGRAPH-%X'%i + char = ('\\U%08X' % i).decode('unicode-escape') + assert unicodedata.name(char) == charname + assert unicodedata.lookup(charname) == char + # Test outside the boundary + for i in first - 1, last + 1: + charname = 'CJK UNIFIED IDEOGRAPH-%X'%i + char = ('\\U%08X' % i).decode('unicode-escape') + try: + unicodedata.name(char) + except ValueError, e: + assert e.message == 'no such name' + raises(KeyError, unicodedata.lookup, charname) + + def test_bug_1704793(self): # from CPython + import unicodedata + assert unicodedata.lookup("GOTHIC LETTER FAIHU") == u'\U00010346' + + def test_normalize(self): + import unicodedata + raises(TypeError, unicodedata.normalize, 'x') + + @py.test.mark.skipif("sys.maxunicode < 0x10ffff") + def test_normalize_wide(self): + import unicodedata + assert unicodedata.normalize('NFC', u'\U000110a5\U000110ba') == u'\U000110ab' + + def test_linebreaks(self): + linebreaks = (0x0a, 0x0b, 0x0c, 0x0d, 0x85, + 0x1c, 0x1d, 0x1e, 0x2028, 0x2029) + for i in linebreaks: + for j in range(-2, 3): + lines = (unichr(i + j) + u'A').splitlines() + if i + j in linebreaks: + assert len(lines) == 2 + else: + assert len(lines) == 1 + + def test_mirrored(self): + import unicodedata + # For no reason, unicodedata.mirrored() returns an int, not a bool + assert repr(unicodedata.mirrored(u' ')) == '0' diff --git a/pypy/module/unicodedata/test_unicodedata.py b/pypy/module/unicodedata/test_unicodedata.py deleted file mode 100644 --- a/pypy/module/unicodedata/test_unicodedata.py +++ /dev/null @@ -1,103 +0,0 @@ - -class AppTestUnicodeData: - spaceconfig = dict(usemodules=('unicodedata',)) - - def test_hangul_syllables(self): - import unicodedata - # Test all leading, vowel and trailing jamo - # but not every combination of them. - for code, name in ((0xAC00, 'HANGUL SYLLABLE GA'), - (0xAE69, 'HANGUL SYLLABLE GGAEG'), - (0xB0D2, 'HANGUL SYLLABLE NYAGG'), - (0xB33B, 'HANGUL SYLLABLE DYAEGS'), - (0xB5A4, 'HANGUL SYLLABLE DDEON'), - (0xB80D, 'HANGUL SYLLABLE RENJ'), - (0xBA76, 'HANGUL SYLLABLE MYEONH'), - (0xBCDF, 'HANGUL SYLLABLE BYED'), - (0xBF48, 'HANGUL SYLLABLE BBOL'), - (0xC1B1, 'HANGUL SYLLABLE SWALG'), - (0xC41A, 'HANGUL SYLLABLE SSWAELM'), - (0xC683, 'HANGUL SYLLABLE OELB'), - (0xC8EC, 'HANGUL SYLLABLE JYOLS'), - (0xCB55, 'HANGUL SYLLABLE JJULT'), - (0xCDBE, 'HANGUL SYLLABLE CWEOLP'), - (0xD027, 'HANGUL SYLLABLE KWELH'), - (0xD290, 'HANGUL SYLLABLE TWIM'), - (0xD4F9, 'HANGUL SYLLABLE PYUB'), - (0xD762, 'HANGUL SYLLABLE HEUBS'), - (0xAE27, 'HANGUL SYLLABLE GYIS'), - (0xB090, 'HANGUL SYLLABLE GGISS'), - (0xB0AD, 'HANGUL SYLLABLE NANG'), - (0xB316, 'HANGUL SYLLABLE DAEJ'), - (0xB57F, 'HANGUL SYLLABLE DDYAC'), - (0xB7E8, 'HANGUL SYLLABLE RYAEK'), - (0xBA51, 'HANGUL SYLLABLE MEOT'), - (0xBCBA, 'HANGUL SYLLABLE BEP'), - (0xBF23, 'HANGUL SYLLABLE BBYEOH'), - (0xD7A3, 'HANGUL SYLLABLE HIH')): - assert unicodedata.name(unichr(code)) == name - assert unicodedata.lookup(name) == unichr(code) - # Test outside the range - raises(ValueError, unicodedata.name, unichr(0xAC00 - 1)) - raises(ValueError, unicodedata.name, unichr(0xD7A3 + 1)) - - def test_cjk(self): - import sys - import unicodedata - cases = ((0x3400, 0x4DB5), - (0x4E00, 0x9FA5)) - if unicodedata.unidata_version >= "5": # don't know the exact limit - cases = ((0x3400, 0x4DB5), - (0x4E00, 0x9FCB), - (0x20000, 0x2A6D6), - (0x2A700, 0x2B734)) - elif unicodedata.unidata_version >= "4.1": - cases = ((0x3400, 0x4DB5), - (0x4E00, 0x9FBB), - (0x20000, 0x2A6D6)) - for first, last in cases: - # Test at and inside the boundary - for i in (first, first + 1, last - 1, last): - charname = 'CJK UNIFIED IDEOGRAPH-%X'%i - char = ('\\U%08X' % i).decode('unicode-escape') - assert unicodedata.name(char) == charname - assert unicodedata.lookup(charname) == char - # Test outside the boundary - for i in first - 1, last + 1: - charname = 'CJK UNIFIED IDEOGRAPH-%X'%i - char = ('\\U%08X' % i).decode('unicode-escape') - try: - unicodedata.name(char) - except ValueError, e: - assert e.message == 'no such name' - raises(KeyError, unicodedata.lookup, charname) - - def test_bug_1704793(self): # from CPython - import unicodedata - assert unicodedata.lookup("GOTHIC LETTER FAIHU") == u'\U00010346' - - def test_normalize(self): - import unicodedata - raises(TypeError, unicodedata.normalize, 'x') - - def test_normalize_wide(self): - import sys, unicodedata - if sys.maxunicode < 0x10ffff: - skip("requires a 'wide' python build.") - assert unicodedata.normalize('NFC', u'\U000110a5\U000110ba') == u'\U000110ab' - - def test_linebreaks(self): - linebreaks = (0x0a, 0x0b, 0x0c, 0x0d, 0x85, - 0x1c, 0x1d, 0x1e, 0x2028, 0x2029) - for i in linebreaks: - for j in range(-2, 3): - lines = (unichr(i + j) + u'A').splitlines() - if i + j in linebreaks: - assert len(lines) == 2 - else: - assert len(lines) == 1 - - def test_mirrored(self): - import unicodedata - # For no reason, unicodedata.mirrored() returns an int, not a bool - assert repr(unicodedata.mirrored(u' ')) == '0' diff --git a/pypy/objspace/std/bytearraytype.py b/pypy/objspace/std/bytearraytype.py --- a/pypy/objspace/std/bytearraytype.py +++ b/pypy/objspace/std/bytearraytype.py @@ -15,6 +15,7 @@ str_splitlines, str_translate) from pypy.objspace.std.listtype import ( list_append, list_extend) +from rpython.rlib.objectmodel import newlist_hint, resizelist_hint bytearray_insert = SMM('insert', 3, @@ -87,8 +88,10 @@ return [c for c in string] # sequence of bytes - data = [] w_iter = space.iter(w_source) + length_hint = space.length_hint(w_source, 0) + data = newlist_hint(length_hint) + extended = 0 while True: try: w_item = space.next(w_iter) @@ -98,6 +101,9 @@ break value = getbytevalue(space, w_item) data.append(value) + extended += 1 + if extended < length_hint: + resizelist_hint(data, extended) return data def descr_bytearray__reduce__(space, w_self): diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py --- a/pypy/objspace/std/dictmultiobject.py +++ b/pypy/objspace/std/dictmultiobject.py @@ -54,6 +54,9 @@ # every module needs its own strategy, because the strategy stores # the version tag strategy = ModuleDictStrategy(space) + elif space.config.objspace.std.withmapdict and instance: + from pypy.objspace.std.mapdict import MapDictStrategy + strategy = space.fromcache(MapDictStrategy) elif instance or strdict or module: assert w_type is None @@ -349,7 +352,7 @@ self.pos = 0 def length(self): - if self.dictimplementation is not None: + if self.dictimplementation is not None and self.len != -1: return self.len - self.pos return 0 diff --git a/pypy/objspace/std/mapdict.py b/pypy/objspace/std/mapdict.py --- a/pypy/objspace/std/mapdict.py +++ b/pypy/objspace/std/mapdict.py @@ -593,6 +593,9 @@ # ____________________________________________________________ # dict implementation +def get_terminator_for_dicts(space): + return DictTerminator(space, None) + class MapDictStrategy(DictStrategy): erase, unerase = rerased.new_erasing_pair("map") @@ -602,13 +605,19 @@ def __init__(self, space): self.space = space + def get_empty_storage(self): + w_result = Object() + terminator = self.space.fromcache(get_terminator_for_dicts) + w_result._init_empty(terminator) + return self.erase(w_result) + def switch_to_object_strategy(self, w_dict): w_obj = self.unerase(w_dict.dstorage) strategy = self.space.fromcache(ObjectDictStrategy) dict_w = strategy.unerase(strategy.get_empty_storage()) w_dict.strategy = strategy w_dict.dstorage = strategy.erase(dict_w) - assert w_obj.getdict(self.space) is w_dict + assert w_obj.getdict(self.space) is w_dict or w_obj._get_mapdict_map().terminator.w_cls is None materialize_r_dict(self.space, w_obj, dict_w) def getitem(self, w_dict, w_key): diff --git a/pypy/objspace/std/setobject.py b/pypy/objspace/std/setobject.py --- a/pypy/objspace/std/setobject.py +++ b/pypy/objspace/std/setobject.py @@ -898,7 +898,7 @@ raise NotImplementedError def length(self): - if self.setimplementation is not None: + if self.setimplementation is not None and self.len != -1: return self.len - self.pos return 0 diff --git a/pypy/objspace/std/test/test_dictmultiobject.py b/pypy/objspace/std/test/test_dictmultiobject.py --- a/pypy/objspace/std/test/test_dictmultiobject.py +++ b/pypy/objspace/std/test/test_dictmultiobject.py @@ -954,6 +954,7 @@ withcelldict = False withmethodcache = False withidentitydict = False + withmapdict = False FakeSpace.config = Config() diff --git a/pypy/objspace/std/test/test_lengthhint.py b/pypy/objspace/std/test/test_lengthhint.py --- a/pypy/objspace/std/test/test_lengthhint.py +++ b/pypy/objspace/std/test/test_lengthhint.py @@ -6,7 +6,7 @@ SIZE = 4 ITEMS = range(SIZE) - def _test_length_hint(self, w_obj): + def _test_length_hint(self, w_obj, w_mutate=None): space = self.space assert space.length_hint(w_obj, 8) == self.SIZE @@ -18,6 +18,13 @@ space.next(w_iter) assert space.length_hint(w_iter, 8) == self.SIZE - 1 + if w_mutate is not None: + # Test handling of collections that enforce length + # immutability during iteration + space.call_function(w_mutate) + space.raises_w(space.w_RuntimeError, space.next, w_iter) + assert space.length_hint(w_iter, 8) == 0 + def test_bytearray(self): space = self.space w_bytearray = space.call_function(space.w_bytearray, @@ -31,16 +38,20 @@ self._test_length_hint(w_dict) def test_dict_iterkeys(self): - w_iterkeys = self.space.appexec([], """(): - return dict.fromkeys(%r).iterkeys() - """ % self.ITEMS) - self._test_length_hint(w_iterkeys) + space = self.space + w_iterkeys, w_mutate = space.fixedview(space.appexec([], """(): + d = dict.fromkeys(%r) + return d.iterkeys(), d.popitem + """ % self.ITEMS), 2) + self._test_length_hint(w_iterkeys, w_mutate) def test_dict_values(self): - w_itervalues = self.space.appexec([], """(): - return dict.fromkeys(%r).itervalues() - """ % self.ITEMS) - self._test_length_hint(w_itervalues) + space = self.space + w_itervalues, w_mutate = space.fixedview(space.appexec([], """(): + d = dict.fromkeys(%r) + return d.itervalues(), d.popitem + """ % self.ITEMS), 2) + self._test_length_hint(w_itervalues, w_mutate) def test_frozenset(self): space = self.space @@ -50,7 +61,8 @@ def test_set(self): space = self.space w_set = space.call_function(space.w_set, space.wrap(self.ITEMS)) - self._test_length_hint(w_set) + w_mutate = space.getattr(w_set, space.wrap('pop')) + self._test_length_hint(w_set, w_mutate) def test_list(self): self._test_length_hint(self.space.wrap(self.ITEMS)) @@ -101,12 +113,19 @@ self._test_length_hint(W_Repeat(space, space.wrap(22), space.wrap(self.SIZE))) - def test_collections_deque(self): + def _setup_deque(self): space = self.space w_deque = W_Deque(space) space.call_method(w_deque, '__init__', space.wrap(self.ITEMS)) - self._test_length_hint(w_deque) - self._test_length_hint(w_deque.reviter()) + w_mutate = space.getattr(w_deque, space.wrap('pop')) + return w_deque, w_mutate + + def test_collections_deque(self): + self._test_length_hint(*self._setup_deque()) + + def test_collections_deque_rev(self): + w_deque, w_mutate = self._setup_deque() + self._test_length_hint(w_deque.reviter(), w_mutate) def test_default(self): space = self.space diff --git a/pypy/objspace/std/test/test_mapdict.py b/pypy/objspace/std/test/test_mapdict.py --- a/pypy/objspace/std/test/test_mapdict.py +++ b/pypy/objspace/std/test/test_mapdict.py @@ -1,7 +1,17 @@ from pypy.objspace.std.test.test_dictmultiobject import FakeSpace, W_DictMultiObject from pypy.objspace.std.mapdict import * +class Config: + class objspace: + class std: + withsmalldicts = False + withcelldict = False + withmethodcache = False + withidentitydict = False + withmapdict = True + space = FakeSpace() +space.config = Config class Class(object): def __init__(self, hasdict=True): @@ -1061,3 +1071,11 @@ return A() """) assert w_dict.user_overridden_class + +def test_newdict_instance(): + w_dict = space.newdict(instance=True) + assert type(w_dict.strategy) is MapDictStrategy + +class TestMapDictImplementationUsingnewdict(BaseTestRDictImplementation): + StrategyClass = MapDictStrategy + # NB: the get_impl method is not overwritten here, as opposed to above diff --git a/pypy/pytest-A.cfg b/pypy/pytest-A.cfg --- a/pypy/pytest-A.cfg +++ b/pypy/pytest-A.cfg @@ -1,5 +1,5 @@ cherrypick = ['interpreter', 'objspace/test', 'objspace/std', 'module'] -interp = ['translator/goal/pypy-c'] +interp = ['goal/pypy-c'] test_driver = ['test_all.py', '-A'] diff --git a/pypy/test_all.py b/pypy/test_all.py --- a/pypy/test_all.py +++ b/pypy/test_all.py @@ -23,7 +23,8 @@ if len(sys.argv) == 1 and os.path.dirname(sys.argv[0]) in '.': print >> sys.stderr, __doc__ sys.exit(2) - + #Add toplevel repository dir to sys.path + sys.path.insert(0,os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) import pytest import pytest_cov sys.exit(pytest.main(plugins=[pytest_cov])) diff --git a/pypy/tool/jitlogparser/parser.py b/pypy/tool/jitlogparser/parser.py --- a/pypy/tool/jitlogparser/parser.py +++ b/pypy/tool/jitlogparser/parser.py @@ -215,6 +215,7 @@ _lineset = None is_bytecode = False inline_level = None + bytecode_name = None # factory method TraceForOpcode = TraceForOpcode @@ -244,23 +245,29 @@ return ",".join([str(len(v)) for v in stack]) def append_to_res(bc): - if not stack: - stack.append([]) - else: - if bc.inline_level is not None and bc.inline_level + 1 != len(stack): - if bc.inline_level < len(stack): + if bc.inline_level is not None: + if bc.inline_level == len(stack) - 1: + pass + elif bc.inline_level > len(stack) - 1: + stack.append([]) + else: + while bc.inline_level + 1 < len(stack): last = stack.pop() stack[-1].append(cls(last, getpath(stack), storage)) - else: - stack.append([]) stack[-1].append(bc) so_far = [] stack = [] + nothing_yet = True for op in operations: if op.name == 'debug_merge_point': if so_far: - append_to_res(cls.TraceForOpcode(so_far, storage, loopname)) + opc = cls.TraceForOpcode(so_far, storage, loopname) + if nothing_yet: + nothing_yet = False + for i in xrange(opc.inline_level + 1): + stack.append([]) + append_to_res(opc) if limit: break so_far = [] @@ -365,7 +372,7 @@ m = re.search('has address ([-\da-f]+)', entry) addr = int(m.group(1), 16) entry = entry.lower() - m = re.search('guard \d+', entry) + m = re.search('guard [\da-f]+', entry) name = m.group(0) else: name = entry[:entry.find('(') - 1].lower() @@ -388,8 +395,8 @@ comm = loop.comment comm = comm.lower() if comm.startswith('# bridge'): - m = re.search('guard \d+', comm) - name = m.group(0) + m = re.search('guard (\d+)', comm) + name = 'guard ' + hex(int(m.group(1)))[2:] elif "(" in comm: name = comm[2:comm.find('(')-1] else: @@ -437,3 +444,7 @@ if line: num, count = line.split(':', 2) mapping[num].count = int(count) + +if __name__ == '__main__': + import_log(sys.argv[1]) + diff --git a/pypy/tool/jitlogparser/test/test_parser.py b/pypy/tool/jitlogparser/test/test_parser.py --- a/pypy/tool/jitlogparser/test/test_parser.py +++ b/pypy/tool/jitlogparser/test/test_parser.py @@ -354,3 +354,25 @@ f = Function.from_operations(loop.operations, LoopStorage()) assert f.chunks[-1].filename == 'x.py' assert f.filename is None + +def test_parse_2_levels_up(): + loop = parse(""" + [] + debug_merge_point(0, 0, 'one') + debug_merge_point(1, 0, 'two') + debug_merge_point(2, 0, 'three') + debug_merge_point(0, 0, 'one') + """) + f = Function.from_operations(loop.operations, LoopStorage()) + assert len(f.chunks) == 3 + +def test_parse_from_inside(): + loop = parse(""" + [] + debug_merge_point(1, 0, 'two') + debug_merge_point(2, 0, 'three') + debug_merge_point(0, 0, 'one') + """) + f = Function.from_operations(loop.operations, LoopStorage()) + assert len(f.chunks) == 2 + diff --git a/pypy/tool/pytest/confpath.py b/pypy/tool/pytest/confpath.py --- a/pypy/tool/pytest/confpath.py +++ b/pypy/tool/pytest/confpath.py @@ -1,8 +1,10 @@ import py import pypy +import rpython from pypy.tool import lib_pypy pypydir = py.path.local(pypy.__file__).dirpath() +rpythondir = py.path.local(rpython.__file__).dirpath() distdir = pypydir.dirpath() testresultdir = distdir.join('testresult') assert pypydir.check(dir=1) diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -11,8 +11,10 @@ import shutil import sys +import os +#Add toplevel repository dir to sys.path +sys.path.insert(0,os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))) import py -import os import fnmatch from rpython.tool.udir import udir @@ -48,7 +50,7 @@ basename = 'pypy-c' if sys.platform == 'win32': basename += '.exe' - pypy_c = basedir.join(basename) + pypy_c = basedir.join('pypy', 'goal', basename) else: pypy_c = py.path.local(override_pypy_c) if not pypy_c.check(): diff --git a/pypy/bin/translatorshell.py b/rpython/bin/translatorshell.py rename from pypy/bin/translatorshell.py rename to rpython/bin/translatorshell.py diff --git a/rpython/jit/backend/arm/test/support.py b/rpython/jit/backend/arm/test/support.py --- a/rpython/jit/backend/arm/test/support.py +++ b/rpython/jit/backend/arm/test/support.py @@ -14,6 +14,9 @@ def check_jumps(self, maxcount): pass +if not getattr(os, 'uname', None): + pytest.skip('cannot run arm tests on non-posix platform') + if os.uname()[1] == 'llaima.local': AS = '~/Code/arm-jit/android/android-ndk-r4b//build/prebuilt/darwin-x86/arm-eabi-4.4.0/arm-eabi/bin/as' else: diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py --- a/rpython/jit/backend/llgraph/runner.py +++ b/rpython/jit/backend/llgraph/runner.py @@ -582,7 +582,7 @@ class LLFrame(object): - _TYPE = lltype.Signed + _TYPE = llmemory.GCREF forced_deadframe = None overflow_flag = False @@ -595,6 +595,22 @@ for box, arg in zip(argboxes, args): self.setenv(box, arg) + def __eq__(self, other): + # this is here to avoid crashes in 'token == TOKEN_TRACING_RESCALL' + from rpython.jit.metainterp.virtualizable import TOKEN_NONE + from rpython.jit.metainterp.virtualizable import TOKEN_TRACING_RESCALL + if isinstance(other, LLFrame): + return self is other + if other == TOKEN_NONE or other == TOKEN_TRACING_RESCALL: + return False + assert 0 + + def __ne__(self, other): + return not (self == other) + + def _identityhash(self): + return hash(self) + def setenv(self, box, arg): if box.type == INT: # typecheck the result @@ -863,7 +879,8 @@ def reset_vable(jd, vable): if jd.index_of_virtualizable != -1: fielddescr = jd.vable_token_descr - self.cpu.bh_setfield_gc(vable, 0, fielddescr) + NULL = lltype.nullptr(llmemory.GCREF.TO) + self.cpu.bh_setfield_gc(vable, NULL, fielddescr) faildescr = self.cpu.get_latest_descr(pframe) if faildescr == self.cpu.done_with_this_frame_descr_int: reset_vable(jd, vable) diff --git a/rpython/jit/backend/llsupport/asmmemmgr.py b/rpython/jit/backend/llsupport/asmmemmgr.py --- a/rpython/jit/backend/llsupport/asmmemmgr.py +++ b/rpython/jit/backend/llsupport/asmmemmgr.py @@ -248,6 +248,12 @@ index += self.SUBBLOCK_SIZE block.data[index] = char + def overwrite32(self, index, val): + self.overwrite(index, chr(val & 0xff)) + self.overwrite(index + 1, chr((val >> 8) & 0xff)) + self.overwrite(index + 2, chr((val >> 16) & 0xff)) + self.overwrite(index + 3, chr((val >> 24) & 0xff)) + def get_relative_pos(self): return self._baserelpos + self._cursubindex diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py --- a/rpython/jit/backend/llsupport/gc.py +++ b/rpython/jit/backend/llsupport/gc.py @@ -122,6 +122,8 @@ for name in ['jf_descr', 'jf_guard_exc', 'jf_force_descr', 'jf_frame_info', 'jf_gcmap']: setattr(descrs, name, cpu.fielddescrof(jitframe.JITFRAME, name)) + descrs.jfi_frame_size = cpu.fielddescrof(jitframe.JITFRAMEINFO, + 'jfi_frame_size') descrs.jfi_frame_depth = cpu.fielddescrof(jitframe.JITFRAMEINFO, 'jfi_frame_depth') return descrs diff --git a/rpython/jit/backend/llsupport/jitframe.py b/rpython/jit/backend/llsupport/jitframe.py --- a/rpython/jit/backend/llsupport/jitframe.py +++ b/rpython/jit/backend/llsupport/jitframe.py @@ -3,7 +3,6 @@ from rpython.rlib.objectmodel import specialize from rpython.rlib.debug import ll_assert -STATICSIZE = 0 # patch from the assembler backend SIZEOFSIGNED = rffi.sizeof(lltype.Signed) IS_32BIT = (SIZEOFSIGNED == 2 ** 31 - 1) @@ -14,10 +13,19 @@ GCMAP = lltype.GcArray(lltype.Unsigned) NULLGCMAP = lltype.nullptr(GCMAP) +def jitframeinfo_set_depth(jfi, base_ofs, new_depth): + jfi.jfi_frame_depth = new_depth + jfi.jfi_frame_size = base_ofs + new_depth * SIZEOFSIGNED + JITFRAMEINFO = lltype.GcStruct( 'JITFRAMEINFO', - # the depth of frame + # the depth of the frame ('jfi_frame_depth', lltype.Signed), + # the total size of the frame, in bytes + ('jfi_frame_size', lltype.Signed), + adtmeths = { + 'set_frame_depth': jitframeinfo_set_depth, + }, ) NULLFRAMEINFO = lltype.nullptr(JITFRAMEINFO) @@ -99,13 +107,15 @@ # 32 possible bits state = (fld >> 3) & 0x1f no = fld >> (3 + 5) - MAX = 31 + MAX = 32 else: # 64 possible bits state = (fld >> 3) & 0x3f no = fld >> (3 + 6) - MAX = 63 + MAX = 64 gcmap = (obj_addr + getofs('jf_gcmap')).address[0] + if not gcmap: + return llmemory.NULL gcmap_lgt = (gcmap + GCMAPLENGTHOFS).signed[0] while no < gcmap_lgt: cur = (gcmap + GCMAPBASEOFS + UNSIGN_SIZE * no).unsigned[0] diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -137,23 +137,42 @@ else: raise NotImplementedError(op.getopname()) + def gen_malloc_frame(self, frame_info, frame, size_box): + descrs = self.gc_ll_descr.getframedescrs(self.cpu) + if self.gc_ll_descr.kind == 'boehm': + op0 = ResOperation(rop.GETFIELD_GC, [history.ConstPtr(frame_info)], + size_box, + descr=descrs.jfi_frame_depth) + self.newops.append(op0) + op1 = ResOperation(rop.NEW_ARRAY, [size_box], frame, + descr=descrs.arraydescr) + self.handle_new_array(descrs.arraydescr, op1) + else: + # we read size in bytes here, not the length + op0 = ResOperation(rop.GETFIELD_GC, [history.ConstPtr(frame_info)], + size_box, + descr=descrs.jfi_frame_size) + self.newops.append(op0) + self.gen_malloc_nursery_varsize(size_box, frame, is_small=True) + self.gen_initialize_tid(frame, descrs.arraydescr.tid) + length_box = history.BoxInt() + op1 = ResOperation(rop.GETFIELD_GC, [history.ConstPtr(frame_info)], + length_box, + descr=descrs.jfi_frame_depth) + self.newops.append(op1) + self.gen_initialize_len(frame, length_box, + descrs.arraydescr.lendescr) + def handle_call_assembler(self, op): descrs = self.gc_ll_descr.getframedescrs(self.cpu) loop_token = op.getdescr() assert isinstance(loop_token, history.JitCellToken) - lgt_box = history.BoxInt() - frame = history.BoxPtr() jfi = loop_token.compiled_loop_token.frame_info llref = lltype.cast_opaque_ptr(llmemory.GCREF, jfi) rgc._make_sure_does_not_move(llref) - op0 = ResOperation(rop.GETFIELD_GC, [history.ConstPtr(llref)], lgt_box, - descr=descrs.jfi_frame_depth) - self.newops.append(op0) - # XXX for now it generates call_malloc_gc, instead of - # call_malloc_nursery, because the array is strange - op1 = ResOperation(rop.NEW_ARRAY, [lgt_box], frame, - descr=descrs.arraydescr) - self.handle_new_array(descrs.arraydescr, op1) + size_box = history.BoxInt() + frame = history.BoxPtr() + self.gen_malloc_frame(llref, frame, size_box) op2 = ResOperation(rop.SETFIELD_GC, [frame, history.ConstPtr(llref)], None, descr=descrs.jf_frame_info) self.newops.append(op2) @@ -167,7 +186,7 @@ assert isinstance(descr, JitCellToken) jd = descr.outermost_jitdriver_sd args = [frame] - if jd.index_of_virtualizable >= 0: + if jd and jd.index_of_virtualizable >= 0: args = [frame, arglist[jd.index_of_virtualizable]] else: args = [frame] @@ -259,6 +278,18 @@ self._gen_call_malloc_gc([ConstInt(addr), v_num_elem], v_result, self.gc_ll_descr.malloc_unicode_descr) + def gen_malloc_nursery_varsize(self, sizebox, v_result, is_small=False): + """ Generate CALL_MALLOC_NURSERY_VARSIZE_SMALL + """ + assert is_small + self.emitting_an_operation_that_can_collect() + op = ResOperation(rop.CALL_MALLOC_NURSERY_VARSIZE_SMALL, + [sizebox], + v_result) + + self.newops.append(op) + self.recent_mallocs[v_result] = None + def gen_malloc_nursery(self, size, v_result): """Try to generate or update a CALL_MALLOC_NURSERY. If that fails, generate a plain CALL_MALLOC_GC instead. diff --git a/rpython/jit/backend/llsupport/test/test_gc.py b/rpython/jit/backend/llsupport/test/test_gc.py --- a/rpython/jit/backend/llsupport/test/test_gc.py +++ b/rpython/jit/backend/llsupport/test/test_gc.py @@ -1,3 +1,4 @@ +import sys from rpython.rtyper.lltypesystem import lltype, llmemory, rstr from rpython.rtyper.lltypesystem.lloperation import llop from rpython.rtyper.annlowlevel import llhelper @@ -270,37 +271,37 @@ return (frame_adr + jitframe.getofs('jf_frame') + jitframe.BASEITEMOFS + jitframe.SIGN_SIZE * no) - PREV_STATICSIZE = jitframe.STATICSIZE - try: - jitframe.STATICSIZE = 3 - frame_info = lltype.malloc(jitframe.JITFRAMEINFO, zero=True) - frame = lltype.malloc(jitframe.JITFRAME, 15, zero=True) - frame.jf_frame_info = frame_info - frame.jf_gcmap = lltype.malloc(jitframe.GCMAP, 2) - frame.jf_gcmap[0] = r_uint(1 | 2 | 8 | 32 | 128) - frame.jf_gcmap[1] = r_uint(2 | 16 | 32 | 128) - frame_adr = llmemory.cast_ptr_to_adr(frame) - all_addrs = [] - next = jitframe.jitframe_trace(frame_adr, llmemory.NULL) - while next: - all_addrs.append(next) - next = jitframe.jitframe_trace(frame_adr, next) - counter = 0 - for name in jitframe.JITFRAME._names: - TP = getattr(jitframe.JITFRAME, name) - if isinstance(TP, lltype.Ptr): # only GC pointers - assert all_addrs[counter] == frame_adr + jitframe.getofs(name) - counter += 1 - # gcpattern - assert all_addrs[6] == indexof(0) - assert all_addrs[7] == indexof(1) - assert all_addrs[8] == indexof(3) - assert all_addrs[9] == indexof(5) - assert all_addrs[10] == indexof(7) - # XXX 32bit - assert all_addrs[11] == indexof(65) + frame_info = lltype.malloc(jitframe.JITFRAMEINFO, zero=True) + frame = lltype.malloc(jitframe.JITFRAME, 15, zero=True) + frame.jf_frame_info = frame_info + frame.jf_gcmap = lltype.malloc(jitframe.GCMAP, 2) + if sys.maxint == 2**31 - 1: + max = r_uint(2 ** 31) + else: + max = r_uint(2 ** 63) + frame.jf_gcmap[0] = r_uint(1 | 2 | 8 | 32 | 128) | max + frame.jf_gcmap[1] = r_uint(2 | 16 | 32 | 128) + frame_adr = llmemory.cast_ptr_to_adr(frame) + all_addrs = [] + next = jitframe.jitframe_trace(frame_adr, llmemory.NULL) + while next: + all_addrs.append(next) + next = jitframe.jitframe_trace(frame_adr, next) + counter = 0 + for name in jitframe.JITFRAME._names: + TP = getattr(jitframe.JITFRAME, name) + if isinstance(TP, lltype.Ptr): # only GC pointers + assert all_addrs[counter] == frame_adr + jitframe.getofs(name) + counter += 1 + # gcpattern + assert all_addrs[6] == indexof(0) + assert all_addrs[7] == indexof(1) + assert all_addrs[8] == indexof(3) + assert all_addrs[9] == indexof(5) + assert all_addrs[10] == indexof(7) + assert all_addrs[11] == indexof(63) + # XXX 32bit + assert all_addrs[12] == indexof(65) - assert len(all_addrs) == 6 + 5 + 4 - # 6 static fields, 4 addresses from gcmap, 2 from gcpattern - finally: - jitframe.STATICSIZE = PREV_STATICSIZE + assert len(all_addrs) == 6 + 6 + 4 + # 6 static fields, 4 addresses from gcmap, 2 from gcpattern diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py --- a/rpython/jit/backend/llsupport/test/test_rewrite.py +++ b/rpython/jit/backend/llsupport/test/test_rewrite.py @@ -76,8 +76,11 @@ ll_frame_info = lltype.cast_opaque_ptr(llmemory.GCREF, frame_info) clt.frame_info = frame_info frame_info.jfi_frame_depth = 13 + frame_info.jfi_frame_size = 255 framedescrs = self.gc_ll_descr.getframedescrs(self.cpu) + framelendescr = framedescrs.arraydescr.lendescr jfi_frame_depth = framedescrs.jfi_frame_depth + jfi_frame_size = framedescrs.jfi_frame_size jf_frame_info = framedescrs.jf_frame_info signedframedescr = self.cpu.signedframedescr floatframedescr = self.cpu.floatframedescr @@ -737,12 +740,13 @@ i2 = call_assembler(i0, f0, descr=casmdescr) """, """ [i0, f0] - i1 = getfield_gc(ConstPtr(ll_frame_info), descr=jfi_frame_depth) - p1 = call_malloc_gc(ConstClass(malloc_array_nonstandard), 1, 2, 0, 0, i1, descr=malloc_array_nonstandard_descr) + i1 = getfield_gc(ConstPtr(ll_frame_info), descr=jfi_frame_size) + p1 = call_malloc_nursery_varsize_small(i1) + setfield_gc(p1, 0, descr=tiddescr) + i2 = getfield_gc(ConstPtr(ll_frame_info), descr=jfi_frame_depth) + setfield_gc(p1, i2, descr=framelendescr) setfield_gc(p1, ConstPtr(ll_frame_info), descr=jf_frame_info) setarrayitem_gc(p1, 0, i0, descr=signedframedescr) setarrayitem_gc(p1, 1, f0, descr=floatframedescr) - i2 = call_assembler(p1, descr=casmdescr) + i3 = call_assembler(p1, descr=casmdescr) """) - # XXX we want call_malloc_nursery actually, but let's not care - # for now, the array is a bit non-standard diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py --- a/rpython/jit/backend/model.py +++ b/rpython/jit/backend/model.py @@ -1,5 +1,4 @@ from rpython.rlib.debug import debug_start, debug_print, debug_stop -from rpython.jit.metainterp import compile from rpython.rtyper.lltypesystem import lltype class CPUTotalTracker(object): @@ -23,7 +22,6 @@ propagate_exception_descr = None def __init__(self): - self.__dict__.update(compile.make_done_loop_tokens()) self.tracker = CPUTotalTracker() def _freeze_(self): diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py --- a/rpython/jit/backend/test/runner_test.py +++ b/rpython/jit/backend/test/runner_test.py @@ -2064,7 +2064,7 @@ def test_cond_call_gc_wb(self): def func_void(a): - record.append(a) + record.append(rffi.cast(lltype.Signed, a)) record = [] # S = lltype.GcStruct('S', ('tid', lltype.Signed)) @@ -2094,13 +2094,13 @@ [BoxPtr(sgcref), ConstPtr(tgcref)], 'void', descr=WriteBarrierDescr()) if cond: - assert record == [s] + assert record == [rffi.cast(lltype.Signed, sgcref)] else: assert record == [] def test_cond_call_gc_wb_array(self): def func_void(a): - record.append(a) + record.append(rffi.cast(lltype.Signed, a)) record = [] # S = lltype.GcStruct('S', ('tid', lltype.Signed)) @@ -2129,13 +2129,13 @@ [BoxPtr(sgcref), ConstInt(123), BoxPtr(sgcref)], 'void', descr=WriteBarrierDescr()) if cond: - assert record == [s] + assert record == [rffi.cast(lltype.Signed, sgcref)] else: assert record == [] def test_cond_call_gc_wb_array_card_marking_fast_path(self): def func_void(a): - record.append(a) + record.append(rffi.cast(lltype.Signed, a)) if cond == 1: # the write barrier sets the flag s.data.tid |= 32768 record = [] @@ -2194,7 +2194,7 @@ [BoxPtr(sgcref), box_index, BoxPtr(sgcref)], 'void', descr=WriteBarrierDescr()) if cond in [0, 1]: - assert record == [s.data] + assert record == [rffi.cast(lltype.Signed, s.data)] else: assert record == [] if cond in [1, 2]: @@ -2222,7 +2222,7 @@ values.append(self.cpu.get_int_value(deadframe, 1)) self.cpu.set_savedata_ref(deadframe, random_gcref) - FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Void) + FUNC = self.FuncType([llmemory.GCREF, lltype.Signed], lltype.Void) func_ptr = llhelper(lltype.Ptr(FUNC), maybe_force) funcbox = self.get_funcbox(self.cpu, func_ptr).constbox() calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, @@ -2230,7 +2230,7 @@ cpu = self.cpu i0 = BoxInt() i1 = BoxInt() - tok = BoxInt() + tok = BoxPtr() faildescr = BasicFailDescr(1) ops = [ ResOperation(rop.FORCE_TOKEN, [], tok), @@ -2267,7 +2267,7 @@ self.cpu.set_savedata_ref(deadframe, random_gcref) return 42 - FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Signed) + FUNC = self.FuncType([llmemory.GCREF, lltype.Signed], lltype.Signed) func_ptr = llhelper(lltype.Ptr(FUNC), maybe_force) funcbox = self.get_funcbox(self.cpu, func_ptr).constbox() calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, @@ -2276,7 +2276,7 @@ i0 = BoxInt() i1 = BoxInt() i2 = BoxInt() - tok = BoxInt() + tok = BoxPtr() faildescr = BasicFailDescr(1) ops = [ ResOperation(rop.FORCE_TOKEN, [], tok), @@ -2315,7 +2315,7 @@ self.cpu.set_savedata_ref(deadframe, random_gcref) return 42.5 - FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Float) + FUNC = self.FuncType([llmemory.GCREF, lltype.Signed], lltype.Float) func_ptr = llhelper(lltype.Ptr(FUNC), maybe_force) funcbox = self.get_funcbox(self.cpu, func_ptr).constbox() calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, @@ -2324,7 +2324,7 @@ i0 = BoxInt() i1 = BoxInt() f2 = BoxFloat() - tok = BoxInt() + tok = BoxPtr() faildescr = BasicFailDescr(1) ops = [ ResOperation(rop.FORCE_TOKEN, [], tok), @@ -2789,6 +2789,7 @@ looptoken = JitCellToken() looptoken.outermost_jitdriver_sd = FakeJitDriverSD() finish_descr = loop.operations[-1].getdescr() + self.cpu.done_with_this_frame_descr_int = BasicFinalDescr() self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) ARGS = [lltype.Signed] * 10 RES = lltype.Signed @@ -2860,6 +2861,7 @@ finish_descr = loop.operations[-1].getdescr() looptoken = JitCellToken() looptoken.outermost_jitdriver_sd = FakeJitDriverSD() + self.cpu.done_with_this_frame_descr_float = BasicFinalDescr() self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) args = [longlong.getfloatstorage(1.2), longlong.getfloatstorage(2.3)] @@ -2952,6 +2954,7 @@ loop = parse(ops) looptoken = JitCellToken() looptoken.outermost_jitdriver_sd = FakeJitDriverSD() + self.cpu.done_with_this_frame_descr_float = BasicFinalDescr() self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) finish_descr = loop.operations[-1].getdescr() args = [longlong.getfloatstorage(1.25), @@ -3699,7 +3702,7 @@ values.append(self.cpu.get_int_value(deadframe, 0)) return 42 - FUNC = self.FuncType([lltype.Signed, lltype.Signed], lltype.Signed) + FUNC = self.FuncType([llmemory.GCREF, lltype.Signed], lltype.Signed) func_ptr = llhelper(lltype.Ptr(FUNC), maybe_force) funcbox = self.get_funcbox(self.cpu, func_ptr).constbox() calldescr = self.cpu.calldescrof(FUNC, FUNC.ARGS, FUNC.RESULT, @@ -3707,7 +3710,7 @@ i0 = BoxInt() i1 = BoxInt() i2 = BoxInt() - tok = BoxInt() + tok = BoxPtr() faildescr = BasicFailDescr(23) ops = [ ResOperation(rop.FORCE_TOKEN, [], tok), diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -94,7 +94,7 @@ self.float_const_neg_addr = 0 self.float_const_abs_addr = 0 self.malloc_slowpath = 0 - self.wb_slowpath = [0, 0, 0, 0] + self.wb_slowpath = [0, 0, 0, 0, 0] self.memcpy_addr = 0 self.setup_failure_recovery() self._debug = False @@ -119,6 +119,8 @@ self._build_failure_recovery(True) self._build_wb_slowpath(False) self._build_wb_slowpath(True) + self._build_wb_slowpath(False, for_frame=True) + # only one of those self._build_stack_check_failure() if self.cpu.supports_floats: self._build_failure_recovery(False, withfloats=True) @@ -201,9 +203,11 @@ base_ofs = self.cpu.get_baseofs_of_frame_field() self._push_all_regs_to_frame(mc, [], self.cpu.supports_floats) assert not IS_X86_32 + # this is the gcmap stored by push_gcmap(mov=True) in _check_stack_frame mc.MOV_rs(ecx.value, WORD) gcmap_ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') mc.MOV_br(gcmap_ofs, ecx.value) + # this is size that we're after, sanity checking only mc.MOV_rs(esi.value, WORD*2) # push first arg mc.LEA_rb(edi.value, -base_ofs) @@ -219,7 +223,7 @@ def _build_malloc_slowpath(self): """ While arriving on slowpath, we have a gcpattern on stack, - nursery_head in eax and the size in edx - eax + nursery_head in eax and the size in edi - eax """ mc = codebuf.MachineCodeBlockWrapper() self._push_all_regs_to_frame(mc, [eax, edi], self.cpu.supports_floats) @@ -240,17 +244,15 @@ mc.SUB_ri(esp.value, 16 - WORD) mc.CALL(imm(addr)) mc.ADD_ri(esp.value, 16 - WORD) - # Note: we check this after the code above, just because the code - # above is more than 127 bytes on 64-bits... - self._pop_all_regs_from_frame(mc, [eax, edi], self.cpu.supports_floats) mc.TEST_rr(eax.value, eax.value) - mc.J_il8(rx86.Conditions['Z'], 0) # patched later + mc.J_il(rx86.Conditions['Z'], 0xfffff) # patched later jz_location = mc.get_relative_pos() # nursery_free_adr = self.cpu.gc_ll_descr.get_nursery_free_addr() - mc.MOV(edi, heap(nursery_free_adr)) # load this in EDX + self._reload_frame_if_necessary(mc) + self._pop_all_regs_from_frame(mc, [eax, edi], self.cpu.supports_floats) + mc.MOV(edi, heap(nursery_free_adr)) # load this in EDI # clear the gc pattern - self._reload_frame_if_necessary(mc) mc.MOV_bi(ofs, 0) mc.RET() # @@ -260,8 +262,7 @@ # but the code we jump to will actually restore the stack # position based on EBP, which will get us out of here for free. offset = mc.get_relative_pos() - jz_location - assert 0 < offset <= 127 - mc.overwrite(jz_location-1, chr(offset)) + mc.overwrite32(jz_location-4, offset) mc.JMP(imm(self.propagate_exception_path)) # rawstart = mc.materialize(self.cpu.asmmemmgr, []) @@ -364,7 +365,7 @@ rawstart = mc.materialize(self.cpu.asmmemmgr, []) self.stack_check_slowpath = rawstart - def _build_wb_slowpath(self, withcards, withfloats=False): + def _build_wb_slowpath(self, withcards, withfloats=False, for_frame=False): descr = self.cpu.gc_ll_descr.write_barrier_descr if descr is None: return @@ -384,61 +385,50 @@ # accordingly. mc = codebuf.MachineCodeBlockWrapper() # - frame_size = (1 + # my argument, considered part of my frame - 1 + # my return address - len(gpr_reg_mgr_cls.save_around_call_regs)) - if withfloats: - frame_size += 16 # X86_32: 16 words for 8 registers; - # X86_64: just 16 registers - if IS_X86_32: - frame_size += 1 # argument to pass to the call - # - # align to a multiple of 16 bytes - frame_size = (frame_size + (CALL_ALIGN-1)) & ~(CALL_ALIGN-1) - # - correct_esp_by = (frame_size - 2) * WORD - mc.SUB_ri(esp.value, correct_esp_by) - # - ofs = correct_esp_by - if withfloats: - for reg in xmm_reg_mgr_cls.save_around_call_regs: - ofs -= 8 - mc.MOVSD_sx(ofs, reg.value) - for reg in gpr_reg_mgr_cls.save_around_call_regs: - ofs -= WORD - mc.MOV_sr(ofs, reg.value) - # - if IS_X86_32: - mc.MOV_rs(eax.value, (frame_size - 1) * WORD) - mc.MOV_sr(0, eax.value) - elif IS_X86_64: - mc.MOV_rs(edi.value, (frame_size - 1) * WORD) + if not for_frame: + self._push_all_regs_to_frame(mc, [], withfloats, callee_only=True) + if IS_X86_32: + XXX + mc.MOV_rs(eax.value, WORD) + mc.MOV_sr(0, eax.value) + elif IS_X86_64: + mc.MOV_rs(edi.value, WORD) + else: + # we're possibly called from the slowpath of malloc, so we have + # one extra CALL on the stack, but one less PUSH, + # save to store stuff 2 locations away on the stack. + mc.MOV_sr(3*WORD, eax.value) + base_ofs = self.cpu.get_baseofs_of_frame_field() + mc.LEA_rb(edi.value, -base_ofs) + mc.CALL(imm(func)) # if withcards: # A final TEST8 before the RET, for the caller. Careful to # not follow this instruction with another one that changes # the status of the CPU flags! - mc.MOV_rs(eax.value, (frame_size - 1) * WORD) + mc.MOV_rs(eax.value, WORD) mc.TEST8(addr_add_const(eax, descr.jit_wb_if_flag_byteofs), imm(-0x80)) # - ofs = correct_esp_by - if withfloats: - for reg in xmm_reg_mgr_cls.save_around_call_regs: - ofs -= 8 - mc.MOVSD_xs(reg.value, ofs) - for reg in gpr_reg_mgr_cls.save_around_call_regs: - ofs -= WORD - mc.MOV_rs(reg.value, ofs) + + if not for_frame: + self._pop_all_regs_from_frame(mc, [], withfloats, callee_only=True) + mc.RET16_i(WORD) + else: + mc.MOV_rs(eax.value, 3 * WORD) + mc.RET() # # ADD esp, correct_esp_by --- but cannot use ADD, because # of its effects on the CPU flags - mc.LEA_rs(esp.value, correct_esp_by) - mc.RET16_i(WORD) + + #mc.LEA_rs(esp.value, WORD) # rawstart = mc.materialize(self.cpu.asmmemmgr, []) - self.wb_slowpath[withcards + 2 * withfloats] = rawstart + if for_frame: + self.wb_slowpath[4] = rawstart + else: + self.wb_slowpath[withcards + 2 * withfloats] = rawstart @staticmethod @rgc.no_collect @@ -517,7 +507,7 @@ clt = CompiledLoopToken(self.cpu, looptoken.number) clt.frame_info = lltype.malloc(jitframe.JITFRAMEINFO) clt.allgcrefs = [] - clt.frame_info.jfi_frame_depth = 0 # for now + clt.frame_info.set_frame_depth(0, 0) # for now looptoken.compiled_loop_token = clt clt._debug_nbargs = len(inputargs) if not we_are_translated(): @@ -537,7 +527,6 @@ rgc._make_sure_does_not_move(lltype.cast_opaque_ptr(llmemory.GCREF, clt.frame_info)) looppos = self.mc.get_relative_pos() - looptoken._x86_loop_code = looppos frame_depth = self._assemble(regalloc, inputargs, operations) self.update_frame_depth(frame_depth + JITFRAME_FIXED_SIZE) # @@ -546,6 +535,7 @@ full_size = self.mc.get_relative_pos() # rawstart = self.materialize_loop(looptoken) + looptoken._x86_loop_code = looppos + rawstart debug_start("jit-backend-addr") debug_print("Loop %d (%s) has address %x to %x (bootstrap %x)" % ( looptoken.number, loopname, @@ -676,13 +666,14 @@ mc.copy_to_raw_memory(rawstart + pos_after_jz - 4) def update_frame_depth(self, frame_depth): - self.current_clt.frame_info.jfi_frame_depth = frame_depth + baseofs = self.cpu.get_baseofs_of_frame_field() + self.current_clt.frame_info.set_frame_depth(baseofs, frame_depth) new_jumping_to = [] for wref in self.current_clt.jumping_to: clt = wref() if clt is not None: - clt.frame_info.jfi_frame_depth = max(frame_depth, - clt.frame_info.jfi_frame_depth) + clt.frame_info.set_frame_depth(baseofs, max(frame_depth, + clt.frame_info.jfi_frame_depth)) new_jumping_to.append(weakref.ref(clt)) self.current_clt.jumping_to = new_jumping_to @@ -866,6 +857,10 @@ self.mc.RET() def _call_header_shadowstack(self, gcrootmap): + # we don't *really* have to do it, since we have the frame + # being referenced by the caller. However, we still do it + # to provide a place where we can read the frame from, in case + # we need to reload it after a collection rst = gcrootmap.get_root_stack_top_addr() if rx86.fits_in_32bits(rst): self.mc.MOV_rj(eax.value, rst) # MOV eax, [rootstacktop] @@ -904,7 +899,8 @@ # copy frame-info data old_fi = oldlooptoken.compiled_loop_token.frame_info new_fi = newlooptoken.compiled_loop_token.frame_info - old_fi.jfi_frame_depth = new_fi.jfi_frame_depth + baseofs = self.cpu.get_baseofs_of_frame_field() + old_fi.set_frame_depth(baseofs, new_fi.jfi_frame_depth) From noreply at buildbot.pypy.org Mon Jan 28 17:16:59 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 28 Jan 2013 17:16:59 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: kill jit2gc['root_iterator'] Message-ID: <20130128161659.066391C0041@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60618:38ad7fc1383e Date: 2013-01-28 18:14 +0200 http://bitbucket.org/pypy/pypy/changeset/38ad7fc1383e/ Log: kill jit2gc['root_iterator'] diff --git a/rpython/rtyper/memory/gctransform/shadowstack.py b/rpython/rtyper/memory/gctransform/shadowstack.py --- a/rpython/rtyper/memory/gctransform/shadowstack.py +++ b/rpython/rtyper/memory/gctransform/shadowstack.py @@ -268,12 +268,6 @@ self.gc_start_fresh_new_state_ptr = getfn(gc_start_fresh_new_state, [], annmodel.s_None, inline=True) - # fish... - translator = gctransformer.translator - if hasattr(translator, '_jit2gc'): - from rpython.rlib._rffi_stacklet import _translate_pointer - root_iterator = translator._jit2gc['root_iterator'] - root_iterator.translateptr = _translate_pointer # ____________________________________________________________ @@ -366,23 +360,18 @@ def get_root_iterator(gctransformer): if hasattr(gctransformer, '_root_iterator'): return gctransformer._root_iterator # if already built - translator = gctransformer.translator - if (hasattr(translator, '_jit2gc') and - 'root_iterator' in translator._jit2gc): - result = translator._jit2gc['root_iterator'] - else: - class RootIterator(object): - def _freeze_(self): - return True - def setcontext(self, context): - pass - def nextleft(self, gc, start, addr): - while addr != start: - addr -= sizeofaddr - if gc.points_to_valid_gc_object(addr): - return addr - return llmemory.NULL - result = RootIterator() + class RootIterator(object): + def _freeze_(self): + return True + def setcontext(self, context): + pass + def nextleft(self, gc, start, addr): + while addr != start: + addr -= sizeofaddr + if gc.points_to_valid_gc_object(addr): + return addr + return llmemory.NULL + result = RootIterator() gctransformer._root_iterator = result return result From noreply at buildbot.pypy.org Mon Jan 28 17:17:00 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 28 Jan 2013 17:17:00 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: merge Message-ID: <20130128161700.58EC41C0041@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60619:aef40e6e1dc8 Date: 2013-01-28 18:16 +0200 http://bitbucket.org/pypy/pypy/changeset/aef40e6e1dc8/ Log: merge diff --git a/rpython/jit/backend/arm/arch.py b/rpython/jit/backend/arm/arch.py --- a/rpython/jit/backend/arm/arch.py +++ b/rpython/jit/backend/arm/arch.py @@ -1,7 +1,3 @@ -from rpython.rtyper.lltypesystem import lltype, rffi -from rpython.rlib.rarithmetic import r_uint - - FUNC_ALIGN = 8 WORD = 4 DOUBLE_WORD = 8 @@ -14,54 +10,13 @@ PC_OFFSET = 8 FORCE_INDEX_OFS = 0 -from rpython.translator.tool.cbuild import ExternalCompilationInfo -eci = ExternalCompilationInfo(post_include_bits=[""" -static int pypy__arm_int_div(int a, int b) { - return a/b; -} -static unsigned int pypy__arm_uint_div(unsigned int a, unsigned int b) { - return a/b; -} -static int pypy__arm_int_mod(int a, int b) { - return a % b; -} -"""]) +# The stack contains the force_index and the, callee saved registers and +# ABI required information +# All the rest of the data is in a GC-managed variable-size "frame". +# This jitframe object's address is always stored in the register FP +# A jitframe is a jit.backend.llsupport.llmodel.JITFRAME = GcArray(Signed). +# Stack frame fixed area +# Currently only the force_index +FRAME_FIXED_SIZE = 1 +JITFRAME_FIXED_SIZE = 16 + 16 * 2 # 16 GPR + 16 VFP Regs (64bit) - -def arm_int_div_emulator(a, b): - return int(a / float(b)) -arm_int_div_sign = lltype.Ptr( - lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed)) -arm_int_div = rffi.llexternal( - "pypy__arm_int_div", [lltype.Signed, lltype.Signed], lltype.Signed, - _callable=arm_int_div_emulator, - compilation_info=eci, - _nowrapper=True, elidable_function=True) - - -def arm_uint_div_emulator(a, b): - return r_uint(a) / r_uint(b) -arm_uint_div_sign = lltype.Ptr( - lltype.FuncType([lltype.Unsigned, lltype.Unsigned], lltype.Unsigned)) -arm_uint_div = rffi.llexternal( - "pypy__arm_uint_div", [lltype.Unsigned, lltype.Unsigned], lltype.Unsigned, - _callable=arm_uint_div_emulator, - compilation_info=eci, - _nowrapper=True, elidable_function=True) - - -def arm_int_mod_emulator(a, b): - sign = 1 - if a < 0: - a = -1 * a - sign = -1 - if b < 0: - b = -1 * b - res = a % b - return sign * res -arm_int_mod_sign = arm_int_div_sign -arm_int_mod = rffi.llexternal( - "pypy__arm_int_mod", [lltype.Signed, lltype.Signed], lltype.Signed, - _callable=arm_int_mod_emulator, - compilation_info=eci, - _nowrapper=True, elidable_function=True) diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py --- a/rpython/jit/backend/arm/assembler.py +++ b/rpython/jit/backend/arm/assembler.py @@ -5,7 +5,8 @@ from rpython.jit.backend.arm import conditions as c from rpython.jit.backend.arm import registers as r from rpython.jit.backend.arm.arch import WORD, DOUBLE_WORD, FUNC_ALIGN, \ - N_REGISTERS_SAVED_BY_MALLOC + N_REGISTERS_SAVED_BY_MALLOC, \ + JITFRAME_FIXED_SIZE, FRAME_FIXED_SIZE from rpython.jit.backend.arm.codebuilder import ARMv7Builder, OverwritingBuilder from rpython.jit.backend.arm.locations import get_fp_offset from rpython.jit.backend.arm.regalloc import (Regalloc, ARMFrameManager, @@ -21,7 +22,7 @@ from rpython.jit.metainterp.resoperation import rop, ResOperation from rpython.rlib import rgc from rpython.rlib.objectmodel import we_are_translated, specialize -from rpython.rtyper.annlowlevel import llhelper +from rpython.rtyper.annlowlevel import llhelper, cast_instance_to_gcref from rpython.rtyper.lltypesystem import lltype, rffi, llmemory from rpython.rtyper.lltypesystem.lloperation import llop from rpython.jit.backend.arm.opassembler import ResOpAssembler @@ -29,9 +30,9 @@ have_debug_prints, fatalerror) from rpython.rlib.jit import AsmInfo from rpython.rlib.objectmodel import compute_unique_id +from rpython.rlib.rarithmetic import intmask, r_uint -# XXX Move to llsupport -from rpython.jit.backend.x86.support import memcpy_fn +from rpython.jit.backend.arm.support import memcpy_fn DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', ('i', lltype.Signed), ('type', lltype.Char), # 'b'ridge, 'l'abel or @@ -41,8 +42,6 @@ class AssemblerARM(ResOpAssembler): - STACK_FIXED_AREA = -1 - debug = True def __init__(self, cpu, translate_support_code=False): @@ -59,33 +58,19 @@ self.datablockwrapper = None self.propagate_exception_path = 0 self.stack_check_slowpath = 0 - self._compute_stack_size() self._debug = False self.loop_run_counters = [] self.debug_counter_descr = cpu.fielddescrof(DEBUG_COUNTER, 'i') - self.force_token_to_dead_frame = {} # XXX temporary hack + self.gcrootmap_retaddr_forced = 0 def set_debug(self, v): r = self._debug self._debug = v return r - def _compute_stack_size(self): - self.STACK_FIXED_AREA = len(r.callee_saved_registers) * WORD - self.STACK_FIXED_AREA += WORD # FORCE_TOKEN - self.STACK_FIXED_AREA += N_REGISTERS_SAVED_BY_MALLOC * WORD - if self.cpu.supports_floats: - self.STACK_FIXED_AREA += (len(r.callee_saved_vfp_registers) - * DOUBLE_WORD) - if self.STACK_FIXED_AREA % 8 != 0: - self.STACK_FIXED_AREA += WORD # Stack alignment - assert self.STACK_FIXED_AREA % 8 == 0 - - def setup(self, looptoken, operations): + def setup(self, looptoken): + assert self.memcpy_addr != 0, 'setup_once() not called?' self.current_clt = looptoken.compiled_loop_token - operations = self.cpu.gc_ll_descr.rewrite_assembler(self.cpu, - operations, self.current_clt.allgcrefs) - assert self.memcpy_addr != 0, 'setup_once() not called?' self.mc = ARMv7Builder() self.pending_guards = [] assert self.datablockwrapper is None @@ -93,7 +78,6 @@ self.datablockwrapper = MachineDataBlockWrapper(self.cpu.asmmemmgr, allblocks) self.target_tokens_currently_compiling = {} - return operations def teardown(self): self.current_clt = None @@ -106,10 +90,11 @@ # Addresses of functions called by new_xxx operations gc_ll_descr = self.cpu.gc_ll_descr gc_ll_descr.initialize() + self.memcpy_addr = self.cpu.cast_ptr_to_int(memcpy_fn) + self._build_failure_recovery(exc=True, withfloats=False) + self._build_failure_recovery(exc=False, withfloats=False) self._build_wb_slowpath(False) self._build_wb_slowpath(True) - self._build_failure_recovery(exc=True, withfloats=False) - self._build_failure_recovery(exc=False, withfloats=False) if self.cpu.supports_floats: self._build_wb_slowpath(False, withfloats=True) self._build_wb_slowpath(True, withfloats=True) @@ -121,7 +106,6 @@ self._build_stack_check_slowpath() if gc_ll_descr.gcrootmap and gc_ll_descr.gcrootmap.is_shadow_stack: self._build_release_gil(gc_ll_descr.gcrootmap) - self.memcpy_addr = self.cpu.cast_ptr_to_int(memcpy_fn) if not self._debug: # if self._debug is already set it means that someone called @@ -130,6 +114,9 @@ debug_start('jit-backend-counts') self.set_debug(have_debug_prints()) debug_stop('jit-backend-counts') + # when finishing, we only have one value at [0], the rest dies + self.gcmap_for_finish = lltype.malloc(jitframe.GCMAP, 1, zero=True) + self.gcmap_for_finish[0] = r_uint(1) def finish_once(self): if self._debug: @@ -218,18 +205,50 @@ self.reacqgil_addr = rffi.cast(lltype.Signed, reacqgil_func) def _build_propagate_exception_path(self): - if self.cpu.propagate_exception_v < 0: + if not self.cpu.propagate_exception_descr: return # not supported (for tests, or non-translated) # mc = ARMv7Builder() # - # Call the helper, which will return a dead frame object with - # the correct exception set, or MemoryError by default - # XXX make sure we return the correct value here + # read and reset the current exception addr = rffi.cast(lltype.Signed, self.cpu.get_propagate_exception()) mc.BL(addr) self.gen_func_epilog(mc=mc) self.propagate_exception_path = mc.materialize(self.cpu.asmmemmgr, []) + # + self._store_and_reset_exception(r.r0) + ofs = self.cpu.get_ofs_of_frame_field('jf_guard_exc') + # make sure ofs fits into a register + assert check_imm_arg(ofs) + self.mc.STR_ri(r.r0.value, r.fp.value, imm=ofs) + propagate_exception_descr = rffi.cast(lltype.Signed, + cast_instance_to_gcref(self.cpu.propagate_exception_descr)) + ofs = self.cpu.get_ofs_of_frame_field('jf_descr') + # make sure ofs fits into a register + assert check_imm_arg(ofs) + self.mc.BKPT() + #base_ofs = self.cpu.get_baseofs_of_frame_field() + #self.mc.MOV_bi(ofs, propagate_exception_descr) + #self.mc.LEA_rb(eax.value, -base_ofs) + # + self._call_footer() + rawstart = self.mc.materialize(self.cpu.asmmemmgr, []) + self.propagate_exception_path = rawstart + self.mc = None + + def _store_and_reset_exception(self, resloc=None): + assert resloc is not r.ip + if resloc is not None: + self.mc.gen_load_int(resloc.value, self.cpu.pos_exc_value()) + self.mc.LDR_ri(resloc.value, resloc.value) + self.mc.MOV(resloc, heap(self.cpu.pos_exc_value())) + + with saved_registers(self.mc, [r.r0]): + self.mc.gen_load_int(r.r0.value, self.cpu.pos_exc_value()) + self.mc.gen_load_int(r.ip.value, 0) + self.mc.STR_ri(r.ip.value, r.r0.value) + self.mc.gen_load_int(r.r0.value, self.cpu.pos_exception()) + self.mc.STR_ri(r.ip.value, r.r0.value) def _build_stack_check_slowpath(self): _, _, slowpathaddr = self.cpu.insert_stack_check() @@ -558,7 +577,7 @@ # We might have an exception pending. Load it into r4 # (this is a register saved across calls) mc.gen_load_int(r.r5.value, self.cpu.pos_exc_value()) - mc.LDR_ri(r.r4.value, self.cpu.pos_exc_value()) + mc.LDR_ri(r.r4.value, r.r5.value) # clear the exc flags mc.gen_load_int(r.r6.value, 0) mc.STR_ri(r.r6.value, r.r5.value) @@ -661,37 +680,35 @@ self.mc.writechar(chr(0)) def gen_func_epilog(self, mc=None, cond=c.AL): - stack_size = self.STACK_FIXED_AREA - stack_size -= len(r.callee_saved_registers) * WORD - if self.cpu.supports_floats: - stack_size -= len(r.callee_saved_vfp_registers) * 2 * WORD - gcrootmap = self.cpu.gc_ll_descr.gcrootmap if mc is None: mc = self.mc if gcrootmap and gcrootmap.is_shadow_stack: self.gen_footer_shadowstack(gcrootmap, mc) - mc.MOV_rr(r.sp.value, r.fp.value, cond=cond) - mc.ADD_ri(r.sp.value, r.sp.value, stack_size, cond=cond) + mc.ADD_ri(r.sp.value, r.sp.value, WORD, cond=cond) # for the force index if self.cpu.supports_floats: mc.VPOP([reg.value for reg in r.callee_saved_vfp_registers], cond=cond) mc.POP([reg.value for reg in r.callee_restored_registers], cond=cond) + mc.BKPT() def gen_func_prolog(self): - stack_size = self.STACK_FIXED_AREA - stack_size -= len(r.callee_saved_registers) * WORD + stack_size = FRAME_FIXED_SIZE * WORD + stack_size += len(r.callee_saved_registers) * WORD if self.cpu.supports_floats: - stack_size -= len(r.callee_saved_vfp_registers) * 2 * WORD + stack_size += len(r.callee_saved_vfp_registers) * 2 * WORD self.mc.PUSH([reg.value for reg in r.callee_saved_registers]) if self.cpu.supports_floats: self.mc.VPUSH([reg.value for reg in r.callee_saved_vfp_registers]) - # here we modify the stack pointer to leave room for the 9 registers - # that are going to be saved here around malloc calls and one word to - # store the force index - self.mc.SUB_ri(r.sp.value, r.sp.value, stack_size) - self.mc.MOV_rr(r.fp.value, r.sp.value) + self.mc.SUB_ri(r.sp.value, r.sp.value, WORD) # for the force index + assert stack_size % 8 == 0 # ensure we keep alignment + + # set fp to point to the JITFRAME + ofs + ofs = self.cpu.get_baseofs_of_frame_field() + assert check_imm_arg(ofs) + self.mc.ADD_ri(r.fp.value, r.r0.value, imm=ofs) + # gcrootmap = self.cpu.gc_ll_descr.gcrootmap if gcrootmap and gcrootmap.is_shadow_stack: self.gen_shadowstack_header(gcrootmap) @@ -754,7 +771,9 @@ # cpu interface def assemble_loop(self, loopname, inputargs, operations, looptoken, log): clt = CompiledLoopToken(self.cpu, looptoken.number) + clt.frame_info = lltype.malloc(jitframe.JITFRAMEINFO) clt.allgcrefs = [] + clt.frame_info.jfi_frame_depth = 0 # for now looptoken.compiled_loop_token = clt clt._debug_nbargs = len(inputargs) @@ -762,38 +781,40 @@ # Arguments should be unique assert len(set(inputargs)) == len(inputargs) - operations = self.setup(looptoken, operations) - if log: + self.setup(looptoken) + if False and log: operations = self._inject_debugging_code(looptoken, operations, 'e', looptoken.number) self._call_header_with_stack_check() - sp_patch_location = self._prepare_sp_patch_position() + #sp_patch_location = self._prepare_sp_patch_position() - regalloc = Regalloc(assembler=self, frame_manager=ARMFrameManager()) - regalloc.prepare_loop(inputargs, operations) + regalloc = Regalloc(assembler=self) + operations = regalloc.prepare_loop(inputargs, operations, looptoken, + clt.allgcrefs) + rgc._make_sure_does_not_move(lltype.cast_opaque_ptr(llmemory.GCREF, + clt.frame_info)) loop_head = self.mc.get_relative_pos() looptoken._arm_loop_code = loop_head # - clt.frame_depth = -1 - frame_depth = self._assemble(operations, regalloc) - clt.frame_depth = frame_depth + frame_depth = self._assemble(regalloc, inputargs, operations) + self.update_frame_depth(frame_depth + JITFRAME_FIXED_SIZE) # size_excluding_failure_stuff = self.mc.get_relative_pos() - self._patch_sp_offset(sp_patch_location, frame_depth) + #self._patch_sp_offset(sp_patch_location, frame_depth) self.write_pending_failure_recoveries() rawstart = self.materialize_loop(looptoken) - looptoken._arm_func_addr = rawstart + looptoken._function_addr = looptoken._arm_func_addr = rawstart self.process_pending_guards(rawstart) self.fixup_target_tokens(rawstart) if log and not we_are_translated(): self.mc._dump_trace(rawstart, - 'loop_%s.asm' % self.cpu.total_compiled_loops) + 'loop.asm') ops_offset = self.mc.ops_offset self.teardown() @@ -809,18 +830,20 @@ return AsmInfo(ops_offset, rawstart + loop_head, size_excluding_failure_stuff - loop_head) - def _assemble(self, operations, regalloc): + def _assemble(self, regalloc, inputargs, operations): regalloc.compute_hint_frame_locations(operations) - self._walk_operations(operations, regalloc) - frame_depth = regalloc.frame_manager.get_frame_depth() + self._walk_operations(inputargs, operations, regalloc) + frame_depth = regalloc.get_final_frame_depth() jump_target_descr = regalloc.jump_target_descr if jump_target_descr is not None: - frame_depth = max(frame_depth, - jump_target_descr._arm_clt.frame_depth) + tgt_depth = jump_target_descr._arm_clt.frame_info.jfi_frame_depth + target_frame_depth = tgt_depth - JITFRAME_FIXED_SIZE + frame_depth = max(frame_depth, target_frame_depth) return frame_depth def assemble_bridge(self, faildescr, inputargs, operations, original_loop_token, log): + assert 0 operations = self.setup(original_loop_token, operations) descr_number = self.cpu.get_fail_descr_number(faildescr) if log: @@ -899,6 +922,17 @@ return self.mc.materialize(self.cpu.asmmemmgr, allblocks, self.cpu.gc_ll_descr.gcrootmap) + def update_frame_depth(self, frame_depth): + self.current_clt.frame_info.jfi_frame_depth = frame_depth + new_jumping_to = [] + for wref in self.current_clt.jumping_to: + clt = wref() + if clt is not None: + clt.frame_info.jfi_frame_depth = max(frame_depth, + clt.frame_info.jfi_frame_depth) + new_jumping_to.append(weakref.ref(clt)) + self.current_clt.jumping_to = new_jumping_to + def write_pending_failure_recoveries(self): for tok in self.pending_guards: #generate the exit stub and the encoded representation @@ -972,7 +1006,7 @@ else: cb.SUB_rr(r.sp.value, base_reg.value, r.ip.value, cond=fcond) - def _walk_operations(self, operations, regalloc): + def _walk_operations(self, inputargs, operations, regalloc): fcond = c.AL self._regalloc = regalloc while regalloc.position() < len(operations) - 1: @@ -1141,10 +1175,10 @@ if not check_imm_arg(offset, size=0xFFF): self.mc.PUSH([r.lr.value], cond=cond) pushed = True - self.mc.gen_load_int(r.lr.value, -offset, cond=cond) + self.mc.gen_load_int(r.lr.value, offset, cond=cond) self.mc.LDR_rr(loc.value, r.fp.value, r.lr.value, cond=cond) else: - self.mc.LDR_ri(loc.value, r.fp.value, imm=-offset, cond=cond) + self.mc.LDR_ri(loc.value, r.fp.value, imm=offset, cond=cond) if pushed: self.mc.POP([r.lr.value], cond=cond) elif loc.is_vfp_reg(): @@ -1364,6 +1398,26 @@ else: return 0 + def push_gcmap(self, mc, gcmap, push=False, mov=False, store=False): + gcmapref = lltype.cast_opaque_ptr(llmemory.GCREF, gcmap) + # keep the ref alive + self.current_clt.allgcrefs.append(gcmapref) + rgc._make_sure_does_not_move(gcmapref) + pass + #if push: + # mc.PUSH(imm(rffi.cast(lltype.Signed, gcmapref))) + #elif mov: + # mc.MOV(RawEspLoc(0, REF), + # imm(rffi.cast(lltype.Signed, gcmapref))) + #else: + # assert store + # ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') + # mc.MOV(raw_stack(ofs), imm(rffi.cast(lltype.Signed, gcmapref))) + + def pop_gcmap(self, mc): + ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') + mc.MOV_bi(ofs, 0) + def not_implemented(msg): os.write(2, '[ARM/asm] %s\n' % msg) diff --git a/rpython/jit/backend/arm/codebuilder.py b/rpython/jit/backend/arm/codebuilder.py --- a/rpython/jit/backend/arm/codebuilder.py +++ b/rpython/jit/backend/arm/codebuilder.py @@ -1,6 +1,6 @@ -from rpython.jit.backend.arm import arch from rpython.jit.backend.arm import conditions as cond from rpython.jit.backend.arm import registers as reg +from rpython.jit.backend.arm import support from rpython.jit.backend.arm.arch import (WORD, FUNC_ALIGN) from rpython.jit.backend.arm.instruction_builder import define_instructions from rpython.jit.backend.llsupport.asmmemmgr import BlockBuilderMixin @@ -17,7 +17,7 @@ def binary_helper_call(name): - function = getattr(arch, 'arm_%s' % name) + function = getattr(support, 'arm_%s' % name) def f(self, c=cond.AL): """Generates a call to a helper function, takes its diff --git a/rpython/jit/backend/arm/locations.py b/rpython/jit/backend/arm/locations.py --- a/rpython/jit/backend/arm/locations.py +++ b/rpython/jit/backend/arm/locations.py @@ -1,5 +1,5 @@ from rpython.jit.metainterp.history import INT, FLOAT -from rpython.jit.backend.arm.arch import WORD, DOUBLE_WORD +from rpython.jit.backend.arm.arch import WORD, DOUBLE_WORD, JITFRAME_FIXED_SIZE class AssemblerLocation(object): @@ -136,9 +136,5 @@ return ImmLocation(i) -def get_fp_offset(i): - if i >= 0: - # Take the FORCE_TOKEN into account - return (1 + i) * WORD - else: - return i * WORD +def get_fp_offset(position): + return WORD * (position + JITFRAME_FIXED_SIZE) diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -317,9 +317,27 @@ return fcond def emit_op_finish(self, op, arglocs, regalloc, fcond): - [argloc] = arglocs - if argloc is not r.r0: #XXX verify this - self.mov_loc_loc(argloc, r.r0, fcond) + base_ofs = self.cpu.get_baseofs_of_frame_field() - WORD + if len(arglocs) == 2: + [return_val, fail_descr_loc] = arglocs + if op.getarg(0).type == FLOAT and not IS_X86_64: + XXX + size = WORD * 2 + else: + size = WORD + self.mc.STR_ri(return_val.value, r.fp.value)#, imm=-base_ofs) + #self.save_into_mem(raw_stack(0), return_val, imm(size)) + else: + [fail_descr_loc] = arglocs + ofs = self.cpu.get_ofs_of_frame_field('jf_descr') + base_ofs = self.cpu.get_baseofs_of_frame_field() + + self.mc.gen_load_int(r.ip.value, fail_descr_loc.value) + # XXX self.mov(fail_descr_loc, RawStackLoc(ofs)) + self.mc.STR_ri(r.ip.value, r.fp.value, imm=ofs) + gcmap = self.gcmap_for_finish + self.push_gcmap(self.mc, gcmap, store=True) + self.mc.SUB_ri(r.r0.value, r.fp.value, base_ofs) # exit function self.gen_func_epilog() return fcond diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py --- a/rpython/jit/backend/arm/regalloc.py +++ b/rpython/jit/backend/arm/regalloc.py @@ -1,3 +1,5 @@ +from rpython.rtyper.annlowlevel import llhelper, cast_instance_to_gcref +from rpython.rlib import rgc from rpython.jit.backend.llsupport.regalloc import FrameManager, \ RegisterManager, TempBox, compute_vars_longevity from rpython.jit.backend.arm import registers as r @@ -180,10 +182,10 @@ class Regalloc(object): - def __init__(self, frame_manager=None, assembler=None): + def __init__(self, assembler=None): self.cpu = assembler.cpu self.assembler = assembler - self.frame_manager = frame_manager + self.frame_manager = None self.jump_target_descr = None self.final_jump_op = None @@ -282,7 +284,12 @@ assert isinstance(value, ConstFloat) return self.vfprm.convert_to_imm(value) - def _prepare(self, inputargs, operations): + def _prepare(self, inputargs, operations, allgcrefs): + self.frame_manager = self.fm = ARMFrameManager() + cpu = self.assembler.cpu + operations = cpu.gc_ll_descr.rewrite_assembler(cpu, operations, + allgcrefs) + # compute longevity of variables longevity, last_real_usage = compute_vars_longevity( inputargs, operations) self.longevity = longevity @@ -291,92 +298,27 @@ asm = self.assembler self.vfprm = VFPRegisterManager(longevity, fm, asm) self.rm = CoreRegisterManager(longevity, fm, asm) + return operations - def prepare_loop(self, inputargs, operations): - self._prepare(inputargs, operations) + def prepare_loop(self, inputargs, operations, looptoken, allgcrefs): + operations = self._prepare(inputargs, operations, allgcrefs) self._set_initial_bindings(inputargs) - self.possibly_free_vars(inputargs) + self.possibly_free_vars(list(inputargs)) + return operations def prepare_bridge(self, inputargs, arglocs, ops): self._prepare(inputargs, ops) self._update_bindings(arglocs, inputargs) + def get_final_frame_depth(self): + return self.frame_manager.get_frame_depth() + def _set_initial_bindings(self, inputargs): - # The first inputargs are passed in registers r0-r3 - # we relly on the soft-float calling convention so we need to move - # float params to the coprocessor. - if self.cpu.use_hf_abi: - self._set_initial_bindings_hf(inputargs) - else: - self._set_initial_bindings_sf(inputargs) - - def _set_initial_bindings_sf(self, inputargs): - - arg_index = 0 - count = 0 - n_register_args = len(r.argument_regs) - cur_frame_pos = 1 - (self.assembler.STACK_FIXED_AREA // WORD) + # the input args are passed in the jitframe for box in inputargs: assert isinstance(box, Box) - # handle inputargs in argument registers - if box.type == FLOAT and arg_index % 2 != 0: - arg_index += 1 # align argument index for float passed - # in register - if arg_index < n_register_args: - if box.type == FLOAT: - loc = r.argument_regs[arg_index] - loc2 = r.argument_regs[arg_index + 1] - vfpreg = self.try_allocate_reg(box) - # move soft-float argument to vfp - self.assembler.mov_to_vfp_loc(loc, loc2, vfpreg) - arg_index += 2 # this argument used two argument registers - else: - loc = r.argument_regs[arg_index] - self.try_allocate_reg(box, selected_reg=loc) - arg_index += 1 - else: - # treat stack args as stack locations with a negative offset - if box.type == FLOAT: - cur_frame_pos -= 2 - if count % 2 != 0: # Stack argument alignment - cur_frame_pos -= 1 - count = 0 - else: - cur_frame_pos -= 1 - count += 1 - loc = self.frame_manager.frame_pos(cur_frame_pos, box.type) - self.frame_manager.set_binding(box, loc) - - def _set_initial_bindings_hf(self, inputargs): - - arg_index = vfp_arg_index = 0 - count = 0 - n_reg_args = len(r.argument_regs) - n_vfp_reg_args = len(r.vfp_argument_regs) - cur_frame_pos = 1 - (self.assembler.STACK_FIXED_AREA // WORD) - for box in inputargs: - assert isinstance(box, Box) - # handle inputargs in argument registers - if box.type != FLOAT and arg_index < n_reg_args: - reg = r.argument_regs[arg_index] - self.try_allocate_reg(box, selected_reg=reg) - arg_index += 1 - elif box.type == FLOAT and vfp_arg_index < n_vfp_reg_args: - reg = r.vfp_argument_regs[vfp_arg_index] - self.try_allocate_reg(box, selected_reg=reg) - vfp_arg_index += 1 - else: - # treat stack args as stack locations with a negative offset - if box.type == FLOAT: - cur_frame_pos -= 2 - if count % 2 != 0: # Stack argument alignment - cur_frame_pos -= 1 - count = 0 - else: - cur_frame_pos -= 1 - count += 1 - loc = self.frame_manager.frame_pos(cur_frame_pos, box.type) - self.frame_manager.set_binding(box, loc) + assert box.type != FLOAT + self.fm.get_new_loc(box) def _update_bindings(self, locs, inputargs): used = {} @@ -644,9 +586,19 @@ return args def prepare_op_finish(self, op, fcond): - loc = self.loc(op.getarg(0)) - self.possibly_free_var(op.getarg(0)) - return [loc] + # the frame is in fp, but we have to point where in the frame is + # the potential argument to FINISH + descr = op.getdescr() + fail_descr = cast_instance_to_gcref(descr) + # we know it does not move, but well + rgc._make_sure_does_not_move(fail_descr) + fail_descr = rffi.cast(lltype.Signed, fail_descr) + if op.numargs() == 1: + loc = self.make_sure_var_in_reg(op.getarg(0)) + locs = [loc, imm(fail_descr)] + else: + locs = [imm(fail_descr)] + return locs def prepare_op_guard_true(self, op, fcond): l0 = self.make_sure_var_in_reg(op.getarg(0)) diff --git a/rpython/jit/backend/arm/runner.py b/rpython/jit/backend/arm/runner.py --- a/rpython/jit/backend/arm/runner.py +++ b/rpython/jit/backend/arm/runner.py @@ -1,12 +1,18 @@ +from rpython.jit.backend.arm.arch import JITFRAME_FIXED_SIZE from rpython.jit.backend.arm.assembler import AssemblerARM from rpython.jit.backend.arm.registers import all_regs, all_vfp_regs +from rpython.jit.backend.llsupport import jitframe +from rpython.jit.backend.llsupport.symbolic import WORD from rpython.jit.backend.llsupport.llmodel import AbstractLLCPU +from rpython.jit.metainterp import history +from rpython.rlib.jit_hooks import LOOP_RUN_CONTAINER +from rpython.rlib.unroll import unrolling_iterable from rpython.rtyper.llinterp import LLInterpreter from rpython.rtyper.lltypesystem import lltype, rffi, llmemory -from rpython.rlib.jit_hooks import LOOP_RUN_CONTAINER -from rpython.jit.backend.arm.arch import FORCE_INDEX_OFS +jitframe.STATICSIZE = JITFRAME_FIXED_SIZE + class AbstractARMCPU(AbstractLLCPU): supports_floats = True @@ -18,14 +24,9 @@ def __init__(self, rtyper, stats, opts=None, translate_support_code=False, gcdescr=None): - if gcdescr is not None: - gcdescr.force_index_ofs = FORCE_INDEX_OFS AbstractLLCPU.__init__(self, rtyper, stats, opts, translate_support_code, gcdescr) - from rpython.jit.backend.llsupport import jitframe - self.deadframe_size_max = llmemory.sizeof(jitframe.DEADFRAME, - self.get_failargs_limit()) def set_debug(self, flag): return self.assembler.set_debug(flag) @@ -64,7 +65,11 @@ setitem(index, null) def make_execute_token(self, *ARGS): - FUNCPTR = lltype.Ptr(lltype.FuncType(ARGS, llmemory.GCREF)) + FUNCPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF], + llmemory.GCREF)) + + lst = [(i, history.getkind(ARG)[0]) for i, ARG in enumerate(ARGS)] + kinds = unrolling_iterable(lst) def execute_token(executable_token, *args): clt = executable_token.compiled_loop_token @@ -74,18 +79,32 @@ assert addr % 8 == 0 func = rffi.cast(FUNCPTR, addr) #llop.debug_print(lltype.Void, ">>>> Entering", addr) + frame_info = clt.frame_info + frame = self.gc_ll_descr.malloc_jitframe(frame_info) + ll_frame = lltype.cast_opaque_ptr(llmemory.GCREF, frame) prev_interpreter = None # help flow space if not self.translate_support_code: prev_interpreter = LLInterpreter.current_interpreter LLInterpreter.current_interpreter = self.debug_ll_interpreter try: - deadframe = func(*args) + num = JITFRAME_FIXED_SIZE * WORD + for i, kind in kinds: + arg = args[i] + if kind == history.INT: + self.set_int_value(ll_frame, num, arg) + elif kind == history.FLOAT: + self.set_float_value(ll_frame, num, arg) + num += WORD # on ARM(32 bit) a FLOAT needs two words + else: + assert kind == history.REF + self.set_ref_value(ll_frame, num, arg) + num += WORD + ll_frame = func(ll_frame) finally: if not self.translate_support_code: LLInterpreter.current_interpreter = prev_interpreter #llop.debug_print(lltype.Void, "<<<< Back") - self.gc_set_extra_threshold() - return deadframe + return ll_frame return execute_token def cast_ptr_to_int(x): diff --git a/rpython/jit/backend/arm/support.py b/rpython/jit/backend/arm/support.py new file mode 100644 --- /dev/null +++ b/rpython/jit/backend/arm/support.py @@ -0,0 +1,61 @@ +from rpython.rtyper.lltypesystem import lltype, rffi, llmemory +from rpython.rlib.rarithmetic import r_uint +from rpython.translator.tool.cbuild import ExternalCompilationInfo + +eci = ExternalCompilationInfo(post_include_bits=[""" +static int pypy__arm_int_div(int a, int b) { + return a/b; +} +static unsigned int pypy__arm_uint_div(unsigned int a, unsigned int b) { + return a/b; +} +static int pypy__arm_int_mod(int a, int b) { + return a % b; +} +"""]) + + +def arm_int_div_emulator(a, b): + return int(a / float(b)) +arm_int_div_sign = lltype.Ptr( + lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed)) +arm_int_div = rffi.llexternal( + "pypy__arm_int_div", [lltype.Signed, lltype.Signed], lltype.Signed, + _callable=arm_int_div_emulator, + compilation_info=eci, + _nowrapper=True, elidable_function=True) + + +def arm_uint_div_emulator(a, b): + return r_uint(a) / r_uint(b) +arm_uint_div_sign = lltype.Ptr( + lltype.FuncType([lltype.Unsigned, lltype.Unsigned], lltype.Unsigned)) +arm_uint_div = rffi.llexternal( + "pypy__arm_uint_div", [lltype.Unsigned, lltype.Unsigned], lltype.Unsigned, + _callable=arm_uint_div_emulator, + compilation_info=eci, + _nowrapper=True, elidable_function=True) + + +def arm_int_mod_emulator(a, b): + sign = 1 + if a < 0: + a = -1 * a + sign = -1 + if b < 0: + b = -1 * b + res = a % b + return sign * res +arm_int_mod_sign = arm_int_div_sign +arm_int_mod = rffi.llexternal( + "pypy__arm_int_mod", [lltype.Signed, lltype.Signed], lltype.Signed, + _callable=arm_int_mod_emulator, + compilation_info=eci, + _nowrapper=True, elidable_function=True) +# ____________________________________________________________ + +memcpy_fn = rffi.llexternal('memcpy', [llmemory.Address, llmemory.Address, + rffi.SIZE_T], lltype.Void, + sandboxsafe=True, _nowrapper=True) + +# ____________________________________________________________ diff --git a/rpython/jit/backend/arm/test/test_assembler.py b/rpython/jit/backend/arm/test/test_assembler.py --- a/rpython/jit/backend/arm/test/test_assembler.py +++ b/rpython/jit/backend/arm/test/test_assembler.py @@ -1,6 +1,6 @@ from rpython.jit.backend.arm import conditions as c from rpython.jit.backend.arm import registers as r -from rpython.jit.backend.arm.arch import arm_int_div +from rpython.jit.backend.arm.support import arm_int_div from rpython.jit.backend.arm.assembler import AssemblerARM from rpython.jit.backend.arm.locations import imm from rpython.jit.backend.arm.test.support import run_asm diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py --- a/rpython/jit/backend/llsupport/llmodel.py +++ b/rpython/jit/backend/llsupport/llmodel.py @@ -335,6 +335,40 @@ # ____________________________________________________________ + def set_int_value(self, newframe, index, value): + """ Note that we keep index multiplied by WORD here mostly + for completeness with get_int_value and friends + """ + descr = self.gc_ll_descr.getframedescrs(self).arraydescr + ofs = self.unpack_arraydescr(descr) + self.write_int_at_mem(newframe, ofs + index, WORD, 1, value) + + def set_ref_value(self, newframe, index, value): + descr = self.gc_ll_descr.getframedescrs(self).arraydescr + ofs = self.unpack_arraydescr(descr) + self.write_ref_at_mem(newframe, ofs + index, value) + + def set_float_value(self, newframe, index, value): + descr = self.gc_ll_descr.getframedescrs(self).arraydescr + ofs = self.unpack_arraydescr(descr) + self.write_float_at_mem(newframe, ofs + index, value) + + @specialize.arg(1) + def get_ofs_of_frame_field(self, name): + descrs = self.gc_ll_descr.getframedescrs(self) + if name.startswith('jfi_'): + base_ofs = 0 # not relative to frame + else: + base_ofs = self.unpack_arraydescr(descrs.arraydescr) + ofs = self.unpack_fielddescr(getattr(descrs, name)) + return ofs - base_ofs + + def get_baseofs_of_frame_field(self): + descrs = self.gc_ll_descr.getframedescrs(self) + base_ofs = self.unpack_arraydescr(descrs.arraydescr) + return base_ofs + # ____________________________________________________________ + def bh_arraylen_gc(self, array, arraydescr): assert isinstance(arraydescr, ArrayDescr) diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py --- a/rpython/jit/backend/test/runner_test.py +++ b/rpython/jit/backend/test/runner_test.py @@ -129,6 +129,8 @@ assert fail.identifier == 1 def test_compile_linear_float_loop(self): + if not self.cpu.supports_floats: + py.test.skip("requires floats") i0 = BoxFloat() i1 = BoxFloat() operations = [ diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py --- a/rpython/jit/backend/x86/runner.py +++ b/rpython/jit/backend/x86/runner.py @@ -107,7 +107,7 @@ lst = [(i, history.getkind(ARG)[0]) for i, ARG in enumerate(ARGS)] kinds = unrolling_iterable(lst) - + def execute_token(executable_token, *args): clt = executable_token.compiled_loop_token assert len(args) == clt._debug_nbargs @@ -160,7 +160,7 @@ def invalidate_loop(self, looptoken): from rpython.jit.backend.x86 import codebuf - + for addr, tgt in looptoken.compiled_loop_token.invalidate_positions: mc = codebuf.MachineCodeBlockWrapper() mc.JMP_l(tgt) @@ -178,38 +178,6 @@ l[i].counter = ll_s.i return l - def set_int_value(self, newframe, index, value): - """ Note that we keep index multiplied by WORD here mostly - for completeness with get_int_value and friends - """ - descr = self.gc_ll_descr.getframedescrs(self).arraydescr - ofs = self.unpack_arraydescr(descr) - self.write_int_at_mem(newframe, ofs + index, WORD, 1, value) - - def set_ref_value(self, newframe, index, value): - descr = self.gc_ll_descr.getframedescrs(self).arraydescr - ofs = self.unpack_arraydescr(descr) - self.write_ref_at_mem(newframe, ofs + index, value) - - def set_float_value(self, newframe, index, value): - descr = self.gc_ll_descr.getframedescrs(self).arraydescr - ofs = self.unpack_arraydescr(descr) - self.write_float_at_mem(newframe, ofs + index, value) - - @specialize.arg(1) - def get_ofs_of_frame_field(self, name): - descrs = self.gc_ll_descr.getframedescrs(self) - if name.startswith('jfi_'): - base_ofs = 0 # not relative to frame - else: - base_ofs = self.unpack_arraydescr(descrs.arraydescr) - ofs = self.unpack_fielddescr(getattr(descrs, name)) - return ofs - base_ofs - - def get_baseofs_of_frame_field(self): - descrs = self.gc_ll_descr.getframedescrs(self) - base_ofs = self.unpack_arraydescr(descrs.arraydescr) - return base_ofs class CPU386(AbstractX86CPU): backend_name = 'x86' From noreply at buildbot.pypy.org Mon Jan 28 18:30:38 2013 From: noreply at buildbot.pypy.org (fijal) Date: Mon, 28 Jan 2013 18:30:38 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: enable continuation module Message-ID: <20130128173038.DFE141C0264@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60620:508d5ce5b5df Date: 2013-01-28 19:27 +0200 http://bitbucket.org/pypy/pypy/changeset/508d5ce5b5df/ Log: enable continuation module diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py --- a/pypy/module/pypyjit/policy.py +++ b/pypy/module/pypyjit/policy.py @@ -106,7 +106,7 @@ 'posix', '_socket', '_sre', '_lsprof', '_weakref', '__pypy__', 'cStringIO', '_collections', 'struct', 'mmap', 'marshal', '_codecs', 'rctime', 'cppyy', - '_cffi_backend', 'pyexpat']: + '_cffi_backend', 'pyexpat', '_continuation']: if modname == 'pypyjit' and 'interp_resop' in rest: return False return True From noreply at buildbot.pypy.org Mon Jan 28 19:42:07 2013 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 28 Jan 2013 19:42:07 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: test differently Message-ID: <20130128184207.0A7851C087E@cobra.cs.uni-duesseldorf.de> Author: Matti Picus Branch: missing-ndarray-attributes Changeset: r60621:76bdd206840c Date: 2013-01-28 20:41 +0200 http://bitbucket.org/pypy/pypy/changeset/76bdd206840c/ Log: test differently diff --git a/pypy/module/micronumpy/arrayimpl/sort.py b/pypy/module/micronumpy/arrayimpl/sort.py --- a/pypy/module/micronumpy/arrayimpl/sort.py +++ b/pypy/module/micronumpy/arrayimpl/sort.py @@ -82,11 +82,11 @@ def argsort_array(arr, space, w_axis): itemtype = arr.dtype.itemtype - if (not isinstance(itemtype, types.Float) and - not isinstance(itemtype, types.Integer)): + if (not arr.dtype.is_int_type() and + not arr.dtype.is_float_type()): # XXX this should probably be changed raise OperationError(space.w_NotImplementedError, - space.wrap("sorting of non-numeric types is not implemented")) + space.wrap("sorting of non-numeric types '%r' is not implemented" % itemtype )) if w_axis is space.w_None: # note that it's fine ot pass None here as we're not going # to pass the result around (None is the link to base in slices) diff --git a/pypy/module/micronumpy/interp_dtype.py b/pypy/module/micronumpy/interp_dtype.py --- a/pypy/module/micronumpy/interp_dtype.py +++ b/pypy/module/micronumpy/interp_dtype.py @@ -155,6 +155,9 @@ def is_complex_type(self): return False + def is_float_type(self): + return (self.kind == FLOATINGLTR or self.float_type is not None) + def is_bool_type(self): return self.kind == BOOLLTR From noreply at buildbot.pypy.org Mon Jan 28 19:42:22 2013 From: noreply at buildbot.pypy.org (mattip) Date: Mon, 28 Jan 2013 19:42:22 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: merge default into branch Message-ID: <20130128184222.9636C1C087E@cobra.cs.uni-duesseldorf.de> Author: Matti Picus Branch: missing-ndarray-attributes Changeset: r60622:336093003972 Date: 2013-01-28 20:41 +0200 http://bitbucket.org/pypy/pypy/changeset/336093003972/ Log: merge default into branch diff too long, truncating to 2000 out of 141776 lines diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -36,6 +36,7 @@ .. branch: task-decorator .. branch: fix-e4fa0b2 .. branch: win32-fixes +.. branch: fix-version-tool .. branch: release-2.0-beta1 diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -343,9 +343,13 @@ signal, the tick counter is set to -1 by C code in signals.h. """ assert isinstance(action, PeriodicAsyncAction) - self._periodic_actions.append(action) + # hack to put the release-the-GIL one at the end of the list, + # and the report-the-signals one at the start of the list. if use_bytecode_counter: + self._periodic_actions.append(action) self.has_bytecode_counter = True + else: + self._periodic_actions.insert(0, action) self._rebuild_action_dispatcher() def getcheckinterval(self): @@ -419,15 +423,6 @@ The action must have been registered at space initalization time.""" self.space.actionflag.fire(self) - def fire_after_thread_switch(self): - """Bit of a hack: fire() the action but only the next time the GIL - is released and re-acquired (i.e. after a potential thread switch). - Don't call this if threads are not enabled. Currently limited to - one action (i.e. reserved for CheckSignalAction from module/signal). - """ - from pypy.module.thread.gil import spacestate - spacestate.action_after_thread_switch = self - def perform(self, executioncontext, frame): """To be overridden.""" diff --git a/pypy/interpreter/test2/test_app_main.py b/pypy/interpreter/test2/test_app_main.py --- a/pypy/interpreter/test2/test_app_main.py +++ b/pypy/interpreter/test2/test_app_main.py @@ -774,7 +774,7 @@ assert data == p + os.sep + '\n' def test_getfilesystemencoding(self): - py.test.skip("this has been failing since forever, but it's not tested nightly because buildbot uses python2.6 :-(") + py.test.skip("encoding is only set if stdout.isatty(), test is flawed") if sys.version_info < (2, 7): skip("test requires Python >= 2.7") p = getscript_in_dir(""" @@ -881,7 +881,6 @@ def test_setup_bootstrap_path(self): import sys - import os old_sys_path = sys.path[:] sys.path.append(self.goal_dir) try: diff --git a/pypy/interpreter/test2/test_targetpypy.py b/pypy/interpreter/test2/test_targetpypy.py --- a/pypy/interpreter/test2/test_targetpypy.py +++ b/pypy/interpreter/test2/test_targetpypy.py @@ -1,5 +1,3 @@ - -import py from pypy.goal.targetpypystandalone import get_entry_point from pypy.config.pypyoption import get_pypy_config @@ -7,6 +5,4 @@ def test_run(self): config = get_pypy_config(translating=False) entry_point = get_entry_point(config)[0] - space = self.space - py.test.skip("not working so far") entry_point(['pypy-c' , '-S', '-c', 'print 3']) diff --git a/pypy/module/_ffi/test/test_funcptr.py b/pypy/module/_ffi/test/test_funcptr.py --- a/pypy/module/_ffi/test/test_funcptr.py +++ b/pypy/module/_ffi/test/test_funcptr.py @@ -96,7 +96,7 @@ libm = CDLL(self.libm_name) pow_addr = libm.getaddressindll('pow') fff = sys.maxint*2-1 - if sys.platform == 'win32': + if sys.platform == 'win32' or sys.platform == 'darwin': fff = sys.maxint*2+1 assert pow_addr == self.pow_addr & fff diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py --- a/pypy/module/_file/test/test_file.py +++ b/pypy/module/_file/test/test_file.py @@ -430,15 +430,15 @@ assert f.subclass_closed def test_readline_unbuffered_should_read_one_line_only(self): - import posix + import os - with self.file(self.temppath, 'w') as f: + with self.file(self.temppath, 'wb') as f: f.write('foo\nbar\n') - with self.file(self.temppath, 'r', 0) as f: + with self.file(self.temppath, 'rb', 0) as f: s = f.readline() assert s == 'foo\n' - s = posix.read(f.fileno(), 10) + s = os.read(f.fileno(), 10) assert s == 'bar\n' def test_flush_at_exit(): diff --git a/pypy/module/_file/test/test_file_extra.py b/pypy/module/_file/test/test_file_extra.py --- a/pypy/module/_file/test/test_file_extra.py +++ b/pypy/module/_file/test/test_file_extra.py @@ -612,6 +612,7 @@ repr(unicode(self.temptestfile))) f.close() + @py.test.mark.skipif("os.name != 'posix'") def test_EAGAIN(self): import _socket, posix s1, s2 = _socket.socketpair() diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -2,7 +2,7 @@ import os class AppTestFileIO: - spaceconfig = dict(usemodules=['_io']) + spaceconfig = dict(usemodules=['_io'] + (['fcntl'] if os.name != 'nt' else [])) def setup_class(cls): tmpfile = udir.join('tmpfile') diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -49,6 +49,8 @@ kwds["libraries"] = [api_library] # '%s' undefined; assuming extern returning int kwds["compile_extra"] = ["/we4013"] + elif sys.platform == 'darwin': + kwds["link_files"] = [str(api_library + '.dylib')] else: kwds["link_files"] = [str(api_library + '.so')] if sys.platform.startswith('linux'): diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py --- a/pypy/module/rctime/test/test_rctime.py +++ b/pypy/module/rctime/test/test_rctime.py @@ -114,7 +114,10 @@ assert rctime.mktime(rctime.localtime(-1)) == -1 res = rctime.mktime((2000, 1, 1, 0, 0, 0, -1, -1, -1)) - assert rctime.ctime(res) == 'Sat Jan 1 00:00:00 2000' + if os.name == 'nt': + assert rctime.ctime(res) == 'Sat Jan 01 00:00:00 2000' + else: + assert rctime.ctime(res) == 'Sat Jan 1 00:00:00 2000' def test_asctime(self): import time as rctime @@ -213,7 +216,7 @@ def test_strftime(self): import time as rctime - import os + import os, sys t = rctime.time() tt = rctime.gmtime(t) @@ -234,6 +237,10 @@ # input to [w]strftime is not kosher. if os.name == 'nt': raises(ValueError, rctime.strftime, '%f') + elif sys.platform == 'darwin': + # darwin strips % of unknown format codes + # http://bugs.python.org/issue9811 + assert rctime.strftime('%f') == 'f' else: assert rctime.strftime('%f') == '%f' diff --git a/pypy/module/signal/__init__.py b/pypy/module/signal/__init__.py --- a/pypy/module/signal/__init__.py +++ b/pypy/module/signal/__init__.py @@ -30,9 +30,10 @@ } def buildloaders(cls): - from pypy.module.signal import interp_signal - for name in interp_signal.signal_names: - signum = getattr(interp_signal, name) + from rpython.rlib import rsignal + + for name in rsignal.signal_names: + signum = getattr(rsignal, name) if signum is not None: Module.interpleveldefs[name] = 'space.wrap(%d)' % (signum,) super(Module, cls).buildloaders() diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py --- a/pypy/module/signal/interp_signal.py +++ b/pypy/module/signal/interp_signal.py @@ -1,25 +1,28 @@ from __future__ import with_statement + +import signal as cpy_signal +import sys + from pypy.interpreter.error import OperationError, exception_from_errno -from pypy.interpreter.executioncontext import AsyncAction, AbstractActionFlag -from pypy.interpreter.executioncontext import PeriodicAsyncAction +from pypy.interpreter.executioncontext import (AsyncAction, AbstractActionFlag, + PeriodicAsyncAction) from pypy.interpreter.gateway import unwrap_spec -import signal as cpy_signal -from rpython.rtyper.lltypesystem import lltype, rffi -from rpython.rtyper.tool import rffi_platform -from rpython.translator.tool.cbuild import ExternalCompilationInfo -from rpython.conftest import cdir -import py -import sys -from rpython.rlib import jit, rposix + +from rpython.rlib import jit, rposix, rgc +from rpython.rlib.objectmodel import we_are_translated from rpython.rlib.rarithmetic import intmask from rpython.rlib.rsignal import * +from rpython.rtyper.lltypesystem import lltype, rffi + WIN32 = sys.platform == 'win32' + class SignalActionFlag(AbstractActionFlag): # This class uses the C-level pypysig_counter variable as the tick # counter. The C-level signal handler will reset it to -1 whenever - # a signal is received. + # a signal is received. This causes CheckSignalAction.perform() to + # be called. def get_ticker(self): p = pypysig_getaddr_occurred() @@ -29,6 +32,11 @@ p = pypysig_getaddr_occurred() p.c_value = value + @staticmethod + def rearm_ticker(): + p = pypysig_getaddr_occurred() + p.c_value = -1 + def decrement_ticker(self, by): p = pypysig_getaddr_occurred() value = p.c_value @@ -44,48 +52,65 @@ class CheckSignalAction(PeriodicAsyncAction): """An action that is automatically invoked when a signal is received.""" + # Note that this is a PeriodicAsyncAction: it means more precisely + # that it is called whenever the C-level ticker becomes < 0. + # Without threads, it is only ever set to -1 when we receive a + # signal. With threads, it also decrements steadily (but slowly). + def __init__(self, space): + "NOT_RPYTHON" AsyncAction.__init__(self, space) self.handlers_w = {} - if space.config.objspace.usemodules.thread: - # need a helper action in case signals arrive in a non-main thread - self.pending_signals = {} - self.reissue_signal_action = ReissueSignalAction(space) - else: - self.reissue_signal_action = None + self.pending_signal = -1 + self.fire_in_main_thread = False + if self.space.config.objspace.usemodules.thread: + from pypy.module.thread import gil + gil.after_thread_switch = self._after_thread_switch + + @rgc.no_collect + def _after_thread_switch(self): + if self.fire_in_main_thread: + if self.space.threadlocals.ismainthread(): + self.fire_in_main_thread = False + SignalActionFlag.rearm_ticker() + # this occurs when we just switched to the main thread + # and there is a signal pending: we force the ticker to + # -1, which should ensure perform() is called quickly. @jit.dont_look_inside def perform(self, executioncontext, frame): - while True: - n = pypysig_poll() - if n < 0: + # Poll for the next signal, if any + n = self.pending_signal + if n < 0: n = pypysig_poll() + while n >= 0: + if self.space.config.objspace.usemodules.thread: + in_main = self.space.threadlocals.ismainthread() + else: + in_main = True + if in_main: + # If we are in the main thread, report the signal now, + # and poll more + self.pending_signal = -1 + self._report_signal(n) + n = self.pending_signal + if n < 0: n = pypysig_poll() + else: + # Otherwise, arrange for perform() to be called again + # after we switch to the main thread. + self.pending_signal = n + self.fire_in_main_thread = True break - self.perform_signal(executioncontext, n) - @jit.dont_look_inside - def perform_signal(self, executioncontext, n): - if self.reissue_signal_action is None: - # no threads: we can report the signal immediately - self.report_signal(n) - else: - main_ec = self.space.threadlocals.getmainthreadvalue() - if executioncontext is main_ec: - # running in the main thread: we can report the - # signal immediately - self.report_signal(n) - else: - # running in another thread: we need to hack a bit - self.pending_signals[n] = None - self.reissue_signal_action.fire_after_thread_switch() - - @jit.dont_look_inside def set_interrupt(self): "Simulates the effect of a SIGINT signal arriving" - ec = self.space.getexecutioncontext() - self.perform_signal(ec, cpy_signal.SIGINT) + if not we_are_translated(): + self.pending_signal = cpy_signal.SIGINT + # ^^^ may override another signal, but it's just for testing + else: + pypysig_pushback(cpy_signal.SIGINT) + self.fire_in_main_thread = True - @jit.dont_look_inside - def report_signal(self, n): + def _report_signal(self, n): try: w_handler = self.handlers_w[n] except KeyError: @@ -100,39 +125,6 @@ w_frame = space.wrap(ec.gettopframe_nohidden()) space.call_function(w_handler, space.wrap(n), w_frame) - @jit.dont_look_inside - def report_pending_signals(self): - # XXX this logic isn't so complicated but I have no clue how - # to test it :-( - pending_signals = self.pending_signals.keys() - self.pending_signals.clear() - try: - while pending_signals: - self.report_signal(pending_signals.pop()) - finally: - # in case of exception, put the undelivered signals back - # into the dict instead of silently swallowing them - if pending_signals: - for n in pending_signals: - self.pending_signals[n] = None - self.reissue_signal_action.fire() - - -class ReissueSignalAction(AsyncAction): - """A special action to help deliver signals to the main thread. If - a non-main thread caught a signal, this action fires after every - thread switch until we land in the main thread. - """ - - def perform(self, executioncontext, frame): - main_ec = self.space.threadlocals.getmainthreadvalue() - if executioncontext is main_ec: - # now running in the main thread: we can really report the signals - self.space.check_signal_action.report_pending_signals() - else: - # still running in some other thread: try again later - self.fire_after_thread_switch() - @unwrap_spec(signum=int) def getsignal(space, signum): @@ -154,6 +146,7 @@ return action.handlers_w[signum] return space.wrap(SIG_DFL) + def default_int_handler(space, w_signum, w_frame): """ default_int_handler(...) @@ -164,22 +157,26 @@ raise OperationError(space.w_KeyboardInterrupt, space.w_None) + @jit.dont_look_inside @unwrap_spec(timeout=int) def alarm(space, timeout): return space.wrap(c_alarm(timeout)) + @jit.dont_look_inside def pause(space): c_pause() return space.w_None + def check_signum_exists(space, signum): if signum in signal_values: return raise OperationError(space.w_ValueError, space.wrap("invalid signal value")) + def check_signum_in_range(space, signum): if 1 <= signum < NSIG: return @@ -201,7 +198,7 @@ A signal handler function is called with two arguments: the first is the signal number, the second is the interrupted stack frame. """ - ec = space.getexecutioncontext() + ec = space.getexecutioncontext() main_ec = space.threadlocals.getmainthreadvalue() old_handler = getsignal(space, signum) @@ -224,13 +221,14 @@ action.handlers_w[signum] = w_handler return old_handler + @jit.dont_look_inside @unwrap_spec(fd=int) def set_wakeup_fd(space, fd): """Sets the fd to be written to (with '\0') when a signal comes in. Returns the old fd. A library can use this to wakeup select or poll. The previous fd is returned. - + The fd must be non-blocking. """ if space.config.objspace.usemodules.thread: @@ -243,6 +241,7 @@ old_fd = pypysig_set_wakeup_fd(fd) return space.wrap(intmask(old_fd)) + @jit.dont_look_inside @unwrap_spec(signum=int, flag=int) def siginterrupt(space, signum, flag): @@ -258,33 +257,38 @@ rffi.setintfield(timeval, 'c_tv_sec', int(d)) rffi.setintfield(timeval, 'c_tv_usec', int((d - int(d)) * 1000000)) + def double_from_timeval(tv): return rffi.getintfield(tv, 'c_tv_sec') + ( rffi.getintfield(tv, 'c_tv_usec') / 1000000.0) + def itimer_retval(space, val): w_value = space.wrap(double_from_timeval(val.c_it_value)) w_interval = space.wrap(double_from_timeval(val.c_it_interval)) return space.newtuple([w_value, w_interval]) + class Cache: def __init__(self, space): self.w_itimererror = space.new_exception_class("signal.ItimerError", space.w_IOError) + def get_itimer_error(space): return space.fromcache(Cache).w_itimererror + @jit.dont_look_inside @unwrap_spec(which=int, first=float, interval=float) def setitimer(space, which, first, interval=0): """setitimer(which, seconds[, interval]) - Sets given itimer (one of ITIMER_REAL, ITIMER_VIRTUAL + or ITIMER_PROF) to fire after value seconds and after that every interval seconds. The itimer can be cleared by setting seconds to zero. - + Returns old values as a tuple: (delay, interval). """ with lltype.scoped_alloc(itimervalP.TO, 1) as new: @@ -298,14 +302,14 @@ if ret != 0: raise exception_from_errno(space, get_itimer_error(space)) + return itimer_retval(space, old[0]) - return itimer_retval(space, old[0]) @jit.dont_look_inside @unwrap_spec(which=int) def getitimer(space, which): """getitimer(which) - + Returns current value of given itimer. """ with lltype.scoped_alloc(itimervalP.TO, 1) as old: diff --git a/pypy/module/signal/test/test_signal.py b/pypy/module/signal/test/test_signal.py --- a/pypy/module/signal/test/test_signal.py +++ b/pypy/module/signal/test/test_signal.py @@ -157,6 +157,8 @@ if sys.platform == 'win32': raises(ValueError, signal, 42, lambda *args: None) raises(ValueError, signal, 7, lambda *args: None) + elif sys.platform == 'darwin': + raises(ValueError, signal, 42, lambda *args: None) else: signal(42, lambda *args: None) signal(42, SIG_DFL) diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -28,6 +28,7 @@ import pypy pypydir = os.path.dirname(os.path.abspath(pypy.__file__)) +pypyroot = os.path.dirname(pypydir) del pypy from rpython.tool.version import get_repo_version_info @@ -68,7 +69,7 @@ CPYTHON_VERSION[0], CPYTHON_VERSION[1], CPYTHON_VERSION[2], - get_repo_version_info()[2], + get_repo_version_info(root=pypyroot)[1], date, time, ver, @@ -91,10 +92,10 @@ return space.wrap(('PyPy', '', '')) def get_repo_info(space): - info = get_repo_version_info() + info = get_repo_version_info(root=pypyroot) if info: - project, repo_tag, repo_version = info - return space.newtuple([space.wrap(project), + repo_tag, repo_version = info + return space.newtuple([space.wrap('PyPy'), space.wrap(repo_tag), space.wrap(repo_version)]) else: diff --git a/pypy/module/thread/gil.py b/pypy/module/thread/gil.py --- a/pypy/module/thread/gil.py +++ b/pypy/module/thread/gil.py @@ -62,22 +62,7 @@ do_yield_thread() -class SpaceState: - - def _cleanup_(self): - self.action_after_thread_switch = None - # ^^^ set by AsyncAction.fire_after_thread_switch() - - def after_thread_switch(self): - # this is support logic for the signal module, to help it deliver - # signals to the main thread. - action = self.action_after_thread_switch - if action is not None: - self.action_after_thread_switch = None - action.fire() - -spacestate = SpaceState() -spacestate._cleanup_() +after_thread_switch = lambda: None # hook for signal.py # Fragile code below. We have to preserve the C-level errno manually... @@ -94,7 +79,7 @@ e = get_errno() thread.gil_acquire() thread.gc_thread_run() - spacestate.after_thread_switch() + after_thread_switch() set_errno(e) after_external_call._gctransformer_hint_cannot_collect_ = True after_external_call._dont_reach_me_in_del_ = True @@ -112,7 +97,7 @@ # the same thread). if thread.gil_yield_thread(): thread.gc_thread_run() - spacestate.after_thread_switch() + after_thread_switch() do_yield_thread._gctransformer_hint_close_stack_ = True do_yield_thread._dont_reach_me_in_del_ = True do_yield_thread._dont_inline_ = True diff --git a/pypy/module/thread/os_local.py b/pypy/module/thread/os_local.py --- a/pypy/module/thread/os_local.py +++ b/pypy/module/thread/os_local.py @@ -98,5 +98,5 @@ local = wref() if local is not None: del local.dicts[ec] - local.last_dict = None - local.last_ec = None + local.last_dict = None + local.last_ec = None diff --git a/pypy/module/thread/threadlocals.py b/pypy/module/thread/threadlocals.py --- a/pypy/module/thread/threadlocals.py +++ b/pypy/module/thread/threadlocals.py @@ -48,6 +48,9 @@ ident = self._mainthreadident return self._valuedict.get(ident, None) + def ismainthread(self): + return thread.get_ident() == self._mainthreadident + def getallvalues(self): return self._valuedict diff --git a/rpython/jit/backend/llsupport/ffisupport.py b/rpython/jit/backend/llsupport/ffisupport.py --- a/rpython/jit/backend/llsupport/ffisupport.py +++ b/rpython/jit/backend/llsupport/ffisupport.py @@ -42,11 +42,14 @@ @specialize.memo() def _get_ffi2descr_dict(cpu): - d = {('v', 0): ('v', None)} + def entry(letter, TYPE): + return (letter, cpu.arraydescrof(rffi.CArray(TYPE)), rffi.sizeof(TYPE)) + # + d = {('v', 0): ('v', None, 1)} if cpu.supports_floats: - d[('f', 0)] = ('f', cpu.arraydescrof(rffi.CArray(lltype.Float))) + d[('f', 0)] = entry('f', lltype.Float) if cpu.supports_singlefloats: - d[('S', 0)] = ('i', cpu.arraydescrof(rffi.CArray(lltype.SingleFloat))) + d[('S', 0)] = entry('i', lltype.SingleFloat) for SIGNED_TYPE in [rffi.SIGNEDCHAR, rffi.SHORT, rffi.INT, @@ -59,7 +62,7 @@ continue key = ('L', 0) kind = 'f' - d[key] = (kind, cpu.arraydescrof(rffi.CArray(SIGNED_TYPE))) + d[key] = entry(kind, SIGNED_TYPE) for UNSIGNED_TYPE in [rffi.UCHAR, rffi.USHORT, rffi.UINT, @@ -68,7 +71,7 @@ key = ('u', rffi.sizeof(UNSIGNED_TYPE)) if key[1] > rffi.sizeof(lltype.Signed): continue - d[key] = ('i', cpu.arraydescrof(rffi.CArray(UNSIGNED_TYPE))) + d[key] = entry('i', UNSIGNED_TYPE) return d def get_arg_descr(cpu, ffi_type): diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -2567,7 +2567,8 @@ self.history.operations.pop() arg_boxes = [] for i in range(cif_description.nargs): - kind, descr = get_arg_descr(self.cpu, cif_description.atypes[i]) + kind, descr, itemsize = get_arg_descr(self.cpu, + cif_description.atypes[i]) if kind == 'i': box_arg = history.BoxInt() elif kind == 'f': @@ -2576,16 +2577,14 @@ assert kind == 'v' continue ofs = cif_description.exchange_args[i] - box_argpos = history.BoxInt() - self.history.record(rop.INT_ADD, - [box_exchange_buffer, ConstInt(ofs)], - box_argpos) + assert ofs % itemsize == 0 # alignment check self.history.record(rop.GETARRAYITEM_RAW, - [box_argpos, ConstInt(0)], + [box_exchange_buffer, + ConstInt(ofs // itemsize)], box_arg, descr) arg_boxes.append(box_arg) # - kind, descr = get_arg_descr(self.cpu, cif_description.rtype) + kind, descr, itemsize = get_arg_descr(self.cpu, cif_description.rtype) if kind == 'i': box_result = history.BoxInt() elif kind == 'f': @@ -2601,12 +2600,10 @@ # if box_result is not None: ofs = cif_description.exchange_result - box_resultpos = history.BoxInt() - self.history.record(rop.INT_ADD, - [box_exchange_buffer, ConstInt(ofs)], - box_resultpos) + assert ofs % itemsize == 0 # alignment check (result) self.history.record(rop.SETARRAYITEM_RAW, - [box_resultpos, ConstInt(0), box_result], + [box_exchange_buffer, + ConstInt(ofs // itemsize), box_result], None, descr) def direct_call_release_gil(self): diff --git a/rpython/jit/metainterp/test/test_warmspot.py b/rpython/jit/metainterp/test/test_warmspot.py --- a/rpython/jit/metainterp/test/test_warmspot.py +++ b/rpython/jit/metainterp/test/test_warmspot.py @@ -401,6 +401,7 @@ assert len(oplabel.getarglist()) == 2 # 'n', 'res' in some order def test_inline_jit_merge_point(self): + py.test.skip("fix the test if you want to re-enable this") # test that the machinery to inline jit_merge_points in callers # works. The final user does not need to mess manually with the # _inline_jit_merge_point_ attribute and similar, it is all nicely @@ -430,6 +431,7 @@ self.check_resops(int_add=4) def test_jitdriver_inline(self): + py.test.skip("fix the test if you want to re-enable this") myjitdriver = JitDriver(greens = [], reds = 'auto') class MyRange(object): def __init__(self, n): @@ -462,6 +464,7 @@ self.check_trace_count(1) def test_jitdriver_inline_twice(self): + py.test.skip("fix the test if you want to re-enable this") myjitdriver = JitDriver(greens = [], reds = 'auto') def jit_merge_point(a, b): @@ -492,6 +495,7 @@ self.check_trace_count(2) def test_jitdriver_inline_exception(self): + py.test.skip("fix the test if you want to re-enable this") # this simulates what happens in a real case scenario: inside the next # we have a call which we cannot inline (e.g. space.next in the case # of W_InterpIterable), but we need to put it in a try/except block. diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -578,6 +578,8 @@ pass def inline(self, call_jit_merge_point): + assert False, "@inline off: see skipped failures in test_warmspot." + # assert self.autoreds, "@inline works only with reds='auto'" self.inline_jit_merge_point = True def decorate(func): diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py --- a/rpython/rlib/ropenssl.py +++ b/rpython/rlib/ropenssl.py @@ -2,6 +2,7 @@ from rpython.rtyper.tool import rffi_platform from rpython.translator.platform import platform from rpython.translator.tool.cbuild import ExternalCompilationInfo +from rpython.rlib.unroll import unrolling_iterable import sys, os @@ -92,8 +93,12 @@ OPENSSL_NO_SSL2 = rffi_platform.Defined("OPENSSL_NO_SSL2") SSL_FILETYPE_PEM = rffi_platform.ConstantInteger("SSL_FILETYPE_PEM") SSL_OP_ALL = rffi_platform.ConstantInteger("SSL_OP_ALL") + SSL_OP_NO_SSLv2 = rffi_platform.ConstantInteger("SSL_OP_NO_SSLv2") + SSL_OP_NO_SSLv3 = rffi_platform.ConstantInteger("SSL_OP_NO_SSLv3") + SSL_OP_NO_TLSv1 = rffi_platform.ConstantInteger("SSL_OP_NO_TLSv1") SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS = rffi_platform.ConstantInteger( "SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS") + HAS_SNI = rffi_platform.Defined("SSL_CTRL_SET_TLSEXT_HOSTNAME") SSL_VERIFY_NONE = rffi_platform.ConstantInteger("SSL_VERIFY_NONE") SSL_VERIFY_PEER = rffi_platform.ConstantInteger("SSL_VERIFY_PEER") SSL_VERIFY_FAIL_IF_NO_PEER_CERT = rffi_platform.ConstantInteger("SSL_VERIFY_FAIL_IF_NO_PEER_CERT") @@ -118,6 +123,9 @@ CRYPTO_LOCK = rffi_platform.ConstantInteger("CRYPTO_LOCK") + OBJ_NAME_TYPE_MD_METH = rffi_platform.ConstantInteger( + "OBJ_NAME_TYPE_MD_METH") + # Some structures, with only the fields used in the _ssl module X509_name_entry_st = rffi_platform.Struct('struct X509_name_entry_st', [('set', rffi.INT)]) @@ -146,6 +154,12 @@ OPENSSL_EXPORT_VAR_AS_FUNCTION = rffi_platform.Defined( "OPENSSL_EXPORT_VAR_AS_FUNCTION") + OBJ_NAME_st = rffi_platform.Struct( + 'OBJ_NAME', + [('alias', rffi.INT), + ('name', rffi.CCHARP), + ]) + for k, v in rffi_platform.configure(CConfig).items(): globals()[k] = v @@ -166,8 +180,10 @@ ASN1_INTEGER = rffi.COpaquePtr('ASN1_INTEGER') GENERAL_NAMES = rffi.COpaquePtr('GENERAL_NAMES') GENERAL_NAME = rffi.CArrayPtr(GENERAL_NAME_st) +OBJ_NAME = rffi.CArrayPtr(OBJ_NAME_st) HAVE_OPENSSL_RAND = OPENSSL_VERSION_NUMBER >= 0x0090500f +HAVE_SSL_CTX_CLEAR_OPTIONS = OPENSSL_VERSION_NUMBER >= 0x009080df def external(name, argtypes, restype, **kw): kw['compilation_info'] = eci @@ -203,12 +219,26 @@ ssl_external('SSLv23_method', [], SSL_METHOD) ssl_external('SSL_CTX_use_PrivateKey_file', [SSL_CTX, rffi.CCHARP, rffi.INT], rffi.INT) ssl_external('SSL_CTX_use_certificate_chain_file', [SSL_CTX, rffi.CCHARP], rffi.INT) +ssl_external('SSL_CTX_get_options', [SSL_CTX], rffi.INT, macro=True) ssl_external('SSL_CTX_set_options', [SSL_CTX, rffi.INT], rffi.INT, macro=True) +if HAVE_SSL_CTX_CLEAR_OPTIONS: + ssl_external('SSL_CTX_clear_options', [SSL_CTX, rffi.INT], rffi.INT, + macro=True) ssl_external('SSL_CTX_ctrl', [SSL_CTX, rffi.INT, rffi.INT, rffi.VOIDP], rffi.INT) ssl_external('SSL_CTX_set_verify', [SSL_CTX, rffi.INT, rffi.VOIDP], lltype.Void) ssl_external('SSL_CTX_get_verify_mode', [SSL_CTX], rffi.INT) +ssl_external('SSL_CTX_set_default_verify_paths', [SSL_CTX], rffi.INT) ssl_external('SSL_CTX_set_cipher_list', [SSL_CTX, rffi.CCHARP], rffi.INT) ssl_external('SSL_CTX_load_verify_locations', [SSL_CTX, rffi.CCHARP, rffi.CCHARP], rffi.INT) +ssl_external('SSL_CTX_check_private_key', [SSL_CTX], rffi.INT) +ssl_external('SSL_CTX_set_session_id_context', [SSL_CTX, rffi.CCHARP, rffi.UINT], rffi.INT) +SSL_CTX_STATS_NAMES = """ + number connect connect_good connect_renegotiate accept accept_good + accept_renegotiate hits misses timeouts cache_full""".split() +SSL_CTX_STATS = unrolling_iterable( + (name, external('SSL_CTX_sess_' + name, [SSL_CTX], rffi.LONG, macro=True)) + for name in SSL_CTX_STATS_NAMES) + ssl_external('SSL_new', [SSL_CTX], SSL) ssl_external('SSL_set_fd', [SSL, rffi.INT], rffi.INT) ssl_external('SSL_set_mode', [SSL, rffi.INT], rffi.INT, macro=True) @@ -224,6 +254,7 @@ ssl_external('SSL_get_error', [SSL, rffi.INT], rffi.INT) ssl_external('SSL_get_shutdown', [SSL], rffi.INT) ssl_external('SSL_set_read_ahead', [SSL, rffi.INT], lltype.Void) +ssl_external('SSL_set_tlsext_host_name', [SSL, rffi.CCHARP], rffi.INT, macro=True) ssl_external('SSL_get_peer_certificate', [SSL], X509) ssl_external('X509_get_subject_name', [X509], X509_NAME) @@ -234,7 +265,7 @@ ssl_external('X509_NAME_ENTRY_get_object', [X509_NAME_ENTRY], ASN1_OBJECT) ssl_external('X509_NAME_ENTRY_get_data', [X509_NAME_ENTRY], ASN1_STRING) ssl_external('i2d_X509', [X509, rffi.CCHARPP], rffi.INT) -ssl_external('X509_free', [X509], lltype.Void) +ssl_external('X509_free', [X509], lltype.Void, threadsafe=False) ssl_external('X509_get_notBefore', [X509], ASN1_TIME, macro=True) ssl_external('X509_get_notAfter', [X509], ASN1_TIME, macro=True) ssl_external('X509_get_serialNumber', [X509], ASN1_INTEGER) @@ -272,9 +303,12 @@ ssl_external('ERR_get_error', [], rffi.INT) ssl_external('ERR_peek_last_error', [], rffi.INT) ssl_external('ERR_error_string', [rffi.ULONG, rffi.CCHARP], rffi.CCHARP) +ssl_external('ERR_clear_error', [], lltype.Void) -ssl_external('SSL_free', [SSL], lltype.Void) -ssl_external('SSL_CTX_free', [SSL_CTX], lltype.Void) +# 'threadsafe=False' here indicates that this function will be called +# with the GIL held, and so is allowed to run in a RPython __del__ method. +ssl_external('SSL_free', [SSL], lltype.Void, threadsafe=False) +ssl_external('SSL_CTX_free', [SSL_CTX], lltype.Void, threadsafe=False) ssl_external('CRYPTO_free', [rffi.VOIDP], lltype.Void) libssl_OPENSSL_free = libssl_CRYPTO_free @@ -316,6 +350,11 @@ EVP_MD_CTX_cleanup = external( 'EVP_MD_CTX_cleanup', [EVP_MD_CTX], rffi.INT, threadsafe=False) +OBJ_NAME_CALLBACK = lltype.Ptr(lltype.FuncType( + [OBJ_NAME, rffi.VOIDP], lltype.Void)) +OBJ_NAME_do_all = external( + 'OBJ_NAME_do_all', [rffi.INT, OBJ_NAME_CALLBACK, rffi.VOIDP], lltype.Void) + # HASH_MALLOC_SIZE is the size of EVP_MD, EVP_MD_CTX plus their points # Used for adding memory pressure. Last number is an (under?)estimate of # EVP_PKEY_CTX's size. diff --git a/rpython/rlib/rsignal.py b/rpython/rlib/rsignal.py --- a/rpython/rlib/rsignal.py +++ b/rpython/rlib/rsignal.py @@ -80,6 +80,8 @@ pypysig_poll = external('pypysig_poll', [], rffi.INT, threadsafe=False) # don't bother releasing the GIL around a call to pypysig_poll: it's # pointless and a performance issue +pypysig_pushback = external('pypysig_pushback', [rffi.INT], lltype.Void, + threadsafe=False) # don't use rffi.LONGP because the JIT doesn't support raw arrays so far struct_name = 'pypysig_long_struct' diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py --- a/rpython/rlib/rthread.py +++ b/rpython/rlib/rthread.py @@ -19,9 +19,9 @@ separate_module_files = [translator_c_dir / 'src' / 'thread.c'], include_dirs = [translator_c_dir], export_symbols = ['RPyThreadGetIdent', 'RPyThreadLockInit', - 'RPyThreadAcquireLock', 'RPyThreadReleaseLock', - 'RPyGilAllocate', 'RPyGilYieldThread', - 'RPyGilRelease', 'RPyGilAcquire', + 'RPyThreadAcquireLock', 'RPyThreadAcquireLockTimed', + 'RPyThreadReleaseLock', 'RPyGilAllocate', + 'RPyGilYieldThread', 'RPyGilRelease', 'RPyGilAcquire', 'RPyThreadGetStackSize', 'RPyThreadSetStackSize', 'RPyOpaqueDealloc_ThreadLock', 'RPyThreadAfterFork'] @@ -61,6 +61,10 @@ c_thread_acquirelock = llexternal('RPyThreadAcquireLock', [TLOCKP, rffi.INT], rffi.INT, threadsafe=True) # release the GIL +c_thread_acquirelock_timed = llexternal('RPyThreadAcquireLockTimed', + [TLOCKP, rffi.LONGLONG, rffi.INT], + rffi.INT, + threadsafe=True) # release the GIL c_thread_releaselock = llexternal('RPyThreadReleaseLock', [TLOCKP], lltype.Void, threadsafe=True) # release the GIL @@ -121,6 +125,12 @@ res = rffi.cast(lltype.Signed, res) return bool(res) + def acquire_timed(self, timeout): + "timeout is in microseconds." + res = c_thread_acquirelock_timed(self._lock, timeout, 1) + res = rffi.cast(lltype.Signed, res) + return bool(res) + def release(self): # Sanity check: the lock must be locked if self.acquire(False): diff --git a/rpython/rlib/test/test_jit.py b/rpython/rlib/test/test_jit.py --- a/rpython/rlib/test/test_jit.py +++ b/rpython/rlib/test/test_jit.py @@ -35,6 +35,7 @@ assert driver.reds == ['a', 'b'] assert driver.numreds == 2 + at py.test.mark.xfail(reason="@inline off: see skipped failures in test_warmspot.") def test_jitdriver_inline(): driver = JitDriver(greens=[], reds='auto') calls = [] @@ -54,6 +55,7 @@ ('bar', 40, 2), ] + at py.test.mark.xfail(reason="@inline off: see skipped failures in test_warmspot.") def test_jitdriver_clone(): def bar(): pass def foo(): pass diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py --- a/rpython/rlib/test/test_rposix.py +++ b/rpython/rlib/test/test_rposix.py @@ -117,7 +117,7 @@ curdir = os.getcwd() try: interpret(f, []) - assert os.getcwdu() == self.ufilename + assert os.getcwdu() == os.path.realpath(self.ufilename) finally: os.chdir(curdir) diff --git a/rpython/rlib/test/test_rthread.py b/rpython/rlib/test/test_rthread.py --- a/rpython/rlib/test/test_rthread.py +++ b/rpython/rlib/test/test_rthread.py @@ -153,6 +153,23 @@ answers = fn() assert answers == expected + def test_acquire_timed(self): + import time + def f(): + l = allocate_lock() + l.acquire(True) + t1 = time.time() + ok = l.acquire_timed(1000000) + t2 = time.time() + delay = t2 - t1 + if ok: + return delay + else: + return -delay + fn = self.getcompiled(f, []) + res = fn() + assert res < -1.0 + #class TestRunDirectly(AbstractThreadTests): # def getcompiled(self, f, argtypes): # return f diff --git a/rpython/rlib/unicodedata/DerivedCoreProperties-3.2.0.txt b/rpython/rlib/unicodedata/DerivedCoreProperties-3.2.0.txt new file mode 100644 --- /dev/null +++ b/rpython/rlib/unicodedata/DerivedCoreProperties-3.2.0.txt @@ -0,0 +1,4218 @@ +# DerivedCoreProperties-3.2.0.txt +# Date: 2002-03-19,23:30:42 GMT [MD] +# +# Unicode Character Database: Derived Property Data +# Generated algorithmically from the Unicode Character Database +# For documentation, see DerivedProperties.html +# Note: Unassigned and Noncharacter codepoints are omitted, +# except when listing Noncharacter or Cn. +# ================================================ + +# ================================================ + +# Derived Property: Math +# Generated from: Sm + Other_Math + +0028 ; Math # Ps LEFT PARENTHESIS +0029 ; Math # Pe RIGHT PARENTHESIS +002A ; Math # Po ASTERISK +002B ; Math # Sm PLUS SIGN +002D ; Math # Pd HYPHEN-MINUS +002F ; Math # Po SOLIDUS +003C..003E ; Math # Sm [3] LESS-THAN SIGN..GREATER-THAN SIGN +005B ; Math # Ps LEFT SQUARE BRACKET +005C ; Math # Po REVERSE SOLIDUS +005D ; Math # Pe RIGHT SQUARE BRACKET +005E ; Math # Sk CIRCUMFLEX ACCENT +007B ; Math # Ps LEFT CURLY BRACKET +007C ; Math # Sm VERTICAL LINE +007D ; Math # Pe RIGHT CURLY BRACKET +007E ; Math # Sm TILDE +00AC ; Math # Sm NOT SIGN +00B1 ; Math # Sm PLUS-MINUS SIGN +00D7 ; Math # Sm MULTIPLICATION SIGN +00F7 ; Math # Sm DIVISION SIGN +03F6 ; Math # Sm GREEK REVERSED LUNATE EPSILON SYMBOL +2016 ; Math # Po DOUBLE VERTICAL LINE +2032..2034 ; Math # Po [3] PRIME..TRIPLE PRIME +2044 ; Math # Sm FRACTION SLASH +2052 ; Math # Sm COMMERCIAL MINUS SIGN +207A..207C ; Math # Sm [3] SUPERSCRIPT PLUS SIGN..SUPERSCRIPT EQUALS SIGN +207D ; Math # Ps SUPERSCRIPT LEFT PARENTHESIS +207E ; Math # Pe SUPERSCRIPT RIGHT PARENTHESIS +208A..208C ; Math # Sm [3] SUBSCRIPT PLUS SIGN..SUBSCRIPT EQUALS SIGN +208D ; Math # Ps SUBSCRIPT LEFT PARENTHESIS +208E ; Math # Pe SUBSCRIPT RIGHT PARENTHESIS +20D0..20DC ; Math # Mn [13] COMBINING LEFT HARPOON ABOVE..COMBINING FOUR DOTS ABOVE +20E1 ; Math # Mn COMBINING LEFT RIGHT ARROW ABOVE +2102 ; Math # L& DOUBLE-STRUCK CAPITAL C +210A..2113 ; Math # L& [10] SCRIPT SMALL G..SCRIPT SMALL L +2115 ; Math # L& DOUBLE-STRUCK CAPITAL N +2119..211D ; Math # L& [5] DOUBLE-STRUCK CAPITAL P..DOUBLE-STRUCK CAPITAL R +2124 ; Math # L& DOUBLE-STRUCK CAPITAL Z +2128 ; Math # L& BLACK-LETTER CAPITAL Z +212C..212D ; Math # L& [2] SCRIPT CAPITAL B..BLACK-LETTER CAPITAL C +212F..2131 ; Math # L& [3] SCRIPT SMALL E..SCRIPT CAPITAL F +2133..2134 ; Math # L& [2] SCRIPT CAPITAL M..SCRIPT SMALL O +2135..2138 ; Math # Lo [4] ALEF SYMBOL..DALET SYMBOL +2140..2144 ; Math # Sm [5] DOUBLE-STRUCK N-ARY SUMMATION..TURNED SANS-SERIF CAPITAL Y +214B ; Math # Sm TURNED AMPERSAND +2190..2194 ; Math # Sm [5] LEFTWARDS ARROW..LEFT RIGHT ARROW +219A..219B ; Math # Sm [2] LEFTWARDS ARROW WITH STROKE..RIGHTWARDS ARROW WITH STROKE +21A0 ; Math # Sm RIGHTWARDS TWO HEADED ARROW +21A3 ; Math # Sm RIGHTWARDS ARROW WITH TAIL +21A6 ; Math # Sm RIGHTWARDS ARROW FROM BAR +21AE ; Math # Sm LEFT RIGHT ARROW WITH STROKE +21CE..21CF ; Math # Sm [2] LEFT RIGHT DOUBLE ARROW WITH STROKE..RIGHTWARDS DOUBLE ARROW WITH STROKE +21D2 ; Math # Sm RIGHTWARDS DOUBLE ARROW +21D4 ; Math # Sm LEFT RIGHT DOUBLE ARROW +21F4..22FF ; Math # Sm [268] RIGHT ARROW WITH SMALL CIRCLE..Z NOTATION BAG MEMBERSHIP +2308..230B ; Math # Sm [4] LEFT CEILING..RIGHT FLOOR +2320..2321 ; Math # Sm [2] TOP HALF INTEGRAL..BOTTOM HALF INTEGRAL +237C ; Math # Sm RIGHT ANGLE WITH DOWNWARDS ZIGZAG ARROW +239B..23B3 ; Math # Sm [25] LEFT PARENTHESIS UPPER HOOK..SUMMATION BOTTOM +25B7 ; Math # Sm WHITE RIGHT-POINTING TRIANGLE +25C1 ; Math # Sm WHITE LEFT-POINTING TRIANGLE +25F8..25FF ; Math # Sm [8] UPPER LEFT TRIANGLE..LOWER RIGHT TRIANGLE +266F ; Math # Sm MUSIC SHARP SIGN +27D0..27E5 ; Math # Sm [22] WHITE DIAMOND WITH CENTRED DOT..WHITE SQUARE WITH RIGHTWARDS TICK +27F0..27FF ; Math # Sm [16] UPWARDS QUADRUPLE ARROW..LONG RIGHTWARDS SQUIGGLE ARROW +2900..2982 ; Math # Sm [131] RIGHTWARDS TWO-HEADED ARROW WITH VERTICAL STROKE..Z NOTATION TYPE COLON +2999..29D7 ; Math # Sm [63] DOTTED FENCE..BLACK HOURGLASS +29DC..29FB ; Math # Sm [32] INCOMPLETE INFINITY..TRIPLE PLUS +29FE..2AFF ; Math # Sm [258] TINY..N-ARY WHITE VERTICAL BAR +FB29 ; Math # Sm HEBREW LETTER ALTERNATIVE PLUS SIGN +FE35 ; Math # Ps PRESENTATION FORM FOR VERTICAL LEFT PARENTHESIS +FE36 ; Math # Pe PRESENTATION FORM FOR VERTICAL RIGHT PARENTHESIS +FE37 ; Math # Ps PRESENTATION FORM FOR VERTICAL LEFT CURLY BRACKET +FE38 ; Math # Pe PRESENTATION FORM FOR VERTICAL RIGHT CURLY BRACKET +FE59 ; Math # Ps SMALL LEFT PARENTHESIS +FE5A ; Math # Pe SMALL RIGHT PARENTHESIS +FE5B ; Math # Ps SMALL LEFT CURLY BRACKET +FE5C ; Math # Pe SMALL RIGHT CURLY BRACKET +FE61 ; Math # Po SMALL ASTERISK +FE62 ; Math # Sm SMALL PLUS SIGN +FE63 ; Math # Pd SMALL HYPHEN-MINUS +FE64..FE66 ; Math # Sm [3] SMALL LESS-THAN SIGN..SMALL EQUALS SIGN +FE68 ; Math # Po SMALL REVERSE SOLIDUS +FF08 ; Math # Ps FULLWIDTH LEFT PARENTHESIS +FF09 ; Math # Pe FULLWIDTH RIGHT PARENTHESIS +FF0A ; Math # Po FULLWIDTH ASTERISK +FF0B ; Math # Sm FULLWIDTH PLUS SIGN +FF0D ; Math # Pd FULLWIDTH HYPHEN-MINUS +FF0F ; Math # Po FULLWIDTH SOLIDUS +FF1C..FF1E ; Math # Sm [3] FULLWIDTH LESS-THAN SIGN..FULLWIDTH GREATER-THAN SIGN +FF3B ; Math # Ps FULLWIDTH LEFT SQUARE BRACKET +FF3C ; Math # Po FULLWIDTH REVERSE SOLIDUS +FF3D ; Math # Pe FULLWIDTH RIGHT SQUARE BRACKET +FF3E ; Math # Sk FULLWIDTH CIRCUMFLEX ACCENT +FF5B ; Math # Ps FULLWIDTH LEFT CURLY BRACKET +FF5C ; Math # Sm FULLWIDTH VERTICAL LINE +FF5D ; Math # Pe FULLWIDTH RIGHT CURLY BRACKET +FF5E ; Math # Sm FULLWIDTH TILDE +FFE2 ; Math # Sm FULLWIDTH NOT SIGN +FFE9..FFEC ; Math # Sm [4] HALFWIDTH LEFTWARDS ARROW..HALFWIDTH DOWNWARDS ARROW +1D400..1D454 ; Math # L& [85] MATHEMATICAL BOLD CAPITAL A..MATHEMATICAL ITALIC SMALL G +1D456..1D49C ; Math # L& [71] MATHEMATICAL ITALIC SMALL I..MATHEMATICAL SCRIPT CAPITAL A +1D49E..1D49F ; Math # L& [2] MATHEMATICAL SCRIPT CAPITAL C..MATHEMATICAL SCRIPT CAPITAL D +1D4A2 ; Math # L& MATHEMATICAL SCRIPT CAPITAL G +1D4A5..1D4A6 ; Math # L& [2] MATHEMATICAL SCRIPT CAPITAL J..MATHEMATICAL SCRIPT CAPITAL K +1D4A9..1D4AC ; Math # L& [4] MATHEMATICAL SCRIPT CAPITAL N..MATHEMATICAL SCRIPT CAPITAL Q +1D4AE..1D4B9 ; Math # L& [12] MATHEMATICAL SCRIPT CAPITAL S..MATHEMATICAL SCRIPT SMALL D +1D4BB ; Math # L& MATHEMATICAL SCRIPT SMALL F +1D4BD..1D4C0 ; Math # L& [4] MATHEMATICAL SCRIPT SMALL H..MATHEMATICAL SCRIPT SMALL K +1D4C2..1D4C3 ; Math # L& [2] MATHEMATICAL SCRIPT SMALL M..MATHEMATICAL SCRIPT SMALL N +1D4C5..1D505 ; Math # L& [65] MATHEMATICAL SCRIPT SMALL P..MATHEMATICAL FRAKTUR CAPITAL B +1D507..1D50A ; Math # L& [4] MATHEMATICAL FRAKTUR CAPITAL D..MATHEMATICAL FRAKTUR CAPITAL G +1D50D..1D514 ; Math # L& [8] MATHEMATICAL FRAKTUR CAPITAL J..MATHEMATICAL FRAKTUR CAPITAL Q +1D516..1D51C ; Math # L& [7] MATHEMATICAL FRAKTUR CAPITAL S..MATHEMATICAL FRAKTUR CAPITAL Y +1D51E..1D539 ; Math # L& [28] MATHEMATICAL FRAKTUR SMALL A..MATHEMATICAL DOUBLE-STRUCK CAPITAL B +1D53B..1D53E ; Math # L& [4] MATHEMATICAL DOUBLE-STRUCK CAPITAL D..MATHEMATICAL DOUBLE-STRUCK CAPITAL G +1D540..1D544 ; Math # L& [5] MATHEMATICAL DOUBLE-STRUCK CAPITAL I..MATHEMATICAL DOUBLE-STRUCK CAPITAL M +1D546 ; Math # L& MATHEMATICAL DOUBLE-STRUCK CAPITAL O +1D54A..1D550 ; Math # L& [7] MATHEMATICAL DOUBLE-STRUCK CAPITAL S..MATHEMATICAL DOUBLE-STRUCK CAPITAL Y +1D552..1D6A3 ; Math # L& [338] MATHEMATICAL DOUBLE-STRUCK SMALL A..MATHEMATICAL MONOSPACE SMALL Z +1D6A8..1D6C0 ; Math # L& [25] MATHEMATICAL BOLD CAPITAL ALPHA..MATHEMATICAL BOLD CAPITAL OMEGA +1D6C1 ; Math # Sm MATHEMATICAL BOLD NABLA +1D6C2..1D6DA ; Math # L& [25] MATHEMATICAL BOLD SMALL ALPHA..MATHEMATICAL BOLD SMALL OMEGA +1D6DB ; Math # Sm MATHEMATICAL BOLD PARTIAL DIFFERENTIAL +1D6DC..1D6FA ; Math # L& [31] MATHEMATICAL BOLD EPSILON SYMBOL..MATHEMATICAL ITALIC CAPITAL OMEGA +1D6FB ; Math # Sm MATHEMATICAL ITALIC NABLA +1D6FC..1D714 ; Math # L& [25] MATHEMATICAL ITALIC SMALL ALPHA..MATHEMATICAL ITALIC SMALL OMEGA +1D715 ; Math # Sm MATHEMATICAL ITALIC PARTIAL DIFFERENTIAL +1D716..1D734 ; Math # L& [31] MATHEMATICAL ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD ITALIC CAPITAL OMEGA +1D735 ; Math # Sm MATHEMATICAL BOLD ITALIC NABLA +1D736..1D74E ; Math # L& [25] MATHEMATICAL BOLD ITALIC SMALL ALPHA..MATHEMATICAL BOLD ITALIC SMALL OMEGA +1D74F ; Math # Sm MATHEMATICAL BOLD ITALIC PARTIAL DIFFERENTIAL +1D750..1D76E ; Math # L& [31] MATHEMATICAL BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD CAPITAL OMEGA +1D76F ; Math # Sm MATHEMATICAL SANS-SERIF BOLD NABLA +1D770..1D788 ; Math # L& [25] MATHEMATICAL SANS-SERIF BOLD SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD SMALL OMEGA +1D789 ; Math # Sm MATHEMATICAL SANS-SERIF BOLD PARTIAL DIFFERENTIAL +1D78A..1D7A8 ; Math # L& [31] MATHEMATICAL SANS-SERIF BOLD EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL OMEGA +1D7A9 ; Math # Sm MATHEMATICAL SANS-SERIF BOLD ITALIC NABLA +1D7AA..1D7C2 ; Math # L& [25] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL OMEGA +1D7C3 ; Math # Sm MATHEMATICAL SANS-SERIF BOLD ITALIC PARTIAL DIFFERENTIAL +1D7C4..1D7C9 ; Math # L& [6] MATHEMATICAL SANS-SERIF BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC PI SYMBOL +1D7CE..1D7FF ; Math # Nd [50] MATHEMATICAL BOLD DIGIT ZERO..MATHEMATICAL MONOSPACE DIGIT NINE + +# Total code points: 1965 + +# ================================================ + +# Derived Property: Alphabetic +# Generated from: Lu+Ll+Lt+Lm+Lo+Nl + Other_Alphabetic + +0041..005A ; Alphabetic # L& [26] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER Z +0061..007A ; Alphabetic # L& [26] LATIN SMALL LETTER A..LATIN SMALL LETTER Z +00AA ; Alphabetic # L& FEMININE ORDINAL INDICATOR +00B5 ; Alphabetic # L& MICRO SIGN +00BA ; Alphabetic # L& MASCULINE ORDINAL INDICATOR +00C0..00D6 ; Alphabetic # L& [23] LATIN CAPITAL LETTER A WITH GRAVE..LATIN CAPITAL LETTER O WITH DIAERESIS +00D8..00F6 ; Alphabetic # L& [31] LATIN CAPITAL LETTER O WITH STROKE..LATIN SMALL LETTER O WITH DIAERESIS +00F8..01BA ; Alphabetic # L& [195] LATIN SMALL LETTER O WITH STROKE..LATIN SMALL LETTER EZH WITH TAIL +01BB ; Alphabetic # Lo LATIN LETTER TWO WITH STROKE +01BC..01BF ; Alphabetic # L& [4] LATIN CAPITAL LETTER TONE FIVE..LATIN LETTER WYNN +01C0..01C3 ; Alphabetic # Lo [4] LATIN LETTER DENTAL CLICK..LATIN LETTER RETROFLEX CLICK +01C4..0220 ; Alphabetic # L& [93] LATIN CAPITAL LETTER DZ WITH CARON..LATIN CAPITAL LETTER N WITH LONG RIGHT LEG +0222..0233 ; Alphabetic # L& [18] LATIN CAPITAL LETTER OU..LATIN SMALL LETTER Y WITH MACRON +0250..02AD ; Alphabetic # L& [94] LATIN SMALL LETTER TURNED A..LATIN LETTER BIDENTAL PERCUSSIVE +02B0..02B8 ; Alphabetic # Lm [9] MODIFIER LETTER SMALL H..MODIFIER LETTER SMALL Y +02BB..02C1 ; Alphabetic # Lm [7] MODIFIER LETTER TURNED COMMA..MODIFIER LETTER REVERSED GLOTTAL STOP +02D0..02D1 ; Alphabetic # Lm [2] MODIFIER LETTER TRIANGULAR COLON..MODIFIER LETTER HALF TRIANGULAR COLON +02E0..02E4 ; Alphabetic # Lm [5] MODIFIER LETTER SMALL GAMMA..MODIFIER LETTER SMALL REVERSED GLOTTAL STOP +02EE ; Alphabetic # Lm MODIFIER LETTER DOUBLE APOSTROPHE +0345 ; Alphabetic # Mn COMBINING GREEK YPOGEGRAMMENI +037A ; Alphabetic # Lm GREEK YPOGEGRAMMENI +0386 ; Alphabetic # L& GREEK CAPITAL LETTER ALPHA WITH TONOS +0388..038A ; Alphabetic # L& [3] GREEK CAPITAL LETTER EPSILON WITH TONOS..GREEK CAPITAL LETTER IOTA WITH TONOS +038C ; Alphabetic # L& GREEK CAPITAL LETTER OMICRON WITH TONOS +038E..03A1 ; Alphabetic # L& [20] GREEK CAPITAL LETTER UPSILON WITH TONOS..GREEK CAPITAL LETTER RHO +03A3..03CE ; Alphabetic # L& [44] GREEK CAPITAL LETTER SIGMA..GREEK SMALL LETTER OMEGA WITH TONOS +03D0..03F5 ; Alphabetic # L& [38] GREEK BETA SYMBOL..GREEK LUNATE EPSILON SYMBOL +0400..0481 ; Alphabetic # L& [130] CYRILLIC CAPITAL LETTER IE WITH GRAVE..CYRILLIC SMALL LETTER KOPPA +048A..04CE ; Alphabetic # L& [69] CYRILLIC CAPITAL LETTER SHORT I WITH TAIL..CYRILLIC SMALL LETTER EM WITH TAIL +04D0..04F5 ; Alphabetic # L& [38] CYRILLIC CAPITAL LETTER A WITH BREVE..CYRILLIC SMALL LETTER CHE WITH DIAERESIS +04F8..04F9 ; Alphabetic # L& [2] CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS..CYRILLIC SMALL LETTER YERU WITH DIAERESIS +0500..050F ; Alphabetic # L& [16] CYRILLIC CAPITAL LETTER KOMI DE..CYRILLIC SMALL LETTER KOMI TJE +0531..0556 ; Alphabetic # L& [38] ARMENIAN CAPITAL LETTER AYB..ARMENIAN CAPITAL LETTER FEH +0559 ; Alphabetic # Lm ARMENIAN MODIFIER LETTER LEFT HALF RING +0561..0587 ; Alphabetic # L& [39] ARMENIAN SMALL LETTER AYB..ARMENIAN SMALL LIGATURE ECH YIWN +05B0..05B9 ; Alphabetic # Mn [10] HEBREW POINT SHEVA..HEBREW POINT HOLAM +05BB..05BD ; Alphabetic # Mn [3] HEBREW POINT QUBUTS..HEBREW POINT METEG +05BF ; Alphabetic # Mn HEBREW POINT RAFE +05C1..05C2 ; Alphabetic # Mn [2] HEBREW POINT SHIN DOT..HEBREW POINT SIN DOT +05C4 ; Alphabetic # Mn HEBREW MARK UPPER DOT +05D0..05EA ; Alphabetic # Lo [27] HEBREW LETTER ALEF..HEBREW LETTER TAV +05F0..05F2 ; Alphabetic # Lo [3] HEBREW LIGATURE YIDDISH DOUBLE VAV..HEBREW LIGATURE YIDDISH DOUBLE YOD +0621..063A ; Alphabetic # Lo [26] ARABIC LETTER HAMZA..ARABIC LETTER GHAIN +0640 ; Alphabetic # Lm ARABIC TATWEEL +0641..064A ; Alphabetic # Lo [10] ARABIC LETTER FEH..ARABIC LETTER YEH +064B..0655 ; Alphabetic # Mn [11] ARABIC FATHATAN..ARABIC HAMZA BELOW +066E..066F ; Alphabetic # Lo [2] ARABIC LETTER DOTLESS BEH..ARABIC LETTER DOTLESS QAF +0670 ; Alphabetic # Mn ARABIC LETTER SUPERSCRIPT ALEF +0671..06D3 ; Alphabetic # Lo [99] ARABIC LETTER ALEF WASLA..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE +06D5 ; Alphabetic # Lo ARABIC LETTER AE +06D6..06DC ; Alphabetic # Mn [7] ARABIC SMALL HIGH LIGATURE SAD WITH LAM WITH ALEF MAKSURA..ARABIC SMALL HIGH SEEN +06E1..06E4 ; Alphabetic # Mn [4] ARABIC SMALL HIGH DOTLESS HEAD OF KHAH..ARABIC SMALL HIGH MADDA +06E5..06E6 ; Alphabetic # Lm [2] ARABIC SMALL WAW..ARABIC SMALL YEH +06E7..06E8 ; Alphabetic # Mn [2] ARABIC SMALL HIGH YEH..ARABIC SMALL HIGH NOON +06ED ; Alphabetic # Mn ARABIC SMALL LOW MEEM +06FA..06FC ; Alphabetic # Lo [3] ARABIC LETTER SHEEN WITH DOT BELOW..ARABIC LETTER GHAIN WITH DOT BELOW +0710 ; Alphabetic # Lo SYRIAC LETTER ALAPH +0711 ; Alphabetic # Mn SYRIAC LETTER SUPERSCRIPT ALAPH +0712..072C ; Alphabetic # Lo [27] SYRIAC LETTER BETH..SYRIAC LETTER TAW +0730..073F ; Alphabetic # Mn [16] SYRIAC PTHAHA ABOVE..SYRIAC RWAHA +0780..07A5 ; Alphabetic # Lo [38] THAANA LETTER HAA..THAANA LETTER WAAVU +07A6..07B0 ; Alphabetic # Mn [11] THAANA ABAFILI..THAANA SUKUN +07B1 ; Alphabetic # Lo THAANA LETTER NAA +0901..0902 ; Alphabetic # Mn [2] DEVANAGARI SIGN CANDRABINDU..DEVANAGARI SIGN ANUSVARA +0903 ; Alphabetic # Mc DEVANAGARI SIGN VISARGA +0905..0939 ; Alphabetic # Lo [53] DEVANAGARI LETTER A..DEVANAGARI LETTER HA +093D ; Alphabetic # Lo DEVANAGARI SIGN AVAGRAHA +093E..0940 ; Alphabetic # Mc [3] DEVANAGARI VOWEL SIGN AA..DEVANAGARI VOWEL SIGN II +0941..0948 ; Alphabetic # Mn [8] DEVANAGARI VOWEL SIGN U..DEVANAGARI VOWEL SIGN AI +0949..094C ; Alphabetic # Mc [4] DEVANAGARI VOWEL SIGN CANDRA O..DEVANAGARI VOWEL SIGN AU +0950 ; Alphabetic # Lo DEVANAGARI OM +0958..0961 ; Alphabetic # Lo [10] DEVANAGARI LETTER QA..DEVANAGARI LETTER VOCALIC LL +0962..0963 ; Alphabetic # Mn [2] DEVANAGARI VOWEL SIGN VOCALIC L..DEVANAGARI VOWEL SIGN VOCALIC LL +0981 ; Alphabetic # Mn BENGALI SIGN CANDRABINDU +0982..0983 ; Alphabetic # Mc [2] BENGALI SIGN ANUSVARA..BENGALI SIGN VISARGA +0985..098C ; Alphabetic # Lo [8] BENGALI LETTER A..BENGALI LETTER VOCALIC L +098F..0990 ; Alphabetic # Lo [2] BENGALI LETTER E..BENGALI LETTER AI +0993..09A8 ; Alphabetic # Lo [22] BENGALI LETTER O..BENGALI LETTER NA +09AA..09B0 ; Alphabetic # Lo [7] BENGALI LETTER PA..BENGALI LETTER RA +09B2 ; Alphabetic # Lo BENGALI LETTER LA +09B6..09B9 ; Alphabetic # Lo [4] BENGALI LETTER SHA..BENGALI LETTER HA +09BE..09C0 ; Alphabetic # Mc [3] BENGALI VOWEL SIGN AA..BENGALI VOWEL SIGN II +09C1..09C4 ; Alphabetic # Mn [4] BENGALI VOWEL SIGN U..BENGALI VOWEL SIGN VOCALIC RR +09C7..09C8 ; Alphabetic # Mc [2] BENGALI VOWEL SIGN E..BENGALI VOWEL SIGN AI +09CB..09CC ; Alphabetic # Mc [2] BENGALI VOWEL SIGN O..BENGALI VOWEL SIGN AU +09D7 ; Alphabetic # Mc BENGALI AU LENGTH MARK +09DC..09DD ; Alphabetic # Lo [2] BENGALI LETTER RRA..BENGALI LETTER RHA +09DF..09E1 ; Alphabetic # Lo [3] BENGALI LETTER YYA..BENGALI LETTER VOCALIC LL +09E2..09E3 ; Alphabetic # Mn [2] BENGALI VOWEL SIGN VOCALIC L..BENGALI VOWEL SIGN VOCALIC LL +09F0..09F1 ; Alphabetic # Lo [2] BENGALI LETTER RA WITH MIDDLE DIAGONAL..BENGALI LETTER RA WITH LOWER DIAGONAL +0A02 ; Alphabetic # Mn GURMUKHI SIGN BINDI +0A05..0A0A ; Alphabetic # Lo [6] GURMUKHI LETTER A..GURMUKHI LETTER UU +0A0F..0A10 ; Alphabetic # Lo [2] GURMUKHI LETTER EE..GURMUKHI LETTER AI +0A13..0A28 ; Alphabetic # Lo [22] GURMUKHI LETTER OO..GURMUKHI LETTER NA +0A2A..0A30 ; Alphabetic # Lo [7] GURMUKHI LETTER PA..GURMUKHI LETTER RA +0A32..0A33 ; Alphabetic # Lo [2] GURMUKHI LETTER LA..GURMUKHI LETTER LLA +0A35..0A36 ; Alphabetic # Lo [2] GURMUKHI LETTER VA..GURMUKHI LETTER SHA +0A38..0A39 ; Alphabetic # Lo [2] GURMUKHI LETTER SA..GURMUKHI LETTER HA +0A3E..0A40 ; Alphabetic # Mc [3] GURMUKHI VOWEL SIGN AA..GURMUKHI VOWEL SIGN II +0A41..0A42 ; Alphabetic # Mn [2] GURMUKHI VOWEL SIGN U..GURMUKHI VOWEL SIGN UU +0A47..0A48 ; Alphabetic # Mn [2] GURMUKHI VOWEL SIGN EE..GURMUKHI VOWEL SIGN AI +0A4B..0A4C ; Alphabetic # Mn [2] GURMUKHI VOWEL SIGN OO..GURMUKHI VOWEL SIGN AU +0A59..0A5C ; Alphabetic # Lo [4] GURMUKHI LETTER KHHA..GURMUKHI LETTER RRA +0A5E ; Alphabetic # Lo GURMUKHI LETTER FA +0A70..0A71 ; Alphabetic # Mn [2] GURMUKHI TIPPI..GURMUKHI ADDAK +0A72..0A74 ; Alphabetic # Lo [3] GURMUKHI IRI..GURMUKHI EK ONKAR +0A81..0A82 ; Alphabetic # Mn [2] GUJARATI SIGN CANDRABINDU..GUJARATI SIGN ANUSVARA +0A83 ; Alphabetic # Mc GUJARATI SIGN VISARGA +0A85..0A8B ; Alphabetic # Lo [7] GUJARATI LETTER A..GUJARATI LETTER VOCALIC R +0A8D ; Alphabetic # Lo GUJARATI VOWEL CANDRA E +0A8F..0A91 ; Alphabetic # Lo [3] GUJARATI LETTER E..GUJARATI VOWEL CANDRA O +0A93..0AA8 ; Alphabetic # Lo [22] GUJARATI LETTER O..GUJARATI LETTER NA +0AAA..0AB0 ; Alphabetic # Lo [7] GUJARATI LETTER PA..GUJARATI LETTER RA +0AB2..0AB3 ; Alphabetic # Lo [2] GUJARATI LETTER LA..GUJARATI LETTER LLA +0AB5..0AB9 ; Alphabetic # Lo [5] GUJARATI LETTER VA..GUJARATI LETTER HA +0ABD ; Alphabetic # Lo GUJARATI SIGN AVAGRAHA +0ABE..0AC0 ; Alphabetic # Mc [3] GUJARATI VOWEL SIGN AA..GUJARATI VOWEL SIGN II +0AC1..0AC5 ; Alphabetic # Mn [5] GUJARATI VOWEL SIGN U..GUJARATI VOWEL SIGN CANDRA E +0AC7..0AC8 ; Alphabetic # Mn [2] GUJARATI VOWEL SIGN E..GUJARATI VOWEL SIGN AI +0AC9 ; Alphabetic # Mc GUJARATI VOWEL SIGN CANDRA O +0ACB..0ACC ; Alphabetic # Mc [2] GUJARATI VOWEL SIGN O..GUJARATI VOWEL SIGN AU +0AD0 ; Alphabetic # Lo GUJARATI OM +0AE0 ; Alphabetic # Lo GUJARATI LETTER VOCALIC RR +0B01 ; Alphabetic # Mn ORIYA SIGN CANDRABINDU +0B02..0B03 ; Alphabetic # Mc [2] ORIYA SIGN ANUSVARA..ORIYA SIGN VISARGA +0B05..0B0C ; Alphabetic # Lo [8] ORIYA LETTER A..ORIYA LETTER VOCALIC L +0B0F..0B10 ; Alphabetic # Lo [2] ORIYA LETTER E..ORIYA LETTER AI +0B13..0B28 ; Alphabetic # Lo [22] ORIYA LETTER O..ORIYA LETTER NA +0B2A..0B30 ; Alphabetic # Lo [7] ORIYA LETTER PA..ORIYA LETTER RA +0B32..0B33 ; Alphabetic # Lo [2] ORIYA LETTER LA..ORIYA LETTER LLA +0B36..0B39 ; Alphabetic # Lo [4] ORIYA LETTER SHA..ORIYA LETTER HA +0B3D ; Alphabetic # Lo ORIYA SIGN AVAGRAHA +0B3E ; Alphabetic # Mc ORIYA VOWEL SIGN AA +0B3F ; Alphabetic # Mn ORIYA VOWEL SIGN I +0B40 ; Alphabetic # Mc ORIYA VOWEL SIGN II +0B41..0B43 ; Alphabetic # Mn [3] ORIYA VOWEL SIGN U..ORIYA VOWEL SIGN VOCALIC R +0B47..0B48 ; Alphabetic # Mc [2] ORIYA VOWEL SIGN E..ORIYA VOWEL SIGN AI +0B4B..0B4C ; Alphabetic # Mc [2] ORIYA VOWEL SIGN O..ORIYA VOWEL SIGN AU +0B56 ; Alphabetic # Mn ORIYA AI LENGTH MARK +0B57 ; Alphabetic # Mc ORIYA AU LENGTH MARK +0B5C..0B5D ; Alphabetic # Lo [2] ORIYA LETTER RRA..ORIYA LETTER RHA +0B5F..0B61 ; Alphabetic # Lo [3] ORIYA LETTER YYA..ORIYA LETTER VOCALIC LL +0B82 ; Alphabetic # Mn TAMIL SIGN ANUSVARA +0B83 ; Alphabetic # Lo TAMIL SIGN VISARGA +0B85..0B8A ; Alphabetic # Lo [6] TAMIL LETTER A..TAMIL LETTER UU +0B8E..0B90 ; Alphabetic # Lo [3] TAMIL LETTER E..TAMIL LETTER AI +0B92..0B95 ; Alphabetic # Lo [4] TAMIL LETTER O..TAMIL LETTER KA +0B99..0B9A ; Alphabetic # Lo [2] TAMIL LETTER NGA..TAMIL LETTER CA +0B9C ; Alphabetic # Lo TAMIL LETTER JA +0B9E..0B9F ; Alphabetic # Lo [2] TAMIL LETTER NYA..TAMIL LETTER TTA +0BA3..0BA4 ; Alphabetic # Lo [2] TAMIL LETTER NNA..TAMIL LETTER TA +0BA8..0BAA ; Alphabetic # Lo [3] TAMIL LETTER NA..TAMIL LETTER PA +0BAE..0BB5 ; Alphabetic # Lo [8] TAMIL LETTER MA..TAMIL LETTER VA +0BB7..0BB9 ; Alphabetic # Lo [3] TAMIL LETTER SSA..TAMIL LETTER HA +0BBE..0BBF ; Alphabetic # Mc [2] TAMIL VOWEL SIGN AA..TAMIL VOWEL SIGN I +0BC0 ; Alphabetic # Mn TAMIL VOWEL SIGN II +0BC1..0BC2 ; Alphabetic # Mc [2] TAMIL VOWEL SIGN U..TAMIL VOWEL SIGN UU +0BC6..0BC8 ; Alphabetic # Mc [3] TAMIL VOWEL SIGN E..TAMIL VOWEL SIGN AI +0BCA..0BCC ; Alphabetic # Mc [3] TAMIL VOWEL SIGN O..TAMIL VOWEL SIGN AU +0BD7 ; Alphabetic # Mc TAMIL AU LENGTH MARK +0C01..0C03 ; Alphabetic # Mc [3] TELUGU SIGN CANDRABINDU..TELUGU SIGN VISARGA +0C05..0C0C ; Alphabetic # Lo [8] TELUGU LETTER A..TELUGU LETTER VOCALIC L +0C0E..0C10 ; Alphabetic # Lo [3] TELUGU LETTER E..TELUGU LETTER AI +0C12..0C28 ; Alphabetic # Lo [23] TELUGU LETTER O..TELUGU LETTER NA +0C2A..0C33 ; Alphabetic # Lo [10] TELUGU LETTER PA..TELUGU LETTER LLA +0C35..0C39 ; Alphabetic # Lo [5] TELUGU LETTER VA..TELUGU LETTER HA +0C3E..0C40 ; Alphabetic # Mn [3] TELUGU VOWEL SIGN AA..TELUGU VOWEL SIGN II +0C41..0C44 ; Alphabetic # Mc [4] TELUGU VOWEL SIGN U..TELUGU VOWEL SIGN VOCALIC RR +0C46..0C48 ; Alphabetic # Mn [3] TELUGU VOWEL SIGN E..TELUGU VOWEL SIGN AI +0C4A..0C4C ; Alphabetic # Mn [3] TELUGU VOWEL SIGN O..TELUGU VOWEL SIGN AU +0C55..0C56 ; Alphabetic # Mn [2] TELUGU LENGTH MARK..TELUGU AI LENGTH MARK +0C60..0C61 ; Alphabetic # Lo [2] TELUGU LETTER VOCALIC RR..TELUGU LETTER VOCALIC LL +0C82..0C83 ; Alphabetic # Mc [2] KANNADA SIGN ANUSVARA..KANNADA SIGN VISARGA +0C85..0C8C ; Alphabetic # Lo [8] KANNADA LETTER A..KANNADA LETTER VOCALIC L +0C8E..0C90 ; Alphabetic # Lo [3] KANNADA LETTER E..KANNADA LETTER AI +0C92..0CA8 ; Alphabetic # Lo [23] KANNADA LETTER O..KANNADA LETTER NA +0CAA..0CB3 ; Alphabetic # Lo [10] KANNADA LETTER PA..KANNADA LETTER LLA +0CB5..0CB9 ; Alphabetic # Lo [5] KANNADA LETTER VA..KANNADA LETTER HA +0CBE ; Alphabetic # Mc KANNADA VOWEL SIGN AA +0CBF ; Alphabetic # Mn KANNADA VOWEL SIGN I +0CC0..0CC4 ; Alphabetic # Mc [5] KANNADA VOWEL SIGN II..KANNADA VOWEL SIGN VOCALIC RR +0CC6 ; Alphabetic # Mn KANNADA VOWEL SIGN E +0CC7..0CC8 ; Alphabetic # Mc [2] KANNADA VOWEL SIGN EE..KANNADA VOWEL SIGN AI +0CCA..0CCB ; Alphabetic # Mc [2] KANNADA VOWEL SIGN O..KANNADA VOWEL SIGN OO +0CCC ; Alphabetic # Mn KANNADA VOWEL SIGN AU +0CD5..0CD6 ; Alphabetic # Mc [2] KANNADA LENGTH MARK..KANNADA AI LENGTH MARK +0CDE ; Alphabetic # Lo KANNADA LETTER FA +0CE0..0CE1 ; Alphabetic # Lo [2] KANNADA LETTER VOCALIC RR..KANNADA LETTER VOCALIC LL +0D02..0D03 ; Alphabetic # Mc [2] MALAYALAM SIGN ANUSVARA..MALAYALAM SIGN VISARGA +0D05..0D0C ; Alphabetic # Lo [8] MALAYALAM LETTER A..MALAYALAM LETTER VOCALIC L +0D0E..0D10 ; Alphabetic # Lo [3] MALAYALAM LETTER E..MALAYALAM LETTER AI +0D12..0D28 ; Alphabetic # Lo [23] MALAYALAM LETTER O..MALAYALAM LETTER NA +0D2A..0D39 ; Alphabetic # Lo [16] MALAYALAM LETTER PA..MALAYALAM LETTER HA +0D3E..0D40 ; Alphabetic # Mc [3] MALAYALAM VOWEL SIGN AA..MALAYALAM VOWEL SIGN II +0D41..0D43 ; Alphabetic # Mn [3] MALAYALAM VOWEL SIGN U..MALAYALAM VOWEL SIGN VOCALIC R +0D46..0D48 ; Alphabetic # Mc [3] MALAYALAM VOWEL SIGN E..MALAYALAM VOWEL SIGN AI +0D4A..0D4C ; Alphabetic # Mc [3] MALAYALAM VOWEL SIGN O..MALAYALAM VOWEL SIGN AU +0D57 ; Alphabetic # Mc MALAYALAM AU LENGTH MARK +0D60..0D61 ; Alphabetic # Lo [2] MALAYALAM LETTER VOCALIC RR..MALAYALAM LETTER VOCALIC LL +0D82..0D83 ; Alphabetic # Mc [2] SINHALA SIGN ANUSVARAYA..SINHALA SIGN VISARGAYA +0D85..0D96 ; Alphabetic # Lo [18] SINHALA LETTER AYANNA..SINHALA LETTER AUYANNA +0D9A..0DB1 ; Alphabetic # Lo [24] SINHALA LETTER ALPAPRAANA KAYANNA..SINHALA LETTER DANTAJA NAYANNA +0DB3..0DBB ; Alphabetic # Lo [9] SINHALA LETTER SANYAKA DAYANNA..SINHALA LETTER RAYANNA +0DBD ; Alphabetic # Lo SINHALA LETTER DANTAJA LAYANNA +0DC0..0DC6 ; Alphabetic # Lo [7] SINHALA LETTER VAYANNA..SINHALA LETTER FAYANNA +0DCF..0DD1 ; Alphabetic # Mc [3] SINHALA VOWEL SIGN AELA-PILLA..SINHALA VOWEL SIGN DIGA AEDA-PILLA +0DD2..0DD4 ; Alphabetic # Mn [3] SINHALA VOWEL SIGN KETTI IS-PILLA..SINHALA VOWEL SIGN KETTI PAA-PILLA +0DD6 ; Alphabetic # Mn SINHALA VOWEL SIGN DIGA PAA-PILLA +0DD8..0DDF ; Alphabetic # Mc [8] SINHALA VOWEL SIGN GAETTA-PILLA..SINHALA VOWEL SIGN GAYANUKITTA +0DF2..0DF3 ; Alphabetic # Mc [2] SINHALA VOWEL SIGN DIGA GAETTA-PILLA..SINHALA VOWEL SIGN DIGA GAYANUKITTA +0E01..0E30 ; Alphabetic # Lo [48] THAI CHARACTER KO KAI..THAI CHARACTER SARA A +0E31 ; Alphabetic # Mn THAI CHARACTER MAI HAN-AKAT +0E32..0E33 ; Alphabetic # Lo [2] THAI CHARACTER SARA AA..THAI CHARACTER SARA AM +0E34..0E3A ; Alphabetic # Mn [7] THAI CHARACTER SARA I..THAI CHARACTER PHINTHU +0E40..0E45 ; Alphabetic # Lo [6] THAI CHARACTER SARA E..THAI CHARACTER LAKKHANGYAO +0E46 ; Alphabetic # Lm THAI CHARACTER MAIYAMOK +0E4D ; Alphabetic # Mn THAI CHARACTER NIKHAHIT +0E81..0E82 ; Alphabetic # Lo [2] LAO LETTER KO..LAO LETTER KHO SUNG +0E84 ; Alphabetic # Lo LAO LETTER KHO TAM +0E87..0E88 ; Alphabetic # Lo [2] LAO LETTER NGO..LAO LETTER CO +0E8A ; Alphabetic # Lo LAO LETTER SO TAM +0E8D ; Alphabetic # Lo LAO LETTER NYO +0E94..0E97 ; Alphabetic # Lo [4] LAO LETTER DO..LAO LETTER THO TAM +0E99..0E9F ; Alphabetic # Lo [7] LAO LETTER NO..LAO LETTER FO SUNG +0EA1..0EA3 ; Alphabetic # Lo [3] LAO LETTER MO..LAO LETTER LO LING +0EA5 ; Alphabetic # Lo LAO LETTER LO LOOT +0EA7 ; Alphabetic # Lo LAO LETTER WO +0EAA..0EAB ; Alphabetic # Lo [2] LAO LETTER SO SUNG..LAO LETTER HO SUNG +0EAD..0EB0 ; Alphabetic # Lo [4] LAO LETTER O..LAO VOWEL SIGN A +0EB1 ; Alphabetic # Mn LAO VOWEL SIGN MAI KAN +0EB2..0EB3 ; Alphabetic # Lo [2] LAO VOWEL SIGN AA..LAO VOWEL SIGN AM +0EB4..0EB9 ; Alphabetic # Mn [6] LAO VOWEL SIGN I..LAO VOWEL SIGN UU +0EBB..0EBC ; Alphabetic # Mn [2] LAO VOWEL SIGN MAI KON..LAO SEMIVOWEL SIGN LO +0EBD ; Alphabetic # Lo LAO SEMIVOWEL SIGN NYO +0EC0..0EC4 ; Alphabetic # Lo [5] LAO VOWEL SIGN E..LAO VOWEL SIGN AI +0EC6 ; Alphabetic # Lm LAO KO LA +0ECD ; Alphabetic # Mn LAO NIGGAHITA +0EDC..0EDD ; Alphabetic # Lo [2] LAO HO NO..LAO HO MO +0F00 ; Alphabetic # Lo TIBETAN SYLLABLE OM +0F40..0F47 ; Alphabetic # Lo [8] TIBETAN LETTER KA..TIBETAN LETTER JA +0F49..0F6A ; Alphabetic # Lo [34] TIBETAN LETTER NYA..TIBETAN LETTER FIXED-FORM RA +0F71..0F7E ; Alphabetic # Mn [14] TIBETAN VOWEL SIGN AA..TIBETAN SIGN RJES SU NGA RO +0F7F ; Alphabetic # Mc TIBETAN SIGN RNAM BCAD +0F80..0F81 ; Alphabetic # Mn [2] TIBETAN VOWEL SIGN REVERSED I..TIBETAN VOWEL SIGN REVERSED II +0F88..0F8B ; Alphabetic # Lo [4] TIBETAN SIGN LCE TSA CAN..TIBETAN SIGN GRU MED RGYINGS +0F90..0F97 ; Alphabetic # Mn [8] TIBETAN SUBJOINED LETTER KA..TIBETAN SUBJOINED LETTER JA +0F99..0FBC ; Alphabetic # Mn [36] TIBETAN SUBJOINED LETTER NYA..TIBETAN SUBJOINED LETTER FIXED-FORM RA +1000..1021 ; Alphabetic # Lo [34] MYANMAR LETTER KA..MYANMAR LETTER A +1023..1027 ; Alphabetic # Lo [5] MYANMAR LETTER I..MYANMAR LETTER E +1029..102A ; Alphabetic # Lo [2] MYANMAR LETTER O..MYANMAR LETTER AU +102C ; Alphabetic # Mc MYANMAR VOWEL SIGN AA +102D..1030 ; Alphabetic # Mn [4] MYANMAR VOWEL SIGN I..MYANMAR VOWEL SIGN UU +1031 ; Alphabetic # Mc MYANMAR VOWEL SIGN E +1032 ; Alphabetic # Mn MYANMAR VOWEL SIGN AI +1036 ; Alphabetic # Mn MYANMAR SIGN ANUSVARA +1038 ; Alphabetic # Mc MYANMAR SIGN VISARGA +1050..1055 ; Alphabetic # Lo [6] MYANMAR LETTER SHA..MYANMAR LETTER VOCALIC LL +1056..1057 ; Alphabetic # Mc [2] MYANMAR VOWEL SIGN VOCALIC R..MYANMAR VOWEL SIGN VOCALIC RR +1058..1059 ; Alphabetic # Mn [2] MYANMAR VOWEL SIGN VOCALIC L..MYANMAR VOWEL SIGN VOCALIC LL +10A0..10C5 ; Alphabetic # L& [38] GEORGIAN CAPITAL LETTER AN..GEORGIAN CAPITAL LETTER HOE +10D0..10F8 ; Alphabetic # Lo [41] GEORGIAN LETTER AN..GEORGIAN LETTER ELIFI +1100..1159 ; Alphabetic # Lo [90] HANGUL CHOSEONG KIYEOK..HANGUL CHOSEONG YEORINHIEUH +115F..11A2 ; Alphabetic # Lo [68] HANGUL CHOSEONG FILLER..HANGUL JUNGSEONG SSANGARAEA +11A8..11F9 ; Alphabetic # Lo [82] HANGUL JONGSEONG KIYEOK..HANGUL JONGSEONG YEORINHIEUH +1200..1206 ; Alphabetic # Lo [7] ETHIOPIC SYLLABLE HA..ETHIOPIC SYLLABLE HO +1208..1246 ; Alphabetic # Lo [63] ETHIOPIC SYLLABLE LA..ETHIOPIC SYLLABLE QO +1248 ; Alphabetic # Lo ETHIOPIC SYLLABLE QWA +124A..124D ; Alphabetic # Lo [4] ETHIOPIC SYLLABLE QWI..ETHIOPIC SYLLABLE QWE +1250..1256 ; Alphabetic # Lo [7] ETHIOPIC SYLLABLE QHA..ETHIOPIC SYLLABLE QHO +1258 ; Alphabetic # Lo ETHIOPIC SYLLABLE QHWA +125A..125D ; Alphabetic # Lo [4] ETHIOPIC SYLLABLE QHWI..ETHIOPIC SYLLABLE QHWE +1260..1286 ; Alphabetic # Lo [39] ETHIOPIC SYLLABLE BA..ETHIOPIC SYLLABLE XO +1288 ; Alphabetic # Lo ETHIOPIC SYLLABLE XWA +128A..128D ; Alphabetic # Lo [4] ETHIOPIC SYLLABLE XWI..ETHIOPIC SYLLABLE XWE +1290..12AE ; Alphabetic # Lo [31] ETHIOPIC SYLLABLE NA..ETHIOPIC SYLLABLE KO +12B0 ; Alphabetic # Lo ETHIOPIC SYLLABLE KWA +12B2..12B5 ; Alphabetic # Lo [4] ETHIOPIC SYLLABLE KWI..ETHIOPIC SYLLABLE KWE +12B8..12BE ; Alphabetic # Lo [7] ETHIOPIC SYLLABLE KXA..ETHIOPIC SYLLABLE KXO +12C0 ; Alphabetic # Lo ETHIOPIC SYLLABLE KXWA +12C2..12C5 ; Alphabetic # Lo [4] ETHIOPIC SYLLABLE KXWI..ETHIOPIC SYLLABLE KXWE +12C8..12CE ; Alphabetic # Lo [7] ETHIOPIC SYLLABLE WA..ETHIOPIC SYLLABLE WO +12D0..12D6 ; Alphabetic # Lo [7] ETHIOPIC SYLLABLE PHARYNGEAL A..ETHIOPIC SYLLABLE PHARYNGEAL O +12D8..12EE ; Alphabetic # Lo [23] ETHIOPIC SYLLABLE ZA..ETHIOPIC SYLLABLE YO +12F0..130E ; Alphabetic # Lo [31] ETHIOPIC SYLLABLE DA..ETHIOPIC SYLLABLE GO +1310 ; Alphabetic # Lo ETHIOPIC SYLLABLE GWA +1312..1315 ; Alphabetic # Lo [4] ETHIOPIC SYLLABLE GWI..ETHIOPIC SYLLABLE GWE +1318..131E ; Alphabetic # Lo [7] ETHIOPIC SYLLABLE GGA..ETHIOPIC SYLLABLE GGO +1320..1346 ; Alphabetic # Lo [39] ETHIOPIC SYLLABLE THA..ETHIOPIC SYLLABLE TZO +1348..135A ; Alphabetic # Lo [19] ETHIOPIC SYLLABLE FA..ETHIOPIC SYLLABLE FYA +13A0..13F4 ; Alphabetic # Lo [85] CHEROKEE LETTER A..CHEROKEE LETTER YV +1401..166C ; Alphabetic # Lo [620] CANADIAN SYLLABICS E..CANADIAN SYLLABICS CARRIER TTSA +166F..1676 ; Alphabetic # Lo [8] CANADIAN SYLLABICS QAI..CANADIAN SYLLABICS NNGAA +1681..169A ; Alphabetic # Lo [26] OGHAM LETTER BEITH..OGHAM LETTER PEITH +16A0..16EA ; Alphabetic # Lo [75] RUNIC LETTER FEHU FEOH FE F..RUNIC LETTER X +16EE..16F0 ; Alphabetic # Nl [3] RUNIC ARLAUG SYMBOL..RUNIC BELGTHOR SYMBOL +1700..170C ; Alphabetic # Lo [13] TAGALOG LETTER A..TAGALOG LETTER YA +170E..1711 ; Alphabetic # Lo [4] TAGALOG LETTER LA..TAGALOG LETTER HA +1712..1713 ; Alphabetic # Mn [2] TAGALOG VOWEL SIGN I..TAGALOG VOWEL SIGN U +1720..1731 ; Alphabetic # Lo [18] HANUNOO LETTER A..HANUNOO LETTER HA +1732..1733 ; Alphabetic # Mn [2] HANUNOO VOWEL SIGN I..HANUNOO VOWEL SIGN U +1740..1751 ; Alphabetic # Lo [18] BUHID LETTER A..BUHID LETTER HA +1752..1753 ; Alphabetic # Mn [2] BUHID VOWEL SIGN I..BUHID VOWEL SIGN U +1760..176C ; Alphabetic # Lo [13] TAGBANWA LETTER A..TAGBANWA LETTER YA +176E..1770 ; Alphabetic # Lo [3] TAGBANWA LETTER LA..TAGBANWA LETTER SA +1772..1773 ; Alphabetic # Mn [2] TAGBANWA VOWEL SIGN I..TAGBANWA VOWEL SIGN U +1780..17B3 ; Alphabetic # Lo [52] KHMER LETTER KA..KHMER INDEPENDENT VOWEL QAU +17B4..17B6 ; Alphabetic # Mc [3] KHMER VOWEL INHERENT AQ..KHMER VOWEL SIGN AA +17B7..17BD ; Alphabetic # Mn [7] KHMER VOWEL SIGN I..KHMER VOWEL SIGN UA +17BE..17C5 ; Alphabetic # Mc [8] KHMER VOWEL SIGN OE..KHMER VOWEL SIGN AU +17C6 ; Alphabetic # Mn KHMER SIGN NIKAHIT +17C7..17C8 ; Alphabetic # Mc [2] KHMER SIGN REAHMUK..KHMER SIGN YUUKALEAPINTU +17D7 ; Alphabetic # Lm KHMER SIGN LEK TOO +17DC ; Alphabetic # Lo KHMER SIGN AVAKRAHASANYA +1820..1842 ; Alphabetic # Lo [35] MONGOLIAN LETTER A..MONGOLIAN LETTER CHI +1843 ; Alphabetic # Lm MONGOLIAN LETTER TODO LONG VOWEL SIGN +1844..1877 ; Alphabetic # Lo [52] MONGOLIAN LETTER TODO E..MONGOLIAN LETTER MANCHU ZHA +1880..18A8 ; Alphabetic # Lo [41] MONGOLIAN LETTER ALI GALI ANUSVARA ONE..MONGOLIAN LETTER MANCHU ALI GALI BHA +18A9 ; Alphabetic # Mn MONGOLIAN LETTER ALI GALI DAGALGA +1E00..1E9B ; Alphabetic # L& [156] LATIN CAPITAL LETTER A WITH RING BELOW..LATIN SMALL LETTER LONG S WITH DOT ABOVE +1EA0..1EF9 ; Alphabetic # L& [90] LATIN CAPITAL LETTER A WITH DOT BELOW..LATIN SMALL LETTER Y WITH TILDE +1F00..1F15 ; Alphabetic # L& [22] GREEK SMALL LETTER ALPHA WITH PSILI..GREEK SMALL LETTER EPSILON WITH DASIA AND OXIA +1F18..1F1D ; Alphabetic # L& [6] GREEK CAPITAL LETTER EPSILON WITH PSILI..GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA +1F20..1F45 ; Alphabetic # L& [38] GREEK SMALL LETTER ETA WITH PSILI..GREEK SMALL LETTER OMICRON WITH DASIA AND OXIA +1F48..1F4D ; Alphabetic # L& [6] GREEK CAPITAL LETTER OMICRON WITH PSILI..GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA +1F50..1F57 ; Alphabetic # L& [8] GREEK SMALL LETTER UPSILON WITH PSILI..GREEK SMALL LETTER UPSILON WITH DASIA AND PERISPOMENI +1F59 ; Alphabetic # L& GREEK CAPITAL LETTER UPSILON WITH DASIA +1F5B ; Alphabetic # L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA +1F5D ; Alphabetic # L& GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA +1F5F..1F7D ; Alphabetic # L& [31] GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI..GREEK SMALL LETTER OMEGA WITH OXIA +1F80..1FB4 ; Alphabetic # L& [53] GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI..GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI +1FB6..1FBC ; Alphabetic # L& [7] GREEK SMALL LETTER ALPHA WITH PERISPOMENI..GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI +1FBE ; Alphabetic # L& GREEK PROSGEGRAMMENI +1FC2..1FC4 ; Alphabetic # L& [3] GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI +1FC6..1FCC ; Alphabetic # L& [7] GREEK SMALL LETTER ETA WITH PERISPOMENI..GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI +1FD0..1FD3 ; Alphabetic # L& [4] GREEK SMALL LETTER IOTA WITH VRACHY..GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA +1FD6..1FDB ; Alphabetic # L& [6] GREEK SMALL LETTER IOTA WITH PERISPOMENI..GREEK CAPITAL LETTER IOTA WITH OXIA +1FE0..1FEC ; Alphabetic # L& [13] GREEK SMALL LETTER UPSILON WITH VRACHY..GREEK CAPITAL LETTER RHO WITH DASIA +1FF2..1FF4 ; Alphabetic # L& [3] GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI..GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI +1FF6..1FFC ; Alphabetic # L& [7] GREEK SMALL LETTER OMEGA WITH PERISPOMENI..GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI +2071 ; Alphabetic # L& SUPERSCRIPT LATIN SMALL LETTER I +207F ; Alphabetic # L& SUPERSCRIPT LATIN SMALL LETTER N +2102 ; Alphabetic # L& DOUBLE-STRUCK CAPITAL C +2107 ; Alphabetic # L& EULER CONSTANT +210A..2113 ; Alphabetic # L& [10] SCRIPT SMALL G..SCRIPT SMALL L +2115 ; Alphabetic # L& DOUBLE-STRUCK CAPITAL N +2119..211D ; Alphabetic # L& [5] DOUBLE-STRUCK CAPITAL P..DOUBLE-STRUCK CAPITAL R +2124 ; Alphabetic # L& DOUBLE-STRUCK CAPITAL Z +2126 ; Alphabetic # L& OHM SIGN +2128 ; Alphabetic # L& BLACK-LETTER CAPITAL Z +212A..212D ; Alphabetic # L& [4] KELVIN SIGN..BLACK-LETTER CAPITAL C +212F..2131 ; Alphabetic # L& [3] SCRIPT SMALL E..SCRIPT CAPITAL F +2133..2134 ; Alphabetic # L& [2] SCRIPT CAPITAL M..SCRIPT SMALL O +2135..2138 ; Alphabetic # Lo [4] ALEF SYMBOL..DALET SYMBOL +2139 ; Alphabetic # L& INFORMATION SOURCE +213D..213F ; Alphabetic # L& [3] DOUBLE-STRUCK SMALL GAMMA..DOUBLE-STRUCK CAPITAL PI +2145..2149 ; Alphabetic # L& [5] DOUBLE-STRUCK ITALIC CAPITAL D..DOUBLE-STRUCK ITALIC SMALL J +2160..2183 ; Alphabetic # Nl [36] ROMAN NUMERAL ONE..ROMAN NUMERAL REVERSED ONE HUNDRED +3005 ; Alphabetic # Lm IDEOGRAPHIC ITERATION MARK +3006 ; Alphabetic # Lo IDEOGRAPHIC CLOSING MARK +3007 ; Alphabetic # Nl IDEOGRAPHIC NUMBER ZERO +3021..3029 ; Alphabetic # Nl [9] HANGZHOU NUMERAL ONE..HANGZHOU NUMERAL NINE +3031..3035 ; Alphabetic # Lm [5] VERTICAL KANA REPEAT MARK..VERTICAL KANA REPEAT MARK LOWER HALF +3038..303A ; Alphabetic # Nl [3] HANGZHOU NUMERAL TEN..HANGZHOU NUMERAL THIRTY +303B ; Alphabetic # Lm VERTICAL IDEOGRAPHIC ITERATION MARK +303C ; Alphabetic # Lo MASU MARK +3041..3096 ; Alphabetic # Lo [86] HIRAGANA LETTER SMALL A..HIRAGANA LETTER SMALL KE +309D..309E ; Alphabetic # Lm [2] HIRAGANA ITERATION MARK..HIRAGANA VOICED ITERATION MARK +309F ; Alphabetic # Lo HIRAGANA DIGRAPH YORI +30A1..30FA ; Alphabetic # Lo [90] KATAKANA LETTER SMALL A..KATAKANA LETTER VO +30FC..30FE ; Alphabetic # Lm [3] KATAKANA-HIRAGANA PROLONGED SOUND MARK..KATAKANA VOICED ITERATION MARK +30FF ; Alphabetic # Lo KATAKANA DIGRAPH KOTO +3105..312C ; Alphabetic # Lo [40] BOPOMOFO LETTER B..BOPOMOFO LETTER GN +3131..318E ; Alphabetic # Lo [94] HANGUL LETTER KIYEOK..HANGUL LETTER ARAEAE +31A0..31B7 ; Alphabetic # Lo [24] BOPOMOFO LETTER BU..BOPOMOFO FINAL LETTER H +31F0..31FF ; Alphabetic # Lo [16] KATAKANA LETTER SMALL KU..KATAKANA LETTER SMALL RO +3400..4DB5 ; Alphabetic # Lo [6582] CJK UNIFIED IDEOGRAPH-3400..CJK UNIFIED IDEOGRAPH-4DB5 +4E00..9FA5 ; Alphabetic # Lo [20902] CJK UNIFIED IDEOGRAPH-4E00..CJK UNIFIED IDEOGRAPH-9FA5 +A000..A48C ; Alphabetic # Lo [1165] YI SYLLABLE IT..YI SYLLABLE YYR +AC00..D7A3 ; Alphabetic # Lo [11172] HANGUL SYLLABLE GA..HANGUL SYLLABLE HIH +F900..FA2D ; Alphabetic # Lo [302] CJK COMPATIBILITY IDEOGRAPH-F900..CJK COMPATIBILITY IDEOGRAPH-FA2D +FA30..FA6A ; Alphabetic # Lo [59] CJK COMPATIBILITY IDEOGRAPH-FA30..CJK COMPATIBILITY IDEOGRAPH-FA6A +FB00..FB06 ; Alphabetic # L& [7] LATIN SMALL LIGATURE FF..LATIN SMALL LIGATURE ST +FB13..FB17 ; Alphabetic # L& [5] ARMENIAN SMALL LIGATURE MEN NOW..ARMENIAN SMALL LIGATURE MEN XEH +FB1D ; Alphabetic # Lo HEBREW LETTER YOD WITH HIRIQ +FB1E ; Alphabetic # Mn HEBREW POINT JUDEO-SPANISH VARIKA +FB1F..FB28 ; Alphabetic # Lo [10] HEBREW LIGATURE YIDDISH YOD YOD PATAH..HEBREW LETTER WIDE TAV +FB2A..FB36 ; Alphabetic # Lo [13] HEBREW LETTER SHIN WITH SHIN DOT..HEBREW LETTER ZAYIN WITH DAGESH +FB38..FB3C ; Alphabetic # Lo [5] HEBREW LETTER TET WITH DAGESH..HEBREW LETTER LAMED WITH DAGESH +FB3E ; Alphabetic # Lo HEBREW LETTER MEM WITH DAGESH +FB40..FB41 ; Alphabetic # Lo [2] HEBREW LETTER NUN WITH DAGESH..HEBREW LETTER SAMEKH WITH DAGESH +FB43..FB44 ; Alphabetic # Lo [2] HEBREW LETTER FINAL PE WITH DAGESH..HEBREW LETTER PE WITH DAGESH +FB46..FBB1 ; Alphabetic # Lo [108] HEBREW LETTER TSADI WITH DAGESH..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE FINAL FORM +FBD3..FD3D ; Alphabetic # Lo [363] ARABIC LETTER NG ISOLATED FORM..ARABIC LIGATURE ALEF WITH FATHATAN ISOLATED FORM +FD50..FD8F ; Alphabetic # Lo [64] ARABIC LIGATURE TEH WITH JEEM WITH MEEM INITIAL FORM..ARABIC LIGATURE MEEM WITH KHAH WITH MEEM INITIAL FORM +FD92..FDC7 ; Alphabetic # Lo [54] ARABIC LIGATURE MEEM WITH JEEM WITH KHAH INITIAL FORM..ARABIC LIGATURE NOON WITH JEEM WITH YEH FINAL FORM +FDF0..FDFB ; Alphabetic # Lo [12] ARABIC LIGATURE SALLA USED AS KORANIC STOP SIGN ISOLATED FORM..ARABIC LIGATURE JALLAJALALOUHOU +FE70..FE74 ; Alphabetic # Lo [5] ARABIC FATHATAN ISOLATED FORM..ARABIC KASRATAN ISOLATED FORM +FE76..FEFC ; Alphabetic # Lo [135] ARABIC FATHA ISOLATED FORM..ARABIC LIGATURE LAM WITH ALEF FINAL FORM +FF21..FF3A ; Alphabetic # L& [26] FULLWIDTH LATIN CAPITAL LETTER A..FULLWIDTH LATIN CAPITAL LETTER Z +FF41..FF5A ; Alphabetic # L& [26] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH LATIN SMALL LETTER Z +FF66..FF6F ; Alphabetic # Lo [10] HALFWIDTH KATAKANA LETTER WO..HALFWIDTH KATAKANA LETTER SMALL TU +FF70 ; Alphabetic # Lm HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK +FF71..FF9D ; Alphabetic # Lo [45] HALFWIDTH KATAKANA LETTER A..HALFWIDTH KATAKANA LETTER N +FF9E..FF9F ; Alphabetic # Lm [2] HALFWIDTH KATAKANA VOICED SOUND MARK..HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK +FFA0..FFBE ; Alphabetic # Lo [31] HALFWIDTH HANGUL FILLER..HALFWIDTH HANGUL LETTER HIEUH +FFC2..FFC7 ; Alphabetic # Lo [6] HALFWIDTH HANGUL LETTER A..HALFWIDTH HANGUL LETTER E +FFCA..FFCF ; Alphabetic # Lo [6] HALFWIDTH HANGUL LETTER YEO..HALFWIDTH HANGUL LETTER OE +FFD2..FFD7 ; Alphabetic # Lo [6] HALFWIDTH HANGUL LETTER YO..HALFWIDTH HANGUL LETTER YU +FFDA..FFDC ; Alphabetic # Lo [3] HALFWIDTH HANGUL LETTER EU..HALFWIDTH HANGUL LETTER I +10300..1031E ; Alphabetic # Lo [31] OLD ITALIC LETTER A..OLD ITALIC LETTER UU +10330..10349 ; Alphabetic # Lo [26] GOTHIC LETTER AHSA..GOTHIC LETTER OTHAL +1034A ; Alphabetic # Nl GOTHIC LETTER NINE HUNDRED +10400..10425 ; Alphabetic # L& [38] DESERET CAPITAL LETTER LONG I..DESERET CAPITAL LETTER ENG +10428..1044D ; Alphabetic # L& [38] DESERET SMALL LETTER LONG I..DESERET SMALL LETTER ENG +1D400..1D454 ; Alphabetic # L& [85] MATHEMATICAL BOLD CAPITAL A..MATHEMATICAL ITALIC SMALL G +1D456..1D49C ; Alphabetic # L& [71] MATHEMATICAL ITALIC SMALL I..MATHEMATICAL SCRIPT CAPITAL A +1D49E..1D49F ; Alphabetic # L& [2] MATHEMATICAL SCRIPT CAPITAL C..MATHEMATICAL SCRIPT CAPITAL D +1D4A2 ; Alphabetic # L& MATHEMATICAL SCRIPT CAPITAL G +1D4A5..1D4A6 ; Alphabetic # L& [2] MATHEMATICAL SCRIPT CAPITAL J..MATHEMATICAL SCRIPT CAPITAL K +1D4A9..1D4AC ; Alphabetic # L& [4] MATHEMATICAL SCRIPT CAPITAL N..MATHEMATICAL SCRIPT CAPITAL Q +1D4AE..1D4B9 ; Alphabetic # L& [12] MATHEMATICAL SCRIPT CAPITAL S..MATHEMATICAL SCRIPT SMALL D +1D4BB ; Alphabetic # L& MATHEMATICAL SCRIPT SMALL F +1D4BD..1D4C0 ; Alphabetic # L& [4] MATHEMATICAL SCRIPT SMALL H..MATHEMATICAL SCRIPT SMALL K +1D4C2..1D4C3 ; Alphabetic # L& [2] MATHEMATICAL SCRIPT SMALL M..MATHEMATICAL SCRIPT SMALL N +1D4C5..1D505 ; Alphabetic # L& [65] MATHEMATICAL SCRIPT SMALL P..MATHEMATICAL FRAKTUR CAPITAL B +1D507..1D50A ; Alphabetic # L& [4] MATHEMATICAL FRAKTUR CAPITAL D..MATHEMATICAL FRAKTUR CAPITAL G +1D50D..1D514 ; Alphabetic # L& [8] MATHEMATICAL FRAKTUR CAPITAL J..MATHEMATICAL FRAKTUR CAPITAL Q +1D516..1D51C ; Alphabetic # L& [7] MATHEMATICAL FRAKTUR CAPITAL S..MATHEMATICAL FRAKTUR CAPITAL Y +1D51E..1D539 ; Alphabetic # L& [28] MATHEMATICAL FRAKTUR SMALL A..MATHEMATICAL DOUBLE-STRUCK CAPITAL B +1D53B..1D53E ; Alphabetic # L& [4] MATHEMATICAL DOUBLE-STRUCK CAPITAL D..MATHEMATICAL DOUBLE-STRUCK CAPITAL G +1D540..1D544 ; Alphabetic # L& [5] MATHEMATICAL DOUBLE-STRUCK CAPITAL I..MATHEMATICAL DOUBLE-STRUCK CAPITAL M +1D546 ; Alphabetic # L& MATHEMATICAL DOUBLE-STRUCK CAPITAL O +1D54A..1D550 ; Alphabetic # L& [7] MATHEMATICAL DOUBLE-STRUCK CAPITAL S..MATHEMATICAL DOUBLE-STRUCK CAPITAL Y +1D552..1D6A3 ; Alphabetic # L& [338] MATHEMATICAL DOUBLE-STRUCK SMALL A..MATHEMATICAL MONOSPACE SMALL Z +1D6A8..1D6C0 ; Alphabetic # L& [25] MATHEMATICAL BOLD CAPITAL ALPHA..MATHEMATICAL BOLD CAPITAL OMEGA +1D6C2..1D6DA ; Alphabetic # L& [25] MATHEMATICAL BOLD SMALL ALPHA..MATHEMATICAL BOLD SMALL OMEGA +1D6DC..1D6FA ; Alphabetic # L& [31] MATHEMATICAL BOLD EPSILON SYMBOL..MATHEMATICAL ITALIC CAPITAL OMEGA +1D6FC..1D714 ; Alphabetic # L& [25] MATHEMATICAL ITALIC SMALL ALPHA..MATHEMATICAL ITALIC SMALL OMEGA +1D716..1D734 ; Alphabetic # L& [31] MATHEMATICAL ITALIC EPSILON SYMBOL..MATHEMATICAL BOLD ITALIC CAPITAL OMEGA +1D736..1D74E ; Alphabetic # L& [25] MATHEMATICAL BOLD ITALIC SMALL ALPHA..MATHEMATICAL BOLD ITALIC SMALL OMEGA +1D750..1D76E ; Alphabetic # L& [31] MATHEMATICAL BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD CAPITAL OMEGA +1D770..1D788 ; Alphabetic # L& [25] MATHEMATICAL SANS-SERIF BOLD SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD SMALL OMEGA +1D78A..1D7A8 ; Alphabetic # L& [31] MATHEMATICAL SANS-SERIF BOLD EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL OMEGA +1D7AA..1D7C2 ; Alphabetic # L& [25] MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ALPHA..MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL OMEGA +1D7C4..1D7C9 ; Alphabetic # L& [6] MATHEMATICAL SANS-SERIF BOLD ITALIC EPSILON SYMBOL..MATHEMATICAL SANS-SERIF BOLD ITALIC PI SYMBOL +20000..2A6D6 ; Alphabetic # Lo [42711] CJK UNIFIED IDEOGRAPH-20000..CJK UNIFIED IDEOGRAPH-2A6D6 +2F800..2FA1D ; Alphabetic # Lo [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D + +# Total code points: 90368 + +# ================================================ + +# Derived Property: Lowercase +# Generated from: Ll + Other_Lowercase + +0061..007A ; Lowercase # L& [26] LATIN SMALL LETTER A..LATIN SMALL LETTER Z +00AA ; Lowercase # L& FEMININE ORDINAL INDICATOR +00B5 ; Lowercase # L& MICRO SIGN +00BA ; Lowercase # L& MASCULINE ORDINAL INDICATOR +00DF..00F6 ; Lowercase # L& [24] LATIN SMALL LETTER SHARP S..LATIN SMALL LETTER O WITH DIAERESIS +00F8..00FF ; Lowercase # L& [8] LATIN SMALL LETTER O WITH STROKE..LATIN SMALL LETTER Y WITH DIAERESIS +0101 ; Lowercase # L& LATIN SMALL LETTER A WITH MACRON +0103 ; Lowercase # L& LATIN SMALL LETTER A WITH BREVE +0105 ; Lowercase # L& LATIN SMALL LETTER A WITH OGONEK +0107 ; Lowercase # L& LATIN SMALL LETTER C WITH ACUTE +0109 ; Lowercase # L& LATIN SMALL LETTER C WITH CIRCUMFLEX +010B ; Lowercase # L& LATIN SMALL LETTER C WITH DOT ABOVE +010D ; Lowercase # L& LATIN SMALL LETTER C WITH CARON +010F ; Lowercase # L& LATIN SMALL LETTER D WITH CARON +0111 ; Lowercase # L& LATIN SMALL LETTER D WITH STROKE +0113 ; Lowercase # L& LATIN SMALL LETTER E WITH MACRON +0115 ; Lowercase # L& LATIN SMALL LETTER E WITH BREVE +0117 ; Lowercase # L& LATIN SMALL LETTER E WITH DOT ABOVE +0119 ; Lowercase # L& LATIN SMALL LETTER E WITH OGONEK +011B ; Lowercase # L& LATIN SMALL LETTER E WITH CARON +011D ; Lowercase # L& LATIN SMALL LETTER G WITH CIRCUMFLEX +011F ; Lowercase # L& LATIN SMALL LETTER G WITH BREVE +0121 ; Lowercase # L& LATIN SMALL LETTER G WITH DOT ABOVE +0123 ; Lowercase # L& LATIN SMALL LETTER G WITH CEDILLA +0125 ; Lowercase # L& LATIN SMALL LETTER H WITH CIRCUMFLEX +0127 ; Lowercase # L& LATIN SMALL LETTER H WITH STROKE +0129 ; Lowercase # L& LATIN SMALL LETTER I WITH TILDE +012B ; Lowercase # L& LATIN SMALL LETTER I WITH MACRON +012D ; Lowercase # L& LATIN SMALL LETTER I WITH BREVE +012F ; Lowercase # L& LATIN SMALL LETTER I WITH OGONEK +0131 ; Lowercase # L& LATIN SMALL LETTER DOTLESS I +0133 ; Lowercase # L& LATIN SMALL LIGATURE IJ +0135 ; Lowercase # L& LATIN SMALL LETTER J WITH CIRCUMFLEX +0137..0138 ; Lowercase # L& [2] LATIN SMALL LETTER K WITH CEDILLA..LATIN SMALL LETTER KRA +013A ; Lowercase # L& LATIN SMALL LETTER L WITH ACUTE +013C ; Lowercase # L& LATIN SMALL LETTER L WITH CEDILLA +013E ; Lowercase # L& LATIN SMALL LETTER L WITH CARON +0140 ; Lowercase # L& LATIN SMALL LETTER L WITH MIDDLE DOT +0142 ; Lowercase # L& LATIN SMALL LETTER L WITH STROKE +0144 ; Lowercase # L& LATIN SMALL LETTER N WITH ACUTE +0146 ; Lowercase # L& LATIN SMALL LETTER N WITH CEDILLA +0148..0149 ; Lowercase # L& [2] LATIN SMALL LETTER N WITH CARON..LATIN SMALL LETTER N PRECEDED BY APOSTROPHE +014B ; Lowercase # L& LATIN SMALL LETTER ENG +014D ; Lowercase # L& LATIN SMALL LETTER O WITH MACRON +014F ; Lowercase # L& LATIN SMALL LETTER O WITH BREVE +0151 ; Lowercase # L& LATIN SMALL LETTER O WITH DOUBLE ACUTE +0153 ; Lowercase # L& LATIN SMALL LIGATURE OE +0155 ; Lowercase # L& LATIN SMALL LETTER R WITH ACUTE +0157 ; Lowercase # L& LATIN SMALL LETTER R WITH CEDILLA +0159 ; Lowercase # L& LATIN SMALL LETTER R WITH CARON +015B ; Lowercase # L& LATIN SMALL LETTER S WITH ACUTE +015D ; Lowercase # L& LATIN SMALL LETTER S WITH CIRCUMFLEX +015F ; Lowercase # L& LATIN SMALL LETTER S WITH CEDILLA +0161 ; Lowercase # L& LATIN SMALL LETTER S WITH CARON +0163 ; Lowercase # L& LATIN SMALL LETTER T WITH CEDILLA +0165 ; Lowercase # L& LATIN SMALL LETTER T WITH CARON +0167 ; Lowercase # L& LATIN SMALL LETTER T WITH STROKE +0169 ; Lowercase # L& LATIN SMALL LETTER U WITH TILDE +016B ; Lowercase # L& LATIN SMALL LETTER U WITH MACRON +016D ; Lowercase # L& LATIN SMALL LETTER U WITH BREVE +016F ; Lowercase # L& LATIN SMALL LETTER U WITH RING ABOVE +0171 ; Lowercase # L& LATIN SMALL LETTER U WITH DOUBLE ACUTE +0173 ; Lowercase # L& LATIN SMALL LETTER U WITH OGONEK +0175 ; Lowercase # L& LATIN SMALL LETTER W WITH CIRCUMFLEX +0177 ; Lowercase # L& LATIN SMALL LETTER Y WITH CIRCUMFLEX +017A ; Lowercase # L& LATIN SMALL LETTER Z WITH ACUTE +017C ; Lowercase # L& LATIN SMALL LETTER Z WITH DOT ABOVE +017E..0180 ; Lowercase # L& [3] LATIN SMALL LETTER Z WITH CARON..LATIN SMALL LETTER B WITH STROKE +0183 ; Lowercase # L& LATIN SMALL LETTER B WITH TOPBAR +0185 ; Lowercase # L& LATIN SMALL LETTER TONE SIX +0188 ; Lowercase # L& LATIN SMALL LETTER C WITH HOOK +018C..018D ; Lowercase # L& [2] LATIN SMALL LETTER D WITH TOPBAR..LATIN SMALL LETTER TURNED DELTA +0192 ; Lowercase # L& LATIN SMALL LETTER F WITH HOOK +0195 ; Lowercase # L& LATIN SMALL LETTER HV +0199..019B ; Lowercase # L& [3] LATIN SMALL LETTER K WITH HOOK..LATIN SMALL LETTER LAMBDA WITH STROKE +019E ; Lowercase # L& LATIN SMALL LETTER N WITH LONG RIGHT LEG +01A1 ; Lowercase # L& LATIN SMALL LETTER O WITH HORN +01A3 ; Lowercase # L& LATIN SMALL LETTER OI +01A5 ; Lowercase # L& LATIN SMALL LETTER P WITH HOOK +01A8 ; Lowercase # L& LATIN SMALL LETTER TONE TWO +01AA..01AB ; Lowercase # L& [2] LATIN LETTER REVERSED ESH LOOP..LATIN SMALL LETTER T WITH PALATAL HOOK +01AD ; Lowercase # L& LATIN SMALL LETTER T WITH HOOK +01B0 ; Lowercase # L& LATIN SMALL LETTER U WITH HORN +01B4 ; Lowercase # L& LATIN SMALL LETTER Y WITH HOOK +01B6 ; Lowercase # L& LATIN SMALL LETTER Z WITH STROKE +01B9..01BA ; Lowercase # L& [2] LATIN SMALL LETTER EZH REVERSED..LATIN SMALL LETTER EZH WITH TAIL +01BD..01BF ; Lowercase # L& [3] LATIN SMALL LETTER TONE FIVE..LATIN LETTER WYNN +01C6 ; Lowercase # L& LATIN SMALL LETTER DZ WITH CARON +01C9 ; Lowercase # L& LATIN SMALL LETTER LJ +01CC ; Lowercase # L& LATIN SMALL LETTER NJ +01CE ; Lowercase # L& LATIN SMALL LETTER A WITH CARON +01D0 ; Lowercase # L& LATIN SMALL LETTER I WITH CARON +01D2 ; Lowercase # L& LATIN SMALL LETTER O WITH CARON +01D4 ; Lowercase # L& LATIN SMALL LETTER U WITH CARON +01D6 ; Lowercase # L& LATIN SMALL LETTER U WITH DIAERESIS AND MACRON +01D8 ; Lowercase # L& LATIN SMALL LETTER U WITH DIAERESIS AND ACUTE +01DA ; Lowercase # L& LATIN SMALL LETTER U WITH DIAERESIS AND CARON +01DC..01DD ; Lowercase # L& [2] LATIN SMALL LETTER U WITH DIAERESIS AND GRAVE..LATIN SMALL LETTER TURNED E +01DF ; Lowercase # L& LATIN SMALL LETTER A WITH DIAERESIS AND MACRON +01E1 ; Lowercase # L& LATIN SMALL LETTER A WITH DOT ABOVE AND MACRON +01E3 ; Lowercase # L& LATIN SMALL LETTER AE WITH MACRON +01E5 ; Lowercase # L& LATIN SMALL LETTER G WITH STROKE +01E7 ; Lowercase # L& LATIN SMALL LETTER G WITH CARON +01E9 ; Lowercase # L& LATIN SMALL LETTER K WITH CARON +01EB ; Lowercase # L& LATIN SMALL LETTER O WITH OGONEK +01ED ; Lowercase # L& LATIN SMALL LETTER O WITH OGONEK AND MACRON +01EF..01F0 ; Lowercase # L& [2] LATIN SMALL LETTER EZH WITH CARON..LATIN SMALL LETTER J WITH CARON +01F3 ; Lowercase # L& LATIN SMALL LETTER DZ +01F5 ; Lowercase # L& LATIN SMALL LETTER G WITH ACUTE +01F9 ; Lowercase # L& LATIN SMALL LETTER N WITH GRAVE +01FB ; Lowercase # L& LATIN SMALL LETTER A WITH RING ABOVE AND ACUTE +01FD ; Lowercase # L& LATIN SMALL LETTER AE WITH ACUTE +01FF ; Lowercase # L& LATIN SMALL LETTER O WITH STROKE AND ACUTE +0201 ; Lowercase # L& LATIN SMALL LETTER A WITH DOUBLE GRAVE +0203 ; Lowercase # L& LATIN SMALL LETTER A WITH INVERTED BREVE +0205 ; Lowercase # L& LATIN SMALL LETTER E WITH DOUBLE GRAVE +0207 ; Lowercase # L& LATIN SMALL LETTER E WITH INVERTED BREVE +0209 ; Lowercase # L& LATIN SMALL LETTER I WITH DOUBLE GRAVE +020B ; Lowercase # L& LATIN SMALL LETTER I WITH INVERTED BREVE +020D ; Lowercase # L& LATIN SMALL LETTER O WITH DOUBLE GRAVE +020F ; Lowercase # L& LATIN SMALL LETTER O WITH INVERTED BREVE +0211 ; Lowercase # L& LATIN SMALL LETTER R WITH DOUBLE GRAVE +0213 ; Lowercase # L& LATIN SMALL LETTER R WITH INVERTED BREVE +0215 ; Lowercase # L& LATIN SMALL LETTER U WITH DOUBLE GRAVE +0217 ; Lowercase # L& LATIN SMALL LETTER U WITH INVERTED BREVE +0219 ; Lowercase # L& LATIN SMALL LETTER S WITH COMMA BELOW +021B ; Lowercase # L& LATIN SMALL LETTER T WITH COMMA BELOW +021D ; Lowercase # L& LATIN SMALL LETTER YOGH +021F ; Lowercase # L& LATIN SMALL LETTER H WITH CARON +0223 ; Lowercase # L& LATIN SMALL LETTER OU +0225 ; Lowercase # L& LATIN SMALL LETTER Z WITH HOOK +0227 ; Lowercase # L& LATIN SMALL LETTER A WITH DOT ABOVE +0229 ; Lowercase # L& LATIN SMALL LETTER E WITH CEDILLA +022B ; Lowercase # L& LATIN SMALL LETTER O WITH DIAERESIS AND MACRON +022D ; Lowercase # L& LATIN SMALL LETTER O WITH TILDE AND MACRON +022F ; Lowercase # L& LATIN SMALL LETTER O WITH DOT ABOVE +0231 ; Lowercase # L& LATIN SMALL LETTER O WITH DOT ABOVE AND MACRON +0233 ; Lowercase # L& LATIN SMALL LETTER Y WITH MACRON +0250..02AD ; Lowercase # L& [94] LATIN SMALL LETTER TURNED A..LATIN LETTER BIDENTAL PERCUSSIVE +02B0..02B8 ; Lowercase # Lm [9] MODIFIER LETTER SMALL H..MODIFIER LETTER SMALL Y +02C0..02C1 ; Lowercase # Lm [2] MODIFIER LETTER GLOTTAL STOP..MODIFIER LETTER REVERSED GLOTTAL STOP +02E0..02E4 ; Lowercase # Lm [5] MODIFIER LETTER SMALL GAMMA..MODIFIER LETTER SMALL REVERSED GLOTTAL STOP +0345 ; Lowercase # Mn COMBINING GREEK YPOGEGRAMMENI +037A ; Lowercase # Lm GREEK YPOGEGRAMMENI +0390 ; Lowercase # L& GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS +03AC..03CE ; Lowercase # L& [35] GREEK SMALL LETTER ALPHA WITH TONOS..GREEK SMALL LETTER OMEGA WITH TONOS +03D0..03D1 ; Lowercase # L& [2] GREEK BETA SYMBOL..GREEK THETA SYMBOL +03D5..03D7 ; Lowercase # L& [3] GREEK PHI SYMBOL..GREEK KAI SYMBOL +03D9 ; Lowercase # L& GREEK SMALL LETTER ARCHAIC KOPPA +03DB ; Lowercase # L& GREEK SMALL LETTER STIGMA +03DD ; Lowercase # L& GREEK SMALL LETTER DIGAMMA +03DF ; Lowercase # L& GREEK SMALL LETTER KOPPA +03E1 ; Lowercase # L& GREEK SMALL LETTER SAMPI +03E3 ; Lowercase # L& COPTIC SMALL LETTER SHEI +03E5 ; Lowercase # L& COPTIC SMALL LETTER FEI +03E7 ; Lowercase # L& COPTIC SMALL LETTER KHEI +03E9 ; Lowercase # L& COPTIC SMALL LETTER HORI +03EB ; Lowercase # L& COPTIC SMALL LETTER GANGIA +03ED ; Lowercase # L& COPTIC SMALL LETTER SHIMA +03EF..03F3 ; Lowercase # L& [5] COPTIC SMALL LETTER DEI..GREEK LETTER YOT +03F5 ; Lowercase # L& GREEK LUNATE EPSILON SYMBOL +0430..045F ; Lowercase # L& [48] CYRILLIC SMALL LETTER A..CYRILLIC SMALL LETTER DZHE +0461 ; Lowercase # L& CYRILLIC SMALL LETTER OMEGA +0463 ; Lowercase # L& CYRILLIC SMALL LETTER YAT +0465 ; Lowercase # L& CYRILLIC SMALL LETTER IOTIFIED E +0467 ; Lowercase # L& CYRILLIC SMALL LETTER LITTLE YUS +0469 ; Lowercase # L& CYRILLIC SMALL LETTER IOTIFIED LITTLE YUS +046B ; Lowercase # L& CYRILLIC SMALL LETTER BIG YUS +046D ; Lowercase # L& CYRILLIC SMALL LETTER IOTIFIED BIG YUS +046F ; Lowercase # L& CYRILLIC SMALL LETTER KSI +0471 ; Lowercase # L& CYRILLIC SMALL LETTER PSI +0473 ; Lowercase # L& CYRILLIC SMALL LETTER FITA +0475 ; Lowercase # L& CYRILLIC SMALL LETTER IZHITSA +0477 ; Lowercase # L& CYRILLIC SMALL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT +0479 ; Lowercase # L& CYRILLIC SMALL LETTER UK +047B ; Lowercase # L& CYRILLIC SMALL LETTER ROUND OMEGA +047D ; Lowercase # L& CYRILLIC SMALL LETTER OMEGA WITH TITLO +047F ; Lowercase # L& CYRILLIC SMALL LETTER OT +0481 ; Lowercase # L& CYRILLIC SMALL LETTER KOPPA +048B ; Lowercase # L& CYRILLIC SMALL LETTER SHORT I WITH TAIL +048D ; Lowercase # L& CYRILLIC SMALL LETTER SEMISOFT SIGN +048F ; Lowercase # L& CYRILLIC SMALL LETTER ER WITH TICK +0491 ; Lowercase # L& CYRILLIC SMALL LETTER GHE WITH UPTURN +0493 ; Lowercase # L& CYRILLIC SMALL LETTER GHE WITH STROKE +0495 ; Lowercase # L& CYRILLIC SMALL LETTER GHE WITH MIDDLE HOOK +0497 ; Lowercase # L& CYRILLIC SMALL LETTER ZHE WITH DESCENDER +0499 ; Lowercase # L& CYRILLIC SMALL LETTER ZE WITH DESCENDER +049B ; Lowercase # L& CYRILLIC SMALL LETTER KA WITH DESCENDER +049D ; Lowercase # L& CYRILLIC SMALL LETTER KA WITH VERTICAL STROKE +049F ; Lowercase # L& CYRILLIC SMALL LETTER KA WITH STROKE +04A1 ; Lowercase # L& CYRILLIC SMALL LETTER BASHKIR KA +04A3 ; Lowercase # L& CYRILLIC SMALL LETTER EN WITH DESCENDER +04A5 ; Lowercase # L& CYRILLIC SMALL LIGATURE EN GHE +04A7 ; Lowercase # L& CYRILLIC SMALL LETTER PE WITH MIDDLE HOOK +04A9 ; Lowercase # L& CYRILLIC SMALL LETTER ABKHASIAN HA +04AB ; Lowercase # L& CYRILLIC SMALL LETTER ES WITH DESCENDER +04AD ; Lowercase # L& CYRILLIC SMALL LETTER TE WITH DESCENDER +04AF ; Lowercase # L& CYRILLIC SMALL LETTER STRAIGHT U +04B1 ; Lowercase # L& CYRILLIC SMALL LETTER STRAIGHT U WITH STROKE +04B3 ; Lowercase # L& CYRILLIC SMALL LETTER HA WITH DESCENDER +04B5 ; Lowercase # L& CYRILLIC SMALL LIGATURE TE TSE +04B7 ; Lowercase # L& CYRILLIC SMALL LETTER CHE WITH DESCENDER +04B9 ; Lowercase # L& CYRILLIC SMALL LETTER CHE WITH VERTICAL STROKE +04BB ; Lowercase # L& CYRILLIC SMALL LETTER SHHA +04BD ; Lowercase # L& CYRILLIC SMALL LETTER ABKHASIAN CHE +04BF ; Lowercase # L& CYRILLIC SMALL LETTER ABKHASIAN CHE WITH DESCENDER +04C2 ; Lowercase # L& CYRILLIC SMALL LETTER ZHE WITH BREVE +04C4 ; Lowercase # L& CYRILLIC SMALL LETTER KA WITH HOOK +04C6 ; Lowercase # L& CYRILLIC SMALL LETTER EL WITH TAIL +04C8 ; Lowercase # L& CYRILLIC SMALL LETTER EN WITH HOOK +04CA ; Lowercase # L& CYRILLIC SMALL LETTER EN WITH TAIL +04CC ; Lowercase # L& CYRILLIC SMALL LETTER KHAKASSIAN CHE +04CE ; Lowercase # L& CYRILLIC SMALL LETTER EM WITH TAIL +04D1 ; Lowercase # L& CYRILLIC SMALL LETTER A WITH BREVE +04D3 ; Lowercase # L& CYRILLIC SMALL LETTER A WITH DIAERESIS +04D5 ; Lowercase # L& CYRILLIC SMALL LIGATURE A IE +04D7 ; Lowercase # L& CYRILLIC SMALL LETTER IE WITH BREVE +04D9 ; Lowercase # L& CYRILLIC SMALL LETTER SCHWA +04DB ; Lowercase # L& CYRILLIC SMALL LETTER SCHWA WITH DIAERESIS +04DD ; Lowercase # L& CYRILLIC SMALL LETTER ZHE WITH DIAERESIS +04DF ; Lowercase # L& CYRILLIC SMALL LETTER ZE WITH DIAERESIS +04E1 ; Lowercase # L& CYRILLIC SMALL LETTER ABKHASIAN DZE +04E3 ; Lowercase # L& CYRILLIC SMALL LETTER I WITH MACRON +04E5 ; Lowercase # L& CYRILLIC SMALL LETTER I WITH DIAERESIS +04E7 ; Lowercase # L& CYRILLIC SMALL LETTER O WITH DIAERESIS +04E9 ; Lowercase # L& CYRILLIC SMALL LETTER BARRED O +04EB ; Lowercase # L& CYRILLIC SMALL LETTER BARRED O WITH DIAERESIS +04ED ; Lowercase # L& CYRILLIC SMALL LETTER E WITH DIAERESIS +04EF ; Lowercase # L& CYRILLIC SMALL LETTER U WITH MACRON +04F1 ; Lowercase # L& CYRILLIC SMALL LETTER U WITH DIAERESIS +04F3 ; Lowercase # L& CYRILLIC SMALL LETTER U WITH DOUBLE ACUTE +04F5 ; Lowercase # L& CYRILLIC SMALL LETTER CHE WITH DIAERESIS +04F9 ; Lowercase # L& CYRILLIC SMALL LETTER YERU WITH DIAERESIS +0501 ; Lowercase # L& CYRILLIC SMALL LETTER KOMI DE +0503 ; Lowercase # L& CYRILLIC SMALL LETTER KOMI DJE +0505 ; Lowercase # L& CYRILLIC SMALL LETTER KOMI ZJE +0507 ; Lowercase # L& CYRILLIC SMALL LETTER KOMI DZJE +0509 ; Lowercase # L& CYRILLIC SMALL LETTER KOMI LJE +050B ; Lowercase # L& CYRILLIC SMALL LETTER KOMI NJE +050D ; Lowercase # L& CYRILLIC SMALL LETTER KOMI SJE +050F ; Lowercase # L& CYRILLIC SMALL LETTER KOMI TJE +0561..0587 ; Lowercase # L& [39] ARMENIAN SMALL LETTER AYB..ARMENIAN SMALL LIGATURE ECH YIWN +1E01 ; Lowercase # L& LATIN SMALL LETTER A WITH RING BELOW +1E03 ; Lowercase # L& LATIN SMALL LETTER B WITH DOT ABOVE +1E05 ; Lowercase # L& LATIN SMALL LETTER B WITH DOT BELOW +1E07 ; Lowercase # L& LATIN SMALL LETTER B WITH LINE BELOW +1E09 ; Lowercase # L& LATIN SMALL LETTER C WITH CEDILLA AND ACUTE +1E0B ; Lowercase # L& LATIN SMALL LETTER D WITH DOT ABOVE +1E0D ; Lowercase # L& LATIN SMALL LETTER D WITH DOT BELOW +1E0F ; Lowercase # L& LATIN SMALL LETTER D WITH LINE BELOW +1E11 ; Lowercase # L& LATIN SMALL LETTER D WITH CEDILLA +1E13 ; Lowercase # L& LATIN SMALL LETTER D WITH CIRCUMFLEX BELOW +1E15 ; Lowercase # L& LATIN SMALL LETTER E WITH MACRON AND GRAVE +1E17 ; Lowercase # L& LATIN SMALL LETTER E WITH MACRON AND ACUTE +1E19 ; Lowercase # L& LATIN SMALL LETTER E WITH CIRCUMFLEX BELOW +1E1B ; Lowercase # L& LATIN SMALL LETTER E WITH TILDE BELOW +1E1D ; Lowercase # L& LATIN SMALL LETTER E WITH CEDILLA AND BREVE +1E1F ; Lowercase # L& LATIN SMALL LETTER F WITH DOT ABOVE +1E21 ; Lowercase # L& LATIN SMALL LETTER G WITH MACRON +1E23 ; Lowercase # L& LATIN SMALL LETTER H WITH DOT ABOVE +1E25 ; Lowercase # L& LATIN SMALL LETTER H WITH DOT BELOW +1E27 ; Lowercase # L& LATIN SMALL LETTER H WITH DIAERESIS +1E29 ; Lowercase # L& LATIN SMALL LETTER H WITH CEDILLA +1E2B ; Lowercase # L& LATIN SMALL LETTER H WITH BREVE BELOW +1E2D ; Lowercase # L& LATIN SMALL LETTER I WITH TILDE BELOW +1E2F ; Lowercase # L& LATIN SMALL LETTER I WITH DIAERESIS AND ACUTE +1E31 ; Lowercase # L& LATIN SMALL LETTER K WITH ACUTE +1E33 ; Lowercase # L& LATIN SMALL LETTER K WITH DOT BELOW +1E35 ; Lowercase # L& LATIN SMALL LETTER K WITH LINE BELOW +1E37 ; Lowercase # L& LATIN SMALL LETTER L WITH DOT BELOW +1E39 ; Lowercase # L& LATIN SMALL LETTER L WITH DOT BELOW AND MACRON +1E3B ; Lowercase # L& LATIN SMALL LETTER L WITH LINE BELOW +1E3D ; Lowercase # L& LATIN SMALL LETTER L WITH CIRCUMFLEX BELOW +1E3F ; Lowercase # L& LATIN SMALL LETTER M WITH ACUTE +1E41 ; Lowercase # L& LATIN SMALL LETTER M WITH DOT ABOVE +1E43 ; Lowercase # L& LATIN SMALL LETTER M WITH DOT BELOW +1E45 ; Lowercase # L& LATIN SMALL LETTER N WITH DOT ABOVE +1E47 ; Lowercase # L& LATIN SMALL LETTER N WITH DOT BELOW +1E49 ; Lowercase # L& LATIN SMALL LETTER N WITH LINE BELOW +1E4B ; Lowercase # L& LATIN SMALL LETTER N WITH CIRCUMFLEX BELOW +1E4D ; Lowercase # L& LATIN SMALL LETTER O WITH TILDE AND ACUTE +1E4F ; Lowercase # L& LATIN SMALL LETTER O WITH TILDE AND DIAERESIS +1E51 ; Lowercase # L& LATIN SMALL LETTER O WITH MACRON AND GRAVE +1E53 ; Lowercase # L& LATIN SMALL LETTER O WITH MACRON AND ACUTE +1E55 ; Lowercase # L& LATIN SMALL LETTER P WITH ACUTE +1E57 ; Lowercase # L& LATIN SMALL LETTER P WITH DOT ABOVE +1E59 ; Lowercase # L& LATIN SMALL LETTER R WITH DOT ABOVE +1E5B ; Lowercase # L& LATIN SMALL LETTER R WITH DOT BELOW +1E5D ; Lowercase # L& LATIN SMALL LETTER R WITH DOT BELOW AND MACRON +1E5F ; Lowercase # L& LATIN SMALL LETTER R WITH LINE BELOW +1E61 ; Lowercase # L& LATIN SMALL LETTER S WITH DOT ABOVE +1E63 ; Lowercase # L& LATIN SMALL LETTER S WITH DOT BELOW +1E65 ; Lowercase # L& LATIN SMALL LETTER S WITH ACUTE AND DOT ABOVE +1E67 ; Lowercase # L& LATIN SMALL LETTER S WITH CARON AND DOT ABOVE +1E69 ; Lowercase # L& LATIN SMALL LETTER S WITH DOT BELOW AND DOT ABOVE +1E6B ; Lowercase # L& LATIN SMALL LETTER T WITH DOT ABOVE +1E6D ; Lowercase # L& LATIN SMALL LETTER T WITH DOT BELOW +1E6F ; Lowercase # L& LATIN SMALL LETTER T WITH LINE BELOW +1E71 ; Lowercase # L& LATIN SMALL LETTER T WITH CIRCUMFLEX BELOW +1E73 ; Lowercase # L& LATIN SMALL LETTER U WITH DIAERESIS BELOW +1E75 ; Lowercase # L& LATIN SMALL LETTER U WITH TILDE BELOW +1E77 ; Lowercase # L& LATIN SMALL LETTER U WITH CIRCUMFLEX BELOW +1E79 ; Lowercase # L& LATIN SMALL LETTER U WITH TILDE AND ACUTE +1E7B ; Lowercase # L& LATIN SMALL LETTER U WITH MACRON AND DIAERESIS +1E7D ; Lowercase # L& LATIN SMALL LETTER V WITH TILDE +1E7F ; Lowercase # L& LATIN SMALL LETTER V WITH DOT BELOW +1E81 ; Lowercase # L& LATIN SMALL LETTER W WITH GRAVE +1E83 ; Lowercase # L& LATIN SMALL LETTER W WITH ACUTE +1E85 ; Lowercase # L& LATIN SMALL LETTER W WITH DIAERESIS +1E87 ; Lowercase # L& LATIN SMALL LETTER W WITH DOT ABOVE +1E89 ; Lowercase # L& LATIN SMALL LETTER W WITH DOT BELOW +1E8B ; Lowercase # L& LATIN SMALL LETTER X WITH DOT ABOVE +1E8D ; Lowercase # L& LATIN SMALL LETTER X WITH DIAERESIS +1E8F ; Lowercase # L& LATIN SMALL LETTER Y WITH DOT ABOVE +1E91 ; Lowercase # L& LATIN SMALL LETTER Z WITH CIRCUMFLEX +1E93 ; Lowercase # L& LATIN SMALL LETTER Z WITH DOT BELOW +1E95..1E9B ; Lowercase # L& [7] LATIN SMALL LETTER Z WITH LINE BELOW..LATIN SMALL LETTER LONG S WITH DOT ABOVE +1EA1 ; Lowercase # L& LATIN SMALL LETTER A WITH DOT BELOW +1EA3 ; Lowercase # L& LATIN SMALL LETTER A WITH HOOK ABOVE +1EA5 ; Lowercase # L& LATIN SMALL LETTER A WITH CIRCUMFLEX AND ACUTE +1EA7 ; Lowercase # L& LATIN SMALL LETTER A WITH CIRCUMFLEX AND GRAVE +1EA9 ; Lowercase # L& LATIN SMALL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE +1EAB ; Lowercase # L& LATIN SMALL LETTER A WITH CIRCUMFLEX AND TILDE +1EAD ; Lowercase # L& LATIN SMALL LETTER A WITH CIRCUMFLEX AND DOT BELOW +1EAF ; Lowercase # L& LATIN SMALL LETTER A WITH BREVE AND ACUTE +1EB1 ; Lowercase # L& LATIN SMALL LETTER A WITH BREVE AND GRAVE +1EB3 ; Lowercase # L& LATIN SMALL LETTER A WITH BREVE AND HOOK ABOVE +1EB5 ; Lowercase # L& LATIN SMALL LETTER A WITH BREVE AND TILDE +1EB7 ; Lowercase # L& LATIN SMALL LETTER A WITH BREVE AND DOT BELOW +1EB9 ; Lowercase # L& LATIN SMALL LETTER E WITH DOT BELOW +1EBB ; Lowercase # L& LATIN SMALL LETTER E WITH HOOK ABOVE +1EBD ; Lowercase # L& LATIN SMALL LETTER E WITH TILDE +1EBF ; Lowercase # L& LATIN SMALL LETTER E WITH CIRCUMFLEX AND ACUTE +1EC1 ; Lowercase # L& LATIN SMALL LETTER E WITH CIRCUMFLEX AND GRAVE +1EC3 ; Lowercase # L& LATIN SMALL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE +1EC5 ; Lowercase # L& LATIN SMALL LETTER E WITH CIRCUMFLEX AND TILDE +1EC7 ; Lowercase # L& LATIN SMALL LETTER E WITH CIRCUMFLEX AND DOT BELOW +1EC9 ; Lowercase # L& LATIN SMALL LETTER I WITH HOOK ABOVE +1ECB ; Lowercase # L& LATIN SMALL LETTER I WITH DOT BELOW +1ECD ; Lowercase # L& LATIN SMALL LETTER O WITH DOT BELOW +1ECF ; Lowercase # L& LATIN SMALL LETTER O WITH HOOK ABOVE +1ED1 ; Lowercase # L& LATIN SMALL LETTER O WITH CIRCUMFLEX AND ACUTE +1ED3 ; Lowercase # L& LATIN SMALL LETTER O WITH CIRCUMFLEX AND GRAVE +1ED5 ; Lowercase # L& LATIN SMALL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE +1ED7 ; Lowercase # L& LATIN SMALL LETTER O WITH CIRCUMFLEX AND TILDE +1ED9 ; Lowercase # L& LATIN SMALL LETTER O WITH CIRCUMFLEX AND DOT BELOW +1EDB ; Lowercase # L& LATIN SMALL LETTER O WITH HORN AND ACUTE +1EDD ; Lowercase # L& LATIN SMALL LETTER O WITH HORN AND GRAVE +1EDF ; Lowercase # L& LATIN SMALL LETTER O WITH HORN AND HOOK ABOVE +1EE1 ; Lowercase # L& LATIN SMALL LETTER O WITH HORN AND TILDE +1EE3 ; Lowercase # L& LATIN SMALL LETTER O WITH HORN AND DOT BELOW +1EE5 ; Lowercase # L& LATIN SMALL LETTER U WITH DOT BELOW +1EE7 ; Lowercase # L& LATIN SMALL LETTER U WITH HOOK ABOVE +1EE9 ; Lowercase # L& LATIN SMALL LETTER U WITH HORN AND ACUTE +1EEB ; Lowercase # L& LATIN SMALL LETTER U WITH HORN AND GRAVE +1EED ; Lowercase # L& LATIN SMALL LETTER U WITH HORN AND HOOK ABOVE +1EEF ; Lowercase # L& LATIN SMALL LETTER U WITH HORN AND TILDE +1EF1 ; Lowercase # L& LATIN SMALL LETTER U WITH HORN AND DOT BELOW +1EF3 ; Lowercase # L& LATIN SMALL LETTER Y WITH GRAVE +1EF5 ; Lowercase # L& LATIN SMALL LETTER Y WITH DOT BELOW +1EF7 ; Lowercase # L& LATIN SMALL LETTER Y WITH HOOK ABOVE +1EF9 ; Lowercase # L& LATIN SMALL LETTER Y WITH TILDE +1F00..1F07 ; Lowercase # L& [8] GREEK SMALL LETTER ALPHA WITH PSILI..GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI +1F10..1F15 ; Lowercase # L& [6] GREEK SMALL LETTER EPSILON WITH PSILI..GREEK SMALL LETTER EPSILON WITH DASIA AND OXIA +1F20..1F27 ; Lowercase # L& [8] GREEK SMALL LETTER ETA WITH PSILI..GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI From noreply at buildbot.pypy.org Mon Jan 28 19:56:37 2013 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 28 Jan 2013 19:56:37 +0100 (CET) Subject: [pypy-commit] pypy default: Slice opcodes don't exist in py3k anymore. Message-ID: <20130128185637.6ED241C12E1@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: Changeset: r60623:5ab94dfe5b22 Date: 2013-01-28 19:53 +0100 http://bitbucket.org/pypy/pypy/changeset/5ab94dfe5b22/ Log: Slice opcodes don't exist in py3k anymore. Use a more generic method to convert "SLICE+0" to identifiers. diff --git a/rpython/tool/stdlib_opcode.py b/rpython/tool/stdlib_opcode.py --- a/rpython/tool/stdlib_opcode.py +++ b/rpython/tool/stdlib_opcode.py @@ -71,10 +71,14 @@ def to_globals(self, globals_dict): """NOT_RPYTHON. Add individual opcodes to the module constants.""" - globals_dict.update(self.opmap) - globals_dict['SLICE'] = self.opmap["SLICE+0"] - globals_dict['STORE_SLICE'] = self.opmap["STORE_SLICE+0"] - globals_dict['DELETE_SLICE'] = self.opmap["DELETE_SLICE+0"] + for name, value in self.opmap.iteritems(): + # Rename 'STORE_SLICE+0' opcodes + if name.endswith('+0'): + name = name[:-2] + # Ignore 'STORE_SLICE+1' opcodes + elif name.endswith(('+1', '+2', '+3')): + continue + globals_dict[name] = value def __str__(self): return "<%s bytecode>" % (self.name,) @@ -83,4 +87,4 @@ from opcode import opmap, HAVE_ARGUMENT -host_bytecode_spec = BytecodeSpec('host', opmap, HAVE_ARGUMENT) \ No newline at end of file +host_bytecode_spec = BytecodeSpec('host', opmap, HAVE_ARGUMENT) From noreply at buildbot.pypy.org Mon Jan 28 19:56:38 2013 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 28 Jan 2013 19:56:38 +0100 (CET) Subject: [pypy-commit] pypy default: Propagate no_nul attribute in str.__getitem__ Message-ID: <20130128185638.C6A1D1C12E1@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: Changeset: r60624:2dd7e774dce8 Date: 2013-01-28 19:53 +0100 http://bitbucket.org/pypy/pypy/changeset/2dd7e774dce8/ Log: Propagate no_nul attribute in str.__getitem__ diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py --- a/rpython/annotator/binaryop.py +++ b/rpython/annotator/binaryop.py @@ -446,7 +446,8 @@ class __extend__(pairtype(SomeChar, SomeChar)): def union((chr1, chr2)): - return SomeChar() + no_nul = chr1.no_nul and chr2.no_nul + return SomeChar(no_nul=no_nul) class __extend__(pairtype(SomeChar, SomeUnicodeCodePoint), @@ -664,14 +665,14 @@ def getitem((str1, int2)): getbookkeeper().count("str_getitem", int2) - return SomeChar() + return SomeChar(no_nul=str1.no_nul) getitem.can_only_throw = [] getitem_key = getitem def getitem_idx((str1, int2)): getbookkeeper().count("str_getitem", int2) - return SomeChar() + return SomeChar(no_nul=str1.no_nul) getitem_idx.can_only_throw = [IndexError] getitem_idx_key = getitem_idx diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py --- a/rpython/annotator/test/test_annrpython.py +++ b/rpython/annotator/test/test_annrpython.py @@ -2068,7 +2068,23 @@ s = a.build_types(f, [annmodel.SomeString(no_nul=True)]) assert isinstance(s, annmodel.SomeString) assert s.no_nul - + + def test_getitem_str0(self): + def f(s, n): + if n == 1: + return s[0] + elif n == 2: + return s[1] + elif n == 3: + return s[1:] + return s + a = self.RPythonAnnotator() + a.translator.config.translation.check_str_without_nul = True + + s = a.build_types(f, [annmodel.SomeString(no_nul=True), + annmodel.SomeInteger()]) + assert isinstance(s, annmodel.SomeString) + assert s.no_nul def test_non_none_and_none_with_isinstance(self): class A(object): From noreply at buildbot.pypy.org Mon Jan 28 19:56:40 2013 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 28 Jan 2013 19:56:40 +0100 (CET) Subject: [pypy-commit] pypy default: Add rcomplex.isfinite() Message-ID: <20130128185640.094A21C12E1@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: Changeset: r60625:83989cc371b2 Date: 2013-01-28 19:53 +0100 http://bitbucket.org/pypy/pypy/changeset/83989cc371b2/ Log: Add rcomplex.isfinite() diff --git a/rpython/rlib/rcomplex.py b/rpython/rlib/rcomplex.py --- a/rpython/rlib/rcomplex.py +++ b/rpython/rlib/rcomplex.py @@ -566,3 +566,6 @@ def c_isnan(r, i): return isnan(r) or isnan(i) + +def c_isfinite(r, i): + return isfinite(r) and isfinite(i) diff --git a/rpython/rlib/test/test_rcomplex.py b/rpython/rlib/test/test_rcomplex.py --- a/rpython/rlib/test/test_rcomplex.py +++ b/rpython/rlib/test/test_rcomplex.py @@ -260,3 +260,24 @@ rAssertAlmostEqual(expected[1], actual[1], abs_err=real_abs_err, msg=error_message) + +def test_isnan(): + assert not c.c_isnan(0, 0) + assert c.c_isnan(float('nan'), 0) + assert c.c_isnan(1, float('nan')) + assert not c.c_isnan(float('inf'), 0) + +def test_isinf(): + assert not c.c_isinf(0, 0) + assert c.c_isinf(float('inf'), 0) + assert c.c_isinf(float('-inf'), 0) + assert c.c_isinf(1, float('inf')) + assert not c.c_isinf(float('nan'), 0) + +def test_isfinite(): + assert c.c_isfinite(0, 0) + assert not c.c_isfinite(float('nan'), 0) + assert not c.c_isfinite(float('-inf'), 0) + assert not c.c_isfinite(0, float('nan')) + assert not c.c_isfinite(0, float('-inf')) + From noreply at buildbot.pypy.org Mon Jan 28 20:00:04 2013 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 28 Jan 2013 20:00:04 +0100 (CET) Subject: [pypy-commit] pypy py3k: hg merge default Message-ID: <20130128190004.D95EE1C1305@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r60626:649d563661a8 Date: 2013-01-28 09:27 +0100 http://bitbucket.org/pypy/pypy/changeset/649d563661a8/ Log: hg merge default diff too long, truncating to 2000 out of 140935 lines diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -34,6 +34,9 @@ .. branch: kill-faking .. branch: improved_ebnfparse_error .. branch: task-decorator +.. branch: fix-e4fa0b2 +.. branch: win32-fixes +.. branch: fix-version-tool .. branch: release-2.0-beta1 diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -760,13 +760,13 @@ def pypy_find_stdlib(s): from os.path import abspath, join, dirname as dn thisfile = abspath(__file__) - root = dn(dn(dn(dn(thisfile)))) + root = dn(dn(dn(thisfile))) return [join(root, 'lib-python', '3.2'), join(root, 'lib_pypy')] def pypy_resolvedirof(s): # we ignore the issue of symlinks; for tests, the executable is always - # translator/goal/app_main.py anyway + # interpreter/app_main.py anyway import os return os.path.abspath(os.path.join(s, '..')) diff --git a/pypy/interpreter/test2/test_app_main.py b/pypy/interpreter/test2/test_app_main.py --- a/pypy/interpreter/test2/test_app_main.py +++ b/pypy/interpreter/test2/test_app_main.py @@ -578,7 +578,7 @@ assert line.rstrip() == 'Not at all. They could be carried.' print('A five ounce bird could not carry a one pound coconut.') """) - py_py = os.path.join(pypydir, 'bin', 'py.py') + py_py = os.path.join(pypydir, 'bin', 'pyinteractive.py') child = self._spawn(sys.executable, [py_py, '-S', path]) child.expect('Are you suggesting coconuts migrate?', timeout=120) child.sendline('Not at all. They could be carried.') @@ -806,7 +806,7 @@ assert data == p + os.sep + '\n' def test_getfilesystemencoding(self): - py.test.skip("this has been failing since forever, but it's not tested nightly because buildbot uses python2.6 :-(") + py.test.skip("encoding is only set if stdout.isatty(), test is flawed") if sys.version_info < (2, 7): skip("test requires Python >= 2.7") p = getscript_in_dir(""" @@ -913,7 +913,6 @@ def test_setup_bootstrap_path(self): import sys - import os old_sys_path = sys.path[:] sys.path.append(self.goal_dir) try: @@ -939,7 +938,7 @@ sys.path.append(self.goal_dir) try: import app_main - pypy_c = os.path.join(self.trunkdir, 'pypy', 'translator', 'goal', 'pypy-c') + pypy_c = os.path.join(self.trunkdir, 'pypy', 'goal', 'pypy-c') app_main.setup_bootstrap_path(pypy_c) newpath = sys.path[:] # we get at least lib_pypy @@ -957,7 +956,7 @@ sys.path.append(self.goal_dir) try: import app_main - pypy_c = os.path.join(self.trunkdir, 'pypy', 'translator', 'goal', 'pypy-c') + pypy_c = os.path.join(self.trunkdir, 'pypy', 'goal', 'pypy-c') app_main.entry_point(pypy_c, [self.foo_py]) # assert it did not crash finally: diff --git a/pypy/interpreter/test2/test_targetpypy.py b/pypy/interpreter/test2/test_targetpypy.py --- a/pypy/interpreter/test2/test_targetpypy.py +++ b/pypy/interpreter/test2/test_targetpypy.py @@ -1,5 +1,3 @@ - -import py from pypy.goal.targetpypystandalone import get_entry_point from pypy.config.pypyoption import get_pypy_config @@ -7,6 +5,4 @@ def test_run(self): config = get_pypy_config(translating=False) entry_point = get_entry_point(config)[0] - space = self.space - py.test.skip("not working so far") entry_point(['pypy-c' , '-S', '-c', 'print 3']) diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -2,7 +2,7 @@ import os class AppTestFileIO: - spaceconfig = dict(usemodules=['_io']) + spaceconfig = dict(usemodules=['_io'] + (['fcntl'] if os.name != 'nt' else [])) def setup_class(cls): tmpfile = udir.join('tmpfile') diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -49,6 +49,8 @@ kwds["libraries"] = [api_library] # '%s' undefined; assuming extern returning int kwds["compile_extra"] = ["/we4013"] + elif sys.platform == 'darwin': + kwds["link_files"] = [str(api_library + '.dylib')] else: kwds["link_files"] = [str(api_library + '.so')] if sys.platform.startswith('linux'): diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py --- a/pypy/module/rctime/test/test_rctime.py +++ b/pypy/module/rctime/test/test_rctime.py @@ -213,7 +213,7 @@ def test_strftime(self): import time as rctime - import os + import os, sys t = rctime.time() tt = rctime.gmtime(t) @@ -234,6 +234,10 @@ # input to [w]strftime is not kosher. if os.name == 'nt': raises(ValueError, rctime.strftime, '%f') + elif sys.platform == 'darwin': + # darwin strips % of unknown format codes + # http://bugs.python.org/issue9811 + assert rctime.strftime('%f') == 'f' else: assert rctime.strftime('%f') == '%f' diff --git a/pypy/module/signal/test/test_signal.py b/pypy/module/signal/test/test_signal.py --- a/pypy/module/signal/test/test_signal.py +++ b/pypy/module/signal/test/test_signal.py @@ -36,7 +36,7 @@ class AppTestSignal: spaceconfig = { - "usemodules": ['signal', 'rctime'], + "usemodules": ['signal', 'rctime'] + (['fcntl'] if os.name != 'nt' else []), } def setup_class(cls): @@ -157,6 +157,8 @@ if sys.platform == 'win32': raises(ValueError, signal, 42, lambda *args: None) raises(ValueError, signal, 7, lambda *args: None) + elif sys.platform == 'darwin': + raises(ValueError, signal, 42, lambda *args: None) else: signal(42, lambda *args: None) signal(42, SIG_DFL) diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -28,6 +28,7 @@ import pypy pypydir = os.path.dirname(os.path.abspath(pypy.__file__)) +pypyroot = os.path.dirname(pypydir) del pypy from rpython.tool.version import get_repo_version_info @@ -67,7 +68,7 @@ CPYTHON_VERSION[0], CPYTHON_VERSION[1], CPYTHON_VERSION[2], - get_repo_version_info()[2], + get_repo_version_info(root=pypyroot)[1], date, time, ver, @@ -90,10 +91,10 @@ return space.wrap(('PyPy', '', '')) def get_repo_info(space): - info = get_repo_version_info() + info = get_repo_version_info(root=pypyroot) if info: - project, repo_tag, repo_version = info - return space.newtuple([space.wrap(project), + repo_tag, repo_version = info + return space.newtuple([space.wrap('PyPy'), space.wrap(repo_tag), space.wrap(repo_version)]) else: diff --git a/pypy/module/thread/test/test_thread.py b/pypy/module/thread/test/test_thread.py --- a/pypy/module/thread/test/test_thread.py +++ b/pypy/module/thread/test/test_thread.py @@ -220,16 +220,24 @@ import signal def f(): - time.sleep(0.5) - _thread.interrupt_main() + for x in range(50): + if waiting: + _thread.interrupt_main() + return + print 'tock...', x # <-force the GIL to be released, as + time.sleep(0.01) # time.sleep doesn't do non-translated def busy_wait(): - for x in range(1000): + waiting.append(None) + for x in range(100): print('tick...', x) # <-force the GIL to be released, as time.sleep(0.01) # time.sleep doesn't do non-translated + waiting.pop() # This is normally called by app_main.py signal.signal(signal.SIGINT, signal.default_int_handler) - _thread.start_new_thread(f, ()) - raises(KeyboardInterrupt, busy_wait) + for i in range(100): + waiting = [] + _thread.start_new_thread(f, ()) + raises(KeyboardInterrupt, busy_wait) diff --git a/pypy/module/unicodedata/interp_ucd.py b/pypy/module/unicodedata/interp_ucd.py --- a/pypy/module/unicodedata/interp_ucd.py +++ b/pypy/module/unicodedata/interp_ucd.py @@ -9,7 +9,7 @@ from rpython.rlib.objectmodel import we_are_translated from rpython.rlib.runicode import MAXUNICODE from rpython.rlib.unicodedata import unicodedb_5_2_0, unicodedb_3_2_0 -from rpython.rlib.runicode import code_to_unichr, ORD +from rpython.rlib.runicode import code_to_unichr, ord_accepts_surrogate import sys @@ -28,8 +28,6 @@ # handling: on narrow unicode builds, a surrogate pair is considered as one # unicode code point. -# The functions below are subtly different from the ones in runicode.py. -# When PyPy implements Python 3 they should be merged. if MAXUNICODE > 0xFFFF: # Target is wide build @@ -41,7 +39,7 @@ if not we_are_translated() and sys.maxunicode == 0xFFFF: # Host CPython is narrow build, accept surrogates try: - return ORD(space.unicode_w(w_unichr)) + return ord_accepts_surrogate(space.unicode_w(w_unichr)) except ValueError: raise OperationError(space.w_TypeError, space.wrap( 'need a single Unicode character as parameter')) @@ -68,7 +66,7 @@ else: # Accept surrogates try: - return ORD(space.unicode_w(w_unichr)) + return ord_accepts_surrogate(space.unicode_w(w_unichr)) except ValueError: raise OperationError(space.w_TypeError, space.wrap( 'need a single Unicode character as parameter')) diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -1,6 +1,6 @@ #!/usr/bin/env python """ A sample script that packages PyPy, provided that it's already built. -It uses 'pypy/translator/goal/pypy-c' and parts of the rest of the working +It uses 'pypy/goal/pypy-c' and parts of the rest of the working copy. Usage: package.py root-pypy-dir [name-of-archive] [name-of-pypy-c] [destination-for-tarball] [pypy-c-path] diff --git a/pypy/tool/release/test/test_package.py b/pypy/tool/release/test/test_package.py --- a/pypy/tool/release/test/test_package.py +++ b/pypy/tool/release/test/test_package.py @@ -15,7 +15,7 @@ basename = 'pypy-c' rename_pypy_c = 'pypy' exe_name_in_archive = 'bin/pypy' - pypy_c = py.path.local(pypydir).join('..', basename) + pypy_c = py.path.local(pypydir).join('goal', basename) if not pypy_c.check(): os.system("echo faked_pypy_c> %s" % (pypy_c,)) pypy_c.chmod(0755) diff --git a/pypy/tool/rundictbenchmarks.py b/pypy/tool/rundictbenchmarks.py --- a/pypy/tool/rundictbenchmarks.py +++ b/pypy/tool/rundictbenchmarks.py @@ -3,7 +3,7 @@ # this file runs some benchmarks with a pypy-c that is assumed to be # built using the MeasuringDictImplementation. -# it should be run with pypy/translator/goal as the cwd, and you'll +# it should be run with pypy/goal as the cwd, and you'll # need to hack a copy of rst2html for yourself (svn docutils # required). diff --git a/rpython/jit/backend/llgraph/llimpl.py b/rpython/jit/backend/llgraph/llimpl.py deleted file mode 100644 --- a/rpython/jit/backend/llgraph/llimpl.py +++ /dev/null @@ -1,1968 +0,0 @@ -""" -The non-RPythonic part of the llgraph backend. -This contains all the code that is directly run -when executing on top of the llinterpreter. -""" - -import weakref -from rpython.flowspace.model import Variable, Constant -from rpython.annotator import model as annmodel -from rpython.jit.metainterp.history import REF, INT, FLOAT -from rpython.jit.metainterp import history -from rpython.jit.codewriter import heaptracker -from rpython.rtyper.lltypesystem import lltype, llmemory, rclass, rstr, rffi -from rpython.rtyper.ootypesystem import ootype -from rpython.rtyper.module.support import LLSupport, OOSupport -from rpython.rtyper.llinterp import LLException -from rpython.rtyper.extregistry import ExtRegistryEntry - -from rpython.jit.metainterp import resoperation -from rpython.jit.metainterp.resoperation import rop -from rpython.jit.backend.llgraph import symbolic -from rpython.jit.codewriter import longlong -from rpython.jit.codewriter.effectinfo import EffectInfo - -from rpython.rlib.objectmodel import ComputedIntSymbolic, we_are_translated -from rpython.rlib.rarithmetic import ovfcheck -from rpython.rlib.rarithmetic import r_longlong, r_ulonglong, r_uint -from rpython.rlib.rtimer import read_timestamp - -import py -from rpython.tool.ansi_print import ansi_log -log = py.log.Producer('runner') -py.log.setconsumer('runner', ansi_log) - -IS_32_BIT = r_ulonglong is not r_uint - - -def _from_opaque(opq): - return opq._obj.externalobj - -_TO_OPAQUE = {} - -def _to_opaque(value): - try: - return value._the_opaque_pointer - except AttributeError: - op = lltype.opaqueptr(_TO_OPAQUE[value.__class__], 'opaque', - externalobj=value) - value._the_opaque_pointer = op - return op - -def _normalize(value): - if isinstance(value, lltype._ptr): - value = lltype.top_container(value._obj) - return value - -def from_opaque_string(s): - if isinstance(s, str): - return s - elif isinstance(s, ootype._string): - return OOSupport.from_rstr(s) - else: - return LLSupport.from_rstr(s) - -FLOAT_ARRAY_TP = lltype.Ptr(lltype.Array(lltype.Float, hints={"nolength": True})) -def maybe_uncast(TP, array): - if array._TYPE.TO.OF != lltype.Float: - # array._TYPE.TO._hints.get("uncast_on_llgraph"): - array = rffi.cast(TP, array) - return array - -# a list of argtypes of all operations - couldn't find any and it's -# very useful. Note however that the table is half-broken here and -# there, in ways that are sometimes a bit hard to fix; that's why -# it is not "official". -TYPES = { - 'int_add' : (('int', 'int'), 'int'), - 'int_sub' : (('int', 'int'), 'int'), - 'int_mul' : (('int', 'int'), 'int'), - 'int_floordiv' : (('int', 'int'), 'int'), - 'int_mod' : (('int', 'int'), 'int'), - 'int_and' : (('int', 'int'), 'int'), - 'int_or' : (('int', 'int'), 'int'), - 'int_xor' : (('int', 'int'), 'int'), - 'int_lshift' : (('int', 'int'), 'int'), - 'int_rshift' : (('int', 'int'), 'int'), - 'int_lt' : (('int', 'int'), 'bool'), - 'int_gt' : (('int', 'int'), 'bool'), - 'int_ge' : (('int', 'int'), 'bool'), - 'int_le' : (('int', 'int'), 'bool'), - 'int_eq' : (('int', 'int'), 'bool'), - 'int_ne' : (('int', 'int'), 'bool'), - 'int_is_true' : (('int',), 'bool'), - 'int_is_zero' : (('int',), 'bool'), - 'int_neg' : (('int',), 'int'), - 'int_invert' : (('int',), 'int'), - 'int_add_ovf' : (('int', 'int'), 'int'), - 'int_sub_ovf' : (('int', 'int'), 'int'), - 'int_mul_ovf' : (('int', 'int'), 'int'), - 'int_force_ge_zero':(('int',), 'int'), - 'uint_add' : (('int', 'int'), 'int'), - 'uint_sub' : (('int', 'int'), 'int'), - 'uint_mul' : (('int', 'int'), 'int'), - 'uint_lt' : (('int', 'int'), 'bool'), - 'uint_le' : (('int', 'int'), 'bool'), - 'uint_eq' : (('int', 'int'), 'bool'), - 'uint_ne' : (('int', 'int'), 'bool'), - 'uint_gt' : (('int', 'int'), 'bool'), - 'uint_ge' : (('int', 'int'), 'bool'), - 'uint_xor' : (('int', 'int'), 'int'), - 'uint_rshift' : (('int', 'int'), 'int'), - 'uint_floordiv' : (('int', 'int'), 'int'), - 'float_add' : (('float', 'float'), 'float'), - 'float_sub' : (('float', 'float'), 'float'), - 'float_mul' : (('float', 'float'), 'float'), - 'float_truediv' : (('float', 'float'), 'float'), - 'float_lt' : (('float', 'float'), 'bool'), - 'float_le' : (('float', 'float'), 'bool'), - 'float_eq' : (('float', 'float'), 'bool'), - 'float_ne' : (('float', 'float'), 'bool'), - 'float_gt' : (('float', 'float'), 'bool'), - 'float_ge' : (('float', 'float'), 'bool'), - 'float_neg' : (('float',), 'float'), - 'float_abs' : (('float',), 'float'), - 'cast_float_to_int':(('float',), 'int'), - 'cast_int_to_float':(('int',), 'float'), - 'same_as' : (('int',), 'int'), # could also be ptr=>ptr - 'new_with_vtable' : (('ref',), 'ref'), - 'new' : ((), 'ref'), - 'new_array' : (('int',), 'ref'), - 'oois' : (('ref', 'ref'), 'bool'), - 'ooisnot' : (('ref', 'ref'), 'bool'), - 'instanceof' : (('ref',), 'bool'), - 'subclassof' : (('ref', 'ref'), 'bool'), - 'runtimenew' : (('ref',), 'ref'), - 'setfield_gc' : (('ref', 'intorptr'), None), - 'getfield_gc' : (('ref',), 'intorptr'), - 'getfield_gc_pure': (('ref',), 'intorptr'), - 'setfield_raw' : (('ref', 'intorptr'), None), - 'getfield_raw' : (('ref',), 'intorptr'), - 'getfield_raw_pure': (('ref',), 'intorptr'), - 'setarrayitem_gc' : (('ref', 'int', 'intorptr'), None), - 'getarrayitem_gc' : (('ref', 'int'), 'intorptr'), - 'getarrayitem_gc_pure' : (('ref', 'int'), 'intorptr'), - 'setarrayitem_raw' : (('ref', 'int', 'intorptr'), None), - 'getarrayitem_raw' : (('ref', 'int'), 'intorptr'), - 'getarrayitem_raw_pure' : (('ref', 'int'), 'intorptr'), - 'arraylen_gc' : (('ref',), 'int'), - 'call' : (('ref', 'varargs'), 'intorptr'), - 'call_assembler' : (('varargs',), 'intorptr'), - 'cond_call_gc_wb' : (('ptr', 'ptr'), None), - 'cond_call_gc_wb_array': (('ptr', 'int', 'ptr'), None), - 'oosend' : (('varargs',), 'intorptr'), - 'oosend_pure' : (('varargs',), 'intorptr'), - 'guard_true' : (('bool',), None), - 'guard_false' : (('bool',), None), - 'guard_value' : (('int', 'int'), None), - 'guard_class' : (('ref', 'ref'), None), - 'guard_no_exception' : ((), None), - 'guard_exception' : (('ref',), 'ref'), - 'guard_no_overflow' : ((), None), - 'guard_overflow' : ((), None), - 'guard_nonnull' : (('ref',), None), - 'guard_isnull' : (('ref',), None), - 'guard_nonnull_class' : (('ref', 'ref'), None), - 'newstr' : (('int',), 'ref'), - 'strlen' : (('ref',), 'int'), - 'strgetitem' : (('ref', 'int'), 'int'), - 'strsetitem' : (('ref', 'int', 'int'), None), - 'newunicode' : (('int',), 'ref'), - 'unicodelen' : (('ref',), 'int'), - 'unicodegetitem' : (('ref', 'int'), 'int'), - 'unicodesetitem' : (('ref', 'int', 'int'), 'int'), - 'cast_ptr_to_int' : (('ref',), 'int'), - 'cast_int_to_ptr' : (('int',), 'ref'), - 'debug_merge_point': (('ref', 'int', 'int'), None), - 'force_token' : ((), 'int'), - 'call_may_force' : (('int', 'varargs'), 'intorptr'), - 'guard_not_forced': ((), None), -} - -# ____________________________________________________________ - -class CompiledLoop(object): - has_been_freed = False - invalid = False - - def __init__(self): - self.inputargs = [] - self.operations = [] - - def getargtypes(self): - return [v.concretetype for v in self.inputargs] - - def __repr__(self): - lines = [] - self.as_text(lines, 1) - return 'CompiledLoop %s:\n%s' % (self.inputargs, '\n'.join(lines)) - - def as_text(self, lines, indent): - for op in self.operations: - lines.append('\t'*indent + repr(op)) - -class Operation(object): - result = None - descr = None - jump_target = None - fail_args = None - - def __init__(self, opnum): - self.opnum = opnum - self.args = [] - - def __repr__(self): - if self.result is not None: - sres = repr0(self.result) + ' = ' - else: - sres = '' - return '{%s%s(%s)}' % (sres, self.getopname(), - ', '.join(map(repr0, self.args))) - - def getopname(self): - try: - return resoperation.opname[self.opnum] - except KeyError: - return '<%d>' % self.opnum - - def is_guard(self): - return rop._GUARD_FIRST <= self.opnum <= rop._GUARD_LAST - - def is_final(self): - return rop._FINAL_FIRST <= self.opnum <= rop._FINAL_LAST - -def repr0(x): - if isinstance(x, list): - return '[' + ', '.join(repr0(y) for y in x) + ']' - elif isinstance(x, Constant): - return '(' + repr0(x.value) + ')' - elif isinstance(x, lltype._ptr): - x = llmemory.cast_ptr_to_adr(x) - if x.ptr: - try: - container = x.ptr._obj._normalizedcontainer() - return '* %s' % (container._TYPE._short_name(),) - except AttributeError: - return repr(x) - else: - return 'NULL' - else: - return repr(x) - -def repr_list(lst, types): - res_l = [] - if types and types[-1] == 'varargs': - types = types[:-1] + ('int',) * (len(lst) - len(types) + 1) - assert len(types) == len(lst) - for elem, tp in zip(lst, types): - if isinstance(elem, Constant): - res_l.append('(%s)' % repr1(elem, tp)) - else: - res_l.append(repr1(elem, tp)) - return '[%s]' % (', '.join(res_l)) - -def repr1(x, tp): - if tp == "intorptr": - TYPE = lltype.typeOf(x) - if isinstance(TYPE, lltype.Ptr) and TYPE.TO._gckind == 'gc': - tp = "ref" - else: - tp = "int" - if tp == 'int': - return str(x) - elif tp == 'void': - return '---' - elif tp == 'ref': - if not x: - return '(* None)' - if isinstance(x, int): - # XXX normalize? - ptr = str(llmemory.cast_int_to_adr(x)) - elif isinstance(ootype.typeOf(x), ootype.OOType): - return repr(x) - else: - if getattr(x, '_fake', None): - return repr(x) - if lltype.typeOf(x) == llmemory.GCREF: - TP = lltype.Ptr(lltype.typeOf(x._obj.container)) - ptr = lltype.cast_opaque_ptr(TP, x) - else: - ptr = x - try: - container = ptr._obj._normalizedcontainer() - return '(* %s)' % (container._TYPE._short_name(),) - except AttributeError: - return '(%r)' % (ptr,) - elif tp == 'bool': - assert x == 0 or x == 1 - return str(bool(x)) - #elif tp == 'fieldname': - # return str(symbolic.TokenToField[x...][1]) - elif tp == 'float': - return str(x) - else: - raise NotImplementedError("tp = %s" % tp) - -_variables = [] - -def compile_start(): - del _variables[:] - return _to_opaque(CompiledLoop()) - -def mark_as_free(loop): - loop = _from_opaque(loop) - assert not loop.has_been_freed - loop.has_been_freed = True - -def compile_start_int_var(loop): - return compile_start_ref_var(loop, lltype.Signed) - -def compile_start_float_var(loop): - return compile_start_ref_var(loop, longlong.FLOATSTORAGE) - -def compile_start_ref_var(loop, TYPE): - loop = _from_opaque(loop) - assert not loop.operations - v = Variable() - v.concretetype = TYPE - loop.inputargs.append(v) - r = len(_variables) - _variables.append(v) - return r - -def compile_started_vars(clt): - if not hasattr(clt, '_debug_argtypes'): # only when compiling the loop - argtypes = [v.concretetype for v in _variables] - try: - clt._debug_argtypes = argtypes - except AttributeError: # when 'clt' is actually a translated - pass # GcStruct - -def compile_add(loop, opnum): - loop = _from_opaque(loop) - loop.operations.append(Operation(opnum)) - -def compile_add_descr(loop, ofs, type, arg_types, extrainfo, width): - from rpython.jit.backend.llgraph.runner import Descr - loop = _from_opaque(loop) - op = loop.operations[-1] - assert isinstance(type, str) and len(type) == 1 - op.descr = Descr(ofs, type, arg_types=arg_types, extrainfo=extrainfo, width=width) - -def compile_add_descr_arg(loop, ofs, type, arg_types): - from rpython.jit.backend.llgraph.runner import Descr - loop = _from_opaque(loop) - op = loop.operations[-1] - assert isinstance(type, str) and len(type) == 1 - op.args.append(Descr(ofs, type, arg_types=arg_types)) - -def compile_add_loop_token(loop, descr): - if we_are_translated(): - raise ValueError("CALL_ASSEMBLER not supported") - loop = _from_opaque(loop) - op = loop.operations[-1] - op.descr = weakref.ref(descr) - -TARGET_TOKENS = weakref.WeakKeyDictionary() - -def compile_add_target_token(loop, descr, clt): - # here, 'clt' is the compiled_loop_token of the original loop that - # we are compiling - loop = _from_opaque(loop) - op = loop.operations[-1] - descrobj = _normalize(descr) - TARGET_TOKENS[descrobj] = loop, len(loop.operations), op.args, clt - -def compile_add_var(loop, intvar): - loop = _from_opaque(loop) - op = loop.operations[-1] - op.args.append(_variables[intvar]) - -def compile_add_int_const(loop, value): - compile_add_ref_const(loop, value, lltype.Signed) - -def compile_add_float_const(loop, value): - compile_add_ref_const(loop, value, longlong.FLOATSTORAGE) - -def compile_add_ref_const(loop, value, TYPE): - loop = _from_opaque(loop) - const = Constant(value) - const.concretetype = TYPE - op = loop.operations[-1] - op.args.append(const) - -def compile_add_int_result(loop): - return compile_add_ref_result(loop, lltype.Signed) - -def compile_add_float_result(loop): - return compile_add_ref_result(loop, longlong.FLOATSTORAGE) - -def compile_add_ref_result(loop, TYPE): - loop = _from_opaque(loop) - v = Variable() - v.concretetype = TYPE - op = loop.operations[-1] - op.result = v - r = len(_variables) - _variables.append(v) - return r - -def compile_add_jump_target(loop, targettoken, source_clt): - loop = _from_opaque(loop) - descrobj = _normalize(targettoken) - (loop_target, target_opindex, target_inputargs, target_clt - ) = TARGET_TOKENS[descrobj] - # - try: - assert source_clt._debug_argtypes == target_clt._debug_argtypes - except AttributeError: # when translated - pass - # - op = loop.operations[-1] - op.jump_target = loop_target - op.jump_target_opindex = target_opindex - op.jump_target_inputargs = target_inputargs - assert op.opnum == rop.JUMP - assert [v.concretetype for v in op.args] == ( - [v.concretetype for v in target_inputargs]) - # - if loop_target == loop: - log.info("compiling new loop") - else: - log.info("compiling new bridge") - -def compile_add_guard_jump_target(loop, loop_target): - loop = _from_opaque(loop) - loop_target = _from_opaque(loop_target) - op = loop.operations[-1] - assert op.is_guard() - op.jump_target = loop_target - -def compile_add_fail(loop, fail_index): - loop = _from_opaque(loop) - index = len(loop.operations)-1 - op = loop.operations[index] - op.fail_index = fail_index - return index - -def compile_add_fail_arg(loop, intvar): - loop = _from_opaque(loop) - op = loop.operations[-1] - if op.fail_args is None: - op.fail_args = [] - if intvar == -1: - op.fail_args.append(None) - else: - op.fail_args.append(_variables[intvar]) - -def compile_redirect_fail(old_loop, old_index, new_loop): - old_loop = _from_opaque(old_loop) - new_loop = _from_opaque(new_loop) - guard_op = old_loop.operations[old_index] - assert guard_op.is_guard() - guard_op.jump_target = new_loop - # check that the bridge's inputargs are of the correct number and - # kind for the guard - if guard_op.fail_args is not None: - argkinds = [v.concretetype for v in guard_op.fail_args if v] - else: - argkinds = [] - assert argkinds == [v.concretetype for v in new_loop.inputargs] - -# ------------------------------ - -class Frame(object): - OPHANDLERS = [None] * (rop._LAST+1) - - def __init__(self, cpu): - self.verbose = False - self.cpu = cpu - self.opindex = 1 - self._forced = False - self._may_force = -1 - - def getenv(self, v): - from rpython.jit.backend.llgraph.runner import Descr - if isinstance(v, Constant): - return v.value - elif isinstance(v, Descr): - return v - else: - return self.env[v] - - def _populate_fail_args(self, op, skip=None): - fail_args = [] - if op.fail_args: - for fail_arg in op.fail_args: - if fail_arg is None: - fail_args.append(None) - elif fail_arg is skip: - fail_args.append(fail_arg.concretetype._defl()) - else: - fail_args.append(self.getenv(fail_arg)) - self.fail_args = fail_args - self.fail_index = op.fail_index - - def execute(self): - """Execute all operations in a loop, - possibly following to other loops as well. - """ - global _last_exception - assert _last_exception is None, "exception left behind" - verbose = True - self.opindex = 0 - while True: - assert not self.loop.has_been_freed - op = self.loop.operations[self.opindex] - args = [self.getenv(v) for v in op.args] - if not op.is_final(): - try: - result = self.execute_operation(op.opnum, args, op.descr, - verbose) - except GuardFailed: - assert op.is_guard() - _stats.exec_conditional_jumps += 1 - if op.jump_target is not None: - # a patched guard, pointing to further code - if op.fail_args: - args = [self.getenv(v) for v in op.fail_args if v] - else: - args = [] - assert len(op.jump_target.inputargs) == len(args) - self.env = dict(zip(op.jump_target.inputargs, args)) - self.loop = op.jump_target - self.opindex = 0 - continue - else: - self._populate_fail_args(op) - # a non-patched guard - if self.verbose: - log.trace('failed: %s' % ( - ', '.join(map(str, fail_args)),)) - return op.fail_index - #verbose = self.verbose - assert (result is None) == (op.result is None) - if op.result is not None: - RESTYPE = op.result.concretetype - if RESTYPE is lltype.Signed: - x = self.as_int(result) - elif RESTYPE is llmemory.GCREF: - x = self.as_ptr(result) - elif RESTYPE is ootype.Object: - x = self.as_object(result) - elif RESTYPE is longlong.FLOATSTORAGE: - x = self.as_floatstorage(result) - else: - raise Exception("op.result.concretetype is %r" - % (RESTYPE,)) - self.env[op.result] = x - self.opindex += 1 - continue - if op.opnum == rop.JUMP: - inputargs = op.jump_target_inputargs - assert len(inputargs) == len(args) - self.env = dict(zip(inputargs, args)) - self.loop = op.jump_target - self.opindex = op.jump_target_opindex - _stats.exec_jumps += 1 - elif op.opnum == rop.FINISH: - if self.verbose: - log.trace('finished: %s' % ( - ', '.join(map(str, args)),)) - self.fail_args = args - return op.fail_index - - else: - assert 0, "unknown final operation %d" % (op.opnum,) - - def execute_operation(self, opnum, values, descr, verbose): - """Execute a single operation. - """ - ophandler = self.OPHANDLERS[opnum] - if ophandler is None: - self._define_impl(opnum) - ophandler = self.OPHANDLERS[opnum] - assert ophandler is not None, "missing impl for op %d" % opnum - opname = resoperation.opname[opnum].lower() - exec_counters = _stats.exec_counters - exec_counters[opname] = exec_counters.get(opname, 0) + 1 - for i in range(len(values)): - if isinstance(values[i], ComputedIntSymbolic): - values[i] = values[i].compute_fn() - res = NotImplemented - try: - res = ophandler(self, descr, *values) - finally: - if 0: # if verbose: - argtypes, restype = TYPES[opname] - if res is None: - resdata = '' - elif res is NotImplemented: - resdata = '*fail*' - else: - resdata = '-> ' + repr1(res, restype) - # fish the types - log.cpu('\t%s %s %s' % (opname, repr_list(values, argtypes), - resdata)) - return res - - def as_int(self, x): - return cast_to_int(x) - - def as_ptr(self, x): - return cast_to_ptr(x) - - def as_object(self, x): - return ootype.cast_to_object(x) - - def as_floatstorage(self, x): - return cast_to_floatstorage(x) - - def log_progress(self): - count = sum(_stats.exec_counters.values()) - count_jumps = _stats.exec_jumps - log.trace('ran %d operations, %d jumps' % (count, count_jumps)) - - # ---------- - - @classmethod - def _define_impl(cls, opnum): - opname = resoperation.opname[opnum] - try: - op = getattr(cls, 'op_' + opname.lower()) # op_guard_true etc. - except AttributeError: - try: - impl = globals()['do_' + opname.lower()] # do_arraylen_gc etc. - def op(self, descr, *args): - if descr is None: - return impl(*args) - else: - return impl(descr, *args) - except KeyError: - op = cls._make_impl_from_blackhole_interp(opname) - cls.OPHANDLERS[opnum] = op - - @classmethod - def _make_impl_from_blackhole_interp(cls, opname): - from rpython.jit.metainterp.blackhole import BlackholeInterpreter - name = 'bhimpl_' + opname.lower() - func = BlackholeInterpreter.__dict__[name] - for argtype in func.argtypes: - assert argtype in ('i', 'r', 'f') - # - def _op_default_implementation(self, descr, *args): - # for all operations implemented in the blackhole interpreter - return func(*args) - # - return _op_default_implementation - - def op_label(self, _, *args): - op = self.loop.operations[self.opindex] - assert op.opnum == rop.LABEL - assert len(op.args) == len(args) - newenv = {} - for v, value in zip(op.args, args): - newenv[v] = value - self.env = newenv - - def op_debug_merge_point(self, _, *args): - from rpython.jit.metainterp.warmspot import get_stats - try: - stats = get_stats() - except AttributeError: - pass - else: - stats.add_merge_point_location(args[1:]) - pass - - def op_guard_true(self, _, value): - if not value: - raise GuardFailed - - def op_guard_false(self, _, value): - if value: - raise GuardFailed - - op_guard_nonnull = op_guard_true - op_guard_isnull = op_guard_false - - def op_guard_class(self, _, value, expected_class): - value = lltype.cast_opaque_ptr(rclass.OBJECTPTR, value) - expected_class = llmemory.cast_adr_to_ptr( - llmemory.cast_int_to_adr(expected_class), - rclass.CLASSTYPE) - if value.typeptr != expected_class: - raise GuardFailed - - def op_guard_nonnull_class(self, _, value, expected_class): - if not value: - raise GuardFailed - self.op_guard_class(_, value, expected_class) - - def op_guard_value(self, _, value, expected_value): - if value != expected_value: - raise GuardFailed - - def op_guard_no_exception(self, _): - if _last_exception: - raise GuardFailed - - def _check_exception(self, expected_exception): - global _last_exception - expected_exception = self._cast_exception(expected_exception) - assert expected_exception - exc = _last_exception - if exc: - got = exc.args[0] - # exact match! - if got != expected_exception: - return False - return True - else: - return False - - def _cast_exception(self, exception): - return llmemory.cast_adr_to_ptr( - llmemory.cast_int_to_adr(exception), - rclass.CLASSTYPE) - - def _issubclass(self, cls1, cls2): - return rclass.ll_issubclass(cls1, cls2) - - def op_guard_exception(self, _, expected_exception): - global _last_exception - if not self._check_exception(expected_exception): - raise GuardFailed - res = _last_exception[1] - _last_exception = None - return res - - def op_guard_no_overflow(self, _): - flag = self.overflow_flag - del self.overflow_flag - if flag: - raise GuardFailed - - def op_guard_overflow(self, _): - flag = self.overflow_flag - del self.overflow_flag - if not flag: - raise GuardFailed - - def op_int_add_ovf(self, _, x, y): - try: - z = ovfcheck(x + y) - except OverflowError: - ovf = True - z = 0 - else: - ovf = False - self.overflow_flag = ovf - return z - - def op_int_sub_ovf(self, _, x, y): - try: - z = ovfcheck(x - y) - except OverflowError: - ovf = True - z = 0 - else: - ovf = False - self.overflow_flag = ovf - return z - - def op_int_mul_ovf(self, _, x, y): - try: - z = ovfcheck(x * y) - except OverflowError: - ovf = True - z = 0 - else: - ovf = False - self.overflow_flag = ovf - return z - - def op_keepalive(self, _, x): - pass - - # ---------- - # delegating to the builtins do_xxx() (done automatically for simple cases) - - def op_getarrayitem_gc(self, arraydescr, array, index): - if arraydescr.typeinfo == REF: - return do_getarrayitem_gc_ptr(array, index) - elif arraydescr.typeinfo == INT: - return do_getarrayitem_gc_int(array, index) - elif arraydescr.typeinfo == FLOAT: - return do_getarrayitem_gc_float(array, index) - else: - raise NotImplementedError - - op_getarrayitem_gc_pure = op_getarrayitem_gc - - def op_getarrayitem_raw(self, arraydescr, array, index): - if arraydescr.typeinfo == REF: - raise NotImplementedError("getarrayitem_raw -> gcref") - elif arraydescr.typeinfo == INT: - return do_getarrayitem_raw_int(array, index, arraydescr.ofs) - elif arraydescr.typeinfo == FLOAT: - return do_getarrayitem_raw_float(array, index) - else: - raise NotImplementedError - - op_getarrayitem_raw_pure = op_getarrayitem_raw - - def op_getfield_gc(self, fielddescr, struct): - if fielddescr.typeinfo == REF: - return do_getfield_gc_ptr(struct, fielddescr.ofs) - elif fielddescr.typeinfo == INT: - return do_getfield_gc_int(struct, fielddescr.ofs) - elif fielddescr.typeinfo == FLOAT: - return do_getfield_gc_float(struct, fielddescr.ofs) - else: - raise NotImplementedError - - op_getfield_gc_pure = op_getfield_gc - - def op_getfield_raw(self, fielddescr, struct): - if fielddescr.typeinfo == REF: - return do_getfield_raw_ptr(struct, fielddescr.ofs) - elif fielddescr.typeinfo == INT: - return do_getfield_raw_int(struct, fielddescr.ofs) - elif fielddescr.typeinfo == FLOAT: - return do_getfield_raw_float(struct, fielddescr.ofs) - else: - raise NotImplementedError - - op_getfield_raw_pure = op_getfield_raw - - def op_raw_store(self, arraydescr, addr, offset, value): - if arraydescr.typeinfo == REF: - raise AssertionError("cannot store GC pointer in raw storage") - elif arraydescr.typeinfo == INT: - do_raw_store_int(addr, offset, arraydescr.ofs, value) - elif arraydescr.typeinfo == FLOAT: - do_raw_store_float(addr, offset, value) - else: - raise NotImplementedError - - def op_raw_load(self, arraydescr, addr, offset): - if arraydescr.typeinfo == REF: - raise AssertionError("cannot store GC pointer in raw storage") - elif arraydescr.typeinfo == INT: - return do_raw_load_int(addr, offset, arraydescr.ofs) - elif arraydescr.typeinfo == FLOAT: - return do_raw_load_float(addr, offset) - else: - raise NotImplementedError - - def op_new(self, size): - return do_new(size.ofs) - - def op_new_with_vtable(self, descr, vtable): - assert descr is None - descr = heaptracker.vtable2descr(self.cpu, vtable) - result = do_new(descr.ofs) - value = lltype.cast_opaque_ptr(rclass.OBJECTPTR, result) - value.typeptr = cast_from_int(rclass.CLASSTYPE, vtable) - return result - - def op_setarrayitem_gc(self, arraydescr, array, index, newvalue): - if arraydescr.typeinfo == REF: - do_setarrayitem_gc_ptr(array, index, newvalue) - elif arraydescr.typeinfo == INT: - do_setarrayitem_gc_int(array, index, newvalue) - elif arraydescr.typeinfo == FLOAT: - do_setarrayitem_gc_float(array, index, newvalue) - else: - raise NotImplementedError - - def op_setarrayitem_raw(self, arraydescr, array, index, newvalue): - if arraydescr.typeinfo == REF: - raise NotImplementedError("setarrayitem_raw <- gcref") - elif arraydescr.typeinfo == INT: - do_setarrayitem_raw_int(array, index, newvalue, arraydescr.ofs) - elif arraydescr.typeinfo == FLOAT: - do_setarrayitem_raw_float(array, index, newvalue) - else: - raise NotImplementedError - - def op_getinteriorfield_gc(self, descr, array, index): - if descr.typeinfo == REF: - return do_getinteriorfield_gc_ptr(array, index, descr.ofs) - elif descr.typeinfo == INT: - return do_getinteriorfield_gc_int(array, index, descr.ofs) - elif descr.typeinfo == FLOAT: - return do_getinteriorfield_gc_float(array, index, descr.ofs) - else: - raise NotImplementedError - - def op_getinteriorfield_raw(self, descr, array, index): - if descr.typeinfo == REF: - return do_getinteriorfield_raw_ptr(array, index, descr.width, descr.ofs) - elif descr.typeinfo == INT: - return do_getinteriorfield_raw_int(array, index, descr.width, descr.ofs) - elif descr.typeinfo == FLOAT: - return do_getinteriorfield_raw_float(array, index, descr.width, descr.ofs) - else: - raise NotImplementedError - - def op_setinteriorfield_gc(self, descr, array, index, newvalue): - if descr.typeinfo == REF: - return do_setinteriorfield_gc_ptr(array, index, descr.ofs, - newvalue) - elif descr.typeinfo == INT: - return do_setinteriorfield_gc_int(array, index, descr.ofs, - newvalue) - elif descr.typeinfo == FLOAT: - return do_setinteriorfield_gc_float(array, index, descr.ofs, - newvalue) - else: - raise NotImplementedError - - def op_setinteriorfield_raw(self, descr, array, index, newvalue): - if descr.typeinfo == REF: - return do_setinteriorfield_raw_ptr(array, index, newvalue, descr.width, descr.ofs) - elif descr.typeinfo == INT: - return do_setinteriorfield_raw_int(array, index, newvalue, descr.width, descr.ofs) - elif descr.typeinfo == FLOAT: - return do_setinteriorfield_raw_float(array, index, newvalue, descr.width, descr.ofs) - else: - raise NotImplementedError - - def op_setfield_gc(self, fielddescr, struct, newvalue): - if fielddescr.typeinfo == REF: - do_setfield_gc_ptr(struct, fielddescr.ofs, newvalue) - elif fielddescr.typeinfo == INT: - do_setfield_gc_int(struct, fielddescr.ofs, newvalue) - elif fielddescr.typeinfo == FLOAT: - do_setfield_gc_float(struct, fielddescr.ofs, newvalue) - else: - raise NotImplementedError - - def op_setfield_raw(self, fielddescr, struct, newvalue): - if fielddescr.typeinfo == REF: - do_setfield_raw_ptr(struct, fielddescr.ofs, newvalue) - elif fielddescr.typeinfo == INT: - do_setfield_raw_int(struct, fielddescr.ofs, newvalue) - elif fielddescr.typeinfo == FLOAT: - do_setfield_raw_float(struct, fielddescr.ofs, newvalue) - else: - raise NotImplementedError - - def op_call(self, calldescr, func, *args): - effectinfo = calldescr.get_extra_info() - if effectinfo is not None and hasattr(effectinfo, 'oopspecindex'): - oopspecindex = effectinfo.oopspecindex - if oopspecindex == EffectInfo.OS_MATH_SQRT: - return do_math_sqrt(args[0]) - return self._do_call(calldescr, func, args, call_with_llptr=False) - - def op_call_release_gil(self, calldescr, func, *args): - return self._do_call(calldescr, func, args, call_with_llptr=True) - - def _do_call(self, calldescr, func, args, call_with_llptr): - global _last_exception - assert _last_exception is None, "exception left behind" - assert _call_args_i == _call_args_r == _call_args_f == [] - args_in_order = [] - for x in args: - T = lltype.typeOf(x) - if T is lltype.Signed: - args_in_order.append('i') - _call_args_i.append(x) - elif T == llmemory.GCREF: - args_in_order.append('r') - _call_args_r.append(x) - elif T is longlong.FLOATSTORAGE: - args_in_order.append('f') - _call_args_f.append(x) - else: - raise TypeError(x) - try: - return _do_call_common(func, args_in_order, calldescr, - call_with_llptr) - except LLException, lle: - _last_exception = lle - d = {'v': None, - REF: lltype.nullptr(llmemory.GCREF.TO), - INT: 0, - FLOAT: 0.0} - return d[calldescr.typeinfo] - - def op_cond_call_gc_wb(self, descr, a, b): - py.test.skip("cond_call_gc_wb not supported") - - def op_cond_call_gc_wb_array(self, descr, a, b, c): - py.test.skip("cond_call_gc_wb_array not supported") - - def op_oosend(self, descr, obj, *args): - raise NotImplementedError("oosend for lltype backend??") - - op_oosend_pure = op_oosend - - def op_new_array(self, arraydescr, count): - return do_new_array(arraydescr.ofs, count) - - def op_force_token(self, descr): - opaque_frame = _to_opaque(self) - return llmemory.cast_ptr_to_adr(opaque_frame) - - def op_read_timestamp(self, descr): - return read_timestamp() - - def op_call_may_force(self, calldescr, func, *args): - assert not self._forced - self._may_force = self.opindex - try: - return self.op_call(calldescr, func, *args) - finally: - self._may_force = -1 - - def op_call_assembler(self, wref_loop_token, *args): - if we_are_translated(): - raise ValueError("CALL_ASSEMBLER not supported") - return self._do_call_assembler(wref_loop_token, *args) - - def _do_call_assembler(self, wref_loop_token, *args): - global _last_exception - loop_token = wref_loop_token() - assert loop_token, "CALL_ASSEMBLER to a target that already died" - ctl = loop_token.compiled_loop_token - if hasattr(ctl, 'redirected'): - return self._do_call_assembler(ctl.redirected, *args) - assert not self._forced - self._may_force = self.opindex - try: - inpargs = _from_opaque(ctl.compiled_version).inputargs - assert len(inpargs) == len(args) - for i, inparg in enumerate(inpargs): - TYPE = inparg.concretetype - if TYPE is lltype.Signed: - set_future_value_int(i, args[i]) - elif isinstance(TYPE, lltype.Ptr): - set_future_value_ref(i, args[i]) - elif TYPE is longlong.FLOATSTORAGE: - set_future_value_float(i, args[i]) - else: - raise Exception("Nonsense type %s" % TYPE) - - failindex = self.cpu._execute_token(loop_token) - jd = loop_token.outermost_jitdriver_sd - assert jd is not None, ("call_assembler(): the loop_token needs " - "to have 'outermost_jitdriver_sd'") - if jd.index_of_virtualizable != -1: - vable = args[jd.index_of_virtualizable] - else: - vable = lltype.nullptr(llmemory.GCREF.TO) - # - # Emulate the fast path - if failindex == self.cpu.done_with_this_frame_int_v: - reset_vable(jd, vable) - return self.cpu.get_latest_value_int(0) - if failindex == self.cpu.done_with_this_frame_ref_v: - reset_vable(jd, vable) - return self.cpu.get_latest_value_ref(0) - if failindex == self.cpu.done_with_this_frame_float_v: - reset_vable(jd, vable) - return self.cpu.get_latest_value_float(0) - if failindex == self.cpu.done_with_this_frame_void_v: - reset_vable(jd, vable) - return None - # - assembler_helper_ptr = jd.assembler_helper_adr.ptr # fish - try: - return assembler_helper_ptr(failindex, vable) - except LLException, lle: - assert _last_exception is None, "exception left behind" - _last_exception = lle - # fish op - op = self.loop.operations[self.opindex] - if op.result is not None: - return 0 - finally: - self._may_force = -1 - - def op_guard_not_forced(self, descr): - forced = self._forced - self._forced = False - if forced: - raise GuardFailed - - def op_guard_not_invalidated(self, descr): - if self.loop.invalid: - raise GuardFailed - -class OOFrame(Frame): - - OPHANDLERS = [None] * (rop._LAST+1) - - def op_new_with_vtable(self, descr, vtable): - assert descr is None - typedescr = get_class_size(self.memocast, vtable) - return ootype.cast_to_object(ootype.new(typedescr.TYPE)) - - def op_new_array(self, typedescr, count): - res = ootype.oonewarray(typedescr.ARRAY, count) - return ootype.cast_to_object(res) - - def op_getfield_gc(self, fielddescr, obj): - TYPE = fielddescr.TYPE - fieldname = fielddescr.fieldname - _, T = TYPE._lookup_field(fieldname) - obj = ootype.cast_from_object(TYPE, obj) - res = getattr(obj, fieldname) - if isinstance(T, ootype.OOType): - return ootype.cast_to_object(res) - return res - - op_getfield_gc_pure = op_getfield_gc - - def op_setfield_gc(self, fielddescr, obj, newvalue): - TYPE = fielddescr.TYPE - fieldname = fielddescr.fieldname - _, T = TYPE._lookup_field(fieldname) - obj = ootype.cast_from_object(TYPE, obj) - if isinstance(ootype.typeOf(newvalue), ootype.OOType): - newvalue = ootype.cast_from_object(T, newvalue) - elif isinstance(T, lltype.Primitive): - newvalue = lltype.cast_primitive(T, newvalue) - setattr(obj, fieldname, newvalue) - - def op_getarrayitem_gc(self, typedescr, obj, index): - array = ootype.cast_from_object(typedescr.ARRAY, obj) - res = array.ll_getitem_fast(index) - if isinstance(typedescr.TYPE, ootype.OOType): - return ootype.cast_to_object(res) - return res - - op_getarrayitem_gc_pure = op_getarrayitem_gc - - def op_setarrayitem_gc(self, typedescr, obj, index, objnewvalue): - array = ootype.cast_from_object(typedescr.ARRAY, obj) - if ootype.typeOf(objnewvalue) == ootype.Object: - newvalue = ootype.cast_from_object(typedescr.TYPE, objnewvalue) - else: - newvalue = objnewvalue - array.ll_setitem_fast(index, newvalue) - - def op_arraylen_gc(self, typedescr, obj): - array = ootype.cast_from_object(typedescr.ARRAY, obj) - return array.ll_length() - - def op_call(self, calldescr, func, *args): - sm = ootype.cast_from_object(calldescr.FUNC, func) - newargs = cast_call_args(calldescr.FUNC.ARGS, args) - res = call_maybe_on_top_of_llinterp(sm, newargs) - if isinstance(calldescr.FUNC.RESULT, ootype.OOType): - return ootype.cast_to_object(res) - return res - - def op_oosend(self, descr, obj, *args): - METH = descr.METH - obj = ootype.cast_from_object(descr.SELFTYPE, obj) - meth = getattr(obj, descr.methname) - newargs = cast_call_args(METH.ARGS, args) - res = call_maybe_on_top_of_llinterp(meth, newargs) - if isinstance(METH.RESULT, ootype.OOType): - return ootype.cast_to_object(res) - return res - - op_oosend_pure = op_oosend - - def op_guard_class(self, _, value, expected_class): - value = ootype.cast_from_object(ootype.ROOT, value) - expected_class = ootype.cast_from_object(ootype.Class, expected_class) - if ootype.classof(value) is not expected_class: - raise GuardFailed - - def op_runtimenew(self, _, cls): - cls = ootype.cast_from_object(ootype.Class, cls) - res = ootype.runtimenew(cls) - return ootype.cast_to_object(res) - - def op_instanceof(self, typedescr, obj): - inst = ootype.cast_from_object(ootype.ROOT, obj) - return ootype.instanceof(inst, typedescr.TYPE) - - def op_subclassof(self, _, obj1, obj2): - cls1 = ootype.cast_from_object(ootype.Class, obj1) - cls2 = ootype.cast_from_object(ootype.Class, obj2) - return ootype.subclassof(cls1, cls2) - - def _cast_exception(self, exception): - return ootype.cast_from_object(ootype.Class, exception) - - def _issubclass(self, cls1, cls2): - return ootype.subclassof(cls1, cls2) - -# ____________________________________________________________ - -def cast_to_int(x): - TP = lltype.typeOf(x) - if isinstance(TP, lltype.Ptr): - return heaptracker.adr2int(llmemory.cast_ptr_to_adr(x)) - if TP == llmemory.Address: - return heaptracker.adr2int(x) - if TP is lltype.SingleFloat: - return longlong.singlefloat2int(x) - return lltype.cast_primitive(lltype.Signed, x) - -def cast_from_int(TYPE, x): - if isinstance(TYPE, lltype.Ptr): - if isinstance(x, (int, long, llmemory.AddressAsInt)): - x = llmemory.cast_int_to_adr(x) - if TYPE is rffi.VOIDP or ( - hasattr(TYPE.TO, '_hints') and - TYPE.TO._hints.get("uncast_on_llgraph")): - # assume that we want a "C-style" cast, without typechecking the value - return rffi.cast(TYPE, x) - return llmemory.cast_adr_to_ptr(x, TYPE) - elif TYPE == llmemory.Address: - if isinstance(x, (int, long, llmemory.AddressAsInt)): - x = llmemory.cast_int_to_adr(x) - assert lltype.typeOf(x) == llmemory.Address - return x - elif TYPE is lltype.SingleFloat: - assert lltype.typeOf(x) is lltype.Signed - return longlong.int2singlefloat(x) - else: - if lltype.typeOf(x) == llmemory.Address: - x = heaptracker.adr2int(x) - return lltype.cast_primitive(TYPE, x) - -def cast_to_ptr(x): - assert isinstance(lltype.typeOf(x), lltype.Ptr) - return lltype.cast_opaque_ptr(llmemory.GCREF, x) - -def cast_from_ptr(TYPE, x): - return lltype.cast_opaque_ptr(TYPE, x) - -def cast_to_floatstorage(x): - if isinstance(x, float): - return longlong.getfloatstorage(x) # common case - if IS_32_BIT: - assert longlong.supports_longlong - if isinstance(x, r_longlong): - return x - if isinstance(x, r_ulonglong): - return rffi.cast(lltype.SignedLongLong, x) - raise TypeError(type(x)) - -def cast_from_floatstorage(TYPE, x): - assert isinstance(x, longlong.r_float_storage) - if TYPE is lltype.Float: - return longlong.getrealfloat(x) - if longlong.is_longlong(TYPE): - return rffi.cast(TYPE, x) - raise TypeError(TYPE) - - -def new_frame(is_oo, cpu): - if is_oo: - frame = OOFrame(cpu) - else: - frame = Frame(cpu) - return _to_opaque(frame) - -_future_values = [] - -def frame_clear(frame, loop): - frame = _from_opaque(frame) - loop = _from_opaque(loop) - assert len(_future_values) == len(loop.inputargs) - frame.loop = loop - frame.env = {} - for i in range(len(loop.inputargs)): - expected_type = loop.inputargs[i].concretetype - assert lltype.typeOf(_future_values[i]) == expected_type - frame.env[loop.inputargs[i]] = _future_values[i] - del _future_values[:] - -def set_future_value_int(index, value): - assert lltype.typeOf(value) is lltype.Signed - set_future_value_ref(index, value) - -def set_future_value_float(index, value): - assert isinstance(value, longlong.r_float_storage) - set_future_value_ref(index, value) - -def set_future_value_ref(index, value): - del _future_values[index:] - assert len(_future_values) == index - _future_values.append(value) - -def frame_execute(frame): - frame = _from_opaque(frame) - if frame.verbose: - values = [frame.env[v] for v in frame.loop.inputargs] - log.trace('Entering CPU frame <- %r' % (values,)) - try: - result = frame.execute() - if frame.verbose: - log.trace('Leaving CPU frame -> #%d' % (result,)) - frame.log_progress() - except Exception, e: - log.ERROR('%s in CPU frame: %s' % (e.__class__.__name__, e)) - # Only invoke pdb when io capturing is not on otherwise py.io complains. - if py.test.config.option.capture == 'no': - import sys, pdb - pdb.post_mortem(sys.exc_info()[2]) - raise - del frame.env - return result - -def frame_int_getvalue(frame, num): - frame = _from_opaque(frame) - assert num >= 0 - x = frame.fail_args[num] - assert lltype.typeOf(x) is lltype.Signed - return x - -def frame_float_getvalue(frame, num): - frame = _from_opaque(frame) - assert num >= 0 - x = frame.fail_args[num] - assert lltype.typeOf(x) is longlong.FLOATSTORAGE - return x - -def frame_ptr_getvalue(frame, num): - frame = _from_opaque(frame) - assert num >= 0 - x = frame.fail_args[num] - assert lltype.typeOf(x) == llmemory.GCREF - return x - -def frame_get_value_count(frame): - frame = _from_opaque(frame) - return len(frame.fail_args) - -def frame_clear_latest_values(frame, count): - frame = _from_opaque(frame) - assert count == len(frame.fail_args) - del frame.fail_args - -_last_exception = None - -def grab_exc_value(): - global _last_exception - if _last_exception is not None: - result = _last_exception.args[1] - _last_exception = None - return lltype.cast_opaque_ptr(llmemory.GCREF, result) - else: - return lltype.nullptr(llmemory.GCREF.TO) - -##_pseudo_exceptions = {} - -##def _get_error(Class): -## if _llinterp.typer is not None: -## llframe = _llinterp.frame_class(None, None, _llinterp) -## try: -## llframe.make_llexception(Class()) -## except LLException, e: -## return e -## else: -## assert 0, "should have raised" -## else: -## # for tests, a random emulated ll_inst will do -## if Class not in _pseudo_exceptions: -## ll_inst = lltype.malloc(rclass.OBJECT, zero=True) -## ll_inst.typeptr = lltype.malloc(rclass.OBJECT_VTABLE, -## immortal=True) -## _pseudo_exceptions[Class] = LLException(ll_inst.typeptr, ll_inst) -## return _pseudo_exceptions[Class] - -##def get_overflow_error_value(): -## return lltype.cast_opaque_ptr(llmemory.GCREF, -## _get_error(OverflowError).args[1]) - -def force(opaque_frame): - frame = _from_opaque(opaque_frame) - assert not frame._forced - frame._forced = True - assert frame._may_force >= 0 - call_op = frame.loop.operations[frame._may_force] - guard_op = frame.loop.operations[frame._may_force+1] - opnum = call_op.opnum - assert opnum == rop.CALL_MAY_FORCE or opnum == rop.CALL_ASSEMBLER - frame._populate_fail_args(guard_op, skip=call_op.result) - return frame.fail_index - -def get_forced_token_frame(force_token): - opaque_frame = llmemory.cast_adr_to_ptr(force_token, - lltype.Ptr(_TO_OPAQUE[Frame])) - return opaque_frame - -def get_frame_forced_token(opaque_frame): - return llmemory.cast_ptr_to_adr(opaque_frame) - -##def cast_adr_to_int(memocast, adr): -## # xxx slow -## assert lltype.typeOf(adr) == llmemory.Address -## memocast = _from_opaque(memocast) -## addresses = memocast.addresses -## for i in xrange(len(addresses)-1, -1, -1): -## if addresses[i] == adr: -## return i -## i = len(addresses) -## addresses.append(adr) -## return i - -##def cast_int_to_adr(memocast, int): -## memocast = _from_opaque(memocast) -## assert 0 <= int < len(memocast.addresses) -## return memocast.addresses[int] - -##def get_class_size(memocast, vtable): -## memocast = _from_opaque(memocast) -## return memocast.vtable_to_size[vtable] - -##def set_class_size(memocast, vtable, size): -## memocast = _from_opaque(memocast) -## memocast.vtable_to_size[vtable] = size - -class GuardFailed(Exception): - pass - -# ____________________________________________________________ - - -def do_same_as(x): - return x - -def do_arraylen_gc(arraydescr, array): - array = array._obj.container - return array.getlength() - -def do_strlen(string): - str = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string) - return len(str.chars) - -def do_strgetitem(string, index): - str = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string) - return ord(str.chars[index]) - -def do_unicodelen(string): - uni = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), string) - return len(uni.chars) - -def do_unicodegetitem(string, index): - uni = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), string) - return ord(uni.chars[index]) - -def do_getarrayitem_gc_int(array, index): - array = array._obj.container - return cast_to_int(array.getitem(index)) - -def do_getarrayitem_raw_int(array, index, itemsize): - array = array.adr.ptr - ITEMTYPE = lltype.typeOf(array).TO.OF - TYPE = symbolic.Size2Type[itemsize] - if TYPE.OF != ITEMTYPE: - array = rffi.cast(lltype.Ptr(TYPE), array) - return cast_to_int(array._obj.getitem(index)) - -def do_getarrayitem_gc_float(array, index): - array = array._obj.container - return cast_to_floatstorage(array.getitem(index)) - -def do_getarrayitem_raw_float(array, index): - array = maybe_uncast(FLOAT_ARRAY_TP, array.adr.ptr) - return cast_to_floatstorage(array._obj.getitem(index)) - -def do_getarrayitem_gc_ptr(array, index): - array = array._obj.container - return cast_to_ptr(array.getitem(index)) - -def _getfield_gc(struct, fieldnum): - STRUCT, fieldname = symbolic.TokenToField[fieldnum] - ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct) - return getattr(ptr, fieldname) - -def do_getfield_gc_int(struct, fieldnum): - return cast_to_int(_getfield_gc(struct, fieldnum)) - -def do_getfield_gc_float(struct, fieldnum): - return cast_to_floatstorage(_getfield_gc(struct, fieldnum)) - -def do_getfield_gc_ptr(struct, fieldnum): - return cast_to_ptr(_getfield_gc(struct, fieldnum)) - -def _getinteriorfield_gc(struct, fieldnum): - STRUCT, fieldname = symbolic.TokenToField[fieldnum] - return getattr(struct, fieldname) - -def do_getinteriorfield_gc_int(array, index, fieldnum): - struct = array._obj.container.getitem(index) - return cast_to_int(_getinteriorfield_gc(struct, fieldnum)) - -def do_getinteriorfield_gc_float(array, index, fieldnum): - struct = array._obj.container.getitem(index) - return cast_to_floatstorage(_getinteriorfield_gc(struct, fieldnum)) - -def do_getinteriorfield_gc_ptr(array, index, fieldnum): - struct = array._obj.container.getitem(index) - return cast_to_ptr(_getinteriorfield_gc(struct, fieldnum)) - -def _getfield_raw(struct, fieldnum): - STRUCT, fieldname = symbolic.TokenToField[fieldnum] - ptr = cast_from_int(lltype.Ptr(STRUCT), struct) - return getattr(ptr, fieldname) - -def do_getfield_raw_int(struct, fieldnum): - return cast_to_int(_getfield_raw(struct, fieldnum)) - -def do_getfield_raw_float(struct, fieldnum): - return cast_to_floatstorage(_getfield_raw(struct, fieldnum)) - -def do_getfield_raw_ptr(struct, fieldnum): - return cast_to_ptr(_getfield_raw(struct, fieldnum)) - -def do_raw_load_int(struct, offset, descrofs): - TYPE = symbolic.Size2Type[descrofs] - ll_p = rffi.cast(rffi.CCHARP, struct) - ll_p = rffi.cast(lltype.Ptr(TYPE), rffi.ptradd(ll_p, offset)) - value = ll_p[0] - return rffi.cast(lltype.Signed, value) - -def do_raw_load_float(struct, offset): - ll_p = rffi.cast(rffi.CCHARP, struct) - ll_p = rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), - rffi.ptradd(ll_p, offset)) - value = ll_p[0] - return value - -def do_raw_store_int(struct, offset, descrofs, value): - TYPE = symbolic.Size2Type[descrofs] - ll_p = rffi.cast(rffi.CCHARP, struct) - ll_p = rffi.cast(lltype.Ptr(TYPE), rffi.ptradd(ll_p, offset)) - ll_p[0] = rffi.cast(TYPE.OF, value) - -def do_raw_store_float(struct, offset, value): - ll_p = rffi.cast(rffi.CCHARP, struct) - ll_p = rffi.cast(rffi.CArrayPtr(longlong.FLOATSTORAGE), - rffi.ptradd(ll_p, offset)) - ll_p[0] = value - -def do_new(size): - TYPE = symbolic.Size2Type[size] - x = lltype.malloc(TYPE, zero=True) - return cast_to_ptr(x) - -def do_new_array(arraynum, count): - TYPE = symbolic.Size2Type[arraynum] - assert count >= 0 # explode if it's not - x = lltype.malloc(TYPE, count, zero=True) - return cast_to_ptr(x) - -def do_setarrayitem_gc_int(array, index, newvalue): - array = array._obj.container - ITEMTYPE = lltype.typeOf(array).OF - newvalue = cast_from_int(ITEMTYPE, newvalue) - array.setitem(index, newvalue) - -def do_setarrayitem_raw_int(array, index, newvalue, itemsize): - array = array.adr.ptr - ITEMTYPE = lltype.typeOf(array).TO.OF - TYPE = symbolic.Size2Type[itemsize] - if TYPE.OF != ITEMTYPE: - array = rffi.cast(lltype.Ptr(TYPE), array) - newvalue = cast_from_int(TYPE.OF, newvalue) - array._obj.setitem(index, newvalue) - -def do_setarrayitem_gc_float(array, index, newvalue): - array = array._obj.container - ITEMTYPE = lltype.typeOf(array).OF - newvalue = cast_from_floatstorage(ITEMTYPE, newvalue) - array.setitem(index, newvalue) - - -def do_setarrayitem_raw_float(array, index, newvalue): - array = maybe_uncast(FLOAT_ARRAY_TP, array.adr.ptr) - ITEMTYPE = lltype.typeOf(array).TO.OF - newvalue = cast_from_floatstorage(ITEMTYPE, newvalue) - array._obj.setitem(index, newvalue) - -def do_setarrayitem_gc_ptr(array, index, newvalue): - array = array._obj.container - ITEMTYPE = lltype.typeOf(array).OF - newvalue = cast_from_ptr(ITEMTYPE, newvalue) - array.setitem(index, newvalue) - -def new_setfield_gc(cast_func): - def do_setfield_gc(struct, fieldnum, newvalue): - STRUCT, fieldname = symbolic.TokenToField[fieldnum] - ptr = lltype.cast_opaque_ptr(lltype.Ptr(STRUCT), struct) - FIELDTYPE = getattr(STRUCT, fieldname) - newvalue = cast_func(FIELDTYPE, newvalue) - setattr(ptr, fieldname, newvalue) - return do_setfield_gc -do_setfield_gc_int = new_setfield_gc(cast_from_int) -do_setfield_gc_float = new_setfield_gc(cast_from_floatstorage) -do_setfield_gc_ptr = new_setfield_gc(cast_from_ptr) - -def new_setinteriorfield_gc(cast_func): - def do_setinteriorfield_gc(array, index, fieldnum, newvalue): - STRUCT, fieldname = symbolic.TokenToField[fieldnum] - struct = array._obj.container.getitem(index) - FIELDTYPE = getattr(STRUCT, fieldname) - setattr(struct, fieldname, cast_func(FIELDTYPE, newvalue)) - return do_setinteriorfield_gc -do_setinteriorfield_gc_int = new_setinteriorfield_gc(cast_from_int) -do_setinteriorfield_gc_float = new_setinteriorfield_gc(cast_from_floatstorage) -do_setinteriorfield_gc_ptr = new_setinteriorfield_gc(cast_from_ptr) - -def do_setfield_raw_int(struct, fieldnum, newvalue): - STRUCT, fieldname = symbolic.TokenToField[fieldnum] - ptr = cast_from_int(lltype.Ptr(STRUCT), struct) - FIELDTYPE = getattr(STRUCT, fieldname) - newvalue = cast_from_int(FIELDTYPE, newvalue) - setattr(ptr, fieldname, newvalue) - -def do_setfield_raw_float(struct, fieldnum, newvalue): - STRUCT, fieldname = symbolic.TokenToField[fieldnum] - ptr = cast_from_int(lltype.Ptr(STRUCT), struct) - FIELDTYPE = getattr(STRUCT, fieldname) - newvalue = cast_from_floatstorage(FIELDTYPE, newvalue) - setattr(ptr, fieldname, newvalue) - -def do_setfield_raw_ptr(struct, fieldnum, newvalue): - STRUCT, fieldname = symbolic.TokenToField[fieldnum] - ptr = cast_from_int(lltype.Ptr(STRUCT), struct) - FIELDTYPE = getattr(STRUCT, fieldname) - newvalue = cast_from_ptr(FIELDTYPE, newvalue) - setattr(ptr, fieldname, newvalue) - -def do_newstr(length): - x = rstr.mallocstr(length) - return cast_to_ptr(x) - -def do_newunicode(length): - return cast_to_ptr(rstr.mallocunicode(length)) - -def do_strsetitem(string, index, newvalue): - str = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), string) - str.chars[index] = chr(newvalue) - -def do_unicodesetitem(string, index, newvalue): - uni = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), string) - uni.chars[index] = unichr(newvalue) - -def do_copystrcontent(src, dst, srcstart, dststart, length): - src = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), src) - dst = lltype.cast_opaque_ptr(lltype.Ptr(rstr.STR), dst) - assert 0 <= srcstart <= srcstart + length <= len(src.chars) - assert 0 <= dststart <= dststart + length <= len(dst.chars) - rstr.copy_string_contents(src, dst, srcstart, dststart, length) - -def do_copyunicodecontent(src, dst, srcstart, dststart, length): - src = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), src) - dst = lltype.cast_opaque_ptr(lltype.Ptr(rstr.UNICODE), dst) - assert 0 <= srcstart <= srcstart + length <= len(src.chars) - assert 0 <= dststart <= dststart + length <= len(dst.chars) - rstr.copy_unicode_contents(src, dst, srcstart, dststart, length) - -def do_math_sqrt(value): - import math - y = cast_from_floatstorage(lltype.Float, value) - x = math.sqrt(y) - return cast_to_floatstorage(x) - -# ---------- call ---------- - -_call_args_i = [] -_call_args_r = [] -_call_args_f = [] - -def do_call_pushint(x): - _call_args_i.append(x) - -def do_call_pushptr(x): - _call_args_r.append(x) - -def do_call_pushfloat(x): - _call_args_f.append(x) - -kind2TYPE = { - 'i': lltype.Signed, - 'f': lltype.Float, - 'L': lltype.SignedLongLong, - 'S': lltype.SingleFloat, From noreply at buildbot.pypy.org Mon Jan 28 20:00:06 2013 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 28 Jan 2013 20:00:06 +0100 (CET) Subject: [pypy-commit] pypy py3k: Now that rpython has its own copy of _marshal.py, Message-ID: <20130128190006.29D311C1305@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r60627:d0b59874de62 Date: 2013-01-28 19:55 +0100 http://bitbucket.org/pypy/pypy/changeset/d0b59874de62/ Log: Now that rpython has its own copy of _marshal.py, undo the python3 specific change we made there. diff --git a/rpython/translator/sandbox/_marshal.py b/rpython/translator/sandbox/_marshal.py --- a/rpython/translator/sandbox/_marshal.py +++ b/rpython/translator/sandbox/_marshal.py @@ -4,12 +4,8 @@ This module contains functions that can read and write Python values in a binary format. The format is specific to Python, but independent of machine architecture issues (e.g., you can write a Python value to a file on a PC, transport the file to a Sun, and read it back there). Details of the format may change between Python versions. """ -# NOTE: This module is used in the Python3 interpreter, but also by -# the "sandboxed" process. It must work for Python2 as well. - import types from _codecs import utf_8_decode, utf_8_encode -import sys try: from __pypy__ import builtinify except ImportError: builtinify = lambda f: f @@ -54,7 +50,7 @@ if func: break else: - raise ValueError("unmarshallable object") + raise ValueError, "unmarshallable object" func(self, x) def w_long64(self, x): @@ -77,7 +73,7 @@ def dump_none(self, x): self._write(TYPE_NONE) - dispatch[type(None)] = dump_none + dispatch[types.NoneType] = dump_none def dump_bool(self, x): if x: @@ -88,7 +84,7 @@ def dump_stopiter(self, x): if x is not StopIteration: - raise ValueError("unmarshallable object") + raise ValueError, "unmarshallable object" self._write(TYPE_STOPITER) dispatch[type(StopIteration)] = dump_stopiter @@ -96,11 +92,10 @@ self._write(TYPE_ELLIPSIS) try: - dispatch[type(Ellipsis)] = dump_ellipsis + dispatch[types.EllipsisType] = dump_ellipsis except NameError: pass - # In Python3, this function is not used; see dump_long() below. def dump_int(self, x): y = x>>31 if y and y != -1: @@ -109,7 +104,7 @@ else: self._write(TYPE_INT) self.w_long(x) - dispatch[int] = dump_int + dispatch[types.IntType] = dump_int def dump_long(self, x): self._write(TYPE_LONG) @@ -124,32 +119,27 @@ self.w_long(len(digits) * sign) for d in digits: self.w_short(d) - try: - long - except NameError: - dispatch[int] = dump_long - else: - dispatch[long] = dump_long + dispatch[types.LongType] = dump_long def dump_float(self, x): write = self._write write(TYPE_FLOAT) - s = repr(x) + s = `x` write(chr(len(s))) write(s) - dispatch[float] = dump_float + dispatch[types.FloatType] = dump_float def dump_complex(self, x): write = self._write write(TYPE_COMPLEX) - s = repr(x.real) + s = `x.real` write(chr(len(s))) write(s) - s = repr(x.imag) + s = `x.imag` write(chr(len(s))) write(s) try: - dispatch[complex] = dump_complex + dispatch[types.ComplexType] = dump_complex except NameError: pass @@ -159,7 +149,7 @@ self._write(TYPE_STRING) self.w_long(len(x)) self._write(x) - dispatch[bytes] = dump_string + dispatch[types.StringType] = dump_string def dump_unicode(self, x): self._write(TYPE_UNICODE) @@ -167,26 +157,21 @@ s, len_s = utf_8_encode(x) self.w_long(len_s) self._write(s) - try: - unicode - except NameError: - dispatch[str] = dump_unicode - else: - dispatch[unicode] = dump_unicode + dispatch[types.UnicodeType] = dump_unicode def dump_tuple(self, x): self._write(TYPE_TUPLE) self.w_long(len(x)) for item in x: self.dump(item) - dispatch[tuple] = dump_tuple + dispatch[types.TupleType] = dump_tuple def dump_list(self, x): self._write(TYPE_LIST) self.w_long(len(x)) for item in x: self.dump(item) - dispatch[list] = dump_list + dispatch[types.ListType] = dump_list def dump_dict(self, x): self._write(TYPE_DICT) @@ -194,7 +179,7 @@ self.dump(key) self.dump(value) self._write(TYPE_NULL) - dispatch[dict] = dump_dict + dispatch[types.DictionaryType] = dump_dict def dump_code(self, x): self._write(TYPE_CODE) @@ -268,7 +253,7 @@ try: return self.dispatch[c](self) except KeyError: - raise ValueError("bad marshal code: %c (%d)" % (c, ord(c))) + raise ValueError, "bad marshal code: %c (%d)" % (c, ord(c)) def r_short(self): lo = ord(self._read(1)) @@ -286,7 +271,7 @@ d = ord(s[3]) x = a | (b<<8) | (c<<16) | (d<<24) if d & 0x80 and x > 0: - x = -((1<<32) - x) + x = -((1L<<32) - x) return int(x) else: return x @@ -296,14 +281,14 @@ b = ord(self._read(1)) c = ord(self._read(1)) d = ord(self._read(1)) - e = int(ord(self._read(1))) - f = int(ord(self._read(1))) - g = int(ord(self._read(1))) - h = int(ord(self._read(1))) + e = long(ord(self._read(1))) + f = long(ord(self._read(1))) + g = long(ord(self._read(1))) + h = long(ord(self._read(1))) x = a | (b<<8) | (c<<16) | (d<<24) x = x | (e<<32) | (f<<40) | (g<<48) | (h<<56) if h & 0x80 and x > 0: - x = -((1<<64) - x) + x = -((1L<<64) - x) return x def load_null(self): @@ -340,10 +325,10 @@ if size < 0: sign = -1 size = -size - x = 0 + x = 0L for i in range(size): d = self.r_short() - x = x | (d<<(i*15)) + x = x | (d<<(i*15L)) return x * sign dispatch[TYPE_LONG] = load_long @@ -370,7 +355,7 @@ def load_interned(self): n = self.r_long() - ret = sys.intern(self._read(n)) + ret = intern(self._read(n)) self._stringtable.append(ret) return ret dispatch[TYPE_INTERNED] = load_interned @@ -475,7 +460,7 @@ self.bufpos += 4 x = a | (b<<8) | (c<<16) | (d<<24) if d & 0x80 and x > 0: - x = -((1<<32) - x) + x = -((1L<<32) - x) return int(x) else: return x @@ -485,14 +470,14 @@ b = ord(_read1(self)) c = ord(_read1(self)) d = ord(_read1(self)) - e = int(ord(_read1(self))) - f = int(ord(_read1(self))) - g = int(ord(_read1(self))) - h = int(ord(_read1(self))) + e = long(ord(_read1(self))) + f = long(ord(_read1(self))) + g = long(ord(_read1(self))) + h = long(ord(_read1(self))) x = a | (b<<8) | (c<<16) | (d<<24) x = x | (e<<32) | (f<<40) | (g<<48) | (h<<56) if h & 0x80 and x > 0: - x = -((1<<64) - x) + x = -((1L<<64) - x) return x _load_dispatch = {} @@ -514,7 +499,7 @@ self.bufpos += 1 return _load_dispatch[c](self) except KeyError: - raise ValueError("bad marshal code: %c (%d)" % (c, ord(c))) + raise ValueError, "bad marshal code: %c (%d)" % (c, ord(c)) except IndexError: raise EOFError @@ -556,10 +541,10 @@ if size < 0: sign = -1 size = -size - x = 0 + x = 0L for i in range(size): d = _r_short(self) - x = x | (d<<(i*15)) + x = x | (d<<(i*15L)) return x * sign dispatch[TYPE_LONG] = load_long @@ -586,7 +571,7 @@ def load_interned(self): n = _r_long(self) - ret = sys.intern(_read(self, n)) + ret = intern(_read(self, n)) self._stringtable.append(ret) return ret dispatch[TYPE_INTERNED] = load_interned From noreply at buildbot.pypy.org Mon Jan 28 20:00:07 2013 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 28 Jan 2013 20:00:07 +0100 (CET) Subject: [pypy-commit] pypy py3k: Kill remaining difference. Message-ID: <20130128190007.6437C1C1305@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r60628:bdf31108b4dd Date: 2013-01-28 19:55 +0100 http://bitbucket.org/pypy/pypy/changeset/bdf31108b4dd/ Log: Kill remaining difference. diff --git a/rpython/translator/test/snippet.py b/rpython/translator/test/snippet.py --- a/rpython/translator/test/snippet.py +++ b/rpython/translator/test/snippet.py @@ -636,6 +636,7 @@ result += 1.0 / n return result + # --------------------(Currently) Non runnable Functions --------------------- def _somebug1(n=int): From noreply at buildbot.pypy.org Mon Jan 28 20:00:08 2013 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 28 Jan 2013 20:00:08 +0100 (CET) Subject: [pypy-commit] pypy py3k: hg merge default Message-ID: <20130128190008.B713E1C1305@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r60629:fcaa58ce7548 Date: 2013-01-28 19:59 +0100 http://bitbucket.org/pypy/pypy/changeset/fcaa58ce7548/ Log: hg merge default diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -343,9 +343,13 @@ signal, the tick counter is set to -1 by C code in signals.h. """ assert isinstance(action, PeriodicAsyncAction) - self._periodic_actions.append(action) + # hack to put the release-the-GIL one at the end of the list, + # and the report-the-signals one at the start of the list. if use_bytecode_counter: + self._periodic_actions.append(action) self.has_bytecode_counter = True + else: + self._periodic_actions.insert(0, action) self._rebuild_action_dispatcher() def getcheckinterval(self): @@ -419,15 +423,6 @@ The action must have been registered at space initalization time.""" self.space.actionflag.fire(self) - def fire_after_thread_switch(self): - """Bit of a hack: fire() the action but only the next time the GIL - is released and re-acquired (i.e. after a potential thread switch). - Don't call this if threads are not enabled. Currently limited to - one action (i.e. reserved for CheckSignalAction from module/signal). - """ - from pypy.module.thread.gil import spacestate - spacestate.action_after_thread_switch = self - def perform(self, executioncontext, frame): """To be overridden.""" diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py --- a/pypy/module/rctime/test/test_rctime.py +++ b/pypy/module/rctime/test/test_rctime.py @@ -114,7 +114,10 @@ assert rctime.mktime(rctime.localtime(-1)) == -1 res = rctime.mktime((2000, 1, 1, 0, 0, 0, -1, -1, -1)) - assert rctime.ctime(res) == 'Sat Jan 1 00:00:00 2000' + if os.name == 'nt': + assert rctime.ctime(res) == 'Sat Jan 01 00:00:00 2000' + else: + assert rctime.ctime(res) == 'Sat Jan 1 00:00:00 2000' def test_asctime(self): import time as rctime diff --git a/pypy/module/signal/__init__.py b/pypy/module/signal/__init__.py --- a/pypy/module/signal/__init__.py +++ b/pypy/module/signal/__init__.py @@ -30,9 +30,10 @@ } def buildloaders(cls): - from pypy.module.signal import interp_signal - for name in interp_signal.signal_names: - signum = getattr(interp_signal, name) + from rpython.rlib import rsignal + + for name in rsignal.signal_names: + signum = getattr(rsignal, name) if signum is not None: Module.interpleveldefs[name] = 'space.wrap(%d)' % (signum,) super(Module, cls).buildloaders() diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py --- a/pypy/module/signal/interp_signal.py +++ b/pypy/module/signal/interp_signal.py @@ -1,25 +1,28 @@ from __future__ import with_statement + +import signal as cpy_signal +import sys + from pypy.interpreter.error import OperationError, exception_from_errno -from pypy.interpreter.executioncontext import AsyncAction, AbstractActionFlag -from pypy.interpreter.executioncontext import PeriodicAsyncAction +from pypy.interpreter.executioncontext import (AsyncAction, AbstractActionFlag, + PeriodicAsyncAction) from pypy.interpreter.gateway import unwrap_spec -import signal as cpy_signal -from rpython.rtyper.lltypesystem import lltype, rffi -from rpython.rtyper.tool import rffi_platform -from rpython.translator.tool.cbuild import ExternalCompilationInfo -from rpython.conftest import cdir -import py -import sys -from rpython.rlib import jit, rposix + +from rpython.rlib import jit, rposix, rgc +from rpython.rlib.objectmodel import we_are_translated from rpython.rlib.rarithmetic import intmask from rpython.rlib.rsignal import * +from rpython.rtyper.lltypesystem import lltype, rffi + WIN32 = sys.platform == 'win32' + class SignalActionFlag(AbstractActionFlag): # This class uses the C-level pypysig_counter variable as the tick # counter. The C-level signal handler will reset it to -1 whenever - # a signal is received. + # a signal is received. This causes CheckSignalAction.perform() to + # be called. def get_ticker(self): p = pypysig_getaddr_occurred() @@ -29,6 +32,11 @@ p = pypysig_getaddr_occurred() p.c_value = value + @staticmethod + def rearm_ticker(): + p = pypysig_getaddr_occurred() + p.c_value = -1 + def decrement_ticker(self, by): p = pypysig_getaddr_occurred() value = p.c_value @@ -44,48 +52,65 @@ class CheckSignalAction(PeriodicAsyncAction): """An action that is automatically invoked when a signal is received.""" + # Note that this is a PeriodicAsyncAction: it means more precisely + # that it is called whenever the C-level ticker becomes < 0. + # Without threads, it is only ever set to -1 when we receive a + # signal. With threads, it also decrements steadily (but slowly). + def __init__(self, space): + "NOT_RPYTHON" AsyncAction.__init__(self, space) self.handlers_w = {} - if space.config.objspace.usemodules.thread: - # need a helper action in case signals arrive in a non-main thread - self.pending_signals = {} - self.reissue_signal_action = ReissueSignalAction(space) - else: - self.reissue_signal_action = None + self.pending_signal = -1 + self.fire_in_main_thread = False + if self.space.config.objspace.usemodules.thread: + from pypy.module.thread import gil + gil.after_thread_switch = self._after_thread_switch + + @rgc.no_collect + def _after_thread_switch(self): + if self.fire_in_main_thread: + if self.space.threadlocals.ismainthread(): + self.fire_in_main_thread = False + SignalActionFlag.rearm_ticker() + # this occurs when we just switched to the main thread + # and there is a signal pending: we force the ticker to + # -1, which should ensure perform() is called quickly. @jit.dont_look_inside def perform(self, executioncontext, frame): - while True: - n = pypysig_poll() - if n < 0: + # Poll for the next signal, if any + n = self.pending_signal + if n < 0: n = pypysig_poll() + while n >= 0: + if self.space.config.objspace.usemodules.thread: + in_main = self.space.threadlocals.ismainthread() + else: + in_main = True + if in_main: + # If we are in the main thread, report the signal now, + # and poll more + self.pending_signal = -1 + self._report_signal(n) + n = self.pending_signal + if n < 0: n = pypysig_poll() + else: + # Otherwise, arrange for perform() to be called again + # after we switch to the main thread. + self.pending_signal = n + self.fire_in_main_thread = True break - self.perform_signal(executioncontext, n) - @jit.dont_look_inside - def perform_signal(self, executioncontext, n): - if self.reissue_signal_action is None: - # no threads: we can report the signal immediately - self.report_signal(n) - else: - main_ec = self.space.threadlocals.getmainthreadvalue() - if executioncontext is main_ec: - # running in the main thread: we can report the - # signal immediately - self.report_signal(n) - else: - # running in another thread: we need to hack a bit - self.pending_signals[n] = None - self.reissue_signal_action.fire_after_thread_switch() - - @jit.dont_look_inside def set_interrupt(self): "Simulates the effect of a SIGINT signal arriving" - ec = self.space.getexecutioncontext() - self.perform_signal(ec, cpy_signal.SIGINT) + if not we_are_translated(): + self.pending_signal = cpy_signal.SIGINT + # ^^^ may override another signal, but it's just for testing + else: + pypysig_pushback(cpy_signal.SIGINT) + self.fire_in_main_thread = True - @jit.dont_look_inside - def report_signal(self, n): + def _report_signal(self, n): try: w_handler = self.handlers_w[n] except KeyError: @@ -100,39 +125,6 @@ w_frame = space.wrap(ec.gettopframe_nohidden()) space.call_function(w_handler, space.wrap(n), w_frame) - @jit.dont_look_inside - def report_pending_signals(self): - # XXX this logic isn't so complicated but I have no clue how - # to test it :-( - pending_signals = self.pending_signals.keys() - self.pending_signals.clear() - try: - while pending_signals: - self.report_signal(pending_signals.pop()) - finally: - # in case of exception, put the undelivered signals back - # into the dict instead of silently swallowing them - if pending_signals: - for n in pending_signals: - self.pending_signals[n] = None - self.reissue_signal_action.fire() - - -class ReissueSignalAction(AsyncAction): - """A special action to help deliver signals to the main thread. If - a non-main thread caught a signal, this action fires after every - thread switch until we land in the main thread. - """ - - def perform(self, executioncontext, frame): - main_ec = self.space.threadlocals.getmainthreadvalue() - if executioncontext is main_ec: - # now running in the main thread: we can really report the signals - self.space.check_signal_action.report_pending_signals() - else: - # still running in some other thread: try again later - self.fire_after_thread_switch() - @unwrap_spec(signum=int) def getsignal(space, signum): @@ -154,6 +146,7 @@ return action.handlers_w[signum] return space.wrap(SIG_DFL) + def default_int_handler(space, w_signum, w_frame): """ default_int_handler(...) @@ -164,22 +157,26 @@ raise OperationError(space.w_KeyboardInterrupt, space.w_None) + @jit.dont_look_inside @unwrap_spec(timeout=int) def alarm(space, timeout): return space.wrap(c_alarm(timeout)) + @jit.dont_look_inside def pause(space): c_pause() return space.w_None + def check_signum_exists(space, signum): if signum in signal_values: return raise OperationError(space.w_ValueError, space.wrap("invalid signal value")) + def check_signum_in_range(space, signum): if 1 <= signum < NSIG: return @@ -201,7 +198,7 @@ A signal handler function is called with two arguments: the first is the signal number, the second is the interrupted stack frame. """ - ec = space.getexecutioncontext() + ec = space.getexecutioncontext() main_ec = space.threadlocals.getmainthreadvalue() old_handler = getsignal(space, signum) @@ -224,13 +221,14 @@ action.handlers_w[signum] = w_handler return old_handler + @jit.dont_look_inside @unwrap_spec(fd=int) def set_wakeup_fd(space, fd): """Sets the fd to be written to (with '\0') when a signal comes in. Returns the old fd. A library can use this to wakeup select or poll. The previous fd is returned. - + The fd must be non-blocking. """ if space.config.objspace.usemodules.thread: @@ -243,6 +241,7 @@ old_fd = pypysig_set_wakeup_fd(fd) return space.wrap(intmask(old_fd)) + @jit.dont_look_inside @unwrap_spec(signum=int, flag=int) def siginterrupt(space, signum, flag): @@ -258,33 +257,38 @@ rffi.setintfield(timeval, 'c_tv_sec', int(d)) rffi.setintfield(timeval, 'c_tv_usec', int((d - int(d)) * 1000000)) + def double_from_timeval(tv): return rffi.getintfield(tv, 'c_tv_sec') + ( rffi.getintfield(tv, 'c_tv_usec') / 1000000.0) + def itimer_retval(space, val): w_value = space.wrap(double_from_timeval(val.c_it_value)) w_interval = space.wrap(double_from_timeval(val.c_it_interval)) return space.newtuple([w_value, w_interval]) + class Cache: def __init__(self, space): self.w_itimererror = space.new_exception_class("signal.ItimerError", space.w_IOError) + def get_itimer_error(space): return space.fromcache(Cache).w_itimererror + @jit.dont_look_inside @unwrap_spec(which=int, first=float, interval=float) def setitimer(space, which, first, interval=0): """setitimer(which, seconds[, interval]) - Sets given itimer (one of ITIMER_REAL, ITIMER_VIRTUAL + or ITIMER_PROF) to fire after value seconds and after that every interval seconds. The itimer can be cleared by setting seconds to zero. - + Returns old values as a tuple: (delay, interval). """ with lltype.scoped_alloc(itimervalP.TO, 1) as new: @@ -298,14 +302,14 @@ if ret != 0: raise exception_from_errno(space, get_itimer_error(space)) + return itimer_retval(space, old[0]) - return itimer_retval(space, old[0]) @jit.dont_look_inside @unwrap_spec(which=int) def getitimer(space, which): """getitimer(which) - + Returns current value of given itimer. """ with lltype.scoped_alloc(itimervalP.TO, 1) as old: diff --git a/pypy/module/thread/gil.py b/pypy/module/thread/gil.py --- a/pypy/module/thread/gil.py +++ b/pypy/module/thread/gil.py @@ -62,22 +62,7 @@ do_yield_thread() -class SpaceState: - - def _cleanup_(self): - self.action_after_thread_switch = None - # ^^^ set by AsyncAction.fire_after_thread_switch() - - def after_thread_switch(self): - # this is support logic for the signal module, to help it deliver - # signals to the main thread. - action = self.action_after_thread_switch - if action is not None: - self.action_after_thread_switch = None - action.fire() - -spacestate = SpaceState() -spacestate._cleanup_() +after_thread_switch = lambda: None # hook for signal.py # Fragile code below. We have to preserve the C-level errno manually... @@ -94,7 +79,7 @@ e = get_errno() thread.gil_acquire() thread.gc_thread_run() - spacestate.after_thread_switch() + after_thread_switch() set_errno(e) after_external_call._gctransformer_hint_cannot_collect_ = True after_external_call._dont_reach_me_in_del_ = True @@ -112,7 +97,7 @@ # the same thread). if thread.gil_yield_thread(): thread.gc_thread_run() - spacestate.after_thread_switch() + after_thread_switch() do_yield_thread._gctransformer_hint_close_stack_ = True do_yield_thread._dont_reach_me_in_del_ = True do_yield_thread._dont_inline_ = True diff --git a/pypy/module/thread/os_local.py b/pypy/module/thread/os_local.py --- a/pypy/module/thread/os_local.py +++ b/pypy/module/thread/os_local.py @@ -98,5 +98,5 @@ local = wref() if local is not None: del local.dicts[ec] - local.last_dict = None - local.last_ec = None + local.last_dict = None + local.last_ec = None diff --git a/pypy/module/thread/threadlocals.py b/pypy/module/thread/threadlocals.py --- a/pypy/module/thread/threadlocals.py +++ b/pypy/module/thread/threadlocals.py @@ -48,6 +48,9 @@ ident = self._mainthreadident return self._valuedict.get(ident, None) + def ismainthread(self): + return thread.get_ident() == self._mainthreadident + def getallvalues(self): return self._valuedict diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py --- a/rpython/annotator/test/test_annrpython.py +++ b/rpython/annotator/test/test_annrpython.py @@ -2068,7 +2068,23 @@ s = a.build_types(f, [annmodel.SomeString(no_nul=True)]) assert isinstance(s, annmodel.SomeString) assert s.no_nul - + + def test_getitem_str0(self): + def f(s, n): + if n == 1: + return s[0] + elif n == 2: + return s[1] + elif n == 3: + return s[1:] + return s + a = self.RPythonAnnotator() + a.translator.config.translation.check_str_without_nul = True + + s = a.build_types(f, [annmodel.SomeString(no_nul=True), + annmodel.SomeInteger()]) + assert isinstance(s, annmodel.SomeString) + assert s.no_nul def test_non_none_and_none_with_isinstance(self): class A(object): diff --git a/rpython/jit/backend/llsupport/ffisupport.py b/rpython/jit/backend/llsupport/ffisupport.py --- a/rpython/jit/backend/llsupport/ffisupport.py +++ b/rpython/jit/backend/llsupport/ffisupport.py @@ -42,11 +42,14 @@ @specialize.memo() def _get_ffi2descr_dict(cpu): - d = {('v', 0): ('v', None)} + def entry(letter, TYPE): + return (letter, cpu.arraydescrof(rffi.CArray(TYPE)), rffi.sizeof(TYPE)) + # + d = {('v', 0): ('v', None, 1)} if cpu.supports_floats: - d[('f', 0)] = ('f', cpu.arraydescrof(rffi.CArray(lltype.Float))) + d[('f', 0)] = entry('f', lltype.Float) if cpu.supports_singlefloats: - d[('S', 0)] = ('i', cpu.arraydescrof(rffi.CArray(lltype.SingleFloat))) + d[('S', 0)] = entry('i', lltype.SingleFloat) for SIGNED_TYPE in [rffi.SIGNEDCHAR, rffi.SHORT, rffi.INT, @@ -59,7 +62,7 @@ continue key = ('L', 0) kind = 'f' - d[key] = (kind, cpu.arraydescrof(rffi.CArray(SIGNED_TYPE))) + d[key] = entry(kind, SIGNED_TYPE) for UNSIGNED_TYPE in [rffi.UCHAR, rffi.USHORT, rffi.UINT, @@ -68,7 +71,7 @@ key = ('u', rffi.sizeof(UNSIGNED_TYPE)) if key[1] > rffi.sizeof(lltype.Signed): continue - d[key] = ('i', cpu.arraydescrof(rffi.CArray(UNSIGNED_TYPE))) + d[key] = entry('i', UNSIGNED_TYPE) return d def get_arg_descr(cpu, ffi_type): diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -2567,7 +2567,8 @@ self.history.operations.pop() arg_boxes = [] for i in range(cif_description.nargs): - kind, descr = get_arg_descr(self.cpu, cif_description.atypes[i]) + kind, descr, itemsize = get_arg_descr(self.cpu, + cif_description.atypes[i]) if kind == 'i': box_arg = history.BoxInt() elif kind == 'f': @@ -2576,16 +2577,14 @@ assert kind == 'v' continue ofs = cif_description.exchange_args[i] - box_argpos = history.BoxInt() - self.history.record(rop.INT_ADD, - [box_exchange_buffer, ConstInt(ofs)], - box_argpos) + assert ofs % itemsize == 0 # alignment check self.history.record(rop.GETARRAYITEM_RAW, - [box_argpos, ConstInt(0)], + [box_exchange_buffer, + ConstInt(ofs // itemsize)], box_arg, descr) arg_boxes.append(box_arg) # - kind, descr = get_arg_descr(self.cpu, cif_description.rtype) + kind, descr, itemsize = get_arg_descr(self.cpu, cif_description.rtype) if kind == 'i': box_result = history.BoxInt() elif kind == 'f': @@ -2601,12 +2600,10 @@ # if box_result is not None: ofs = cif_description.exchange_result - box_resultpos = history.BoxInt() - self.history.record(rop.INT_ADD, - [box_exchange_buffer, ConstInt(ofs)], - box_resultpos) + assert ofs % itemsize == 0 # alignment check (result) self.history.record(rop.SETARRAYITEM_RAW, - [box_resultpos, ConstInt(0), box_result], + [box_exchange_buffer, + ConstInt(ofs // itemsize), box_result], None, descr) def direct_call_release_gil(self): diff --git a/rpython/jit/metainterp/test/test_warmspot.py b/rpython/jit/metainterp/test/test_warmspot.py --- a/rpython/jit/metainterp/test/test_warmspot.py +++ b/rpython/jit/metainterp/test/test_warmspot.py @@ -401,6 +401,7 @@ assert len(oplabel.getarglist()) == 2 # 'n', 'res' in some order def test_inline_jit_merge_point(self): + py.test.skip("fix the test if you want to re-enable this") # test that the machinery to inline jit_merge_points in callers # works. The final user does not need to mess manually with the # _inline_jit_merge_point_ attribute and similar, it is all nicely @@ -430,6 +431,7 @@ self.check_resops(int_add=4) def test_jitdriver_inline(self): + py.test.skip("fix the test if you want to re-enable this") myjitdriver = JitDriver(greens = [], reds = 'auto') class MyRange(object): def __init__(self, n): @@ -462,6 +464,7 @@ self.check_trace_count(1) def test_jitdriver_inline_twice(self): + py.test.skip("fix the test if you want to re-enable this") myjitdriver = JitDriver(greens = [], reds = 'auto') def jit_merge_point(a, b): @@ -492,6 +495,7 @@ self.check_trace_count(2) def test_jitdriver_inline_exception(self): + py.test.skip("fix the test if you want to re-enable this") # this simulates what happens in a real case scenario: inside the next # we have a call which we cannot inline (e.g. space.next in the case # of W_InterpIterable), but we need to put it in a try/except block. diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -578,6 +578,8 @@ pass def inline(self, call_jit_merge_point): + assert False, "@inline off: see skipped failures in test_warmspot." + # assert self.autoreds, "@inline works only with reds='auto'" self.inline_jit_merge_point = True def decorate(func): diff --git a/rpython/rlib/rsignal.py b/rpython/rlib/rsignal.py --- a/rpython/rlib/rsignal.py +++ b/rpython/rlib/rsignal.py @@ -80,6 +80,8 @@ pypysig_poll = external('pypysig_poll', [], rffi.INT, threadsafe=False) # don't bother releasing the GIL around a call to pypysig_poll: it's # pointless and a performance issue +pypysig_pushback = external('pypysig_pushback', [rffi.INT], lltype.Void, + threadsafe=False) # don't use rffi.LONGP because the JIT doesn't support raw arrays so far struct_name = 'pypysig_long_struct' diff --git a/rpython/rlib/test/test_jit.py b/rpython/rlib/test/test_jit.py --- a/rpython/rlib/test/test_jit.py +++ b/rpython/rlib/test/test_jit.py @@ -35,6 +35,7 @@ assert driver.reds == ['a', 'b'] assert driver.numreds == 2 + at py.test.mark.xfail(reason="@inline off: see skipped failures in test_warmspot.") def test_jitdriver_inline(): driver = JitDriver(greens=[], reds='auto') calls = [] @@ -54,6 +55,7 @@ ('bar', 40, 2), ] + at py.test.mark.xfail(reason="@inline off: see skipped failures in test_warmspot.") def test_jitdriver_clone(): def bar(): pass def foo(): pass diff --git a/rpython/rlib/test/test_rcomplex.py b/rpython/rlib/test/test_rcomplex.py --- a/rpython/rlib/test/test_rcomplex.py +++ b/rpython/rlib/test/test_rcomplex.py @@ -260,3 +260,24 @@ rAssertAlmostEqual(expected[1], actual[1], abs_err=real_abs_err, msg=error_message) + +def test_isnan(): + assert not c.c_isnan(0, 0) + assert c.c_isnan(float('nan'), 0) + assert c.c_isnan(1, float('nan')) + assert not c.c_isnan(float('inf'), 0) + +def test_isinf(): + assert not c.c_isinf(0, 0) + assert c.c_isinf(float('inf'), 0) + assert c.c_isinf(float('-inf'), 0) + assert c.c_isinf(1, float('inf')) + assert not c.c_isinf(float('nan'), 0) + +def test_isfinite(): + assert c.c_isfinite(0, 0) + assert not c.c_isfinite(float('nan'), 0) + assert not c.c_isfinite(float('-inf'), 0) + assert not c.c_isfinite(0, float('nan')) + assert not c.c_isfinite(0, float('-inf')) + diff --git a/rpython/rtyper/lltypesystem/ll2ctypes.py b/rpython/rtyper/lltypesystem/ll2ctypes.py --- a/rpython/rtyper/lltypesystem/ll2ctypes.py +++ b/rpython/rtyper/lltypesystem/ll2ctypes.py @@ -32,6 +32,15 @@ except ImportError: class tlsobject(object): pass +try: + from threading import RLock +except ImportError: + class RLock(object): + def __enter__(self): + pass + def __exit__(self, *args): + pass +rlock = RLock() _POSIX = os.name == "posix" _MS_WINDOWS = os.name == "nt" @@ -694,10 +703,11 @@ return None def lltype2ctypes(llobj, normalize=True): - """Convert the lltype object 'llobj' to its ctypes equivalent. - 'normalize' should only be False in tests, where we want to - inspect the resulting ctypes object manually. - """ + """Convert the lltype object 'llobj' to its ctypes equivalent. + 'normalize' should only be False in tests, where we want to + inspect the resulting ctypes object manually. + """ + with rlock: if isinstance(llobj, lltype._uninitialized): return uninitialized2ctypes(llobj.TYPE) if isinstance(llobj, llmemory.AddressAsInt): @@ -875,9 +885,10 @@ return llobj def ctypes2lltype(T, cobj): - """Convert the ctypes object 'cobj' to its lltype equivalent. - 'T' is the expected lltype type. - """ + """Convert the ctypes object 'cobj' to its lltype equivalent. + 'T' is the expected lltype type. + """ + with rlock: if T is lltype.Void: return None if isinstance(T, lltype.Typedef): @@ -1176,10 +1187,11 @@ #self.funcptr = ... set later def __call__(self, *argvalues): - if self.trampoline is None: - # lazily build the corresponding ctypes function object - cfunc = get_ctypes_callable(self.funcptr, self.calling_conv) - self.trampoline = get_ctypes_trampoline(self.FUNCTYPE, cfunc) + with rlock: + if self.trampoline is None: + # lazily build the corresponding ctypes function object + cfunc = get_ctypes_callable(self.funcptr, self.calling_conv) + self.trampoline = get_ctypes_trampoline(self.FUNCTYPE, cfunc) # perform the call return self.trampoline(*argvalues) @@ -1215,8 +1227,9 @@ return ctypes2lltype(RESULT, cres) return invoke_via_ctypes + def force_cast(RESTYPE, value): - """Cast a value to a result type, trying to use the same rules as C.""" + with rlock: if not isinstance(RESTYPE, lltype.LowLevelType): raise TypeError("rffi.cast() first arg should be a TYPE") if isinstance(value, llmemory.AddressAsInt): diff --git a/rpython/tool/stdlib_opcode.py b/rpython/tool/stdlib_opcode.py --- a/rpython/tool/stdlib_opcode.py +++ b/rpython/tool/stdlib_opcode.py @@ -71,7 +71,14 @@ def to_globals(self, globals_dict): """NOT_RPYTHON. Add individual opcodes to the module constants.""" - globals_dict.update(self.opmap) + for name, value in self.opmap.iteritems(): + # Rename 'STORE_SLICE+0' opcodes + if name.endswith('+0'): + name = name[:-2] + # Ignore 'STORE_SLICE+1' opcodes + elif name.endswith(('+1', '+2', '+3')): + continue + globals_dict[name] = value def __str__(self): return "<%s bytecode>" % (self.name,) diff --git a/rpython/translator/c/src/signals.c b/rpython/translator/c/src/signals.c --- a/rpython/translator/c/src/signals.c +++ b/rpython/translator/c/src/signals.c @@ -71,7 +71,7 @@ #endif } -static void signal_setflag_handler(int signum) +void pypysig_pushback(int signum) { if (0 <= signum && signum < NSIG) { @@ -79,6 +79,11 @@ pypysig_occurred = 1; pypysig_counter.value = -1; } +} + +static void signal_setflag_handler(int signum) +{ + pypysig_pushback(signum); if (wakeup_fd != -1) { diff --git a/rpython/translator/c/src/signals.h b/rpython/translator/c/src/signals.h --- a/rpython/translator/c/src/signals.h +++ b/rpython/translator/c/src/signals.h @@ -11,6 +11,7 @@ /* utility to poll for signals that arrived */ int pypysig_poll(void); /* => signum or -1 */ +void pypysig_pushback(int signum); /* When a signal is received, pypysig_counter is set to -1. */ /* This is a struct for the JIT. See rsignal.py. */ diff --git a/rpython/translator/c/src/thread_nt.c b/rpython/translator/c/src/thread_nt.c --- a/rpython/translator/c/src/thread_nt.c +++ b/rpython/translator/c/src/thread_nt.c @@ -5,6 +5,7 @@ /* Eliminated some memory leaks, gsw at agere.com */ #include +#include #include #include @@ -112,7 +113,7 @@ mutex->sem = NULL ; /* Just in case */ } -DWORD EnterNonRecursiveMutex(PNRMUTEX mutex, BOOL wait) +DWORD EnterNonRecursiveMutex(PNRMUTEX mutex, DWORD milliseconds) { return WaitForSingleObject(mutex->sem, milliseconds); } @@ -151,26 +152,28 @@ { /* Fow now, intr_flag does nothing on Windows, and lock acquires are * uninterruptible. */ - PyLockStatus success; - PY_TIMEOUT_T milliseconds; + RPyLockStatus success; + RPY_TIMEOUT_T milliseconds; if (microseconds >= 0) { milliseconds = microseconds / 1000; if (microseconds % 1000 > 0) ++milliseconds; - if ((DWORD) milliseconds != milliseconds) - Py_FatalError("Timeout too large for a DWORD, " - "please check PY_TIMEOUT_MAX"); + if ((DWORD) milliseconds != milliseconds) { + fprintf(stderr, "Timeout too large for a DWORD, " + "please check RPY_TIMEOUT_MAX"); + abort(); + } } else milliseconds = INFINITE; if (lock && EnterNonRecursiveMutex( lock, (DWORD)milliseconds) == WAIT_OBJECT_0) { - success = PY_LOCK_ACQUIRED; + success = RPY_LOCK_ACQUIRED; } else { - success = PY_LOCK_FAILURE; + success = RPY_LOCK_FAILURE; } return success; From noreply at buildbot.pypy.org Mon Jan 28 20:53:15 2013 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 28 Jan 2013 20:53:15 +0100 (CET) Subject: [pypy-commit] pypy default: encode() should raise UnicodeEncodeError. Message-ID: <20130128195315.A53231C0041@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: Changeset: r60630:fa5a0d941349 Date: 2013-01-28 20:21 +0100 http://bitbucket.org/pypy/pypy/changeset/fa5a0d941349/ Log: encode() should raise UnicodeEncodeError. (questions for Antonio: why are lone surrogate not allowed? Do all callers expect a RPython exception? And shouldn't RPython follow python2 semantics?) diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py --- a/rpython/rtyper/rstr.py +++ b/rpython/rtyper/rstr.py @@ -80,7 +80,7 @@ assert s is not None bytes = self.runicode_encode_utf_8( s, len(s), 'strict', - errorhandler=self.ll_raise_unicode_exception_decode, + errorhandler=self.ll_raise_unicode_exception_encode, allow_surrogates=False) return self.ll.llstr(bytes) diff --git a/rpython/rtyper/test/test_runicode.py b/rpython/rtyper/test/test_runicode.py --- a/rpython/rtyper/test/test_runicode.py +++ b/rpython/rtyper/test/test_runicode.py @@ -106,6 +106,12 @@ assert self.ll_to_string(self.interpret(f, [38])) == f(38) + def g(n): + x = u'\ud800' + unichr(n) + return x.encode('utf-8') + + self.interpret_raises(UnicodeEncodeError, g, [38]) + def test_utf_8_encoding_annotation(self): from rpython.rlib.runicode import unicode_encode_utf_8 def errorhandler(errors, encoding, msg, u, From noreply at buildbot.pypy.org Mon Jan 28 20:53:17 2013 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 28 Jan 2013 20:53:17 +0100 (CET) Subject: [pypy-commit] pypy default: Add rposix.putenv() and unsetenv(), which support unicode strings. Message-ID: <20130128195317.08DAB1C0041@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: Changeset: r60631:3b27811312df Date: 2013-01-28 20:38 +0100 http://bitbucket.org/pypy/pypy/changeset/3b27811312df/ Log: Add rposix.putenv() and unsetenv(), which support unicode strings. Python2.7 don't use them, though. diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -247,6 +247,20 @@ else: return nt._getfullpathname(path.as_bytes()) + at specialize.argtype(0, 1) +def putenv(name, value): + if isinstance(name, str): + os.environ[name] = value + else: + os.environ[name.as_bytes()] = value.as_bytes() + + at specialize.argtype(0) +def unsetenv(name): + if isinstance(name, str): + del os.environ[name] + else: + del os.environ[name.as_bytes()] + if os.name == 'nt': from rpython.rlib import rwin32 os_kill = rwin32.os_kill diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py --- a/rpython/rlib/test/test_rposix.py +++ b/rpython/rlib/test/test_rposix.py @@ -141,3 +141,10 @@ assert rposix.is_valid_fd(fd) == 1 fid.close() assert rposix.is_valid_fd(fd) == 0 + + def test_putenv(self): + def f(): + rposix.putenv(self.path, self.path) + rposix.unsetenv(self.path) + + interpret(f, []) # does not crash From noreply at buildbot.pypy.org Mon Jan 28 20:53:18 2013 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 28 Jan 2013 20:53:18 +0100 (CET) Subject: [pypy-commit] pypy default: Add str0 and unicode0 to signature types. Message-ID: <20130128195318.386031C0041@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: Changeset: r60632:56b7dba4721d Date: 2013-01-28 20:52 +0100 http://bitbucket.org/pypy/pypy/changeset/56b7dba4721d/ Log: Add str0 and unicode0 to signature types. diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py --- a/rpython/annotator/annrpython.py +++ b/rpython/annotator/annrpython.py @@ -113,6 +113,9 @@ if policy is None: from rpython.annotator.policy import AnnotatorPolicy policy = AnnotatorPolicy() + # XXX hack + annmodel.TLS.check_str_without_nul = ( + self.translator.config.translation.check_str_without_nul) graph, inputcells = self.get_call_parameters(function, args_s, policy) self.build_graph_types(graph, inputcells, complete_now=False) self.complete_helpers(policy) diff --git a/rpython/rlib/test/test_signature.py b/rpython/rlib/test/test_signature.py --- a/rpython/rlib/test/test_signature.py +++ b/rpython/rlib/test/test_signature.py @@ -9,6 +9,7 @@ def annotate_at(f, policy=None): t = TranslationContext() + t.config.translation.check_str_without_nul = True a = t.buildannotator(policy=policy) a.annotate_helper(f, [model.s_ImpossibleValue]*f.func_code.co_argcount, policy=policy) return a @@ -112,6 +113,12 @@ return len(u) assert getsig(f) == [model.SomeUnicodeString(), model.SomeInteger()] +def test_str0(): + @signature(types.unicode0(), returns=types.str0()) + def f(u): + return 'str' + assert getsig(f) == [model.SomeUnicodeString(no_nul=True), + model.SomeString(no_nul=True)] def test_ptr(): policy = LowLevelAnnotatorPolicy() diff --git a/rpython/rlib/types.py b/rpython/rlib/types.py --- a/rpython/rlib/types.py +++ b/rpython/rlib/types.py @@ -24,9 +24,15 @@ def unicode(): return model.SomeUnicodeString() +def unicode0(): + return model.SomeUnicodeString(no_nul=True) + def str(): return model.SomeString() +def str0(): + return model.SomeString(no_nul=True) + def char(): return model.SomeChar() From noreply at buildbot.pypy.org Mon Jan 28 22:01:27 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 28 Jan 2013 22:01:27 +0100 (CET) Subject: [pypy-commit] pypy default: skip rather than xfail @inline tests Message-ID: <20130128210127.3D2131C0264@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60633:9151be93f949 Date: 2013-01-28 16:01 -0500 http://bitbucket.org/pypy/pypy/changeset/9151be93f949/ Log: skip rather than xfail @inline tests diff --git a/rpython/rlib/test/test_jit.py b/rpython/rlib/test/test_jit.py --- a/rpython/rlib/test/test_jit.py +++ b/rpython/rlib/test/test_jit.py @@ -35,8 +35,8 @@ assert driver.reds == ['a', 'b'] assert driver.numreds == 2 - at py.test.mark.xfail(reason="@inline off: see skipped failures in test_warmspot.") def test_jitdriver_inline(): + py.test.skip("@inline off: see skipped failures in test_warmspot.") driver = JitDriver(greens=[], reds='auto') calls = [] def foo(a, b): @@ -55,8 +55,8 @@ ('bar', 40, 2), ] - at py.test.mark.xfail(reason="@inline off: see skipped failures in test_warmspot.") def test_jitdriver_clone(): + py.test.skip("@inline off: see skipped failures in test_warmspot.") def bar(): pass def foo(): pass driver = JitDriver(greens=[], reds=[]) From noreply at buildbot.pypy.org Mon Jan 28 22:44:52 2013 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 28 Jan 2013 22:44:52 +0100 (CET) Subject: [pypy-commit] pypy default: Clone rffi.liststr2charpp(): in the py3k branch, this function is used both at normal rpython level Message-ID: <20130128214452.93D801C0041@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: Changeset: r60634:9f7de2ebbe58 Date: 2013-01-28 22:32 +0100 http://bitbucket.org/pypy/pypy/changeset/9f7de2ebbe58/ Log: Clone rffi.liststr2charpp(): in the py3k branch, this function is used both at normal rpython level (in _posixsubprocess module) and at a lower-level (in ll_os.py). It seems that strings differ between these levels, and two identical functions are needed. diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py --- a/rpython/rtyper/lltypesystem/rffi.py +++ b/rpython/rtyper/lltypesystem/rffi.py @@ -867,6 +867,8 @@ array[len(l)] = lltype.nullptr(CCHARP.TO) return array liststr2charpp._annenforceargs_ = [[annmodel.s_Str0]] # List of strings +# Make a copy for the ll_os.py module +ll_liststr2charpp = func_with_new_name(liststr2charpp, 'll_liststr2charpp') def free_charpp(ref): """ frees list of char**, NULL terminated diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py --- a/rpython/rtyper/module/ll_os.py +++ b/rpython/rtyper/module/ll_os.py @@ -306,7 +306,7 @@ rffi.INT, compilation_info = eci) def execv_llimpl(path, args): - l_args = rffi.liststr2charpp(args) + l_args = rffi.ll_liststr2charpp(args) os_execv(path, l_args) rffi.free_charpp(l_args) raise OSError(rposix.get_errno(), "execv failed") @@ -332,8 +332,8 @@ envstr = "%s=%s" % item envstrs.append(envstr) - l_args = rffi.liststr2charpp(args) - l_env = rffi.liststr2charpp(envstrs) + l_args = rffi.ll_liststr2charpp(args) + l_env = rffi.ll_liststr2charpp(envstrs) os_execve(path, l_args, l_env) # XXX untested @@ -357,7 +357,7 @@ def spawnv_llimpl(mode, path, args): mode = rffi.cast(rffi.INT, mode) - l_args = rffi.liststr2charpp(args) + l_args = rffi.ll_liststr2charpp(args) childpid = os_spawnv(mode, path, l_args) rffi.free_charpp(l_args) if childpid == -1: @@ -380,8 +380,8 @@ envstrs.append("%s=%s" % item) mode = rffi.cast(rffi.INT, mode) - l_args = rffi.liststr2charpp(args) - l_env = rffi.liststr2charpp(envstrs) + l_args = rffi.ll_liststr2charpp(args) + l_env = rffi.ll_liststr2charpp(envstrs) childpid = os_spawnve(mode, path, l_args, l_env) rffi.free_charpp(l_env) rffi.free_charpp(l_args) From noreply at buildbot.pypy.org Mon Jan 28 22:46:31 2013 From: noreply at buildbot.pypy.org (amauryfa) Date: Mon, 28 Jan 2013 22:46:31 +0100 (CET) Subject: [pypy-commit] pypy py3k: hg merge default Message-ID: <20130128214631.72BF81C0041@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3k Changeset: r60635:5f3eacb6f86c Date: 2013-01-28 22:46 +0100 http://bitbucket.org/pypy/pypy/changeset/5f3eacb6f86c/ Log: hg merge default diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py --- a/rpython/annotator/annrpython.py +++ b/rpython/annotator/annrpython.py @@ -113,6 +113,9 @@ if policy is None: from rpython.annotator.policy import AnnotatorPolicy policy = AnnotatorPolicy() + # XXX hack + annmodel.TLS.check_str_without_nul = ( + self.translator.config.translation.check_str_without_nul) graph, inputcells = self.get_call_parameters(function, args_s, policy) self.build_graph_types(graph, inputcells, complete_now=False) self.complete_helpers(policy) diff --git a/rpython/rlib/test/test_jit.py b/rpython/rlib/test/test_jit.py --- a/rpython/rlib/test/test_jit.py +++ b/rpython/rlib/test/test_jit.py @@ -35,8 +35,8 @@ assert driver.reds == ['a', 'b'] assert driver.numreds == 2 - at py.test.mark.xfail(reason="@inline off: see skipped failures in test_warmspot.") def test_jitdriver_inline(): + py.test.skip("@inline off: see skipped failures in test_warmspot.") driver = JitDriver(greens=[], reds='auto') calls = [] def foo(a, b): @@ -55,8 +55,8 @@ ('bar', 40, 2), ] - at py.test.mark.xfail(reason="@inline off: see skipped failures in test_warmspot.") def test_jitdriver_clone(): + py.test.skip("@inline off: see skipped failures in test_warmspot.") def bar(): pass def foo(): pass driver = JitDriver(greens=[], reds=[]) diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py --- a/rpython/rlib/test/test_rposix.py +++ b/rpython/rlib/test/test_rposix.py @@ -141,3 +141,10 @@ assert rposix.is_valid_fd(fd) == 1 fid.close() assert rposix.is_valid_fd(fd) == 0 + + def test_putenv(self): + def f(): + rposix.putenv(self.path, self.path) + rposix.unsetenv(self.path) + + interpret(f, []) # does not crash diff --git a/rpython/rlib/test/test_signature.py b/rpython/rlib/test/test_signature.py --- a/rpython/rlib/test/test_signature.py +++ b/rpython/rlib/test/test_signature.py @@ -9,6 +9,7 @@ def annotate_at(f, policy=None): t = TranslationContext() + t.config.translation.check_str_without_nul = True a = t.buildannotator(policy=policy) a.annotate_helper(f, [model.s_ImpossibleValue]*f.func_code.co_argcount, policy=policy) return a @@ -112,6 +113,12 @@ return len(u) assert getsig(f) == [model.SomeUnicodeString(), model.SomeInteger()] +def test_str0(): + @signature(types.unicode0(), returns=types.str0()) + def f(u): + return 'str' + assert getsig(f) == [model.SomeUnicodeString(no_nul=True), + model.SomeString(no_nul=True)] def test_ptr(): policy = LowLevelAnnotatorPolicy() diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py --- a/rpython/rtyper/rstr.py +++ b/rpython/rtyper/rstr.py @@ -88,7 +88,6 @@ startingpos, endingpos): raise UnicodeEncodeError(encoding, u, startingpos, endingpos, msg) - class __extend__(annmodel.SomeString): def rtyper_makerepr(self, rtyper): return rtyper.type_system.rstr.string_repr diff --git a/rpython/rtyper/test/test_runicode.py b/rpython/rtyper/test/test_runicode.py --- a/rpython/rtyper/test/test_runicode.py +++ b/rpython/rtyper/test/test_runicode.py @@ -106,6 +106,12 @@ assert self.ll_to_string(self.interpret(f, [38])) == f(38) + def g(n): + x = u'\ud800' + unichr(n) + return x.encode('utf-8') + + self.interpret_raises(UnicodeEncodeError, g, [38]) + def test_utf_8_encoding_annotation(self): from rpython.rlib.runicode import unicode_encode_utf_8 def errorhandler(errors, encoding, msg, u, From noreply at buildbot.pypy.org Mon Jan 28 23:00:54 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 28 Jan 2013 23:00:54 +0100 (CET) Subject: [pypy-commit] pypy default: fix test_chmod on win32 Message-ID: <20130128220054.92B311C0041@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60636:ff950660ad43 Date: 2013-01-28 16:59 -0500 http://bitbucket.org/pypy/pypy/changeset/ff950660ad43/ Log: fix test_chmod on win32 diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -829,8 +829,12 @@ f = open(self.path, "w") f.write("this is a test") f.close() - os.chmod(self.path, 0200) - assert (os.stat(self.path).st_mode & 0777) == 0200 + if sys.platform == 'win32': + os.chmod(self.path, 0400) + assert (os.stat(self.path).st_mode & 0600) == 0400 + else: + os.chmod(self.path, 0200) + assert (os.stat(self.path).st_mode & 0777) == 0200 if hasattr(os, 'fchmod'): def test_fchmod(self): From noreply at buildbot.pypy.org Mon Jan 28 23:00:55 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 28 Jan 2013 23:00:55 +0100 (CET) Subject: [pypy-commit] pypy default: merge heads Message-ID: <20130128220055.C25721C039A@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60637:fbe59aa7bdd5 Date: 2013-01-28 17:00 -0500 http://bitbucket.org/pypy/pypy/changeset/fbe59aa7bdd5/ Log: merge heads diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py --- a/rpython/rtyper/lltypesystem/rffi.py +++ b/rpython/rtyper/lltypesystem/rffi.py @@ -867,6 +867,8 @@ array[len(l)] = lltype.nullptr(CCHARP.TO) return array liststr2charpp._annenforceargs_ = [[annmodel.s_Str0]] # List of strings +# Make a copy for the ll_os.py module +ll_liststr2charpp = func_with_new_name(liststr2charpp, 'll_liststr2charpp') def free_charpp(ref): """ frees list of char**, NULL terminated diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py --- a/rpython/rtyper/module/ll_os.py +++ b/rpython/rtyper/module/ll_os.py @@ -306,7 +306,7 @@ rffi.INT, compilation_info = eci) def execv_llimpl(path, args): - l_args = rffi.liststr2charpp(args) + l_args = rffi.ll_liststr2charpp(args) os_execv(path, l_args) rffi.free_charpp(l_args) raise OSError(rposix.get_errno(), "execv failed") @@ -332,8 +332,8 @@ envstr = "%s=%s" % item envstrs.append(envstr) - l_args = rffi.liststr2charpp(args) - l_env = rffi.liststr2charpp(envstrs) + l_args = rffi.ll_liststr2charpp(args) + l_env = rffi.ll_liststr2charpp(envstrs) os_execve(path, l_args, l_env) # XXX untested @@ -357,7 +357,7 @@ def spawnv_llimpl(mode, path, args): mode = rffi.cast(rffi.INT, mode) - l_args = rffi.liststr2charpp(args) + l_args = rffi.ll_liststr2charpp(args) childpid = os_spawnv(mode, path, l_args) rffi.free_charpp(l_args) if childpid == -1: @@ -380,8 +380,8 @@ envstrs.append("%s=%s" % item) mode = rffi.cast(rffi.INT, mode) - l_args = rffi.liststr2charpp(args) - l_env = rffi.liststr2charpp(envstrs) + l_args = rffi.ll_liststr2charpp(args) + l_env = rffi.ll_liststr2charpp(envstrs) childpid = os_spawnve(mode, path, l_args, l_env) rffi.free_charpp(l_env) rffi.free_charpp(l_args) From noreply at buildbot.pypy.org Mon Jan 28 23:25:09 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 28 Jan 2013 23:25:09 +0100 (CET) Subject: [pypy-commit] pypy default: replace use of commands.getoutput with subprocess.check_output Message-ID: <20130128222509.E3CDD1C0264@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60638:7bf65ca5bf56 Date: 2013-01-28 17:22 -0500 http://bitbucket.org/pypy/pypy/changeset/7bf65ca5bf56/ Log: replace use of commands.getoutput with subprocess.check_output diff --git a/pypy/doc/test/test_whatsnew.py b/pypy/doc/test/test_whatsnew.py --- a/pypy/doc/test/test_whatsnew.py +++ b/pypy/doc/test/test_whatsnew.py @@ -1,6 +1,6 @@ import py import pypy -from commands import getoutput +from subprocess import check_output ROOT = py.path.local(pypy.__file__).dirpath().dirpath() @@ -29,8 +29,8 @@ merge() and \ branch(default)) and \ not branch(default)' % (startrev, endrev) - cmd = r'hg log -R "%s" -r "%s" --template "{branches}\n"' % (path, revset) - out = getoutput(cmd) + cmd = ['hg', 'log', '-R', str(path), '-r', revset, '--template', '{branches}\n'] + out = check_output(cmd) branches = set(map(str.strip, out.splitlines())) return branches diff --git a/pypy/tool/clean_old_branches.py b/pypy/tool/clean_old_branches.py --- a/pypy/tool/clean_old_branches.py +++ b/pypy/tool/clean_old_branches.py @@ -6,15 +6,15 @@ import os import sys -import commands +import subprocess if not os.path.isdir('.hg'): print 'Must run this script from the top-level directory.' sys.exit(1) def heads(): - result = commands.getoutput( - "hg heads --topo --closed --template '{node|short}:{branches}:{extras}\n'") + result = subprocess.check_output( + ['hg', 'heads', '--topo', '--closed', '--template', '{node|short}:{branches}:{extras}\n']) result = result.splitlines(False) result = [s.split(':', 2) for s in result] for line in result: From noreply at buildbot.pypy.org Mon Jan 28 23:44:14 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Mon, 28 Jan 2013 23:44:14 +0100 (CET) Subject: [pypy-commit] pypy default: win32 test fix Message-ID: <20130128224414.90B781C0041@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60639:447da353c5a8 Date: 2013-01-28 17:44 -0500 http://bitbucket.org/pypy/pypy/changeset/447da353c5a8/ Log: win32 test fix diff --git a/pypy/module/_multiprocessing/test/test_connection.py b/pypy/module/_multiprocessing/test/test_connection.py --- a/pypy/module/_multiprocessing/test/test_connection.py +++ b/pypy/module/_multiprocessing/test/test_connection.py @@ -68,7 +68,7 @@ assert rhandle.readable class AppTestWinpipeConnection(BaseConnectionTest): - spaceconfig = dict(usemodules=('_multiprocessing', 'thread')) + spaceconfig = dict(usemodules=('_multiprocessing', 'thread', 'signal')) def setup_class(cls): if sys.platform != "win32": From noreply at buildbot.pypy.org Tue Jan 29 00:20:14 2013 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 29 Jan 2013 00:20:14 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: only classes with _attrs_=() can go into make_sort_classes Message-ID: <20130128232014.6F32F1C039A@cobra.cs.uni-duesseldorf.de> Author: Matti Picus Branch: missing-ndarray-attributes Changeset: r60640:d7e093a28980 Date: 2013-01-29 01:17 +0200 http://bitbucket.org/pypy/pypy/changeset/d7e093a28980/ Log: only classes with _attrs_=() can go into make_sort_classes diff --git a/pypy/module/micronumpy/arrayimpl/sort.py b/pypy/module/micronumpy/arrayimpl/sort.py --- a/pypy/module/micronumpy/arrayimpl/sort.py +++ b/pypy/module/micronumpy/arrayimpl/sort.py @@ -82,11 +82,17 @@ def argsort_array(arr, space, w_axis): itemtype = arr.dtype.itemtype - if (not arr.dtype.is_int_type() and - not arr.dtype.is_float_type()): - # XXX this should probably be changed + # this is for runtime + if not (arr.dtype.is_int_type() or \ + arr.dtype.is_float_type() or \ + arr.dtype.is_complex_type()): raise OperationError(space.w_NotImplementedError, - space.wrap("sorting of non-numeric types '%r' is not implemented" % itemtype )) + space.wrap("sorting of non-numeric types " + \ + "'%r' is not implemented" % arr.dtype )) + # this is for translation + assert isinstance(itemtype, types.Float) or \ + isinstance(itemtype, types.Integer) or \ + isinstance(itemtype, types.ComplexFloating) if w_axis is space.w_None: # note that it's fine ot pass None here as we're not going # to pass the result around (None is the link to base in slices) @@ -113,7 +119,8 @@ if axis < 0: axis = len(shape) + axis - 1 if axis < 0 or axis > len(shape): - raise OperationError(space.w_IndexError("Wrong axis %d" % axis)) + raise OperationError(space.w_IndexError, space.wrap( + "Wrong axis %d" % axis)) iterable_shape = shape[:axis] + [0] + shape[axis + 1:] iter = AxisIterator(arr, iterable_shape, axis, False) index_impl = index_arr.implementation From noreply at buildbot.pypy.org Tue Jan 29 00:20:15 2013 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 29 Jan 2013 00:20:15 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: hack to give test_zjit a fighting chance Message-ID: <20130128232015.A87C81C039A@cobra.cs.uni-duesseldorf.de> Author: Matti Picus Branch: missing-ndarray-attributes Changeset: r60641:84175fe407d3 Date: 2013-01-29 01:19 +0200 http://bitbucket.org/pypy/pypy/changeset/84175fe407d3/ Log: hack to give test_zjit a fighting chance diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py --- a/rpython/jit/metainterp/compile.py +++ b/rpython/jit/metainterp/compile.py @@ -26,7 +26,7 @@ def show_procedures(metainterp_sd, procedure=None, error=None): # debugging - if option.view or option.viewloops: + if option and (option.view or option.viewloops): if error: errmsg = error.__class__.__name__ if str(error): From noreply at buildbot.pypy.org Tue Jan 29 00:48:07 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 29 Jan 2013 00:48:07 +0100 (CET) Subject: [pypy-commit] pypy default: skip these tests if not running on unix Message-ID: <20130128234807.8D8441C0041@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60642:45879262d4c1 Date: 2013-01-28 18:47 -0500 http://bitbucket.org/pypy/pypy/changeset/45879262d4c1/ Log: skip these tests if not running on unix diff --git a/pypy/module/crypt/test/test_crypt.py b/pypy/module/crypt/test/test_crypt.py --- a/pypy/module/crypt/test/test_crypt.py +++ b/pypy/module/crypt/test/test_crypt.py @@ -1,6 +1,11 @@ +import os, py + +if os.name != 'posix': + py.test.skip('crypt module only available on unix') + class AppTestCrypt: spaceconfig = dict(usemodules=['crypt']) - + def test_crypt(self): import crypt res = crypt.crypt("pass", "ab") diff --git a/pypy/module/fcntl/test/test_fcntl.py b/pypy/module/fcntl/test/test_fcntl.py --- a/pypy/module/fcntl/test/test_fcntl.py +++ b/pypy/module/fcntl/test/test_fcntl.py @@ -1,9 +1,8 @@ -import os +import os, py from rpython.tool.udir import udir -if os.name == "nt": - from py.test import skip - skip("fcntl module is not available on Windows") +if os.name != 'posix': + py.test.skip("fcntl module only available on unix") def teardown_module(mod): for i in "abcde": diff --git a/pypy/module/pwd/test/test_pwd.py b/pypy/module/pwd/test/test_pwd.py --- a/pypy/module/pwd/test/test_pwd.py +++ b/pypy/module/pwd/test/test_pwd.py @@ -1,3 +1,8 @@ +import os, py + +if os.name != 'posix': + py.test.skip('pwd module only available on unix') + class AppTestPwd: spaceconfig = dict(usemodules=['pwd']) diff --git a/pypy/module/termios/test/test_termios.py b/pypy/module/termios/test/test_termios.py --- a/pypy/module/termios/test/test_termios.py +++ b/pypy/module/termios/test/test_termios.py @@ -1,9 +1,11 @@ - import py -import sys +import sys, os from pypy.conftest import pypydir from rpython.tool.udir import udir +if os.name != 'posix': + py.test.skip('termios module only available on unix') + class TestTermios(object): def setup_class(cls): try: From noreply at buildbot.pypy.org Tue Jan 29 02:09:15 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Tue, 29 Jan 2013 02:09:15 +0100 (CET) Subject: [pypy-commit] pypy py3k: read arbitrary .py files in binary mode Message-ID: <20130129010915.8C4411C0041@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60643:94b14a170aad Date: 2013-01-28 17:06 -0800 http://bitbucket.org/pypy/pypy/changeset/94b14a170aad/ Log: read arbitrary .py files in binary mode diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -587,9 +587,8 @@ python_startup = readenv and os.getenv('PYTHONSTARTUP') if python_startup: try: - f = open(python_startup) - startup = f.read() - f.close() + with open(python_startup, 'rb') as f: + startup = f.read() except IOError as e: print("Could not open PYTHONSTARTUP", file=sys.stderr) print("IOError:", e, file=sys.stderr) @@ -642,7 +641,7 @@ else: # no. That's the normal path, "pypy stuff.py". def execfile(filename, namespace): - with open(filename) as f: + with open(filename, 'rb') as f: code = f.read() exec_(compile(code, filename, 'exec'), namespace) args = (execfile, filename, mainmodule.__dict__) From noreply at buildbot.pypy.org Tue Jan 29 02:09:16 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Tue, 29 Jan 2013 02:09:16 +0100 (CET) Subject: [pypy-commit] pypy py3k: emit different NameErrors for LOAD_GLOBAL/NAME Message-ID: <20130129010916.E86E71C0041@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60644:f5ceeba72c47 Date: 2013-01-28 17:07 -0800 http://bitbucket.org/pypy/pypy/changeset/f5ceeba72c47/ Log: emit different NameErrors for LOAD_GLOBAL/NAME diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -655,25 +655,26 @@ if w_value is not None: self.pushvalue(w_value) return - self.LOAD_GLOBAL(nameindex, next_instr) # fall-back + # fall-back + self.LOAD_GLOBAL(nameindex, next_instr, "name '%s' is not defined") - def _load_global(self, varname): + def _load_global(self, varname, errmsg): w_value = self.space.finditem_str(self.w_globals, varname) if w_value is None: # not in the globals, now look in the built-ins w_value = self.get_builtin().getdictvalue(self.space, varname) if w_value is None: - self._load_global_failed(varname) + self._load_global_failed(errmsg, varname) return w_value _load_global._always_inline_ = True - def _load_global_failed(self, varname): - message = "global name '%s' is not defined" - raise operationerrfmt(self.space.w_NameError, message, varname) + def _load_global_failed(self, errmsg, varname): + raise operationerrfmt(self.space.w_NameError, errmsg, varname) _load_global_failed._dont_inline_ = True - def LOAD_GLOBAL(self, nameindex, next_instr): - self.pushvalue(self._load_global(self.getname_u(nameindex))) + def LOAD_GLOBAL(self, nameindex, next_instr, + errmsg="global name '%s' is not defined"): + self.pushvalue(self._load_global(self.getname_u(nameindex), errmsg)) LOAD_GLOBAL._always_inline_ = True def DELETE_FAST(self, varindex, next_instr): From noreply at buildbot.pypy.org Tue Jan 29 02:09:18 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Tue, 29 Jan 2013 02:09:18 +0100 (CET) Subject: [pypy-commit] pypy py3k: cpython issue3574: fix latin1 decoding, it's no longer special cased Message-ID: <20130129010918.300DF1C0041@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60645:0dacceb8afd5 Date: 2013-01-28 17:07 -0800 http://bitbucket.org/pypy/pypy/changeset/0dacceb8afd5/ Log: cpython issue3574: fix latin1 decoding, it's no longer special cased diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py b/pypy/interpreter/astcompiler/test/test_astbuilder.py --- a/pypy/interpreter/astcompiler/test/test_astbuilder.py +++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py @@ -1134,6 +1134,16 @@ s = ast_from_node(space, tree, info).body[0].value assert isinstance(s, ast.Str) assert space.eq_w(s.s, space.wrap(japan)) + + def test_issue3574(self): + space = self.space + source = u'# coding: Latin-1\nu = "Ç"\n' + info = pyparse.CompileInfo("", "exec") + tree = self.parser.parse_source(source.encode("Latin-1"), info) + assert info.encoding == "iso-8859-1" + s = ast_from_node(space, tree, info).body[0].value + assert isinstance(s, ast.Str) + assert space.eq_w(s.s, space.wrap(u'Ç')) def test_string_bug(self): py.test.py3k_skip('fixme') diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py --- a/pypy/interpreter/pyparser/parsestring.py +++ b/pypy/interpreter/pyparser/parsestring.py @@ -5,7 +5,6 @@ def parsestr(space, encoding, s): """Parses a string or unicode literal, and return a wrapped value. - If encoding=iso8859-1, the source string is also in this encoding. If encoding=None, the source string is ascii only. In other cases, the source string is in utf-8 encoding. @@ -49,8 +48,7 @@ q -= 2 if unicode_literal and not rawmode: # XXX Py_UnicodeFlag is ignored for now - if encoding is None or encoding == "iso-8859-1": - # 'unicode_escape' expects latin-1 bytes, string is ready. + if encoding is None: buf = s bufp = ps bufq = q From noreply at buildbot.pypy.org Tue Jan 29 02:09:19 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Tue, 29 Jan 2013 02:09:19 +0100 (CET) Subject: [pypy-commit] pypy py3k: cpython issue3297: fix parsing of surrogates w/ wide builds Message-ID: <20130129010919.623421C0041@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60646:1b48d48dc26c Date: 2013-01-28 17:08 -0800 http://bitbucket.org/pypy/pypy/changeset/1b48d48dc26c/ Log: cpython issue3297: fix parsing of surrogates w/ wide builds diff --git a/pypy/interpreter/pyparser/parsestring.py b/pypy/interpreter/pyparser/parsestring.py --- a/pypy/interpreter/pyparser/parsestring.py +++ b/pypy/interpreter/pyparser/parsestring.py @@ -1,3 +1,4 @@ +# coding: utf-8 from pypy.interpreter.error import OperationError from pypy.interpreter import unicodehelper from rpython.rlib.rstring import StringBuilder @@ -58,7 +59,10 @@ # latin-1; So multibyte sequences must be escaped. lis = [] # using a list to assemble the value end = q - # Worst case: "\XX" may become "\u005c\uHHLL" (12 bytes) + # Worst case: + # "ä" (2 bytes) may become "\U000000E4" (10 bytes), or 1:5 + # "\ä" (3 bytes) may become "\u005c\U000000E4" (16 bytes), + # or ~1:6 while ps < end: if s[ps] == '\\': lis.append(s[ps]) @@ -70,13 +74,15 @@ # instead. lis.append("u005c") if ord(s[ps]) & 0x80: # XXX inefficient - w, ps = decode_utf8(space, s, ps, end, "utf-16-be") + w, ps = decode_utf8(space, s, ps, end, "utf-32-be") rn = len(w) - assert rn % 2 == 0 - for i in range(0, rn, 2): - lis.append('\\u') + assert rn % 4 == 0 + for i in range(0, rn, 4): + lis.append('\\U') lis.append(hexbyte(ord(w[i]))) lis.append(hexbyte(ord(w[i+1]))) + lis.append(hexbyte(ord(w[i+2]))) + lis.append(hexbyte(ord(w[i+3]))) else: lis.append(s[ps]) ps += 1 diff --git a/pypy/interpreter/test/test_exec.py b/pypy/interpreter/test/test_exec.py --- a/pypy/interpreter/test/test_exec.py +++ b/pypy/interpreter/test/test_exec.py @@ -199,3 +199,11 @@ x = ns['x'] assert len(x) == 6 assert ord(x[0]) == 0x0439 + + def test_issue3297(self): + c = compile("a, b = '\U0001010F', '\\U0001010F'", "dummy", "exec") + d = {} + exec(c, d) + assert d['a'] == d['b'] + assert len(d['a']) == len(d['b']) + assert ascii(d['a']) == ascii(d['b']) From noreply at buildbot.pypy.org Tue Jan 29 03:23:54 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Tue, 29 Jan 2013 03:23:54 +0100 (CET) Subject: [pypy-commit] pypy default: missing import Message-ID: <20130129022354.65DCF1C087E@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: Changeset: r60647:4e6eeb12cce6 Date: 2013-01-28 18:23 -0800 http://bitbucket.org/pypy/pypy/changeset/4e6eeb12cce6/ Log: missing import diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -823,6 +823,7 @@ if hasattr(os, 'chmod'): def test_chmod(self): + import sys os = self.posix os.unlink(self.path) raises(OSError, os.chmod, self.path, 0600) From noreply at buildbot.pypy.org Tue Jan 29 04:20:10 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 29 Jan 2013 04:20:10 +0100 (CET) Subject: [pypy-commit] pypy default: pep8 Message-ID: <20130129032010.B3C041C0264@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60648:63be88a51b60 Date: 2013-01-28 22:18 -0500 http://bitbucket.org/pypy/pypy/changeset/63be88a51b60/ Log: pep8 diff --git a/pypy/module/crypt/test/test_crypt.py b/pypy/module/crypt/test/test_crypt.py --- a/pypy/module/crypt/test/test_crypt.py +++ b/pypy/module/crypt/test/test_crypt.py @@ -1,4 +1,5 @@ -import os, py +import os +import py if os.name != 'posix': py.test.skip('crypt module only available on unix') diff --git a/pypy/module/fcntl/test/test_fcntl.py b/pypy/module/fcntl/test/test_fcntl.py --- a/pypy/module/fcntl/test/test_fcntl.py +++ b/pypy/module/fcntl/test/test_fcntl.py @@ -1,4 +1,5 @@ -import os, py +import os +import py from rpython.tool.udir import udir if os.name != 'posix': diff --git a/pypy/module/pwd/test/test_pwd.py b/pypy/module/pwd/test/test_pwd.py --- a/pypy/module/pwd/test/test_pwd.py +++ b/pypy/module/pwd/test/test_pwd.py @@ -1,4 +1,5 @@ -import os, py +import os +import py if os.name != 'posix': py.test.skip('pwd module only available on unix') diff --git a/pypy/module/termios/test/test_termios.py b/pypy/module/termios/test/test_termios.py --- a/pypy/module/termios/test/test_termios.py +++ b/pypy/module/termios/test/test_termios.py @@ -1,5 +1,6 @@ +import os +import sys import py -import sys, os from pypy.conftest import pypydir from rpython.tool.udir import udir From noreply at buildbot.pypy.org Tue Jan 29 06:32:00 2013 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 29 Jan 2013 06:32:00 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: add a failing test Message-ID: <20130129053200.5CCDC1C087E@cobra.cs.uni-duesseldorf.de> Author: Matti Picus Branch: missing-ndarray-attributes Changeset: r60649:9a8d7248bf7b Date: 2013-01-29 07:31 +0200 http://bitbucket.org/pypy/pypy/changeset/9a8d7248bf7b/ Log: add a failing test diff --git a/pypy/module/micronumpy/arrayimpl/sort.py b/pypy/module/micronumpy/arrayimpl/sort.py --- a/pypy/module/micronumpy/arrayimpl/sort.py +++ b/pypy/module/micronumpy/arrayimpl/sort.py @@ -88,7 +88,7 @@ arr.dtype.is_complex_type()): raise OperationError(space.w_NotImplementedError, space.wrap("sorting of non-numeric types " + \ - "'%r' is not implemented" % arr.dtype )) + "'%s' is not implemented" % arr.dtype.get_name() )) # this is for translation assert isinstance(itemtype, types.Float) or \ isinstance(itemtype, types.Integer) or \ diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -346,6 +346,10 @@ b = a.copy() assert (b == a).all() + a = array(['abc', 'def','xyz'], dtype='S3') + b = a.copy() + assert (b == a).all() + def test_iterator_init(self): from _numpypy import array a = array(range(5)) From noreply at buildbot.pypy.org Tue Jan 29 07:08:06 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Tue, 29 Jan 2013 07:08:06 +0100 (CET) Subject: [pypy-commit] pypy py3k: backout f5ceeba72c47 (fails translation) Message-ID: <20130129060806.3557E1C0207@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60650:800cb0287bf7 Date: 2013-01-28 22:07 -0800 http://bitbucket.org/pypy/pypy/changeset/800cb0287bf7/ Log: backout f5ceeba72c47 (fails translation) diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -655,26 +655,25 @@ if w_value is not None: self.pushvalue(w_value) return - # fall-back - self.LOAD_GLOBAL(nameindex, next_instr, "name '%s' is not defined") + self.LOAD_GLOBAL(nameindex, next_instr) # fall-back - def _load_global(self, varname, errmsg): + def _load_global(self, varname): w_value = self.space.finditem_str(self.w_globals, varname) if w_value is None: # not in the globals, now look in the built-ins w_value = self.get_builtin().getdictvalue(self.space, varname) if w_value is None: - self._load_global_failed(errmsg, varname) + self._load_global_failed(varname) return w_value _load_global._always_inline_ = True - def _load_global_failed(self, errmsg, varname): - raise operationerrfmt(self.space.w_NameError, errmsg, varname) + def _load_global_failed(self, varname): + message = "global name '%s' is not defined" + raise operationerrfmt(self.space.w_NameError, message, varname) _load_global_failed._dont_inline_ = True - def LOAD_GLOBAL(self, nameindex, next_instr, - errmsg="global name '%s' is not defined"): - self.pushvalue(self._load_global(self.getname_u(nameindex), errmsg)) + def LOAD_GLOBAL(self, nameindex, next_instr): + self.pushvalue(self._load_global(self.getname_u(nameindex))) LOAD_GLOBAL._always_inline_ = True def DELETE_FAST(self, varindex, next_instr): From noreply at buildbot.pypy.org Tue Jan 29 07:08:07 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Tue, 29 Jan 2013 07:08:07 +0100 (CET) Subject: [pypy-commit] pypy py3k: emit different NameErrors for LOAD_GLOBAL/NAME try 2 Message-ID: <20130129060807.6C6AD1C0207@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60651:76d9858633b2 Date: 2013-01-28 22:07 -0800 http://bitbucket.org/pypy/pypy/changeset/76d9858633b2/ Log: emit different NameErrors for LOAD_GLOBAL/NAME try 2 diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -655,15 +655,23 @@ if w_value is not None: self.pushvalue(w_value) return - self.LOAD_GLOBAL(nameindex, next_instr) # fall-back + # fall-back + self.LOAD_GLOBAL(nameindex, next_instr, self._load_name_failed) - def _load_global(self, varname): + def _load_name_failed(self, varname): + message = "name '%s' is not defined" + raise operationerrfmt(self.space.w_NameError, message, varname) + _load_name_failed._dont_inline_ = True + + def _load_global(self, varname, error_handler): w_value = self.space.finditem_str(self.w_globals, varname) if w_value is None: # not in the globals, now look in the built-ins w_value = self.get_builtin().getdictvalue(self.space, varname) if w_value is None: - self._load_global_failed(varname) + if error_handler is None: + error_handler = self._load_global_failed + error_handler(varname) return w_value _load_global._always_inline_ = True @@ -672,8 +680,9 @@ raise operationerrfmt(self.space.w_NameError, message, varname) _load_global_failed._dont_inline_ = True - def LOAD_GLOBAL(self, nameindex, next_instr): - self.pushvalue(self._load_global(self.getname_u(nameindex))) + def LOAD_GLOBAL(self, nameindex, next_instr, error_handler=None): + self.pushvalue(self._load_global(self.getname_u(nameindex), + error_handler)) LOAD_GLOBAL._always_inline_ = True def DELETE_FAST(self, varindex, next_instr): From noreply at buildbot.pypy.org Tue Jan 29 08:16:41 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 29 Jan 2013 08:16:41 +0100 (CET) Subject: [pypy-commit] pypy default: backout 7bf65ca5bf56 for python 2.6 compat Message-ID: <20130129071641.B30251C0207@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60652:be67791c3bb2 Date: 2013-01-29 01:49 -0500 http://bitbucket.org/pypy/pypy/changeset/be67791c3bb2/ Log: backout 7bf65ca5bf56 for python 2.6 compat diff --git a/pypy/doc/test/test_whatsnew.py b/pypy/doc/test/test_whatsnew.py --- a/pypy/doc/test/test_whatsnew.py +++ b/pypy/doc/test/test_whatsnew.py @@ -1,6 +1,6 @@ import py import pypy -from subprocess import check_output +from commands import getoutput ROOT = py.path.local(pypy.__file__).dirpath().dirpath() @@ -29,8 +29,8 @@ merge() and \ branch(default)) and \ not branch(default)' % (startrev, endrev) - cmd = ['hg', 'log', '-R', str(path), '-r', revset, '--template', '{branches}\n'] - out = check_output(cmd) + cmd = r'hg log -R "%s" -r "%s" --template "{branches}\n"' % (path, revset) + out = getoutput(cmd) branches = set(map(str.strip, out.splitlines())) return branches diff --git a/pypy/tool/clean_old_branches.py b/pypy/tool/clean_old_branches.py --- a/pypy/tool/clean_old_branches.py +++ b/pypy/tool/clean_old_branches.py @@ -6,15 +6,15 @@ import os import sys -import subprocess +import commands if not os.path.isdir('.hg'): print 'Must run this script from the top-level directory.' sys.exit(1) def heads(): - result = subprocess.check_output( - ['hg', 'heads', '--topo', '--closed', '--template', '{node|short}:{branches}:{extras}\n']) + result = commands.getoutput( + "hg heads --topo --closed --template '{node|short}:{branches}:{extras}\n'") result = result.splitlines(False) result = [s.split(':', 2) for s in result] for line in result: From noreply at buildbot.pypy.org Tue Jan 29 08:16:43 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 29 Jan 2013 08:16:43 +0100 (CET) Subject: [pypy-commit] pypy default: fix lib-python/conftest.py for running tests untranslated Message-ID: <20130129071643.078D81C0264@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60653:b50f2e3d6c2e Date: 2013-01-29 01:49 -0500 http://bitbucket.org/pypy/pypy/changeset/b50f2e3d6c2e/ Log: fix lib-python/conftest.py for running tests untranslated diff --git a/lib-python/conftest.py b/lib-python/conftest.py --- a/lib-python/conftest.py +++ b/lib-python/conftest.py @@ -61,7 +61,7 @@ usemodules = '', skip=None): self.basename = basename - self._usemodules = usemodules.split() + ['signal'] + self._usemodules = usemodules.split() + ['signal', 'rctime', 'itertools', '_socket'] self._compiler = compiler self.core = core self.skip = skip @@ -103,29 +103,29 @@ testmap = [ RegrTest('test___all__.py', core=True), RegrTest('test___future__.py', core=True), - RegrTest('test__locale.py', skip=skip_win32), + RegrTest('test__locale.py', usemodules='_locale', skip=skip_win32), RegrTest('test_abc.py'), RegrTest('test_abstract_numbers.py'), RegrTest('test_aepack.py', skip=True), RegrTest('test_aifc.py'), - RegrTest('test_argparse.py'), RegrTest('test_al.py', skip=True), - RegrTest('test_ast.py', core=True), - RegrTest('test_anydbm.py'), + RegrTest('test_anydbm.py', usemodules='struct'), RegrTest('test_applesingle.py', skip=True), - RegrTest('test_array.py', core=True, usemodules='struct array'), + RegrTest('test_argparse.py', usemodules='binascii'), + RegrTest('test_array.py', core=True, usemodules='struct array binascii'), RegrTest('test_ascii_formatd.py'), - RegrTest('test_asynchat.py', usemodules='thread'), - RegrTest('test_asyncore.py'), + RegrTest('test_ast.py', core=True, usemodules='struct'), + RegrTest('test_asynchat.py', usemodules='select fcntl'), + RegrTest('test_asyncore.py', usemodules='select fcntl'), RegrTest('test_atexit.py', core=True), RegrTest('test_audioop.py', skip=True), RegrTest('test_augassign.py', core=True), - RegrTest('test_base64.py'), + RegrTest('test_base64.py', usemodules='struct'), RegrTest('test_bastion.py'), + RegrTest('test_bigaddrspace.py'), + RegrTest('test_bigmem.py'), RegrTest('test_binascii.py', usemodules='binascii'), - RegrTest('test_binhex.py'), - RegrTest('test_binop.py', core=True), RegrTest('test_bisect.py', core=True, usemodules='_bisect'), RegrTest('test_bool.py', core=True), @@ -134,22 +134,23 @@ RegrTest('test_bsddb3.py', skip="unsupported extension module"), RegrTest('test_buffer.py'), RegrTest('test_bufio.py', core=True), - RegrTest('test_builtin.py', core=True), - RegrTest('test_bytes.py'), + RegrTest('test_builtin.py', core=True, usemodules='binascii'), + RegrTest('test_bytes.py', usemodules='struct binascii'), RegrTest('test_bz2.py', usemodules='bz2'), RegrTest('test_calendar.py'), RegrTest('test_call.py', core=True), RegrTest('test_capi.py', skip="not applicable"), RegrTest('test_cd.py', skip=True), RegrTest('test_cfgparser.py'), - RegrTest('test_cgi.py'), RegrTest('test_charmapcodec.py', core=True), RegrTest('test_cl.py', skip=True), RegrTest('test_class.py', core=True), RegrTest('test_cmath.py', core=True), RegrTest('test_cmd.py'), + RegrTest('test_cmd_line.py'), RegrTest('test_cmd_line_script.py'), + RegrTest('test_code.py', core=True), RegrTest('test_codeccallbacks.py', core=True), RegrTest('test_codecencodings_cn.py', usemodules='_multibytecodec'), RegrTest('test_codecencodings_hk.py', usemodules='_multibytecodec'), @@ -157,7 +158,6 @@ RegrTest('test_codecencodings_jp.py', usemodules='_multibytecodec'), RegrTest('test_codecencodings_kr.py', usemodules='_multibytecodec'), RegrTest('test_codecencodings_tw.py', usemodules='_multibytecodec'), - RegrTest('test_codecmaps_cn.py', usemodules='_multibytecodec'), RegrTest('test_codecmaps_hk.py', usemodules='_multibytecodec'), RegrTest('test_codecmaps_jp.py', usemodules='_multibytecodec'), @@ -165,6 +165,7 @@ RegrTest('test_codecmaps_tw.py', usemodules='_multibytecodec'), RegrTest('test_codecs.py', core=True, usemodules='_multibytecodec'), RegrTest('test_codeop.py', core=True), + RegrTest('test_coding.py', core=True), RegrTest('test_coercion.py', core=True), RegrTest('test_collections.py'), RegrTest('test_colorsys.py'), @@ -174,22 +175,24 @@ RegrTest('test_compileall.py'), RegrTest('test_compiler.py', core=False, skip="slowly deprecating compiler"), RegrTest('test_complex.py', core=True), - + RegrTest('test_complex_args.py'), RegrTest('test_contains.py', core=True), + RegrTest('test_contextlib.py', usemodules="thread"), RegrTest('test_cookie.py'), RegrTest('test_cookielib.py'), RegrTest('test_copy.py', core=True), RegrTest('test_copy_reg.py', core=True), RegrTest('test_cpickle.py', core=True), - RegrTest('test_cprofile.py'), + RegrTest('test_cprofile.py'), RegrTest('test_crypt.py', usemodules='crypt', skip=skip_win32), RegrTest('test_csv.py', usemodules='_csv'), - + RegrTest('test_ctypes.py', usemodules="_rawffi thread"), RegrTest('test_curses.py', skip="unsupported extension module"), RegrTest('test_datetime.py'), RegrTest('test_dbm.py'), RegrTest('test_decimal.py'), RegrTest('test_decorators.py', core=True), + RegrTest('test_defaultdict.py', usemodules='_collections'), RegrTest('test_deque.py', core=True, usemodules='_collections'), RegrTest('test_descr.py', core=True, usemodules='_weakref'), RegrTest('test_descrtut.py', core=True), @@ -208,12 +211,13 @@ RegrTest('test_dummy_thread.py', core=True), RegrTest('test_dummy_threading.py', core=True), RegrTest('test_email.py'), - RegrTest('test_email_codecs.py'), + RegrTest('test_email_renamed.py'), RegrTest('test_enumerate.py', core=True), RegrTest('test_eof.py', core=True), RegrTest('test_epoll.py'), RegrTest('test_errno.py', usemodules="errno"), + RegrTest('test_exception_variations.py'), RegrTest('test_exceptions.py', core=True), RegrTest('test_extcall.py', core=True), RegrTest('test_fcntl.py', usemodules='fcntl', skip=skip_win32), @@ -222,6 +226,7 @@ RegrTest('test_filecmp.py', core=True), RegrTest('test_fileinput.py', core=True), RegrTest('test_fileio.py'), + RegrTest('test_float.py', core=True), RegrTest('test_fnmatch.py', core=True), RegrTest('test_fork1.py', usemodules="thread"), RegrTest('test_format.py', core=True), @@ -230,6 +235,7 @@ RegrTest('test_frozen.py', skip="unsupported extension module"), RegrTest('test_ftplib.py'), RegrTest('test_funcattrs.py', core=True), + RegrTest('test_functools.py'), RegrTest('test_future.py', core=True), RegrTest('test_future1.py', core=True), RegrTest('test_future2.py', core=True), @@ -245,24 +251,19 @@ RegrTest('test_genexps.py', core=True, usemodules='_weakref'), RegrTest('test_getargs.py', skip="unsupported extension module"), RegrTest('test_getargs2.py', skip="unsupported extension module"), - RegrTest('test_getopt.py', core=True), RegrTest('test_gettext.py'), - RegrTest('test_gl.py', skip=True), RegrTest('test_glob.py', core=True), RegrTest('test_global.py', core=True), RegrTest('test_grammar.py', core=True), RegrTest('test_grp.py', skip=skip_win32), - - RegrTest('test_gzip.py'), + RegrTest('test_gzip.py', usemodules='zlib'), RegrTest('test_hash.py', core=True), RegrTest('test_hashlib.py', core=True), - RegrTest('test_heapq.py', core=True), RegrTest('test_hmac.py'), RegrTest('test_hotshot.py', skip="unsupported extension module"), - RegrTest('test_htmllib.py'), RegrTest('test_htmlparser.py'), RegrTest('test_httplib.py'), @@ -274,6 +275,7 @@ RegrTest('test_import.py', core=True), RegrTest('test_importhooks.py', core=True), RegrTest('test_importlib.py'), + RegrTest('test_index.py'), RegrTest('test_inspect.py'), RegrTest('test_int.py', core=True), RegrTest('test_int_literal.py', core=True), @@ -307,7 +309,7 @@ RegrTest('test_mhlib.py'), RegrTest('test_mimetools.py'), RegrTest('test_mimetypes.py'), - RegrTest('test_MimeWriter.py', core=False), + RegrTest('test_MimeWriter.py', core=False, usemodules='binascii'), RegrTest('test_minidom.py'), RegrTest('test_mmap.py', usemodules="mmap"), RegrTest('test_module.py', core=True), @@ -324,11 +326,11 @@ RegrTest('test_nis.py', skip="unsupported extension module"), RegrTest('test_normalization.py'), RegrTest('test_ntpath.py'), + RegrTest('test_old_mailbox.py'), RegrTest('test_opcodes.py', core=True), RegrTest('test_openpty.py'), RegrTest('test_operator.py', core=True), RegrTest('test_optparse.py'), - RegrTest('test_os.py', core=True), RegrTest('test_ossaudiodev.py', skip="unsupported extension module"), RegrTest('test_parser.py', skip="slowly deprecating compiler"), @@ -338,12 +340,14 @@ RegrTest('test_pep263.py'), RegrTest('test_pep277.py'), RegrTest('test_pep292.py'), + RegrTest('test_pep352.py'), RegrTest('test_pickle.py', core=True), RegrTest('test_pickletools.py', core=False), RegrTest('test_pipes.py'), RegrTest('test_pkg.py', core=True), RegrTest('test_pkgimport.py', core=True), RegrTest('test_pkgutil.py'), + RegrTest('test_platform.py'), RegrTest('test_plistlib.py', skip="unsupported module"), RegrTest('test_poll.py', skip=skip_win32), RegrTest('test_popen.py'), @@ -374,8 +378,8 @@ RegrTest('test_rfc822.py'), RegrTest('test_richcmp.py', core=True), RegrTest('test_rlcompleter.py'), - RegrTest('test_robotparser.py'), + RegrTest('test_runpy.py'), RegrTest('test_sax.py'), RegrTest('test_scope.py', core=True), RegrTest('test_scriptpackages.py', skip="unsupported extension module"), @@ -389,26 +393,24 @@ RegrTest('test_shlex.py'), RegrTest('test_shutil.py'), RegrTest('test_signal.py'), - RegrTest('test_SimpleHTTPServer.py'), + RegrTest('test_SimpleHTTPServer.py', usemodules='binascii'), RegrTest('test_site.py', core=False), RegrTest('test_slice.py', core=True), RegrTest('test_smtplib.py'), RegrTest('test_smtpnet.py'), RegrTest('test_socket.py', usemodules='thread _weakref'), - RegrTest('test_socketserver.py', usemodules='thread'), - RegrTest('test_softspace.py', core=True), RegrTest('test_sort.py', core=True), + RegrTest('test_sqlite.py', usemodules="thread _rawffi zlib"), RegrTest('test_ssl.py', usemodules='_ssl _socket select'), + RegrTest('test_startfile.py'), RegrTest('test_str.py', core=True), - RegrTest('test_strftime.py'), RegrTest('test_string.py', core=True), - RegrTest('test_StringIO.py', core=True, usemodules='cStringIO'), + RegrTest('test_StringIO.py', core=True, usemodules='cStringIO array'), RegrTest('test_stringprep.py'), RegrTest('test_strop.py', skip="deprecated"), - RegrTest('test_strptime.py'), RegrTest('test_strtod.py'), RegrTest('test_struct.py', usemodules='struct'), @@ -420,33 +422,30 @@ RegrTest('test_symtable.py', skip="implementation detail"), RegrTest('test_syntax.py', core=True), RegrTest('test_sys.py', core=True, usemodules='struct'), + RegrTest('test_sys_setprofile.py', core=True), RegrTest('test_sys_settrace.py', core=True), - RegrTest('test_sys_setprofile.py', core=True), RegrTest('test_sysconfig.py'), + RegrTest('test_tarfile.py'), RegrTest('test_tcl.py', skip="unsupported extension module"), - RegrTest('test_tarfile.py'), RegrTest('test_telnetlib.py'), RegrTest('test_tempfile.py'), - RegrTest('test_textwrap.py'), RegrTest('test_thread.py', usemodules="thread", core=True), RegrTest('test_threaded_import.py', usemodules="thread", core=True), RegrTest('test_threadedtempfile.py', usemodules="thread", core=False), - RegrTest('test_threading.py', usemodules="thread", core=True), RegrTest('test_threading_local.py', usemodules="thread", core=True), RegrTest('test_threadsignals.py', usemodules="thread"), - RegrTest('test_time.py', core=True), RegrTest('test_timeout.py'), RegrTest('test_tk.py'), - RegrTest('test_ttk_guionly.py'), - RegrTest('test_ttk_textonly.py'), RegrTest('test_tokenize.py'), RegrTest('test_trace.py'), RegrTest('test_traceback.py', core=True), RegrTest('test_transformer.py', core=True), + RegrTest('test_ttk_guionly.py'), + RegrTest('test_ttk_textonly.py'), RegrTest('test_tuple.py', core=True), RegrTest('test_typechecks.py'), RegrTest('test_types.py', core=True), @@ -462,6 +461,7 @@ RegrTest('test_unpack.py', core=True), RegrTest('test_urllib.py'), RegrTest('test_urllib2.py'), + RegrTest('test_urllib2_localnet.py', usemodules="thread"), RegrTest('test_urllib2net.py'), RegrTest('test_urllibnet.py'), RegrTest('test_urlparse.py'), @@ -469,60 +469,35 @@ RegrTest('test_userlist.py', core=True), RegrTest('test_userstring.py', core=True), RegrTest('test_uu.py'), - + RegrTest('test_uuid.py'), + RegrTest('test_wait3.py', usemodules="thread"), + RegrTest('test_wait4.py', usemodules="thread"), RegrTest('test_warnings.py', core=True), RegrTest('test_wave.py', skip="unsupported extension module"), RegrTest('test_weakref.py', core=True, usemodules='_weakref'), RegrTest('test_weakset.py'), - RegrTest('test_whichdb.py'), RegrTest('test_winreg.py', skip=only_win32), RegrTest('test_winsound.py', skip="unsupported extension module"), - RegrTest('test_xmllib.py'), - RegrTest('test_xmlrpc.py'), - - RegrTest('test_xpickle.py'), - RegrTest('test_xrange.py', core=True), - RegrTest('test_zipfile.py'), - RegrTest('test_zipimport.py', usemodules='zlib zipimport'), - RegrTest('test_zipimport_support.py', usemodules='zlib zipimport'), - RegrTest('test_zlib.py', usemodules='zlib'), - - RegrTest('test_bigaddrspace.py'), - RegrTest('test_bigmem.py'), - RegrTest('test_cmd_line.py'), - RegrTest('test_code.py'), - RegrTest('test_coding.py'), - RegrTest('test_complex_args.py'), - RegrTest('test_contextlib.py', usemodules="thread"), - RegrTest('test_ctypes.py', usemodules="_rawffi thread"), - RegrTest('test_defaultdict.py', usemodules='_collections'), - RegrTest('test_email_renamed.py'), - RegrTest('test_exception_variations.py'), - RegrTest('test_float.py'), - RegrTest('test_functools.py'), - RegrTest('test_index.py'), - RegrTest('test_old_mailbox.py'), - RegrTest('test_pep352.py'), - RegrTest('test_platform.py'), - RegrTest('test_runpy.py'), - RegrTest('test_sqlite.py', usemodules="thread _rawffi zlib"), - RegrTest('test_startfile.py', skip="bogus test"), - RegrTest('test_structmembers.py', skip="depends on _testcapi"), - RegrTest('test_urllib2_localnet.py', usemodules="thread"), - RegrTest('test_uuid.py'), - RegrTest('test_wait3.py', usemodules="thread"), - RegrTest('test_wait4.py', usemodules="thread"), RegrTest('test_with.py'), RegrTest('test_wsgiref.py'), RegrTest('test_xdrlib.py'), RegrTest('test_xml_etree.py'), RegrTest('test_xml_etree_c.py'), + RegrTest('test_xmllib.py'), + RegrTest('test_xmlrpc.py'), + RegrTest('test_xpickle.py'), + RegrTest('test_xrange.py', core=True), + RegrTest('test_zipfile.py'), RegrTest('test_zipfile64.py'), + RegrTest('test_zipimport.py', usemodules='zlib zipimport'), + RegrTest('test_zipimport_support.py', usemodules='zlib zipimport'), + RegrTest('test_zlib.py', usemodules='zlib'), ] def check_testmap_complete(): listed_names = dict.fromkeys([regrtest.basename for regrtest in testmap]) + assert len(listed_names) == len(testmap) listed_names['test_support.py'] = True # ignore this missing = [] for path in testdir.listdir(fil='test_*.py'): @@ -578,7 +553,7 @@ def getinvocation(self, regrtest): fspath = regrtest.getfspath() python = sys.executable - pypy_script = pypydir.join('bin', 'py.py') + pypy_script = pypydir.join('bin', 'pyinteractive.py') alarm_script = pypydir.join('tool', 'alarm.py') if sys.platform == 'win32': watchdog_name = 'watchdog_nt.py' From noreply at buildbot.pypy.org Tue Jan 29 08:16:44 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 29 Jan 2013 08:16:44 +0100 (CET) Subject: [pypy-commit] pypy py3k: merge default Message-ID: <20130129071644.553871C039A@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: py3k Changeset: r60654:80098444cbc8 Date: 2013-01-29 02:14 -0500 http://bitbucket.org/pypy/pypy/changeset/80098444cbc8/ Log: merge default diff --git a/lib-python/conftest.py b/lib-python/conftest.py --- a/lib-python/conftest.py +++ b/lib-python/conftest.py @@ -61,7 +61,8 @@ usemodules = '', skip=None): self.basename = basename - self._usemodules = usemodules.split() + ['signal', '_warnings'] + self._usemodules = usemodules.split() + ['signal', 'rctime', 'binascii', '_socket', + 'select', 'fcntl', '_posixsubprocess'] self._compiler = compiler self.core = core self.skip = skip @@ -103,19 +104,19 @@ testmap = [ RegrTest('test___all__.py', core=True), RegrTest('test___future__.py', core=True), - RegrTest('test__locale.py', skip=skip_win32), + RegrTest('test__locale.py', usemodules='_locale', skip=skip_win32), RegrTest('test_abc.py'), RegrTest('test_abstract_numbers.py'), RegrTest('test_aifc.py'), - RegrTest('test_argparse.py'), - RegrTest('test_array.py', core=True, usemodules='struct array'), - RegrTest('test_ast.py', core=True), - RegrTest('test_asynchat.py', usemodules='thread'), - RegrTest('test_asyncore.py'), + RegrTest('test_argparse.py', usemodules='binascii'), + RegrTest('test_array.py', core=True, usemodules='struct array binascii'), + RegrTest('test_ast.py', core=True, usemodules='struct'), + RegrTest('test_asynchat.py', usemodules='select fcntl'), + RegrTest('test_asyncore.py', usemodules='select fcntl'), RegrTest('test_atexit.py', core=True), RegrTest('test_audioop.py', skip=True), RegrTest('test_augassign.py', core=True), - RegrTest('test_base64.py'), + RegrTest('test_base64.py', usemodules='struct'), RegrTest('test_bigaddrspace.py'), RegrTest('test_bigmem.py'), RegrTest('test_binascii.py', usemodules='binascii'), @@ -124,8 +125,8 @@ RegrTest('test_bisect.py', core=True, usemodules='_bisect'), RegrTest('test_bool.py', core=True), RegrTest('test_bufio.py', core=True), - RegrTest('test_builtin.py', core=True), - RegrTest('test_bytes.py'), + RegrTest('test_builtin.py', core=True, usemodules='binascii'), + RegrTest('test_bytes.py', usemodules='struct binascii'), RegrTest('test_bz2.py', usemodules='bz2'), RegrTest('test_calendar.py'), RegrTest('test_call.py', core=True), @@ -371,7 +372,7 @@ RegrTest('test_sort.py', core=True), RegrTest('test_sqlite.py', usemodules="thread _rawffi zlib"), RegrTest('test_ssl.py', usemodules='_ssl _socket select'), - RegrTest('test_startfile.py'), # skip="bogus test"? + RegrTest('test_startfile.py'), RegrTest('test_strftime.py'), RegrTest('test_string.py', core=True), RegrTest('test_stringprep.py'), @@ -461,6 +462,7 @@ def check_testmap_complete(): listed_names = dict.fromkeys([regrtest.basename for regrtest in testmap]) + assert len(listed_names) == len(testmap) listed_names['test_support.py'] = True # ignore this missing = [] for path in testdir.listdir(fil='test_*.py'): @@ -516,7 +518,7 @@ def getinvocation(self, regrtest): fspath = regrtest.getfspath() python = sys.executable - pypy_script = pypydir.join('bin', 'py.py') + pypy_script = pypydir.join('bin', 'pyinteractive.py') alarm_script = pypydir.join('tool', 'alarm.py') if sys.platform == 'win32': watchdog_name = 'watchdog_nt.py' diff --git a/pypy/module/_multiprocessing/test/test_connection.py b/pypy/module/_multiprocessing/test/test_connection.py --- a/pypy/module/_multiprocessing/test/test_connection.py +++ b/pypy/module/_multiprocessing/test/test_connection.py @@ -67,7 +67,7 @@ assert rhandle.readable class AppTestWinpipeConnection(BaseConnectionTest): - spaceconfig = dict(usemodules=('_multiprocessing', 'thread')) + spaceconfig = dict(usemodules=('_multiprocessing', 'thread', 'signal')) def setup_class(cls): if sys.platform != "win32": diff --git a/pypy/module/crypt/test/test_crypt.py b/pypy/module/crypt/test/test_crypt.py --- a/pypy/module/crypt/test/test_crypt.py +++ b/pypy/module/crypt/test/test_crypt.py @@ -1,6 +1,12 @@ +import os +import py + +if os.name != 'posix': + py.test.skip('crypt module only available on unix') + class AppTestCrypt: spaceconfig = dict(usemodules=['crypt']) - + def test_crypt(self): import crypt res = crypt.crypt("pass", "ab") diff --git a/pypy/module/fcntl/test/test_fcntl.py b/pypy/module/fcntl/test/test_fcntl.py --- a/pypy/module/fcntl/test/test_fcntl.py +++ b/pypy/module/fcntl/test/test_fcntl.py @@ -1,9 +1,9 @@ import os +import py from rpython.tool.udir import udir -if os.name == "nt": - from py.test import skip - skip("fcntl module is not available on Windows") +if os.name != 'posix': + py.test.skip("fcntl module only available on unix") def teardown_module(mod): for i in "abcde": diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -797,14 +797,19 @@ if hasattr(os, 'chmod'): def test_chmod(self): + import sys os = self.posix os.unlink(self.path) raises(OSError, os.chmod, self.path, 0o600) f = open(self.path, "w") f.write("this is a test") f.close() - os.chmod(self.path, 0o200) - assert (os.stat(self.path).st_mode & 0o777) == 0o200 + if sys.platform == 'win32': + os.chmod(self.path, 0o400) + assert (os.stat(self.path).st_mode & 0o600) == 0o400 + else: + os.chmod(self.path, 0o200) + assert (os.stat(self.path).st_mode & 0o777) == 0o200 if hasattr(os, 'fchmod'): def test_fchmod(self): diff --git a/pypy/module/pwd/test/test_pwd.py b/pypy/module/pwd/test/test_pwd.py --- a/pypy/module/pwd/test/test_pwd.py +++ b/pypy/module/pwd/test/test_pwd.py @@ -1,3 +1,9 @@ +import os +import py + +if os.name != 'posix': + py.test.skip('pwd module only available on unix') + class AppTestPwd: spaceconfig = dict(usemodules=['pwd']) diff --git a/pypy/module/termios/test/test_termios.py b/pypy/module/termios/test/test_termios.py --- a/pypy/module/termios/test/test_termios.py +++ b/pypy/module/termios/test/test_termios.py @@ -1,9 +1,12 @@ - +import os +import sys import py -import sys from pypy.conftest import pypydir from rpython.tool.udir import udir +if os.name != 'posix': + py.test.skip('termios module only available on unix') + class TestTermios(object): def setup_class(cls): try: From noreply at buildbot.pypy.org Tue Jan 29 08:55:09 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Tue, 29 Jan 2013 08:55:09 +0100 (CET) Subject: [pypy-commit] pypy quiet-rpython: -q/--quiet option flag for rpython to make its output less spammy Message-ID: <20130129075509.6FE7E1C0264@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: quiet-rpython Changeset: r60655:29f82492ab85 Date: 2013-01-28 17:15 +0100 http://bitbucket.org/pypy/pypy/changeset/29f82492ab85/ Log: -q/--quiet option flag for rpython to make its output less spammy diff --git a/rpython/tool/ansi_print.py b/rpython/tool/ansi_print.py --- a/rpython/tool/ansi_print.py +++ b/rpython/tool/ansi_print.py @@ -4,7 +4,18 @@ import sys from py.io import ansi_print -from rpython.tool.ansi_mandelbrot import Driver +from rpython.tool.ansi_mandelbrot import Driver as MandelbrotDriver + +class SpinningDriver(object): + def __init__(self): + self.states = ['|', '/', '-', '\\'] + self.state = 0 + def reset(self): + self.state = 0 + def dot(self): + sys.stderr.write(self.states[self.state] + '\b') + self.state += 1 + self.state %= len(self.states) class AnsiLog: wrote_dot = False # XXX sharing state with all instances @@ -20,17 +31,32 @@ 'info': ((35,), False), 'stub': ((34,), False), } + + log_on_quiet = [ + "ERROR", + "Error", + "info", + ] def __init__(self, kw_to_color={}, file=None): self.kw_to_color = self.KW_TO_COLOR.copy() self.kw_to_color.update(kw_to_color) self.file = file - self.fancy = True self.isatty = getattr(sys.stderr, 'isatty', lambda: False) - if self.fancy and self.isatty(): - self.mandelbrot_driver = Driver() - else: - self.mandelbrot_driver = None + self.driver = None + self.set_option(fancy=True, quiet=False) + + def set_option(self, fancy=None, quiet=None): + if fancy is not None: + self.fancy = fancy + if quiet is not None: + self.quiet = quiet + + self.driver = None + if self.isatty and self.fancy: + self.driver = MandelbrotDriver() + if self.isatty and self.quiet: + self.driver = SpinningDriver() def __call__(self, msg): tty = self.isatty() @@ -55,10 +81,10 @@ return elif 'dot' in keywords: if tty: - if self.fancy: + if self.driver is not None: if not AnsiLog.wrote_dot: - self.mandelbrot_driver.reset() - self.mandelbrot_driver.dot() + self.driver.reset() + self.driver.dot() else: ansi_print(".", tuple(esc), file=self.file, newline=False, flush=flush) AnsiLog.wrote_dot = True @@ -67,8 +93,9 @@ AnsiLog.wrote_dot = False sys.stderr.write("\n") esc = tuple(esc) - for line in msg.content().splitlines(): - ansi_print("[%s] %s" %(":".join(keywords), line), esc, - file=self.file, newline=newline, flush=flush) + if not self.quiet or any([kw in self.log_on_quiet for kw in keywords]): + for line in msg.content().splitlines(): + ansi_print("[%s] %s" %(":".join(keywords), line), esc, + file=self.file, newline=newline, flush=flush) ansi_log = AnsiLog() diff --git a/rpython/translator/goal/translate.py b/rpython/translator/goal/translate.py --- a/rpython/translator/goal/translate.py +++ b/rpython/translator/goal/translate.py @@ -74,6 +74,8 @@ cmdline="-h --help", negation=False), BoolOption("fullhelp", "show full help message and exit", default=False, cmdline="--full-help", negation=False), + BoolOption("quiet", "be less verbose and don't paint fractals", default=False, + cmdline="-q --quiet", negation=False), ArbitraryOption("goals", "XXX", defaultfactory=list), # xxx default goals ['annotate', 'rtype', 'backendopt', 'source', 'compile'] @@ -117,6 +119,8 @@ options, args = opt_parser.parse_args() + ansi_log.set_option(quiet=translateconfig.quiet) + # set goals and skipped_goals reset = False for name, _, _, _ in GOALS: diff --git a/rpython/translator/platform/__init__.py b/rpython/translator/platform/__init__.py --- a/rpython/translator/platform/__init__.py +++ b/rpython/translator/platform/__init__.py @@ -7,7 +7,7 @@ from rpython.tool.udir import udir log = py.log.Producer("platform") - +py.log.setconsumer("platform", ansi_log) class CompilationError(Exception): def __init__(self, out, err): From noreply at buildbot.pypy.org Tue Jan 29 08:55:10 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Tue, 29 Jan 2013 08:55:10 +0100 (CET) Subject: [pypy-commit] pypy quiet-rpython: Added init logging keyword Message-ID: <20130129075510.A1D201C0264@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: quiet-rpython Changeset: r60656:656335af6d22 Date: 2013-01-28 18:04 +0100 http://bitbucket.org/pypy/pypy/changeset/656335af6d22/ Log: Added init logging keyword diff --git a/rpython/tool/ansi_print.py b/rpython/tool/ansi_print.py --- a/rpython/tool/ansi_print.py +++ b/rpython/tool/ansi_print.py @@ -30,12 +30,13 @@ 'Error': ((1, 31), False), 'info': ((35,), False), 'stub': ((34,), False), + 'init': ((1, 34), False), } log_on_quiet = [ "ERROR", "Error", - "info", + "init", ] def __init__(self, kw_to_color={}, file=None): diff --git a/rpython/translator/driver.py b/rpython/translator/driver.py --- a/rpython/translator/driver.py +++ b/rpython/translator/driver.py @@ -268,10 +268,10 @@ def _do(self, goal, func, *args, **kwds): title = func.task_title if goal in self.done: - self.log.info("already done: %s" % title) + self.log.init("already done: %s" % title) return else: - self.log.info("%s..." % title) + self.log.init("%s" % title) debug_start('translation-task') debug_print('starting', goal) self.timer.start_event(goal) @@ -300,7 +300,7 @@ #import gc; gc.dump_rpy_heap('rpyheap-after-%s.dump' % goal) return res - @taskdef([], "Annotating&simplifying") + @taskdef([], "Annotating & simplifying") def task_annotate(self): """ Annotate """ diff --git a/rpython/translator/goal/translate.py b/rpython/translator/goal/translate.py --- a/rpython/translator/goal/translate.py +++ b/rpython/translator/goal/translate.py @@ -92,7 +92,7 @@ py.log.setconsumer("translation", ansi_log) def load_target(targetspec): - log.info("Translating target as defined by %s" % targetspec) + log.init("Translating target %s" % targetspec) if not targetspec.endswith('.py'): targetspec += '.py' thismod = sys.modules[__name__] From noreply at buildbot.pypy.org Tue Jan 29 08:55:11 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Tue, 29 Jan 2013 08:55:11 +0100 (CET) Subject: [pypy-commit] pypy quiet-rpython: Added new log drivers and log.step Message-ID: <20130129075511.C67131C0264@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: quiet-rpython Changeset: r60657:f96e644c7df3 Date: 2013-01-29 08:42 +0100 http://bitbucket.org/pypy/pypy/changeset/f96e644c7df3/ Log: Added new log drivers and log.step diff --git a/rpython/jit/codewriter/codewriter.py b/rpython/jit/codewriter/codewriter.py --- a/rpython/jit/codewriter/codewriter.py +++ b/rpython/jit/codewriter/codewriter.py @@ -65,14 +65,14 @@ self.print_ssa_repr(ssarepr, portal_jd, verbose) def make_jitcodes(self, verbose=False): - log.info("making JitCodes...") + log.info("Producing JitCodes") self.callcontrol.grab_initial_jitcodes() count = 0 for graph, jitcode in self.callcontrol.enum_pending_graphs(): self.transform_graph_to_jitcode(graph, jitcode, verbose) count += 1 - if not count % 500: - log.info("Produced %d jitcodes" % count) + if not count % 10: + log.dot() self.assembler.finished(self.callcontrol.callinfocollection) heaptracker.finish_registering(self.cpu) log.info("there are %d JitCode instances." % count) diff --git a/rpython/rtyper/rtyper.py b/rpython/rtyper/rtyper.py --- a/rpython/rtyper/rtyper.py +++ b/rpython/rtyper/rtyper.py @@ -271,8 +271,7 @@ if self.typererrors: self.dump_typererrors(to_log=True) raise TyperError("there were %d error" % len(self.typererrors)) - self.log.event('-=- specialized %d%s blocks -=-' % ( - blockcount, newtext)) + self.log.dot() annmixlevel = self.annmixlevel del self.annmixlevel if annmixlevel is not None: diff --git a/rpython/tool/ansi_mandelbrot.py b/rpython/tool/ansi_mandelbrot.py --- a/rpython/tool/ansi_mandelbrot.py +++ b/rpython/tool/ansi_mandelbrot.py @@ -118,6 +118,7 @@ def reset(self, cnt=0): """ Resets to the beginning of the line and drops cnt lines internally. """ self.mandelbrot.reset(cnt) + print >>sys.stderr def catchup(self): """ Fills the current line. """ diff --git a/rpython/tool/ansi_print.py b/rpython/tool/ansi_print.py --- a/rpython/tool/ansi_print.py +++ b/rpython/tool/ansi_print.py @@ -17,6 +17,12 @@ self.state += 1 self.state %= len(self.states) +class DotDriver(object): + def dot(self): + sys.stderr.write('.') + def reset(self): + pass + class AnsiLog: wrote_dot = False # XXX sharing state with all instances @@ -30,13 +36,13 @@ 'Error': ((1, 31), False), 'info': ((35,), False), 'stub': ((34,), False), - 'init': ((1, 34), False), + 'step': ((1, 34), False), } log_on_quiet = [ "ERROR", "Error", - "init", + "step", ] def __init__(self, kw_to_color={}, file=None): @@ -54,10 +60,18 @@ self.quiet = quiet self.driver = None - if self.isatty and self.fancy: - self.driver = MandelbrotDriver() - if self.isatty and self.quiet: - self.driver = SpinningDriver() + if self.isatty: + self.driver = DotDriver() + if self.fancy: + self.driver = MandelbrotDriver() + if self.quiet: + self.driver = SpinningDriver() + + def dot(self): + if self.driver is None: + return + self.driver.dot() + AnsiLog.wrote_dot = True def __call__(self, msg): tty = self.isatty() @@ -81,22 +95,17 @@ print >> sys.stderr return elif 'dot' in keywords: - if tty: - if self.driver is not None: - if not AnsiLog.wrote_dot: - self.driver.reset() - self.driver.dot() - else: - ansi_print(".", tuple(esc), file=self.file, newline=False, flush=flush) - AnsiLog.wrote_dot = True - return - if AnsiLog.wrote_dot: - AnsiLog.wrote_dot = False - sys.stderr.write("\n") + self.dot() + return esc = tuple(esc) if not self.quiet or any([kw in self.log_on_quiet for kw in keywords]): + if AnsiLog.wrote_dot and self.driver is not None: + self.driver.reset() + AnsiLog.wrote_dot = False for line in msg.content().splitlines(): ansi_print("[%s] %s" %(":".join(keywords), line), esc, file=self.file, newline=newline, flush=flush) + else: + self.dot() ansi_log = AnsiLog() diff --git a/rpython/translator/driver.py b/rpython/translator/driver.py --- a/rpython/translator/driver.py +++ b/rpython/translator/driver.py @@ -268,12 +268,10 @@ def _do(self, goal, func, *args, **kwds): title = func.task_title if goal in self.done: - self.log.init("already done: %s" % title) + self.log.step("already done: %s" % title) return else: - self.log.init("%s" % title) - debug_start('translation-task') - debug_print('starting', goal) + self.log.step("%s" % title) self.timer.start_event(goal) try: instrument = False @@ -291,7 +289,6 @@ assert False, 'we should not get here' finally: try: - debug_stop('translation-task') self.timer.end_event(goal) except (KeyboardInterrupt, SystemExit): raise @@ -514,7 +511,7 @@ str(newsoname.new(ext='lib'))) self.c_entryp = newexename self.log.info('usession directory: %s' % (udir,)) - self.log.info("created: %s" % (self.c_entryp,)) + self.log.step("created: %s" % (self.c_entryp,)) @taskdef(['source_c'], "Compiling c source") def task_compile_c(self): diff --git a/rpython/translator/goal/translate.py b/rpython/translator/goal/translate.py --- a/rpython/translator/goal/translate.py +++ b/rpython/translator/goal/translate.py @@ -92,7 +92,7 @@ py.log.setconsumer("translation", ansi_log) def load_target(targetspec): - log.init("Translating target %s" % targetspec) + log.step("Translating target %s" % targetspec) if not targetspec.endswith('.py'): targetspec += '.py' thismod = sys.modules[__name__] From noreply at buildbot.pypy.org Tue Jan 29 10:35:57 2013 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 29 Jan 2013 10:35:57 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: Comment Message-ID: <20130129093557.B91DB1C039A@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: jitframe-on-heap Changeset: r60658:6a2167ae591e Date: 2013-01-29 10:35 +0100 http://bitbucket.org/pypy/pypy/changeset/6a2167ae591e/ Log: Comment diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py --- a/rpython/jit/backend/llsupport/llmodel.py +++ b/rpython/jit/backend/llsupport/llmodel.py @@ -60,6 +60,7 @@ assert size <= frame.jf_frame_info.jfi_frame_depth new_frame = jitframe.JITFRAME.allocate(frame.jf_frame_info) # XXX now we know, rewrite this + # YYY it doesn't matter at all, it's fine that way too # we need to do this, because we're not sure what things # are GC pointers and which ones are not llop.gc_writebarrier_before_copy(lltype.Bool, frame, new_frame, From noreply at buildbot.pypy.org Tue Jan 29 10:35:59 2013 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 29 Jan 2013 10:35:59 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: merge heads Message-ID: <20130129093559.480A11C039A@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: jitframe-on-heap Changeset: r60659:8df690cd860f Date: 2013-01-29 10:35 +0100 http://bitbucket.org/pypy/pypy/changeset/8df690cd860f/ Log: merge heads diff --git a/pypy/module/_continuation/interp_continuation.py b/pypy/module/_continuation/interp_continuation.py --- a/pypy/module/_continuation/interp_continuation.py +++ b/pypy/module/_continuation/interp_continuation.py @@ -34,7 +34,7 @@ if self.sthread is not None: raise geterror(self.space, "continulet already __init__ialized") sthread = build_sthread(self.space) - workaround_disable_jit(sthread) + #workaround_disable_jit(sthread) # # hackish: build the frame "by hand", passing it the correct arguments space = self.space @@ -77,7 +77,7 @@ global_state.clear() raise geterror(self.space, "continulet already finished") self.check_sthread() - workaround_disable_jit(self.sthread) + #workaround_disable_jit(self.sthread) # global_state.origin = self if to is None: diff --git a/pypy/module/pypyjit/policy.py b/pypy/module/pypyjit/policy.py --- a/pypy/module/pypyjit/policy.py +++ b/pypy/module/pypyjit/policy.py @@ -106,7 +106,7 @@ 'posix', '_socket', '_sre', '_lsprof', '_weakref', '__pypy__', 'cStringIO', '_collections', 'struct', 'mmap', 'marshal', '_codecs', 'rctime', 'cppyy', - '_cffi_backend', 'pyexpat']: + '_cffi_backend', 'pyexpat', '_continuation']: if modname == 'pypyjit' and 'interp_resop' in rest: return False return True diff --git a/rpython/jit/backend/arm/arch.py b/rpython/jit/backend/arm/arch.py --- a/rpython/jit/backend/arm/arch.py +++ b/rpython/jit/backend/arm/arch.py @@ -1,7 +1,3 @@ -from rpython.rtyper.lltypesystem import lltype, rffi -from rpython.rlib.rarithmetic import r_uint - - FUNC_ALIGN = 8 WORD = 4 DOUBLE_WORD = 8 @@ -14,54 +10,13 @@ PC_OFFSET = 8 FORCE_INDEX_OFS = 0 -from rpython.translator.tool.cbuild import ExternalCompilationInfo -eci = ExternalCompilationInfo(post_include_bits=[""" -static int pypy__arm_int_div(int a, int b) { - return a/b; -} -static unsigned int pypy__arm_uint_div(unsigned int a, unsigned int b) { - return a/b; -} -static int pypy__arm_int_mod(int a, int b) { - return a % b; -} -"""]) +# The stack contains the force_index and the, callee saved registers and +# ABI required information +# All the rest of the data is in a GC-managed variable-size "frame". +# This jitframe object's address is always stored in the register FP +# A jitframe is a jit.backend.llsupport.llmodel.JITFRAME = GcArray(Signed). +# Stack frame fixed area +# Currently only the force_index +FRAME_FIXED_SIZE = 1 +JITFRAME_FIXED_SIZE = 16 + 16 * 2 # 16 GPR + 16 VFP Regs (64bit) - -def arm_int_div_emulator(a, b): - return int(a / float(b)) -arm_int_div_sign = lltype.Ptr( - lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed)) -arm_int_div = rffi.llexternal( - "pypy__arm_int_div", [lltype.Signed, lltype.Signed], lltype.Signed, - _callable=arm_int_div_emulator, - compilation_info=eci, - _nowrapper=True, elidable_function=True) - - -def arm_uint_div_emulator(a, b): - return r_uint(a) / r_uint(b) -arm_uint_div_sign = lltype.Ptr( - lltype.FuncType([lltype.Unsigned, lltype.Unsigned], lltype.Unsigned)) -arm_uint_div = rffi.llexternal( - "pypy__arm_uint_div", [lltype.Unsigned, lltype.Unsigned], lltype.Unsigned, - _callable=arm_uint_div_emulator, - compilation_info=eci, - _nowrapper=True, elidable_function=True) - - -def arm_int_mod_emulator(a, b): - sign = 1 - if a < 0: - a = -1 * a - sign = -1 - if b < 0: - b = -1 * b - res = a % b - return sign * res -arm_int_mod_sign = arm_int_div_sign -arm_int_mod = rffi.llexternal( - "pypy__arm_int_mod", [lltype.Signed, lltype.Signed], lltype.Signed, - _callable=arm_int_mod_emulator, - compilation_info=eci, - _nowrapper=True, elidable_function=True) diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py --- a/rpython/jit/backend/arm/assembler.py +++ b/rpython/jit/backend/arm/assembler.py @@ -5,7 +5,8 @@ from rpython.jit.backend.arm import conditions as c from rpython.jit.backend.arm import registers as r from rpython.jit.backend.arm.arch import WORD, DOUBLE_WORD, FUNC_ALIGN, \ - N_REGISTERS_SAVED_BY_MALLOC + N_REGISTERS_SAVED_BY_MALLOC, \ + JITFRAME_FIXED_SIZE, FRAME_FIXED_SIZE from rpython.jit.backend.arm.codebuilder import ARMv7Builder, OverwritingBuilder from rpython.jit.backend.arm.locations import get_fp_offset from rpython.jit.backend.arm.regalloc import (Regalloc, ARMFrameManager, @@ -21,7 +22,7 @@ from rpython.jit.metainterp.resoperation import rop, ResOperation from rpython.rlib import rgc from rpython.rlib.objectmodel import we_are_translated, specialize -from rpython.rtyper.annlowlevel import llhelper +from rpython.rtyper.annlowlevel import llhelper, cast_instance_to_gcref from rpython.rtyper.lltypesystem import lltype, rffi, llmemory from rpython.rtyper.lltypesystem.lloperation import llop from rpython.jit.backend.arm.opassembler import ResOpAssembler @@ -29,9 +30,9 @@ have_debug_prints, fatalerror) from rpython.rlib.jit import AsmInfo from rpython.rlib.objectmodel import compute_unique_id +from rpython.rlib.rarithmetic import intmask, r_uint -# XXX Move to llsupport -from rpython.jit.backend.x86.support import memcpy_fn +from rpython.jit.backend.arm.support import memcpy_fn DEBUG_COUNTER = lltype.Struct('DEBUG_COUNTER', ('i', lltype.Signed), ('type', lltype.Char), # 'b'ridge, 'l'abel or @@ -41,8 +42,6 @@ class AssemblerARM(ResOpAssembler): - STACK_FIXED_AREA = -1 - debug = True def __init__(self, cpu, translate_support_code=False): @@ -59,33 +58,19 @@ self.datablockwrapper = None self.propagate_exception_path = 0 self.stack_check_slowpath = 0 - self._compute_stack_size() self._debug = False self.loop_run_counters = [] self.debug_counter_descr = cpu.fielddescrof(DEBUG_COUNTER, 'i') - self.force_token_to_dead_frame = {} # XXX temporary hack + self.gcrootmap_retaddr_forced = 0 def set_debug(self, v): r = self._debug self._debug = v return r - def _compute_stack_size(self): - self.STACK_FIXED_AREA = len(r.callee_saved_registers) * WORD - self.STACK_FIXED_AREA += WORD # FORCE_TOKEN - self.STACK_FIXED_AREA += N_REGISTERS_SAVED_BY_MALLOC * WORD - if self.cpu.supports_floats: - self.STACK_FIXED_AREA += (len(r.callee_saved_vfp_registers) - * DOUBLE_WORD) - if self.STACK_FIXED_AREA % 8 != 0: - self.STACK_FIXED_AREA += WORD # Stack alignment - assert self.STACK_FIXED_AREA % 8 == 0 - - def setup(self, looptoken, operations): + def setup(self, looptoken): + assert self.memcpy_addr != 0, 'setup_once() not called?' self.current_clt = looptoken.compiled_loop_token - operations = self.cpu.gc_ll_descr.rewrite_assembler(self.cpu, - operations, self.current_clt.allgcrefs) - assert self.memcpy_addr != 0, 'setup_once() not called?' self.mc = ARMv7Builder() self.pending_guards = [] assert self.datablockwrapper is None @@ -93,7 +78,6 @@ self.datablockwrapper = MachineDataBlockWrapper(self.cpu.asmmemmgr, allblocks) self.target_tokens_currently_compiling = {} - return operations def teardown(self): self.current_clt = None @@ -106,10 +90,11 @@ # Addresses of functions called by new_xxx operations gc_ll_descr = self.cpu.gc_ll_descr gc_ll_descr.initialize() + self.memcpy_addr = self.cpu.cast_ptr_to_int(memcpy_fn) + self._build_failure_recovery(exc=True, withfloats=False) + self._build_failure_recovery(exc=False, withfloats=False) self._build_wb_slowpath(False) self._build_wb_slowpath(True) - self._build_failure_recovery(exc=True, withfloats=False) - self._build_failure_recovery(exc=False, withfloats=False) if self.cpu.supports_floats: self._build_wb_slowpath(False, withfloats=True) self._build_wb_slowpath(True, withfloats=True) @@ -121,7 +106,6 @@ self._build_stack_check_slowpath() if gc_ll_descr.gcrootmap and gc_ll_descr.gcrootmap.is_shadow_stack: self._build_release_gil(gc_ll_descr.gcrootmap) - self.memcpy_addr = self.cpu.cast_ptr_to_int(memcpy_fn) if not self._debug: # if self._debug is already set it means that someone called @@ -130,6 +114,9 @@ debug_start('jit-backend-counts') self.set_debug(have_debug_prints()) debug_stop('jit-backend-counts') + # when finishing, we only have one value at [0], the rest dies + self.gcmap_for_finish = lltype.malloc(jitframe.GCMAP, 1, zero=True) + self.gcmap_for_finish[0] = r_uint(1) def finish_once(self): if self._debug: @@ -218,18 +205,50 @@ self.reacqgil_addr = rffi.cast(lltype.Signed, reacqgil_func) def _build_propagate_exception_path(self): - if self.cpu.propagate_exception_v < 0: + if not self.cpu.propagate_exception_descr: return # not supported (for tests, or non-translated) # mc = ARMv7Builder() # - # Call the helper, which will return a dead frame object with - # the correct exception set, or MemoryError by default - # XXX make sure we return the correct value here + # read and reset the current exception addr = rffi.cast(lltype.Signed, self.cpu.get_propagate_exception()) mc.BL(addr) self.gen_func_epilog(mc=mc) self.propagate_exception_path = mc.materialize(self.cpu.asmmemmgr, []) + # + self._store_and_reset_exception(r.r0) + ofs = self.cpu.get_ofs_of_frame_field('jf_guard_exc') + # make sure ofs fits into a register + assert check_imm_arg(ofs) + self.mc.STR_ri(r.r0.value, r.fp.value, imm=ofs) + propagate_exception_descr = rffi.cast(lltype.Signed, + cast_instance_to_gcref(self.cpu.propagate_exception_descr)) + ofs = self.cpu.get_ofs_of_frame_field('jf_descr') + # make sure ofs fits into a register + assert check_imm_arg(ofs) + self.mc.BKPT() + #base_ofs = self.cpu.get_baseofs_of_frame_field() + #self.mc.MOV_bi(ofs, propagate_exception_descr) + #self.mc.LEA_rb(eax.value, -base_ofs) + # + self._call_footer() + rawstart = self.mc.materialize(self.cpu.asmmemmgr, []) + self.propagate_exception_path = rawstart + self.mc = None + + def _store_and_reset_exception(self, resloc=None): + assert resloc is not r.ip + if resloc is not None: + self.mc.gen_load_int(resloc.value, self.cpu.pos_exc_value()) + self.mc.LDR_ri(resloc.value, resloc.value) + self.mc.MOV(resloc, heap(self.cpu.pos_exc_value())) + + with saved_registers(self.mc, [r.r0]): + self.mc.gen_load_int(r.r0.value, self.cpu.pos_exc_value()) + self.mc.gen_load_int(r.ip.value, 0) + self.mc.STR_ri(r.ip.value, r.r0.value) + self.mc.gen_load_int(r.r0.value, self.cpu.pos_exception()) + self.mc.STR_ri(r.ip.value, r.r0.value) def _build_stack_check_slowpath(self): _, _, slowpathaddr = self.cpu.insert_stack_check() @@ -558,7 +577,7 @@ # We might have an exception pending. Load it into r4 # (this is a register saved across calls) mc.gen_load_int(r.r5.value, self.cpu.pos_exc_value()) - mc.LDR_ri(r.r4.value, self.cpu.pos_exc_value()) + mc.LDR_ri(r.r4.value, r.r5.value) # clear the exc flags mc.gen_load_int(r.r6.value, 0) mc.STR_ri(r.r6.value, r.r5.value) @@ -661,37 +680,35 @@ self.mc.writechar(chr(0)) def gen_func_epilog(self, mc=None, cond=c.AL): - stack_size = self.STACK_FIXED_AREA - stack_size -= len(r.callee_saved_registers) * WORD - if self.cpu.supports_floats: - stack_size -= len(r.callee_saved_vfp_registers) * 2 * WORD - gcrootmap = self.cpu.gc_ll_descr.gcrootmap if mc is None: mc = self.mc if gcrootmap and gcrootmap.is_shadow_stack: self.gen_footer_shadowstack(gcrootmap, mc) - mc.MOV_rr(r.sp.value, r.fp.value, cond=cond) - mc.ADD_ri(r.sp.value, r.sp.value, stack_size, cond=cond) + mc.ADD_ri(r.sp.value, r.sp.value, WORD, cond=cond) # for the force index if self.cpu.supports_floats: mc.VPOP([reg.value for reg in r.callee_saved_vfp_registers], cond=cond) mc.POP([reg.value for reg in r.callee_restored_registers], cond=cond) + mc.BKPT() def gen_func_prolog(self): - stack_size = self.STACK_FIXED_AREA - stack_size -= len(r.callee_saved_registers) * WORD + stack_size = FRAME_FIXED_SIZE * WORD + stack_size += len(r.callee_saved_registers) * WORD if self.cpu.supports_floats: - stack_size -= len(r.callee_saved_vfp_registers) * 2 * WORD + stack_size += len(r.callee_saved_vfp_registers) * 2 * WORD self.mc.PUSH([reg.value for reg in r.callee_saved_registers]) if self.cpu.supports_floats: self.mc.VPUSH([reg.value for reg in r.callee_saved_vfp_registers]) - # here we modify the stack pointer to leave room for the 9 registers - # that are going to be saved here around malloc calls and one word to - # store the force index - self.mc.SUB_ri(r.sp.value, r.sp.value, stack_size) - self.mc.MOV_rr(r.fp.value, r.sp.value) + self.mc.SUB_ri(r.sp.value, r.sp.value, WORD) # for the force index + assert stack_size % 8 == 0 # ensure we keep alignment + + # set fp to point to the JITFRAME + ofs + ofs = self.cpu.get_baseofs_of_frame_field() + assert check_imm_arg(ofs) + self.mc.ADD_ri(r.fp.value, r.r0.value, imm=ofs) + # gcrootmap = self.cpu.gc_ll_descr.gcrootmap if gcrootmap and gcrootmap.is_shadow_stack: self.gen_shadowstack_header(gcrootmap) @@ -754,7 +771,9 @@ # cpu interface def assemble_loop(self, loopname, inputargs, operations, looptoken, log): clt = CompiledLoopToken(self.cpu, looptoken.number) + clt.frame_info = lltype.malloc(jitframe.JITFRAMEINFO) clt.allgcrefs = [] + clt.frame_info.jfi_frame_depth = 0 # for now looptoken.compiled_loop_token = clt clt._debug_nbargs = len(inputargs) @@ -762,38 +781,40 @@ # Arguments should be unique assert len(set(inputargs)) == len(inputargs) - operations = self.setup(looptoken, operations) - if log: + self.setup(looptoken) + if False and log: operations = self._inject_debugging_code(looptoken, operations, 'e', looptoken.number) self._call_header_with_stack_check() - sp_patch_location = self._prepare_sp_patch_position() + #sp_patch_location = self._prepare_sp_patch_position() - regalloc = Regalloc(assembler=self, frame_manager=ARMFrameManager()) - regalloc.prepare_loop(inputargs, operations) + regalloc = Regalloc(assembler=self) + operations = regalloc.prepare_loop(inputargs, operations, looptoken, + clt.allgcrefs) + rgc._make_sure_does_not_move(lltype.cast_opaque_ptr(llmemory.GCREF, + clt.frame_info)) loop_head = self.mc.get_relative_pos() looptoken._arm_loop_code = loop_head # - clt.frame_depth = -1 - frame_depth = self._assemble(operations, regalloc) - clt.frame_depth = frame_depth + frame_depth = self._assemble(regalloc, inputargs, operations) + self.update_frame_depth(frame_depth + JITFRAME_FIXED_SIZE) # size_excluding_failure_stuff = self.mc.get_relative_pos() - self._patch_sp_offset(sp_patch_location, frame_depth) + #self._patch_sp_offset(sp_patch_location, frame_depth) self.write_pending_failure_recoveries() rawstart = self.materialize_loop(looptoken) - looptoken._arm_func_addr = rawstart + looptoken._function_addr = looptoken._arm_func_addr = rawstart self.process_pending_guards(rawstart) self.fixup_target_tokens(rawstart) if log and not we_are_translated(): self.mc._dump_trace(rawstart, - 'loop_%s.asm' % self.cpu.total_compiled_loops) + 'loop.asm') ops_offset = self.mc.ops_offset self.teardown() @@ -809,18 +830,20 @@ return AsmInfo(ops_offset, rawstart + loop_head, size_excluding_failure_stuff - loop_head) - def _assemble(self, operations, regalloc): + def _assemble(self, regalloc, inputargs, operations): regalloc.compute_hint_frame_locations(operations) - self._walk_operations(operations, regalloc) - frame_depth = regalloc.frame_manager.get_frame_depth() + self._walk_operations(inputargs, operations, regalloc) + frame_depth = regalloc.get_final_frame_depth() jump_target_descr = regalloc.jump_target_descr if jump_target_descr is not None: - frame_depth = max(frame_depth, - jump_target_descr._arm_clt.frame_depth) + tgt_depth = jump_target_descr._arm_clt.frame_info.jfi_frame_depth + target_frame_depth = tgt_depth - JITFRAME_FIXED_SIZE + frame_depth = max(frame_depth, target_frame_depth) return frame_depth def assemble_bridge(self, faildescr, inputargs, operations, original_loop_token, log): + assert 0 operations = self.setup(original_loop_token, operations) descr_number = self.cpu.get_fail_descr_number(faildescr) if log: @@ -899,6 +922,17 @@ return self.mc.materialize(self.cpu.asmmemmgr, allblocks, self.cpu.gc_ll_descr.gcrootmap) + def update_frame_depth(self, frame_depth): + self.current_clt.frame_info.jfi_frame_depth = frame_depth + new_jumping_to = [] + for wref in self.current_clt.jumping_to: + clt = wref() + if clt is not None: + clt.frame_info.jfi_frame_depth = max(frame_depth, + clt.frame_info.jfi_frame_depth) + new_jumping_to.append(weakref.ref(clt)) + self.current_clt.jumping_to = new_jumping_to + def write_pending_failure_recoveries(self): for tok in self.pending_guards: #generate the exit stub and the encoded representation @@ -972,7 +1006,7 @@ else: cb.SUB_rr(r.sp.value, base_reg.value, r.ip.value, cond=fcond) - def _walk_operations(self, operations, regalloc): + def _walk_operations(self, inputargs, operations, regalloc): fcond = c.AL self._regalloc = regalloc while regalloc.position() < len(operations) - 1: @@ -1141,10 +1175,10 @@ if not check_imm_arg(offset, size=0xFFF): self.mc.PUSH([r.lr.value], cond=cond) pushed = True - self.mc.gen_load_int(r.lr.value, -offset, cond=cond) + self.mc.gen_load_int(r.lr.value, offset, cond=cond) self.mc.LDR_rr(loc.value, r.fp.value, r.lr.value, cond=cond) else: - self.mc.LDR_ri(loc.value, r.fp.value, imm=-offset, cond=cond) + self.mc.LDR_ri(loc.value, r.fp.value, imm=offset, cond=cond) if pushed: self.mc.POP([r.lr.value], cond=cond) elif loc.is_vfp_reg(): @@ -1364,6 +1398,26 @@ else: return 0 + def push_gcmap(self, mc, gcmap, push=False, mov=False, store=False): + gcmapref = lltype.cast_opaque_ptr(llmemory.GCREF, gcmap) + # keep the ref alive + self.current_clt.allgcrefs.append(gcmapref) + rgc._make_sure_does_not_move(gcmapref) + pass + #if push: + # mc.PUSH(imm(rffi.cast(lltype.Signed, gcmapref))) + #elif mov: + # mc.MOV(RawEspLoc(0, REF), + # imm(rffi.cast(lltype.Signed, gcmapref))) + #else: + # assert store + # ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') + # mc.MOV(raw_stack(ofs), imm(rffi.cast(lltype.Signed, gcmapref))) + + def pop_gcmap(self, mc): + ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') + mc.MOV_bi(ofs, 0) + def not_implemented(msg): os.write(2, '[ARM/asm] %s\n' % msg) diff --git a/rpython/jit/backend/arm/codebuilder.py b/rpython/jit/backend/arm/codebuilder.py --- a/rpython/jit/backend/arm/codebuilder.py +++ b/rpython/jit/backend/arm/codebuilder.py @@ -1,6 +1,6 @@ -from rpython.jit.backend.arm import arch from rpython.jit.backend.arm import conditions as cond from rpython.jit.backend.arm import registers as reg +from rpython.jit.backend.arm import support from rpython.jit.backend.arm.arch import (WORD, FUNC_ALIGN) from rpython.jit.backend.arm.instruction_builder import define_instructions from rpython.jit.backend.llsupport.asmmemmgr import BlockBuilderMixin @@ -17,7 +17,7 @@ def binary_helper_call(name): - function = getattr(arch, 'arm_%s' % name) + function = getattr(support, 'arm_%s' % name) def f(self, c=cond.AL): """Generates a call to a helper function, takes its diff --git a/rpython/jit/backend/arm/locations.py b/rpython/jit/backend/arm/locations.py --- a/rpython/jit/backend/arm/locations.py +++ b/rpython/jit/backend/arm/locations.py @@ -1,5 +1,5 @@ from rpython.jit.metainterp.history import INT, FLOAT -from rpython.jit.backend.arm.arch import WORD, DOUBLE_WORD +from rpython.jit.backend.arm.arch import WORD, DOUBLE_WORD, JITFRAME_FIXED_SIZE class AssemblerLocation(object): @@ -136,9 +136,5 @@ return ImmLocation(i) -def get_fp_offset(i): - if i >= 0: - # Take the FORCE_TOKEN into account - return (1 + i) * WORD - else: - return i * WORD +def get_fp_offset(position): + return WORD * (position + JITFRAME_FIXED_SIZE) diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -317,9 +317,27 @@ return fcond def emit_op_finish(self, op, arglocs, regalloc, fcond): - [argloc] = arglocs - if argloc is not r.r0: #XXX verify this - self.mov_loc_loc(argloc, r.r0, fcond) + base_ofs = self.cpu.get_baseofs_of_frame_field() - WORD + if len(arglocs) == 2: + [return_val, fail_descr_loc] = arglocs + if op.getarg(0).type == FLOAT and not IS_X86_64: + XXX + size = WORD * 2 + else: + size = WORD + self.mc.STR_ri(return_val.value, r.fp.value)#, imm=-base_ofs) + #self.save_into_mem(raw_stack(0), return_val, imm(size)) + else: + [fail_descr_loc] = arglocs + ofs = self.cpu.get_ofs_of_frame_field('jf_descr') + base_ofs = self.cpu.get_baseofs_of_frame_field() + + self.mc.gen_load_int(r.ip.value, fail_descr_loc.value) + # XXX self.mov(fail_descr_loc, RawStackLoc(ofs)) + self.mc.STR_ri(r.ip.value, r.fp.value, imm=ofs) + gcmap = self.gcmap_for_finish + self.push_gcmap(self.mc, gcmap, store=True) + self.mc.SUB_ri(r.r0.value, r.fp.value, base_ofs) # exit function self.gen_func_epilog() return fcond diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py --- a/rpython/jit/backend/arm/regalloc.py +++ b/rpython/jit/backend/arm/regalloc.py @@ -1,3 +1,5 @@ +from rpython.rtyper.annlowlevel import llhelper, cast_instance_to_gcref +from rpython.rlib import rgc from rpython.jit.backend.llsupport.regalloc import FrameManager, \ RegisterManager, TempBox, compute_vars_longevity from rpython.jit.backend.arm import registers as r @@ -180,10 +182,10 @@ class Regalloc(object): - def __init__(self, frame_manager=None, assembler=None): + def __init__(self, assembler=None): self.cpu = assembler.cpu self.assembler = assembler - self.frame_manager = frame_manager + self.frame_manager = None self.jump_target_descr = None self.final_jump_op = None @@ -282,7 +284,12 @@ assert isinstance(value, ConstFloat) return self.vfprm.convert_to_imm(value) - def _prepare(self, inputargs, operations): + def _prepare(self, inputargs, operations, allgcrefs): + self.frame_manager = self.fm = ARMFrameManager() + cpu = self.assembler.cpu + operations = cpu.gc_ll_descr.rewrite_assembler(cpu, operations, + allgcrefs) + # compute longevity of variables longevity, last_real_usage = compute_vars_longevity( inputargs, operations) self.longevity = longevity @@ -291,92 +298,27 @@ asm = self.assembler self.vfprm = VFPRegisterManager(longevity, fm, asm) self.rm = CoreRegisterManager(longevity, fm, asm) + return operations - def prepare_loop(self, inputargs, operations): - self._prepare(inputargs, operations) + def prepare_loop(self, inputargs, operations, looptoken, allgcrefs): + operations = self._prepare(inputargs, operations, allgcrefs) self._set_initial_bindings(inputargs) - self.possibly_free_vars(inputargs) + self.possibly_free_vars(list(inputargs)) + return operations def prepare_bridge(self, inputargs, arglocs, ops): self._prepare(inputargs, ops) self._update_bindings(arglocs, inputargs) + def get_final_frame_depth(self): + return self.frame_manager.get_frame_depth() + def _set_initial_bindings(self, inputargs): - # The first inputargs are passed in registers r0-r3 - # we relly on the soft-float calling convention so we need to move - # float params to the coprocessor. - if self.cpu.use_hf_abi: - self._set_initial_bindings_hf(inputargs) - else: - self._set_initial_bindings_sf(inputargs) - - def _set_initial_bindings_sf(self, inputargs): - - arg_index = 0 - count = 0 - n_register_args = len(r.argument_regs) - cur_frame_pos = 1 - (self.assembler.STACK_FIXED_AREA // WORD) + # the input args are passed in the jitframe for box in inputargs: assert isinstance(box, Box) - # handle inputargs in argument registers - if box.type == FLOAT and arg_index % 2 != 0: - arg_index += 1 # align argument index for float passed - # in register - if arg_index < n_register_args: - if box.type == FLOAT: - loc = r.argument_regs[arg_index] - loc2 = r.argument_regs[arg_index + 1] - vfpreg = self.try_allocate_reg(box) - # move soft-float argument to vfp - self.assembler.mov_to_vfp_loc(loc, loc2, vfpreg) - arg_index += 2 # this argument used two argument registers - else: - loc = r.argument_regs[arg_index] - self.try_allocate_reg(box, selected_reg=loc) - arg_index += 1 - else: - # treat stack args as stack locations with a negative offset - if box.type == FLOAT: - cur_frame_pos -= 2 - if count % 2 != 0: # Stack argument alignment - cur_frame_pos -= 1 - count = 0 - else: - cur_frame_pos -= 1 - count += 1 - loc = self.frame_manager.frame_pos(cur_frame_pos, box.type) - self.frame_manager.set_binding(box, loc) - - def _set_initial_bindings_hf(self, inputargs): - - arg_index = vfp_arg_index = 0 - count = 0 - n_reg_args = len(r.argument_regs) - n_vfp_reg_args = len(r.vfp_argument_regs) - cur_frame_pos = 1 - (self.assembler.STACK_FIXED_AREA // WORD) - for box in inputargs: - assert isinstance(box, Box) - # handle inputargs in argument registers - if box.type != FLOAT and arg_index < n_reg_args: - reg = r.argument_regs[arg_index] - self.try_allocate_reg(box, selected_reg=reg) - arg_index += 1 - elif box.type == FLOAT and vfp_arg_index < n_vfp_reg_args: - reg = r.vfp_argument_regs[vfp_arg_index] - self.try_allocate_reg(box, selected_reg=reg) - vfp_arg_index += 1 - else: - # treat stack args as stack locations with a negative offset - if box.type == FLOAT: - cur_frame_pos -= 2 - if count % 2 != 0: # Stack argument alignment - cur_frame_pos -= 1 - count = 0 - else: - cur_frame_pos -= 1 - count += 1 - loc = self.frame_manager.frame_pos(cur_frame_pos, box.type) - self.frame_manager.set_binding(box, loc) + assert box.type != FLOAT + self.fm.get_new_loc(box) def _update_bindings(self, locs, inputargs): used = {} @@ -644,9 +586,19 @@ return args def prepare_op_finish(self, op, fcond): - loc = self.loc(op.getarg(0)) - self.possibly_free_var(op.getarg(0)) - return [loc] + # the frame is in fp, but we have to point where in the frame is + # the potential argument to FINISH + descr = op.getdescr() + fail_descr = cast_instance_to_gcref(descr) + # we know it does not move, but well + rgc._make_sure_does_not_move(fail_descr) + fail_descr = rffi.cast(lltype.Signed, fail_descr) + if op.numargs() == 1: + loc = self.make_sure_var_in_reg(op.getarg(0)) + locs = [loc, imm(fail_descr)] + else: + locs = [imm(fail_descr)] + return locs def prepare_op_guard_true(self, op, fcond): l0 = self.make_sure_var_in_reg(op.getarg(0)) diff --git a/rpython/jit/backend/arm/runner.py b/rpython/jit/backend/arm/runner.py --- a/rpython/jit/backend/arm/runner.py +++ b/rpython/jit/backend/arm/runner.py @@ -1,12 +1,18 @@ +from rpython.jit.backend.arm.arch import JITFRAME_FIXED_SIZE from rpython.jit.backend.arm.assembler import AssemblerARM from rpython.jit.backend.arm.registers import all_regs, all_vfp_regs +from rpython.jit.backend.llsupport import jitframe +from rpython.jit.backend.llsupport.symbolic import WORD from rpython.jit.backend.llsupport.llmodel import AbstractLLCPU +from rpython.jit.metainterp import history +from rpython.rlib.jit_hooks import LOOP_RUN_CONTAINER +from rpython.rlib.unroll import unrolling_iterable from rpython.rtyper.llinterp import LLInterpreter from rpython.rtyper.lltypesystem import lltype, rffi, llmemory -from rpython.rlib.jit_hooks import LOOP_RUN_CONTAINER -from rpython.jit.backend.arm.arch import FORCE_INDEX_OFS +jitframe.STATICSIZE = JITFRAME_FIXED_SIZE + class AbstractARMCPU(AbstractLLCPU): supports_floats = True @@ -18,14 +24,9 @@ def __init__(self, rtyper, stats, opts=None, translate_support_code=False, gcdescr=None): - if gcdescr is not None: - gcdescr.force_index_ofs = FORCE_INDEX_OFS AbstractLLCPU.__init__(self, rtyper, stats, opts, translate_support_code, gcdescr) - from rpython.jit.backend.llsupport import jitframe - self.deadframe_size_max = llmemory.sizeof(jitframe.DEADFRAME, - self.get_failargs_limit()) def set_debug(self, flag): return self.assembler.set_debug(flag) @@ -64,7 +65,11 @@ setitem(index, null) def make_execute_token(self, *ARGS): - FUNCPTR = lltype.Ptr(lltype.FuncType(ARGS, llmemory.GCREF)) + FUNCPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF], + llmemory.GCREF)) + + lst = [(i, history.getkind(ARG)[0]) for i, ARG in enumerate(ARGS)] + kinds = unrolling_iterable(lst) def execute_token(executable_token, *args): clt = executable_token.compiled_loop_token @@ -74,18 +79,32 @@ assert addr % 8 == 0 func = rffi.cast(FUNCPTR, addr) #llop.debug_print(lltype.Void, ">>>> Entering", addr) + frame_info = clt.frame_info + frame = self.gc_ll_descr.malloc_jitframe(frame_info) + ll_frame = lltype.cast_opaque_ptr(llmemory.GCREF, frame) prev_interpreter = None # help flow space if not self.translate_support_code: prev_interpreter = LLInterpreter.current_interpreter LLInterpreter.current_interpreter = self.debug_ll_interpreter try: - deadframe = func(*args) + num = JITFRAME_FIXED_SIZE * WORD + for i, kind in kinds: + arg = args[i] + if kind == history.INT: + self.set_int_value(ll_frame, num, arg) + elif kind == history.FLOAT: + self.set_float_value(ll_frame, num, arg) + num += WORD # on ARM(32 bit) a FLOAT needs two words + else: + assert kind == history.REF + self.set_ref_value(ll_frame, num, arg) + num += WORD + ll_frame = func(ll_frame) finally: if not self.translate_support_code: LLInterpreter.current_interpreter = prev_interpreter #llop.debug_print(lltype.Void, "<<<< Back") - self.gc_set_extra_threshold() - return deadframe + return ll_frame return execute_token def cast_ptr_to_int(x): diff --git a/rpython/jit/backend/arm/support.py b/rpython/jit/backend/arm/support.py new file mode 100644 --- /dev/null +++ b/rpython/jit/backend/arm/support.py @@ -0,0 +1,61 @@ +from rpython.rtyper.lltypesystem import lltype, rffi, llmemory +from rpython.rlib.rarithmetic import r_uint +from rpython.translator.tool.cbuild import ExternalCompilationInfo + +eci = ExternalCompilationInfo(post_include_bits=[""" +static int pypy__arm_int_div(int a, int b) { + return a/b; +} +static unsigned int pypy__arm_uint_div(unsigned int a, unsigned int b) { + return a/b; +} +static int pypy__arm_int_mod(int a, int b) { + return a % b; +} +"""]) + + +def arm_int_div_emulator(a, b): + return int(a / float(b)) +arm_int_div_sign = lltype.Ptr( + lltype.FuncType([lltype.Signed, lltype.Signed], lltype.Signed)) +arm_int_div = rffi.llexternal( + "pypy__arm_int_div", [lltype.Signed, lltype.Signed], lltype.Signed, + _callable=arm_int_div_emulator, + compilation_info=eci, + _nowrapper=True, elidable_function=True) + + +def arm_uint_div_emulator(a, b): + return r_uint(a) / r_uint(b) +arm_uint_div_sign = lltype.Ptr( + lltype.FuncType([lltype.Unsigned, lltype.Unsigned], lltype.Unsigned)) +arm_uint_div = rffi.llexternal( + "pypy__arm_uint_div", [lltype.Unsigned, lltype.Unsigned], lltype.Unsigned, + _callable=arm_uint_div_emulator, + compilation_info=eci, + _nowrapper=True, elidable_function=True) + + +def arm_int_mod_emulator(a, b): + sign = 1 + if a < 0: + a = -1 * a + sign = -1 + if b < 0: + b = -1 * b + res = a % b + return sign * res +arm_int_mod_sign = arm_int_div_sign +arm_int_mod = rffi.llexternal( + "pypy__arm_int_mod", [lltype.Signed, lltype.Signed], lltype.Signed, + _callable=arm_int_mod_emulator, + compilation_info=eci, + _nowrapper=True, elidable_function=True) +# ____________________________________________________________ + +memcpy_fn = rffi.llexternal('memcpy', [llmemory.Address, llmemory.Address, + rffi.SIZE_T], lltype.Void, + sandboxsafe=True, _nowrapper=True) + +# ____________________________________________________________ diff --git a/rpython/jit/backend/arm/test/test_assembler.py b/rpython/jit/backend/arm/test/test_assembler.py --- a/rpython/jit/backend/arm/test/test_assembler.py +++ b/rpython/jit/backend/arm/test/test_assembler.py @@ -1,6 +1,6 @@ from rpython.jit.backend.arm import conditions as c from rpython.jit.backend.arm import registers as r -from rpython.jit.backend.arm.arch import arm_int_div +from rpython.jit.backend.arm.support import arm_int_div from rpython.jit.backend.arm.assembler import AssemblerARM from rpython.jit.backend.arm.locations import imm from rpython.jit.backend.arm.test.support import run_asm diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py --- a/rpython/jit/backend/llsupport/llmodel.py +++ b/rpython/jit/backend/llsupport/llmodel.py @@ -336,6 +336,40 @@ # ____________________________________________________________ + def set_int_value(self, newframe, index, value): + """ Note that we keep index multiplied by WORD here mostly + for completeness with get_int_value and friends + """ + descr = self.gc_ll_descr.getframedescrs(self).arraydescr + ofs = self.unpack_arraydescr(descr) + self.write_int_at_mem(newframe, ofs + index, WORD, 1, value) + + def set_ref_value(self, newframe, index, value): + descr = self.gc_ll_descr.getframedescrs(self).arraydescr + ofs = self.unpack_arraydescr(descr) + self.write_ref_at_mem(newframe, ofs + index, value) + + def set_float_value(self, newframe, index, value): + descr = self.gc_ll_descr.getframedescrs(self).arraydescr + ofs = self.unpack_arraydescr(descr) + self.write_float_at_mem(newframe, ofs + index, value) + + @specialize.arg(1) + def get_ofs_of_frame_field(self, name): + descrs = self.gc_ll_descr.getframedescrs(self) + if name.startswith('jfi_'): + base_ofs = 0 # not relative to frame + else: + base_ofs = self.unpack_arraydescr(descrs.arraydescr) + ofs = self.unpack_fielddescr(getattr(descrs, name)) + return ofs - base_ofs + + def get_baseofs_of_frame_field(self): + descrs = self.gc_ll_descr.getframedescrs(self) + base_ofs = self.unpack_arraydescr(descrs.arraydescr) + return base_ofs + # ____________________________________________________________ + def bh_arraylen_gc(self, array, arraydescr): assert isinstance(arraydescr, ArrayDescr) diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py --- a/rpython/jit/backend/model.py +++ b/rpython/jit/backend/model.py @@ -1,5 +1,4 @@ from rpython.rlib.debug import debug_start, debug_print, debug_stop -from rpython.jit.metainterp import compile from rpython.rtyper.lltypesystem import lltype class CPUTotalTracker(object): @@ -23,7 +22,6 @@ propagate_exception_descr = None def __init__(self): - self.__dict__.update(compile.make_done_loop_tokens()) self.tracker = CPUTotalTracker() def _freeze_(self): diff --git a/rpython/jit/backend/test/runner_test.py b/rpython/jit/backend/test/runner_test.py --- a/rpython/jit/backend/test/runner_test.py +++ b/rpython/jit/backend/test/runner_test.py @@ -129,6 +129,8 @@ assert fail.identifier == 1 def test_compile_linear_float_loop(self): + if not self.cpu.supports_floats: + py.test.skip("requires floats") i0 = BoxFloat() i1 = BoxFloat() operations = [ @@ -2787,6 +2789,7 @@ looptoken = JitCellToken() looptoken.outermost_jitdriver_sd = FakeJitDriverSD() finish_descr = loop.operations[-1].getdescr() + self.cpu.done_with_this_frame_descr_int = BasicFinalDescr() self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) ARGS = [lltype.Signed] * 10 RES = lltype.Signed @@ -2858,6 +2861,7 @@ finish_descr = loop.operations[-1].getdescr() looptoken = JitCellToken() looptoken.outermost_jitdriver_sd = FakeJitDriverSD() + self.cpu.done_with_this_frame_descr_float = BasicFinalDescr() self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) args = [longlong.getfloatstorage(1.2), longlong.getfloatstorage(2.3)] @@ -2950,6 +2954,7 @@ loop = parse(ops) looptoken = JitCellToken() looptoken.outermost_jitdriver_sd = FakeJitDriverSD() + self.cpu.done_with_this_frame_descr_float = BasicFinalDescr() self.cpu.compile_loop(loop.inputargs, loop.operations, looptoken) finish_descr = loop.operations[-1].getdescr() args = [longlong.getfloatstorage(1.25), diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -2271,7 +2271,9 @@ else: raise AssertionError(kind) - value = rffi.cast(lltype.Signed, cast_instance_to_gcref(value)) + gcref = cast_instance_to_gcref(value) + rgc._make_sure_does_not_move(gcref) + value = rffi.cast(lltype.Signed, gcref) base_ofs = self.cpu.get_baseofs_of_frame_field() ofs = self.cpu.get_ofs_of_frame_field('jf_descr') self.mc.CMP_mi((eax.value, base_ofs + ofs), value) @@ -2302,8 +2304,7 @@ fielddescr = jd.vable_token_descr assert isinstance(fielddescr, FieldDescr) vtoken_ofs = fielddescr.offset - vable_ofs = (jd.index_of_virtualizable + JITFRAME_FIXED_SIZE) * WORD - self.mc.MOV_rm(edx.value, (eax.value, vable_ofs)) + self.mc.MOV(edx, vloc) # we know vloc is on the current frame self.mc.MOV_mi((edx.value, vtoken_ofs), 0) # in the line above, TOKEN_NONE = 0 # diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py --- a/rpython/jit/backend/x86/runner.py +++ b/rpython/jit/backend/x86/runner.py @@ -107,7 +107,7 @@ lst = [(i, history.getkind(ARG)[0]) for i, ARG in enumerate(ARGS)] kinds = unrolling_iterable(lst) - + def execute_token(executable_token, *args): clt = executable_token.compiled_loop_token assert len(args) == clt._debug_nbargs @@ -160,7 +160,7 @@ def invalidate_loop(self, looptoken): from rpython.jit.backend.x86 import codebuf - + for addr, tgt in looptoken.compiled_loop_token.invalidate_positions: mc = codebuf.MachineCodeBlockWrapper() mc.JMP_l(tgt) @@ -178,38 +178,6 @@ l[i].counter = ll_s.i return l - def set_int_value(self, newframe, index, value): - """ Note that we keep index multiplied by WORD here mostly - for completeness with get_int_value and friends - """ - descr = self.gc_ll_descr.getframedescrs(self).arraydescr - ofs = self.unpack_arraydescr(descr) - self.write_int_at_mem(newframe, ofs + index, WORD, 1, value) - - def set_ref_value(self, newframe, index, value): - descr = self.gc_ll_descr.getframedescrs(self).arraydescr - ofs = self.unpack_arraydescr(descr) - self.write_ref_at_mem(newframe, ofs + index, value) - - def set_float_value(self, newframe, index, value): - descr = self.gc_ll_descr.getframedescrs(self).arraydescr - ofs = self.unpack_arraydescr(descr) - self.write_float_at_mem(newframe, ofs + index, value) - - @specialize.arg(1) - def get_ofs_of_frame_field(self, name): - descrs = self.gc_ll_descr.getframedescrs(self) - if name.startswith('jfi_'): - base_ofs = 0 # not relative to frame - else: - base_ofs = self.unpack_arraydescr(descrs.arraydescr) - ofs = self.unpack_fielddescr(getattr(descrs, name)) - return ofs - base_ofs - - def get_baseofs_of_frame_field(self): - descrs = self.gc_ll_descr.getframedescrs(self) - base_ofs = self.unpack_arraydescr(descrs.arraydescr) - return base_ofs class CPU386(AbstractX86CPU): backend_name = 'x86' diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -1420,6 +1420,11 @@ self._addr2name_values = [] self.__dict__.update(compile.make_done_loop_tokens()) + for val in ['int', 'float', 'ref', 'void']: + fullname = 'done_with_this_frame_descr_' + val + setattr(self.cpu, fullname, getattr(self, fullname)) + d = self.exit_frame_with_exception_descr_ref + self.cpu.exit_frame_with_exception_descr_ref = d def _freeze_(self): return True diff --git a/rpython/rtyper/memory/gctransform/shadowstack.py b/rpython/rtyper/memory/gctransform/shadowstack.py --- a/rpython/rtyper/memory/gctransform/shadowstack.py +++ b/rpython/rtyper/memory/gctransform/shadowstack.py @@ -268,12 +268,6 @@ self.gc_start_fresh_new_state_ptr = getfn(gc_start_fresh_new_state, [], annmodel.s_None, inline=True) - # fish... - translator = gctransformer.translator - if hasattr(translator, '_jit2gc'): - from rpython.rlib._rffi_stacklet import _translate_pointer - root_iterator = translator._jit2gc['root_iterator'] - root_iterator.translateptr = _translate_pointer # ____________________________________________________________ @@ -366,23 +360,18 @@ def get_root_iterator(gctransformer): if hasattr(gctransformer, '_root_iterator'): return gctransformer._root_iterator # if already built - translator = gctransformer.translator - if (hasattr(translator, '_jit2gc') and - 'root_iterator' in translator._jit2gc): - result = translator._jit2gc['root_iterator'] - else: - class RootIterator(object): - def _freeze_(self): - return True - def setcontext(self, context): - pass - def nextleft(self, gc, start, addr): - while addr != start: - addr -= sizeofaddr - if gc.points_to_valid_gc_object(addr): - return addr - return llmemory.NULL - result = RootIterator() + class RootIterator(object): + def _freeze_(self): + return True + def setcontext(self, context): + pass + def nextleft(self, gc, start, addr): + while addr != start: + addr -= sizeofaddr + if gc.points_to_valid_gc_object(addr): + return addr + return llmemory.NULL + result = RootIterator() gctransformer._root_iterator = result return result From noreply at buildbot.pypy.org Tue Jan 29 10:44:47 2013 From: noreply at buildbot.pypy.org (arigo) Date: Tue, 29 Jan 2013 10:44:47 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: Future-proofing code. Message-ID: <20130129094447.501A51C0041@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: jitframe-on-heap Changeset: r60660:0e72a368a1b0 Date: 2013-01-29 10:44 +0100 http://bitbucket.org/pypy/pypy/changeset/0e72a368a1b0/ Log: Future-proofing code. diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py --- a/rpython/jit/backend/llsupport/gc.py +++ b/rpython/jit/backend/llsupport/gc.py @@ -139,7 +139,6 @@ """ Allocate a new frame, overwritten by tests """ frame = jitframe.JITFRAME.allocate(frame_info) - llop.gc_assume_young_pointers(lltype.Void, frame) return frame class JitFrameDescrs: diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py --- a/rpython/jit/backend/llsupport/llmodel.py +++ b/rpython/jit/backend/llsupport/llmodel.py @@ -59,18 +59,13 @@ frame = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, frame) assert size <= frame.jf_frame_info.jfi_frame_depth new_frame = jitframe.JITFRAME.allocate(frame.jf_frame_info) - # XXX now we know, rewrite this - # YYY it doesn't matter at all, it's fine that way too - # we need to do this, because we're not sure what things - # are GC pointers and which ones are not - llop.gc_writebarrier_before_copy(lltype.Bool, frame, new_frame, - 0, 0, len(frame.jf_frame)) i = 0 while i < len(frame.jf_frame): new_frame.jf_frame[i] = frame.jf_frame[i] i += 1 new_frame.jf_savedata = frame.jf_savedata # all other fields are empty + llop.gc_assume_young_pointers(lltype.Void, new_frame) return lltype.cast_opaque_ptr(llmemory.GCREF, new_frame) if not translate_support_code: diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py --- a/rpython/jit/backend/x86/runner.py +++ b/rpython/jit/backend/x86/runner.py @@ -1,6 +1,7 @@ import py from rpython.rlib.unroll import unrolling_iterable from rpython.rtyper.lltypesystem import lltype, llmemory, rffi +from rpython.rtyper.lltypesystem.lloperation import llop from rpython.rtyper.llinterp import LLInterpreter from rpython.rlib.objectmodel import specialize from rpython.rlib.jit_hooks import LOOP_RUN_CONTAINER @@ -136,6 +137,9 @@ assert kind == history.REF self.set_ref_value(ll_frame, num, arg) num += WORD + # no GC operation between gc_assume_young_pointers and + # the actual call to assembler! + llop.gc_assume_young_pointers(lltype.Void, frame) ll_frame = func(ll_frame) finally: if not self.translate_support_code: From noreply at buildbot.pypy.org Tue Jan 29 10:48:10 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 29 Jan 2013 10:48:10 +0100 (CET) Subject: [pypy-commit] pypy default: reduce unnecessary skips in lib-python tests Message-ID: <20130129094810.381C31C12FC@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60661:ab1b94dab0d1 Date: 2013-01-29 04:47 -0500 http://bitbucket.org/pypy/pypy/changeset/ab1b94dab0d1/ Log: reduce unnecessary skips in lib-python tests diff --git a/lib-python/conftest.py b/lib-python/conftest.py --- a/lib-python/conftest.py +++ b/lib-python/conftest.py @@ -93,24 +93,17 @@ m.test_main() ''' % locals()) -if sys.platform == 'win32': - skip_win32 = "Not supported on Windows" - only_win32 = False -else: - skip_win32 = False - only_win32 = "Only on Windows" - testmap = [ RegrTest('test___all__.py', core=True), RegrTest('test___future__.py', core=True), - RegrTest('test__locale.py', usemodules='_locale', skip=skip_win32), + RegrTest('test__locale.py', usemodules='_locale'), RegrTest('test_abc.py'), RegrTest('test_abstract_numbers.py'), - RegrTest('test_aepack.py', skip=True), + RegrTest('test_aepack.py'), RegrTest('test_aifc.py'), - RegrTest('test_al.py', skip=True), + RegrTest('test_al.py'), RegrTest('test_anydbm.py', usemodules='struct'), - RegrTest('test_applesingle.py', skip=True), + RegrTest('test_applesingle.py'), RegrTest('test_argparse.py', usemodules='binascii'), RegrTest('test_array.py', core=True, usemodules='struct array binascii'), RegrTest('test_ascii_formatd.py'), @@ -118,7 +111,7 @@ RegrTest('test_asynchat.py', usemodules='select fcntl'), RegrTest('test_asyncore.py', usemodules='select fcntl'), RegrTest('test_atexit.py', core=True), - RegrTest('test_audioop.py', skip=True), + RegrTest('test_audioop.py', skip="unsupported extension module"), RegrTest('test_augassign.py', core=True), RegrTest('test_base64.py', usemodules='struct'), RegrTest('test_bastion.py'), @@ -129,9 +122,9 @@ RegrTest('test_binop.py', core=True), RegrTest('test_bisect.py', core=True, usemodules='_bisect'), RegrTest('test_bool.py', core=True), - RegrTest('test_bsddb.py', skip="unsupported extension module"), - RegrTest('test_bsddb185.py', skip="unsupported extension module"), - RegrTest('test_bsddb3.py', skip="unsupported extension module"), + RegrTest('test_bsddb.py'), + RegrTest('test_bsddb185.py'), + RegrTest('test_bsddb3.py'), RegrTest('test_buffer.py'), RegrTest('test_bufio.py', core=True), RegrTest('test_builtin.py', core=True, usemodules='binascii'), @@ -140,11 +133,11 @@ RegrTest('test_calendar.py'), RegrTest('test_call.py', core=True), RegrTest('test_capi.py', skip="not applicable"), - RegrTest('test_cd.py', skip=True), + RegrTest('test_cd.py'), RegrTest('test_cfgparser.py'), RegrTest('test_cgi.py'), RegrTest('test_charmapcodec.py', core=True), - RegrTest('test_cl.py', skip=True), + RegrTest('test_cl.py'), RegrTest('test_class.py', core=True), RegrTest('test_cmath.py', core=True), RegrTest('test_cmd.py'), @@ -184,10 +177,10 @@ RegrTest('test_copy_reg.py', core=True), RegrTest('test_cpickle.py', core=True), RegrTest('test_cprofile.py'), - RegrTest('test_crypt.py', usemodules='crypt', skip=skip_win32), + RegrTest('test_crypt.py', usemodules='crypt'), RegrTest('test_csv.py', usemodules='_csv'), RegrTest('test_ctypes.py', usemodules="_rawffi thread"), - RegrTest('test_curses.py', skip="unsupported extension module"), + RegrTest('test_curses.py'), RegrTest('test_datetime.py'), RegrTest('test_dbm.py'), RegrTest('test_decimal.py'), @@ -203,7 +196,7 @@ RegrTest('test_dircache.py', core=True), RegrTest('test_dis.py'), RegrTest('test_distutils.py', skip=True), - RegrTest('test_dl.py', skip=True), + RegrTest('test_dl.py'), RegrTest('test_doctest.py', usemodules="thread"), RegrTest('test_doctest2.py'), RegrTest('test_docxmlrpc.py'), @@ -220,7 +213,7 @@ RegrTest('test_exception_variations.py'), RegrTest('test_exceptions.py', core=True), RegrTest('test_extcall.py', core=True), - RegrTest('test_fcntl.py', usemodules='fcntl', skip=skip_win32), + RegrTest('test_fcntl.py', usemodules='fcntl'), RegrTest('test_file.py', usemodules="posix", core=True), RegrTest('test_file2k.py', usemodules="posix", core=True), RegrTest('test_filecmp.py', core=True), @@ -245,19 +238,19 @@ RegrTest('test_future_builtins.py'), RegrTest('test_gc.py', usemodules='_weakref', skip="implementation detail"), RegrTest('test_gdb.py', skip="not applicable"), - RegrTest('test_gdbm.py', skip="unsupported extension module"), + RegrTest('test_gdbm.py'), RegrTest('test_generators.py', core=True, usemodules='thread _weakref'), RegrTest('test_genericpath.py'), RegrTest('test_genexps.py', core=True, usemodules='_weakref'), - RegrTest('test_getargs.py', skip="unsupported extension module"), - RegrTest('test_getargs2.py', skip="unsupported extension module"), + RegrTest('test_getargs.py'), + RegrTest('test_getargs2.py', usemodules='binascii', skip=True), RegrTest('test_getopt.py', core=True), RegrTest('test_gettext.py'), - RegrTest('test_gl.py', skip=True), + RegrTest('test_gl.py'), RegrTest('test_glob.py', core=True), RegrTest('test_global.py', core=True), RegrTest('test_grammar.py', core=True), - RegrTest('test_grp.py', skip=skip_win32), + RegrTest('test_grp.py'), RegrTest('test_gzip.py', usemodules='zlib'), RegrTest('test_hash.py', core=True), RegrTest('test_hashlib.py', core=True), @@ -268,9 +261,9 @@ RegrTest('test_htmlparser.py'), RegrTest('test_httplib.py'), RegrTest('test_httpservers.py'), - RegrTest('test_imageop.py', skip="unsupported extension module"), + RegrTest('test_imageop.py'), RegrTest('test_imaplib.py'), - RegrTest('test_imgfile.py', skip="unsupported extension module"), + RegrTest('test_imgfile.py'), RegrTest('test_imp.py', core=True, usemodules='thread'), RegrTest('test_import.py', core=True), RegrTest('test_importhooks.py', core=True), @@ -298,7 +291,7 @@ RegrTest('test_long_future.py', core=True), RegrTest('test_longexp.py', core=True), RegrTest('test_macos.py'), - RegrTest('test_macostools.py', skip=True), + RegrTest('test_macostools.py'), RegrTest('test_macpath.py'), RegrTest('test_mailbox.py'), RegrTest('test_marshal.py', core=True), @@ -314,16 +307,15 @@ RegrTest('test_mmap.py', usemodules="mmap"), RegrTest('test_module.py', core=True), RegrTest('test_modulefinder.py'), - RegrTest('test_msilib.py', skip=only_win32), + RegrTest('test_msilib.py'), RegrTest('test_multibytecodec.py', usemodules='_multibytecodec'), - RegrTest('test_multibytecodec_support.py', skip="not a test"), RegrTest('test_multifile.py'), - RegrTest('test_multiprocessing.py', skip="FIXME leaves subprocesses"), + RegrTest('test_multiprocessing.py', skip=True), RegrTest('test_mutants.py', core="possibly"), RegrTest('test_mutex.py'), RegrTest('test_netrc.py'), RegrTest('test_new.py', core=True), - RegrTest('test_nis.py', skip="unsupported extension module"), + RegrTest('test_nis.py'), RegrTest('test_normalization.py'), RegrTest('test_ntpath.py'), RegrTest('test_old_mailbox.py'), @@ -332,7 +324,7 @@ RegrTest('test_operator.py', core=True), RegrTest('test_optparse.py'), RegrTest('test_os.py', core=True), - RegrTest('test_ossaudiodev.py', skip="unsupported extension module"), + RegrTest('test_ossaudiodev.py'), RegrTest('test_parser.py', skip="slowly deprecating compiler"), RegrTest('test_pdb.py'), RegrTest('test_peepholer.py'), @@ -348,8 +340,8 @@ RegrTest('test_pkgimport.py', core=True), RegrTest('test_pkgutil.py'), RegrTest('test_platform.py'), - RegrTest('test_plistlib.py', skip="unsupported module"), - RegrTest('test_poll.py', skip=skip_win32), + RegrTest('test_plistlib.py'), + RegrTest('test_poll.py'), RegrTest('test_popen.py'), RegrTest('test_popen2.py'), RegrTest('test_poplib.py'), @@ -361,8 +353,8 @@ RegrTest('test_profile.py'), RegrTest('test_property.py', core=True), RegrTest('test_pstats.py'), - RegrTest('test_pty.py', skip="unsupported extension module"), - RegrTest('test_pwd.py', usemodules="pwd", skip=skip_win32), + RegrTest('test_pty.py', skip=True), + RegrTest('test_pwd.py', usemodules="pwd"), RegrTest('test_py3kwarn.py'), RegrTest('test_py_compile.py'), RegrTest('test_pyclbr.py'), @@ -374,7 +366,7 @@ RegrTest('test_re.py', core=True), RegrTest('test_readline.py'), RegrTest('test_repr.py', core=True), - RegrTest('test_resource.py', skip=skip_win32), + RegrTest('test_resource.py'), RegrTest('test_rfc822.py'), RegrTest('test_richcmp.py', core=True), RegrTest('test_rlcompleter.py'), @@ -382,7 +374,7 @@ RegrTest('test_runpy.py'), RegrTest('test_sax.py'), RegrTest('test_scope.py', core=True), - RegrTest('test_scriptpackages.py', skip="unsupported extension module"), + RegrTest('test_scriptpackages.py'), RegrTest('test_select.py'), RegrTest('test_set.py', core=True), RegrTest('test_sets.py'), @@ -417,7 +409,7 @@ RegrTest('test_structmembers.py', skip="CPython specific"), RegrTest('test_structseq.py'), RegrTest('test_subprocess.py', usemodules='signal'), - RegrTest('test_sunaudiodev.py', skip=True), + RegrTest('test_sunaudiodev.py'), RegrTest('test_sundry.py'), RegrTest('test_symtable.py', skip="implementation detail"), RegrTest('test_syntax.py', core=True), @@ -426,7 +418,7 @@ RegrTest('test_sys_settrace.py', core=True), RegrTest('test_sysconfig.py'), RegrTest('test_tarfile.py'), - RegrTest('test_tcl.py', skip="unsupported extension module"), + RegrTest('test_tcl.py'), RegrTest('test_telnetlib.py'), RegrTest('test_tempfile.py'), RegrTest('test_textwrap.py'), @@ -473,12 +465,12 @@ RegrTest('test_wait3.py', usemodules="thread"), RegrTest('test_wait4.py', usemodules="thread"), RegrTest('test_warnings.py', core=True), - RegrTest('test_wave.py', skip="unsupported extension module"), + RegrTest('test_wave.py'), RegrTest('test_weakref.py', core=True, usemodules='_weakref'), RegrTest('test_weakset.py'), RegrTest('test_whichdb.py'), - RegrTest('test_winreg.py', skip=only_win32), - RegrTest('test_winsound.py', skip="unsupported extension module"), + RegrTest('test_winreg.py'), + RegrTest('test_winsound.py'), RegrTest('test_with.py'), RegrTest('test_wsgiref.py'), RegrTest('test_xdrlib.py'), @@ -498,7 +490,9 @@ def check_testmap_complete(): listed_names = dict.fromkeys([regrtest.basename for regrtest in testmap]) assert len(listed_names) == len(testmap) - listed_names['test_support.py'] = True # ignore this + # names to ignore + listed_names['test_support.py'] = True + listed_names['test_multibytecodec_support.py'] = True missing = [] for path in testdir.listdir(fil='test_*.py'): name = path.basename From noreply at buildbot.pypy.org Tue Jan 29 11:45:12 2013 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 29 Jan 2013 11:45:12 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: kill the hack to update frame_info and instead make it official to have size as an argument to realloc frame Message-ID: <20130129104512.327961C0207@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60662:1af1c9069875 Date: 2013-01-29 12:44 +0200 http://bitbucket.org/pypy/pypy/changeset/1af1c9069875/ Log: kill the hack to update frame_info and instead make it official to have size as an argument to realloc frame diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py --- a/rpython/jit/backend/arm/assembler.py +++ b/rpython/jit/backend/arm/assembler.py @@ -924,14 +924,6 @@ def update_frame_depth(self, frame_depth): self.current_clt.frame_info.jfi_frame_depth = frame_depth - new_jumping_to = [] - for wref in self.current_clt.jumping_to: - clt = wref() - if clt is not None: - clt.frame_info.jfi_frame_depth = max(frame_depth, - clt.frame_info.jfi_frame_depth) - new_jumping_to.append(weakref.ref(clt)) - self.current_clt.jumping_to = new_jumping_to def write_pending_failure_recoveries(self): for tok in self.pending_guards: diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py --- a/rpython/jit/backend/llsupport/llmodel.py +++ b/rpython/jit/backend/llsupport/llmodel.py @@ -57,7 +57,11 @@ def realloc_frame(frame, size): frame = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, frame) - assert size <= frame.jf_frame_info.jfi_frame_depth + if size > frame.jf_frame_info.jfi_frame_depth: + # update the frame_info size, which is for whatever reason + # not up to date + base_ofs = self.get_baseofs_of_frame_field() + frame.jf_frame_info.set_frame_depth(base_ofs, size) new_frame = jitframe.JITFRAME.allocate(frame.jf_frame_info) i = 0 while i < len(frame.jf_frame): diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py --- a/rpython/jit/backend/model.py +++ b/rpython/jit/backend/model.py @@ -295,7 +295,6 @@ self.cpu = cpu self.number = number self.bridges_count = 0 - self.jumping_to = [] # a list of weakrefs who jump here # This growing list gives the 'descr_number' of all fail descrs # that belong to this loop or to a bridge attached to it. # Filled by the frontend calling record_faildescr_index(). diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -668,14 +668,6 @@ def update_frame_depth(self, frame_depth): baseofs = self.cpu.get_baseofs_of_frame_field() self.current_clt.frame_info.set_frame_depth(baseofs, frame_depth) - new_jumping_to = [] - for wref in self.current_clt.jumping_to: - clt = wref() - if clt is not None: - clt.frame_info.set_frame_depth(baseofs, max(frame_depth, - clt.frame_info.jfi_frame_depth)) - new_jumping_to.append(weakref.ref(clt)) - self.current_clt.jumping_to = new_jumping_to def _check_frame_depth(self, mc, gcmap): """ check if the frame is of enough depth to follow this bridge. @@ -2468,10 +2460,6 @@ curpos = self.mc.get_relative_pos() + 5 self.mc.JMP_l(target - curpos) else: - clt = self.current_clt - assert clt is not None - target_token._x86_clt.jumping_to.append( - weakref.ref(clt)) self.mc.JMP(imm(target)) def malloc_cond(self, nursery_free_adr, nursery_top_adr, size, gcmap): From noreply at buildbot.pypy.org Tue Jan 29 12:12:09 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 29 Jan 2013 12:12:09 +0100 (CET) Subject: [pypy-commit] pypy default: enable lib-python test_pty Message-ID: <20130129111209.2983A1C0041@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60663:5d685d27c0a9 Date: 2013-01-29 06:03 -0500 http://bitbucket.org/pypy/pypy/changeset/5d685d27c0a9/ Log: enable lib-python test_pty diff --git a/lib-python/conftest.py b/lib-python/conftest.py --- a/lib-python/conftest.py +++ b/lib-python/conftest.py @@ -353,7 +353,7 @@ RegrTest('test_profile.py'), RegrTest('test_property.py', core=True), RegrTest('test_pstats.py'), - RegrTest('test_pty.py', skip=True), + RegrTest('test_pty.py', usemodules='fcntl termios select'), RegrTest('test_pwd.py', usemodules="pwd"), RegrTest('test_py3kwarn.py'), RegrTest('test_py_compile.py'), From noreply at buildbot.pypy.org Tue Jan 29 12:12:10 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 29 Jan 2013 12:12:10 +0100 (CET) Subject: [pypy-commit] pypy default: don't raise errors on socket.close() to match cpython Message-ID: <20130129111210.723A61C0041@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60664:466f3b01ca3d Date: 2013-01-29 06:11 -0500 http://bitbucket.org/pypy/pypy/changeset/466f3b01ca3d/ Log: don't raise errors on socket.close() to match cpython diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -181,7 +181,8 @@ try: self.close() except SocketError, e: - raise converted_error(space, e) + # cpython doesn't return any errors on close + pass def connect_w(self, space, w_addr): """connect(address) @@ -448,7 +449,7 @@ w_addr = space.w_None return space.newtuple([space.wrap(readlgt), w_addr]) except SocketError, e: - raise converted_error(space, e) + raise converted_error(space, e) @unwrap_spec(cmd=int) def ioctl_w(self, space, cmd, w_option): diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -362,13 +362,14 @@ assert isinstance(s.fileno(), int) def test_socket_close(self): - import _socket + import _socket, os s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM, 0) fileno = s.fileno() assert s.fileno() >= 0 s.close() assert s.fileno() < 0 s.close() + raises(OSError, os.close, fileno) def test_socket_close_error(self): import _socket, os @@ -376,7 +377,7 @@ skip("Windows sockets are not files") s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM, 0) os.close(s.fileno()) - raises(_socket.error, s.close) + s.close() def test_socket_connect(self): import _socket, os From noreply at buildbot.pypy.org Tue Jan 29 12:18:54 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 29 Jan 2013 12:18:54 +0100 (CET) Subject: [pypy-commit] pypy default: oops this doesnt make sense for windows Message-ID: <20130129111854.45FE01C13CF@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60665:9384b83b4442 Date: 2013-01-29 06:17 -0500 http://bitbucket.org/pypy/pypy/changeset/9384b83b4442/ Log: oops this doesnt make sense for windows diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -369,7 +369,8 @@ s.close() assert s.fileno() < 0 s.close() - raises(OSError, os.close, fileno) + if os.name != 'nt': + raises(OSError, os.close, fileno) def test_socket_close_error(self): import _socket, os From noreply at buildbot.pypy.org Tue Jan 29 13:14:31 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 29 Jan 2013 13:14:31 +0100 (CET) Subject: [pypy-commit] pypy vendor/stdlib: update python 2 stdlib to tip (2.7.3+) Message-ID: <20130129121431.C2B4E1C13C4@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: vendor/stdlib Changeset: r60666:8a3b5bfd266e Date: 2013-01-29 02:32 -0500 http://bitbucket.org/pypy/pypy/changeset/8a3b5bfd266e/ Log: update python 2 stdlib to tip (2.7.3+) diff too long, truncating to 2000 out of 17105 lines diff --git a/lib-python/2.7/BaseHTTPServer.py b/lib-python/2.7/BaseHTTPServer.py --- a/lib-python/2.7/BaseHTTPServer.py +++ b/lib-python/2.7/BaseHTTPServer.py @@ -447,13 +447,13 @@ specified as subsequent arguments (it's just like printf!). - The client host and current date/time are prefixed to - every message. + The client ip address and current date/time are prefixed to every + message. """ sys.stderr.write("%s - - [%s] %s\n" % - (self.address_string(), + (self.client_address[0], self.log_date_time_string(), format%args)) diff --git a/lib-python/2.7/CGIHTTPServer.py b/lib-python/2.7/CGIHTTPServer.py --- a/lib-python/2.7/CGIHTTPServer.py +++ b/lib-python/2.7/CGIHTTPServer.py @@ -84,9 +84,11 @@ path begins with one of the strings in self.cgi_directories (and the next character is a '/' or the end of the string). """ - splitpath = _url_collapse_path_split(self.path) - if splitpath[0] in self.cgi_directories: - self.cgi_info = splitpath + collapsed_path = _url_collapse_path(self.path) + dir_sep = collapsed_path.find('/', 1) + head, tail = collapsed_path[:dir_sep], collapsed_path[dir_sep+1:] + if head in self.cgi_directories: + self.cgi_info = head, tail return True return False @@ -298,44 +300,46 @@ self.log_message("CGI script exited OK") -# TODO(gregory.p.smith): Move this into an appropriate library. -def _url_collapse_path_split(path): +def _url_collapse_path(path): """ Given a URL path, remove extra '/'s and '.' path elements and collapse - any '..' references. + any '..' references and returns a colllapsed path. Implements something akin to RFC-2396 5.2 step 6 to parse relative paths. + The utility of this function is limited to is_cgi method and helps + preventing some security attacks. Returns: A tuple of (head, tail) where tail is everything after the final / and head is everything before it. Head will always start with a '/' and, if it contains anything else, never have a trailing '/'. Raises: IndexError if too many '..' occur within the path. + """ # Similar to os.path.split(os.path.normpath(path)) but specific to URL # path semantics rather than local operating system semantics. - path_parts = [] - for part in path.split('/'): - if part == '.': - path_parts.append('') - else: - path_parts.append(part) - # Filter out blank non trailing parts before consuming the '..'. - path_parts = [part for part in path_parts[:-1] if part] + path_parts[-1:] + path_parts = path.split('/') + head_parts = [] + for part in path_parts[:-1]: + if part == '..': + head_parts.pop() # IndexError if more '..' than prior parts + elif part and part != '.': + head_parts.append( part ) if path_parts: tail_part = path_parts.pop() + if tail_part: + if tail_part == '..': + head_parts.pop() + tail_part = '' + elif tail_part == '.': + tail_part = '' else: tail_part = '' - head_parts = [] - for part in path_parts: - if part == '..': - head_parts.pop() - else: - head_parts.append(part) - if tail_part and tail_part == '..': - head_parts.pop() - tail_part = '' - return ('/' + '/'.join(head_parts), tail_part) + + splitpath = ('/' + '/'.join(head_parts), tail_part) + collapsed_path = "/".join(splitpath) + + return collapsed_path nobody = None diff --git a/lib-python/2.7/Cookie.py b/lib-python/2.7/Cookie.py --- a/lib-python/2.7/Cookie.py +++ b/lib-python/2.7/Cookie.py @@ -390,7 +390,7 @@ from time import gmtime, time now = time() year, month, day, hh, mm, ss, wd, y, z = gmtime(now + future) - return "%s, %02d-%3s-%4d %02d:%02d:%02d GMT" % \ + return "%s, %02d %3s %4d %02d:%02d:%02d GMT" % \ (weekdayname[wd], day, monthname[month], year, hh, mm, ss) @@ -539,7 +539,7 @@ r"(?P" # Start of group 'val' r'"(?:[^\\"]|\\.)*"' # Any doublequoted string r"|" # or - r"\w{3},\s[\w\d-]{9,11}\s[\d:]{8}\sGMT" # Special case for "expires" attr + r"\w{3},\s[\s\w\d-]{9,11}\s[\d:]{8}\sGMT" # Special case for "expires" attr r"|" # or ""+ _LegalCharsPatt +"*" # Any word or empty string r")" # End of group 'val' diff --git a/lib-python/2.7/HTMLParser.py b/lib-python/2.7/HTMLParser.py --- a/lib-python/2.7/HTMLParser.py +++ b/lib-python/2.7/HTMLParser.py @@ -22,13 +22,13 @@ starttagopen = re.compile('<[a-zA-Z]') piclose = re.compile('>') commentclose = re.compile(r'--\s*>') -tagfind = re.compile('[a-zA-Z][-.a-zA-Z0-9:_]*') +tagfind = re.compile('([a-zA-Z][-.a-zA-Z0-9:_]*)(?:\s|/(?!>))*') # see http://www.w3.org/TR/html5/tokenization.html#tag-open-state # and http://www.w3.org/TR/html5/tokenization.html#tag-name-state tagfind_tolerant = re.compile('[a-zA-Z][^\t\n\r\f />\x00]*') attrfind = re.compile( - r'[\s/]*((?<=[\'"\s/])[^\s/>][^\s/=>]*)(\s*=+\s*' + r'((?<=[\'"\s/])[^\s/>][^\s/=>]*)(\s*=+\s*' r'(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?(?:\s|/(?!>))*') locatestarttagend = re.compile(r""" @@ -289,7 +289,7 @@ match = tagfind.match(rawdata, i+1) assert match, 'unexpected call to parse_starttag()' k = match.end() - self.lasttag = tag = rawdata[i+1:k].lower() + self.lasttag = tag = match.group(1).lower() while k < endpos: m = attrfind.match(rawdata, k) diff --git a/lib-python/2.7/SimpleXMLRPCServer.py b/lib-python/2.7/SimpleXMLRPCServer.py --- a/lib-python/2.7/SimpleXMLRPCServer.py +++ b/lib-python/2.7/SimpleXMLRPCServer.py @@ -1,4 +1,4 @@ -"""Simple XML-RPC Server. +r"""Simple XML-RPC Server. This module can be used to create simple XML-RPC servers by creating a server and either installing functions, a diff --git a/lib-python/2.7/SocketServer.py b/lib-python/2.7/SocketServer.py --- a/lib-python/2.7/SocketServer.py +++ b/lib-python/2.7/SocketServer.py @@ -133,6 +133,7 @@ import select import sys import os +import errno try: import threading except ImportError: @@ -147,6 +148,15 @@ "ThreadingUnixStreamServer", "ThreadingUnixDatagramServer"]) +def _eintr_retry(func, *args): + """restart a system call interrupted by EINTR""" + while True: + try: + return func(*args) + except (OSError, select.error) as e: + if e.args[0] != errno.EINTR: + raise + class BaseServer: """Base class for server classes. @@ -222,7 +232,8 @@ # connecting to the socket to wake this up instead of # polling. Polling reduces our responsiveness to a # shutdown request and wastes cpu at all other times. - r, w, e = select.select([self], [], [], poll_interval) + r, w, e = _eintr_retry(select.select, [self], [], [], + poll_interval) if self in r: self._handle_request_noblock() finally: @@ -262,7 +273,7 @@ timeout = self.timeout elif self.timeout is not None: timeout = min(timeout, self.timeout) - fd_sets = select.select([self], [], [], timeout) + fd_sets = _eintr_retry(select.select, [self], [], [], timeout) if not fd_sets[0]: self.handle_timeout() return @@ -690,7 +701,12 @@ def finish(self): if not self.wfile.closed: - self.wfile.flush() + try: + self.wfile.flush() + except socket.error: + # An final socket error may have occurred here, such as + # the local error ECONNABORTED. + pass self.wfile.close() self.rfile.close() diff --git a/lib-python/2.7/_LWPCookieJar.py b/lib-python/2.7/_LWPCookieJar.py --- a/lib-python/2.7/_LWPCookieJar.py +++ b/lib-python/2.7/_LWPCookieJar.py @@ -48,7 +48,7 @@ class LWPCookieJar(FileCookieJar): """ - The LWPCookieJar saves a sequence of"Set-Cookie3" lines. + The LWPCookieJar saves a sequence of "Set-Cookie3" lines. "Set-Cookie3" is the format used by the libwww-perl libary, not known to be compatible with any browser, but which is easy to read and doesn't lose information about RFC 2965 cookies. @@ -60,7 +60,7 @@ """ def as_lwp_str(self, ignore_discard=True, ignore_expires=True): - """Return cookies as a string of "\n"-separated "Set-Cookie3" headers. + """Return cookies as a string of "\\n"-separated "Set-Cookie3" headers. ignore_discard and ignore_expires: see docstring for FileCookieJar.save diff --git a/lib-python/2.7/__future__.py b/lib-python/2.7/__future__.py --- a/lib-python/2.7/__future__.py +++ b/lib-python/2.7/__future__.py @@ -112,7 +112,7 @@ CO_FUTURE_DIVISION) absolute_import = _Feature((2, 5, 0, "alpha", 1), - (2, 7, 0, "alpha", 0), + (3, 0, 0, "alpha", 0), CO_FUTURE_ABSOLUTE_IMPORT) with_statement = _Feature((2, 5, 0, "alpha", 1), diff --git a/lib-python/2.7/_pyio.py b/lib-python/2.7/_pyio.py --- a/lib-python/2.7/_pyio.py +++ b/lib-python/2.7/_pyio.py @@ -340,8 +340,10 @@ This method has no effect if the file is already closed. """ if not self.__closed: - self.flush() - self.__closed = True + try: + self.flush() + finally: + self.__closed = True def __del__(self): """Destructor. Calls close().""" @@ -883,12 +885,18 @@ return pos def readable(self): + if self.closed: + raise ValueError("I/O operation on closed file.") return True def writable(self): + if self.closed: + raise ValueError("I/O operation on closed file.") return True def seekable(self): + if self.closed: + raise ValueError("I/O operation on closed file.") return True @@ -1451,7 +1459,7 @@ enabled. With this enabled, on input, the lines endings '\n', '\r', or '\r\n' are translated to '\n' before being returned to the caller. Conversely, on output, '\n' is translated to the system - default line seperator, os.linesep. If newline is any other of its + default line separator, os.linesep. If newline is any other of its legal values, that newline becomes the newline when the file is read and it is returned untranslated. On output, '\n' is converted to the newline. @@ -1546,6 +1554,8 @@ return self._buffer def seekable(self): + if self.closed: + raise ValueError("I/O operation on closed file.") return self._seekable def readable(self): @@ -1560,8 +1570,10 @@ def close(self): if self.buffer is not None and not self.closed: - self.flush() - self.buffer.close() + try: + self.flush() + finally: + self.buffer.close() @property def closed(self): diff --git a/lib-python/2.7/_strptime.py b/lib-python/2.7/_strptime.py --- a/lib-python/2.7/_strptime.py +++ b/lib-python/2.7/_strptime.py @@ -326,7 +326,8 @@ if len(data_string) != found.end(): raise ValueError("unconverted data remains: %s" % data_string[found.end():]) - year = 1900 + + year = None month = day = 1 hour = minute = second = fraction = 0 tz = -1 @@ -425,6 +426,12 @@ else: tz = value break + leap_year_fix = False + if year is None and month == 2 and day == 29: + year = 1904 # 1904 is first leap year of 20th century + leap_year_fix = True + elif year is None: + year = 1900 # If we know the week of the year and what day of that week, we can figure # out the Julian day of the year. if julian == -1 and week_of_year != -1 and weekday != -1: @@ -446,6 +453,12 @@ day = datetime_result.day if weekday == -1: weekday = datetime_date(year, month, day).weekday() + if leap_year_fix: + # the caller didn't supply a year but asked for Feb 29th. We couldn't + # use the default of 1900 for computations. We set it back to ensure + # that February 29th is smaller than March 1st. + year = 1900 + return (time.struct_time((year, month, day, hour, minute, second, weekday, julian, tz)), fraction) diff --git a/lib-python/2.7/_weakrefset.py b/lib-python/2.7/_weakrefset.py --- a/lib-python/2.7/_weakrefset.py +++ b/lib-python/2.7/_weakrefset.py @@ -63,7 +63,7 @@ yield item def __len__(self): - return sum(x() is not None for x in self.data) + return len(self.data) - len(self._pending_removals) def __contains__(self, item): try: @@ -116,36 +116,21 @@ def update(self, other): if self._pending_removals: self._commit_removals() - if isinstance(other, self.__class__): - self.data.update(other.data) - else: - for element in other: - self.add(element) + for element in other: + self.add(element) def __ior__(self, other): self.update(other) return self - # Helper functions for simple delegating methods. - def _apply(self, other, method): - if not isinstance(other, self.__class__): - other = self.__class__(other) - newdata = method(other.data) - newset = self.__class__() - newset.data = newdata + def difference(self, other): + newset = self.copy() + newset.difference_update(other) return newset - - def difference(self, other): - return self._apply(other, self.data.difference) __sub__ = difference def difference_update(self, other): - if self._pending_removals: - self._commit_removals() - if self is other: - self.data.clear() - else: - self.data.difference_update(ref(item) for item in other) + self.__isub__(other) def __isub__(self, other): if self._pending_removals: self._commit_removals() @@ -156,13 +141,11 @@ return self def intersection(self, other): - return self._apply(other, self.data.intersection) + return self.__class__(item for item in other if item in self) __and__ = intersection def intersection_update(self, other): - if self._pending_removals: - self._commit_removals() - self.data.intersection_update(ref(item) for item in other) + self.__iand__(other) def __iand__(self, other): if self._pending_removals: self._commit_removals() @@ -171,17 +154,17 @@ def issubset(self, other): return self.data.issubset(ref(item) for item in other) - __lt__ = issubset + __le__ = issubset - def __le__(self, other): - return self.data <= set(ref(item) for item in other) + def __lt__(self, other): + return self.data < set(ref(item) for item in other) def issuperset(self, other): return self.data.issuperset(ref(item) for item in other) - __gt__ = issuperset + __ge__ = issuperset - def __ge__(self, other): - return self.data >= set(ref(item) for item in other) + def __gt__(self, other): + return self.data > set(ref(item) for item in other) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -189,27 +172,24 @@ return self.data == set(ref(item) for item in other) def symmetric_difference(self, other): - return self._apply(other, self.data.symmetric_difference) + newset = self.copy() + newset.symmetric_difference_update(other) + return newset __xor__ = symmetric_difference def symmetric_difference_update(self, other): - if self._pending_removals: - self._commit_removals() - if self is other: - self.data.clear() - else: - self.data.symmetric_difference_update(ref(item) for item in other) + self.__ixor__(other) def __ixor__(self, other): if self._pending_removals: self._commit_removals() if self is other: self.data.clear() else: - self.data.symmetric_difference_update(ref(item) for item in other) + self.data.symmetric_difference_update(ref(item, self._remove) for item in other) return self def union(self, other): - return self._apply(other, self.data.union) + return self.__class__(e for s in (self, other) for e in s) __or__ = union def isdisjoint(self, other): diff --git a/lib-python/2.7/aifc.py b/lib-python/2.7/aifc.py --- a/lib-python/2.7/aifc.py +++ b/lib-python/2.7/aifc.py @@ -732,22 +732,28 @@ self._patchheader() def close(self): - self._ensure_header_written(0) - if self._datawritten & 1: - # quick pad to even size - self._file.write(chr(0)) - self._datawritten = self._datawritten + 1 - self._writemarkers() - if self._nframeswritten != self._nframes or \ - self._datalength != self._datawritten or \ - self._marklength: - self._patchheader() - if self._comp: - self._comp.CloseCompressor() - self._comp = None - # Prevent ref cycles - self._convert = None - self._file.close() + if self._file is None: + return + try: + self._ensure_header_written(0) + if self._datawritten & 1: + # quick pad to even size + self._file.write(chr(0)) + self._datawritten = self._datawritten + 1 + self._writemarkers() + if self._nframeswritten != self._nframes or \ + self._datalength != self._datawritten or \ + self._marklength: + self._patchheader() + if self._comp: + self._comp.CloseCompressor() + self._comp = None + finally: + # Prevent ref cycles + self._convert = None + f = self._file + self._file = None + f.close() # # Internal methods. diff --git a/lib-python/2.7/argparse.py b/lib-python/2.7/argparse.py --- a/lib-python/2.7/argparse.py +++ b/lib-python/2.7/argparse.py @@ -740,10 +740,10 @@ - default -- The value to be produced if the option is not specified. - - type -- The type which the command-line arguments should be converted - to, should be one of 'string', 'int', 'float', 'complex' or a - callable object that accepts a single string argument. If None, - 'string' is assumed. + - type -- A callable that accepts a single string argument, and + returns the converted value. The standard Python types str, int, + float, and complex are useful examples of such callables. If None, + str is used. - choices -- A container of values that should be allowed. If not None, after a command-line argument has been converted to the appropriate @@ -1692,9 +1692,12 @@ return args def parse_known_args(self, args=None, namespace=None): - # args default to the system args if args is None: + # args default to the system args args = _sys.argv[1:] + else: + # make sure that args are mutable + args = list(args) # default Namespace built from parser defaults if namespace is None: @@ -1705,10 +1708,7 @@ if action.dest is not SUPPRESS: if not hasattr(namespace, action.dest): if action.default is not SUPPRESS: - default = action.default - if isinstance(action.default, basestring): - default = self._get_value(action, default) - setattr(namespace, action.dest, default) + setattr(namespace, action.dest, action.default) # add any parser defaults that aren't present for dest in self._defaults: @@ -1936,12 +1936,23 @@ if positionals: self.error(_('too few arguments')) - # make sure all required actions were present + # make sure all required actions were present, and convert defaults. for action in self._actions: - if action.required: - if action not in seen_actions: + if action not in seen_actions: + if action.required: name = _get_action_name(action) self.error(_('argument %s is required') % name) + else: + # Convert action default now instead of doing it before + # parsing arguments to avoid calling convert functions + # twice (which may fail) if the argument was given, but + # only if it was defined already in the namespace + if (action.default is not None and + isinstance(action.default, basestring) and + hasattr(namespace, action.dest) and + action.default is getattr(namespace, action.dest)): + setattr(namespace, action.dest, + self._get_value(action, action.default)) # make sure all required groups had one option present for group in self._mutually_exclusive_groups: @@ -1967,7 +1978,7 @@ for arg_string in arg_strings: # for regular arguments, just add them back into the list - if arg_string[0] not in self.fromfile_prefix_chars: + if not arg_string or arg_string[0] not in self.fromfile_prefix_chars: new_arg_strings.append(arg_string) # replace arguments referencing files with the file content @@ -2174,9 +2185,12 @@ # Value conversion methods # ======================== def _get_values(self, action, arg_strings): - # for everything but PARSER args, strip out '--' + # for everything but PARSER, REMAINDER args, strip out first '--' if action.nargs not in [PARSER, REMAINDER]: - arg_strings = [s for s in arg_strings if s != '--'] + try: + arg_strings.remove('--') + except ValueError: + pass # optional argument produces a default when not present if not arg_strings and action.nargs == OPTIONAL: diff --git a/lib-python/2.7/asyncore.py b/lib-python/2.7/asyncore.py --- a/lib-python/2.7/asyncore.py +++ b/lib-python/2.7/asyncore.py @@ -225,6 +225,7 @@ debug = False connected = False accepting = False + connecting = False closing = False addr = None ignore_log_types = frozenset(['warning']) @@ -248,7 +249,7 @@ try: self.addr = sock.getpeername() except socket.error, err: - if err.args[0] == ENOTCONN: + if err.args[0] in (ENOTCONN, EINVAL): # To handle the case where we got an unconnected # socket. self.connected = False @@ -342,9 +343,11 @@ def connect(self, address): self.connected = False + self.connecting = True err = self.socket.connect_ex(address) if err in (EINPROGRESS, EALREADY, EWOULDBLOCK) \ or err == EINVAL and os.name in ('nt', 'ce'): + self.addr = address return if err in (0, EISCONN): self.addr = address @@ -390,7 +393,7 @@ else: return data except socket.error, why: - # winsock sometimes throws ENOTCONN + # winsock sometimes raises ENOTCONN if why.args[0] in _DISCONNECTED: self.handle_close() return '' @@ -400,6 +403,7 @@ def close(self): self.connected = False self.accepting = False + self.connecting = False self.del_channel() try: self.socket.close() @@ -438,7 +442,8 @@ # sockets that are connected self.handle_accept() elif not self.connected: - self.handle_connect_event() + if self.connecting: + self.handle_connect_event() self.handle_read() else: self.handle_read() @@ -449,6 +454,7 @@ raise socket.error(err, _strerror(err)) self.handle_connect() self.connected = True + self.connecting = False def handle_write_event(self): if self.accepting: @@ -457,12 +463,8 @@ return if not self.connected: - #check for errors - err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) - if err != 0: - raise socket.error(err, _strerror(err)) - - self.handle_connect_event() + if self.connecting: + self.handle_connect_event() self.handle_write() def handle_expt_event(self): diff --git a/lib-python/2.7/bdb.py b/lib-python/2.7/bdb.py --- a/lib-python/2.7/bdb.py +++ b/lib-python/2.7/bdb.py @@ -24,6 +24,7 @@ self.skip = set(skip) if skip else None self.breaks = {} self.fncache = {} + self.frame_returning = None def canonic(self, filename): if filename == "<" + filename[1:-1] + ">": @@ -82,7 +83,11 @@ def dispatch_return(self, frame, arg): if self.stop_here(frame) or frame == self.returnframe: - self.user_return(frame, arg) + try: + self.frame_returning = frame + self.user_return(frame, arg) + finally: + self.frame_returning = None if self.quitting: raise BdbQuit return self.trace_dispatch @@ -186,6 +191,14 @@ def set_step(self): """Stop after one line of code.""" + # Issue #13183: pdb skips frames after hitting a breakpoint and running + # step commands. + # Restore the trace function in the caller (that may not have been set + # for performance reasons) when returning from the current frame. + if self.frame_returning: + caller_frame = self.frame_returning.f_back + if caller_frame and not caller_frame.f_trace: + caller_frame.f_trace = self.trace_dispatch self._set_stopinfo(None, None) def set_next(self, frame): diff --git a/lib-python/2.7/calendar.py b/lib-python/2.7/calendar.py --- a/lib-python/2.7/calendar.py +++ b/lib-python/2.7/calendar.py @@ -161,7 +161,11 @@ oneday = datetime.timedelta(days=1) while True: yield date - date += oneday + try: + date += oneday + except OverflowError: + # Adding one day could fail after datetime.MAXYEAR + break if date.month != month and date.weekday() == self.firstweekday: break diff --git a/lib-python/2.7/cgi.py b/lib-python/2.7/cgi.py --- a/lib-python/2.7/cgi.py +++ b/lib-python/2.7/cgi.py @@ -37,7 +37,6 @@ from operator import attrgetter import sys import os -import urllib import UserDict import urlparse diff --git a/lib-python/2.7/cgitb.py b/lib-python/2.7/cgitb.py --- a/lib-python/2.7/cgitb.py +++ b/lib-python/2.7/cgitb.py @@ -295,14 +295,19 @@ if self.logdir is not None: suffix = ['.txt', '.html'][self.format=="html"] (fd, path) = tempfile.mkstemp(suffix=suffix, dir=self.logdir) + try: file = os.fdopen(fd, 'w') file.write(doc) file.close() - msg = '

    %s contains the description of this error.' % path + msg = '%s contains the description of this error.' % path except: - msg = '

    Tried to save traceback to %s, but failed.' % path - self.file.write(msg + '\n') + msg = 'Tried to save traceback to %s, but failed.' % path + + if self.format == 'html': + self.file.write('

    %s

    \n' % msg) + else: + self.file.write(msg + '\n') try: self.file.flush() except: pass diff --git a/lib-python/2.7/cmd.py b/lib-python/2.7/cmd.py --- a/lib-python/2.7/cmd.py +++ b/lib-python/2.7/cmd.py @@ -294,6 +294,7 @@ return list(commands | topics) def do_help(self, arg): + 'List available commands with "help" or detailed help with "help cmd".' if arg: # XXX check arg syntax try: diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py --- a/lib-python/2.7/collections.py +++ b/lib-python/2.7/collections.py @@ -6,11 +6,12 @@ __all__ += _abcoll.__all__ from _collections import deque, defaultdict -from operator import itemgetter as _itemgetter +from operator import itemgetter as _itemgetter, eq as _eq from keyword import iskeyword as _iskeyword import sys as _sys import heapq as _heapq from itertools import repeat as _repeat, chain as _chain, starmap as _starmap +from itertools import imap as _imap try: from thread import get_ident as _get_ident @@ -50,49 +51,45 @@ self.__map = {} self.__update(*args, **kwds) - def __setitem__(self, key, value, PREV=0, NEXT=1, dict_setitem=dict.__setitem__): + def __setitem__(self, key, value, dict_setitem=dict.__setitem__): 'od.__setitem__(i, y) <==> od[i]=y' # Setting a new item creates a new link at the end of the linked list, # and the inherited dictionary is updated with the new key/value pair. if key not in self: root = self.__root - last = root[PREV] - last[NEXT] = root[PREV] = self.__map[key] = [last, root, key] - dict_setitem(self, key, value) + last = root[0] + last[1] = root[0] = self.__map[key] = [last, root, key] + return dict_setitem(self, key, value) - def __delitem__(self, key, PREV=0, NEXT=1, dict_delitem=dict.__delitem__): + def __delitem__(self, key, dict_delitem=dict.__delitem__): 'od.__delitem__(y) <==> del od[y]' # Deleting an existing item uses self.__map to find the link which gets # removed by updating the links in the predecessor and successor nodes. dict_delitem(self, key) link_prev, link_next, key = self.__map.pop(key) - link_prev[NEXT] = link_next - link_next[PREV] = link_prev + link_prev[1] = link_next # update link_prev[NEXT] + link_next[0] = link_prev # update link_next[PREV] def __iter__(self): 'od.__iter__() <==> iter(od)' # Traverse the linked list in order. - NEXT, KEY = 1, 2 root = self.__root - curr = root[NEXT] + curr = root[1] # start at the first node while curr is not root: - yield curr[KEY] - curr = curr[NEXT] + yield curr[2] # yield the curr[KEY] + curr = curr[1] # move to next node def __reversed__(self): 'od.__reversed__() <==> reversed(od)' # Traverse the linked list in reverse order. - PREV, KEY = 0, 2 root = self.__root - curr = root[PREV] + curr = root[0] # start at the last node while curr is not root: - yield curr[KEY] - curr = curr[PREV] + yield curr[2] # yield the curr[KEY] + curr = curr[0] # move to previous node def clear(self): 'od.clear() -> None. Remove all items from od.' - for node in self.__map.itervalues(): - del node[:] root = self.__root root[:] = [root, root, None] self.__map.clear() @@ -208,7 +205,7 @@ ''' if isinstance(other, OrderedDict): - return len(self)==len(other) and self.items() == other.items() + return dict.__eq__(self, other) and all(_imap(_eq, self, other)) return dict.__eq__(self, other) def __ne__(self, other): @@ -234,10 +231,60 @@ ### namedtuple ################################################################################ +_class_template = '''\ +class {typename}(tuple): + '{typename}({arg_list})' + + __slots__ = () + + _fields = {field_names!r} + + def __new__(_cls, {arg_list}): + 'Create new instance of {typename}({arg_list})' + return _tuple.__new__(_cls, ({arg_list})) + + @classmethod + def _make(cls, iterable, new=tuple.__new__, len=len): + 'Make a new {typename} object from a sequence or iterable' + result = new(cls, iterable) + if len(result) != {num_fields:d}: + raise TypeError('Expected {num_fields:d} arguments, got %d' % len(result)) + return result + + def __repr__(self): + 'Return a nicely formatted representation string' + return '{typename}({repr_fmt})' % self + + def _asdict(self): + 'Return a new OrderedDict which maps field names to their values' + return OrderedDict(zip(self._fields, self)) + + __dict__ = property(_asdict) + + def _replace(_self, **kwds): + 'Return a new {typename} object replacing specified fields with new values' + result = _self._make(map(kwds.pop, {field_names!r}, _self)) + if kwds: + raise ValueError('Got unexpected field names: %r' % kwds.keys()) + return result + + def __getnewargs__(self): + 'Return self as a plain tuple. Used by copy and pickle.' + return tuple(self) + +{field_defs} +''' + +_repr_template = '{name}=%r' + +_field_template = '''\ + {name} = _property(_itemgetter({index:d}), doc='Alias for field number {index:d}') +''' + def namedtuple(typename, field_names, verbose=False, rename=False): """Returns a new subclass of tuple with named fields. - >>> Point = namedtuple('Point', 'x y') + >>> Point = namedtuple('Point', ['x', 'y']) >>> Point.__doc__ # docstring for the new class 'Point(x, y)' >>> p = Point(11, y=22) # instantiate with positional args or keywords @@ -258,83 +305,63 @@ """ - # Parse and validate the field names. Validation serves two purposes, - # generating informative error messages and preventing template injection attacks. + # Validate the field names. At the user's option, either generate an error + # message or automatically replace the field name with a valid name. if isinstance(field_names, basestring): - field_names = field_names.replace(',', ' ').split() # names separated by whitespace and/or commas - field_names = tuple(map(str, field_names)) + field_names = field_names.replace(',', ' ').split() + field_names = map(str, field_names) if rename: - names = list(field_names) seen = set() - for i, name in enumerate(names): - if (not all(c.isalnum() or c=='_' for c in name) or _iskeyword(name) - or not name or name[0].isdigit() or name.startswith('_') + for index, name in enumerate(field_names): + if (not all(c.isalnum() or c=='_' for c in name) + or _iskeyword(name) + or not name + or name[0].isdigit() + or name.startswith('_') or name in seen): - names[i] = '_%d' % i + field_names[index] = '_%d' % index seen.add(name) - field_names = tuple(names) - for name in (typename,) + field_names: + for name in [typename] + field_names: if not all(c.isalnum() or c=='_' for c in name): - raise ValueError('Type names and field names can only contain alphanumeric characters and underscores: %r' % name) + raise ValueError('Type names and field names can only contain ' + 'alphanumeric characters and underscores: %r' % name) if _iskeyword(name): - raise ValueError('Type names and field names cannot be a keyword: %r' % name) + raise ValueError('Type names and field names cannot be a ' + 'keyword: %r' % name) if name[0].isdigit(): - raise ValueError('Type names and field names cannot start with a number: %r' % name) - seen_names = set() + raise ValueError('Type names and field names cannot start with ' + 'a number: %r' % name) + seen = set() for name in field_names: if name.startswith('_') and not rename: - raise ValueError('Field names cannot start with an underscore: %r' % name) - if name in seen_names: + raise ValueError('Field names cannot start with an underscore: ' + '%r' % name) + if name in seen: raise ValueError('Encountered duplicate field name: %r' % name) - seen_names.add(name) + seen.add(name) - # Create and fill-in the class template - numfields = len(field_names) - argtxt = repr(field_names).replace("'", "")[1:-1] # tuple repr without parens or quotes - reprtxt = ', '.join('%s=%%r' % name for name in field_names) - template = '''class %(typename)s(tuple): - '%(typename)s(%(argtxt)s)' \n - __slots__ = () \n - _fields = %(field_names)r \n - def __new__(_cls, %(argtxt)s): - 'Create new instance of %(typename)s(%(argtxt)s)' - return _tuple.__new__(_cls, (%(argtxt)s)) \n - @classmethod - def _make(cls, iterable, new=tuple.__new__, len=len): - 'Make a new %(typename)s object from a sequence or iterable' - result = new(cls, iterable) - if len(result) != %(numfields)d: - raise TypeError('Expected %(numfields)d arguments, got %%d' %% len(result)) - return result \n - def __repr__(self): - 'Return a nicely formatted representation string' - return '%(typename)s(%(reprtxt)s)' %% self \n - def _asdict(self): - 'Return a new OrderedDict which maps field names to their values' - return OrderedDict(zip(self._fields, self)) \n - __dict__ = property(_asdict) \n - def _replace(_self, **kwds): - 'Return a new %(typename)s object replacing specified fields with new values' - result = _self._make(map(kwds.pop, %(field_names)r, _self)) - if kwds: - raise ValueError('Got unexpected field names: %%r' %% kwds.keys()) - return result \n - def __getnewargs__(self): - 'Return self as a plain tuple. Used by copy and pickle.' - return tuple(self) \n\n''' % locals() - for i, name in enumerate(field_names): - template += " %s = _property(_itemgetter(%d), doc='Alias for field number %d')\n" % (name, i, i) + # Fill-in the class template + class_definition = _class_template.format( + typename = typename, + field_names = tuple(field_names), + num_fields = len(field_names), + arg_list = repr(tuple(field_names)).replace("'", "")[1:-1], + repr_fmt = ', '.join(_repr_template.format(name=name) + for name in field_names), + field_defs = '\n'.join(_field_template.format(index=index, name=name) + for index, name in enumerate(field_names)) + ) if verbose: - print template + print class_definition - # Execute the template string in a temporary namespace and - # support tracing utilities by setting a value for frame.f_globals['__name__'] + # Execute the template string in a temporary namespace and support + # tracing utilities by setting a value for frame.f_globals['__name__'] namespace = dict(_itemgetter=_itemgetter, __name__='namedtuple_%s' % typename, OrderedDict=OrderedDict, _property=property, _tuple=tuple) try: - exec template in namespace - except SyntaxError, e: - raise SyntaxError(e.message + ':\n' + template) + exec class_definition in namespace + except SyntaxError as e: + raise SyntaxError(e.message + ':\n' + class_definition) result = namespace[typename] # For pickling to work, the __module__ variable needs to be set to the frame diff --git a/lib-python/2.7/compiler/consts.py b/lib-python/2.7/compiler/consts.py --- a/lib-python/2.7/compiler/consts.py +++ b/lib-python/2.7/compiler/consts.py @@ -5,7 +5,7 @@ SC_LOCAL = 1 SC_GLOBAL_IMPLICIT = 2 -SC_GLOBAL_EXPLICT = 3 +SC_GLOBAL_EXPLICIT = 3 SC_FREE = 4 SC_CELL = 5 SC_UNKNOWN = 6 diff --git a/lib-python/2.7/compiler/pycodegen.py b/lib-python/2.7/compiler/pycodegen.py --- a/lib-python/2.7/compiler/pycodegen.py +++ b/lib-python/2.7/compiler/pycodegen.py @@ -7,7 +7,7 @@ from compiler import ast, parse, walk, syntax from compiler import pyassem, misc, future, symbols -from compiler.consts import SC_LOCAL, SC_GLOBAL_IMPLICIT, SC_GLOBAL_EXPLICT, \ +from compiler.consts import SC_LOCAL, SC_GLOBAL_IMPLICIT, SC_GLOBAL_EXPLICIT, \ SC_FREE, SC_CELL from compiler.consts import (CO_VARARGS, CO_VARKEYWORDS, CO_NEWLOCALS, CO_NESTED, CO_GENERATOR, CO_FUTURE_DIVISION, @@ -283,7 +283,7 @@ self.emit(prefix + '_NAME', name) else: self.emit(prefix + '_FAST', name) - elif scope == SC_GLOBAL_EXPLICT: + elif scope == SC_GLOBAL_EXPLICIT: self.emit(prefix + '_GLOBAL', name) elif scope == SC_GLOBAL_IMPLICIT: if not self.optimized: diff --git a/lib-python/2.7/compiler/symbols.py b/lib-python/2.7/compiler/symbols.py --- a/lib-python/2.7/compiler/symbols.py +++ b/lib-python/2.7/compiler/symbols.py @@ -1,7 +1,7 @@ """Module symbol-table generator""" from compiler import ast -from compiler.consts import SC_LOCAL, SC_GLOBAL_IMPLICIT, SC_GLOBAL_EXPLICT, \ +from compiler.consts import SC_LOCAL, SC_GLOBAL_IMPLICIT, SC_GLOBAL_EXPLICIT, \ SC_FREE, SC_CELL, SC_UNKNOWN from compiler.misc import mangle import types @@ -90,7 +90,7 @@ The scope of a name could be LOCAL, GLOBAL, FREE, or CELL. """ if name in self.globals: - return SC_GLOBAL_EXPLICT + return SC_GLOBAL_EXPLICIT if name in self.cells: return SC_CELL if name in self.defs: diff --git a/lib-python/2.7/cookielib.py b/lib-python/2.7/cookielib.py --- a/lib-python/2.7/cookielib.py +++ b/lib-python/2.7/cookielib.py @@ -1,4 +1,4 @@ -"""HTTP cookie handling for web clients. +r"""HTTP cookie handling for web clients. This module has (now fairly distant) origins in Gisle Aas' Perl module HTTP::Cookies, from the libwww-perl library. diff --git a/lib-python/2.7/ctypes/test/test_bitfields.py b/lib-python/2.7/ctypes/test/test_bitfields.py --- a/lib-python/2.7/ctypes/test/test_bitfields.py +++ b/lib-python/2.7/ctypes/test/test_bitfields.py @@ -240,5 +240,25 @@ _anonymous_ = ["_"] _fields_ = [("_", X)] + @unittest.skipUnless(hasattr(ctypes, "c_uint32"), "c_int32 is required") + def test_uint32(self): + class X(Structure): + _fields_ = [("a", c_uint32, 32)] + x = X() + x.a = 10 + self.assertEqual(x.a, 10) + x.a = 0xFDCBA987 + self.assertEqual(x.a, 0xFDCBA987) + + @unittest.skipUnless(hasattr(ctypes, "c_uint64"), "c_int64 is required") + def test_uint64(self): + class X(Structure): + _fields_ = [("a", c_uint64, 64)] + x = X() + x.a = 10 + self.assertEqual(x.a, 10) + x.a = 0xFEDCBA9876543211 + self.assertEqual(x.a, 0xFEDCBA9876543211) + if __name__ == "__main__": unittest.main() diff --git a/lib-python/2.7/ctypes/test/test_numbers.py b/lib-python/2.7/ctypes/test/test_numbers.py --- a/lib-python/2.7/ctypes/test/test_numbers.py +++ b/lib-python/2.7/ctypes/test/test_numbers.py @@ -216,6 +216,16 @@ # probably be changed: self.assertRaises(TypeError, c_int, c_long(42)) + def test_float_overflow(self): + import sys + big_int = int(sys.float_info.max) * 2 + for t in float_types + [c_longdouble]: + self.assertRaises(OverflowError, t, big_int) + if (hasattr(t, "__ctype_be__")): + self.assertRaises(OverflowError, t.__ctype_be__, big_int) + if (hasattr(t, "__ctype_le__")): + self.assertRaises(OverflowError, t.__ctype_le__, big_int) + ## def test_perf(self): ## check_perf() diff --git a/lib-python/2.7/ctypes/test/test_structures.py b/lib-python/2.7/ctypes/test/test_structures.py --- a/lib-python/2.7/ctypes/test/test_structures.py +++ b/lib-python/2.7/ctypes/test/test_structures.py @@ -1,6 +1,7 @@ import unittest from ctypes import * from struct import calcsize +import _testcapi class SubclassesTest(unittest.TestCase): def test_subclass(self): @@ -199,6 +200,14 @@ "_pack_": -1} self.assertRaises(ValueError, type(Structure), "X", (Structure,), d) + # Issue 15989 + d = {"_fields_": [("a", c_byte)], + "_pack_": _testcapi.INT_MAX + 1} + self.assertRaises(ValueError, type(Structure), "X", (Structure,), d) + d = {"_fields_": [("a", c_byte)], + "_pack_": _testcapi.UINT_MAX + 2} + self.assertRaises(ValueError, type(Structure), "X", (Structure,), d) + def test_initializers(self): class Person(Structure): _fields_ = [("name", c_char*6), diff --git a/lib-python/2.7/ctypes/test/test_win32.py b/lib-python/2.7/ctypes/test/test_win32.py --- a/lib-python/2.7/ctypes/test/test_win32.py +++ b/lib-python/2.7/ctypes/test/test_win32.py @@ -3,6 +3,7 @@ from ctypes import * from ctypes.test import is_resource_enabled import unittest, sys +from test import test_support as support import _ctypes_test @@ -60,7 +61,9 @@ def test_COMError(self): from _ctypes import COMError - self.assertEqual(COMError.__doc__, "Raised when a COM method call failed.") + if support.HAVE_DOCSTRINGS: + self.assertEqual(COMError.__doc__, + "Raised when a COM method call failed.") ex = COMError(-1, "text", ("details",)) self.assertEqual(ex.hresult, -1) diff --git a/lib-python/2.7/curses/__init__.py b/lib-python/2.7/curses/__init__.py --- a/lib-python/2.7/curses/__init__.py +++ b/lib-python/2.7/curses/__init__.py @@ -5,7 +5,7 @@ import curses from curses import textpad - curses.initwin() + curses.initscr() ... """ diff --git a/lib-python/2.7/decimal.py b/lib-python/2.7/decimal.py --- a/lib-python/2.7/decimal.py +++ b/lib-python/2.7/decimal.py @@ -1581,7 +1581,13 @@ def __float__(self): """Float representation.""" - return float(str(self)) + if self._isnan(): + if self.is_snan(): + raise ValueError("Cannot convert signaling NaN to float") + s = "-nan" if self._sign else "nan" + else: + s = str(self) + return float(s) def __int__(self): """Converts self to an int, truncating if necessary.""" diff --git a/lib-python/2.7/distutils/command/bdist_rpm.py b/lib-python/2.7/distutils/command/bdist_rpm.py --- a/lib-python/2.7/distutils/command/bdist_rpm.py +++ b/lib-python/2.7/distutils/command/bdist_rpm.py @@ -379,16 +379,28 @@ self.spawn(rpm_cmd) if not self.dry_run: + if self.distribution.has_ext_modules(): + pyversion = get_python_version() + else: + pyversion = 'any' + if not self.binary_only: srpm = os.path.join(rpm_dir['SRPMS'], source_rpm) assert(os.path.exists(srpm)) self.move_file(srpm, self.dist_dir) + filename = os.path.join(self.dist_dir, source_rpm) + self.distribution.dist_files.append( + ('bdist_rpm', pyversion, filename)) if not self.source_only: for rpm in binary_rpms: rpm = os.path.join(rpm_dir['RPMS'], rpm) if os.path.exists(rpm): self.move_file(rpm, self.dist_dir) + filename = os.path.join(self.dist_dir, + os.path.basename(rpm)) + self.distribution.dist_files.append( + ('bdist_rpm', pyversion, filename)) # run() def _dist_path(self, path): diff --git a/lib-python/2.7/distutils/command/check.py b/lib-python/2.7/distutils/command/check.py --- a/lib-python/2.7/distutils/command/check.py +++ b/lib-python/2.7/distutils/command/check.py @@ -26,6 +26,9 @@ def system_message(self, level, message, *children, **kwargs): self.messages.append((level, message, children, kwargs)) + return nodes.system_message(message, level=level, + type=self.levels[level], + *children, **kwargs) HAS_DOCUTILS = True except ImportError: diff --git a/lib-python/2.7/distutils/config.py b/lib-python/2.7/distutils/config.py --- a/lib-python/2.7/distutils/config.py +++ b/lib-python/2.7/distutils/config.py @@ -42,16 +42,11 @@ def _store_pypirc(self, username, password): """Creates a default .pypirc file.""" rc = self._get_rc_file() - f = open(rc, 'w') + f = os.fdopen(os.open(rc, os.O_CREAT | os.O_WRONLY, 0600), 'w') try: f.write(DEFAULT_PYPIRC % (username, password)) finally: f.close() - try: - os.chmod(rc, 0600) - except OSError: - # should do something better here - pass def _read_pypirc(self): """Reads the .pypirc file.""" diff --git a/lib-python/2.7/distutils/dir_util.py b/lib-python/2.7/distutils/dir_util.py --- a/lib-python/2.7/distutils/dir_util.py +++ b/lib-python/2.7/distutils/dir_util.py @@ -144,6 +144,10 @@ src_name = os.path.join(src, n) dst_name = os.path.join(dst, n) + if n.startswith('.nfs'): + # skip NFS rename files + continue + if preserve_symlinks and os.path.islink(src_name): link_dest = os.readlink(src_name) if verbose >= 1: diff --git a/lib-python/2.7/distutils/tests/test_bdist_dumb.py b/lib-python/2.7/distutils/tests/test_bdist_dumb.py --- a/lib-python/2.7/distutils/tests/test_bdist_dumb.py +++ b/lib-python/2.7/distutils/tests/test_bdist_dumb.py @@ -1,8 +1,10 @@ """Tests for distutils.command.bdist_dumb.""" +import os +import sys +import zipfile import unittest -import sys -import os +from test.test_support import run_unittest # zlib is not used here, but if it's not available # test_simple_built will fail @@ -11,8 +13,6 @@ except ImportError: zlib = None -from test.test_support import run_unittest - from distutils.core import Distribution from distutils.command.bdist_dumb import bdist_dumb from distutils.tests import support @@ -73,15 +73,23 @@ # see what we have dist_created = os.listdir(os.path.join(pkg_dir, 'dist')) - base = "%s.%s" % (dist.get_fullname(), cmd.plat_name) + base = "%s.%s.zip" % (dist.get_fullname(), cmd.plat_name) if os.name == 'os2': base = base.replace(':', '-') - wanted = ['%s.zip' % base] - self.assertEqual(dist_created, wanted) + self.assertEqual(dist_created, [base]) # now let's check what we have in the zip file - # XXX to be done + fp = zipfile.ZipFile(os.path.join('dist', base)) + try: + contents = fp.namelist() + finally: + fp.close() + + contents = sorted(os.path.basename(fn) for fn in contents) + wanted = ['foo-0.1-py%s.%s.egg-info' % sys.version_info[:2], + 'foo.py', 'foo.pyc'] + self.assertEqual(contents, sorted(wanted)) def test_finalize_options(self): pkg_dir, dist = self.create_dist() diff --git a/lib-python/2.7/distutils/tests/test_bdist_msi.py b/lib-python/2.7/distutils/tests/test_bdist_msi.py --- a/lib-python/2.7/distutils/tests/test_bdist_msi.py +++ b/lib-python/2.7/distutils/tests/test_bdist_msi.py @@ -1,12 +1,11 @@ """Tests for distutils.command.bdist_msi.""" +import sys import unittest -import sys - from test.test_support import run_unittest - from distutils.tests import support - at unittest.skipUnless(sys.platform=="win32", "These tests are only for win32") + + at unittest.skipUnless(sys.platform == 'win32', 'these tests require Windows') class BDistMSITestCase(support.TempdirManager, support.LoggingSilencer, unittest.TestCase): @@ -14,10 +13,11 @@ def test_minimal(self): # minimal test XXX need more tests from distutils.command.bdist_msi import bdist_msi - pkg_pth, dist = self.create_dist() + project_dir, dist = self.create_dist() cmd = bdist_msi(dist) cmd.ensure_finalized() + def test_suite(): return unittest.makeSuite(BDistMSITestCase) diff --git a/lib-python/2.7/distutils/tests/test_bdist_rpm.py b/lib-python/2.7/distutils/tests/test_bdist_rpm.py --- a/lib-python/2.7/distutils/tests/test_bdist_rpm.py +++ b/lib-python/2.7/distutils/tests/test_bdist_rpm.py @@ -79,6 +79,10 @@ dist_created = os.listdir(os.path.join(pkg_dir, 'dist')) self.assertTrue('foo-0.1-1.noarch.rpm' in dist_created) + # bug #2945: upload ignores bdist_rpm files + self.assertIn(('bdist_rpm', 'any', 'dist/foo-0.1-1.src.rpm'), dist.dist_files) + self.assertIn(('bdist_rpm', 'any', 'dist/foo-0.1-1.noarch.rpm'), dist.dist_files) + def test_no_optimize_flag(self): # XXX I am unable yet to make this test work without @@ -118,6 +122,11 @@ dist_created = os.listdir(os.path.join(pkg_dir, 'dist')) self.assertTrue('foo-0.1-1.noarch.rpm' in dist_created) + + # bug #2945: upload ignores bdist_rpm files + self.assertIn(('bdist_rpm', 'any', 'dist/foo-0.1-1.src.rpm'), dist.dist_files) + self.assertIn(('bdist_rpm', 'any', 'dist/foo-0.1-1.noarch.rpm'), dist.dist_files) + os.remove(os.path.join(pkg_dir, 'dist', 'foo-0.1-1.noarch.rpm')) def test_suite(): diff --git a/lib-python/2.7/distutils/tests/test_build_ext.py b/lib-python/2.7/distutils/tests/test_build_ext.py --- a/lib-python/2.7/distutils/tests/test_build_ext.py +++ b/lib-python/2.7/distutils/tests/test_build_ext.py @@ -77,8 +77,9 @@ self.assertEqual(xx.foo(2, 5), 7) self.assertEqual(xx.foo(13,15), 28) self.assertEqual(xx.new().demo(), None) - doc = 'This is a template module just for instruction.' - self.assertEqual(xx.__doc__, doc) + if test_support.HAVE_DOCSTRINGS: + doc = 'This is a template module just for instruction.' + self.assertEqual(xx.__doc__, doc) self.assertTrue(isinstance(xx.Null(), xx.Null)) self.assertTrue(isinstance(xx.Str(), xx.Str)) diff --git a/lib-python/2.7/distutils/tests/test_dir_util.py b/lib-python/2.7/distutils/tests/test_dir_util.py --- a/lib-python/2.7/distutils/tests/test_dir_util.py +++ b/lib-python/2.7/distutils/tests/test_dir_util.py @@ -101,6 +101,24 @@ remove_tree(self.root_target, verbose=0) remove_tree(self.target2, verbose=0) + def test_copy_tree_skips_nfs_temp_files(self): + mkpath(self.target, verbose=0) + + a_file = os.path.join(self.target, 'ok.txt') + nfs_file = os.path.join(self.target, '.nfs123abc') + for f in a_file, nfs_file: + fh = open(f, 'w') + try: + fh.write('some content') + finally: + fh.close() + + copy_tree(self.target, self.target2) + self.assertEqual(os.listdir(self.target2), ['ok.txt']) + + remove_tree(self.root_target, verbose=0) + remove_tree(self.target2, verbose=0) + def test_ensure_relative(self): if os.sep == '/': self.assertEqual(ensure_relative('/home/foo'), 'home/foo') diff --git a/lib-python/2.7/distutils/tests/test_install.py b/lib-python/2.7/distutils/tests/test_install.py --- a/lib-python/2.7/distutils/tests/test_install.py +++ b/lib-python/2.7/distutils/tests/test_install.py @@ -86,19 +86,17 @@ self.old_expand = os.path.expanduser os.path.expanduser = _expanduser - try: - # this is the actual test - self._test_user_site() - finally: + def cleanup(): site.USER_BASE = self.old_user_base site.USER_SITE = self.old_user_site install_module.USER_BASE = self.old_user_base install_module.USER_SITE = self.old_user_site os.path.expanduser = self.old_expand - def _test_user_site(self): + self.addCleanup(cleanup) + for key in ('nt_user', 'unix_user', 'os2_home'): - self.assertTrue(key in INSTALL_SCHEMES) + self.assertIn(key, INSTALL_SCHEMES) dist = Distribution({'name': 'xx'}) cmd = install(dist) @@ -106,14 +104,14 @@ # making sure the user option is there options = [name for name, short, lable in cmd.user_options] - self.assertTrue('user' in options) + self.assertIn('user', options) # setting a value cmd.user = 1 # user base and site shouldn't be created yet - self.assertTrue(not os.path.exists(self.user_base)) - self.assertTrue(not os.path.exists(self.user_site)) + self.assertFalse(os.path.exists(self.user_base)) + self.assertFalse(os.path.exists(self.user_site)) # let's run finalize cmd.ensure_finalized() @@ -122,8 +120,8 @@ self.assertTrue(os.path.exists(self.user_base)) self.assertTrue(os.path.exists(self.user_site)) - self.assertTrue('userbase' in cmd.config_vars) - self.assertTrue('usersite' in cmd.config_vars) + self.assertIn('userbase', cmd.config_vars) + self.assertIn('usersite', cmd.config_vars) def test_handle_extra_path(self): dist = Distribution({'name': 'xx', 'extra_path': 'path,dirs'}) @@ -176,15 +174,16 @@ def test_record(self): install_dir = self.mkdtemp() - project_dir, dist = self.create_dist(scripts=['hello']) - self.addCleanup(os.chdir, os.getcwd()) + project_dir, dist = self.create_dist(py_modules=['hello'], + scripts=['sayhi']) os.chdir(project_dir) - self.write_file('hello', "print('o hai')") + self.write_file('hello.py', "def main(): print 'o hai'") + self.write_file('sayhi', 'from hello import main; main()') cmd = install(dist) dist.command_obj['install'] = cmd cmd.root = install_dir - cmd.record = os.path.join(project_dir, 'RECORD') + cmd.record = os.path.join(project_dir, 'filelist') cmd.ensure_finalized() cmd.run() @@ -195,7 +194,7 @@ f.close() found = [os.path.basename(line) for line in content.splitlines()] - expected = ['hello', + expected = ['hello.py', 'hello.pyc', 'sayhi', 'UNKNOWN-0.0.0-py%s.%s.egg-info' % sys.version_info[:2]] self.assertEqual(found, expected) @@ -203,7 +202,6 @@ install_dir = self.mkdtemp() project_dir, dist = self.create_dist(ext_modules=[ Extension('xx', ['xxmodule.c'])]) - self.addCleanup(os.chdir, os.getcwd()) os.chdir(project_dir) support.copy_xxmodule_c(project_dir) @@ -215,7 +213,7 @@ dist.command_obj['install'] = cmd dist.command_obj['build_ext'] = buildextcmd cmd.root = install_dir - cmd.record = os.path.join(project_dir, 'RECORD') + cmd.record = os.path.join(project_dir, 'filelist') cmd.ensure_finalized() cmd.run() @@ -241,6 +239,7 @@ install_module.DEBUG = False self.assertTrue(len(self.logs) > old_logs_len) + def test_suite(): return unittest.makeSuite(InstallTestCase) diff --git a/lib-python/2.7/distutils/tests/test_msvc9compiler.py b/lib-python/2.7/distutils/tests/test_msvc9compiler.py --- a/lib-python/2.7/distutils/tests/test_msvc9compiler.py +++ b/lib-python/2.7/distutils/tests/test_msvc9compiler.py @@ -104,7 +104,7 @@ unittest.TestCase): def test_no_compiler(self): - # makes sure query_vcvarsall throws + # makes sure query_vcvarsall raises # a DistutilsPlatformError if the compiler # is not found from distutils.msvc9compiler import query_vcvarsall diff --git a/lib-python/2.7/distutils/tests/test_register.py b/lib-python/2.7/distutils/tests/test_register.py --- a/lib-python/2.7/distutils/tests/test_register.py +++ b/lib-python/2.7/distutils/tests/test_register.py @@ -1,6 +1,5 @@ # -*- encoding: utf8 -*- """Tests for distutils.command.register.""" -import sys import os import unittest import getpass @@ -11,11 +10,14 @@ from distutils.command import register as register_module from distutils.command.register import register -from distutils.core import Distribution from distutils.errors import DistutilsSetupError -from distutils.tests import support -from distutils.tests.test_config import PYPIRC, PyPIRCCommandTestCase +from distutils.tests.test_config import PyPIRCCommandTestCase + +try: + import docutils +except ImportError: + docutils = None PYPIRC_NOPASSWORD = """\ [distutils] @@ -192,6 +194,7 @@ self.assertEqual(headers['Content-length'], '290') self.assertTrue('tarek' in req.data) + @unittest.skipUnless(docutils is not None, 'needs docutils') def test_strict(self): # testing the script option # when on, the register command stops if @@ -204,13 +207,6 @@ cmd.strict = 1 self.assertRaises(DistutilsSetupError, cmd.run) - # we don't test the reSt feature if docutils - # is not installed - try: - import docutils - except ImportError: - return - # metadata are OK but long_description is broken metadata = {'url': 'xxx', 'author': 'xxx', 'author_email': u'éxéxé', @@ -264,6 +260,21 @@ finally: del register_module.raw_input + @unittest.skipUnless(docutils is not None, 'needs docutils') + def test_register_invalid_long_description(self): + description = ':funkie:`str`' # mimic Sphinx-specific markup + metadata = {'url': 'xxx', 'author': 'xxx', + 'author_email': 'xxx', + 'name': 'xxx', 'version': 'xxx', + 'long_description': description} + cmd = self._get_cmd(metadata) + cmd.ensure_finalized() + cmd.strict = True + inputs = RawInputs('2', 'tarek', 'tarek at ziade.org') + register_module.raw_input = inputs + self.addCleanup(delattr, register_module, 'raw_input') + self.assertRaises(DistutilsSetupError, cmd.run) + def test_check_metadata_deprecated(self): # makes sure make_metadata is deprecated cmd = self._get_cmd() diff --git a/lib-python/2.7/distutils/tests/test_sdist.py b/lib-python/2.7/distutils/tests/test_sdist.py --- a/lib-python/2.7/distutils/tests/test_sdist.py +++ b/lib-python/2.7/distutils/tests/test_sdist.py @@ -6,6 +6,7 @@ import zipfile from os.path import join from textwrap import dedent +from test.test_support import captured_stdout, check_warnings, run_unittest # zlib is not used here, but if it's not available # the tests that use zipfile may fail @@ -21,7 +22,6 @@ except ImportError: UID_GID_SUPPORT = False -from test.test_support import captured_stdout, check_warnings, run_unittest from distutils.command.sdist import sdist, show_formats from distutils.core import Distribution @@ -91,9 +91,8 @@ @unittest.skipUnless(zlib, "requires zlib") def test_prune_file_list(self): - # this test creates a package with some vcs dirs in it - # and launch sdist to make sure they get pruned - # on all systems + # this test creates a project with some VCS dirs and an NFS rename + # file, then launches sdist to check they get pruned on all systems # creating VCS directories with some files in them os.mkdir(join(self.tmp_dir, 'somecode', '.svn')) @@ -107,6 +106,8 @@ self.write_file((self.tmp_dir, 'somecode', '.git', 'ok'), 'xxx') + self.write_file((self.tmp_dir, 'somecode', '.nfs0001'), 'xxx') + # now building a sdist dist, cmd = self.get_cmd() @@ -375,7 +376,7 @@ # the following tests make sure there is a nice error message instead # of a traceback when parsing an invalid manifest template - def _test_template(self, content): + def _check_template(self, content): dist, cmd = self.get_cmd() os.chdir(self.tmp_dir) self.write_file('MANIFEST.in', content) @@ -386,17 +387,17 @@ self.assertEqual(len(warnings), 1) def test_invalid_template_unknown_command(self): - self._test_template('taunt knights *') + self._check_template('taunt knights *') def test_invalid_template_wrong_arguments(self): # this manifest command takes one argument - self._test_template('prune') + self._check_template('prune') @unittest.skipIf(os.name != 'nt', 'test relevant for Windows only') def test_invalid_template_wrong_path(self): # on Windows, trailing slashes are not allowed # this used to crash instead of raising a warning: #8286 - self._test_template('include examples/') + self._check_template('include examples/') @unittest.skipUnless(zlib, "requires zlib") def test_get_file_list(self): diff --git a/lib-python/2.7/doctest.py b/lib-python/2.7/doctest.py --- a/lib-python/2.7/doctest.py +++ b/lib-python/2.7/doctest.py @@ -2314,7 +2314,8 @@ return "Doctest: " + self._dt_test.name class SkipDocTestCase(DocTestCase): - def __init__(self): + def __init__(self, module): + self.module = module DocTestCase.__init__(self, None) def setUp(self): @@ -2324,7 +2325,10 @@ pass def shortDescription(self): - return "Skipping tests from %s" % module.__name__ + return "Skipping tests from %s" % self.module.__name__ + + __str__ = shortDescription + def DocTestSuite(module=None, globs=None, extraglobs=None, test_finder=None, **options): @@ -2372,12 +2376,17 @@ if not tests and sys.flags.optimize >=2: # Skip doctests when running with -O2 suite = unittest.TestSuite() - suite.addTest(SkipDocTestCase()) + suite.addTest(SkipDocTestCase(module)) return suite elif not tests: # Why do we want to do this? Because it reveals a bug that might # otherwise be hidden. - raise ValueError(module, "has no tests") + # It is probably a bug that this exception is not also raised if the + # number of doctest examples in tests is zero (i.e. if no doctest + # examples were found). However, we should probably not be raising + # an exception at all here, though it is too late to make this change + # for a maintenance release. See also issue #14649. + raise ValueError(module, "has no docstrings") tests.sort() suite = unittest.TestSuite() diff --git a/lib-python/2.7/email/_parseaddr.py b/lib-python/2.7/email/_parseaddr.py --- a/lib-python/2.7/email/_parseaddr.py +++ b/lib-python/2.7/email/_parseaddr.py @@ -13,7 +13,7 @@ 'quote', ] -import time +import time, calendar SPACE = ' ' EMPTYSTRING = '' @@ -150,13 +150,13 @@ def mktime_tz(data): - """Turn a 10-tuple as returned by parsedate_tz() into a UTC timestamp.""" + """Turn a 10-tuple as returned by parsedate_tz() into a POSIX timestamp.""" if data[9] is None: # No zone info, so localtime is better assumption than GMT return time.mktime(data[:8] + (-1,)) else: - t = time.mktime(data[:8] + (0,)) - return t - data[9] - time.timezone + t = calendar.timegm(data) + return t - data[9] def quote(str): diff --git a/lib-python/2.7/email/base64mime.py b/lib-python/2.7/email/base64mime.py --- a/lib-python/2.7/email/base64mime.py +++ b/lib-python/2.7/email/base64mime.py @@ -130,7 +130,7 @@ verbatim (this is the default). Each line of encoded text will end with eol, which defaults to "\\n". Set - this to "\r\n" if you will be using the result of this function directly + this to "\\r\\n" if you will be using the result of this function directly in an email. """ if not s: diff --git a/lib-python/2.7/email/feedparser.py b/lib-python/2.7/email/feedparser.py --- a/lib-python/2.7/email/feedparser.py +++ b/lib-python/2.7/email/feedparser.py @@ -13,7 +13,7 @@ data. When you have no more data to push into the parser, call .close(). This completes the parsing and returns the root message object. -The other advantage of this parser is that it will never throw a parsing +The other advantage of this parser is that it will never raise a parsing exception. Instead, when it finds something unexpected, it adds a 'defect' to the current message. Defects are just instances that live on the message object's .defects attribute. @@ -214,7 +214,7 @@ # supposed to see in the body of the message. self._parse_headers(headers) # Headers-only parsing is a backwards compatibility hack, which was - # necessary in the older parser, which could throw errors. All + # necessary in the older parser, which could raise errors. All # remaining lines in the input are thrown into the message body. if self._headersonly: lines = [] diff --git a/lib-python/2.7/email/generator.py b/lib-python/2.7/email/generator.py --- a/lib-python/2.7/email/generator.py +++ b/lib-python/2.7/email/generator.py @@ -212,7 +212,11 @@ msg.set_boundary(boundary) # If there's a preamble, write it out, with a trailing CRLF if msg.preamble is not None: - print >> self._fp, msg.preamble + if self._mangle_from_: + preamble = fcre.sub('>From ', msg.preamble) + else: + preamble = msg.preamble + print >> self._fp, preamble # dash-boundary transport-padding CRLF print >> self._fp, '--' + boundary # body-part @@ -230,7 +234,11 @@ self._fp.write('\n--' + boundary + '--') if msg.epilogue is not None: print >> self._fp - self._fp.write(msg.epilogue) + if self._mangle_from_: + epilogue = fcre.sub('>From ', msg.epilogue) + else: + epilogue = msg.epilogue + self._fp.write(epilogue) def _handle_multipart_signed(self, msg): # The contents of signed parts has to stay unmodified in order to keep diff --git a/lib-python/2.7/email/test/test_email.py b/lib-python/2.7/email/test/test_email.py --- a/lib-python/2.7/email/test/test_email.py +++ b/lib-python/2.7/email/test/test_email.py @@ -9,6 +9,7 @@ import difflib import unittest import warnings +import textwrap from cStringIO import StringIO import email @@ -948,6 +949,28 @@ Blah blah blah """) + def test_mangle_from_in_preamble_and_epilog(self): + s = StringIO() + g = Generator(s, mangle_from_=True) + msg = email.message_from_string(textwrap.dedent("""\ + From: foo at bar.com + Mime-Version: 1.0 + Content-Type: multipart/mixed; boundary=XXX + + From somewhere unknown + + --XXX + Content-Type: text/plain + + foo + + --XXX-- + + From somewhere unknowable + """)) + g.flatten(msg) + self.assertEqual(len([1 for x in s.getvalue().split('\n') + if x.startswith('>From ')]), 2) # Test the basic MIMEAudio class @@ -2262,6 +2285,12 @@ eq(time.localtime(t)[:6], timetup[:6]) eq(int(time.strftime('%Y', timetup[:9])), 2003) + def test_mktime_tz(self): + self.assertEqual(Utils.mktime_tz((1970, 1, 1, 0, 0, 0, + -1, -1, -1, 0)), 0) + self.assertEqual(Utils.mktime_tz((1970, 1, 1, 0, 0, 0, + -1, -1, -1, 1234)), -1234) + def test_parsedate_y2k(self): """Test for parsing a date with a two-digit year. diff --git a/lib-python/2.7/email/utils.py b/lib-python/2.7/email/utils.py --- a/lib-python/2.7/email/utils.py +++ b/lib-python/2.7/email/utils.py @@ -63,7 +63,7 @@ """Decodes a base64 string. This function is equivalent to base64.decodestring and it's retained only - for backward compatibility. It used to remove the last \n of the decoded + for backward compatibility. It used to remove the last \\n of the decoded string, if it had any (see issue 7143). """ if not s: @@ -73,7 +73,7 @@ def fix_eols(s): - """Replace all line-ending characters with \r\n.""" + """Replace all line-ending characters with \\r\\n.""" # Fix newlines with no preceding carriage return s = re.sub(r'(? Author: Brian Kearns Branch: stdlib-2.7.4 Changeset: r60667:2f22446ff07e Date: 2013-01-29 06:57 -0500 http://bitbucket.org/pypy/pypy/changeset/2f22446ff07e/ Log: branch to start merging/testing what will become stdlib-2.7.4 From noreply at buildbot.pypy.org Tue Jan 29 13:14:35 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 29 Jan 2013 13:14:35 +0100 (CET) Subject: [pypy-commit] pypy stdlib-2.7.4: merge stdlib 2.7.3+ Message-ID: <20130129121435.4D47C1C13C4@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: stdlib-2.7.4 Changeset: r60668:075ffb98394d Date: 2013-01-29 07:08 -0500 http://bitbucket.org/pypy/pypy/changeset/075ffb98394d/ Log: merge stdlib 2.7.3+ diff too long, truncating to 2000 out of 17108 lines diff --git a/lib-python/2.7/BaseHTTPServer.py b/lib-python/2.7/BaseHTTPServer.py --- a/lib-python/2.7/BaseHTTPServer.py +++ b/lib-python/2.7/BaseHTTPServer.py @@ -447,13 +447,13 @@ specified as subsequent arguments (it's just like printf!). - The client host and current date/time are prefixed to - every message. + The client ip address and current date/time are prefixed to every + message. """ sys.stderr.write("%s - - [%s] %s\n" % - (self.address_string(), + (self.client_address[0], self.log_date_time_string(), format%args)) diff --git a/lib-python/2.7/CGIHTTPServer.py b/lib-python/2.7/CGIHTTPServer.py --- a/lib-python/2.7/CGIHTTPServer.py +++ b/lib-python/2.7/CGIHTTPServer.py @@ -84,9 +84,11 @@ path begins with one of the strings in self.cgi_directories (and the next character is a '/' or the end of the string). """ - splitpath = _url_collapse_path_split(self.path) - if splitpath[0] in self.cgi_directories: - self.cgi_info = splitpath + collapsed_path = _url_collapse_path(self.path) + dir_sep = collapsed_path.find('/', 1) + head, tail = collapsed_path[:dir_sep], collapsed_path[dir_sep+1:] + if head in self.cgi_directories: + self.cgi_info = head, tail return True return False @@ -298,44 +300,46 @@ self.log_message("CGI script exited OK") -# TODO(gregory.p.smith): Move this into an appropriate library. -def _url_collapse_path_split(path): +def _url_collapse_path(path): """ Given a URL path, remove extra '/'s and '.' path elements and collapse - any '..' references. + any '..' references and returns a colllapsed path. Implements something akin to RFC-2396 5.2 step 6 to parse relative paths. + The utility of this function is limited to is_cgi method and helps + preventing some security attacks. Returns: A tuple of (head, tail) where tail is everything after the final / and head is everything before it. Head will always start with a '/' and, if it contains anything else, never have a trailing '/'. Raises: IndexError if too many '..' occur within the path. + """ # Similar to os.path.split(os.path.normpath(path)) but specific to URL # path semantics rather than local operating system semantics. - path_parts = [] - for part in path.split('/'): - if part == '.': - path_parts.append('') - else: - path_parts.append(part) - # Filter out blank non trailing parts before consuming the '..'. - path_parts = [part for part in path_parts[:-1] if part] + path_parts[-1:] + path_parts = path.split('/') + head_parts = [] + for part in path_parts[:-1]: + if part == '..': + head_parts.pop() # IndexError if more '..' than prior parts + elif part and part != '.': + head_parts.append( part ) if path_parts: tail_part = path_parts.pop() + if tail_part: + if tail_part == '..': + head_parts.pop() + tail_part = '' + elif tail_part == '.': + tail_part = '' else: tail_part = '' - head_parts = [] - for part in path_parts: - if part == '..': - head_parts.pop() - else: - head_parts.append(part) - if tail_part and tail_part == '..': - head_parts.pop() - tail_part = '' - return ('/' + '/'.join(head_parts), tail_part) + + splitpath = ('/' + '/'.join(head_parts), tail_part) + collapsed_path = "/".join(splitpath) + + return collapsed_path nobody = None diff --git a/lib-python/2.7/Cookie.py b/lib-python/2.7/Cookie.py --- a/lib-python/2.7/Cookie.py +++ b/lib-python/2.7/Cookie.py @@ -390,7 +390,7 @@ from time import gmtime, time now = time() year, month, day, hh, mm, ss, wd, y, z = gmtime(now + future) - return "%s, %02d-%3s-%4d %02d:%02d:%02d GMT" % \ + return "%s, %02d %3s %4d %02d:%02d:%02d GMT" % \ (weekdayname[wd], day, monthname[month], year, hh, mm, ss) @@ -539,7 +539,7 @@ r"(?P" # Start of group 'val' r'"(?:[^\\"]|\\.)*"' # Any doublequoted string r"|" # or - r"\w{3},\s[\w\d-]{9,11}\s[\d:]{8}\sGMT" # Special case for "expires" attr + r"\w{3},\s[\s\w\d-]{9,11}\s[\d:]{8}\sGMT" # Special case for "expires" attr r"|" # or ""+ _LegalCharsPatt +"*" # Any word or empty string r")" # End of group 'val' diff --git a/lib-python/2.7/HTMLParser.py b/lib-python/2.7/HTMLParser.py --- a/lib-python/2.7/HTMLParser.py +++ b/lib-python/2.7/HTMLParser.py @@ -22,13 +22,13 @@ starttagopen = re.compile('<[a-zA-Z]') piclose = re.compile('>') commentclose = re.compile(r'--\s*>') -tagfind = re.compile('[a-zA-Z][-.a-zA-Z0-9:_]*') +tagfind = re.compile('([a-zA-Z][-.a-zA-Z0-9:_]*)(?:\s|/(?!>))*') # see http://www.w3.org/TR/html5/tokenization.html#tag-open-state # and http://www.w3.org/TR/html5/tokenization.html#tag-name-state tagfind_tolerant = re.compile('[a-zA-Z][^\t\n\r\f />\x00]*') attrfind = re.compile( - r'[\s/]*((?<=[\'"\s/])[^\s/>][^\s/=>]*)(\s*=+\s*' + r'((?<=[\'"\s/])[^\s/>][^\s/=>]*)(\s*=+\s*' r'(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?(?:\s|/(?!>))*') locatestarttagend = re.compile(r""" @@ -289,7 +289,7 @@ match = tagfind.match(rawdata, i+1) assert match, 'unexpected call to parse_starttag()' k = match.end() - self.lasttag = tag = rawdata[i+1:k].lower() + self.lasttag = tag = match.group(1).lower() while k < endpos: m = attrfind.match(rawdata, k) diff --git a/lib-python/2.7/SimpleXMLRPCServer.py b/lib-python/2.7/SimpleXMLRPCServer.py --- a/lib-python/2.7/SimpleXMLRPCServer.py +++ b/lib-python/2.7/SimpleXMLRPCServer.py @@ -1,4 +1,4 @@ -"""Simple XML-RPC Server. +r"""Simple XML-RPC Server. This module can be used to create simple XML-RPC servers by creating a server and either installing functions, a diff --git a/lib-python/2.7/SocketServer.py b/lib-python/2.7/SocketServer.py --- a/lib-python/2.7/SocketServer.py +++ b/lib-python/2.7/SocketServer.py @@ -133,6 +133,7 @@ import select import sys import os +import errno try: import threading except ImportError: @@ -147,6 +148,15 @@ "ThreadingUnixStreamServer", "ThreadingUnixDatagramServer"]) +def _eintr_retry(func, *args): + """restart a system call interrupted by EINTR""" + while True: + try: + return func(*args) + except (OSError, select.error) as e: + if e.args[0] != errno.EINTR: + raise + class BaseServer: """Base class for server classes. @@ -222,7 +232,8 @@ # connecting to the socket to wake this up instead of # polling. Polling reduces our responsiveness to a # shutdown request and wastes cpu at all other times. - r, w, e = select.select([self], [], [], poll_interval) + r, w, e = _eintr_retry(select.select, [self], [], [], + poll_interval) if self in r: self._handle_request_noblock() finally: @@ -262,7 +273,7 @@ timeout = self.timeout elif self.timeout is not None: timeout = min(timeout, self.timeout) - fd_sets = select.select([self], [], [], timeout) + fd_sets = _eintr_retry(select.select, [self], [], [], timeout) if not fd_sets[0]: self.handle_timeout() return @@ -690,7 +701,12 @@ def finish(self): if not self.wfile.closed: - self.wfile.flush() + try: + self.wfile.flush() + except socket.error: + # An final socket error may have occurred here, such as + # the local error ECONNABORTED. + pass self.wfile.close() self.rfile.close() diff --git a/lib-python/2.7/_LWPCookieJar.py b/lib-python/2.7/_LWPCookieJar.py --- a/lib-python/2.7/_LWPCookieJar.py +++ b/lib-python/2.7/_LWPCookieJar.py @@ -48,7 +48,7 @@ class LWPCookieJar(FileCookieJar): """ - The LWPCookieJar saves a sequence of"Set-Cookie3" lines. + The LWPCookieJar saves a sequence of "Set-Cookie3" lines. "Set-Cookie3" is the format used by the libwww-perl libary, not known to be compatible with any browser, but which is easy to read and doesn't lose information about RFC 2965 cookies. @@ -60,7 +60,7 @@ """ def as_lwp_str(self, ignore_discard=True, ignore_expires=True): - """Return cookies as a string of "\n"-separated "Set-Cookie3" headers. + """Return cookies as a string of "\\n"-separated "Set-Cookie3" headers. ignore_discard and ignore_expires: see docstring for FileCookieJar.save diff --git a/lib-python/2.7/__future__.py b/lib-python/2.7/__future__.py --- a/lib-python/2.7/__future__.py +++ b/lib-python/2.7/__future__.py @@ -112,7 +112,7 @@ CO_FUTURE_DIVISION) absolute_import = _Feature((2, 5, 0, "alpha", 1), - (2, 7, 0, "alpha", 0), + (3, 0, 0, "alpha", 0), CO_FUTURE_ABSOLUTE_IMPORT) with_statement = _Feature((2, 5, 0, "alpha", 1), diff --git a/lib-python/2.7/_pyio.py b/lib-python/2.7/_pyio.py --- a/lib-python/2.7/_pyio.py +++ b/lib-python/2.7/_pyio.py @@ -340,8 +340,10 @@ This method has no effect if the file is already closed. """ if not self.__closed: - self.flush() - self.__closed = True + try: + self.flush() + finally: + self.__closed = True def __del__(self): """Destructor. Calls close().""" @@ -883,12 +885,18 @@ return pos def readable(self): + if self.closed: + raise ValueError("I/O operation on closed file.") return True def writable(self): + if self.closed: + raise ValueError("I/O operation on closed file.") return True def seekable(self): + if self.closed: + raise ValueError("I/O operation on closed file.") return True @@ -1451,7 +1459,7 @@ enabled. With this enabled, on input, the lines endings '\n', '\r', or '\r\n' are translated to '\n' before being returned to the caller. Conversely, on output, '\n' is translated to the system - default line seperator, os.linesep. If newline is any other of its + default line separator, os.linesep. If newline is any other of its legal values, that newline becomes the newline when the file is read and it is returned untranslated. On output, '\n' is converted to the newline. @@ -1546,6 +1554,8 @@ return self._buffer def seekable(self): + if self.closed: + raise ValueError("I/O operation on closed file.") return self._seekable def readable(self): @@ -1560,8 +1570,10 @@ def close(self): if self.buffer is not None and not self.closed: - self.flush() - self.buffer.close() + try: + self.flush() + finally: + self.buffer.close() @property def closed(self): diff --git a/lib-python/2.7/_strptime.py b/lib-python/2.7/_strptime.py --- a/lib-python/2.7/_strptime.py +++ b/lib-python/2.7/_strptime.py @@ -326,7 +326,8 @@ if len(data_string) != found.end(): raise ValueError("unconverted data remains: %s" % data_string[found.end():]) - year = 1900 + + year = None month = day = 1 hour = minute = second = fraction = 0 tz = -1 @@ -425,6 +426,12 @@ else: tz = value break + leap_year_fix = False + if year is None and month == 2 and day == 29: + year = 1904 # 1904 is first leap year of 20th century + leap_year_fix = True + elif year is None: + year = 1900 # If we know the week of the year and what day of that week, we can figure # out the Julian day of the year. if julian == -1 and week_of_year != -1 and weekday != -1: @@ -446,6 +453,12 @@ day = datetime_result.day if weekday == -1: weekday = datetime_date(year, month, day).weekday() + if leap_year_fix: + # the caller didn't supply a year but asked for Feb 29th. We couldn't + # use the default of 1900 for computations. We set it back to ensure + # that February 29th is smaller than March 1st. + year = 1900 + return (time.struct_time((year, month, day, hour, minute, second, weekday, julian, tz)), fraction) diff --git a/lib-python/2.7/_weakrefset.py b/lib-python/2.7/_weakrefset.py --- a/lib-python/2.7/_weakrefset.py +++ b/lib-python/2.7/_weakrefset.py @@ -63,7 +63,7 @@ yield item def __len__(self): - return sum(x() is not None for x in self.data) + return len(self.data) - len(self._pending_removals) def __contains__(self, item): try: @@ -116,36 +116,21 @@ def update(self, other): if self._pending_removals: self._commit_removals() - if isinstance(other, self.__class__): - self.data.update(other.data) - else: - for element in other: - self.add(element) + for element in other: + self.add(element) def __ior__(self, other): self.update(other) return self - # Helper functions for simple delegating methods. - def _apply(self, other, method): - if not isinstance(other, self.__class__): - other = self.__class__(other) - newdata = method(other.data) - newset = self.__class__() - newset.data = newdata + def difference(self, other): + newset = self.copy() + newset.difference_update(other) return newset - - def difference(self, other): - return self._apply(other, self.data.difference) __sub__ = difference def difference_update(self, other): - if self._pending_removals: - self._commit_removals() - if self is other: - self.data.clear() - else: - self.data.difference_update(ref(item) for item in other) + self.__isub__(other) def __isub__(self, other): if self._pending_removals: self._commit_removals() @@ -156,13 +141,11 @@ return self def intersection(self, other): - return self._apply(other, self.data.intersection) + return self.__class__(item for item in other if item in self) __and__ = intersection def intersection_update(self, other): - if self._pending_removals: - self._commit_removals() - self.data.intersection_update(ref(item) for item in other) + self.__iand__(other) def __iand__(self, other): if self._pending_removals: self._commit_removals() @@ -171,17 +154,17 @@ def issubset(self, other): return self.data.issubset(ref(item) for item in other) - __lt__ = issubset + __le__ = issubset - def __le__(self, other): - return self.data <= set(ref(item) for item in other) + def __lt__(self, other): + return self.data < set(ref(item) for item in other) def issuperset(self, other): return self.data.issuperset(ref(item) for item in other) - __gt__ = issuperset + __ge__ = issuperset - def __ge__(self, other): - return self.data >= set(ref(item) for item in other) + def __gt__(self, other): + return self.data > set(ref(item) for item in other) def __eq__(self, other): if not isinstance(other, self.__class__): @@ -189,27 +172,24 @@ return self.data == set(ref(item) for item in other) def symmetric_difference(self, other): - return self._apply(other, self.data.symmetric_difference) + newset = self.copy() + newset.symmetric_difference_update(other) + return newset __xor__ = symmetric_difference def symmetric_difference_update(self, other): - if self._pending_removals: - self._commit_removals() - if self is other: - self.data.clear() - else: - self.data.symmetric_difference_update(ref(item) for item in other) + self.__ixor__(other) def __ixor__(self, other): if self._pending_removals: self._commit_removals() if self is other: self.data.clear() else: - self.data.symmetric_difference_update(ref(item) for item in other) + self.data.symmetric_difference_update(ref(item, self._remove) for item in other) return self def union(self, other): - return self._apply(other, self.data.union) + return self.__class__(e for s in (self, other) for e in s) __or__ = union def isdisjoint(self, other): diff --git a/lib-python/2.7/aifc.py b/lib-python/2.7/aifc.py --- a/lib-python/2.7/aifc.py +++ b/lib-python/2.7/aifc.py @@ -732,22 +732,28 @@ self._patchheader() def close(self): - self._ensure_header_written(0) - if self._datawritten & 1: - # quick pad to even size - self._file.write(chr(0)) - self._datawritten = self._datawritten + 1 - self._writemarkers() - if self._nframeswritten != self._nframes or \ - self._datalength != self._datawritten or \ - self._marklength: - self._patchheader() - if self._comp: - self._comp.CloseCompressor() - self._comp = None - # Prevent ref cycles - self._convert = None - self._file.close() + if self._file is None: + return + try: + self._ensure_header_written(0) + if self._datawritten & 1: + # quick pad to even size + self._file.write(chr(0)) + self._datawritten = self._datawritten + 1 + self._writemarkers() + if self._nframeswritten != self._nframes or \ + self._datalength != self._datawritten or \ + self._marklength: + self._patchheader() + if self._comp: + self._comp.CloseCompressor() + self._comp = None + finally: + # Prevent ref cycles + self._convert = None + f = self._file + self._file = None + f.close() # # Internal methods. diff --git a/lib-python/2.7/argparse.py b/lib-python/2.7/argparse.py --- a/lib-python/2.7/argparse.py +++ b/lib-python/2.7/argparse.py @@ -740,10 +740,10 @@ - default -- The value to be produced if the option is not specified. - - type -- The type which the command-line arguments should be converted - to, should be one of 'string', 'int', 'float', 'complex' or a - callable object that accepts a single string argument. If None, - 'string' is assumed. + - type -- A callable that accepts a single string argument, and + returns the converted value. The standard Python types str, int, + float, and complex are useful examples of such callables. If None, + str is used. - choices -- A container of values that should be allowed. If not None, after a command-line argument has been converted to the appropriate @@ -1692,9 +1692,12 @@ return args def parse_known_args(self, args=None, namespace=None): - # args default to the system args if args is None: + # args default to the system args args = _sys.argv[1:] + else: + # make sure that args are mutable + args = list(args) # default Namespace built from parser defaults if namespace is None: @@ -1705,10 +1708,7 @@ if action.dest is not SUPPRESS: if not hasattr(namespace, action.dest): if action.default is not SUPPRESS: - default = action.default - if isinstance(action.default, basestring): - default = self._get_value(action, default) - setattr(namespace, action.dest, default) + setattr(namespace, action.dest, action.default) # add any parser defaults that aren't present for dest in self._defaults: @@ -1936,12 +1936,23 @@ if positionals: self.error(_('too few arguments')) - # make sure all required actions were present + # make sure all required actions were present, and convert defaults. for action in self._actions: - if action.required: - if action not in seen_actions: + if action not in seen_actions: + if action.required: name = _get_action_name(action) self.error(_('argument %s is required') % name) + else: + # Convert action default now instead of doing it before + # parsing arguments to avoid calling convert functions + # twice (which may fail) if the argument was given, but + # only if it was defined already in the namespace + if (action.default is not None and + isinstance(action.default, basestring) and + hasattr(namespace, action.dest) and + action.default is getattr(namespace, action.dest)): + setattr(namespace, action.dest, + self._get_value(action, action.default)) # make sure all required groups had one option present for group in self._mutually_exclusive_groups: @@ -1967,7 +1978,7 @@ for arg_string in arg_strings: # for regular arguments, just add them back into the list - if arg_string[0] not in self.fromfile_prefix_chars: + if not arg_string or arg_string[0] not in self.fromfile_prefix_chars: new_arg_strings.append(arg_string) # replace arguments referencing files with the file content @@ -2174,9 +2185,12 @@ # Value conversion methods # ======================== def _get_values(self, action, arg_strings): - # for everything but PARSER args, strip out '--' + # for everything but PARSER, REMAINDER args, strip out first '--' if action.nargs not in [PARSER, REMAINDER]: - arg_strings = [s for s in arg_strings if s != '--'] + try: + arg_strings.remove('--') + except ValueError: + pass # optional argument produces a default when not present if not arg_strings and action.nargs == OPTIONAL: diff --git a/lib-python/2.7/asyncore.py b/lib-python/2.7/asyncore.py --- a/lib-python/2.7/asyncore.py +++ b/lib-python/2.7/asyncore.py @@ -225,6 +225,7 @@ debug = False connected = False accepting = False + connecting = False closing = False addr = None ignore_log_types = frozenset(['warning']) @@ -248,7 +249,7 @@ try: self.addr = sock.getpeername() except socket.error, err: - if err.args[0] == ENOTCONN: + if err.args[0] in (ENOTCONN, EINVAL): # To handle the case where we got an unconnected # socket. self.connected = False @@ -342,9 +343,11 @@ def connect(self, address): self.connected = False + self.connecting = True err = self.socket.connect_ex(address) if err in (EINPROGRESS, EALREADY, EWOULDBLOCK) \ or err == EINVAL and os.name in ('nt', 'ce'): + self.addr = address return if err in (0, EISCONN): self.addr = address @@ -390,7 +393,7 @@ else: return data except socket.error, why: - # winsock sometimes throws ENOTCONN + # winsock sometimes raises ENOTCONN if why.args[0] in _DISCONNECTED: self.handle_close() return '' @@ -400,6 +403,7 @@ def close(self): self.connected = False self.accepting = False + self.connecting = False self.del_channel() try: self.socket.close() @@ -438,7 +442,8 @@ # sockets that are connected self.handle_accept() elif not self.connected: - self.handle_connect_event() + if self.connecting: + self.handle_connect_event() self.handle_read() else: self.handle_read() @@ -449,6 +454,7 @@ raise socket.error(err, _strerror(err)) self.handle_connect() self.connected = True + self.connecting = False def handle_write_event(self): if self.accepting: @@ -457,12 +463,8 @@ return if not self.connected: - #check for errors - err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) - if err != 0: - raise socket.error(err, _strerror(err)) - - self.handle_connect_event() + if self.connecting: + self.handle_connect_event() self.handle_write() def handle_expt_event(self): diff --git a/lib-python/2.7/bdb.py b/lib-python/2.7/bdb.py --- a/lib-python/2.7/bdb.py +++ b/lib-python/2.7/bdb.py @@ -24,6 +24,7 @@ self.skip = set(skip) if skip else None self.breaks = {} self.fncache = {} + self.frame_returning = None def canonic(self, filename): if filename == "<" + filename[1:-1] + ">": @@ -82,7 +83,11 @@ def dispatch_return(self, frame, arg): if self.stop_here(frame) or frame == self.returnframe: - self.user_return(frame, arg) + try: + self.frame_returning = frame + self.user_return(frame, arg) + finally: + self.frame_returning = None if self.quitting: raise BdbQuit return self.trace_dispatch @@ -186,6 +191,14 @@ def set_step(self): """Stop after one line of code.""" + # Issue #13183: pdb skips frames after hitting a breakpoint and running + # step commands. + # Restore the trace function in the caller (that may not have been set + # for performance reasons) when returning from the current frame. + if self.frame_returning: + caller_frame = self.frame_returning.f_back + if caller_frame and not caller_frame.f_trace: + caller_frame.f_trace = self.trace_dispatch self._set_stopinfo(None, None) def set_next(self, frame): diff --git a/lib-python/2.7/calendar.py b/lib-python/2.7/calendar.py --- a/lib-python/2.7/calendar.py +++ b/lib-python/2.7/calendar.py @@ -161,7 +161,11 @@ oneday = datetime.timedelta(days=1) while True: yield date - date += oneday + try: + date += oneday + except OverflowError: + # Adding one day could fail after datetime.MAXYEAR + break if date.month != month and date.weekday() == self.firstweekday: break diff --git a/lib-python/2.7/cgi.py b/lib-python/2.7/cgi.py --- a/lib-python/2.7/cgi.py +++ b/lib-python/2.7/cgi.py @@ -37,7 +37,6 @@ from operator import attrgetter import sys import os -import urllib import UserDict import urlparse diff --git a/lib-python/2.7/cgitb.py b/lib-python/2.7/cgitb.py --- a/lib-python/2.7/cgitb.py +++ b/lib-python/2.7/cgitb.py @@ -295,14 +295,19 @@ if self.logdir is not None: suffix = ['.txt', '.html'][self.format=="html"] (fd, path) = tempfile.mkstemp(suffix=suffix, dir=self.logdir) + try: file = os.fdopen(fd, 'w') file.write(doc) file.close() - msg = '

    %s contains the description of this error.' % path + msg = '%s contains the description of this error.' % path except: - msg = '

    Tried to save traceback to %s, but failed.' % path - self.file.write(msg + '\n') + msg = 'Tried to save traceback to %s, but failed.' % path + + if self.format == 'html': + self.file.write('

    %s

    \n' % msg) + else: + self.file.write(msg + '\n') try: self.file.flush() except: pass diff --git a/lib-python/2.7/cmd.py b/lib-python/2.7/cmd.py --- a/lib-python/2.7/cmd.py +++ b/lib-python/2.7/cmd.py @@ -294,6 +294,7 @@ return list(commands | topics) def do_help(self, arg): + 'List available commands with "help" or detailed help with "help cmd".' if arg: # XXX check arg syntax try: diff --git a/lib-python/2.7/collections.py b/lib-python/2.7/collections.py --- a/lib-python/2.7/collections.py +++ b/lib-python/2.7/collections.py @@ -6,11 +6,12 @@ __all__ += _abcoll.__all__ from _collections import deque, defaultdict -from operator import itemgetter as _itemgetter +from operator import itemgetter as _itemgetter, eq as _eq from keyword import iskeyword as _iskeyword import sys as _sys import heapq as _heapq from itertools import repeat as _repeat, chain as _chain, starmap as _starmap +from itertools import imap as _imap try: from thread import get_ident as _get_ident @@ -50,49 +51,45 @@ self.__map = {} self.__update(*args, **kwds) - def __setitem__(self, key, value, PREV=0, NEXT=1, dict_setitem=dict.__setitem__): + def __setitem__(self, key, value, dict_setitem=dict.__setitem__): 'od.__setitem__(i, y) <==> od[i]=y' # Setting a new item creates a new link at the end of the linked list, # and the inherited dictionary is updated with the new key/value pair. if key not in self: root = self.__root - last = root[PREV] - last[NEXT] = root[PREV] = self.__map[key] = [last, root, key] - dict_setitem(self, key, value) + last = root[0] + last[1] = root[0] = self.__map[key] = [last, root, key] + return dict_setitem(self, key, value) - def __delitem__(self, key, PREV=0, NEXT=1, dict_delitem=dict.__delitem__): + def __delitem__(self, key, dict_delitem=dict.__delitem__): 'od.__delitem__(y) <==> del od[y]' # Deleting an existing item uses self.__map to find the link which gets # removed by updating the links in the predecessor and successor nodes. dict_delitem(self, key) link_prev, link_next, key = self.__map.pop(key) - link_prev[NEXT] = link_next - link_next[PREV] = link_prev + link_prev[1] = link_next # update link_prev[NEXT] + link_next[0] = link_prev # update link_next[PREV] def __iter__(self): 'od.__iter__() <==> iter(od)' # Traverse the linked list in order. - NEXT, KEY = 1, 2 root = self.__root - curr = root[NEXT] + curr = root[1] # start at the first node while curr is not root: - yield curr[KEY] - curr = curr[NEXT] + yield curr[2] # yield the curr[KEY] + curr = curr[1] # move to next node def __reversed__(self): 'od.__reversed__() <==> reversed(od)' # Traverse the linked list in reverse order. - PREV, KEY = 0, 2 root = self.__root - curr = root[PREV] + curr = root[0] # start at the last node while curr is not root: - yield curr[KEY] - curr = curr[PREV] + yield curr[2] # yield the curr[KEY] + curr = curr[0] # move to previous node def clear(self): 'od.clear() -> None. Remove all items from od.' - for node in self.__map.itervalues(): - del node[:] root = self.__root root[:] = [root, root, None] self.__map.clear() @@ -208,7 +205,7 @@ ''' if isinstance(other, OrderedDict): - return len(self)==len(other) and self.items() == other.items() + return dict.__eq__(self, other) and all(_imap(_eq, self, other)) return dict.__eq__(self, other) def __ne__(self, other): @@ -234,10 +231,60 @@ ### namedtuple ################################################################################ +_class_template = '''\ +class {typename}(tuple): + '{typename}({arg_list})' + + __slots__ = () + + _fields = {field_names!r} + + def __new__(_cls, {arg_list}): + 'Create new instance of {typename}({arg_list})' + return _tuple.__new__(_cls, ({arg_list})) + + @classmethod + def _make(cls, iterable, new=tuple.__new__, len=len): + 'Make a new {typename} object from a sequence or iterable' + result = new(cls, iterable) + if len(result) != {num_fields:d}: + raise TypeError('Expected {num_fields:d} arguments, got %d' % len(result)) + return result + + def __repr__(self): + 'Return a nicely formatted representation string' + return '{typename}({repr_fmt})' % self + + def _asdict(self): + 'Return a new OrderedDict which maps field names to their values' + return OrderedDict(zip(self._fields, self)) + + __dict__ = property(_asdict) + + def _replace(_self, **kwds): + 'Return a new {typename} object replacing specified fields with new values' + result = _self._make(map(kwds.pop, {field_names!r}, _self)) + if kwds: + raise ValueError('Got unexpected field names: %r' % kwds.keys()) + return result + + def __getnewargs__(self): + 'Return self as a plain tuple. Used by copy and pickle.' + return tuple(self) + +{field_defs} +''' + +_repr_template = '{name}=%r' + +_field_template = '''\ + {name} = _property(_itemgetter({index:d}), doc='Alias for field number {index:d}') +''' + def namedtuple(typename, field_names, verbose=False, rename=False): """Returns a new subclass of tuple with named fields. - >>> Point = namedtuple('Point', 'x y') + >>> Point = namedtuple('Point', ['x', 'y']) >>> Point.__doc__ # docstring for the new class 'Point(x, y)' >>> p = Point(11, y=22) # instantiate with positional args or keywords @@ -258,83 +305,63 @@ """ - # Parse and validate the field names. Validation serves two purposes, - # generating informative error messages and preventing template injection attacks. + # Validate the field names. At the user's option, either generate an error + # message or automatically replace the field name with a valid name. if isinstance(field_names, basestring): - field_names = field_names.replace(',', ' ').split() # names separated by whitespace and/or commas - field_names = tuple(map(str, field_names)) + field_names = field_names.replace(',', ' ').split() + field_names = map(str, field_names) if rename: - names = list(field_names) seen = set() - for i, name in enumerate(names): - if (not all(c.isalnum() or c=='_' for c in name) or _iskeyword(name) - or not name or name[0].isdigit() or name.startswith('_') + for index, name in enumerate(field_names): + if (not all(c.isalnum() or c=='_' for c in name) + or _iskeyword(name) + or not name + or name[0].isdigit() + or name.startswith('_') or name in seen): - names[i] = '_%d' % i + field_names[index] = '_%d' % index seen.add(name) - field_names = tuple(names) - for name in (typename,) + field_names: + for name in [typename] + field_names: if not all(c.isalnum() or c=='_' for c in name): - raise ValueError('Type names and field names can only contain alphanumeric characters and underscores: %r' % name) + raise ValueError('Type names and field names can only contain ' + 'alphanumeric characters and underscores: %r' % name) if _iskeyword(name): - raise ValueError('Type names and field names cannot be a keyword: %r' % name) + raise ValueError('Type names and field names cannot be a ' + 'keyword: %r' % name) if name[0].isdigit(): - raise ValueError('Type names and field names cannot start with a number: %r' % name) - seen_names = set() + raise ValueError('Type names and field names cannot start with ' + 'a number: %r' % name) + seen = set() for name in field_names: if name.startswith('_') and not rename: - raise ValueError('Field names cannot start with an underscore: %r' % name) - if name in seen_names: + raise ValueError('Field names cannot start with an underscore: ' + '%r' % name) + if name in seen: raise ValueError('Encountered duplicate field name: %r' % name) - seen_names.add(name) + seen.add(name) - # Create and fill-in the class template - numfields = len(field_names) - argtxt = repr(field_names).replace("'", "")[1:-1] # tuple repr without parens or quotes - reprtxt = ', '.join('%s=%%r' % name for name in field_names) - template = '''class %(typename)s(tuple): - '%(typename)s(%(argtxt)s)' \n - __slots__ = () \n - _fields = %(field_names)r \n - def __new__(_cls, %(argtxt)s): - 'Create new instance of %(typename)s(%(argtxt)s)' - return _tuple.__new__(_cls, (%(argtxt)s)) \n - @classmethod - def _make(cls, iterable, new=tuple.__new__, len=len): - 'Make a new %(typename)s object from a sequence or iterable' - result = new(cls, iterable) - if len(result) != %(numfields)d: - raise TypeError('Expected %(numfields)d arguments, got %%d' %% len(result)) - return result \n - def __repr__(self): - 'Return a nicely formatted representation string' - return '%(typename)s(%(reprtxt)s)' %% self \n - def _asdict(self): - 'Return a new OrderedDict which maps field names to their values' - return OrderedDict(zip(self._fields, self)) \n - __dict__ = property(_asdict) \n - def _replace(_self, **kwds): - 'Return a new %(typename)s object replacing specified fields with new values' - result = _self._make(map(kwds.pop, %(field_names)r, _self)) - if kwds: - raise ValueError('Got unexpected field names: %%r' %% kwds.keys()) - return result \n - def __getnewargs__(self): - 'Return self as a plain tuple. Used by copy and pickle.' - return tuple(self) \n\n''' % locals() - for i, name in enumerate(field_names): - template += " %s = _property(_itemgetter(%d), doc='Alias for field number %d')\n" % (name, i, i) + # Fill-in the class template + class_definition = _class_template.format( + typename = typename, + field_names = tuple(field_names), + num_fields = len(field_names), + arg_list = repr(tuple(field_names)).replace("'", "")[1:-1], + repr_fmt = ', '.join(_repr_template.format(name=name) + for name in field_names), + field_defs = '\n'.join(_field_template.format(index=index, name=name) + for index, name in enumerate(field_names)) + ) if verbose: - print template + print class_definition - # Execute the template string in a temporary namespace and - # support tracing utilities by setting a value for frame.f_globals['__name__'] + # Execute the template string in a temporary namespace and support + # tracing utilities by setting a value for frame.f_globals['__name__'] namespace = dict(_itemgetter=_itemgetter, __name__='namedtuple_%s' % typename, OrderedDict=OrderedDict, _property=property, _tuple=tuple) try: - exec template in namespace - except SyntaxError, e: - raise SyntaxError(e.message + ':\n' + template) + exec class_definition in namespace + except SyntaxError as e: + raise SyntaxError(e.message + ':\n' + class_definition) result = namespace[typename] # For pickling to work, the __module__ variable needs to be set to the frame diff --git a/lib-python/2.7/compiler/consts.py b/lib-python/2.7/compiler/consts.py --- a/lib-python/2.7/compiler/consts.py +++ b/lib-python/2.7/compiler/consts.py @@ -5,7 +5,7 @@ SC_LOCAL = 1 SC_GLOBAL_IMPLICIT = 2 -SC_GLOBAL_EXPLICT = 3 +SC_GLOBAL_EXPLICIT = 3 SC_FREE = 4 SC_CELL = 5 SC_UNKNOWN = 6 diff --git a/lib-python/2.7/compiler/pycodegen.py b/lib-python/2.7/compiler/pycodegen.py --- a/lib-python/2.7/compiler/pycodegen.py +++ b/lib-python/2.7/compiler/pycodegen.py @@ -7,7 +7,7 @@ from compiler import ast, parse, walk, syntax from compiler import pyassem, misc, future, symbols -from compiler.consts import SC_LOCAL, SC_GLOBAL_IMPLICIT, SC_GLOBAL_EXPLICT, \ +from compiler.consts import SC_LOCAL, SC_GLOBAL_IMPLICIT, SC_GLOBAL_EXPLICIT, \ SC_FREE, SC_CELL from compiler.consts import (CO_VARARGS, CO_VARKEYWORDS, CO_NEWLOCALS, CO_NESTED, CO_GENERATOR, CO_FUTURE_DIVISION, @@ -283,7 +283,7 @@ self.emit(prefix + '_NAME', name) else: self.emit(prefix + '_FAST', name) - elif scope == SC_GLOBAL_EXPLICT: + elif scope == SC_GLOBAL_EXPLICIT: self.emit(prefix + '_GLOBAL', name) elif scope == SC_GLOBAL_IMPLICIT: if not self.optimized: diff --git a/lib-python/2.7/compiler/symbols.py b/lib-python/2.7/compiler/symbols.py --- a/lib-python/2.7/compiler/symbols.py +++ b/lib-python/2.7/compiler/symbols.py @@ -1,7 +1,7 @@ """Module symbol-table generator""" from compiler import ast -from compiler.consts import SC_LOCAL, SC_GLOBAL_IMPLICIT, SC_GLOBAL_EXPLICT, \ +from compiler.consts import SC_LOCAL, SC_GLOBAL_IMPLICIT, SC_GLOBAL_EXPLICIT, \ SC_FREE, SC_CELL, SC_UNKNOWN from compiler.misc import mangle import types @@ -90,7 +90,7 @@ The scope of a name could be LOCAL, GLOBAL, FREE, or CELL. """ if name in self.globals: - return SC_GLOBAL_EXPLICT + return SC_GLOBAL_EXPLICIT if name in self.cells: return SC_CELL if name in self.defs: diff --git a/lib-python/2.7/cookielib.py b/lib-python/2.7/cookielib.py --- a/lib-python/2.7/cookielib.py +++ b/lib-python/2.7/cookielib.py @@ -1,4 +1,4 @@ -"""HTTP cookie handling for web clients. +r"""HTTP cookie handling for web clients. This module has (now fairly distant) origins in Gisle Aas' Perl module HTTP::Cookies, from the libwww-perl library. diff --git a/lib-python/2.7/ctypes/test/test_bitfields.py b/lib-python/2.7/ctypes/test/test_bitfields.py --- a/lib-python/2.7/ctypes/test/test_bitfields.py +++ b/lib-python/2.7/ctypes/test/test_bitfields.py @@ -246,5 +246,25 @@ _anonymous_ = ["_"] _fields_ = [("_", X)] + @unittest.skipUnless(hasattr(ctypes, "c_uint32"), "c_int32 is required") + def test_uint32(self): + class X(Structure): + _fields_ = [("a", c_uint32, 32)] + x = X() + x.a = 10 + self.assertEqual(x.a, 10) + x.a = 0xFDCBA987 + self.assertEqual(x.a, 0xFDCBA987) + + @unittest.skipUnless(hasattr(ctypes, "c_uint64"), "c_int64 is required") + def test_uint64(self): + class X(Structure): + _fields_ = [("a", c_uint64, 64)] + x = X() + x.a = 10 + self.assertEqual(x.a, 10) + x.a = 0xFEDCBA9876543211 + self.assertEqual(x.a, 0xFEDCBA9876543211) + if __name__ == "__main__": unittest.main() diff --git a/lib-python/2.7/ctypes/test/test_numbers.py b/lib-python/2.7/ctypes/test/test_numbers.py --- a/lib-python/2.7/ctypes/test/test_numbers.py +++ b/lib-python/2.7/ctypes/test/test_numbers.py @@ -221,6 +221,16 @@ # probably be changed: self.assertRaises(TypeError, c_int, c_long(42)) + def test_float_overflow(self): + import sys + big_int = int(sys.float_info.max) * 2 + for t in float_types + [c_longdouble]: + self.assertRaises(OverflowError, t, big_int) + if (hasattr(t, "__ctype_be__")): + self.assertRaises(OverflowError, t.__ctype_be__, big_int) + if (hasattr(t, "__ctype_le__")): + self.assertRaises(OverflowError, t.__ctype_le__, big_int) + ## def test_perf(self): ## check_perf() diff --git a/lib-python/2.7/ctypes/test/test_structures.py b/lib-python/2.7/ctypes/test/test_structures.py --- a/lib-python/2.7/ctypes/test/test_structures.py +++ b/lib-python/2.7/ctypes/test/test_structures.py @@ -1,6 +1,7 @@ import unittest from ctypes import * from struct import calcsize +import _testcapi class SubclassesTest(unittest.TestCase): def test_subclass(self): @@ -199,6 +200,14 @@ "_pack_": -1} self.assertRaises(ValueError, type(Structure), "X", (Structure,), d) + # Issue 15989 + d = {"_fields_": [("a", c_byte)], + "_pack_": _testcapi.INT_MAX + 1} + self.assertRaises(ValueError, type(Structure), "X", (Structure,), d) + d = {"_fields_": [("a", c_byte)], + "_pack_": _testcapi.UINT_MAX + 2} + self.assertRaises(ValueError, type(Structure), "X", (Structure,), d) + def test_initializers(self): class Person(Structure): _fields_ = [("name", c_char*6), diff --git a/lib-python/2.7/ctypes/test/test_win32.py b/lib-python/2.7/ctypes/test/test_win32.py --- a/lib-python/2.7/ctypes/test/test_win32.py +++ b/lib-python/2.7/ctypes/test/test_win32.py @@ -3,6 +3,7 @@ from ctypes import * from ctypes.test import is_resource_enabled import unittest, sys +from test import test_support as support import _ctypes_test @@ -60,7 +61,9 @@ def test_COMError(self): from _ctypes import COMError - self.assertEqual(COMError.__doc__, "Raised when a COM method call failed.") + if support.HAVE_DOCSTRINGS: + self.assertEqual(COMError.__doc__, + "Raised when a COM method call failed.") ex = COMError(-1, "text", ("details",)) self.assertEqual(ex.hresult, -1) diff --git a/lib-python/2.7/curses/__init__.py b/lib-python/2.7/curses/__init__.py --- a/lib-python/2.7/curses/__init__.py +++ b/lib-python/2.7/curses/__init__.py @@ -5,7 +5,7 @@ import curses from curses import textpad - curses.initwin() + curses.initscr() ... """ diff --git a/lib-python/2.7/decimal.py b/lib-python/2.7/decimal.py --- a/lib-python/2.7/decimal.py +++ b/lib-python/2.7/decimal.py @@ -1581,7 +1581,13 @@ def __float__(self): """Float representation.""" - return float(str(self)) + if self._isnan(): + if self.is_snan(): + raise ValueError("Cannot convert signaling NaN to float") + s = "-nan" if self._sign else "nan" + else: + s = str(self) + return float(s) def __int__(self): """Converts self to an int, truncating if necessary.""" diff --git a/lib-python/2.7/distutils/command/bdist_rpm.py b/lib-python/2.7/distutils/command/bdist_rpm.py --- a/lib-python/2.7/distutils/command/bdist_rpm.py +++ b/lib-python/2.7/distutils/command/bdist_rpm.py @@ -379,16 +379,28 @@ self.spawn(rpm_cmd) if not self.dry_run: + if self.distribution.has_ext_modules(): + pyversion = get_python_version() + else: + pyversion = 'any' + if not self.binary_only: srpm = os.path.join(rpm_dir['SRPMS'], source_rpm) assert(os.path.exists(srpm)) self.move_file(srpm, self.dist_dir) + filename = os.path.join(self.dist_dir, source_rpm) + self.distribution.dist_files.append( + ('bdist_rpm', pyversion, filename)) if not self.source_only: for rpm in binary_rpms: rpm = os.path.join(rpm_dir['RPMS'], rpm) if os.path.exists(rpm): self.move_file(rpm, self.dist_dir) + filename = os.path.join(self.dist_dir, + os.path.basename(rpm)) + self.distribution.dist_files.append( + ('bdist_rpm', pyversion, filename)) # run() def _dist_path(self, path): diff --git a/lib-python/2.7/distutils/command/check.py b/lib-python/2.7/distutils/command/check.py --- a/lib-python/2.7/distutils/command/check.py +++ b/lib-python/2.7/distutils/command/check.py @@ -26,6 +26,9 @@ def system_message(self, level, message, *children, **kwargs): self.messages.append((level, message, children, kwargs)) + return nodes.system_message(message, level=level, + type=self.levels[level], + *children, **kwargs) HAS_DOCUTILS = True except ImportError: diff --git a/lib-python/2.7/distutils/config.py b/lib-python/2.7/distutils/config.py --- a/lib-python/2.7/distutils/config.py +++ b/lib-python/2.7/distutils/config.py @@ -42,16 +42,11 @@ def _store_pypirc(self, username, password): """Creates a default .pypirc file.""" rc = self._get_rc_file() - f = open(rc, 'w') + f = os.fdopen(os.open(rc, os.O_CREAT | os.O_WRONLY, 0600), 'w') try: f.write(DEFAULT_PYPIRC % (username, password)) finally: f.close() - try: - os.chmod(rc, 0600) - except OSError: - # should do something better here - pass def _read_pypirc(self): """Reads the .pypirc file.""" diff --git a/lib-python/2.7/distutils/dir_util.py b/lib-python/2.7/distutils/dir_util.py --- a/lib-python/2.7/distutils/dir_util.py +++ b/lib-python/2.7/distutils/dir_util.py @@ -144,6 +144,10 @@ src_name = os.path.join(src, n) dst_name = os.path.join(dst, n) + if n.startswith('.nfs'): + # skip NFS rename files + continue + if preserve_symlinks and os.path.islink(src_name): link_dest = os.readlink(src_name) if verbose >= 1: diff --git a/lib-python/2.7/distutils/tests/test_bdist_dumb.py b/lib-python/2.7/distutils/tests/test_bdist_dumb.py --- a/lib-python/2.7/distutils/tests/test_bdist_dumb.py +++ b/lib-python/2.7/distutils/tests/test_bdist_dumb.py @@ -1,8 +1,10 @@ """Tests for distutils.command.bdist_dumb.""" +import os +import sys +import zipfile import unittest -import sys -import os +from test.test_support import run_unittest # zlib is not used here, but if it's not available # test_simple_built will fail @@ -11,8 +13,6 @@ except ImportError: zlib = None -from test.test_support import run_unittest - from distutils.core import Distribution from distutils.command.bdist_dumb import bdist_dumb from distutils.tests import support @@ -73,15 +73,23 @@ # see what we have dist_created = os.listdir(os.path.join(pkg_dir, 'dist')) - base = "%s.%s" % (dist.get_fullname(), cmd.plat_name) + base = "%s.%s.zip" % (dist.get_fullname(), cmd.plat_name) if os.name == 'os2': base = base.replace(':', '-') - wanted = ['%s.zip' % base] - self.assertEqual(dist_created, wanted) + self.assertEqual(dist_created, [base]) # now let's check what we have in the zip file - # XXX to be done + fp = zipfile.ZipFile(os.path.join('dist', base)) + try: + contents = fp.namelist() + finally: + fp.close() + + contents = sorted(os.path.basename(fn) for fn in contents) + wanted = ['foo-0.1-py%s.%s.egg-info' % sys.version_info[:2], + 'foo.py', 'foo.pyc'] + self.assertEqual(contents, sorted(wanted)) def test_finalize_options(self): pkg_dir, dist = self.create_dist() diff --git a/lib-python/2.7/distutils/tests/test_bdist_msi.py b/lib-python/2.7/distutils/tests/test_bdist_msi.py --- a/lib-python/2.7/distutils/tests/test_bdist_msi.py +++ b/lib-python/2.7/distutils/tests/test_bdist_msi.py @@ -1,12 +1,11 @@ """Tests for distutils.command.bdist_msi.""" +import sys import unittest -import sys - from test.test_support import run_unittest - from distutils.tests import support - at unittest.skipUnless(sys.platform=="win32", "These tests are only for win32") + + at unittest.skipUnless(sys.platform == 'win32', 'these tests require Windows') class BDistMSITestCase(support.TempdirManager, support.LoggingSilencer, unittest.TestCase): @@ -14,10 +13,11 @@ def test_minimal(self): # minimal test XXX need more tests from distutils.command.bdist_msi import bdist_msi - pkg_pth, dist = self.create_dist() + project_dir, dist = self.create_dist() cmd = bdist_msi(dist) cmd.ensure_finalized() + def test_suite(): return unittest.makeSuite(BDistMSITestCase) diff --git a/lib-python/2.7/distutils/tests/test_bdist_rpm.py b/lib-python/2.7/distutils/tests/test_bdist_rpm.py --- a/lib-python/2.7/distutils/tests/test_bdist_rpm.py +++ b/lib-python/2.7/distutils/tests/test_bdist_rpm.py @@ -79,6 +79,10 @@ dist_created = os.listdir(os.path.join(pkg_dir, 'dist')) self.assertTrue('foo-0.1-1.noarch.rpm' in dist_created) + # bug #2945: upload ignores bdist_rpm files + self.assertIn(('bdist_rpm', 'any', 'dist/foo-0.1-1.src.rpm'), dist.dist_files) + self.assertIn(('bdist_rpm', 'any', 'dist/foo-0.1-1.noarch.rpm'), dist.dist_files) + def test_no_optimize_flag(self): # XXX I am unable yet to make this test work without @@ -118,6 +122,11 @@ dist_created = os.listdir(os.path.join(pkg_dir, 'dist')) self.assertTrue('foo-0.1-1.noarch.rpm' in dist_created) + + # bug #2945: upload ignores bdist_rpm files + self.assertIn(('bdist_rpm', 'any', 'dist/foo-0.1-1.src.rpm'), dist.dist_files) + self.assertIn(('bdist_rpm', 'any', 'dist/foo-0.1-1.noarch.rpm'), dist.dist_files) + os.remove(os.path.join(pkg_dir, 'dist', 'foo-0.1-1.noarch.rpm')) def test_suite(): diff --git a/lib-python/2.7/distutils/tests/test_build_ext.py b/lib-python/2.7/distutils/tests/test_build_ext.py --- a/lib-python/2.7/distutils/tests/test_build_ext.py +++ b/lib-python/2.7/distutils/tests/test_build_ext.py @@ -77,8 +77,9 @@ self.assertEqual(xx.foo(2, 5), 7) self.assertEqual(xx.foo(13,15), 28) self.assertEqual(xx.new().demo(), None) - doc = 'This is a template module just for instruction.' - self.assertEqual(xx.__doc__, doc) + if test_support.HAVE_DOCSTRINGS: + doc = 'This is a template module just for instruction.' + self.assertEqual(xx.__doc__, doc) self.assertTrue(isinstance(xx.Null(), xx.Null)) self.assertTrue(isinstance(xx.Str(), xx.Str)) diff --git a/lib-python/2.7/distutils/tests/test_dir_util.py b/lib-python/2.7/distutils/tests/test_dir_util.py --- a/lib-python/2.7/distutils/tests/test_dir_util.py +++ b/lib-python/2.7/distutils/tests/test_dir_util.py @@ -101,6 +101,24 @@ remove_tree(self.root_target, verbose=0) remove_tree(self.target2, verbose=0) + def test_copy_tree_skips_nfs_temp_files(self): + mkpath(self.target, verbose=0) + + a_file = os.path.join(self.target, 'ok.txt') + nfs_file = os.path.join(self.target, '.nfs123abc') + for f in a_file, nfs_file: + fh = open(f, 'w') + try: + fh.write('some content') + finally: + fh.close() + + copy_tree(self.target, self.target2) + self.assertEqual(os.listdir(self.target2), ['ok.txt']) + + remove_tree(self.root_target, verbose=0) + remove_tree(self.target2, verbose=0) + def test_ensure_relative(self): if os.sep == '/': self.assertEqual(ensure_relative('/home/foo'), 'home/foo') diff --git a/lib-python/2.7/distutils/tests/test_install.py b/lib-python/2.7/distutils/tests/test_install.py --- a/lib-python/2.7/distutils/tests/test_install.py +++ b/lib-python/2.7/distutils/tests/test_install.py @@ -88,19 +88,17 @@ self.old_expand = os.path.expanduser os.path.expanduser = _expanduser - try: - # this is the actual test - self._test_user_site() - finally: + def cleanup(): site.USER_BASE = self.old_user_base site.USER_SITE = self.old_user_site install_module.USER_BASE = self.old_user_base install_module.USER_SITE = self.old_user_site os.path.expanduser = self.old_expand - def _test_user_site(self): + self.addCleanup(cleanup) + for key in ('nt_user', 'unix_user', 'os2_home'): - self.assertTrue(key in INSTALL_SCHEMES) + self.assertIn(key, INSTALL_SCHEMES) dist = Distribution({'name': 'xx'}) cmd = install(dist) @@ -108,14 +106,14 @@ # making sure the user option is there options = [name for name, short, lable in cmd.user_options] - self.assertTrue('user' in options) + self.assertIn('user', options) # setting a value cmd.user = 1 # user base and site shouldn't be created yet - self.assertTrue(not os.path.exists(self.user_base)) - self.assertTrue(not os.path.exists(self.user_site)) + self.assertFalse(os.path.exists(self.user_base)) + self.assertFalse(os.path.exists(self.user_site)) # let's run finalize cmd.ensure_finalized() @@ -124,8 +122,8 @@ self.assertTrue(os.path.exists(self.user_base)) self.assertTrue(os.path.exists(self.user_site)) - self.assertTrue('userbase' in cmd.config_vars) - self.assertTrue('usersite' in cmd.config_vars) + self.assertIn('userbase', cmd.config_vars) + self.assertIn('usersite', cmd.config_vars) def test_handle_extra_path(self): dist = Distribution({'name': 'xx', 'extra_path': 'path,dirs'}) @@ -178,15 +176,16 @@ def test_record(self): install_dir = self.mkdtemp() - project_dir, dist = self.create_dist(scripts=['hello']) - self.addCleanup(os.chdir, os.getcwd()) + project_dir, dist = self.create_dist(py_modules=['hello'], + scripts=['sayhi']) os.chdir(project_dir) - self.write_file('hello', "print('o hai')") + self.write_file('hello.py', "def main(): print 'o hai'") + self.write_file('sayhi', 'from hello import main; main()') cmd = install(dist) dist.command_obj['install'] = cmd cmd.root = install_dir - cmd.record = os.path.join(project_dir, 'RECORD') + cmd.record = os.path.join(project_dir, 'filelist') cmd.ensure_finalized() cmd.run() @@ -197,7 +196,7 @@ f.close() found = [os.path.basename(line) for line in content.splitlines()] - expected = ['hello', + expected = ['hello.py', 'hello.pyc', 'sayhi', 'UNKNOWN-0.0.0-py%s.%s.egg-info' % sys.version_info[:2]] self.assertEqual(found, expected) @@ -205,7 +204,6 @@ install_dir = self.mkdtemp() project_dir, dist = self.create_dist(ext_modules=[ Extension('xx', ['xxmodule.c'])]) - self.addCleanup(os.chdir, os.getcwd()) os.chdir(project_dir) support.copy_xxmodule_c(project_dir) @@ -217,7 +215,7 @@ dist.command_obj['install'] = cmd dist.command_obj['build_ext'] = buildextcmd cmd.root = install_dir - cmd.record = os.path.join(project_dir, 'RECORD') + cmd.record = os.path.join(project_dir, 'filelist') cmd.ensure_finalized() cmd.run() @@ -243,6 +241,7 @@ install_module.DEBUG = False self.assertTrue(len(self.logs) > old_logs_len) + def test_suite(): return unittest.makeSuite(InstallTestCase) diff --git a/lib-python/2.7/distutils/tests/test_msvc9compiler.py b/lib-python/2.7/distutils/tests/test_msvc9compiler.py --- a/lib-python/2.7/distutils/tests/test_msvc9compiler.py +++ b/lib-python/2.7/distutils/tests/test_msvc9compiler.py @@ -104,7 +104,7 @@ unittest.TestCase): def test_no_compiler(self): - # makes sure query_vcvarsall throws + # makes sure query_vcvarsall raises # a DistutilsPlatformError if the compiler # is not found from distutils.msvc9compiler import query_vcvarsall diff --git a/lib-python/2.7/distutils/tests/test_register.py b/lib-python/2.7/distutils/tests/test_register.py --- a/lib-python/2.7/distutils/tests/test_register.py +++ b/lib-python/2.7/distutils/tests/test_register.py @@ -1,6 +1,5 @@ # -*- encoding: utf8 -*- """Tests for distutils.command.register.""" -import sys import os import unittest import getpass @@ -11,11 +10,14 @@ from distutils.command import register as register_module from distutils.command.register import register -from distutils.core import Distribution from distutils.errors import DistutilsSetupError -from distutils.tests import support -from distutils.tests.test_config import PYPIRC, PyPIRCCommandTestCase +from distutils.tests.test_config import PyPIRCCommandTestCase + +try: + import docutils +except ImportError: + docutils = None PYPIRC_NOPASSWORD = """\ [distutils] @@ -192,6 +194,7 @@ self.assertEqual(headers['Content-length'], '290') self.assertTrue('tarek' in req.data) + @unittest.skipUnless(docutils is not None, 'needs docutils') def test_strict(self): # testing the script option # when on, the register command stops if @@ -204,13 +207,6 @@ cmd.strict = 1 self.assertRaises(DistutilsSetupError, cmd.run) - # we don't test the reSt feature if docutils - # is not installed - try: - import docutils - except ImportError: - return - # metadata are OK but long_description is broken metadata = {'url': 'xxx', 'author': 'xxx', 'author_email': u'éxéxé', @@ -264,6 +260,21 @@ finally: del register_module.raw_input + @unittest.skipUnless(docutils is not None, 'needs docutils') + def test_register_invalid_long_description(self): + description = ':funkie:`str`' # mimic Sphinx-specific markup + metadata = {'url': 'xxx', 'author': 'xxx', + 'author_email': 'xxx', + 'name': 'xxx', 'version': 'xxx', + 'long_description': description} + cmd = self._get_cmd(metadata) + cmd.ensure_finalized() + cmd.strict = True + inputs = RawInputs('2', 'tarek', 'tarek at ziade.org') + register_module.raw_input = inputs + self.addCleanup(delattr, register_module, 'raw_input') + self.assertRaises(DistutilsSetupError, cmd.run) + def test_check_metadata_deprecated(self): # makes sure make_metadata is deprecated cmd = self._get_cmd() diff --git a/lib-python/2.7/distutils/tests/test_sdist.py b/lib-python/2.7/distutils/tests/test_sdist.py --- a/lib-python/2.7/distutils/tests/test_sdist.py +++ b/lib-python/2.7/distutils/tests/test_sdist.py @@ -6,6 +6,7 @@ import zipfile from os.path import join from textwrap import dedent +from test.test_support import captured_stdout, check_warnings, run_unittest # zlib is not used here, but if it's not available # the tests that use zipfile may fail @@ -21,7 +22,6 @@ except ImportError: UID_GID_SUPPORT = False -from test.test_support import captured_stdout, check_warnings, run_unittest from distutils.command.sdist import sdist, show_formats from distutils.core import Distribution @@ -91,9 +91,8 @@ @unittest.skipUnless(zlib, "requires zlib") def test_prune_file_list(self): - # this test creates a package with some vcs dirs in it - # and launch sdist to make sure they get pruned - # on all systems + # this test creates a project with some VCS dirs and an NFS rename + # file, then launches sdist to check they get pruned on all systems # creating VCS directories with some files in them os.mkdir(join(self.tmp_dir, 'somecode', '.svn')) @@ -107,6 +106,8 @@ self.write_file((self.tmp_dir, 'somecode', '.git', 'ok'), 'xxx') + self.write_file((self.tmp_dir, 'somecode', '.nfs0001'), 'xxx') + # now building a sdist dist, cmd = self.get_cmd() @@ -375,7 +376,7 @@ # the following tests make sure there is a nice error message instead # of a traceback when parsing an invalid manifest template - def _test_template(self, content): + def _check_template(self, content): dist, cmd = self.get_cmd() os.chdir(self.tmp_dir) self.write_file('MANIFEST.in', content) @@ -386,17 +387,17 @@ self.assertEqual(len(warnings), 1) def test_invalid_template_unknown_command(self): - self._test_template('taunt knights *') + self._check_template('taunt knights *') def test_invalid_template_wrong_arguments(self): # this manifest command takes one argument - self._test_template('prune') + self._check_template('prune') @unittest.skipIf(os.name != 'nt', 'test relevant for Windows only') def test_invalid_template_wrong_path(self): # on Windows, trailing slashes are not allowed # this used to crash instead of raising a warning: #8286 - self._test_template('include examples/') + self._check_template('include examples/') @unittest.skipUnless(zlib, "requires zlib") def test_get_file_list(self): diff --git a/lib-python/2.7/doctest.py b/lib-python/2.7/doctest.py --- a/lib-python/2.7/doctest.py +++ b/lib-python/2.7/doctest.py @@ -2314,7 +2314,8 @@ return "Doctest: " + self._dt_test.name class SkipDocTestCase(DocTestCase): - def __init__(self): + def __init__(self, module): + self.module = module DocTestCase.__init__(self, None) def setUp(self): @@ -2324,7 +2325,10 @@ pass def shortDescription(self): - return "Skipping tests from %s" % module.__name__ + return "Skipping tests from %s" % self.module.__name__ + + __str__ = shortDescription + def DocTestSuite(module=None, globs=None, extraglobs=None, test_finder=None, **options): @@ -2372,12 +2376,17 @@ if not tests and sys.flags.optimize >=2: # Skip doctests when running with -O2 suite = unittest.TestSuite() - suite.addTest(SkipDocTestCase()) + suite.addTest(SkipDocTestCase(module)) return suite elif not tests: # Why do we want to do this? Because it reveals a bug that might # otherwise be hidden. - raise ValueError(module, "has no tests") + # It is probably a bug that this exception is not also raised if the + # number of doctest examples in tests is zero (i.e. if no doctest + # examples were found). However, we should probably not be raising + # an exception at all here, though it is too late to make this change + # for a maintenance release. See also issue #14649. + raise ValueError(module, "has no docstrings") tests.sort() suite = unittest.TestSuite() diff --git a/lib-python/2.7/email/_parseaddr.py b/lib-python/2.7/email/_parseaddr.py --- a/lib-python/2.7/email/_parseaddr.py +++ b/lib-python/2.7/email/_parseaddr.py @@ -13,7 +13,7 @@ 'quote', ] -import time +import time, calendar SPACE = ' ' EMPTYSTRING = '' @@ -150,13 +150,13 @@ def mktime_tz(data): - """Turn a 10-tuple as returned by parsedate_tz() into a UTC timestamp.""" + """Turn a 10-tuple as returned by parsedate_tz() into a POSIX timestamp.""" if data[9] is None: # No zone info, so localtime is better assumption than GMT return time.mktime(data[:8] + (-1,)) else: - t = time.mktime(data[:8] + (0,)) - return t - data[9] - time.timezone + t = calendar.timegm(data) + return t - data[9] def quote(str): diff --git a/lib-python/2.7/email/base64mime.py b/lib-python/2.7/email/base64mime.py --- a/lib-python/2.7/email/base64mime.py +++ b/lib-python/2.7/email/base64mime.py @@ -130,7 +130,7 @@ verbatim (this is the default). Each line of encoded text will end with eol, which defaults to "\\n". Set - this to "\r\n" if you will be using the result of this function directly + this to "\\r\\n" if you will be using the result of this function directly in an email. """ if not s: diff --git a/lib-python/2.7/email/feedparser.py b/lib-python/2.7/email/feedparser.py --- a/lib-python/2.7/email/feedparser.py +++ b/lib-python/2.7/email/feedparser.py @@ -13,7 +13,7 @@ data. When you have no more data to push into the parser, call .close(). This completes the parsing and returns the root message object. -The other advantage of this parser is that it will never throw a parsing +The other advantage of this parser is that it will never raise a parsing exception. Instead, when it finds something unexpected, it adds a 'defect' to the current message. Defects are just instances that live on the message object's .defects attribute. @@ -214,7 +214,7 @@ # supposed to see in the body of the message. self._parse_headers(headers) # Headers-only parsing is a backwards compatibility hack, which was - # necessary in the older parser, which could throw errors. All + # necessary in the older parser, which could raise errors. All # remaining lines in the input are thrown into the message body. if self._headersonly: lines = [] diff --git a/lib-python/2.7/email/generator.py b/lib-python/2.7/email/generator.py --- a/lib-python/2.7/email/generator.py +++ b/lib-python/2.7/email/generator.py @@ -212,7 +212,11 @@ msg.set_boundary(boundary) # If there's a preamble, write it out, with a trailing CRLF if msg.preamble is not None: - print >> self._fp, msg.preamble + if self._mangle_from_: + preamble = fcre.sub('>From ', msg.preamble) + else: + preamble = msg.preamble + print >> self._fp, preamble # dash-boundary transport-padding CRLF print >> self._fp, '--' + boundary # body-part @@ -230,7 +234,11 @@ self._fp.write('\n--' + boundary + '--') if msg.epilogue is not None: print >> self._fp - self._fp.write(msg.epilogue) + if self._mangle_from_: + epilogue = fcre.sub('>From ', msg.epilogue) + else: + epilogue = msg.epilogue + self._fp.write(epilogue) def _handle_multipart_signed(self, msg): # The contents of signed parts has to stay unmodified in order to keep diff --git a/lib-python/2.7/email/test/test_email.py b/lib-python/2.7/email/test/test_email.py --- a/lib-python/2.7/email/test/test_email.py +++ b/lib-python/2.7/email/test/test_email.py @@ -9,6 +9,7 @@ import difflib import unittest import warnings +import textwrap from cStringIO import StringIO import email @@ -948,6 +949,28 @@ Blah blah blah """) + def test_mangle_from_in_preamble_and_epilog(self): + s = StringIO() + g = Generator(s, mangle_from_=True) + msg = email.message_from_string(textwrap.dedent("""\ + From: foo at bar.com + Mime-Version: 1.0 + Content-Type: multipart/mixed; boundary=XXX + + From somewhere unknown + + --XXX + Content-Type: text/plain + + foo + + --XXX-- + + From somewhere unknowable + """)) + g.flatten(msg) + self.assertEqual(len([1 for x in s.getvalue().split('\n') + if x.startswith('>From ')]), 2) # Test the basic MIMEAudio class @@ -2262,6 +2285,12 @@ eq(time.localtime(t)[:6], timetup[:6]) eq(int(time.strftime('%Y', timetup[:9])), 2003) + def test_mktime_tz(self): + self.assertEqual(Utils.mktime_tz((1970, 1, 1, 0, 0, 0, + -1, -1, -1, 0)), 0) + self.assertEqual(Utils.mktime_tz((1970, 1, 1, 0, 0, 0, + -1, -1, -1, 1234)), -1234) + def test_parsedate_y2k(self): """Test for parsing a date with a two-digit year. diff --git a/lib-python/2.7/email/utils.py b/lib-python/2.7/email/utils.py --- a/lib-python/2.7/email/utils.py +++ b/lib-python/2.7/email/utils.py @@ -63,7 +63,7 @@ """Decodes a base64 string. This function is equivalent to base64.decodestring and it's retained only - for backward compatibility. It used to remove the last \n of the decoded + for backward compatibility. It used to remove the last \\n of the decoded string, if it had any (see issue 7143). """ if not s: @@ -73,7 +73,7 @@ def fix_eols(s): - """Replace all line-ending characters with \r\n.""" + """Replace all line-ending characters with \\r\\n.""" # Fix newlines with no preceding carriage return s = re.sub(r'(? Author: Brian Kearns Branch: stdlib-2.7.4 Changeset: r60669:8eea657cf3f5 Date: 2013-01-29 07:10 -0500 http://bitbucket.org/pypy/pypy/changeset/8eea657cf3f5/ Log: add new lib-python tests to conftest.py diff --git a/lib-python/conftest.py b/lib-python/conftest.py --- a/lib-python/conftest.py +++ b/lib-python/conftest.py @@ -216,6 +216,7 @@ RegrTest('test_fcntl.py', usemodules='fcntl'), RegrTest('test_file.py', usemodules="posix", core=True), RegrTest('test_file2k.py', usemodules="posix", core=True), + RegrTest('test_file_eintr.py'), RegrTest('test_filecmp.py', core=True), RegrTest('test_fileinput.py', core=True), RegrTest('test_fileio.py'), @@ -433,6 +434,7 @@ RegrTest('test_timeout.py'), RegrTest('test_tk.py'), RegrTest('test_tokenize.py'), + RegrTest('test_tools.py'), RegrTest('test_trace.py'), RegrTest('test_traceback.py', core=True), RegrTest('test_transformer.py', core=True), From noreply at buildbot.pypy.org Tue Jan 29 13:14:40 2013 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 29 Jan 2013 13:14:40 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: make sure we don't annotate unpack_arraydescr too early. RPython is weird like that Message-ID: <20130129121440.BF5FF1C13C4@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60670:9d96565a36db Date: 2013-01-29 14:14 +0200 http://bitbucket.org/pypy/pypy/changeset/9d96565a36db/ Log: make sure we don't annotate unpack_arraydescr too early. RPython is weird like that diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py --- a/rpython/jit/backend/llsupport/llmodel.py +++ b/rpython/jit/backend/llsupport/llmodel.py @@ -54,13 +54,13 @@ def _setup_frame_realloc(self, translate_support_code): FUNC_TP = lltype.Ptr(lltype.FuncType([llmemory.GCREF, lltype.Signed], llmemory.GCREF)) + base_ofs = self.get_baseofs_of_frame_field() def realloc_frame(frame, size): frame = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, frame) if size > frame.jf_frame_info.jfi_frame_depth: # update the frame_info size, which is for whatever reason # not up to date - base_ofs = self.get_baseofs_of_frame_field() frame.jf_frame_info.set_frame_depth(base_ofs, size) new_frame = jitframe.JITFRAME.allocate(frame.jf_frame_info) i = 0 From noreply at buildbot.pypy.org Tue Jan 29 13:16:16 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Tue, 29 Jan 2013 13:16:16 +0100 (CET) Subject: [pypy-commit] pypy quiet-rpython: hg merge default Message-ID: <20130129121616.EE7841C13C4@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: quiet-rpython Changeset: r60671:f169915b9457 Date: 2013-01-29 13:15 +0100 http://bitbucket.org/pypy/pypy/changeset/f169915b9457/ Log: hg merge default diff --git a/lib-python/conftest.py b/lib-python/conftest.py --- a/lib-python/conftest.py +++ b/lib-python/conftest.py @@ -61,7 +61,7 @@ usemodules = '', skip=None): self.basename = basename - self._usemodules = usemodules.split() + ['signal'] + self._usemodules = usemodules.split() + ['signal', 'rctime', 'itertools', '_socket'] self._compiler = compiler self.core = core self.skip = skip @@ -93,63 +93,57 @@ m.test_main() ''' % locals()) -if sys.platform == 'win32': - skip_win32 = "Not supported on Windows" - only_win32 = False -else: - skip_win32 = False - only_win32 = "Only on Windows" - testmap = [ RegrTest('test___all__.py', core=True), RegrTest('test___future__.py', core=True), - RegrTest('test__locale.py', skip=skip_win32), + RegrTest('test__locale.py', usemodules='_locale'), RegrTest('test_abc.py'), RegrTest('test_abstract_numbers.py'), - RegrTest('test_aepack.py', skip=True), + RegrTest('test_aepack.py'), RegrTest('test_aifc.py'), - RegrTest('test_argparse.py'), - RegrTest('test_al.py', skip=True), - RegrTest('test_ast.py', core=True), - RegrTest('test_anydbm.py'), - RegrTest('test_applesingle.py', skip=True), - RegrTest('test_array.py', core=True, usemodules='struct array'), + RegrTest('test_al.py'), + RegrTest('test_anydbm.py', usemodules='struct'), + RegrTest('test_applesingle.py'), + RegrTest('test_argparse.py', usemodules='binascii'), + RegrTest('test_array.py', core=True, usemodules='struct array binascii'), RegrTest('test_ascii_formatd.py'), - RegrTest('test_asynchat.py', usemodules='thread'), - RegrTest('test_asyncore.py'), + RegrTest('test_ast.py', core=True, usemodules='struct'), + RegrTest('test_asynchat.py', usemodules='select fcntl'), + RegrTest('test_asyncore.py', usemodules='select fcntl'), RegrTest('test_atexit.py', core=True), - RegrTest('test_audioop.py', skip=True), + RegrTest('test_audioop.py', skip="unsupported extension module"), RegrTest('test_augassign.py', core=True), - RegrTest('test_base64.py'), + RegrTest('test_base64.py', usemodules='struct'), RegrTest('test_bastion.py'), + RegrTest('test_bigaddrspace.py'), + RegrTest('test_bigmem.py'), RegrTest('test_binascii.py', usemodules='binascii'), - RegrTest('test_binhex.py'), - RegrTest('test_binop.py', core=True), RegrTest('test_bisect.py', core=True, usemodules='_bisect'), RegrTest('test_bool.py', core=True), - RegrTest('test_bsddb.py', skip="unsupported extension module"), - RegrTest('test_bsddb185.py', skip="unsupported extension module"), - RegrTest('test_bsddb3.py', skip="unsupported extension module"), + RegrTest('test_bsddb.py'), + RegrTest('test_bsddb185.py'), + RegrTest('test_bsddb3.py'), RegrTest('test_buffer.py'), RegrTest('test_bufio.py', core=True), - RegrTest('test_builtin.py', core=True), - RegrTest('test_bytes.py'), + RegrTest('test_builtin.py', core=True, usemodules='binascii'), + RegrTest('test_bytes.py', usemodules='struct binascii'), RegrTest('test_bz2.py', usemodules='bz2'), RegrTest('test_calendar.py'), RegrTest('test_call.py', core=True), RegrTest('test_capi.py', skip="not applicable"), - RegrTest('test_cd.py', skip=True), + RegrTest('test_cd.py'), RegrTest('test_cfgparser.py'), - RegrTest('test_cgi.py'), RegrTest('test_charmapcodec.py', core=True), - RegrTest('test_cl.py', skip=True), + RegrTest('test_cl.py'), RegrTest('test_class.py', core=True), RegrTest('test_cmath.py', core=True), RegrTest('test_cmd.py'), + RegrTest('test_cmd_line.py'), RegrTest('test_cmd_line_script.py'), + RegrTest('test_code.py', core=True), RegrTest('test_codeccallbacks.py', core=True), RegrTest('test_codecencodings_cn.py', usemodules='_multibytecodec'), RegrTest('test_codecencodings_hk.py', usemodules='_multibytecodec'), @@ -157,7 +151,6 @@ RegrTest('test_codecencodings_jp.py', usemodules='_multibytecodec'), RegrTest('test_codecencodings_kr.py', usemodules='_multibytecodec'), RegrTest('test_codecencodings_tw.py', usemodules='_multibytecodec'), - RegrTest('test_codecmaps_cn.py', usemodules='_multibytecodec'), RegrTest('test_codecmaps_hk.py', usemodules='_multibytecodec'), RegrTest('test_codecmaps_jp.py', usemodules='_multibytecodec'), @@ -165,6 +158,7 @@ RegrTest('test_codecmaps_tw.py', usemodules='_multibytecodec'), RegrTest('test_codecs.py', core=True, usemodules='_multibytecodec'), RegrTest('test_codeop.py', core=True), + RegrTest('test_coding.py', core=True), RegrTest('test_coercion.py', core=True), RegrTest('test_collections.py'), RegrTest('test_colorsys.py'), @@ -174,22 +168,24 @@ RegrTest('test_compileall.py'), RegrTest('test_compiler.py', core=False, skip="slowly deprecating compiler"), RegrTest('test_complex.py', core=True), - + RegrTest('test_complex_args.py'), RegrTest('test_contains.py', core=True), + RegrTest('test_contextlib.py', usemodules="thread"), RegrTest('test_cookie.py'), RegrTest('test_cookielib.py'), RegrTest('test_copy.py', core=True), RegrTest('test_copy_reg.py', core=True), RegrTest('test_cpickle.py', core=True), - RegrTest('test_cprofile.py'), - RegrTest('test_crypt.py', usemodules='crypt', skip=skip_win32), + RegrTest('test_cprofile.py'), + RegrTest('test_crypt.py', usemodules='crypt'), RegrTest('test_csv.py', usemodules='_csv'), - - RegrTest('test_curses.py', skip="unsupported extension module"), + RegrTest('test_ctypes.py', usemodules="_rawffi thread"), + RegrTest('test_curses.py'), RegrTest('test_datetime.py'), RegrTest('test_dbm.py'), RegrTest('test_decimal.py'), RegrTest('test_decorators.py', core=True), + RegrTest('test_defaultdict.py', usemodules='_collections'), RegrTest('test_deque.py', core=True, usemodules='_collections'), RegrTest('test_descr.py', core=True, usemodules='_weakref'), RegrTest('test_descrtut.py', core=True), @@ -200,7 +196,7 @@ RegrTest('test_dircache.py', core=True), RegrTest('test_dis.py'), RegrTest('test_distutils.py', skip=True), - RegrTest('test_dl.py', skip=True), + RegrTest('test_dl.py'), RegrTest('test_doctest.py', usemodules="thread"), RegrTest('test_doctest2.py'), RegrTest('test_docxmlrpc.py'), @@ -208,20 +204,22 @@ RegrTest('test_dummy_thread.py', core=True), RegrTest('test_dummy_threading.py', core=True), RegrTest('test_email.py'), - RegrTest('test_email_codecs.py'), + RegrTest('test_email_renamed.py'), RegrTest('test_enumerate.py', core=True), RegrTest('test_eof.py', core=True), RegrTest('test_epoll.py'), RegrTest('test_errno.py', usemodules="errno"), + RegrTest('test_exception_variations.py'), RegrTest('test_exceptions.py', core=True), RegrTest('test_extcall.py', core=True), - RegrTest('test_fcntl.py', usemodules='fcntl', skip=skip_win32), + RegrTest('test_fcntl.py', usemodules='fcntl'), RegrTest('test_file.py', usemodules="posix", core=True), RegrTest('test_file2k.py', usemodules="posix", core=True), RegrTest('test_filecmp.py', core=True), RegrTest('test_fileinput.py', core=True), RegrTest('test_fileio.py'), + RegrTest('test_float.py', core=True), RegrTest('test_fnmatch.py', core=True), RegrTest('test_fork1.py', usemodules="thread"), RegrTest('test_format.py', core=True), @@ -230,6 +228,7 @@ RegrTest('test_frozen.py', skip="unsupported extension module"), RegrTest('test_ftplib.py'), RegrTest('test_funcattrs.py', core=True), + RegrTest('test_functools.py'), RegrTest('test_future.py', core=True), RegrTest('test_future1.py', core=True), RegrTest('test_future2.py', core=True), @@ -239,41 +238,37 @@ RegrTest('test_future_builtins.py'), RegrTest('test_gc.py', usemodules='_weakref', skip="implementation detail"), RegrTest('test_gdb.py', skip="not applicable"), - RegrTest('test_gdbm.py', skip="unsupported extension module"), + RegrTest('test_gdbm.py'), RegrTest('test_generators.py', core=True, usemodules='thread _weakref'), RegrTest('test_genericpath.py'), RegrTest('test_genexps.py', core=True, usemodules='_weakref'), - RegrTest('test_getargs.py', skip="unsupported extension module"), - RegrTest('test_getargs2.py', skip="unsupported extension module"), - + RegrTest('test_getargs.py'), + RegrTest('test_getargs2.py', usemodules='binascii', skip=True), RegrTest('test_getopt.py', core=True), RegrTest('test_gettext.py'), - - RegrTest('test_gl.py', skip=True), + RegrTest('test_gl.py'), RegrTest('test_glob.py', core=True), RegrTest('test_global.py', core=True), RegrTest('test_grammar.py', core=True), - RegrTest('test_grp.py', skip=skip_win32), - - RegrTest('test_gzip.py'), + RegrTest('test_grp.py'), + RegrTest('test_gzip.py', usemodules='zlib'), RegrTest('test_hash.py', core=True), RegrTest('test_hashlib.py', core=True), - RegrTest('test_heapq.py', core=True), RegrTest('test_hmac.py'), RegrTest('test_hotshot.py', skip="unsupported extension module"), - RegrTest('test_htmllib.py'), RegrTest('test_htmlparser.py'), RegrTest('test_httplib.py'), RegrTest('test_httpservers.py'), - RegrTest('test_imageop.py', skip="unsupported extension module"), + RegrTest('test_imageop.py'), RegrTest('test_imaplib.py'), - RegrTest('test_imgfile.py', skip="unsupported extension module"), + RegrTest('test_imgfile.py'), RegrTest('test_imp.py', core=True, usemodules='thread'), RegrTest('test_import.py', core=True), RegrTest('test_importhooks.py', core=True), RegrTest('test_importlib.py'), + RegrTest('test_index.py'), RegrTest('test_inspect.py'), RegrTest('test_int.py', core=True), RegrTest('test_int_literal.py', core=True), @@ -296,7 +291,7 @@ RegrTest('test_long_future.py', core=True), RegrTest('test_longexp.py', core=True), RegrTest('test_macos.py'), - RegrTest('test_macostools.py', skip=True), + RegrTest('test_macostools.py'), RegrTest('test_macpath.py'), RegrTest('test_mailbox.py'), RegrTest('test_marshal.py', core=True), @@ -307,30 +302,29 @@ RegrTest('test_mhlib.py'), RegrTest('test_mimetools.py'), RegrTest('test_mimetypes.py'), - RegrTest('test_MimeWriter.py', core=False), + RegrTest('test_MimeWriter.py', core=False, usemodules='binascii'), RegrTest('test_minidom.py'), RegrTest('test_mmap.py', usemodules="mmap"), RegrTest('test_module.py', core=True), RegrTest('test_modulefinder.py'), - RegrTest('test_msilib.py', skip=only_win32), + RegrTest('test_msilib.py'), RegrTest('test_multibytecodec.py', usemodules='_multibytecodec'), - RegrTest('test_multibytecodec_support.py', skip="not a test"), RegrTest('test_multifile.py'), - RegrTest('test_multiprocessing.py', skip="FIXME leaves subprocesses"), + RegrTest('test_multiprocessing.py', skip=True), RegrTest('test_mutants.py', core="possibly"), RegrTest('test_mutex.py'), RegrTest('test_netrc.py'), RegrTest('test_new.py', core=True), - RegrTest('test_nis.py', skip="unsupported extension module"), + RegrTest('test_nis.py'), RegrTest('test_normalization.py'), RegrTest('test_ntpath.py'), + RegrTest('test_old_mailbox.py'), RegrTest('test_opcodes.py', core=True), RegrTest('test_openpty.py'), RegrTest('test_operator.py', core=True), RegrTest('test_optparse.py'), - RegrTest('test_os.py', core=True), - RegrTest('test_ossaudiodev.py', skip="unsupported extension module"), + RegrTest('test_ossaudiodev.py'), RegrTest('test_parser.py', skip="slowly deprecating compiler"), RegrTest('test_pdb.py'), RegrTest('test_peepholer.py'), @@ -338,14 +332,16 @@ RegrTest('test_pep263.py'), RegrTest('test_pep277.py'), RegrTest('test_pep292.py'), + RegrTest('test_pep352.py'), RegrTest('test_pickle.py', core=True), RegrTest('test_pickletools.py', core=False), RegrTest('test_pipes.py'), RegrTest('test_pkg.py', core=True), RegrTest('test_pkgimport.py', core=True), RegrTest('test_pkgutil.py'), - RegrTest('test_plistlib.py', skip="unsupported module"), - RegrTest('test_poll.py', skip=skip_win32), + RegrTest('test_platform.py'), + RegrTest('test_plistlib.py'), + RegrTest('test_poll.py'), RegrTest('test_popen.py'), RegrTest('test_popen2.py'), RegrTest('test_poplib.py'), @@ -357,8 +353,8 @@ RegrTest('test_profile.py'), RegrTest('test_property.py', core=True), RegrTest('test_pstats.py'), - RegrTest('test_pty.py', skip="unsupported extension module"), - RegrTest('test_pwd.py', usemodules="pwd", skip=skip_win32), + RegrTest('test_pty.py', usemodules='fcntl termios select'), + RegrTest('test_pwd.py', usemodules="pwd"), RegrTest('test_py3kwarn.py'), RegrTest('test_py_compile.py'), RegrTest('test_pyclbr.py'), @@ -370,15 +366,15 @@ RegrTest('test_re.py', core=True), RegrTest('test_readline.py'), RegrTest('test_repr.py', core=True), - RegrTest('test_resource.py', skip=skip_win32), + RegrTest('test_resource.py'), RegrTest('test_rfc822.py'), RegrTest('test_richcmp.py', core=True), RegrTest('test_rlcompleter.py'), - RegrTest('test_robotparser.py'), + RegrTest('test_runpy.py'), RegrTest('test_sax.py'), RegrTest('test_scope.py', core=True), - RegrTest('test_scriptpackages.py', skip="unsupported extension module"), + RegrTest('test_scriptpackages.py'), RegrTest('test_select.py'), RegrTest('test_set.py', core=True), RegrTest('test_sets.py'), @@ -389,64 +385,59 @@ RegrTest('test_shlex.py'), RegrTest('test_shutil.py'), RegrTest('test_signal.py'), - RegrTest('test_SimpleHTTPServer.py'), + RegrTest('test_SimpleHTTPServer.py', usemodules='binascii'), RegrTest('test_site.py', core=False), RegrTest('test_slice.py', core=True), RegrTest('test_smtplib.py'), RegrTest('test_smtpnet.py'), RegrTest('test_socket.py', usemodules='thread _weakref'), - RegrTest('test_socketserver.py', usemodules='thread'), - RegrTest('test_softspace.py', core=True), RegrTest('test_sort.py', core=True), + RegrTest('test_sqlite.py', usemodules="thread _rawffi zlib"), RegrTest('test_ssl.py', usemodules='_ssl _socket select'), + RegrTest('test_startfile.py'), RegrTest('test_str.py', core=True), - RegrTest('test_strftime.py'), RegrTest('test_string.py', core=True), - RegrTest('test_StringIO.py', core=True, usemodules='cStringIO'), + RegrTest('test_StringIO.py', core=True, usemodules='cStringIO array'), RegrTest('test_stringprep.py'), RegrTest('test_strop.py', skip="deprecated"), - RegrTest('test_strptime.py'), RegrTest('test_strtod.py'), RegrTest('test_struct.py', usemodules='struct'), RegrTest('test_structmembers.py', skip="CPython specific"), RegrTest('test_structseq.py'), RegrTest('test_subprocess.py', usemodules='signal'), - RegrTest('test_sunaudiodev.py', skip=True), + RegrTest('test_sunaudiodev.py'), RegrTest('test_sundry.py'), RegrTest('test_symtable.py', skip="implementation detail"), RegrTest('test_syntax.py', core=True), RegrTest('test_sys.py', core=True, usemodules='struct'), + RegrTest('test_sys_setprofile.py', core=True), RegrTest('test_sys_settrace.py', core=True), - RegrTest('test_sys_setprofile.py', core=True), RegrTest('test_sysconfig.py'), - RegrTest('test_tcl.py', skip="unsupported extension module"), RegrTest('test_tarfile.py'), + RegrTest('test_tcl.py'), RegrTest('test_telnetlib.py'), RegrTest('test_tempfile.py'), - RegrTest('test_textwrap.py'), RegrTest('test_thread.py', usemodules="thread", core=True), RegrTest('test_threaded_import.py', usemodules="thread", core=True), RegrTest('test_threadedtempfile.py', usemodules="thread", core=False), - RegrTest('test_threading.py', usemodules="thread", core=True), RegrTest('test_threading_local.py', usemodules="thread", core=True), RegrTest('test_threadsignals.py', usemodules="thread"), - RegrTest('test_time.py', core=True), RegrTest('test_timeout.py'), RegrTest('test_tk.py'), - RegrTest('test_ttk_guionly.py'), - RegrTest('test_ttk_textonly.py'), RegrTest('test_tokenize.py'), RegrTest('test_trace.py'), RegrTest('test_traceback.py', core=True), RegrTest('test_transformer.py', core=True), + RegrTest('test_ttk_guionly.py'), + RegrTest('test_ttk_textonly.py'), RegrTest('test_tuple.py', core=True), RegrTest('test_typechecks.py'), RegrTest('test_types.py', core=True), @@ -462,6 +453,7 @@ RegrTest('test_unpack.py', core=True), RegrTest('test_urllib.py'), RegrTest('test_urllib2.py'), + RegrTest('test_urllib2_localnet.py', usemodules="thread"), RegrTest('test_urllib2net.py'), RegrTest('test_urllibnet.py'), RegrTest('test_urlparse.py'), @@ -469,61 +461,38 @@ RegrTest('test_userlist.py', core=True), RegrTest('test_userstring.py', core=True), RegrTest('test_uu.py'), - - RegrTest('test_warnings.py', core=True), - RegrTest('test_wave.py', skip="unsupported extension module"), - RegrTest('test_weakref.py', core=True, usemodules='_weakref'), - RegrTest('test_weakset.py'), - - RegrTest('test_whichdb.py'), - RegrTest('test_winreg.py', skip=only_win32), - RegrTest('test_winsound.py', skip="unsupported extension module"), - RegrTest('test_xmllib.py'), - RegrTest('test_xmlrpc.py'), - - RegrTest('test_xpickle.py'), - RegrTest('test_xrange.py', core=True), - RegrTest('test_zipfile.py'), - RegrTest('test_zipimport.py', usemodules='zlib zipimport'), - RegrTest('test_zipimport_support.py', usemodules='zlib zipimport'), - RegrTest('test_zlib.py', usemodules='zlib'), - - RegrTest('test_bigaddrspace.py'), - RegrTest('test_bigmem.py'), - RegrTest('test_cmd_line.py'), - RegrTest('test_code.py'), - RegrTest('test_coding.py'), - RegrTest('test_complex_args.py'), - RegrTest('test_contextlib.py', usemodules="thread"), - RegrTest('test_ctypes.py', usemodules="_rawffi thread"), - RegrTest('test_defaultdict.py', usemodules='_collections'), - RegrTest('test_email_renamed.py'), - RegrTest('test_exception_variations.py'), - RegrTest('test_float.py'), - RegrTest('test_functools.py'), - RegrTest('test_index.py'), - RegrTest('test_old_mailbox.py'), - RegrTest('test_pep352.py'), - RegrTest('test_platform.py'), - RegrTest('test_runpy.py'), - RegrTest('test_sqlite.py', usemodules="thread _rawffi zlib"), - RegrTest('test_startfile.py', skip="bogus test"), - RegrTest('test_structmembers.py', skip="depends on _testcapi"), - RegrTest('test_urllib2_localnet.py', usemodules="thread"), RegrTest('test_uuid.py'), RegrTest('test_wait3.py', usemodules="thread"), RegrTest('test_wait4.py', usemodules="thread"), + RegrTest('test_warnings.py', core=True), + RegrTest('test_wave.py'), + RegrTest('test_weakref.py', core=True, usemodules='_weakref'), + RegrTest('test_weakset.py'), + RegrTest('test_whichdb.py'), + RegrTest('test_winreg.py'), + RegrTest('test_winsound.py'), RegrTest('test_with.py'), RegrTest('test_wsgiref.py'), RegrTest('test_xdrlib.py'), RegrTest('test_xml_etree.py'), RegrTest('test_xml_etree_c.py'), + RegrTest('test_xmllib.py'), + RegrTest('test_xmlrpc.py'), + RegrTest('test_xpickle.py'), + RegrTest('test_xrange.py', core=True), + RegrTest('test_zipfile.py'), RegrTest('test_zipfile64.py'), + RegrTest('test_zipimport.py', usemodules='zlib zipimport'), + RegrTest('test_zipimport_support.py', usemodules='zlib zipimport'), + RegrTest('test_zlib.py', usemodules='zlib'), ] def check_testmap_complete(): listed_names = dict.fromkeys([regrtest.basename for regrtest in testmap]) - listed_names['test_support.py'] = True # ignore this + assert len(listed_names) == len(testmap) + # names to ignore + listed_names['test_support.py'] = True + listed_names['test_multibytecodec_support.py'] = True missing = [] for path in testdir.listdir(fil='test_*.py'): name = path.basename @@ -578,7 +547,7 @@ def getinvocation(self, regrtest): fspath = regrtest.getfspath() python = sys.executable - pypy_script = pypydir.join('bin', 'py.py') + pypy_script = pypydir.join('bin', 'pyinteractive.py') alarm_script = pypydir.join('tool', 'alarm.py') if sys.platform == 'win32': watchdog_name = 'watchdog_nt.py' diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -343,9 +343,13 @@ signal, the tick counter is set to -1 by C code in signals.h. """ assert isinstance(action, PeriodicAsyncAction) - self._periodic_actions.append(action) + # hack to put the release-the-GIL one at the end of the list, + # and the report-the-signals one at the start of the list. if use_bytecode_counter: + self._periodic_actions.append(action) self.has_bytecode_counter = True + else: + self._periodic_actions.insert(0, action) self._rebuild_action_dispatcher() def getcheckinterval(self): @@ -419,15 +423,6 @@ The action must have been registered at space initalization time.""" self.space.actionflag.fire(self) - def fire_after_thread_switch(self): - """Bit of a hack: fire() the action but only the next time the GIL - is released and re-acquired (i.e. after a potential thread switch). - Don't call this if threads are not enabled. Currently limited to - one action (i.e. reserved for CheckSignalAction from module/signal). - """ - from pypy.module.thread.gil import spacestate - spacestate.action_after_thread_switch = self - def perform(self, executioncontext, frame): """To be overridden.""" diff --git a/pypy/module/_multiprocessing/test/test_connection.py b/pypy/module/_multiprocessing/test/test_connection.py --- a/pypy/module/_multiprocessing/test/test_connection.py +++ b/pypy/module/_multiprocessing/test/test_connection.py @@ -68,7 +68,7 @@ assert rhandle.readable class AppTestWinpipeConnection(BaseConnectionTest): - spaceconfig = dict(usemodules=('_multiprocessing', 'thread')) + spaceconfig = dict(usemodules=('_multiprocessing', 'thread', 'signal')) def setup_class(cls): if sys.platform != "win32": diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -181,7 +181,8 @@ try: self.close() except SocketError, e: - raise converted_error(space, e) + # cpython doesn't return any errors on close + pass def connect_w(self, space, w_addr): """connect(address) @@ -448,7 +449,7 @@ w_addr = space.w_None return space.newtuple([space.wrap(readlgt), w_addr]) except SocketError, e: - raise converted_error(space, e) + raise converted_error(space, e) @unwrap_spec(cmd=int) def ioctl_w(self, space, cmd, w_option): diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -362,13 +362,15 @@ assert isinstance(s.fileno(), int) def test_socket_close(self): - import _socket + import _socket, os s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM, 0) fileno = s.fileno() assert s.fileno() >= 0 s.close() assert s.fileno() < 0 s.close() + if os.name != 'nt': + raises(OSError, os.close, fileno) def test_socket_close_error(self): import _socket, os @@ -376,7 +378,7 @@ skip("Windows sockets are not files") s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM, 0) os.close(s.fileno()) - raises(_socket.error, s.close) + s.close() def test_socket_connect(self): import _socket, os diff --git a/pypy/module/crypt/test/test_crypt.py b/pypy/module/crypt/test/test_crypt.py --- a/pypy/module/crypt/test/test_crypt.py +++ b/pypy/module/crypt/test/test_crypt.py @@ -1,6 +1,12 @@ +import os +import py + +if os.name != 'posix': + py.test.skip('crypt module only available on unix') + class AppTestCrypt: spaceconfig = dict(usemodules=['crypt']) - + def test_crypt(self): import crypt res = crypt.crypt("pass", "ab") diff --git a/pypy/module/fcntl/test/test_fcntl.py b/pypy/module/fcntl/test/test_fcntl.py --- a/pypy/module/fcntl/test/test_fcntl.py +++ b/pypy/module/fcntl/test/test_fcntl.py @@ -1,9 +1,9 @@ import os +import py from rpython.tool.udir import udir -if os.name == "nt": - from py.test import skip - skip("fcntl module is not available on Windows") +if os.name != 'posix': + py.test.skip("fcntl module only available on unix") def teardown_module(mod): for i in "abcde": diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -823,14 +823,19 @@ if hasattr(os, 'chmod'): def test_chmod(self): + import sys os = self.posix os.unlink(self.path) raises(OSError, os.chmod, self.path, 0600) f = open(self.path, "w") f.write("this is a test") f.close() - os.chmod(self.path, 0200) - assert (os.stat(self.path).st_mode & 0777) == 0200 + if sys.platform == 'win32': + os.chmod(self.path, 0400) + assert (os.stat(self.path).st_mode & 0600) == 0400 + else: + os.chmod(self.path, 0200) + assert (os.stat(self.path).st_mode & 0777) == 0200 if hasattr(os, 'fchmod'): def test_fchmod(self): diff --git a/pypy/module/pwd/test/test_pwd.py b/pypy/module/pwd/test/test_pwd.py --- a/pypy/module/pwd/test/test_pwd.py +++ b/pypy/module/pwd/test/test_pwd.py @@ -1,3 +1,9 @@ +import os +import py + +if os.name != 'posix': + py.test.skip('pwd module only available on unix') + class AppTestPwd: spaceconfig = dict(usemodules=['pwd']) diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py --- a/pypy/module/signal/interp_signal.py +++ b/pypy/module/signal/interp_signal.py @@ -8,7 +8,8 @@ PeriodicAsyncAction) from pypy.interpreter.gateway import unwrap_spec -from rpython.rlib import jit, rposix +from rpython.rlib import jit, rposix, rgc +from rpython.rlib.objectmodel import we_are_translated from rpython.rlib.rarithmetic import intmask from rpython.rlib.rsignal import * from rpython.rtyper.lltypesystem import lltype, rffi @@ -31,6 +32,11 @@ p = pypysig_getaddr_occurred() p.c_value = value + @staticmethod + def rearm_ticker(): + p = pypysig_getaddr_occurred() + p.c_value = -1 + def decrement_ticker(self, by): p = pypysig_getaddr_occurred() value = p.c_value @@ -46,41 +52,63 @@ class CheckSignalAction(PeriodicAsyncAction): """An action that is automatically invoked when a signal is received.""" + # Note that this is a PeriodicAsyncAction: it means more precisely + # that it is called whenever the C-level ticker becomes < 0. + # Without threads, it is only ever set to -1 when we receive a + # signal. With threads, it also decrements steadily (but slowly). + def __init__(self, space): + "NOT_RPYTHON" AsyncAction.__init__(self, space) self.handlers_w = {} - self.emulated_sigint = False + self.pending_signal = -1 + self.fire_in_main_thread = False + if self.space.config.objspace.usemodules.thread: + from pypy.module.thread import gil + gil.after_thread_switch = self._after_thread_switch + + @rgc.no_collect + def _after_thread_switch(self): + if self.fire_in_main_thread: + if self.space.threadlocals.ismainthread(): + self.fire_in_main_thread = False + SignalActionFlag.rearm_ticker() + # this occurs when we just switched to the main thread + # and there is a signal pending: we force the ticker to + # -1, which should ensure perform() is called quickly. @jit.dont_look_inside def perform(self, executioncontext, frame): - if self.space.config.objspace.usemodules.thread: - main_ec = self.space.threadlocals.getmainthreadvalue() - in_main = executioncontext is main_ec - else: - in_main = True - # If we are in the main thread, poll and report the signals now. - if in_main: - if self.emulated_sigint: - self.emulated_sigint = False - self._report_signal(cpy_signal.SIGINT) - while True: - n = pypysig_poll() - if n < 0: - break + # Poll for the next signal, if any + n = self.pending_signal + if n < 0: n = pypysig_poll() + while n >= 0: + if self.space.config.objspace.usemodules.thread: + in_main = self.space.threadlocals.ismainthread() + else: + in_main = True + if in_main: + # If we are in the main thread, report the signal now, + # and poll more + self.pending_signal = -1 self._report_signal(n) - else: - # Otherwise, don't call pypysig_poll() at all. Instead, - # arrange for perform() to be called again after a thread - # switch. It might be called again and again, until we - # land in the main thread. - self.fire_after_thread_switch() + n = self.pending_signal + if n < 0: n = pypysig_poll() + else: + # Otherwise, arrange for perform() to be called again + # after we switch to the main thread. + self.pending_signal = n + self.fire_in_main_thread = True + break - @jit.dont_look_inside def set_interrupt(self): "Simulates the effect of a SIGINT signal arriving" - ec = self.space.getexecutioncontext() - self.emulated_sigint = True - self.perform(ec, None) + if not we_are_translated(): + self.pending_signal = cpy_signal.SIGINT + # ^^^ may override another signal, but it's just for testing + else: + pypysig_pushback(cpy_signal.SIGINT) + self.fire_in_main_thread = True def _report_signal(self, n): try: diff --git a/pypy/module/termios/test/test_termios.py b/pypy/module/termios/test/test_termios.py --- a/pypy/module/termios/test/test_termios.py +++ b/pypy/module/termios/test/test_termios.py @@ -1,9 +1,12 @@ - +import os +import sys import py -import sys from pypy.conftest import pypydir from rpython.tool.udir import udir +if os.name != 'posix': + py.test.skip('termios module only available on unix') + class TestTermios(object): def setup_class(cls): try: diff --git a/pypy/module/thread/gil.py b/pypy/module/thread/gil.py --- a/pypy/module/thread/gil.py +++ b/pypy/module/thread/gil.py @@ -62,22 +62,7 @@ do_yield_thread() -class SpaceState: - - def _cleanup_(self): - self.action_after_thread_switch = None - # ^^^ set by AsyncAction.fire_after_thread_switch() - - def after_thread_switch(self): - # this is support logic for the signal module, to help it deliver - # signals to the main thread. - action = self.action_after_thread_switch - if action is not None: - self.action_after_thread_switch = None - action.fire() - -spacestate = SpaceState() -spacestate._cleanup_() +after_thread_switch = lambda: None # hook for signal.py # Fragile code below. We have to preserve the C-level errno manually... @@ -94,7 +79,7 @@ e = get_errno() thread.gil_acquire() thread.gc_thread_run() - spacestate.after_thread_switch() + after_thread_switch() set_errno(e) after_external_call._gctransformer_hint_cannot_collect_ = True after_external_call._dont_reach_me_in_del_ = True @@ -112,7 +97,7 @@ # the same thread). if thread.gil_yield_thread(): thread.gc_thread_run() - spacestate.after_thread_switch() + after_thread_switch() do_yield_thread._gctransformer_hint_close_stack_ = True do_yield_thread._dont_reach_me_in_del_ = True do_yield_thread._dont_inline_ = True diff --git a/pypy/module/thread/threadlocals.py b/pypy/module/thread/threadlocals.py --- a/pypy/module/thread/threadlocals.py +++ b/pypy/module/thread/threadlocals.py @@ -48,6 +48,9 @@ ident = self._mainthreadident return self._valuedict.get(ident, None) + def ismainthread(self): + return thread.get_ident() == self._mainthreadident + def getallvalues(self): return self._valuedict diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py --- a/rpython/annotator/annrpython.py +++ b/rpython/annotator/annrpython.py @@ -113,6 +113,9 @@ if policy is None: from rpython.annotator.policy import AnnotatorPolicy policy = AnnotatorPolicy() + # XXX hack + annmodel.TLS.check_str_without_nul = ( + self.translator.config.translation.check_str_without_nul) graph, inputcells = self.get_call_parameters(function, args_s, policy) self.build_graph_types(graph, inputcells, complete_now=False) self.complete_helpers(policy) diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py --- a/rpython/annotator/binaryop.py +++ b/rpython/annotator/binaryop.py @@ -446,7 +446,8 @@ class __extend__(pairtype(SomeChar, SomeChar)): def union((chr1, chr2)): - return SomeChar() + no_nul = chr1.no_nul and chr2.no_nul + return SomeChar(no_nul=no_nul) class __extend__(pairtype(SomeChar, SomeUnicodeCodePoint), @@ -664,14 +665,14 @@ def getitem((str1, int2)): getbookkeeper().count("str_getitem", int2) - return SomeChar() + return SomeChar(no_nul=str1.no_nul) getitem.can_only_throw = [] getitem_key = getitem def getitem_idx((str1, int2)): getbookkeeper().count("str_getitem", int2) - return SomeChar() + return SomeChar(no_nul=str1.no_nul) getitem_idx.can_only_throw = [IndexError] getitem_idx_key = getitem_idx diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py --- a/rpython/annotator/test/test_annrpython.py +++ b/rpython/annotator/test/test_annrpython.py @@ -2068,7 +2068,23 @@ s = a.build_types(f, [annmodel.SomeString(no_nul=True)]) assert isinstance(s, annmodel.SomeString) assert s.no_nul - + + def test_getitem_str0(self): + def f(s, n): + if n == 1: + return s[0] + elif n == 2: + return s[1] + elif n == 3: + return s[1:] + return s + a = self.RPythonAnnotator() + a.translator.config.translation.check_str_without_nul = True + + s = a.build_types(f, [annmodel.SomeString(no_nul=True), + annmodel.SomeInteger()]) + assert isinstance(s, annmodel.SomeString) + assert s.no_nul def test_non_none_and_none_with_isinstance(self): class A(object): diff --git a/rpython/rlib/rcomplex.py b/rpython/rlib/rcomplex.py --- a/rpython/rlib/rcomplex.py +++ b/rpython/rlib/rcomplex.py @@ -566,3 +566,6 @@ def c_isnan(r, i): return isnan(r) or isnan(i) + +def c_isfinite(r, i): + return isfinite(r) and isfinite(i) diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py --- a/rpython/rlib/rposix.py +++ b/rpython/rlib/rposix.py @@ -247,6 +247,20 @@ else: return nt._getfullpathname(path.as_bytes()) + at specialize.argtype(0, 1) +def putenv(name, value): + if isinstance(name, str): + os.environ[name] = value + else: + os.environ[name.as_bytes()] = value.as_bytes() + + at specialize.argtype(0) +def unsetenv(name): + if isinstance(name, str): + del os.environ[name] + else: + del os.environ[name.as_bytes()] + if os.name == 'nt': from rpython.rlib import rwin32 os_kill = rwin32.os_kill diff --git a/rpython/rlib/rsignal.py b/rpython/rlib/rsignal.py --- a/rpython/rlib/rsignal.py +++ b/rpython/rlib/rsignal.py @@ -80,6 +80,8 @@ pypysig_poll = external('pypysig_poll', [], rffi.INT, threadsafe=False) # don't bother releasing the GIL around a call to pypysig_poll: it's # pointless and a performance issue +pypysig_pushback = external('pypysig_pushback', [rffi.INT], lltype.Void, + threadsafe=False) # don't use rffi.LONGP because the JIT doesn't support raw arrays so far struct_name = 'pypysig_long_struct' diff --git a/rpython/rlib/test/test_jit.py b/rpython/rlib/test/test_jit.py --- a/rpython/rlib/test/test_jit.py +++ b/rpython/rlib/test/test_jit.py @@ -35,8 +35,8 @@ assert driver.reds == ['a', 'b'] assert driver.numreds == 2 - at py.test.mark.xfail(reason="@inline off: see skipped failures in test_warmspot.") def test_jitdriver_inline(): + py.test.skip("@inline off: see skipped failures in test_warmspot.") driver = JitDriver(greens=[], reds='auto') calls = [] def foo(a, b): @@ -55,8 +55,8 @@ ('bar', 40, 2), ] - at py.test.mark.xfail(reason="@inline off: see skipped failures in test_warmspot.") def test_jitdriver_clone(): + py.test.skip("@inline off: see skipped failures in test_warmspot.") def bar(): pass def foo(): pass driver = JitDriver(greens=[], reds=[]) diff --git a/rpython/rlib/test/test_rcomplex.py b/rpython/rlib/test/test_rcomplex.py --- a/rpython/rlib/test/test_rcomplex.py +++ b/rpython/rlib/test/test_rcomplex.py @@ -260,3 +260,24 @@ rAssertAlmostEqual(expected[1], actual[1], abs_err=real_abs_err, msg=error_message) + +def test_isnan(): + assert not c.c_isnan(0, 0) + assert c.c_isnan(float('nan'), 0) + assert c.c_isnan(1, float('nan')) + assert not c.c_isnan(float('inf'), 0) + +def test_isinf(): + assert not c.c_isinf(0, 0) + assert c.c_isinf(float('inf'), 0) + assert c.c_isinf(float('-inf'), 0) + assert c.c_isinf(1, float('inf')) + assert not c.c_isinf(float('nan'), 0) + +def test_isfinite(): + assert c.c_isfinite(0, 0) + assert not c.c_isfinite(float('nan'), 0) + assert not c.c_isfinite(float('-inf'), 0) + assert not c.c_isfinite(0, float('nan')) + assert not c.c_isfinite(0, float('-inf')) + diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py --- a/rpython/rlib/test/test_rposix.py +++ b/rpython/rlib/test/test_rposix.py @@ -141,3 +141,10 @@ assert rposix.is_valid_fd(fd) == 1 fid.close() assert rposix.is_valid_fd(fd) == 0 + + def test_putenv(self): + def f(): + rposix.putenv(self.path, self.path) + rposix.unsetenv(self.path) + + interpret(f, []) # does not crash diff --git a/rpython/rlib/test/test_signature.py b/rpython/rlib/test/test_signature.py --- a/rpython/rlib/test/test_signature.py +++ b/rpython/rlib/test/test_signature.py @@ -9,6 +9,7 @@ def annotate_at(f, policy=None): t = TranslationContext() + t.config.translation.check_str_without_nul = True a = t.buildannotator(policy=policy) a.annotate_helper(f, [model.s_ImpossibleValue]*f.func_code.co_argcount, policy=policy) return a @@ -112,6 +113,12 @@ return len(u) assert getsig(f) == [model.SomeUnicodeString(), model.SomeInteger()] +def test_str0(): + @signature(types.unicode0(), returns=types.str0()) + def f(u): + return 'str' + assert getsig(f) == [model.SomeUnicodeString(no_nul=True), + model.SomeString(no_nul=True)] def test_ptr(): policy = LowLevelAnnotatorPolicy() diff --git a/rpython/rlib/types.py b/rpython/rlib/types.py --- a/rpython/rlib/types.py +++ b/rpython/rlib/types.py @@ -24,9 +24,15 @@ def unicode(): return model.SomeUnicodeString() +def unicode0(): + return model.SomeUnicodeString(no_nul=True) + def str(): return model.SomeString() +def str0(): + return model.SomeString(no_nul=True) + def char(): return model.SomeChar() diff --git a/rpython/rtyper/lltypesystem/ll2ctypes.py b/rpython/rtyper/lltypesystem/ll2ctypes.py --- a/rpython/rtyper/lltypesystem/ll2ctypes.py +++ b/rpython/rtyper/lltypesystem/ll2ctypes.py @@ -32,6 +32,15 @@ except ImportError: class tlsobject(object): pass +try: + from threading import RLock +except ImportError: + class RLock(object): + def __enter__(self): + pass + def __exit__(self, *args): + pass +rlock = RLock() _POSIX = os.name == "posix" _MS_WINDOWS = os.name == "nt" @@ -694,10 +703,11 @@ return None def lltype2ctypes(llobj, normalize=True): - """Convert the lltype object 'llobj' to its ctypes equivalent. - 'normalize' should only be False in tests, where we want to - inspect the resulting ctypes object manually. - """ + """Convert the lltype object 'llobj' to its ctypes equivalent. + 'normalize' should only be False in tests, where we want to + inspect the resulting ctypes object manually. + """ + with rlock: if isinstance(llobj, lltype._uninitialized): return uninitialized2ctypes(llobj.TYPE) if isinstance(llobj, llmemory.AddressAsInt): @@ -875,9 +885,10 @@ return llobj def ctypes2lltype(T, cobj): - """Convert the ctypes object 'cobj' to its lltype equivalent. - 'T' is the expected lltype type. - """ + """Convert the ctypes object 'cobj' to its lltype equivalent. + 'T' is the expected lltype type. + """ + with rlock: if T is lltype.Void: return None if isinstance(T, lltype.Typedef): @@ -1176,10 +1187,11 @@ #self.funcptr = ... set later def __call__(self, *argvalues): - if self.trampoline is None: - # lazily build the corresponding ctypes function object - cfunc = get_ctypes_callable(self.funcptr, self.calling_conv) - self.trampoline = get_ctypes_trampoline(self.FUNCTYPE, cfunc) + with rlock: + if self.trampoline is None: + # lazily build the corresponding ctypes function object + cfunc = get_ctypes_callable(self.funcptr, self.calling_conv) + self.trampoline = get_ctypes_trampoline(self.FUNCTYPE, cfunc) # perform the call return self.trampoline(*argvalues) @@ -1215,8 +1227,9 @@ return ctypes2lltype(RESULT, cres) return invoke_via_ctypes + def force_cast(RESTYPE, value): - """Cast a value to a result type, trying to use the same rules as C.""" + with rlock: if not isinstance(RESTYPE, lltype.LowLevelType): raise TypeError("rffi.cast() first arg should be a TYPE") if isinstance(value, llmemory.AddressAsInt): diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py --- a/rpython/rtyper/lltypesystem/rffi.py +++ b/rpython/rtyper/lltypesystem/rffi.py @@ -867,6 +867,8 @@ array[len(l)] = lltype.nullptr(CCHARP.TO) return array liststr2charpp._annenforceargs_ = [[annmodel.s_Str0]] # List of strings +# Make a copy for the ll_os.py module +ll_liststr2charpp = func_with_new_name(liststr2charpp, 'll_liststr2charpp') def free_charpp(ref): """ frees list of char**, NULL terminated diff --git a/rpython/rtyper/module/ll_os.py b/rpython/rtyper/module/ll_os.py --- a/rpython/rtyper/module/ll_os.py +++ b/rpython/rtyper/module/ll_os.py @@ -306,7 +306,7 @@ rffi.INT, compilation_info = eci) def execv_llimpl(path, args): - l_args = rffi.liststr2charpp(args) + l_args = rffi.ll_liststr2charpp(args) os_execv(path, l_args) rffi.free_charpp(l_args) raise OSError(rposix.get_errno(), "execv failed") @@ -332,8 +332,8 @@ envstr = "%s=%s" % item envstrs.append(envstr) - l_args = rffi.liststr2charpp(args) - l_env = rffi.liststr2charpp(envstrs) + l_args = rffi.ll_liststr2charpp(args) + l_env = rffi.ll_liststr2charpp(envstrs) os_execve(path, l_args, l_env) # XXX untested @@ -357,7 +357,7 @@ def spawnv_llimpl(mode, path, args): mode = rffi.cast(rffi.INT, mode) - l_args = rffi.liststr2charpp(args) + l_args = rffi.ll_liststr2charpp(args) childpid = os_spawnv(mode, path, l_args) rffi.free_charpp(l_args) if childpid == -1: @@ -380,8 +380,8 @@ envstrs.append("%s=%s" % item) mode = rffi.cast(rffi.INT, mode) - l_args = rffi.liststr2charpp(args) - l_env = rffi.liststr2charpp(envstrs) + l_args = rffi.ll_liststr2charpp(args) + l_env = rffi.ll_liststr2charpp(envstrs) childpid = os_spawnve(mode, path, l_args, l_env) rffi.free_charpp(l_env) rffi.free_charpp(l_args) diff --git a/rpython/rtyper/rstr.py b/rpython/rtyper/rstr.py --- a/rpython/rtyper/rstr.py +++ b/rpython/rtyper/rstr.py @@ -80,7 +80,7 @@ assert s is not None bytes = self.runicode_encode_utf_8( s, len(s), 'strict', - errorhandler=self.ll_raise_unicode_exception_decode, + errorhandler=self.ll_raise_unicode_exception_encode, allow_surrogates=False) return self.ll.llstr(bytes) diff --git a/rpython/rtyper/test/test_runicode.py b/rpython/rtyper/test/test_runicode.py --- a/rpython/rtyper/test/test_runicode.py +++ b/rpython/rtyper/test/test_runicode.py @@ -106,6 +106,12 @@ assert self.ll_to_string(self.interpret(f, [38])) == f(38) + def g(n): + x = u'\ud800' + unichr(n) + return x.encode('utf-8') + + self.interpret_raises(UnicodeEncodeError, g, [38]) + def test_utf_8_encoding_annotation(self): from rpython.rlib.runicode import unicode_encode_utf_8 def errorhandler(errors, encoding, msg, u, diff --git a/rpython/tool/stdlib_opcode.py b/rpython/tool/stdlib_opcode.py --- a/rpython/tool/stdlib_opcode.py +++ b/rpython/tool/stdlib_opcode.py @@ -71,10 +71,14 @@ def to_globals(self, globals_dict): """NOT_RPYTHON. Add individual opcodes to the module constants.""" - globals_dict.update(self.opmap) - globals_dict['SLICE'] = self.opmap["SLICE+0"] - globals_dict['STORE_SLICE'] = self.opmap["STORE_SLICE+0"] - globals_dict['DELETE_SLICE'] = self.opmap["DELETE_SLICE+0"] + for name, value in self.opmap.iteritems(): + # Rename 'STORE_SLICE+0' opcodes + if name.endswith('+0'): + name = name[:-2] + # Ignore 'STORE_SLICE+1' opcodes + elif name.endswith(('+1', '+2', '+3')): + continue + globals_dict[name] = value def __str__(self): return "<%s bytecode>" % (self.name,) @@ -83,4 +87,4 @@ from opcode import opmap, HAVE_ARGUMENT -host_bytecode_spec = BytecodeSpec('host', opmap, HAVE_ARGUMENT) \ No newline at end of file +host_bytecode_spec = BytecodeSpec('host', opmap, HAVE_ARGUMENT) diff --git a/rpython/translator/c/src/signals.c b/rpython/translator/c/src/signals.c --- a/rpython/translator/c/src/signals.c +++ b/rpython/translator/c/src/signals.c @@ -71,7 +71,7 @@ #endif } -static void signal_setflag_handler(int signum) +void pypysig_pushback(int signum) { if (0 <= signum && signum < NSIG) { @@ -79,6 +79,11 @@ pypysig_occurred = 1; pypysig_counter.value = -1; } +} + +static void signal_setflag_handler(int signum) +{ + pypysig_pushback(signum); if (wakeup_fd != -1) { diff --git a/rpython/translator/c/src/signals.h b/rpython/translator/c/src/signals.h --- a/rpython/translator/c/src/signals.h +++ b/rpython/translator/c/src/signals.h @@ -11,6 +11,7 @@ /* utility to poll for signals that arrived */ int pypysig_poll(void); /* => signum or -1 */ +void pypysig_pushback(int signum); /* When a signal is received, pypysig_counter is set to -1. */ /* This is a struct for the JIT. See rsignal.py. */ From noreply at buildbot.pypy.org Tue Jan 29 13:17:55 2013 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 29 Jan 2013 13:17:55 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: enfore the args here Message-ID: <20130129121755.0FC4D1C13C4@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60672:98d856650ece Date: 2013-01-29 14:17 +0200 http://bitbucket.org/pypy/pypy/changeset/98d856650ece/ Log: enfore the args here diff --git a/rpython/jit/backend/llsupport/jitframe.py b/rpython/jit/backend/llsupport/jitframe.py --- a/rpython/jit/backend/llsupport/jitframe.py +++ b/rpython/jit/backend/llsupport/jitframe.py @@ -2,6 +2,7 @@ from rpython.rtyper.annlowlevel import llhelper from rpython.rlib.objectmodel import specialize from rpython.rlib.debug import ll_assert +from rpython.rlib.objectmodel import enforceargs SIZEOFSIGNED = rffi.sizeof(lltype.Signed) IS_32BIT = (SIZEOFSIGNED == 2 ** 31 - 1) @@ -13,6 +14,7 @@ GCMAP = lltype.GcArray(lltype.Unsigned) NULLGCMAP = lltype.nullptr(GCMAP) + at enforceargs(None, int, int) def jitframeinfo_set_depth(jfi, base_ofs, new_depth): jfi.jfi_frame_depth = new_depth jfi.jfi_frame_size = base_ofs + new_depth * SIZEOFSIGNED From noreply at buildbot.pypy.org Tue Jan 29 13:42:16 2013 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 29 Jan 2013 13:42:16 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: use single logic statements, translation produces incorrect code otherwise Message-ID: <20130129124216.68F071C0041@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: missing-ndarray-attributes Changeset: r60673:7f7a5ee5c4f8 Date: 2013-01-29 14:42 +0200 http://bitbucket.org/pypy/pypy/changeset/7f7a5ee5c4f8/ Log: use single logic statements, translation produces incorrect code otherwise diff --git a/pypy/module/micronumpy/arrayimpl/sort.py b/pypy/module/micronumpy/arrayimpl/sort.py --- a/pypy/module/micronumpy/arrayimpl/sort.py +++ b/pypy/module/micronumpy/arrayimpl/sort.py @@ -82,10 +82,16 @@ def argsort_array(arr, space, w_axis): itemtype = arr.dtype.itemtype - # this is for runtime - if not (arr.dtype.is_int_type() or \ - arr.dtype.is_float_type() or \ - arr.dtype.is_complex_type()): + # this is for runtime - written like this to simplify + # logic since translation could not handle it as one + # statement + if arr.dtype.is_int_type(): + pass + elif arr.dtype.is_float_type(): + pass + elif arr.dtype.is_complex_type(): + pass + else: raise OperationError(space.w_NotImplementedError, space.wrap("sorting of non-numeric types " + \ "'%s' is not implemented" % arr.dtype.get_name() )) diff --git a/pypy/tool/watchdog_nt.py b/pypy/tool/watchdog_nt.py new file mode 100644 --- /dev/null +++ b/pypy/tool/watchdog_nt.py @@ -0,0 +1,33 @@ +import sys, os +import threading +import ctypes + +def childkill(pid): + global timedout + timedout = True + sys.stderr.write("==== test running for %d seconds ====\n" % timeout) + sys.stderr.write("="*26 + "timedout" + "="*26 + "\n") + ctypes.windll.kernel32.TerminateProcess(pid, 1) + +if __name__ == '__main__': + PROCESS_TERMINATE = 0x1 + + timeout = float(sys.argv[1]) + timedout = False + + pid = os.spawnv(os.P_NOWAIT, sys.argv[2], sys.argv[2:]) + + t = threading.Timer(timeout, childkill, (pid,)) + t.start() + while True: + try: + pid, status = os.waitpid(pid, 0) + except KeyboardInterrupt: + continue + else: + t.cancel() + break + + #print 'status ', status >> 8 + sys.exit(status >> 8) + From noreply at buildbot.pypy.org Tue Jan 29 14:07:14 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Tue, 29 Jan 2013 14:07:14 +0100 (CET) Subject: [pypy-commit] pypy default: typos already in upstream Message-ID: <20130129130714.53B871C039A@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60674:4b3a8e3689cf Date: 2013-01-29 08:06 -0500 http://bitbucket.org/pypy/pypy/changeset/4b3a8e3689cf/ Log: typos already in upstream diff --git a/lib-python/2.7/inspect.py b/lib-python/2.7/inspect.py --- a/lib-python/2.7/inspect.py +++ b/lib-python/2.7/inspect.py @@ -960,7 +960,7 @@ raise TypeError('%s() takes exactly 0 arguments ' '(%d given)' % (f_name, num_total)) else: - raise TypeError('%s() takes no argument (%d given)' % + raise TypeError('%s() takes no arguments (%d given)' % (f_name, num_total)) for arg in args: if isinstance(arg, str) and arg in named: diff --git a/lib-python/2.7/site.py b/lib-python/2.7/site.py --- a/lib-python/2.7/site.py +++ b/lib-python/2.7/site.py @@ -75,6 +75,7 @@ USER_SITE = None USER_BASE = None + def makepath(*paths): dir = os.path.join(*paths) try: diff --git a/lib-python/2.7/test/test_itertools.py b/lib-python/2.7/test/test_itertools.py --- a/lib-python/2.7/test/test_itertools.py +++ b/lib-python/2.7/test/test_itertools.py @@ -533,11 +533,11 @@ self.assertEqual(list(izip()), zip()) self.assertRaises(TypeError, izip, 3) self.assertRaises(TypeError, izip, range(3), 3) - self.assertEqual([tuple(list(pair)) for pair in izip('abc', 'def')], zip('abc', 'def')) self.assertEqual([pair for pair in izip('abc', 'def')], zip('abc', 'def')) + @test_support.impl_detail("tuple reuse is specific to CPython") def test_izip_tuple_reuse(self): ids = map(id, izip('abc', 'def')) @@ -588,6 +588,7 @@ zip('abc', 'def')) self.assertEqual([pair for pair in izip_longest('abc', 'def')], zip('abc', 'def')) + @test_support.impl_detail("tuple reuse is specific to CPython") def test_izip_longest_tuple_reuse(self): ids = map(id, izip_longest('abc', 'def')) diff --git a/lib-python/2.7/test/test_support.py b/lib-python/2.7/test/test_support.py --- a/lib-python/2.7/test/test_support.py +++ b/lib-python/2.7/test/test_support.py @@ -1085,7 +1085,6 @@ else: runner = BasicTestRunner() - result = runner.run(suite) if not result.wasSuccessful(): if len(result.errors) == 1 and not result.failures: From noreply at buildbot.pypy.org Tue Jan 29 14:39:15 2013 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 29 Jan 2013 14:39:15 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: make gcmap and frame_info raw-allocated structures Message-ID: <20130129133915.5E27B1C0041@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60675:ae1decf6800b Date: 2013-01-29 14:22 +0200 http://bitbucket.org/pypy/pypy/changeset/ae1decf6800b/ Log: make gcmap and frame_info raw-allocated structures diff --git a/rpython/jit/backend/llsupport/jitframe.py b/rpython/jit/backend/llsupport/jitframe.py --- a/rpython/jit/backend/llsupport/jitframe.py +++ b/rpython/jit/backend/llsupport/jitframe.py @@ -11,7 +11,7 @@ # compiled loop token (in fact we could use this as a compiled loop token # XXX do this -GCMAP = lltype.GcArray(lltype.Unsigned) +GCMAP = lltype.Array(lltype.Unsigned) NULLGCMAP = lltype.nullptr(GCMAP) @enforceargs(None, int, int) @@ -19,7 +19,7 @@ jfi.jfi_frame_depth = new_depth jfi.jfi_frame_size = base_ofs + new_depth * SIZEOFSIGNED -JITFRAMEINFO = lltype.GcStruct( +JITFRAMEINFO = lltype.Struct( 'JITFRAMEINFO', # the depth of the frame ('jfi_frame_depth', lltype.Signed), @@ -84,25 +84,19 @@ def jitframe_trace(obj_addr, prev): if prev == llmemory.NULL: (obj_addr + getofs('jf_gc_trace_state')).signed[0] = 0 - return obj_addr + getofs('jf_frame_info') + return obj_addr + getofs('jf_descr') fld = (obj_addr + getofs('jf_gc_trace_state')).signed[0] state = fld & 0x7 # 3bits of possible states if state == 0: (obj_addr + getofs('jf_gc_trace_state')).signed[0] = 1 - return obj_addr + getofs('jf_descr') + return obj_addr + getofs('jf_force_descr') elif state == 1: (obj_addr + getofs('jf_gc_trace_state')).signed[0] = 2 - return obj_addr + getofs('jf_force_descr') + return obj_addr + getofs('jf_savedata') elif state == 2: (obj_addr + getofs('jf_gc_trace_state')).signed[0] = 3 - return obj_addr + getofs('jf_gcmap') - elif state == 3: - (obj_addr + getofs('jf_gc_trace_state')).signed[0] = 4 - return obj_addr + getofs('jf_savedata') - elif state == 4: - (obj_addr + getofs('jf_gc_trace_state')).signed[0] = 5 return obj_addr + getofs('jf_guard_exc') - ll_assert(state == 5, "invalid state") + ll_assert(state == 3, "invalid state") # bit pattern # decode the pattern if IS_32BIT: @@ -127,9 +121,9 @@ # found it # save new state if IS_32BIT: - new_state = 5 | ((state + 1) << 3) | (no << 8) + new_state = 3 | ((state + 1) << 3) | (no << 8) else: - new_state = 5 | ((state + 1) << 3) | (no << 9) + new_state = 3 | ((state + 1) << 3) | (no << 9) (obj_addr + getofs('jf_gc_trace_state')).signed[0] = new_state return (obj_addr + getofs('jf_frame') + BASEITEMOFS + SIGN_SIZE * (no * SIZEOFSIGNED * 8 + state)) diff --git a/rpython/jit/backend/llsupport/test/test_gc.py b/rpython/jit/backend/llsupport/test/test_gc.py --- a/rpython/jit/backend/llsupport/test/test_gc.py +++ b/rpython/jit/backend/llsupport/test/test_gc.py @@ -271,10 +271,10 @@ return (frame_adr + jitframe.getofs('jf_frame') + jitframe.BASEITEMOFS + jitframe.SIGN_SIZE * no) - frame_info = lltype.malloc(jitframe.JITFRAMEINFO, zero=True) + frame_info = lltype.malloc(jitframe.JITFRAMEINFO, zero=True, flavor='raw') frame = lltype.malloc(jitframe.JITFRAME, 15, zero=True) frame.jf_frame_info = frame_info - frame.jf_gcmap = lltype.malloc(jitframe.GCMAP, 2) + frame.jf_gcmap = lltype.malloc(jitframe.GCMAP, 2, flavor='raw') if sys.maxint == 2**31 - 1: max = r_uint(2 ** 31) else: @@ -290,18 +290,20 @@ counter = 0 for name in jitframe.JITFRAME._names: TP = getattr(jitframe.JITFRAME, name) - if isinstance(TP, lltype.Ptr): # only GC pointers + if isinstance(TP, lltype.Ptr) and TP.TO._gckind == 'gc': assert all_addrs[counter] == frame_adr + jitframe.getofs(name) counter += 1 # gcpattern - assert all_addrs[6] == indexof(0) - assert all_addrs[7] == indexof(1) - assert all_addrs[8] == indexof(3) - assert all_addrs[9] == indexof(5) - assert all_addrs[10] == indexof(7) - assert all_addrs[11] == indexof(63) + assert all_addrs[4] == indexof(0) + assert all_addrs[5] == indexof(1) + assert all_addrs[6] == indexof(3) + assert all_addrs[7] == indexof(5) + assert all_addrs[8] == indexof(7) + assert all_addrs[9] == indexof(63) # XXX 32bit - assert all_addrs[12] == indexof(65) + assert all_addrs[10] == indexof(65) - assert len(all_addrs) == 6 + 6 + 4 - # 6 static fields, 4 addresses from gcmap, 2 from gcpattern + assert len(all_addrs) == 4 + 6 + 4 + # 4 static fields, 4 addresses from gcmap, 2 from gcpattern + lltype.free(frame_info, flavor='raw') + lltype.free(frame.jf_gcmap, flavor='raw') From noreply at buildbot.pypy.org Tue Jan 29 14:39:16 2013 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 29 Jan 2013 14:39:16 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: update rewrite Message-ID: <20130129133916.A6B6D1C0041@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60676:feba5b4744d2 Date: 2013-01-29 14:29 +0200 http://bitbucket.org/pypy/pypy/changeset/feba5b4744d2/ Log: update rewrite diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -1,5 +1,4 @@ from rpython.rlib.rarithmetic import ovfcheck -from rpython.rlib import rgc from rpython.rtyper.lltypesystem import lltype, llmemory from rpython.jit.metainterp import history from rpython.jit.metainterp.history import ConstInt, BoxPtr, ConstPtr @@ -140,7 +139,7 @@ def gen_malloc_frame(self, frame_info, frame, size_box): descrs = self.gc_ll_descr.getframedescrs(self.cpu) if self.gc_ll_descr.kind == 'boehm': - op0 = ResOperation(rop.GETFIELD_GC, [history.ConstPtr(frame_info)], + op0 = ResOperation(rop.GETFIELD_GC, [history.ConstInt(frame_info)], size_box, descr=descrs.jfi_frame_depth) self.newops.append(op0) @@ -149,14 +148,14 @@ self.handle_new_array(descrs.arraydescr, op1) else: # we read size in bytes here, not the length - op0 = ResOperation(rop.GETFIELD_GC, [history.ConstPtr(frame_info)], + op0 = ResOperation(rop.GETFIELD_GC, [history.ConstInt(frame_info)], size_box, descr=descrs.jfi_frame_size) self.newops.append(op0) self.gen_malloc_nursery_varsize(size_box, frame, is_small=True) self.gen_initialize_tid(frame, descrs.arraydescr.tid) length_box = history.BoxInt() - op1 = ResOperation(rop.GETFIELD_GC, [history.ConstPtr(frame_info)], + op1 = ResOperation(rop.GETFIELD_GC, [history.ConstInt(frame_info)], length_box, descr=descrs.jfi_frame_depth) self.newops.append(op1) @@ -168,12 +167,11 @@ loop_token = op.getdescr() assert isinstance(loop_token, history.JitCellToken) jfi = loop_token.compiled_loop_token.frame_info - llref = lltype.cast_opaque_ptr(llmemory.GCREF, jfi) - rgc._make_sure_does_not_move(llref) + llfi = heaptracker.adr2int(llmemory.cast_ptr_to_adr(jfi)) size_box = history.BoxInt() frame = history.BoxPtr() - self.gen_malloc_frame(llref, frame, size_box) - op2 = ResOperation(rop.SETFIELD_GC, [frame, history.ConstPtr(llref)], + self.gen_malloc_frame(llfi, frame, size_box) + op2 = ResOperation(rop.SETFIELD_GC, [frame, history.ConstInt(llfi)], None, descr=descrs.jf_frame_info) self.newops.append(op2) arglist = op.getarglist() diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py --- a/rpython/jit/backend/llsupport/test/test_rewrite.py +++ b/rpython/jit/backend/llsupport/test/test_rewrite.py @@ -9,7 +9,7 @@ from rpython.jit.metainterp.optimizeopt.util import equaloplists from rpython.jit.codewriter.heaptracker import register_known_gctype from rpython.jit.metainterp.history import JitCellToken, FLOAT -from rpython.rtyper.lltypesystem import lltype, rclass, llmemory +from rpython.rtyper.lltypesystem import lltype, rclass, rffi from rpython.jit.backend.x86.arch import WORD class Evaluator(object): @@ -72,8 +72,7 @@ casmdescr = JitCellToken() clt = FakeLoopToken() - frame_info = lltype.malloc(jitframe.JITFRAMEINFO) - ll_frame_info = lltype.cast_opaque_ptr(llmemory.GCREF, frame_info) + frame_info = lltype.malloc(jitframe.JITFRAMEINFO, flavor='raw') clt.frame_info = frame_info frame_info.jfi_frame_depth = 13 frame_info.jfi_frame_size = 255 @@ -100,6 +99,7 @@ ops.operations, []) equaloplists(operations, expected.operations) + lltype.free(frame_info, flavor='raw') class FakeTracker(object): pass @@ -740,12 +740,12 @@ i2 = call_assembler(i0, f0, descr=casmdescr) """, """ [i0, f0] - i1 = getfield_gc(ConstPtr(ll_frame_info), descr=jfi_frame_size) + i1 = getfield_gc(ConstClass(frame_info), descr=jfi_frame_size) p1 = call_malloc_nursery_varsize_small(i1) setfield_gc(p1, 0, descr=tiddescr) - i2 = getfield_gc(ConstPtr(ll_frame_info), descr=jfi_frame_depth) + i2 = getfield_gc(ConstClass(frame_info), descr=jfi_frame_depth) setfield_gc(p1, i2, descr=framelendescr) - setfield_gc(p1, ConstPtr(ll_frame_info), descr=jf_frame_info) + setfield_gc(p1, ConstClass(frame_info), descr=jf_frame_info) setarrayitem_gc(p1, 0, i0, descr=signedframedescr) setarrayitem_gc(p1, 1, f0, descr=floatframedescr) i3 = call_assembler(p1, descr=casmdescr) From noreply at buildbot.pypy.org Tue Jan 29 14:39:17 2013 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 29 Jan 2013 14:39:17 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: raw-allocate gcmap Message-ID: <20130129133917.D68E11C0041@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60677:46d62088d527 Date: 2013-01-29 15:23 +0200 http://bitbucket.org/pypy/pypy/changeset/46d62088d527/ Log: raw-allocate gcmap diff --git a/rpython/jit/backend/llsupport/jitframe.py b/rpython/jit/backend/llsupport/jitframe.py --- a/rpython/jit/backend/llsupport/jitframe.py +++ b/rpython/jit/backend/llsupport/jitframe.py @@ -19,6 +19,8 @@ jfi.jfi_frame_depth = new_depth jfi.jfi_frame_size = base_ofs + new_depth * SIZEOFSIGNED +JITFRAMEINFO_SIZE = 2 * SIZEOFSIGNED # make sure this stays correct + JITFRAMEINFO = lltype.Struct( 'JITFRAMEINFO', # the depth of the frame @@ -31,6 +33,7 @@ ) NULLFRAMEINFO = lltype.nullptr(JITFRAMEINFO) +JITFRAMEINFOPTR = lltype.Ptr(JITFRAMEINFO) # the JITFRAME that's stored on the heap. See backend//arch.py for # detailed explanation how it is on your architecture diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -44,23 +44,21 @@ class GuardToken(object): - def __init__(self, faildescr, failargs, fail_locs, exc, frame_depth, - is_guard_not_invalidated, is_guard_not_forced): + def __init__(self, gcmap, faildescr, failargs, fail_locs, exc, + frame_depth, is_guard_not_invalidated, is_guard_not_forced): self.faildescr = faildescr self.failargs = failargs self.fail_locs = fail_locs - self.gcmap = self.compute_gcmap(failargs, fail_locs, frame_depth) + self.gcmap = self.compute_gcmap(gcmap, failargs, + fail_locs, frame_depth) self.exc = exc self.is_guard_not_invalidated = is_guard_not_invalidated self.is_guard_not_forced = is_guard_not_forced - def compute_gcmap(self, failargs, fail_locs, frame_depth): + def compute_gcmap(self, gcmap, failargs, fail_locs, frame_depth): # note that regalloc has a very similar compute, but # one that does iteration over all bindings, so slightly different, # eh - size = frame_depth + JITFRAME_FIXED_SIZE - gcmap = lltype.malloc(jitframe.GCMAP, size // WORD // 8 + 1, - zero=True) input_i = 0 for i in range(len(failargs)): arg = failargs[i] @@ -143,7 +141,8 @@ self.set_debug(have_debug_prints()) debug_stop('jit-backend-counts') # when finishing, we only have one value at [0], the rest dies - self.gcmap_for_finish = lltype.malloc(jitframe.GCMAP, 1, zero=True) + self.gcmap_for_finish = lltype.malloc(jitframe.GCMAP, 1, zero=True, + flavor='raw', immortal=True) self.gcmap_for_finish[0] = r_uint(1) def setup(self, looptoken): @@ -505,9 +504,6 @@ # about not storing on 'self' attributes that will live only # for the duration of compiling one loop or a one bridge. clt = CompiledLoopToken(self.cpu, looptoken.number) - clt.frame_info = lltype.malloc(jitframe.JITFRAMEINFO) - clt.allgcrefs = [] - clt.frame_info.set_frame_depth(0, 0) # for now looptoken.compiled_loop_token = clt clt._debug_nbargs = len(inputargs) if not we_are_translated(): @@ -515,6 +511,13 @@ assert len(set(inputargs)) == len(inputargs) self.setup(looptoken) + + frame_info = self.datablockwrapper.malloc_aligned( + jitframe.JITFRAMEINFO_SIZE, alignment=WORD) + clt.frame_info = rffi.cast(jitframe.JITFRAMEINFOPTR, frame_info) + clt.allgcrefs = [] + clt.frame_info.set_frame_depth(0, 0) # for now + if log: operations = self._inject_debugging_code(looptoken, operations, 'e', looptoken.number) @@ -524,8 +527,6 @@ self._call_header_with_stack_check() operations = regalloc.prepare_loop(inputargs, operations, looptoken, clt.allgcrefs) - rgc._make_sure_does_not_move(lltype.cast_opaque_ptr(llmemory.GCREF, - clt.frame_info)) looppos = self.mc.get_relative_pos() frame_depth = self._assemble(regalloc, inputargs, operations) self.update_frame_depth(frame_depth + JITFRAME_FIXED_SIZE) @@ -1842,9 +1843,19 @@ guard_opnum == rop.GUARD_NOT_FORCED) is_guard_not_invalidated = guard_opnum == rop.GUARD_NOT_INVALIDATED is_guard_not_forced = guard_opnum == rop.GUARD_NOT_FORCED - return GuardToken(faildescr, failargs, fail_locs, exc, frame_depth, + gcmap = self.allocate_gcmap(frame_depth) + return GuardToken(gcmap, faildescr, failargs, + fail_locs, exc, frame_depth, is_guard_not_invalidated, is_guard_not_forced) + def allocate_gcmap(self, frame_depth): + size = frame_depth + JITFRAME_FIXED_SIZE + malloc_size = (size // WORD // 8 + 1) + 1 + rawgcmap = self.datablockwrapper.malloc_aligned(WORD * malloc_size, + WORD) + gcmap = rffi.cast(lltype.Ptr(jitframe.GCMAP), rawgcmap) + return gcmap + def generate_propagate_error_64(self): assert WORD == 8 startpos = self.mc.get_relative_pos() @@ -1882,6 +1893,8 @@ positions[i] = v * WORD # write down the positions of locs guardtok.faildescr.rd_locs = positions + # we want the descr to keep alive + guardtok.faildescr.rd_loop_token = self.current_clt #if WORD == 4: # mc.PUSH(imm(fail_descr)) # mc.PUSH(imm(gcpattern)) @@ -1893,19 +1906,15 @@ return startpos def push_gcmap(self, mc, gcmap, push=False, mov=False, store=False): - gcmapref = lltype.cast_opaque_ptr(llmemory.GCREF, gcmap) - # keep the ref alive - self.current_clt.allgcrefs.append(gcmapref) - rgc._make_sure_does_not_move(gcmapref) if push: - mc.PUSH(imm(rffi.cast(lltype.Signed, gcmapref))) + mc.PUSH(imm(rffi.cast(lltype.Signed, gcmap))) elif mov: mc.MOV(RawEspLoc(0, REF), - imm(rffi.cast(lltype.Signed, gcmapref))) + imm(rffi.cast(lltype.Signed, gcmap))) else: assert store ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') - mc.MOV(raw_stack(ofs), imm(rffi.cast(lltype.Signed, gcmapref))) + mc.MOV(raw_stack(ofs), imm(rffi.cast(lltype.Signed, gcmap))) def pop_gcmap(self, mc): ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -896,9 +896,7 @@ def get_gcmap(self, forbidden_regs=[]): frame_depth = self.fm.get_frame_depth() - size = frame_depth + JITFRAME_FIXED_SIZE - gcmap = lltype.malloc(GCMAP, size // WORD // 8 + 1, - zero=True) + gcmap = self.assembler.allocate_gcmap(frame_depth) for box, loc in self.rm.reg_bindings.iteritems(): if loc in forbidden_regs: continue From noreply at buildbot.pypy.org Tue Jan 29 14:39:19 2013 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 29 Jan 2013 14:39:19 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: * check the frame depth before jump Message-ID: <20130129133919.0C3401C0041@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60678:eab8486a061c Date: 2013-01-29 15:38 +0200 http://bitbucket.org/pypy/pypy/changeset/eab8486a061c/ Log: * check the frame depth before jump * initialize the gcmap correctly diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py --- a/rpython/jit/backend/model.py +++ b/rpython/jit/backend/model.py @@ -288,8 +288,6 @@ asmmemmgr_blocks = None asmmemmgr_gcroots = 0 - frame_depth = 0 - def __init__(self, cpu, number): cpu.tracker.total_compiled_loops += 1 self.cpu = cpu diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -670,7 +670,7 @@ baseofs = self.cpu.get_baseofs_of_frame_field() self.current_clt.frame_info.set_frame_depth(baseofs, frame_depth) - def _check_frame_depth(self, mc, gcmap): + def _check_frame_depth(self, mc, gcmap, expected_size=-1): """ check if the frame is of enough depth to follow this bridge. Otherwise reallocate the frame in a helper. There are other potential solutions @@ -679,12 +679,18 @@ descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) ofs = self.cpu.unpack_fielddescr(descrs.arraydescr.lendescr) base_ofs = self.cpu.get_baseofs_of_frame_field() - mc.CMP_bi(ofs - base_ofs, 0xffffff) + if expected_size == -1: + mc.CMP_bi(ofs - base_ofs, 0xffffff) + else: + mc.CMP_bi(ofs - base_ofs, expected_size) stack_check_cmp_ofs = mc.get_relative_pos() - 4 assert not IS_X86_32 mc.J_il8(rx86.Conditions['GE'], 0) jg_location = mc.get_relative_pos() - mc.MOV_si(WORD, 0xffffff) + if expected_size == -1: + mc.MOV_si(WORD, 0xffffff) + else: + mc.MOV_si(WORD, expected_size) ofs2 = mc.get_relative_pos() - 4 self.push_gcmap(mc, gcmap, mov=True) mc.CALL(imm(self._stack_check_failure)) @@ -1853,7 +1859,12 @@ malloc_size = (size // WORD // 8 + 1) + 1 rawgcmap = self.datablockwrapper.malloc_aligned(WORD * malloc_size, WORD) + # set the length field + rffi.cast(rffi.CArrayPtr(lltype.Signed), rawgcmap)[0] = malloc_size - 1 gcmap = rffi.cast(lltype.Ptr(jitframe.GCMAP), rawgcmap) + # zero the area + for i in range(malloc_size - 1): + gcmap[i] = r_uint(0) return gcmap def generate_propagate_error_64(self): @@ -2469,6 +2480,13 @@ curpos = self.mc.get_relative_pos() + 5 self.mc.JMP_l(target - curpos) else: + if target_token._x86_clt is not self.current_clt: + # We can have a frame coming from god knows where that's + # passed to a jump to another loop. Make sure it has the + # correct depth + expected_size = target_token._x86_clt.frame_info.jfi_frame_depth + self._check_frame_depth(self.mc, self._regalloc.get_gcmap(), + expected_size=expected_size) self.mc.JMP(imm(target)) def malloc_cond(self, nursery_free_adr, nursery_top_adr, size, gcmap): diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -1332,6 +1332,7 @@ #jump_op = self.final_jump_op #if jump_op is not None and jump_op.getdescr() is descr: # self._compute_hint_frame_locations_from_descr(descr) + def consider_keepalive(self, op): pass From noreply at buildbot.pypy.org Tue Jan 29 14:57:11 2013 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 29 Jan 2013 14:57:11 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fix this test Message-ID: <20130129135711.76F1E1C0041@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60679:8b7fabbcacbd Date: 2013-01-29 15:56 +0200 http://bitbucket.org/pypy/pypy/changeset/8b7fabbcacbd/ Log: fix this test diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -596,9 +596,6 @@ (descr_number, rawstart, rawstart + codeendpos)) debug_stop("jit-backend-addr") self.patch_pending_failure_recoveries(rawstart) - if not we_are_translated(): - # for the benefit of tests - faildescr._x86_bridge_frame_depth = frame_depth # patch the jump from original guard self.patch_jump_for_descr(faildescr, rawstart) ops_offset = self.mc.ops_offset diff --git a/rpython/jit/backend/x86/test/test_recompilation.py b/rpython/jit/backend/x86/test/test_recompilation.py --- a/rpython/jit/backend/x86/test/test_recompilation.py +++ b/rpython/jit/backend/x86/test/test_recompilation.py @@ -33,7 +33,7 @@ jump(i1, descr=targettoken) ''' loop = self.interpret(ops, [0]) - previous = loop._jitcelltoken.compiled_loop_token.frame_depth + previous = loop._jitcelltoken.compiled_loop_token.frame_info.jfi_frame_depth assert self.getint(0) == 20 ops = ''' [i1] @@ -43,14 +43,15 @@ i6 = int_add(i5, 1) i7 = int_add(i5, i4) force_spill(i5) + force_spill(i6) + force_spill(i7) i8 = int_add(i7, 1) i9 = int_add(i8, 1) guard_false(i3, descr=fdescr2) [i3, i4, i5, i6, i7, i8, i9] finish() ''' - bridge = self.attach_bridge(ops, loop, -2) - descr = loop.operations[3].getdescr() - new = descr._x86_bridge_frame_depth + self.attach_bridge(ops, loop, -2) + new = loop._jitcelltoken.compiled_loop_token.frame_info.jfi_frame_depth # the force_spill() forces the stack to grow assert new > previous fail = self.run(loop, 0) @@ -103,20 +104,32 @@ [i97, i3] i10 = int_mul(i3, 2) i8 = int_add(i3, 1) + i15 = int_add(i3, 1) + i16 = int_add(i3, 1) + i17 = int_add(i3, 1) + i18 = int_add(i3, 1) i6 = int_add(i8, i10) i7 = int_add(i3, i6) force_spill(i6) force_spill(i7) force_spill(i8) + force_spill(i10) i12 = int_add(i7, i8) i11 = int_add(i12, i6) + force_spill(i11) + force_spill(i12) + force_spill(i15) + force_spill(i16) + force_spill(i17) + force_spill(i18) + guard_true(i18) [i3, i12, i11, i10, i6, i7, i18, i17, i16] jump(i3, i12, i11, i10, i6, i7, descr=targettoken) ''' - loop_frame_depth = loop._jitcelltoken.compiled_loop_token.frame_depth + loop_frame_depth = loop._jitcelltoken.compiled_loop_token.frame_info.jfi_frame_depth bridge = self.attach_bridge(ops, loop, 6) - guard_op = loop.operations[6] # the force_spill() forces the stack to grow - assert guard_op.getdescr()._x86_bridge_frame_depth > loop_frame_depth + bridge_frame_depth = loop._jitcelltoken.compiled_loop_token.frame_info.jfi_frame_depth + assert bridge_frame_depth > loop_frame_depth self.run(loop, 0, 0, 0, 0, 0, 0) assert self.getint(0) == 1 assert self.getint(1) == 20 From noreply at buildbot.pypy.org Tue Jan 29 15:16:30 2013 From: noreply at buildbot.pypy.org (bivab) Date: Tue, 29 Jan 2013 15:16:30 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fix import Message-ID: <20130129141630.C0D101C0207@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: jitframe-on-heap Changeset: r60680:3f47e2c24688 Date: 2013-01-29 14:50 +0100 http://bitbucket.org/pypy/pypy/changeset/3f47e2c24688/ Log: fix import diff --git a/rpython/jit/backend/arm/codebuilder.py b/rpython/jit/backend/arm/codebuilder.py --- a/rpython/jit/backend/arm/codebuilder.py +++ b/rpython/jit/backend/arm/codebuilder.py @@ -1,7 +1,7 @@ from rpython.jit.backend.arm import conditions as cond from rpython.jit.backend.arm import registers as reg from rpython.jit.backend.arm import support -from rpython.jit.backend.arm.arch import (WORD, FUNC_ALIGN) +from rpython.jit.backend.arm.arch import (WORD, FUNC_ALIGN, PC_OFFSET) from rpython.jit.backend.arm.instruction_builder import define_instructions from rpython.jit.backend.llsupport.asmmemmgr import BlockBuilderMixin from rpython.rlib.objectmodel import we_are_translated @@ -186,7 +186,7 @@ def B_offs(self, target_ofs, c=cond.AL): pos = self.currpos() - target_ofs = target_ofs - (pos + arch.PC_OFFSET) + target_ofs = target_ofs - (pos + PC_OFFSET) assert target_ofs & 0x3 == 0 self.write32(c << 28 | 0xA << 24 | (target_ofs >> 2) & 0xFFFFFF) From noreply at buildbot.pypy.org Tue Jan 29 15:16:32 2013 From: noreply at buildbot.pypy.org (bivab) Date: Tue, 29 Jan 2013 15:16:32 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: remove deprecated code Message-ID: <20130129141632.0B1721C0207@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: jitframe-on-heap Changeset: r60681:a0f202829fbb Date: 2013-01-29 14:53 +0100 http://bitbucket.org/pypy/pypy/changeset/a0f202829fbb/ Log: remove deprecated code diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py --- a/rpython/jit/backend/arm/assembler.py +++ b/rpython/jit/backend/arm/assembler.py @@ -118,6 +118,9 @@ self.gcmap_for_finish = lltype.malloc(jitframe.GCMAP, 1, zero=True) self.gcmap_for_finish[0] = r_uint(1) + def setup_failure_recovery(self): + self.failure_recovery_code = [0, 0, 0, 0] + def finish_once(self): if self._debug: debug_start('jit-backend-counts') @@ -334,199 +337,6 @@ rawstart = mc.materialize(self.cpu.asmmemmgr, []) self.wb_slowpath[withcards + 2 * withfloats] = rawstart - def setup_failure_recovery(self): - - #@rgc.no_collect -- XXX still true, but hacked gc_set_extra_threshold - def failure_recovery_func(mem_loc, frame_pointer, stack_pointer): - """mem_loc is a structure in memory describing where the values for - the failargs are stored. frame loc is the address of the frame - pointer for the frame to be decoded frame """ - vfp_registers = rffi.cast(rffi.LONGP, stack_pointer) - registers = rffi.ptradd(vfp_registers, 2*len(r.all_vfp_regs)) - registers = rffi.cast(rffi.LONGP, registers) - bytecode = rffi.cast(rffi.UCHARP, mem_loc) - return self.grab_frame_values(self.cpu, bytecode, frame_pointer, - registers, vfp_registers) - self.failure_recovery_code = [0, 0, 0, 0] - - self.failure_recovery_func = failure_recovery_func - - _FAILURE_RECOVERY_FUNC = lltype.Ptr(lltype.FuncType([rffi.LONGP] * 3, - llmemory.GCREF)) - - @staticmethod - #@rgc.no_collect -- XXX still true, but hacked gc_set_extra_threshold - def grab_frame_values(cpu, bytecode, frame_pointer, - registers, vfp_registers): - # no malloc allowed here!! xxx apart from one, hacking a lot - force_index = rffi.cast(lltype.Signed, frame_pointer) - num = 0 - deadframe = lltype.nullptr(jitframe.DEADFRAME) - # step 1: lots of mess just to count the final value of 'num' - bytecode1 = bytecode - while 1: - code = rffi.cast(lltype.Signed, bytecode1[0]) - bytecode1 = rffi.ptradd(bytecode1, 1) - if code >= AssemblerARM.CODE_FROMSTACK: - while code > 0x7F: - code = rffi.cast(lltype.Signed, bytecode1[0]) - bytecode1 = rffi.ptradd(bytecode1, 1) - else: - kind = code & 3 - if kind == AssemblerARM.DESCR_SPECIAL: - if code == AssemblerARM.CODE_HOLE: - num += 1 - continue - if code == AssemblerARM.CODE_INPUTARG: - continue - if code == AssemblerARM.CODE_FORCED: - # resuming from a GUARD_NOT_FORCED - token = force_index - deadframe = ( - cpu.assembler.force_token_to_dead_frame.pop(token)) - deadframe = lltype.cast_opaque_ptr( - jitframe.DEADFRAMEPTR, deadframe) - continue - assert code == AssemblerARM.CODE_STOP - break - num += 1 - - # allocate the deadframe - if not deadframe: - # Remove the "reserve" at the end of the nursery. This means - # that it is guaranteed that the following malloc() works - # without requiring a collect(), but it needs to be re-added - # as soon as possible. - cpu.gc_clear_extra_threshold() - assert num <= cpu.get_failargs_limit() - try: - deadframe = lltype.malloc(jitframe.DEADFRAME, num) - except MemoryError: - fatalerror("memory usage error in grab_frame_values") - # fill it - code_inputarg = False - num = 0 - value_hi = 0 - while 1: - # decode the next instruction from the bytecode - code = rffi.cast(lltype.Signed, bytecode[0]) - bytecode = rffi.ptradd(bytecode, 1) - if code >= AssemblerARM.CODE_FROMSTACK: - if code > 0x7F: - shift = 7 - code &= 0x7F - while True: - nextcode = rffi.cast(lltype.Signed, bytecode[0]) - bytecode = rffi.ptradd(bytecode, 1) - code |= (nextcode & 0x7F) << shift - shift += 7 - if nextcode <= 0x7F: - break - # load the value from the stack - kind = code & 3 - code = (code - AssemblerARM.CODE_FROMSTACK) >> 2 - if code_inputarg: - code = ~code - code_inputarg = False - stackloc = force_index - get_fp_offset(int(code)) - value = rffi.cast(rffi.LONGP, stackloc)[0] - if kind == AssemblerARM.DESCR_FLOAT: - assert WORD == 4 - value_hi = value - value = rffi.cast(rffi.LONGP, stackloc - WORD)[0] - else: - kind = code & 3 - if kind == AssemblerARM.DESCR_SPECIAL: - if code == AssemblerARM.CODE_HOLE: - num += 1 - continue - if code == AssemblerARM.CODE_INPUTARG: - code_inputarg = True - continue - if code == AssemblerARM.CODE_FORCED: - continue - assert code == AssemblerARM.CODE_STOP - break - # 'code' identifies a register: load its value - code >>= 2 - if kind == AssemblerARM.DESCR_FLOAT: - if WORD == 4: - value = vfp_registers[2*code] - value_hi = vfp_registers[2*code + 1] - else: - value = registers[code] - else: - value = registers[code] - # store the loaded value into fail_boxes_ - if kind == AssemblerARM.DESCR_INT: - deadframe.jf_values[num].int = value - elif kind == AssemblerARM.DESCR_REF: - deadframe.jf_values[num].ref = rffi.cast(llmemory.GCREF, value) - elif kind == AssemblerARM.DESCR_FLOAT: - assert WORD == 4 - assert not longlong.is_64_bit - floatvalue = rffi.cast(lltype.SignedLongLong, value_hi) - floatvalue <<= 32 - floatvalue |= rffi.cast(lltype.SignedLongLong, - rffi.cast(lltype.Unsigned, value)) - deadframe.jf_values[num].float = floatvalue - else: - assert 0, "bogus kind" - num += 1 - # - assert num == len(deadframe.jf_values) - if not we_are_translated(): - assert bytecode[4] == 0xCC - fail_index = rffi.cast(rffi.INTP, bytecode)[0] - fail_descr = cpu.get_fail_descr_from_number(fail_index) - deadframe.jf_descr = fail_descr.hide(cpu) - return lltype.cast_opaque_ptr(llmemory.GCREF, deadframe) - - def decode_inputargs(self, code): - descr_to_box_type = [REF, INT, FLOAT] - bytecode = rffi.cast(rffi.UCHARP, code) - arglocs = [] - code_inputarg = False - while 1: - # decode the next instruction from the bytecode - code = rffi.cast(lltype.Signed, bytecode[0]) - bytecode = rffi.ptradd(bytecode, 1) - if code >= self.CODE_FROMSTACK: - # 'code' identifies a stack location - if code > 0x7F: - shift = 7 - code &= 0x7F - while True: - nextcode = rffi.cast(lltype.Signed, bytecode[0]) - bytecode = rffi.ptradd(bytecode, 1) - code |= (nextcode & 0x7F) << shift - shift += 7 - if nextcode <= 0x7F: - break - kind = code & 3 - code = (code - self.CODE_FROMSTACK) >> 2 - if code_inputarg: - code = ~code - code_inputarg = False - loc = ARMFrameManager.frame_pos(code, descr_to_box_type[kind]) - elif code == self.CODE_STOP: - break - elif code == self.CODE_HOLE: - continue - elif code == self.CODE_INPUTARG: - code_inputarg = True - continue - else: - # 'code' identifies a register - kind = code & 3 - code >>= 2 - if kind == self.DESCR_FLOAT: - loc = r.all_vfp_regs[code] - else: - loc = r.all_regs[code] - arglocs.append(loc) - return arglocs[:] - def _build_malloc_slowpath(self): mc = ARMv7Builder() if self.cpu.supports_floats: From noreply at buildbot.pypy.org Tue Jan 29 15:16:33 2013 From: noreply at buildbot.pypy.org (bivab) Date: Tue, 29 Jan 2013 15:16:33 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: support for new guards handling Message-ID: <20130129141633.3B4A91C0207@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: jitframe-on-heap Changeset: r60682:f40c4e68b860 Date: 2013-01-29 14:56 +0100 http://bitbucket.org/pypy/pypy/changeset/f40c4e68b860/ Log: support for new guards handling diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py --- a/rpython/jit/backend/arm/assembler.py +++ b/rpython/jit/backend/arm/assembler.py @@ -8,7 +8,7 @@ N_REGISTERS_SAVED_BY_MALLOC, \ JITFRAME_FIXED_SIZE, FRAME_FIXED_SIZE from rpython.jit.backend.arm.codebuilder import ARMv7Builder, OverwritingBuilder -from rpython.jit.backend.arm.locations import get_fp_offset +from rpython.jit.backend.arm.locations import get_fp_offset, imm from rpython.jit.backend.arm.regalloc import (Regalloc, ARMFrameManager, CoreRegisterManager, check_imm_arg, operations as regalloc_operations, @@ -372,48 +372,63 @@ # see ../x86/assembler.py:propagate_memoryerror_if_eax_is_null self.mc.CMP_ri(r.r0.value, 0) self.mc.B(self.propagate_exception_path, c=c.EQ) + + def _push_all_regs_to_jitframe(self, mc, ignored_regs, withfloats, + callee_only=False): + if callee_only: + regs = CoreRegisterManager.save_around_call_regs + else: + regs = CoreRegisterManager.all_regs + # use STMDB ops here + for i, gpr in enumerate(regs): + if gpr in ignored_regs: + continue + mc.STR_ri(gpr.value, r.fp.value, i * WORD) + if withfloats: + assert 0, 'implement me' def _build_failure_recovery(self, exc, withfloats=False): mc = ARMv7Builder() - failure_recovery = llhelper(self._FAILURE_RECOVERY_FUNC, - self.failure_recovery_func) + self._push_all_regs_to_jitframe(mc, [], withfloats) self._insert_checks(mc) - if withfloats: - f = r.all_vfp_regs - else: - f = [] - with saved_registers(mc, r.all_regs, f): - if exc: - # We might have an exception pending. Load it into r4 - # (this is a register saved across calls) - mc.gen_load_int(r.r5.value, self.cpu.pos_exc_value()) - mc.LDR_ri(r.r4.value, r.r5.value) - # clear the exc flags - mc.gen_load_int(r.r6.value, 0) - mc.STR_ri(r.r6.value, r.r5.value) - mc.gen_load_int(r.r5.value, self.cpu.pos_exception()) - mc.STR_ri(r.r6.value, r.r5.value) - # move mem block address, to r0 to pass as - mc.MOV_rr(r.r0.value, r.lr.value) - # pass the current frame pointer as second param - mc.MOV_rr(r.r1.value, r.fp.value) - # pass the current stack pointer as third param - mc.MOV_rr(r.r2.value, r.sp.value) - self._insert_checks(mc) - mc.BL(rffi.cast(lltype.Signed, failure_recovery)) - if exc: - # save ebx into 'jf_guard_exc' - from rpython.jit.backend.llsupport.descr import unpack_fielddescr - descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) - offset, size, _ = unpack_fielddescr(descrs.jf_guard_exc) - mc.STR_rr(r.r4.value, r.r0.value, offset, cond=c.AL) - mc.MOV_rr(r.ip.value, r.r0.value) - mc.MOV_rr(r.r0.value, r.ip.value) - self.gen_func_epilog(mc=mc) - rawstart = mc.materialize(self.cpu.asmmemmgr, [], - self.cpu.gc_ll_descr.gcrootmap) + + if exc: + # We might have an exception pending. Load it into r4 + # (this is a register saved across calls) + mc.gen_load_int(r.r5.value, self.cpu.pos_exc_value()) + mc.LDR_ri(r.r4.value, r.r5.value) + # clear the exc flags + mc.gen_load_int(r.r6.value, 0) + mc.STR_ri(r.r6.value, r.r5.value) # pos_exc_value is still in r5 + mc.gen_load_int(r.r5.value, self.cpu.pos_exception()) + mc.STR_ri(r.r6.value, r.r5.value) + # save r4 into 'jf_guard_exc' + offset = self.cpu.get_ofs_of_frame_field('jf_guard_exc') + assert check_imm_arg(abs(offset)) + mc.STR_ri(r.r4.value, r.fp.value, imm=offset) + # now we return from the complete frame, which starts from + # _call_header_with_stack_check(). The LEA in _call_footer below + # throws away most of the frame, including all the PUSHes that we + # did just above. + ofs = self.cpu.get_ofs_of_frame_field('jf_descr') + assert check_imm_arg(abs(ofs)) + ofs2 = self.cpu.get_ofs_of_frame_field('jf_gcmap') + assert check_imm_arg(abs(ofs2)) + base_ofs = self.cpu.get_baseofs_of_frame_field() + # store the gcmap + mc.POP([r.ip.value]) + mc.STR_ri(r.ip.value, r.fp.value, imm=ofs2) + # store the descr + mc.POP([r.ip.value]) + mc.STR_ri(r.ip.value, r.fp.value, imm=ofs) + + # set return value + assert check_imm_arg(base_ofs) + mc.SUB_ri(r.r0.value, r.fp.value, base_ofs) + + self.gen_func_epilog(mc) + rawstart = mc.materialize(self.cpu.asmmemmgr, []) self.failure_recovery_code[exc + 2 * withfloats] = rawstart - self.mc = None DESCR_REF = 0x00 DESCR_INT = 0x01 @@ -459,30 +474,37 @@ def generate_quick_failure(self, guardtok, fcond=c.AL): - assert isinstance(guardtok.save_exc, bool) - fail_index = self.cpu.get_fail_descr_number(guardtok.descr) + assert isinstance(guardtok.exc, bool) startpos = self.mc.currpos() withfloats = False for box in guardtok.failargs: if box is not None and box.type == FLOAT: withfloats = True break - exc = guardtok.save_exc + exc = guardtok.exc target = self.failure_recovery_code[exc + 2 * withfloats] - assert target != 0 + fail_descr = cast_instance_to_gcref(guardtok.faildescr) + fail_descr = rffi.cast(lltype.Signed, fail_descr) + positions = [0] * len(guardtok.fail_locs) + for i, loc in enumerate(guardtok.fail_locs): + if loc is None: + positions[i] = -1 + elif loc.is_stack(): + positions[i] = loc.value + else: + if loc.is_reg(): + assert loc is not r.fp # for now + v = loc.value + else: + assert 0, 'fix for floats' + assert loc.is_vfp_reg() + #v = len(VFPRegisterManager.all_regs) + loc.value + positions[i] = v * WORD + # write down the positions of locs + guardtok.faildescr.rd_locs = positions + self.regalloc_push(imm(fail_descr)) + self.push_gcmap(self.mc, gcmap=guardtok.gcmap, push=True) self.mc.BL(target) - # write tight data that describes the failure recovery - if guardtok.is_guard_not_forced: - self.mc.writechar(chr(self.CODE_FORCED)) - self.write_failure_recovery_description(guardtok.descr, - guardtok.failargs, guardtok.faillocs[1:]) - self.mc.write32(fail_index) - # for testing the decoding, write a final byte 0xCC - if not we_are_translated(): - self.mc.writechar('\xCC') - faillocs = [loc for loc in guardtok.faillocs if loc is not None] - guardtok.descr._arm_debug_faillocs = faillocs - self.align() return startpos def align(self): @@ -747,13 +769,11 @@ for tok in self.pending_guards: #generate the exit stub and the encoded representation tok.pos_recovery_stub = self.generate_quick_failure(tok) - # store info on the descr - tok.descr._arm_current_frame_depth = tok.faillocs[0].getint() def process_pending_guards(self, block_start): clt = self.current_clt for tok in self.pending_guards: - descr = tok.descr + descr = tok.faildescr assert isinstance(descr, AbstractFailDescr) failure_recovery_pos = block_start + tok.pos_recovery_stub descr._arm_failure_recovery_block = failure_recovery_pos @@ -1213,16 +1233,18 @@ # keep the ref alive self.current_clt.allgcrefs.append(gcmapref) rgc._make_sure_does_not_move(gcmapref) - pass - #if push: - # mc.PUSH(imm(rffi.cast(lltype.Signed, gcmapref))) - #elif mov: - # mc.MOV(RawEspLoc(0, REF), - # imm(rffi.cast(lltype.Signed, gcmapref))) - #else: - # assert store - # ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') - # mc.MOV(raw_stack(ofs), imm(rffi.cast(lltype.Signed, gcmapref))) + if push: + mc.gen_load_int(r.ip.value, rffi.cast(lltype.Signed, gcmapref)) + mc.PUSH([r.ip.value]) + elif mov: + assert 0 + mc.MOV(RawEspLoc(0, REF), + imm(rffi.cast(lltype.Signed, gcmapref))) + else: + assert store + ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') + mc.gen_load_int(r.ip.value, rffi.cast(lltype.Signed, gcmapref)) + mc.STR_ri(r.ip.value, r.fp.value, imm=ofs) def pop_gcmap(self, mc): ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -2,7 +2,7 @@ from rpython.jit.backend.arm import conditions as c from rpython.jit.backend.arm import registers as r from rpython.jit.backend.arm import shift -from rpython.jit.backend.arm.arch import WORD, DOUBLE_WORD +from rpython.jit.backend.arm.arch import WORD, DOUBLE_WORD, JITFRAME_FIXED_SIZE from rpython.jit.backend.arm.helper.assembler import (gen_emit_op_by_helper_call, gen_emit_op_unary_cmp, @@ -21,7 +21,7 @@ from rpython.jit.backend.arm.jump import remap_frame_layout from rpython.jit.backend.arm.regalloc import TempInt, TempPtr from rpython.jit.backend.arm.locations import imm -from rpython.jit.backend.llsupport import symbolic +from rpython.jit.backend.llsupport import symbolic, jitframe from rpython.jit.backend.llsupport.descr import InteriorFieldDescr from rpython.jit.metainterp.history import (Box, AbstractFailDescr, INT, FLOAT, REF) @@ -35,18 +35,43 @@ class GuardToken(object): - def __init__(self, descr, failargs, faillocs, offset, save_exc, fcond=c.AL, + def __init__(self, faildescr, failargs, fail_locs, offset, exc, fcond=c.AL, is_guard_not_invalidated=False, is_guard_not_forced=False): - assert isinstance(save_exc, bool) - self.descr = descr + assert isinstance(exc, bool) + self.faildescr = faildescr + self.failargs = failargs + self.fail_locs = fail_locs[1:] self.offset = offset + self.gcmap = self.compute_gcmap(failargs, fail_locs, fail_locs[0].value) + self.exc = exc self.is_guard_not_invalidated = is_guard_not_invalidated self.is_guard_not_forced = is_guard_not_forced - self.failargs = failargs - self.faillocs = faillocs - self.save_exc = save_exc self.fcond = fcond + def compute_gcmap(self, failargs, fail_locs, frame_depth): + # note that regalloc has a very similar compute, but + # one that does iteration over all bindings, so slightly different, + # eh + size = frame_depth + JITFRAME_FIXED_SIZE + gcmap = lltype.malloc(jitframe.GCMAP, size // WORD // 8 + 1, + zero=True) + input_i = 0 + for i in range(len(failargs)): + arg = failargs[i] + if arg is None: + continue + loc = fail_locs[input_i] + input_i += 1 + if arg.type == REF: + loc = fail_locs[i] + if loc.is_reg(): + val = loc.value + else: + assert 0, 'ffuu, implement' + val = loc.value // WORD + gcmap[val // WORD // 8] |= r_uint(1) << (val % (WORD * 8)) + return gcmap + class ResOpAssembler(object): @@ -211,9 +236,9 @@ self.mc.BKPT() self.pending_guards.append(GuardToken(descr, failargs=op.getfailargs(), - faillocs=arglocs, + fail_locs=arglocs, offset=pos, - save_exc=save_exc, + exc=save_exc, is_guard_not_invalidated=is_guard_not_invalidated, is_guard_not_forced=is_guard_not_forced, fcond=fcond)) diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py --- a/rpython/jit/backend/arm/regalloc.py +++ b/rpython/jit/backend/arm/regalloc.py @@ -709,6 +709,7 @@ # optimization only: fill in the 'hint_frame_locations' dictionary # of rm and xrm based on the JUMP at the end of the loop, by looking # at where we would like the boxes to be after the jump. + return # XXX disabled for now op = operations[-1] if op.getopnum() != rop.JUMP: return From noreply at buildbot.pypy.org Tue Jan 29 15:16:34 2013 From: noreply at buildbot.pypy.org (bivab) Date: Tue, 29 Jan 2013 15:16:34 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: merge heads Message-ID: <20130129141634.671A91C0207@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: jitframe-on-heap Changeset: r60683:59d2f0246fa5 Date: 2013-01-29 15:13 +0100 http://bitbucket.org/pypy/pypy/changeset/59d2f0246fa5/ Log: merge heads diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py --- a/rpython/jit/backend/arm/assembler.py +++ b/rpython/jit/backend/arm/assembler.py @@ -756,14 +756,6 @@ def update_frame_depth(self, frame_depth): self.current_clt.frame_info.jfi_frame_depth = frame_depth - new_jumping_to = [] - for wref in self.current_clt.jumping_to: - clt = wref() - if clt is not None: - clt.frame_info.jfi_frame_depth = max(frame_depth, - clt.frame_info.jfi_frame_depth) - new_jumping_to.append(weakref.ref(clt)) - self.current_clt.jumping_to = new_jumping_to def write_pending_failure_recoveries(self): for tok in self.pending_guards: diff --git a/rpython/jit/backend/llsupport/jitframe.py b/rpython/jit/backend/llsupport/jitframe.py --- a/rpython/jit/backend/llsupport/jitframe.py +++ b/rpython/jit/backend/llsupport/jitframe.py @@ -2,6 +2,7 @@ from rpython.rtyper.annlowlevel import llhelper from rpython.rlib.objectmodel import specialize from rpython.rlib.debug import ll_assert +from rpython.rlib.objectmodel import enforceargs SIZEOFSIGNED = rffi.sizeof(lltype.Signed) IS_32BIT = (SIZEOFSIGNED == 2 ** 31 - 1) @@ -10,14 +11,17 @@ # compiled loop token (in fact we could use this as a compiled loop token # XXX do this -GCMAP = lltype.GcArray(lltype.Unsigned) +GCMAP = lltype.Array(lltype.Unsigned) NULLGCMAP = lltype.nullptr(GCMAP) + at enforceargs(None, int, int) def jitframeinfo_set_depth(jfi, base_ofs, new_depth): jfi.jfi_frame_depth = new_depth jfi.jfi_frame_size = base_ofs + new_depth * SIZEOFSIGNED -JITFRAMEINFO = lltype.GcStruct( +JITFRAMEINFO_SIZE = 2 * SIZEOFSIGNED # make sure this stays correct + +JITFRAMEINFO = lltype.Struct( 'JITFRAMEINFO', # the depth of the frame ('jfi_frame_depth', lltype.Signed), @@ -29,6 +33,7 @@ ) NULLFRAMEINFO = lltype.nullptr(JITFRAMEINFO) +JITFRAMEINFOPTR = lltype.Ptr(JITFRAMEINFO) # the JITFRAME that's stored on the heap. See backend//arch.py for # detailed explanation how it is on your architecture @@ -82,25 +87,19 @@ def jitframe_trace(obj_addr, prev): if prev == llmemory.NULL: (obj_addr + getofs('jf_gc_trace_state')).signed[0] = 0 - return obj_addr + getofs('jf_frame_info') + return obj_addr + getofs('jf_descr') fld = (obj_addr + getofs('jf_gc_trace_state')).signed[0] state = fld & 0x7 # 3bits of possible states if state == 0: (obj_addr + getofs('jf_gc_trace_state')).signed[0] = 1 - return obj_addr + getofs('jf_descr') + return obj_addr + getofs('jf_force_descr') elif state == 1: (obj_addr + getofs('jf_gc_trace_state')).signed[0] = 2 - return obj_addr + getofs('jf_force_descr') + return obj_addr + getofs('jf_savedata') elif state == 2: (obj_addr + getofs('jf_gc_trace_state')).signed[0] = 3 - return obj_addr + getofs('jf_gcmap') - elif state == 3: - (obj_addr + getofs('jf_gc_trace_state')).signed[0] = 4 - return obj_addr + getofs('jf_savedata') - elif state == 4: - (obj_addr + getofs('jf_gc_trace_state')).signed[0] = 5 return obj_addr + getofs('jf_guard_exc') - ll_assert(state == 5, "invalid state") + ll_assert(state == 3, "invalid state") # bit pattern # decode the pattern if IS_32BIT: @@ -125,9 +124,9 @@ # found it # save new state if IS_32BIT: - new_state = 5 | ((state + 1) << 3) | (no << 8) + new_state = 3 | ((state + 1) << 3) | (no << 8) else: - new_state = 5 | ((state + 1) << 3) | (no << 9) + new_state = 3 | ((state + 1) << 3) | (no << 9) (obj_addr + getofs('jf_gc_trace_state')).signed[0] = new_state return (obj_addr + getofs('jf_frame') + BASEITEMOFS + SIGN_SIZE * (no * SIZEOFSIGNED * 8 + state)) diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py --- a/rpython/jit/backend/llsupport/llmodel.py +++ b/rpython/jit/backend/llsupport/llmodel.py @@ -54,10 +54,14 @@ def _setup_frame_realloc(self, translate_support_code): FUNC_TP = lltype.Ptr(lltype.FuncType([llmemory.GCREF, lltype.Signed], llmemory.GCREF)) + base_ofs = self.get_baseofs_of_frame_field() def realloc_frame(frame, size): frame = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, frame) - assert size <= frame.jf_frame_info.jfi_frame_depth + if size > frame.jf_frame_info.jfi_frame_depth: + # update the frame_info size, which is for whatever reason + # not up to date + frame.jf_frame_info.set_frame_depth(base_ofs, size) new_frame = jitframe.JITFRAME.allocate(frame.jf_frame_info) i = 0 while i < len(frame.jf_frame): diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py --- a/rpython/jit/backend/llsupport/rewrite.py +++ b/rpython/jit/backend/llsupport/rewrite.py @@ -1,5 +1,4 @@ from rpython.rlib.rarithmetic import ovfcheck -from rpython.rlib import rgc from rpython.rtyper.lltypesystem import lltype, llmemory from rpython.jit.metainterp import history from rpython.jit.metainterp.history import ConstInt, BoxPtr, ConstPtr @@ -140,7 +139,7 @@ def gen_malloc_frame(self, frame_info, frame, size_box): descrs = self.gc_ll_descr.getframedescrs(self.cpu) if self.gc_ll_descr.kind == 'boehm': - op0 = ResOperation(rop.GETFIELD_GC, [history.ConstPtr(frame_info)], + op0 = ResOperation(rop.GETFIELD_GC, [history.ConstInt(frame_info)], size_box, descr=descrs.jfi_frame_depth) self.newops.append(op0) @@ -149,14 +148,14 @@ self.handle_new_array(descrs.arraydescr, op1) else: # we read size in bytes here, not the length - op0 = ResOperation(rop.GETFIELD_GC, [history.ConstPtr(frame_info)], + op0 = ResOperation(rop.GETFIELD_GC, [history.ConstInt(frame_info)], size_box, descr=descrs.jfi_frame_size) self.newops.append(op0) self.gen_malloc_nursery_varsize(size_box, frame, is_small=True) self.gen_initialize_tid(frame, descrs.arraydescr.tid) length_box = history.BoxInt() - op1 = ResOperation(rop.GETFIELD_GC, [history.ConstPtr(frame_info)], + op1 = ResOperation(rop.GETFIELD_GC, [history.ConstInt(frame_info)], length_box, descr=descrs.jfi_frame_depth) self.newops.append(op1) @@ -168,12 +167,11 @@ loop_token = op.getdescr() assert isinstance(loop_token, history.JitCellToken) jfi = loop_token.compiled_loop_token.frame_info - llref = lltype.cast_opaque_ptr(llmemory.GCREF, jfi) - rgc._make_sure_does_not_move(llref) + llfi = heaptracker.adr2int(llmemory.cast_ptr_to_adr(jfi)) size_box = history.BoxInt() frame = history.BoxPtr() - self.gen_malloc_frame(llref, frame, size_box) - op2 = ResOperation(rop.SETFIELD_GC, [frame, history.ConstPtr(llref)], + self.gen_malloc_frame(llfi, frame, size_box) + op2 = ResOperation(rop.SETFIELD_GC, [frame, history.ConstInt(llfi)], None, descr=descrs.jf_frame_info) self.newops.append(op2) arglist = op.getarglist() diff --git a/rpython/jit/backend/llsupport/test/test_gc.py b/rpython/jit/backend/llsupport/test/test_gc.py --- a/rpython/jit/backend/llsupport/test/test_gc.py +++ b/rpython/jit/backend/llsupport/test/test_gc.py @@ -271,10 +271,10 @@ return (frame_adr + jitframe.getofs('jf_frame') + jitframe.BASEITEMOFS + jitframe.SIGN_SIZE * no) - frame_info = lltype.malloc(jitframe.JITFRAMEINFO, zero=True) + frame_info = lltype.malloc(jitframe.JITFRAMEINFO, zero=True, flavor='raw') frame = lltype.malloc(jitframe.JITFRAME, 15, zero=True) frame.jf_frame_info = frame_info - frame.jf_gcmap = lltype.malloc(jitframe.GCMAP, 2) + frame.jf_gcmap = lltype.malloc(jitframe.GCMAP, 2, flavor='raw') if sys.maxint == 2**31 - 1: max = r_uint(2 ** 31) else: @@ -290,18 +290,20 @@ counter = 0 for name in jitframe.JITFRAME._names: TP = getattr(jitframe.JITFRAME, name) - if isinstance(TP, lltype.Ptr): # only GC pointers + if isinstance(TP, lltype.Ptr) and TP.TO._gckind == 'gc': assert all_addrs[counter] == frame_adr + jitframe.getofs(name) counter += 1 # gcpattern - assert all_addrs[6] == indexof(0) - assert all_addrs[7] == indexof(1) - assert all_addrs[8] == indexof(3) - assert all_addrs[9] == indexof(5) - assert all_addrs[10] == indexof(7) - assert all_addrs[11] == indexof(63) + assert all_addrs[4] == indexof(0) + assert all_addrs[5] == indexof(1) + assert all_addrs[6] == indexof(3) + assert all_addrs[7] == indexof(5) + assert all_addrs[8] == indexof(7) + assert all_addrs[9] == indexof(63) # XXX 32bit - assert all_addrs[12] == indexof(65) + assert all_addrs[10] == indexof(65) - assert len(all_addrs) == 6 + 6 + 4 - # 6 static fields, 4 addresses from gcmap, 2 from gcpattern + assert len(all_addrs) == 4 + 6 + 4 + # 4 static fields, 4 addresses from gcmap, 2 from gcpattern + lltype.free(frame_info, flavor='raw') + lltype.free(frame.jf_gcmap, flavor='raw') diff --git a/rpython/jit/backend/llsupport/test/test_rewrite.py b/rpython/jit/backend/llsupport/test/test_rewrite.py --- a/rpython/jit/backend/llsupport/test/test_rewrite.py +++ b/rpython/jit/backend/llsupport/test/test_rewrite.py @@ -9,7 +9,7 @@ from rpython.jit.metainterp.optimizeopt.util import equaloplists from rpython.jit.codewriter.heaptracker import register_known_gctype from rpython.jit.metainterp.history import JitCellToken, FLOAT -from rpython.rtyper.lltypesystem import lltype, rclass, llmemory +from rpython.rtyper.lltypesystem import lltype, rclass, rffi from rpython.jit.backend.x86.arch import WORD class Evaluator(object): @@ -72,8 +72,7 @@ casmdescr = JitCellToken() clt = FakeLoopToken() - frame_info = lltype.malloc(jitframe.JITFRAMEINFO) - ll_frame_info = lltype.cast_opaque_ptr(llmemory.GCREF, frame_info) + frame_info = lltype.malloc(jitframe.JITFRAMEINFO, flavor='raw') clt.frame_info = frame_info frame_info.jfi_frame_depth = 13 frame_info.jfi_frame_size = 255 @@ -100,6 +99,7 @@ ops.operations, []) equaloplists(operations, expected.operations) + lltype.free(frame_info, flavor='raw') class FakeTracker(object): pass @@ -740,12 +740,12 @@ i2 = call_assembler(i0, f0, descr=casmdescr) """, """ [i0, f0] - i1 = getfield_gc(ConstPtr(ll_frame_info), descr=jfi_frame_size) + i1 = getfield_gc(ConstClass(frame_info), descr=jfi_frame_size) p1 = call_malloc_nursery_varsize_small(i1) setfield_gc(p1, 0, descr=tiddescr) - i2 = getfield_gc(ConstPtr(ll_frame_info), descr=jfi_frame_depth) + i2 = getfield_gc(ConstClass(frame_info), descr=jfi_frame_depth) setfield_gc(p1, i2, descr=framelendescr) - setfield_gc(p1, ConstPtr(ll_frame_info), descr=jf_frame_info) + setfield_gc(p1, ConstClass(frame_info), descr=jf_frame_info) setarrayitem_gc(p1, 0, i0, descr=signedframedescr) setarrayitem_gc(p1, 1, f0, descr=floatframedescr) i3 = call_assembler(p1, descr=casmdescr) diff --git a/rpython/jit/backend/model.py b/rpython/jit/backend/model.py --- a/rpython/jit/backend/model.py +++ b/rpython/jit/backend/model.py @@ -288,14 +288,11 @@ asmmemmgr_blocks = None asmmemmgr_gcroots = 0 - frame_depth = 0 - def __init__(self, cpu, number): cpu.tracker.total_compiled_loops += 1 self.cpu = cpu self.number = number self.bridges_count = 0 - self.jumping_to = [] # a list of weakrefs who jump here # This growing list gives the 'descr_number' of all fail descrs # that belong to this loop or to a bridge attached to it. # Filled by the frontend calling record_faildescr_index(). diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -44,23 +44,21 @@ class GuardToken(object): - def __init__(self, faildescr, failargs, fail_locs, exc, frame_depth, - is_guard_not_invalidated, is_guard_not_forced): + def __init__(self, gcmap, faildescr, failargs, fail_locs, exc, + frame_depth, is_guard_not_invalidated, is_guard_not_forced): self.faildescr = faildescr self.failargs = failargs self.fail_locs = fail_locs - self.gcmap = self.compute_gcmap(failargs, fail_locs, frame_depth) + self.gcmap = self.compute_gcmap(gcmap, failargs, + fail_locs, frame_depth) self.exc = exc self.is_guard_not_invalidated = is_guard_not_invalidated self.is_guard_not_forced = is_guard_not_forced - def compute_gcmap(self, failargs, fail_locs, frame_depth): + def compute_gcmap(self, gcmap, failargs, fail_locs, frame_depth): # note that regalloc has a very similar compute, but # one that does iteration over all bindings, so slightly different, # eh - size = frame_depth + JITFRAME_FIXED_SIZE - gcmap = lltype.malloc(jitframe.GCMAP, size // WORD // 8 + 1, - zero=True) input_i = 0 for i in range(len(failargs)): arg = failargs[i] @@ -143,7 +141,8 @@ self.set_debug(have_debug_prints()) debug_stop('jit-backend-counts') # when finishing, we only have one value at [0], the rest dies - self.gcmap_for_finish = lltype.malloc(jitframe.GCMAP, 1, zero=True) + self.gcmap_for_finish = lltype.malloc(jitframe.GCMAP, 1, zero=True, + flavor='raw', immortal=True) self.gcmap_for_finish[0] = r_uint(1) def setup(self, looptoken): @@ -505,9 +504,6 @@ # about not storing on 'self' attributes that will live only # for the duration of compiling one loop or a one bridge. clt = CompiledLoopToken(self.cpu, looptoken.number) - clt.frame_info = lltype.malloc(jitframe.JITFRAMEINFO) - clt.allgcrefs = [] - clt.frame_info.set_frame_depth(0, 0) # for now looptoken.compiled_loop_token = clt clt._debug_nbargs = len(inputargs) if not we_are_translated(): @@ -515,6 +511,13 @@ assert len(set(inputargs)) == len(inputargs) self.setup(looptoken) + + frame_info = self.datablockwrapper.malloc_aligned( + jitframe.JITFRAMEINFO_SIZE, alignment=WORD) + clt.frame_info = rffi.cast(jitframe.JITFRAMEINFOPTR, frame_info) + clt.allgcrefs = [] + clt.frame_info.set_frame_depth(0, 0) # for now + if log: operations = self._inject_debugging_code(looptoken, operations, 'e', looptoken.number) @@ -524,8 +527,6 @@ self._call_header_with_stack_check() operations = regalloc.prepare_loop(inputargs, operations, looptoken, clt.allgcrefs) - rgc._make_sure_does_not_move(lltype.cast_opaque_ptr(llmemory.GCREF, - clt.frame_info)) looppos = self.mc.get_relative_pos() frame_depth = self._assemble(regalloc, inputargs, operations) self.update_frame_depth(frame_depth + JITFRAME_FIXED_SIZE) @@ -595,9 +596,6 @@ (descr_number, rawstart, rawstart + codeendpos)) debug_stop("jit-backend-addr") self.patch_pending_failure_recoveries(rawstart) - if not we_are_translated(): - # for the benefit of tests - faildescr._x86_bridge_frame_depth = frame_depth # patch the jump from original guard self.patch_jump_for_descr(faildescr, rawstart) ops_offset = self.mc.ops_offset @@ -668,16 +666,8 @@ def update_frame_depth(self, frame_depth): baseofs = self.cpu.get_baseofs_of_frame_field() self.current_clt.frame_info.set_frame_depth(baseofs, frame_depth) - new_jumping_to = [] - for wref in self.current_clt.jumping_to: - clt = wref() - if clt is not None: - clt.frame_info.set_frame_depth(baseofs, max(frame_depth, - clt.frame_info.jfi_frame_depth)) - new_jumping_to.append(weakref.ref(clt)) - self.current_clt.jumping_to = new_jumping_to - def _check_frame_depth(self, mc, gcmap): + def _check_frame_depth(self, mc, gcmap, expected_size=-1): """ check if the frame is of enough depth to follow this bridge. Otherwise reallocate the frame in a helper. There are other potential solutions @@ -686,12 +676,18 @@ descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) ofs = self.cpu.unpack_fielddescr(descrs.arraydescr.lendescr) base_ofs = self.cpu.get_baseofs_of_frame_field() - mc.CMP_bi(ofs - base_ofs, 0xffffff) + if expected_size == -1: + mc.CMP_bi(ofs - base_ofs, 0xffffff) + else: + mc.CMP_bi(ofs - base_ofs, expected_size) stack_check_cmp_ofs = mc.get_relative_pos() - 4 assert not IS_X86_32 mc.J_il8(rx86.Conditions['GE'], 0) jg_location = mc.get_relative_pos() - mc.MOV_si(WORD, 0xffffff) + if expected_size == -1: + mc.MOV_si(WORD, 0xffffff) + else: + mc.MOV_si(WORD, expected_size) ofs2 = mc.get_relative_pos() - 4 self.push_gcmap(mc, gcmap, mov=True) mc.CALL(imm(self._stack_check_failure)) @@ -1850,9 +1846,24 @@ guard_opnum == rop.GUARD_NOT_FORCED) is_guard_not_invalidated = guard_opnum == rop.GUARD_NOT_INVALIDATED is_guard_not_forced = guard_opnum == rop.GUARD_NOT_FORCED - return GuardToken(faildescr, failargs, fail_locs, exc, frame_depth, + gcmap = self.allocate_gcmap(frame_depth) + return GuardToken(gcmap, faildescr, failargs, + fail_locs, exc, frame_depth, is_guard_not_invalidated, is_guard_not_forced) + def allocate_gcmap(self, frame_depth): + size = frame_depth + JITFRAME_FIXED_SIZE + malloc_size = (size // WORD // 8 + 1) + 1 + rawgcmap = self.datablockwrapper.malloc_aligned(WORD * malloc_size, + WORD) + # set the length field + rffi.cast(rffi.CArrayPtr(lltype.Signed), rawgcmap)[0] = malloc_size - 1 + gcmap = rffi.cast(lltype.Ptr(jitframe.GCMAP), rawgcmap) + # zero the area + for i in range(malloc_size - 1): + gcmap[i] = r_uint(0) + return gcmap + def generate_propagate_error_64(self): assert WORD == 8 startpos = self.mc.get_relative_pos() @@ -1890,6 +1901,8 @@ positions[i] = v * WORD # write down the positions of locs guardtok.faildescr.rd_locs = positions + # we want the descr to keep alive + guardtok.faildescr.rd_loop_token = self.current_clt #if WORD == 4: # mc.PUSH(imm(fail_descr)) # mc.PUSH(imm(gcpattern)) @@ -1901,19 +1914,15 @@ return startpos def push_gcmap(self, mc, gcmap, push=False, mov=False, store=False): - gcmapref = lltype.cast_opaque_ptr(llmemory.GCREF, gcmap) - # keep the ref alive - self.current_clt.allgcrefs.append(gcmapref) - rgc._make_sure_does_not_move(gcmapref) if push: - mc.PUSH(imm(rffi.cast(lltype.Signed, gcmapref))) + mc.PUSH(imm(rffi.cast(lltype.Signed, gcmap))) elif mov: mc.MOV(RawEspLoc(0, REF), - imm(rffi.cast(lltype.Signed, gcmapref))) + imm(rffi.cast(lltype.Signed, gcmap))) else: assert store ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') - mc.MOV(raw_stack(ofs), imm(rffi.cast(lltype.Signed, gcmapref))) + mc.MOV(raw_stack(ofs), imm(rffi.cast(lltype.Signed, gcmap))) def pop_gcmap(self, mc): ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') @@ -2468,10 +2477,13 @@ curpos = self.mc.get_relative_pos() + 5 self.mc.JMP_l(target - curpos) else: - clt = self.current_clt - assert clt is not None - target_token._x86_clt.jumping_to.append( - weakref.ref(clt)) + if target_token._x86_clt is not self.current_clt: + # We can have a frame coming from god knows where that's + # passed to a jump to another loop. Make sure it has the + # correct depth + expected_size = target_token._x86_clt.frame_info.jfi_frame_depth + self._check_frame_depth(self.mc, self._regalloc.get_gcmap(), + expected_size=expected_size) self.mc.JMP(imm(target)) def malloc_cond(self, nursery_free_adr, nursery_top_adr, size, gcmap): diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -896,9 +896,7 @@ def get_gcmap(self, forbidden_regs=[]): frame_depth = self.fm.get_frame_depth() - size = frame_depth + JITFRAME_FIXED_SIZE - gcmap = lltype.malloc(GCMAP, size // WORD // 8 + 1, - zero=True) + gcmap = self.assembler.allocate_gcmap(frame_depth) for box, loc in self.rm.reg_bindings.iteritems(): if loc in forbidden_regs: continue @@ -1334,6 +1332,7 @@ #jump_op = self.final_jump_op #if jump_op is not None and jump_op.getdescr() is descr: # self._compute_hint_frame_locations_from_descr(descr) + def consider_keepalive(self, op): pass diff --git a/rpython/jit/backend/x86/test/test_recompilation.py b/rpython/jit/backend/x86/test/test_recompilation.py --- a/rpython/jit/backend/x86/test/test_recompilation.py +++ b/rpython/jit/backend/x86/test/test_recompilation.py @@ -33,7 +33,7 @@ jump(i1, descr=targettoken) ''' loop = self.interpret(ops, [0]) - previous = loop._jitcelltoken.compiled_loop_token.frame_depth + previous = loop._jitcelltoken.compiled_loop_token.frame_info.jfi_frame_depth assert self.getint(0) == 20 ops = ''' [i1] @@ -43,14 +43,15 @@ i6 = int_add(i5, 1) i7 = int_add(i5, i4) force_spill(i5) + force_spill(i6) + force_spill(i7) i8 = int_add(i7, 1) i9 = int_add(i8, 1) guard_false(i3, descr=fdescr2) [i3, i4, i5, i6, i7, i8, i9] finish() ''' - bridge = self.attach_bridge(ops, loop, -2) - descr = loop.operations[3].getdescr() - new = descr._x86_bridge_frame_depth + self.attach_bridge(ops, loop, -2) + new = loop._jitcelltoken.compiled_loop_token.frame_info.jfi_frame_depth # the force_spill() forces the stack to grow assert new > previous fail = self.run(loop, 0) @@ -103,20 +104,32 @@ [i97, i3] i10 = int_mul(i3, 2) i8 = int_add(i3, 1) + i15 = int_add(i3, 1) + i16 = int_add(i3, 1) + i17 = int_add(i3, 1) + i18 = int_add(i3, 1) i6 = int_add(i8, i10) i7 = int_add(i3, i6) force_spill(i6) force_spill(i7) force_spill(i8) + force_spill(i10) i12 = int_add(i7, i8) i11 = int_add(i12, i6) + force_spill(i11) + force_spill(i12) + force_spill(i15) + force_spill(i16) + force_spill(i17) + force_spill(i18) + guard_true(i18) [i3, i12, i11, i10, i6, i7, i18, i17, i16] jump(i3, i12, i11, i10, i6, i7, descr=targettoken) ''' - loop_frame_depth = loop._jitcelltoken.compiled_loop_token.frame_depth + loop_frame_depth = loop._jitcelltoken.compiled_loop_token.frame_info.jfi_frame_depth bridge = self.attach_bridge(ops, loop, 6) - guard_op = loop.operations[6] # the force_spill() forces the stack to grow - assert guard_op.getdescr()._x86_bridge_frame_depth > loop_frame_depth + bridge_frame_depth = loop._jitcelltoken.compiled_loop_token.frame_info.jfi_frame_depth + assert bridge_frame_depth > loop_frame_depth self.run(loop, 0, 0, 0, 0, 0, 0) assert self.getint(0) == 1 assert self.getint(1) == 20 From noreply at buildbot.pypy.org Tue Jan 29 15:21:10 2013 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 29 Jan 2013 15:21:10 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: happily ignore the immortal parameter Message-ID: <20130129142110.D575F1C0207@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60684:c88fd8172c57 Date: 2013-01-29 16:19 +0200 http://bitbucket.org/pypy/pypy/changeset/c88fd8172c57/ Log: happily ignore the immortal parameter diff --git a/rpython/annotator/builtin.py b/rpython/annotator/builtin.py --- a/rpython/annotator/builtin.py +++ b/rpython/annotator/builtin.py @@ -405,7 +405,7 @@ from rpython.rtyper.lltypesystem import lltype def malloc(s_T, s_n=None, s_flavor=None, s_zero=None, s_track_allocation=None, - s_add_memory_pressure=None): + s_add_memory_pressure=None, s_immortal=None): assert (s_n is None or s_n.knowntype == int or issubclass(s_n.knowntype, rpython.rlib.rarithmetic.base_int)) assert s_T.is_constant() diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py --- a/rpython/rtyper/rbuiltin.py +++ b/rpython/rtyper/rbuiltin.py @@ -357,7 +357,7 @@ # annotation of low-level types def rtype_malloc(hop, i_flavor=None, i_zero=None, i_track_allocation=None, - i_add_memory_pressure=None): + i_add_memory_pressure=None, i_immortal=None): assert hop.args_s[0].is_constant() vlist = [hop.inputarg(lltype.Void, arg=0)] opname = 'malloc' From noreply at buildbot.pypy.org Tue Jan 29 15:21:12 2013 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 29 Jan 2013 15:21:12 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: merge Message-ID: <20130129142112.0CA571C0207@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60685:fad2cb706322 Date: 2013-01-29 16:20 +0200 http://bitbucket.org/pypy/pypy/changeset/fad2cb706322/ Log: merge diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py --- a/rpython/jit/backend/arm/assembler.py +++ b/rpython/jit/backend/arm/assembler.py @@ -8,7 +8,7 @@ N_REGISTERS_SAVED_BY_MALLOC, \ JITFRAME_FIXED_SIZE, FRAME_FIXED_SIZE from rpython.jit.backend.arm.codebuilder import ARMv7Builder, OverwritingBuilder -from rpython.jit.backend.arm.locations import get_fp_offset +from rpython.jit.backend.arm.locations import get_fp_offset, imm from rpython.jit.backend.arm.regalloc import (Regalloc, ARMFrameManager, CoreRegisterManager, check_imm_arg, operations as regalloc_operations, @@ -118,6 +118,9 @@ self.gcmap_for_finish = lltype.malloc(jitframe.GCMAP, 1, zero=True) self.gcmap_for_finish[0] = r_uint(1) + def setup_failure_recovery(self): + self.failure_recovery_code = [0, 0, 0, 0] + def finish_once(self): if self._debug: debug_start('jit-backend-counts') @@ -334,199 +337,6 @@ rawstart = mc.materialize(self.cpu.asmmemmgr, []) self.wb_slowpath[withcards + 2 * withfloats] = rawstart - def setup_failure_recovery(self): - - #@rgc.no_collect -- XXX still true, but hacked gc_set_extra_threshold - def failure_recovery_func(mem_loc, frame_pointer, stack_pointer): - """mem_loc is a structure in memory describing where the values for - the failargs are stored. frame loc is the address of the frame - pointer for the frame to be decoded frame """ - vfp_registers = rffi.cast(rffi.LONGP, stack_pointer) - registers = rffi.ptradd(vfp_registers, 2*len(r.all_vfp_regs)) - registers = rffi.cast(rffi.LONGP, registers) - bytecode = rffi.cast(rffi.UCHARP, mem_loc) - return self.grab_frame_values(self.cpu, bytecode, frame_pointer, - registers, vfp_registers) - self.failure_recovery_code = [0, 0, 0, 0] - - self.failure_recovery_func = failure_recovery_func - - _FAILURE_RECOVERY_FUNC = lltype.Ptr(lltype.FuncType([rffi.LONGP] * 3, - llmemory.GCREF)) - - @staticmethod - #@rgc.no_collect -- XXX still true, but hacked gc_set_extra_threshold - def grab_frame_values(cpu, bytecode, frame_pointer, - registers, vfp_registers): - # no malloc allowed here!! xxx apart from one, hacking a lot - force_index = rffi.cast(lltype.Signed, frame_pointer) - num = 0 - deadframe = lltype.nullptr(jitframe.DEADFRAME) - # step 1: lots of mess just to count the final value of 'num' - bytecode1 = bytecode - while 1: - code = rffi.cast(lltype.Signed, bytecode1[0]) - bytecode1 = rffi.ptradd(bytecode1, 1) - if code >= AssemblerARM.CODE_FROMSTACK: - while code > 0x7F: - code = rffi.cast(lltype.Signed, bytecode1[0]) - bytecode1 = rffi.ptradd(bytecode1, 1) - else: - kind = code & 3 - if kind == AssemblerARM.DESCR_SPECIAL: - if code == AssemblerARM.CODE_HOLE: - num += 1 - continue - if code == AssemblerARM.CODE_INPUTARG: - continue - if code == AssemblerARM.CODE_FORCED: - # resuming from a GUARD_NOT_FORCED - token = force_index - deadframe = ( - cpu.assembler.force_token_to_dead_frame.pop(token)) - deadframe = lltype.cast_opaque_ptr( - jitframe.DEADFRAMEPTR, deadframe) - continue - assert code == AssemblerARM.CODE_STOP - break - num += 1 - - # allocate the deadframe - if not deadframe: - # Remove the "reserve" at the end of the nursery. This means - # that it is guaranteed that the following malloc() works - # without requiring a collect(), but it needs to be re-added - # as soon as possible. - cpu.gc_clear_extra_threshold() - assert num <= cpu.get_failargs_limit() - try: - deadframe = lltype.malloc(jitframe.DEADFRAME, num) - except MemoryError: - fatalerror("memory usage error in grab_frame_values") - # fill it - code_inputarg = False - num = 0 - value_hi = 0 - while 1: - # decode the next instruction from the bytecode - code = rffi.cast(lltype.Signed, bytecode[0]) - bytecode = rffi.ptradd(bytecode, 1) - if code >= AssemblerARM.CODE_FROMSTACK: - if code > 0x7F: - shift = 7 - code &= 0x7F - while True: - nextcode = rffi.cast(lltype.Signed, bytecode[0]) - bytecode = rffi.ptradd(bytecode, 1) - code |= (nextcode & 0x7F) << shift - shift += 7 - if nextcode <= 0x7F: - break - # load the value from the stack - kind = code & 3 - code = (code - AssemblerARM.CODE_FROMSTACK) >> 2 - if code_inputarg: - code = ~code - code_inputarg = False - stackloc = force_index - get_fp_offset(int(code)) - value = rffi.cast(rffi.LONGP, stackloc)[0] - if kind == AssemblerARM.DESCR_FLOAT: - assert WORD == 4 - value_hi = value - value = rffi.cast(rffi.LONGP, stackloc - WORD)[0] - else: - kind = code & 3 - if kind == AssemblerARM.DESCR_SPECIAL: - if code == AssemblerARM.CODE_HOLE: - num += 1 - continue - if code == AssemblerARM.CODE_INPUTARG: - code_inputarg = True - continue - if code == AssemblerARM.CODE_FORCED: - continue - assert code == AssemblerARM.CODE_STOP - break - # 'code' identifies a register: load its value - code >>= 2 - if kind == AssemblerARM.DESCR_FLOAT: - if WORD == 4: - value = vfp_registers[2*code] - value_hi = vfp_registers[2*code + 1] - else: - value = registers[code] - else: - value = registers[code] - # store the loaded value into fail_boxes_ - if kind == AssemblerARM.DESCR_INT: - deadframe.jf_values[num].int = value - elif kind == AssemblerARM.DESCR_REF: - deadframe.jf_values[num].ref = rffi.cast(llmemory.GCREF, value) - elif kind == AssemblerARM.DESCR_FLOAT: - assert WORD == 4 - assert not longlong.is_64_bit - floatvalue = rffi.cast(lltype.SignedLongLong, value_hi) - floatvalue <<= 32 - floatvalue |= rffi.cast(lltype.SignedLongLong, - rffi.cast(lltype.Unsigned, value)) - deadframe.jf_values[num].float = floatvalue - else: - assert 0, "bogus kind" - num += 1 - # - assert num == len(deadframe.jf_values) - if not we_are_translated(): - assert bytecode[4] == 0xCC - fail_index = rffi.cast(rffi.INTP, bytecode)[0] - fail_descr = cpu.get_fail_descr_from_number(fail_index) - deadframe.jf_descr = fail_descr.hide(cpu) - return lltype.cast_opaque_ptr(llmemory.GCREF, deadframe) - - def decode_inputargs(self, code): - descr_to_box_type = [REF, INT, FLOAT] - bytecode = rffi.cast(rffi.UCHARP, code) - arglocs = [] - code_inputarg = False - while 1: - # decode the next instruction from the bytecode - code = rffi.cast(lltype.Signed, bytecode[0]) - bytecode = rffi.ptradd(bytecode, 1) - if code >= self.CODE_FROMSTACK: - # 'code' identifies a stack location - if code > 0x7F: - shift = 7 - code &= 0x7F - while True: - nextcode = rffi.cast(lltype.Signed, bytecode[0]) - bytecode = rffi.ptradd(bytecode, 1) - code |= (nextcode & 0x7F) << shift - shift += 7 - if nextcode <= 0x7F: - break - kind = code & 3 - code = (code - self.CODE_FROMSTACK) >> 2 - if code_inputarg: - code = ~code - code_inputarg = False - loc = ARMFrameManager.frame_pos(code, descr_to_box_type[kind]) - elif code == self.CODE_STOP: - break - elif code == self.CODE_HOLE: - continue - elif code == self.CODE_INPUTARG: - code_inputarg = True - continue - else: - # 'code' identifies a register - kind = code & 3 - code >>= 2 - if kind == self.DESCR_FLOAT: - loc = r.all_vfp_regs[code] - else: - loc = r.all_regs[code] - arglocs.append(loc) - return arglocs[:] - def _build_malloc_slowpath(self): mc = ARMv7Builder() if self.cpu.supports_floats: @@ -562,48 +372,63 @@ # see ../x86/assembler.py:propagate_memoryerror_if_eax_is_null self.mc.CMP_ri(r.r0.value, 0) self.mc.B(self.propagate_exception_path, c=c.EQ) + + def _push_all_regs_to_jitframe(self, mc, ignored_regs, withfloats, + callee_only=False): + if callee_only: + regs = CoreRegisterManager.save_around_call_regs + else: + regs = CoreRegisterManager.all_regs + # use STMDB ops here + for i, gpr in enumerate(regs): + if gpr in ignored_regs: + continue + mc.STR_ri(gpr.value, r.fp.value, i * WORD) + if withfloats: + assert 0, 'implement me' def _build_failure_recovery(self, exc, withfloats=False): mc = ARMv7Builder() - failure_recovery = llhelper(self._FAILURE_RECOVERY_FUNC, - self.failure_recovery_func) + self._push_all_regs_to_jitframe(mc, [], withfloats) self._insert_checks(mc) - if withfloats: - f = r.all_vfp_regs - else: - f = [] - with saved_registers(mc, r.all_regs, f): - if exc: - # We might have an exception pending. Load it into r4 - # (this is a register saved across calls) - mc.gen_load_int(r.r5.value, self.cpu.pos_exc_value()) - mc.LDR_ri(r.r4.value, r.r5.value) - # clear the exc flags - mc.gen_load_int(r.r6.value, 0) - mc.STR_ri(r.r6.value, r.r5.value) - mc.gen_load_int(r.r5.value, self.cpu.pos_exception()) - mc.STR_ri(r.r6.value, r.r5.value) - # move mem block address, to r0 to pass as - mc.MOV_rr(r.r0.value, r.lr.value) - # pass the current frame pointer as second param - mc.MOV_rr(r.r1.value, r.fp.value) - # pass the current stack pointer as third param - mc.MOV_rr(r.r2.value, r.sp.value) - self._insert_checks(mc) - mc.BL(rffi.cast(lltype.Signed, failure_recovery)) - if exc: - # save ebx into 'jf_guard_exc' - from rpython.jit.backend.llsupport.descr import unpack_fielddescr - descrs = self.cpu.gc_ll_descr.getframedescrs(self.cpu) - offset, size, _ = unpack_fielddescr(descrs.jf_guard_exc) - mc.STR_rr(r.r4.value, r.r0.value, offset, cond=c.AL) - mc.MOV_rr(r.ip.value, r.r0.value) - mc.MOV_rr(r.r0.value, r.ip.value) - self.gen_func_epilog(mc=mc) - rawstart = mc.materialize(self.cpu.asmmemmgr, [], - self.cpu.gc_ll_descr.gcrootmap) + + if exc: + # We might have an exception pending. Load it into r4 + # (this is a register saved across calls) + mc.gen_load_int(r.r5.value, self.cpu.pos_exc_value()) + mc.LDR_ri(r.r4.value, r.r5.value) + # clear the exc flags + mc.gen_load_int(r.r6.value, 0) + mc.STR_ri(r.r6.value, r.r5.value) # pos_exc_value is still in r5 + mc.gen_load_int(r.r5.value, self.cpu.pos_exception()) + mc.STR_ri(r.r6.value, r.r5.value) + # save r4 into 'jf_guard_exc' + offset = self.cpu.get_ofs_of_frame_field('jf_guard_exc') + assert check_imm_arg(abs(offset)) + mc.STR_ri(r.r4.value, r.fp.value, imm=offset) + # now we return from the complete frame, which starts from + # _call_header_with_stack_check(). The LEA in _call_footer below + # throws away most of the frame, including all the PUSHes that we + # did just above. + ofs = self.cpu.get_ofs_of_frame_field('jf_descr') + assert check_imm_arg(abs(ofs)) + ofs2 = self.cpu.get_ofs_of_frame_field('jf_gcmap') + assert check_imm_arg(abs(ofs2)) + base_ofs = self.cpu.get_baseofs_of_frame_field() + # store the gcmap + mc.POP([r.ip.value]) + mc.STR_ri(r.ip.value, r.fp.value, imm=ofs2) + # store the descr + mc.POP([r.ip.value]) + mc.STR_ri(r.ip.value, r.fp.value, imm=ofs) + + # set return value + assert check_imm_arg(base_ofs) + mc.SUB_ri(r.r0.value, r.fp.value, base_ofs) + + self.gen_func_epilog(mc) + rawstart = mc.materialize(self.cpu.asmmemmgr, []) self.failure_recovery_code[exc + 2 * withfloats] = rawstart - self.mc = None DESCR_REF = 0x00 DESCR_INT = 0x01 @@ -649,30 +474,37 @@ def generate_quick_failure(self, guardtok, fcond=c.AL): - assert isinstance(guardtok.save_exc, bool) - fail_index = self.cpu.get_fail_descr_number(guardtok.descr) + assert isinstance(guardtok.exc, bool) startpos = self.mc.currpos() withfloats = False for box in guardtok.failargs: if box is not None and box.type == FLOAT: withfloats = True break - exc = guardtok.save_exc + exc = guardtok.exc target = self.failure_recovery_code[exc + 2 * withfloats] - assert target != 0 + fail_descr = cast_instance_to_gcref(guardtok.faildescr) + fail_descr = rffi.cast(lltype.Signed, fail_descr) + positions = [0] * len(guardtok.fail_locs) + for i, loc in enumerate(guardtok.fail_locs): + if loc is None: + positions[i] = -1 + elif loc.is_stack(): + positions[i] = loc.value + else: + if loc.is_reg(): + assert loc is not r.fp # for now + v = loc.value + else: + assert 0, 'fix for floats' + assert loc.is_vfp_reg() + #v = len(VFPRegisterManager.all_regs) + loc.value + positions[i] = v * WORD + # write down the positions of locs + guardtok.faildescr.rd_locs = positions + self.regalloc_push(imm(fail_descr)) + self.push_gcmap(self.mc, gcmap=guardtok.gcmap, push=True) self.mc.BL(target) - # write tight data that describes the failure recovery - if guardtok.is_guard_not_forced: - self.mc.writechar(chr(self.CODE_FORCED)) - self.write_failure_recovery_description(guardtok.descr, - guardtok.failargs, guardtok.faillocs[1:]) - self.mc.write32(fail_index) - # for testing the decoding, write a final byte 0xCC - if not we_are_translated(): - self.mc.writechar('\xCC') - faillocs = [loc for loc in guardtok.faillocs if loc is not None] - guardtok.descr._arm_debug_faillocs = faillocs - self.align() return startpos def align(self): @@ -929,13 +761,11 @@ for tok in self.pending_guards: #generate the exit stub and the encoded representation tok.pos_recovery_stub = self.generate_quick_failure(tok) - # store info on the descr - tok.descr._arm_current_frame_depth = tok.faillocs[0].getint() def process_pending_guards(self, block_start): clt = self.current_clt for tok in self.pending_guards: - descr = tok.descr + descr = tok.faildescr assert isinstance(descr, AbstractFailDescr) failure_recovery_pos = block_start + tok.pos_recovery_stub descr._arm_failure_recovery_block = failure_recovery_pos @@ -1395,16 +1225,18 @@ # keep the ref alive self.current_clt.allgcrefs.append(gcmapref) rgc._make_sure_does_not_move(gcmapref) - pass - #if push: - # mc.PUSH(imm(rffi.cast(lltype.Signed, gcmapref))) - #elif mov: - # mc.MOV(RawEspLoc(0, REF), - # imm(rffi.cast(lltype.Signed, gcmapref))) - #else: - # assert store - # ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') - # mc.MOV(raw_stack(ofs), imm(rffi.cast(lltype.Signed, gcmapref))) + if push: + mc.gen_load_int(r.ip.value, rffi.cast(lltype.Signed, gcmapref)) + mc.PUSH([r.ip.value]) + elif mov: + assert 0 + mc.MOV(RawEspLoc(0, REF), + imm(rffi.cast(lltype.Signed, gcmapref))) + else: + assert store + ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') + mc.gen_load_int(r.ip.value, rffi.cast(lltype.Signed, gcmapref)) + mc.STR_ri(r.ip.value, r.fp.value, imm=ofs) def pop_gcmap(self, mc): ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') diff --git a/rpython/jit/backend/arm/codebuilder.py b/rpython/jit/backend/arm/codebuilder.py --- a/rpython/jit/backend/arm/codebuilder.py +++ b/rpython/jit/backend/arm/codebuilder.py @@ -1,7 +1,7 @@ from rpython.jit.backend.arm import conditions as cond from rpython.jit.backend.arm import registers as reg from rpython.jit.backend.arm import support -from rpython.jit.backend.arm.arch import (WORD, FUNC_ALIGN) +from rpython.jit.backend.arm.arch import (WORD, FUNC_ALIGN, PC_OFFSET) from rpython.jit.backend.arm.instruction_builder import define_instructions from rpython.jit.backend.llsupport.asmmemmgr import BlockBuilderMixin from rpython.rlib.objectmodel import we_are_translated @@ -186,7 +186,7 @@ def B_offs(self, target_ofs, c=cond.AL): pos = self.currpos() - target_ofs = target_ofs - (pos + arch.PC_OFFSET) + target_ofs = target_ofs - (pos + PC_OFFSET) assert target_ofs & 0x3 == 0 self.write32(c << 28 | 0xA << 24 | (target_ofs >> 2) & 0xFFFFFF) diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -2,7 +2,7 @@ from rpython.jit.backend.arm import conditions as c from rpython.jit.backend.arm import registers as r from rpython.jit.backend.arm import shift -from rpython.jit.backend.arm.arch import WORD, DOUBLE_WORD +from rpython.jit.backend.arm.arch import WORD, DOUBLE_WORD, JITFRAME_FIXED_SIZE from rpython.jit.backend.arm.helper.assembler import (gen_emit_op_by_helper_call, gen_emit_op_unary_cmp, @@ -21,7 +21,7 @@ from rpython.jit.backend.arm.jump import remap_frame_layout from rpython.jit.backend.arm.regalloc import TempInt, TempPtr from rpython.jit.backend.arm.locations import imm -from rpython.jit.backend.llsupport import symbolic +from rpython.jit.backend.llsupport import symbolic, jitframe from rpython.jit.backend.llsupport.descr import InteriorFieldDescr from rpython.jit.metainterp.history import (Box, AbstractFailDescr, INT, FLOAT, REF) @@ -35,18 +35,43 @@ class GuardToken(object): - def __init__(self, descr, failargs, faillocs, offset, save_exc, fcond=c.AL, + def __init__(self, faildescr, failargs, fail_locs, offset, exc, fcond=c.AL, is_guard_not_invalidated=False, is_guard_not_forced=False): - assert isinstance(save_exc, bool) - self.descr = descr + assert isinstance(exc, bool) + self.faildescr = faildescr + self.failargs = failargs + self.fail_locs = fail_locs[1:] self.offset = offset + self.gcmap = self.compute_gcmap(failargs, fail_locs, fail_locs[0].value) + self.exc = exc self.is_guard_not_invalidated = is_guard_not_invalidated self.is_guard_not_forced = is_guard_not_forced - self.failargs = failargs - self.faillocs = faillocs - self.save_exc = save_exc self.fcond = fcond + def compute_gcmap(self, failargs, fail_locs, frame_depth): + # note that regalloc has a very similar compute, but + # one that does iteration over all bindings, so slightly different, + # eh + size = frame_depth + JITFRAME_FIXED_SIZE + gcmap = lltype.malloc(jitframe.GCMAP, size // WORD // 8 + 1, + zero=True) + input_i = 0 + for i in range(len(failargs)): + arg = failargs[i] + if arg is None: + continue + loc = fail_locs[input_i] + input_i += 1 + if arg.type == REF: + loc = fail_locs[i] + if loc.is_reg(): + val = loc.value + else: + assert 0, 'ffuu, implement' + val = loc.value // WORD + gcmap[val // WORD // 8] |= r_uint(1) << (val % (WORD * 8)) + return gcmap + class ResOpAssembler(object): @@ -211,9 +236,9 @@ self.mc.BKPT() self.pending_guards.append(GuardToken(descr, failargs=op.getfailargs(), - faillocs=arglocs, + fail_locs=arglocs, offset=pos, - save_exc=save_exc, + exc=save_exc, is_guard_not_invalidated=is_guard_not_invalidated, is_guard_not_forced=is_guard_not_forced, fcond=fcond)) diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py --- a/rpython/jit/backend/arm/regalloc.py +++ b/rpython/jit/backend/arm/regalloc.py @@ -709,6 +709,7 @@ # optimization only: fill in the 'hint_frame_locations' dictionary # of rm and xrm based on the JUMP at the end of the loop, by looking # at where we would like the boxes to be after the jump. + return # XXX disabled for now op = operations[-1] if op.getopnum() != rop.JUMP: return From noreply at buildbot.pypy.org Tue Jan 29 15:23:59 2013 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 29 Jan 2013 15:23:59 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: more lack of support Message-ID: <20130129142359.ABB441C039A@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60686:792b152292a5 Date: 2013-01-29 16:23 +0200 http://bitbucket.org/pypy/pypy/changeset/792b152292a5/ Log: more lack of support diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py --- a/rpython/rtyper/rbuiltin.py +++ b/rpython/rtyper/rbuiltin.py @@ -361,13 +361,15 @@ assert hop.args_s[0].is_constant() vlist = [hop.inputarg(lltype.Void, arg=0)] opname = 'malloc' - v_flavor, v_zero, v_track_allocation, v_add_memory_pressure = parse_kwds( + kwds_v = parse_kwds( hop, (i_flavor, lltype.Void), (i_zero, None), (i_track_allocation, None), - (i_add_memory_pressure, None)) - + (i_add_memory_pressure, None), + (i_immortal, None)) + (v_flavor, v_zero, v_track_allocation, + v_add_memory_pressure, v_immortal) = kwds_v flags = {'flavor': 'gc'} if v_flavor is not None: flags['flavor'] = v_flavor.value From noreply at buildbot.pypy.org Tue Jan 29 15:30:13 2013 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 29 Jan 2013 15:30:13 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: meh Message-ID: <20130129143013.37CB51C12FC@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60687:89b4e02965bc Date: 2013-01-29 16:29 +0200 http://bitbucket.org/pypy/pypy/changeset/89b4e02965bc/ Log: meh diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -141,7 +141,7 @@ self.set_debug(have_debug_prints()) debug_stop('jit-backend-counts') # when finishing, we only have one value at [0], the rest dies - self.gcmap_for_finish = lltype.malloc(jitframe.GCMAP, 1, zero=True, + self.gcmap_for_finish = lltype.malloc(jitframe.GCMAP, 1, flavor='raw', immortal=True) self.gcmap_for_finish[0] = r_uint(1) From noreply at buildbot.pypy.org Tue Jan 29 16:01:23 2013 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 29 Jan 2013 16:01:23 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: implement pass-through convert_to for String, Unicode boxes Message-ID: <20130129150123.780CA1C13E3@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: missing-ndarray-attributes Changeset: r60688:76b82f8c1076 Date: 2013-01-29 16:52 +0200 http://bitbucket.org/pypy/pypy/changeset/76b82f8c1076/ Log: implement pass-through convert_to for String, Unicode boxes diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py --- a/pypy/module/micronumpy/interp_boxes.py +++ b/pypy/module/micronumpy/interp_boxes.py @@ -287,6 +287,11 @@ for i in range(len(arg)): arr.storage[i] = arg[i] return W_StringBox(arr, 0, arr.dtype) + + def convert_to(self, dtype): + from pypy.modules.micronumpy import types + assert isinstance(dtype.itemtype, types.StringType) + return self class W_UnicodeBox(W_CharacterBox): def descr__new__unicode_box(space, w_subtype, w_arg): @@ -300,6 +305,11 @@ # arr.storage[i] = arg[i] return W_UnicodeBox(arr, 0, arr.dtype) + def convert_to(self, dtype): + from pypy.modules.micronumpy import types + assert isinstance(dtype.itemtype, types.UnicodeType) + return self + class W_ComplexFloatingBox(W_InexactBox): _attrs_ = () From noreply at buildbot.pypy.org Tue Jan 29 16:01:25 2013 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 29 Jan 2013 16:01:25 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: simplify, the problem is probably not here anyway Message-ID: <20130129150125.110F91C13E3@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: missing-ndarray-attributes Changeset: r60689:0e05cbe257dc Date: 2013-01-29 17:02 +0200 http://bitbucket.org/pypy/pypy/changeset/0e05cbe257dc/ Log: simplify, the problem is probably not here anyway diff --git a/pypy/module/micronumpy/arrayimpl/sort.py b/pypy/module/micronumpy/arrayimpl/sort.py --- a/pypy/module/micronumpy/arrayimpl/sort.py +++ b/pypy/module/micronumpy/arrayimpl/sort.py @@ -82,23 +82,14 @@ def argsort_array(arr, space, w_axis): itemtype = arr.dtype.itemtype - # this is for runtime - written like this to simplify - # logic since translation could not handle it as one - # statement - if arr.dtype.is_int_type(): - pass - elif arr.dtype.is_float_type(): - pass - elif arr.dtype.is_complex_type(): - pass + if isinstance(itemtype, types.Float) or \ + isinstance(itemtype, types.Integer) or \ + isinstance(itemtype, types.ComplexFloating): + pass else: raise OperationError(space.w_NotImplementedError, space.wrap("sorting of non-numeric types " + \ "'%s' is not implemented" % arr.dtype.get_name() )) - # this is for translation - assert isinstance(itemtype, types.Float) or \ - isinstance(itemtype, types.Integer) or \ - isinstance(itemtype, types.ComplexFloating) if w_axis is space.w_None: # note that it's fine ot pass None here as we're not going # to pass the result around (None is the link to base in slices) From noreply at buildbot.pypy.org Tue Jan 29 16:53:23 2013 From: noreply at buildbot.pypy.org (cfbolz) Date: Tue, 29 Jan 2013 16:53:23 +0100 (CET) Subject: [pypy-commit] pypy default: a sanity check: a guard_value of a virtual object will always fail, thus the Message-ID: <20130129155323.AB0BB1C0264@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: Changeset: r60690:2ef0f6edb727 Date: 2013-01-29 16:53 +0100 http://bitbucket.org/pypy/pypy/changeset/2ef0f6edb727/ Log: a sanity check: a guard_value of a virtual object will always fail, thus the loop is invalid. diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -239,6 +239,8 @@ def optimize_GUARD_VALUE(self, op): value = self.getvalue(op.getarg(0)) + if value.is_virtual(): + raise InvalidLoop('A promote of a virtual (a recently allocated object) never makes sense!') if value.last_guard: # there already has been a guard_nonnull or guard_class or # guard_nonnull_class on this value, which is rather silly. diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -2623,7 +2623,7 @@ jump(p2) """ self.raises(InvalidLoop, self.optimize_loop, - ops, ops) + ops, "crash!") def test_invalid_loop_2(self): ops = """ @@ -2635,7 +2635,7 @@ jump(p2) """ self.raises(InvalidLoop, self.optimize_loop, - ops, ops) + ops, "crash!") def test_invalid_loop_3(self): ops = """ @@ -2648,8 +2648,17 @@ setfield_gc(p3, p4, descr=nextdescr) jump(p3) """ - self.raises(InvalidLoop, self.optimize_loop, ops, ops) - + self.raises(InvalidLoop, self.optimize_loop, ops, "crash!") + + def test_invalid_loop_guard_value_of_virtual(self): + ops = """ + [p1] + p2 = new_with_vtable(ConstClass(node_vtable)) + guard_value(p2, ConstPtr(myptr)) [] + jump(p2) + """ + self.raises(InvalidLoop, self.optimize_loop, + ops, "crash!") def test_merge_guard_class_guard_value(self): ops = """ From noreply at buildbot.pypy.org Tue Jan 29 17:06:27 2013 From: noreply at buildbot.pypy.org (bivab) Date: Tue, 29 Jan 2013 17:06:27 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: change addressing of values now stored in the jitframe Message-ID: <20130129160627.7F9141C039A@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: jitframe-on-heap Changeset: r60691:95b4d5a70499 Date: 2013-01-29 16:55 +0100 http://bitbucket.org/pypy/pypy/changeset/95b4d5a70499/ Log: change addressing of values now stored in the jitframe diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py --- a/rpython/jit/backend/arm/assembler.py +++ b/rpython/jit/backend/arm/assembler.py @@ -975,13 +975,13 @@ offset = loc.value if not check_imm_arg(offset, size=0xFFF): self.mc.PUSH([temp.value], cond=cond) - self.mc.gen_load_int(temp.value, -offset, cond=cond) + self.mc.gen_load_int(temp.value, offset, cond=cond) self.mc.STR_rr(prev_loc.value, r.fp.value, temp.value, cond=cond) self.mc.POP([temp.value], cond=cond) else: self.mc.STR_ri(prev_loc.value, r.fp.value, - imm=-offset, cond=cond) + imm=offset, cond=cond) else: assert 0, 'unsupported case' @@ -1089,15 +1089,15 @@ offset = vfp_loc.value if not check_imm_arg(offset, size=0xFFF): self.mc.PUSH([r.ip.value], cond=cond) - self.mc.gen_load_int(r.ip.value, -offset, cond=cond) + self.mc.gen_load_int(r.ip.value, offset, cond=cond) self.mc.LDR_rr(reg1.value, r.fp.value, r.ip.value, cond=cond) self.mc.ADD_ri(r.ip.value, r.ip.value, imm=WORD, cond=cond) self.mc.LDR_rr(reg2.value, r.fp.value, r.ip.value, cond=cond) self.mc.POP([r.ip.value], cond=cond) else: - self.mc.LDR_ri(reg1.value, r.fp.value, imm=-offset, cond=cond) + self.mc.LDR_ri(reg1.value, r.fp.value, imm=offset, cond=cond) self.mc.LDR_ri(reg2.value, r.fp.value, - imm=-offset + WORD, cond=cond) + imm=offset + WORD, cond=cond) else: assert 0, 'unsupported case' @@ -1112,15 +1112,15 @@ offset = vfp_loc.value if not check_imm_arg(offset, size=0xFFF): self.mc.PUSH([r.ip.value], cond=cond) - self.mc.gen_load_int(r.ip.value, -offset, cond=cond) + self.mc.gen_load_int(r.ip.value, offset, cond=cond) self.mc.STR_rr(reg1.value, r.fp.value, r.ip.value, cond=cond) self.mc.ADD_ri(r.ip.value, r.ip.value, imm=WORD, cond=cond) self.mc.STR_rr(reg2.value, r.fp.value, r.ip.value, cond=cond) self.mc.POP([r.ip.value], cond=cond) else: - self.mc.STR_ri(reg1.value, r.fp.value, imm=-offset, cond=cond) + self.mc.STR_ri(reg1.value, r.fp.value, imm=offset, cond=cond) self.mc.STR_ri(reg2.value, r.fp.value, - imm=-offset + WORD, cond=cond) + imm=offset + WORD, cond=cond) else: assert 0, 'unsupported case' From noreply at buildbot.pypy.org Tue Jan 29 17:06:28 2013 From: noreply at buildbot.pypy.org (bivab) Date: Tue, 29 Jan 2013 17:06:28 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: port most of the latest changes to arm, regarding among other things gcmap initialization Message-ID: <20130129160628.D65931C039A@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: jitframe-on-heap Changeset: r60692:c4e0750d01c6 Date: 2013-01-29 17:00 +0100 http://bitbucket.org/pypy/pypy/changeset/c4e0750d01c6/ Log: port most of the latest changes to arm, regarding among other things gcmap initialization diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py --- a/rpython/jit/backend/arm/assembler.py +++ b/rpython/jit/backend/arm/assembler.py @@ -115,7 +115,8 @@ self.set_debug(have_debug_prints()) debug_stop('jit-backend-counts') # when finishing, we only have one value at [0], the rest dies - self.gcmap_for_finish = lltype.malloc(jitframe.GCMAP, 1, zero=True) + self.gcmap_for_finish = lltype.malloc(jitframe.GCMAP, 1, + flavor='raw', immortal=True) self.gcmap_for_finish[0] = r_uint(1) def setup_failure_recovery(self): @@ -502,6 +503,8 @@ positions[i] = v * WORD # write down the positions of locs guardtok.faildescr.rd_locs = positions + # we want the descr to keep alive + guardtok.faildescr.rd_loop_token = self.current_clt self.regalloc_push(imm(fail_descr)) self.push_gcmap(self.mc, gcmap=guardtok.gcmap, push=True) self.mc.BL(target) @@ -603,9 +606,6 @@ # cpu interface def assemble_loop(self, loopname, inputargs, operations, looptoken, log): clt = CompiledLoopToken(self.cpu, looptoken.number) - clt.frame_info = lltype.malloc(jitframe.JITFRAMEINFO) - clt.allgcrefs = [] - clt.frame_info.jfi_frame_depth = 0 # for now looptoken.compiled_loop_token = clt clt._debug_nbargs = len(inputargs) @@ -614,6 +614,13 @@ assert len(set(inputargs)) == len(inputargs) self.setup(looptoken) + + frame_info = self.datablockwrapper.malloc_aligned( + jitframe.JITFRAMEINFO_SIZE, alignment=WORD) + clt.frame_info = rffi.cast(jitframe.JITFRAMEINFOPTR, frame_info) + clt.allgcrefs = [] + clt.frame_info.set_frame_depth(0, 0) # for now + if False and log: operations = self._inject_debugging_code(looptoken, operations, 'e', looptoken.number) @@ -624,8 +631,6 @@ regalloc = Regalloc(assembler=self) operations = regalloc.prepare_loop(inputargs, operations, looptoken, clt.allgcrefs) - rgc._make_sure_does_not_move(lltype.cast_opaque_ptr(llmemory.GCREF, - clt.frame_info)) loop_head = self.mc.get_relative_pos() looptoken._arm_loop_code = loop_head @@ -755,7 +760,8 @@ self.cpu.gc_ll_descr.gcrootmap) def update_frame_depth(self, frame_depth): - self.current_clt.frame_info.jfi_frame_depth = frame_depth + baseofs = self.cpu.get_baseofs_of_frame_field() + self.current_clt.frame_info.set_frame_depth(baseofs, frame_depth) def write_pending_failure_recoveries(self): for tok in self.pending_guards: @@ -1118,6 +1124,7 @@ self.mc.STR_rr(reg2.value, r.fp.value, r.ip.value, cond=cond) self.mc.POP([r.ip.value], cond=cond) else: + assert 0, 'verify this code' self.mc.STR_ri(reg1.value, r.fp.value, imm=offset, cond=cond) self.mc.STR_ri(reg2.value, r.fp.value, imm=offset + WORD, cond=cond) @@ -1221,21 +1228,17 @@ return 0 def push_gcmap(self, mc, gcmap, push=False, mov=False, store=False): - gcmapref = lltype.cast_opaque_ptr(llmemory.GCREF, gcmap) - # keep the ref alive - self.current_clt.allgcrefs.append(gcmapref) - rgc._make_sure_does_not_move(gcmapref) + ptr = rffi.cast(lltype.Signed, gcmap) if push: - mc.gen_load_int(r.ip.value, rffi.cast(lltype.Signed, gcmapref)) + mc.gen_load_int(r.ip.value, ptr) mc.PUSH([r.ip.value]) elif mov: assert 0 - mc.MOV(RawEspLoc(0, REF), - imm(rffi.cast(lltype.Signed, gcmapref))) + mc.MOV(RawEspLoc(0, REF), ptr) else: assert store ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') - mc.gen_load_int(r.ip.value, rffi.cast(lltype.Signed, gcmapref)) + mc.gen_load_int(r.ip.value, ptr) mc.STR_ri(r.ip.value, r.fp.value, imm=ofs) def pop_gcmap(self, mc): diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -30,31 +30,31 @@ from rpython.rlib.objectmodel import we_are_translated from rpython.rlib import rgc from rpython.rtyper.lltypesystem import rstr, rffi, lltype, llmemory +from rpython.rlib.rarithmetic import r_uint NO_FORCE_INDEX = -1 class GuardToken(object): - def __init__(self, faildescr, failargs, fail_locs, offset, exc, fcond=c.AL, - is_guard_not_invalidated=False, is_guard_not_forced=False): + def __init__(self, gcmap, faildescr, failargs, fail_locs, offset, exc, + frame_depth, fcond=c.AL, is_guard_not_invalidated=False, + is_guard_not_forced=False): assert isinstance(exc, bool) self.faildescr = faildescr self.failargs = failargs self.fail_locs = fail_locs[1:] self.offset = offset - self.gcmap = self.compute_gcmap(failargs, fail_locs, fail_locs[0].value) + self.gcmap = self.compute_gcmap(gcmap, failargs, + fail_locs, frame_depth) self.exc = exc self.is_guard_not_invalidated = is_guard_not_invalidated self.is_guard_not_forced = is_guard_not_forced self.fcond = fcond - def compute_gcmap(self, failargs, fail_locs, frame_depth): + def compute_gcmap(self, gcmap, failargs, fail_locs, frame_depth): # note that regalloc has a very similar compute, but # one that does iteration over all bindings, so slightly different, # eh - size = frame_depth + JITFRAME_FIXED_SIZE - gcmap = lltype.malloc(jitframe.GCMAP, size // WORD // 8 + 1, - zero=True) input_i = 0 for i in range(len(failargs)): arg = failargs[i] @@ -234,16 +234,32 @@ self.mc.NOP() else: self.mc.BKPT() - self.pending_guards.append(GuardToken(descr, + gcmap = self.allocate_gcmap(arglocs[0].value) + self.pending_guards.append(GuardToken(gcmap, + descr, failargs=op.getfailargs(), - fail_locs=arglocs, + fail_locs=arglocs[1:], offset=pos, exc=save_exc, + frame_depth=arglocs[0].value, is_guard_not_invalidated=is_guard_not_invalidated, is_guard_not_forced=is_guard_not_forced, fcond=fcond)) return c.AL + def allocate_gcmap(self, frame_depth): + size = frame_depth + JITFRAME_FIXED_SIZE + malloc_size = (size // WORD // 8 + 1) + 1 + rawgcmap = self.datablockwrapper.malloc_aligned(WORD * malloc_size, + WORD) + # set the length field + rffi.cast(rffi.CArrayPtr(lltype.Signed), rawgcmap)[0] = malloc_size - 1 + gcmap = rffi.cast(lltype.Ptr(jitframe.GCMAP), rawgcmap) + # zero the area + for i in range(malloc_size - 1): + gcmap[i] = r_uint(0) + return gcmap + def _emit_guard_overflow(self, guard, failargs, fcond): if guard.getopnum() == rop.GUARD_OVERFLOW: fcond = self._emit_guard(guard, failargs, c.VS, save_exc=False) @@ -327,18 +343,18 @@ # so that the locations [ebp+8..] of the input arguments are valid # stack locations both before and after the jump. # - descr = op.getdescr() - assert isinstance(descr, TargetToken) + target_token = op.getdescr() + assert isinstance(target_token, TargetToken) assert fcond == c.AL my_nbargs = self.current_clt._debug_nbargs - target_nbargs = descr._arm_clt._debug_nbargs + target_nbargs = target_token._arm_clt._debug_nbargs assert my_nbargs == target_nbargs self._insert_checks() - if descr in self.target_tokens_currently_compiling: - self.mc.B_offs(descr._arm_loop_code, fcond) + if target_token in self.target_tokens_currently_compiling: + self.mc.B_offs(target_token._arm_loop_code, fcond) else: - self.mc.B(descr._arm_loop_code, fcond) + self.mc.B(target_token._arm_loop_code, fcond) return fcond def emit_op_finish(self, op, arglocs, regalloc, fcond): diff --git a/rpython/jit/backend/arm/runner.py b/rpython/jit/backend/arm/runner.py --- a/rpython/jit/backend/arm/runner.py +++ b/rpython/jit/backend/arm/runner.py @@ -9,6 +9,7 @@ from rpython.rlib.unroll import unrolling_iterable from rpython.rtyper.llinterp import LLInterpreter from rpython.rtyper.lltypesystem import lltype, rffi, llmemory +from rpython.rtyper.lltypesystem.lloperation import llop jitframe.STATICSIZE = JITFRAME_FIXED_SIZE @@ -99,6 +100,9 @@ assert kind == history.REF self.set_ref_value(ll_frame, num, arg) num += WORD + # no GC operation between gc_assume_young_pointers and + # the actual call to assembler! + llop.gc_assume_young_pointers(lltype.Void, frame) ll_frame = func(ll_frame) finally: if not self.translate_support_code: From noreply at buildbot.pypy.org Tue Jan 29 17:15:52 2013 From: noreply at buildbot.pypy.org (fijal) Date: Tue, 29 Jan 2013 17:15:52 +0100 (CET) Subject: [pypy-commit] pypy incremental-nursery-cleanup: clean nursery in stages Message-ID: <20130129161552.B67FB1C0041@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: incremental-nursery-cleanup Changeset: r60693:7291c14e7c4a Date: 2013-01-29 18:13 +0200 http://bitbucket.org/pypy/pypy/changeset/7291c14e7c4a/ Log: clean nursery in stages diff --git a/rpython/rtyper/memory/gc/minimark.py b/rpython/rtyper/memory/gc/minimark.py --- a/rpython/rtyper/memory/gc/minimark.py +++ b/rpython/rtyper/memory/gc/minimark.py @@ -2,41 +2,45 @@ Environment variables can be used to fine-tune the following parameters: - PYPY_GC_NURSERY The nursery size. Defaults to '4MB'. Small values - (like 1 or 1KB) are useful for debugging. + PYPY_GC_NURSERY The nursery size. Defaults to '4MB'. Small values + (like 1 or 1KB) are useful for debugging. - PYPY_GC_MAJOR_COLLECT Major collection memory factor. Default is '1.82', - which means trigger a major collection when the - memory consumed equals 1.82 times the memory - really used at the end of the previous major - collection. + PYPY_GC_NURSERY_CLEANUP The interval at which nursery is cleaned up. Must + be smaller than the nursery size and bigger than the + biggest object we can allotate in the nursery. - PYPY_GC_GROWTH Major collection threshold's max growth rate. - Default is '1.4'. Useful to collect more often - than normally on sudden memory growth, e.g. when - there is a temporary peak in memory usage. + PYPY_GC_MAJOR_COLLECT Major collection memory factor. Default is '1.82', + which means trigger a major collection when the + memory consumed equals 1.82 times the memory + really used at the end of the previous major + collection. - PYPY_GC_MAX The max heap size. If coming near this limit, it - will first collect more often, then raise an - RPython MemoryError, and if that is not enough, - crash the program with a fatal error. Try values - like '1.6GB'. + PYPY_GC_GROWTH Major collection threshold's max growth rate. + Default is '1.4'. Useful to collect more often + than normally on sudden memory growth, e.g. when + there is a temporary peak in memory usage. - PYPY_GC_MAX_DELTA The major collection threshold will never be set - to more than PYPY_GC_MAX_DELTA the amount really - used after a collection. Defaults to 1/8th of the - total RAM size (which is constrained to be at most - 2/3/4GB on 32-bit systems). Try values like '200MB'. + PYPY_GC_MAX The max heap size. If coming near this limit, it + will first collect more often, then raise an + RPython MemoryError, and if that is not enough, + crash the program with a fatal error. Try values + like '1.6GB'. - PYPY_GC_MIN Don't collect while the memory size is below this - limit. Useful to avoid spending all the time in - the GC in very small programs. Defaults to 8 - times the nursery. + PYPY_GC_MAX_DELTA The major collection threshold will never be set + to more than PYPY_GC_MAX_DELTA the amount really + used after a collection. Defaults to 1/8th of the + total RAM size (which is constrained to be at most + 2/3/4GB on 32-bit systems). Try values like '200MB'. - PYPY_GC_DEBUG Enable extra checks around collections that are - too slow for normal use. Values are 0 (off), - 1 (on major collections) or 2 (also on minor - collections). + PYPY_GC_MIN Don't collect while the memory size is below this + limit. Useful to avoid spending all the time in + the GC in very small programs. Defaults to 8 + times the nursery. + + PYPY_GC_DEBUG Enable extra checks around collections that are + too slow for normal use. Values are 0 (off), + 1 (on major collections) or 2 (also on minor + collections). """ # XXX Should find a way to bound the major collection threshold by the # XXX total addressable size. Maybe by keeping some minimarkpage arenas @@ -208,11 +212,18 @@ # minimal allocated size of the nursery is 2x the following # number (by default, at least 132KB on 32-bit and 264KB on 64-bit). "large_object": (16384+512)*WORD, + + # This is the chunk that we cleanup in the nursery. The point is + # to avoid having to trash all the caches just to zero the nursery, + # so we trade it by cleaning it bit-by-bit, as we progress through + # nursery. Has to fit at least one large object + "nursery_cleanup": 32768 * WORD, } def __init__(self, config, read_from_env=False, nursery_size=32*WORD, + nursery_cleanup=8*WORD, page_size=16*WORD, arena_size=64*WORD, small_request_threshold=5*WORD, @@ -226,6 +237,7 @@ assert small_request_threshold % WORD == 0 self.read_from_env = read_from_env self.nursery_size = nursery_size + self.nursery_cleanup = nursery_cleanup self.small_request_threshold = small_request_threshold self.major_collection_threshold = major_collection_threshold self.growth_rate_max = growth_rate_max @@ -248,6 +260,7 @@ self.nursery = NULL self.nursery_free = NULL self.nursery_top = NULL + self.nursery_real_top = NULL self.debug_tiny_nursery = -1 self.debug_rotating_nurseries = None self.extra_threshold = 0 @@ -341,6 +354,10 @@ if newsize < minsize: self.debug_tiny_nursery = newsize & ~(WORD-1) newsize = minsize + + nursery_cleanup = env.read_from_env('PYPY_GC_NURSERY_CLEANUP') + if nursery_cleanup >= 0: + self.nursery_cleanup = nursery_cleanup # major_coll = env.read_float_from_env('PYPY_GC_MAJOR_COLLECT') if major_coll > 1.0: @@ -394,7 +411,8 @@ # the current position in the nursery: self.nursery_free = self.nursery # the end of the nursery: - self.nursery_top = self.nursery + self.nursery_size + self.nursery_top = self.nursery + self.nursery_cleanup + self.nursery_real_top = self.nursery + self.nursery_size # initialize the threshold self.min_heap_size = max(self.min_heap_size, self.nursery_size * self.major_collection_threshold) @@ -458,7 +476,8 @@ newnurs = self.debug_rotating_nurseries.pop(0) llarena.arena_protect(newnurs, self._nursery_memory_size(), False) self.nursery = newnurs - self.nursery_top = self.nursery + self.nursery_size + self.nursery_top = self.nursery + self.nursery_cleanup + self.nursery_real_top = self.nursery + self.nursery_size debug_print("switching from nursery", oldnurs, "to nursery", self.nursery, "size", self.nursery_size) @@ -501,7 +520,7 @@ result = self.nursery_free self.nursery_free = result + totalsize if self.nursery_free > self.nursery_top: - result = self.collect_and_reserve(totalsize) + result = self.collect_and_reserve(result, totalsize) # # Build the object. llarena.arena_reserve(result, totalsize) @@ -560,7 +579,7 @@ result = self.nursery_free self.nursery_free = result + totalsize if self.nursery_free > self.nursery_top: - result = self.collect_and_reserve(totalsize) + result = self.collect_and_reserve(result, totalsize) # # Build the object. llarena.arena_reserve(result, totalsize) @@ -579,12 +598,22 @@ if gen > 0: self.major_collection() - def collect_and_reserve(self, totalsize): + def move_nursery_top_and_malloc(self, totalsize): + llarena.arena_reset(self.nursery_top, self.nursery_cleanup, 2) + self.nursery_top += self.nursery_cleanup + + def collect_and_reserve(self, prev_result, totalsize): """To call when nursery_free overflows nursery_top. + First check if the nursery_top is the real top, otherwise we + can just move the top of one cleanup and continue + Do a minor collection, and possibly also a major collection, and finally reserve 'totalsize' bytes at the start of the now-empty nursery. """ + if self.nursery_top < self.nursery_real_top: + self.move_nursery_top_and_malloc(totalsize) + return prev_result self.minor_collection() # if self.get_total_memory_used() > self.next_major_collection_threshold: @@ -756,7 +785,7 @@ if self.next_major_collection_threshold < 0: # cannot trigger a full collection now, but we can ensure # that one will occur very soon - self.nursery_free = self.nursery_top + self.nursery_free = self.nursery_real_top def can_malloc_nonmovable(self): return True @@ -836,7 +865,7 @@ def is_in_nursery(self, addr): ll_assert(llmemory.cast_adr_to_int(addr) & 1 == 0, "odd-valued (i.e. tagged) pointer unexpected here") - return self.nursery <= addr < self.nursery_top + return self.nursery <= addr < self.nursery_real_top def appears_to_be_young(self, addr): # "is a valid addr to a young object?" @@ -856,7 +885,7 @@ if not self.is_valid_gc_object(addr): return False - if self.nursery <= addr < self.nursery_top: + if self.nursery <= addr < self.nursery_real_top: return True # addr is in the nursery # # Else, it may be in the set 'young_rawmalloced_objects' @@ -1263,10 +1292,12 @@ self.free_young_rawmalloced_objects() # # All live nursery objects are out, and the rest dies. Fill - # the whole nursery with zero and reset the current nursery pointer. - llarena.arena_reset(self.nursery, self.nursery_size, 2) + # the nursery up to the cleanup point with zeros + llarena.arena_reset(self.nursery, self.nursery_size, 0) + llarena.arena_reset(self.nursery, self.nursery_cleanup, 2) self.debug_rotate_nursery() self.nursery_free = self.nursery + self.nursery_top = self.nursery + self.nursery_cleanup # debug_print("minor collect, total memory used:", self.get_total_memory_used()) @@ -1826,10 +1857,10 @@ ll_assert(reserved_size <= self.nonlarge_max, "set_extra_threshold: too big!") diff = reserved_size - self.extra_threshold - if diff > 0 and self.nursery_free + diff > self.nursery_top: + if diff > 0 and self.nursery_free + diff > self.nursery_real_top: self.minor_collection() self.nursery_size -= diff - self.nursery_top -= diff + self.nursery_real_top -= diff self.extra_threshold += diff From noreply at buildbot.pypy.org Tue Jan 29 17:21:19 2013 From: noreply at buildbot.pypy.org (bivab) Date: Tue, 29 Jan 2013 17:21:19 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: update tests Message-ID: <20130129162119.4B8871C0041@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: jitframe-on-heap Changeset: r60694:bc556656791e Date: 2013-01-29 17:12 +0100 http://bitbucket.org/pypy/pypy/changeset/bc556656791e/ Log: update tests diff --git a/rpython/jit/backend/arm/test/test_arch.py b/rpython/jit/backend/arm/test/test_arch.py --- a/rpython/jit/backend/arm/test/test_arch.py +++ b/rpython/jit/backend/arm/test/test_arch.py @@ -1,23 +1,23 @@ -from rpython.jit.backend.arm import arch +from rpython.jit.backend.arm import support def test_mod(): - assert arch.arm_int_mod(10, 2) == 0 - assert arch.arm_int_mod(11, 2) == 1 - assert arch.arm_int_mod(11, 3) == 2 + assert support.arm_int_mod(10, 2) == 0 + assert support.arm_int_mod(11, 2) == 1 + assert support.arm_int_mod(11, 3) == 2 def test_mod2(): - assert arch.arm_int_mod(-10, 2) == 0 - assert arch.arm_int_mod(-11, 2) == -1 - assert arch.arm_int_mod(-11, 3) == -2 + assert support.arm_int_mod(-10, 2) == 0 + assert support.arm_int_mod(-11, 2) == -1 + assert support.arm_int_mod(-11, 3) == -2 def test_mod3(): - assert arch.arm_int_mod(10, -2) == 0 - assert arch.arm_int_mod(11, -2) == 1 - assert arch.arm_int_mod(11, -3) == 2 + assert support.arm_int_mod(10, -2) == 0 + assert support.arm_int_mod(11, -2) == 1 + assert support.arm_int_mod(11, -3) == 2 def test_div(): - assert arch.arm_int_div(-7, 2) == -3 - assert arch.arm_int_div(9, 2) == 4 - assert arch.arm_int_div(10, 5) == 2 + assert support.arm_int_div(-7, 2) == -3 + assert support.arm_int_div(9, 2) == 4 + assert support.arm_int_div(10, 5) == 2 diff --git a/rpython/jit/backend/arm/test/test_jump.py b/rpython/jit/backend/arm/test/test_jump.py --- a/rpython/jit/backend/arm/test/test_jump.py +++ b/rpython/jit/backend/arm/test/test_jump.py @@ -1,6 +1,5 @@ import random import py -from rpython.jit.backend.x86.test.test_jump import MockAssembler from rpython.jit.backend.arm.registers import * from rpython.jit.backend.arm.locations import * from rpython.jit.backend.arm.regalloc import ARMFrameManager @@ -9,6 +8,35 @@ frame_pos = ARMFrameManager.frame_pos +class MockAssembler: + def __init__(self): + self.ops = [] + + def regalloc_mov(self, from_loc, to_loc): + self.ops.append(('mov', from_loc, to_loc)) + + def regalloc_push(self, loc): + self.ops.append(('push', loc)) + + def regalloc_pop(self, loc): + self.ops.append(('pop', loc)) + + def got(self, expected): + print '------------------------ comparing ---------------------------' + for op1, op2 in zip(self.ops, expected): + print '%-38s| %-38s' % (op1, op2) + if op1 == op2: + continue + assert len(op1) == len(op2) + for x, y in zip(op1, op2): + if isinstance(x, StackLoc) and isinstance(y, MODRM): + assert x.byte == y.byte + assert x.extradata == y.extradata + else: + assert x == y + assert len(self.ops) == len(expected) + return True + class TestJump(object): def setup_method(self, m): self.assembler = MockAssembler() From noreply at buildbot.pypy.org Tue Jan 29 17:28:44 2013 From: noreply at buildbot.pypy.org (bivab) Date: Tue, 29 Jan 2013 17:28:44 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: pfff Message-ID: <20130129162844.830FE1C0041@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: jitframe-on-heap Changeset: r60695:afb17ae77659 Date: 2013-01-29 17:26 +0100 http://bitbucket.org/pypy/pypy/changeset/afb17ae77659/ Log: pfff diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -42,7 +42,7 @@ assert isinstance(exc, bool) self.faildescr = faildescr self.failargs = failargs - self.fail_locs = fail_locs[1:] + self.fail_locs = fail_locs self.offset = offset self.gcmap = self.compute_gcmap(gcmap, failargs, fail_locs, frame_depth) From noreply at buildbot.pypy.org Tue Jan 29 18:54:22 2013 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 29 Jan 2013 18:54:22 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: fix copy for StringType Message-ID: <20130129175422.503A21C0264@cobra.cs.uni-duesseldorf.de> Author: Matti Picus Branch: missing-ndarray-attributes Changeset: r60696:b0a0486d2dbd Date: 2013-01-29 19:36 +0200 http://bitbucket.org/pypy/pypy/changeset/b0a0486d2dbd/ Log: fix copy for StringType diff --git a/pypy/module/micronumpy/interp_boxes.py b/pypy/module/micronumpy/interp_boxes.py --- a/pypy/module/micronumpy/interp_boxes.py +++ b/pypy/module/micronumpy/interp_boxes.py @@ -289,7 +289,7 @@ return W_StringBox(arr, 0, arr.dtype) def convert_to(self, dtype): - from pypy.modules.micronumpy import types + from pypy.module.micronumpy import types assert isinstance(dtype.itemtype, types.StringType) return self @@ -306,7 +306,7 @@ return W_UnicodeBox(arr, 0, arr.dtype) def convert_to(self, dtype): - from pypy.modules.micronumpy import types + from pypy.module.micronumpy import types assert isinstance(dtype.itemtype, types.UnicodeType) return self diff --git a/pypy/module/micronumpy/test/test_numarray.py b/pypy/module/micronumpy/test/test_numarray.py --- a/pypy/module/micronumpy/test/test_numarray.py +++ b/pypy/module/micronumpy/test/test_numarray.py @@ -348,7 +348,7 @@ a = array(['abc', 'def','xyz'], dtype='S3') b = a.copy() - assert (b == a).all() + assert b[0] == a[0] def test_iterator_init(self): from _numpypy import array From noreply at buildbot.pypy.org Tue Jan 29 22:37:43 2013 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 29 Jan 2013 22:37:43 +0100 (CET) Subject: [pypy-commit] pypy default: trying to fix fron thread/thread_nt split Message-ID: <20130129213743.6BB841C0207@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: Changeset: r60697:d6b59575b135 Date: 2013-01-29 23:37 +0100 http://bitbucket.org/pypy/pypy/changeset/d6b59575b135/ Log: trying to fix fron thread/thread_nt split diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -731,7 +731,11 @@ global_objects.append('%s %s = NULL;' % (typ, name)) global_code = '\n'.join(global_objects) - prologue = ("#include \n" + if sys.platform == "win32": + prologue = ("#include \n" + "#include \n") + else: + prologue = ("#include \n" "#include \n") code = (prologue + struct_declaration_code + @@ -922,8 +926,8 @@ kwds["link_extra"] = ["msvcrt.lib"] elif sys.platform.startswith('linux'): compile_extra.append("-Werror=implicit-function-declaration") + compile_extra.append('-g') export_symbols_eci.append('pypyAPI') - compile_extra.append('-g') else: kwds["includes"] = ['Python.h'] # this is our Python.h diff --git a/rpython/translator/c/src/thread_nt.c b/rpython/translator/c/src/thread_nt.c --- a/rpython/translator/c/src/thread_nt.c +++ b/rpython/translator/c/src/thread_nt.c @@ -13,6 +13,7 @@ /* * Thread support. */ +#include "thread.h" typedef struct RPyOpaque_ThreadLock NRMUTEX, *PNRMUTEX; diff --git a/rpython/translator/c/src/thread_nt.h b/rpython/translator/c/src/thread_nt.h --- a/rpython/translator/c/src/thread_nt.h +++ b/rpython/translator/c/src/thread_nt.h @@ -9,6 +9,7 @@ } NRMUTEX, *PNRMUTEX; /* prototypes */ +long RPyThreadGetIdent(void); long RPyThreadStart(void (*func)(void)); int RPyThreadLockInit(struct RPyOpaque_ThreadLock *lock); void RPyOpaqueDealloc_ThreadLock(struct RPyOpaque_ThreadLock *lock); From noreply at buildbot.pypy.org Tue Jan 29 23:24:58 2013 From: noreply at buildbot.pypy.org (mattip) Date: Tue, 29 Jan 2013 23:24:58 +0100 (CET) Subject: [pypy-commit] pypy default: after review (amaury) Message-ID: <20130129222458.DFA7C1C0041@cobra.cs.uni-duesseldorf.de> Author: Matti Picus Branch: Changeset: r60698:b783959e50bf Date: 2013-01-30 00:24 +0200 http://bitbucket.org/pypy/pypy/changeset/b783959e50bf/ Log: after review (amaury) diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -731,11 +731,7 @@ global_objects.append('%s %s = NULL;' % (typ, name)) global_code = '\n'.join(global_objects) - if sys.platform == "win32": - prologue = ("#include \n" - "#include \n") - else: - prologue = ("#include \n" + prologue = ("#include \n" "#include \n") code = (prologue + struct_declaration_code + diff --git a/rpython/translator/c/src/thread_nt.c b/rpython/translator/c/src/thread_nt.c --- a/rpython/translator/c/src/thread_nt.c +++ b/rpython/translator/c/src/thread_nt.c @@ -13,7 +13,7 @@ /* * Thread support. */ -#include "thread.h" +/* In rpython, this file is pulled in by thread.c */ typedef struct RPyOpaque_ThreadLock NRMUTEX, *PNRMUTEX; From noreply at buildbot.pypy.org Wed Jan 30 00:01:59 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 30 Jan 2013 00:01:59 +0100 (CET) Subject: [pypy-commit] pypy py3k: merge default Message-ID: <20130129230159.C4BD31C0264@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: py3k Changeset: r60699:4f314755eb52 Date: 2013-01-29 18:01 -0500 http://bitbucket.org/pypy/pypy/changeset/4f314755eb52/ Log: merge default diff --git a/lib-python/2.7/inspect.py b/lib-python/2.7/inspect.py --- a/lib-python/2.7/inspect.py +++ b/lib-python/2.7/inspect.py @@ -960,7 +960,7 @@ raise TypeError('%s() takes exactly 0 arguments ' '(%d given)' % (f_name, num_total)) else: - raise TypeError('%s() takes no argument (%d given)' % + raise TypeError('%s() takes no arguments (%d given)' % (f_name, num_total)) for arg in args: if isinstance(arg, str) and arg in named: diff --git a/lib-python/2.7/site.py b/lib-python/2.7/site.py --- a/lib-python/2.7/site.py +++ b/lib-python/2.7/site.py @@ -75,6 +75,7 @@ USER_SITE = None USER_BASE = None + def makepath(*paths): dir = os.path.join(*paths) try: diff --git a/lib-python/2.7/test/test_itertools.py b/lib-python/2.7/test/test_itertools.py --- a/lib-python/2.7/test/test_itertools.py +++ b/lib-python/2.7/test/test_itertools.py @@ -533,11 +533,11 @@ self.assertEqual(list(izip()), zip()) self.assertRaises(TypeError, izip, 3) self.assertRaises(TypeError, izip, range(3), 3) - self.assertEqual([tuple(list(pair)) for pair in izip('abc', 'def')], zip('abc', 'def')) self.assertEqual([pair for pair in izip('abc', 'def')], zip('abc', 'def')) + @test_support.impl_detail("tuple reuse is specific to CPython") def test_izip_tuple_reuse(self): ids = map(id, izip('abc', 'def')) @@ -588,6 +588,7 @@ zip('abc', 'def')) self.assertEqual([pair for pair in izip_longest('abc', 'def')], zip('abc', 'def')) + @test_support.impl_detail("tuple reuse is specific to CPython") def test_izip_longest_tuple_reuse(self): ids = map(id, izip_longest('abc', 'def')) diff --git a/lib-python/2.7/test/test_support.py b/lib-python/2.7/test/test_support.py --- a/lib-python/2.7/test/test_support.py +++ b/lib-python/2.7/test/test_support.py @@ -1085,7 +1085,6 @@ else: runner = BasicTestRunner() - result = runner.run(suite) if not result.wasSuccessful(): if len(result.errors) == 1 and not result.failures: diff --git a/lib-python/conftest.py b/lib-python/conftest.py --- a/lib-python/conftest.py +++ b/lib-python/conftest.py @@ -94,17 +94,10 @@ m.test_main() ''' % locals()) -if sys.platform == 'win32': - skip_win32 = "Not supported on Windows" - only_win32 = False -else: - skip_win32 = False - only_win32 = "Only on Windows" - testmap = [ RegrTest('test___all__.py', core=True), RegrTest('test___future__.py', core=True), - RegrTest('test__locale.py', usemodules='_locale', skip=skip_win32), + RegrTest('test__locale.py', usemodules='_locale'), RegrTest('test_abc.py'), RegrTest('test_abstract_numbers.py'), RegrTest('test_aifc.py'), @@ -114,7 +107,7 @@ RegrTest('test_asynchat.py', usemodules='select fcntl'), RegrTest('test_asyncore.py', usemodules='select fcntl'), RegrTest('test_atexit.py', core=True), - RegrTest('test_audioop.py', skip=True), + RegrTest('test_audioop.py', skip="unsupported extension module"), RegrTest('test_augassign.py', core=True), RegrTest('test_base64.py', usemodules='struct'), RegrTest('test_bigaddrspace.py'), @@ -167,10 +160,10 @@ RegrTest('test_copy.py', core=True), RegrTest('test_copyreg.py', core=True), RegrTest('test_cprofile.py'), - RegrTest('test_crypt.py', usemodules='crypt', skip=skip_win32), + RegrTest('test_crypt.py', usemodules='crypt'), RegrTest('test_csv.py', usemodules='_csv'), RegrTest('test_ctypes.py', usemodules="_rawffi thread"), - RegrTest('test_curses.py', skip="unsupported extension module"), + RegrTest('test_curses.py'), RegrTest('test_datetime.py'), RegrTest('test_dbm.py'), RegrTest('test_dbm_dumb.py'), @@ -202,7 +195,7 @@ RegrTest('test_exception_variations.py'), RegrTest('test_exceptions.py', core=True), RegrTest('test_extcall.py', core=True), - RegrTest('test_fcntl.py', usemodules='fcntl', skip=skip_win32), + RegrTest('test_fcntl.py', usemodules='fcntl'), RegrTest('test_file.py', usemodules="posix", core=True), RegrTest('test_filecmp.py', core=True), RegrTest('test_fileinput.py', core=True), @@ -228,13 +221,13 @@ RegrTest('test_generators.py', core=True, usemodules='thread _weakref'), RegrTest('test_genericpath.py'), RegrTest('test_genexps.py', core=True, usemodules='_weakref'), - RegrTest('test_getargs2.py', skip="unsupported extension module"), + RegrTest('test_getargs2.py', usemodules='binascii', skip=True), RegrTest('test_getopt.py', core=True), RegrTest('test_gettext.py'), RegrTest('test_glob.py', core=True), RegrTest('test_global.py', core=True), RegrTest('test_grammar.py', core=True), - RegrTest('test_grp.py', skip=skip_win32), + RegrTest('test_grp.py'), RegrTest('test_gzip.py', usemodules='zlib'), RegrTest('test_hash.py', core=True), RegrTest('test_hashlib.py', core=True), @@ -285,13 +278,12 @@ RegrTest('test_mmap.py', usemodules="mmap"), RegrTest('test_module.py', core=True), RegrTest('test_modulefinder.py'), - RegrTest('test_msilib.py', skip=only_win32), + RegrTest('test_msilib.py'), RegrTest('test_multibytecodec.py', usemodules='_multibytecodec'), - RegrTest('test_multibytecodec_support.py', skip="not a test"), - RegrTest('test_multiprocessing.py', skip="FIXME leaves subprocesses"), + RegrTest('test_multiprocessing.py', skip=True), RegrTest('test_mutants.py', core="possibly"), RegrTest('test_netrc.py'), - RegrTest('test_nis.py', skip="unsupported extension module"), + RegrTest('test_nis.py'), RegrTest('test_nntplib.py'), RegrTest('test_normalization.py'), RegrTest('test_ntpath.py'), @@ -301,7 +293,7 @@ RegrTest('test_operator.py', core=True), RegrTest('test_optparse.py'), RegrTest('test_os.py', core=True), - RegrTest('test_ossaudiodev.py', skip="unsupported extension module"), + RegrTest('test_ossaudiodev.py'), RegrTest('test_osx_env.py'), RegrTest('test_parser.py', skip="slowly deprecating compiler"), RegrTest('test_pdb.py'), @@ -320,8 +312,8 @@ RegrTest('test_pkgimport.py', core=True), RegrTest('test_pkgutil.py'), RegrTest('test_platform.py'), - RegrTest('test_plistlib.py', skip="unsupported module"), - RegrTest('test_poll.py', skip=skip_win32), + RegrTest('test_plistlib.py'), + RegrTest('test_poll.py'), RegrTest('test_popen.py'), RegrTest('test_poplib.py'), RegrTest('test_posix.py', usemodules="_rawffi"), @@ -332,8 +324,8 @@ RegrTest('test_profile.py'), RegrTest('test_property.py', core=True), RegrTest('test_pstats.py'), - RegrTest('test_pty.py', skip="unsupported extension module"), - RegrTest('test_pwd.py', usemodules="pwd", skip=skip_win32), + RegrTest('test_pty.py', usemodules='fcntl termios select'), + RegrTest('test_pwd.py', usemodules="pwd"), RegrTest('test_py_compile.py'), RegrTest('test_pyclbr.py'), RegrTest('test_pydoc.py'), @@ -346,7 +338,7 @@ RegrTest('test_re.py', core=True), RegrTest('test_readline.py'), RegrTest('test_reprlib.py', core=True), - RegrTest('test_resource.py', skip=skip_win32), + RegrTest('test_resource.py'), RegrTest('test_richcmp.py', core=True), RegrTest('test_rlcompleter.py'), RegrTest('test_robotparser.py'), @@ -383,7 +375,7 @@ RegrTest('test_structmembers.py', skip="CPython specific"), RegrTest('test_structseq.py'), RegrTest('test_subprocess.py', usemodules='signal'), - RegrTest('test_sunau.py', skip=True), + RegrTest('test_sunau.py', skip="unsupported extension module"), RegrTest('test_sundry.py'), RegrTest('test_super.py', core=True), RegrTest('test_symtable.py', skip="implementation detail"), @@ -394,7 +386,7 @@ RegrTest('test_sysconfig.py'), RegrTest('test_syslog.py'), RegrTest('test_tarfile.py'), - RegrTest('test_tcl.py', skip="unsupported extension module"), + RegrTest('test_tcl.py'), RegrTest('test_telnetlib.py'), RegrTest('test_tempfile.py'), RegrTest('test_textwrap.py'), @@ -441,11 +433,11 @@ RegrTest('test_wait3.py', usemodules="thread"), RegrTest('test_wait4.py', usemodules="thread"), RegrTest('test_warnings.py', core=True), - RegrTest('test_wave.py', skip="unsupported extension module"), + RegrTest('test_wave.py'), RegrTest('test_weakref.py', core=True, usemodules='_weakref'), RegrTest('test_weakset.py'), - RegrTest('test_winreg.py', skip=only_win32), - RegrTest('test_winsound.py', skip="unsupported extension module"), + RegrTest('test_winreg.py'), + RegrTest('test_winsound.py'), RegrTest('test_with.py'), RegrTest('test_wsgiref.py'), RegrTest('test_xdrlib.py'), @@ -463,7 +455,9 @@ def check_testmap_complete(): listed_names = dict.fromkeys([regrtest.basename for regrtest in testmap]) assert len(listed_names) == len(testmap) - listed_names['test_support.py'] = True # ignore this + # names to ignore + listed_names['test_support.py'] = True + listed_names['test_multibytecodec_support.py'] = True missing = [] for path in testdir.listdir(fil='test_*.py'): name = path.basename diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -196,7 +196,8 @@ try: self.close() except SocketError, e: - raise converted_error(space, e) + # cpython doesn't return any errors on close + pass def connect_w(self, space, w_addr): """connect(address) @@ -456,7 +457,7 @@ w_addr = space.w_None return space.newtuple([space.wrap(readlgt), w_addr]) except SocketError, e: - raise converted_error(space, e) + raise converted_error(space, e) @unwrap_spec(cmd=int) def ioctl_w(self, space, cmd, w_option): diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -346,13 +346,15 @@ assert isinstance(s.fileno(), int) def test_socket_close(self): - import _socket + import _socket, os s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM, 0) fileno = s.fileno() assert s.fileno() >= 0 s.close() assert s.fileno() < 0 s.close() + if os.name != 'nt': + raises(OSError, os.close, fileno) def test_socket_close_error(self): import _socket, os @@ -360,7 +362,7 @@ skip("Windows sockets are not files") s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM, 0) os.close(s.fileno()) - raises(_socket.error, s.close) + s.close() def test_socket_connect(self): import _socket, os diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -920,8 +920,8 @@ kwds["link_extra"] = ["msvcrt.lib"] elif sys.platform.startswith('linux'): compile_extra.append("-Werror=implicit-function-declaration") + compile_extra.append('-g') export_symbols_eci.append('pypyAPI') - compile_extra.append('-g') else: kwds["includes"] = ['Python.h'] # this is our Python.h diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -239,6 +239,8 @@ def optimize_GUARD_VALUE(self, op): value = self.getvalue(op.getarg(0)) + if value.is_virtual(): + raise InvalidLoop('A promote of a virtual (a recently allocated object) never makes sense!') if value.last_guard: # there already has been a guard_nonnull or guard_class or # guard_nonnull_class on this value, which is rather silly. diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -2623,7 +2623,7 @@ jump(p2) """ self.raises(InvalidLoop, self.optimize_loop, - ops, ops) + ops, "crash!") def test_invalid_loop_2(self): ops = """ @@ -2635,7 +2635,7 @@ jump(p2) """ self.raises(InvalidLoop, self.optimize_loop, - ops, ops) + ops, "crash!") def test_invalid_loop_3(self): ops = """ @@ -2648,8 +2648,17 @@ setfield_gc(p3, p4, descr=nextdescr) jump(p3) """ - self.raises(InvalidLoop, self.optimize_loop, ops, ops) - + self.raises(InvalidLoop, self.optimize_loop, ops, "crash!") + + def test_invalid_loop_guard_value_of_virtual(self): + ops = """ + [p1] + p2 = new_with_vtable(ConstClass(node_vtable)) + guard_value(p2, ConstPtr(myptr)) [] + jump(p2) + """ + self.raises(InvalidLoop, self.optimize_loop, + ops, "crash!") def test_merge_guard_class_guard_value(self): ops = """ diff --git a/rpython/translator/c/src/thread_nt.c b/rpython/translator/c/src/thread_nt.c --- a/rpython/translator/c/src/thread_nt.c +++ b/rpython/translator/c/src/thread_nt.c @@ -13,6 +13,7 @@ /* * Thread support. */ +/* In rpython, this file is pulled in by thread.c */ typedef struct RPyOpaque_ThreadLock NRMUTEX, *PNRMUTEX; diff --git a/rpython/translator/c/src/thread_nt.h b/rpython/translator/c/src/thread_nt.h --- a/rpython/translator/c/src/thread_nt.h +++ b/rpython/translator/c/src/thread_nt.h @@ -9,6 +9,7 @@ } NRMUTEX, *PNRMUTEX; /* prototypes */ +long RPyThreadGetIdent(void); long RPyThreadStart(void (*func)(void)); int RPyThreadLockInit(struct RPyOpaque_ThreadLock *lock); void RPyOpaqueDealloc_ThreadLock(struct RPyOpaque_ThreadLock *lock); From noreply at buildbot.pypy.org Wed Jan 30 01:55:14 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 30 Jan 2013 01:55:14 +0100 (CET) Subject: [pypy-commit] pypy default: unskip test_capi.py while adding individual skips for non-functioning tests Message-ID: <20130130005514.4D7401C0041@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60700:39876dee7626 Date: 2013-01-29 19:54 -0500 http://bitbucket.org/pypy/pypy/changeset/39876dee7626/ Log: unskip test_capi.py while adding individual skips for non- functioning tests diff --git a/lib-python/2.7/test/test_capi.py b/lib-python/2.7/test/test_capi.py --- a/lib-python/2.7/test/test_capi.py +++ b/lib-python/2.7/test/test_capi.py @@ -13,6 +13,20 @@ threading = None import _testcapi +skips = [] +if test_support.check_impl_detail(pypy=True): + skips += [ + 'test_broken_memoryview', + 'test_capsule', + 'test_lazy_hash_inheritance', + 'test_long_api', + 'test_longlong_api', + 'test_null_strings', + 'test_widechar', + 'TestThreadState', + 'TestPendingCalls', + ] + @unittest.skipUnless(threading, 'Threading required for this test.') class TestPendingCalls(unittest.TestCase): @@ -99,7 +113,7 @@ def test_main(): for name in dir(_testcapi): - if name.startswith('test_'): + if name.startswith('test_') and name not in skips: test = getattr(_testcapi, name) if test_support.verbose: print "internal", name @@ -126,7 +140,7 @@ raise test_support.TestFailed, \ "Couldn't find main thread correctly in the list" - if threading: + if threading and 'TestThreadState' not in skips: import thread import time TestThreadState() @@ -134,7 +148,8 @@ t.start() t.join() - test_support.run_unittest(TestPendingCalls) + if 'TestPendingCalls' not in skips: + test_support.run_unittest(TestPendingCalls) if __name__ == "__main__": test_main() diff --git a/lib-python/conftest.py b/lib-python/conftest.py --- a/lib-python/conftest.py +++ b/lib-python/conftest.py @@ -132,7 +132,7 @@ RegrTest('test_bz2.py', usemodules='bz2'), RegrTest('test_calendar.py'), RegrTest('test_call.py', core=True), - RegrTest('test_capi.py', skip="not applicable"), + RegrTest('test_capi.py'), RegrTest('test_cd.py'), RegrTest('test_cfgparser.py'), RegrTest('test_cgi.py'), From noreply at buildbot.pypy.org Wed Jan 30 05:15:33 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 30 Jan 2013 05:15:33 +0100 (CET) Subject: [pypy-commit] pypy default: silence warnings from timeout_killer Message-ID: <20130130041533.A08D11C0207@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60701:80aa79e32429 Date: 2013-01-29 21:35 -0500 http://bitbucket.org/pypy/pypy/changeset/80aa79e32429/ Log: silence warnings from timeout_killer diff --git a/pypy/module/thread/test/support.py b/pypy/module/thread/test/support.py --- a/pypy/module/thread/test/support.py +++ b/pypy/module/thread/test/support.py @@ -2,6 +2,7 @@ import time import thread import os +import errno from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.module.thread import gil @@ -28,7 +29,12 @@ def kill(): for x in range(delay * 10): time.sleep(0.1) - os.kill(pid, 0) + try: + os.kill(pid, 0) + except OSError, e: + if e.errno == errno.ESRCH: # no such process + return + raise os.kill(pid, 9) print "process %s killed!" % (pid,) thread.start_new_thread(kill, ()) From noreply at buildbot.pypy.org Wed Jan 30 05:15:35 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 30 Jan 2013 05:15:35 +0100 (CET) Subject: [pypy-commit] pypy default: clean up intermittently failing test_fork Message-ID: <20130130041535.3B91A1C0207@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60702:3f31a9cbf0bf Date: 2013-01-29 23:13 -0500 http://bitbucket.org/pypy/pypy/changeset/3f31a9cbf0bf/ Log: clean up intermittently failing test_fork diff --git a/pypy/module/thread/test/test_fork.py b/pypy/module/thread/test/test_fork.py --- a/pypy/module/thread/test/test_fork.py +++ b/pypy/module/thread/test/test_fork.py @@ -1,7 +1,7 @@ from pypy.module.thread.test.support import GenericTestThread class AppTestFork(GenericTestThread): - def test_fork(self): + def test_fork_with_thread(self): # XXX This test depends on a multicore machine, as busy_thread must # aquire the GIL the instant that the main thread releases it. # It will incorrectly pass if the GIL is not grabbed in time. @@ -12,45 +12,48 @@ if not hasattr(os, 'fork'): skip("No fork on this platform") - run = True - done = [] def busy_thread(): while run: time.sleep(0) done.append(None) - try: - thread.start_new(busy_thread, ()) + for i in range(1): + run = True + done = [] + try: + thread.start_new(busy_thread, ()) + print 'sleep' - pid = os.fork() - - if pid == 0: - os._exit(0) - - else: - time.sleep(1) - spid, status = os.waitpid(pid, os.WNOHANG) - assert spid == pid - finally: - run = False - self.waitfor(lambda: done) + pid = os.fork() + if pid == 0: + os._exit(0) + else: + self.timeout_killer(pid, 5) + exitcode = os.waitpid(pid, 0)[1] + assert exitcode == 0 # if 9, process was killed by timer! + finally: + run = False + self.waitfor(lambda: done) + assert done def test_forked_can_thread(self): "Checks that a forked interpreter can start a thread" - import os, thread, time + import thread + import os if not hasattr(os, 'fork'): skip("No fork on this platform") - # pre-allocate some locks - thread.start_new_thread(lambda: None, ()) + for i in range(10): + # pre-allocate some locks + thread.start_new_thread(lambda: None, ()) + print 'sleep' - pid = os.fork() - if pid == 0: - print 'in child' - thread.start_new_thread(lambda: None, ()) - os._exit(0) - else: - self.timeout_killer(pid, 5) - exitcode = os.waitpid(pid, 0)[1] - assert exitcode == 0 # if 9, process was killed by timer! + pid = os.fork() + if pid == 0: + thread.start_new_thread(lambda: None, ()) + os._exit(0) + else: + self.timeout_killer(pid, 5) + exitcode = os.waitpid(pid, 0)[1] + assert exitcode == 0 # if 9, process was killed by timer! From noreply at buildbot.pypy.org Wed Jan 30 05:25:14 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Wed, 30 Jan 2013 05:25:14 +0100 (CET) Subject: [pypy-commit] pypy default: some cleanups and style changes Message-ID: <20130130042514.14FF71C0264@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: Changeset: r60703:248f9bb5ec68 Date: 2013-01-29 20:22 -0800 http://bitbucket.org/pypy/pypy/changeset/248f9bb5ec68/ Log: some cleanups and style changes diff --git a/rpython/jit/metainterp/virtualizable.py b/rpython/jit/metainterp/virtualizable.py --- a/rpython/jit/metainterp/virtualizable.py +++ b/rpython/jit/metainterp/virtualizable.py @@ -1,14 +1,12 @@ +from rpython.jit.metainterp import history +from rpython.jit.metainterp.typesystem import deref, fieldType, arrayItem +from rpython.jit.metainterp.warmstate import wrap, unwrap +from rpython.rlib.unroll import unrolling_iterable +from rpython.rtyper import rvirtualizable2 from rpython.rtyper.lltypesystem import lltype, llmemory from rpython.rtyper.ootypesystem import ootype -from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance from rpython.rtyper.rclass import IR_IMMUTABLE_ARRAY, IR_IMMUTABLE -from rpython.rtyper import rvirtualizable2 -from rpython.rlib.unroll import unrolling_iterable -from rpython.rlib.nonconst import NonConstant -from rpython.jit.metainterp.typesystem import deref, fieldType, arrayItem -from rpython.jit.metainterp import history -from rpython.jit.metainterp.warmstate import wrap, unwrap -from rpython.rlib.objectmodel import specialize + class VirtualizableInfo(object): TOKEN_NONE = 0 # must be 0 -- see also x86.call_assembler @@ -76,7 +74,7 @@ getlength = cpu.ts.getlength getarrayitem = cpu.ts.getarrayitem setarrayitem = cpu.ts.setarrayitem - # + def read_boxes(cpu, virtualizable): assert lltype.typeOf(virtualizable) == llmemory.GCREF virtualizable = cast_gcref_to_vtype(virtualizable) @@ -89,7 +87,7 @@ for i in range(getlength(lst)): boxes.append(wrap(cpu, getarrayitem(lst, i))) return boxes - # + def write_boxes(virtualizable, boxes): virtualizable = cast_gcref_to_vtype(virtualizable) i = 0 @@ -104,7 +102,7 @@ setarrayitem(lst, j, x) i = i + 1 assert len(boxes) == i + 1 - # + def write_from_resume_data_partial(virtualizable, reader, numb): virtualizable = cast_gcref_to_vtype(virtualizable) # Load values from the reader (see resume.py) described by @@ -117,7 +115,7 @@ assert i >= 0 for ARRAYITEMTYPE, fieldname in unroll_array_fields_rev: lst = getattr(virtualizable, fieldname) - for j in range(getlength(lst)-1, -1, -1): + for j in range(getlength(lst) - 1, -1, -1): i -= 1 assert i >= 0 x = reader.load_value_of_type(ARRAYITEMTYPE, numb.nums[i]) @@ -128,7 +126,7 @@ x = reader.load_value_of_type(FIELDTYPE, numb.nums[i]) setattr(virtualizable, fieldname, x) return i - # + def load_list_of_boxes(virtualizable, reader, numb): virtualizable = cast_gcref_to_vtype(virtualizable) # Uses 'virtualizable' only to know the length of the arrays; @@ -140,10 +138,10 @@ boxes = [reader.decode_box_of_type(self.VTYPEPTR, numb.nums[i])] for ARRAYITEMTYPE, fieldname in unroll_array_fields_rev: lst = getattr(virtualizable, fieldname) - for j in range(getlength(lst)-1, -1, -1): + for j in range(getlength(lst) - 1, -1, -1): i -= 1 assert i >= 0 - box = reader.decode_box_of_type(ARRAYITEMTYPE,numb.nums[i]) + box = reader.decode_box_of_type(ARRAYITEMTYPE, numb.nums[i]) boxes.append(box) for FIELDTYPE, fieldname in unroll_static_fields_rev: i -= 1 @@ -152,7 +150,7 @@ boxes.append(box) boxes.reverse() return boxes - # + def check_boxes(virtualizable, boxes): virtualizable = cast_gcref_to_vtype(virtualizable) # for debugging @@ -168,7 +166,7 @@ assert getarrayitem(lst, j) == x i = i + 1 assert len(boxes) == i + 1 - # + def get_index_in_array(virtualizable, arrayindex, index): virtualizable = cast_gcref_to_vtype(virtualizable) index += self.num_static_extra_boxes @@ -180,7 +178,7 @@ index += getlength(lst) j = j + 1 assert False, "invalid arrayindex" - # + def get_array_length(virtualizable, arrayindex): virtualizable = cast_gcref_to_vtype(virtualizable) j = 0 @@ -188,16 +186,16 @@ if arrayindex == j: lst = getattr(virtualizable, fieldname) return getlength(lst) - j = j + 1 + j += 1 assert False, "invalid arrayindex" - # + unroll_static_fields = unrolling_iterable(zip(FIELDTYPES, static_fields)) unroll_array_fields = unrolling_iterable(zip(ARRAYITEMTYPES, array_fields)) unroll_static_fields_rev = unrolling_iterable( reversed(list(unroll_static_fields))) - unroll_array_fields_rev = unrolling_iterable( + unroll_array_fields_rev = unrolling_iterable( reversed(list(unroll_array_fields))) self.read_boxes = read_boxes self.write_boxes = write_boxes @@ -291,7 +289,7 @@ def unwrap_virtualizable_box(self, virtualizable_box): return virtualizable_box.getref(llmemory.GCREF) - + def is_vtypeptr(self, TYPE): return rvirtualizable2.match_virtualizable_type(TYPE, self.VTYPEPTR) From noreply at buildbot.pypy.org Wed Jan 30 05:25:15 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Wed, 30 Jan 2013 05:25:15 +0100 (CET) Subject: [pypy-commit] pypy default: merged upstream Message-ID: <20130130042515.49BDD1C0264@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: Changeset: r60704:fe4735de4d71 Date: 2013-01-29 20:25 -0800 http://bitbucket.org/pypy/pypy/changeset/fe4735de4d71/ Log: merged upstream diff --git a/pypy/module/thread/test/support.py b/pypy/module/thread/test/support.py --- a/pypy/module/thread/test/support.py +++ b/pypy/module/thread/test/support.py @@ -2,6 +2,7 @@ import time import thread import os +import errno from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.module.thread import gil @@ -28,7 +29,12 @@ def kill(): for x in range(delay * 10): time.sleep(0.1) - os.kill(pid, 0) + try: + os.kill(pid, 0) + except OSError, e: + if e.errno == errno.ESRCH: # no such process + return + raise os.kill(pid, 9) print "process %s killed!" % (pid,) thread.start_new_thread(kill, ()) diff --git a/pypy/module/thread/test/test_fork.py b/pypy/module/thread/test/test_fork.py --- a/pypy/module/thread/test/test_fork.py +++ b/pypy/module/thread/test/test_fork.py @@ -1,7 +1,7 @@ from pypy.module.thread.test.support import GenericTestThread class AppTestFork(GenericTestThread): - def test_fork(self): + def test_fork_with_thread(self): # XXX This test depends on a multicore machine, as busy_thread must # aquire the GIL the instant that the main thread releases it. # It will incorrectly pass if the GIL is not grabbed in time. @@ -12,45 +12,48 @@ if not hasattr(os, 'fork'): skip("No fork on this platform") - run = True - done = [] def busy_thread(): while run: time.sleep(0) done.append(None) - try: - thread.start_new(busy_thread, ()) + for i in range(1): + run = True + done = [] + try: + thread.start_new(busy_thread, ()) + print 'sleep' - pid = os.fork() - - if pid == 0: - os._exit(0) - - else: - time.sleep(1) - spid, status = os.waitpid(pid, os.WNOHANG) - assert spid == pid - finally: - run = False - self.waitfor(lambda: done) + pid = os.fork() + if pid == 0: + os._exit(0) + else: + self.timeout_killer(pid, 5) + exitcode = os.waitpid(pid, 0)[1] + assert exitcode == 0 # if 9, process was killed by timer! + finally: + run = False + self.waitfor(lambda: done) + assert done def test_forked_can_thread(self): "Checks that a forked interpreter can start a thread" - import os, thread, time + import thread + import os if not hasattr(os, 'fork'): skip("No fork on this platform") - # pre-allocate some locks - thread.start_new_thread(lambda: None, ()) + for i in range(10): + # pre-allocate some locks + thread.start_new_thread(lambda: None, ()) + print 'sleep' - pid = os.fork() - if pid == 0: - print 'in child' - thread.start_new_thread(lambda: None, ()) - os._exit(0) - else: - self.timeout_killer(pid, 5) - exitcode = os.waitpid(pid, 0)[1] - assert exitcode == 0 # if 9, process was killed by timer! + pid = os.fork() + if pid == 0: + thread.start_new_thread(lambda: None, ()) + os._exit(0) + else: + self.timeout_killer(pid, 5) + exitcode = os.waitpid(pid, 0)[1] + assert exitcode == 0 # if 9, process was killed by timer! From noreply at buildbot.pypy.org Wed Jan 30 06:30:37 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Wed, 30 Jan 2013 06:30:37 +0100 (CET) Subject: [pypy-commit] pypy default: remove an import * Message-ID: <20130130053037.940A61C0207@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: Changeset: r60705:ed9f229da85e Date: 2013-01-29 21:26 -0800 http://bitbucket.org/pypy/pypy/changeset/ed9f229da85e/ Log: remove an import * diff --git a/rpython/flowspace/framestate.py b/rpython/flowspace/framestate.py --- a/rpython/flowspace/framestate.py +++ b/rpython/flowspace/framestate.py @@ -1,5 +1,6 @@ +from rpython.flowspace.model import Variable, Constant from rpython.rlib.unroll import SpecTag -from rpython.flowspace.model import * + class FrameState: def __init__(self, mergeable, blocklist, next_instr): @@ -86,6 +87,7 @@ raise TypeError('union of %r and %r' % (w1.__class__.__name__, w2.__class__.__name__)) + # ____________________________________________________________ # # We have to flatten out the state of the frame into a list of @@ -102,6 +104,7 @@ PICKLE_TAGS = {} UNPICKLE_TAGS = {} + def recursively_flatten(space, lst): from rpython.flowspace.flowcontext import SuspendedUnroller i = 0 @@ -117,14 +120,15 @@ except: tag = PICKLE_TAGS[key] = Constant(PickleTag()) UNPICKLE_TAGS[tag] = key - lst[i:i+1] = [tag] + vars + lst[i:i + 1] = [tag] + vars + def recursively_unflatten(space, lst): - for i in xrange(len(lst)-1, -1, -1): + for i in xrange(len(lst) - 1, -1, -1): item = lst[i] if item in UNPICKLE_TAGS: unrollerclass, argcount = UNPICKLE_TAGS[item] - arguments = lst[i+1: i+1+argcount] - del lst[i+1: i+1+argcount] + arguments = lst[i + 1:i + 1 + argcount] + del lst[i + 1:i + 1 + argcount] unroller = unrollerclass.state_pack_variables(space, *arguments) lst[i] = unroller From noreply at buildbot.pypy.org Wed Jan 30 06:30:38 2013 From: noreply at buildbot.pypy.org (alex_gaynor) Date: Wed, 30 Jan 2013 06:30:38 +0100 (CET) Subject: [pypy-commit] pypy default: Remove trailing whitespace and an import * Message-ID: <20130130053038.EA07D1C0207@cobra.cs.uni-duesseldorf.de> Author: Alex Gaynor Branch: Changeset: r60706:814ba5730127 Date: 2013-01-29 21:29 -0800 http://bitbucket.org/pypy/pypy/changeset/814ba5730127/ Log: Remove trailing whitespace and an import * diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py --- a/rpython/jit/metainterp/optimizeopt/unroll.py +++ b/rpython/jit/metainterp/optimizeopt/unroll.py @@ -1,16 +1,16 @@ -from rpython.jit.codewriter.effectinfo import EffectInfo +import sys + +from rpython.jit.metainterp.compile import ResumeGuardDescr +from rpython.jit.metainterp.history import TargetToken, JitCellToken, Const +from rpython.jit.metainterp.inliner import Inliner +from rpython.jit.metainterp.optimize import InvalidLoop +from rpython.jit.metainterp.optimizeopt.generalize import KillHugeIntBounds +from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer, Optimization from rpython.jit.metainterp.optimizeopt.virtualstate import VirtualStateAdder, ShortBoxes, BadVirtualState -from rpython.jit.metainterp.compile import ResumeGuardDescr -from rpython.jit.metainterp.history import TreeLoop, TargetToken, JitCellToken -from rpython.jit.metainterp.jitexc import JitException -from rpython.jit.metainterp.optimize import InvalidLoop -from rpython.rlib.debug import debug_print, debug_start, debug_stop -from rpython.jit.metainterp.optimizeopt.optimizer import * -from rpython.jit.metainterp.optimizeopt.generalize import KillHugeIntBounds -from rpython.jit.metainterp.inliner import Inliner from rpython.jit.metainterp.resoperation import rop, ResOperation from rpython.jit.metainterp.resume import Snapshot -import sys, os +from rpython.rlib.debug import debug_print, debug_start, debug_stop + # FIXME: Introduce some VirtualOptimizer super class instead @@ -19,6 +19,7 @@ opt.inline_short_preamble = inline_short_preamble opt.propagate_all_forward() + class UnrollableOptimizer(Optimizer): def setup(self): self.importable_values = {} @@ -51,7 +52,7 @@ distinction anymore)""" inline_short_preamble = True - + def __init__(self, metainterp_sd, loop, optimizations): self.optimizer = UnrollableOptimizer(metainterp_sd, loop, optimizations) self.boxes_created_this_iteration = None @@ -59,14 +60,14 @@ def fix_snapshot(self, jump_args, snapshot): if snapshot is None: return None - snapshot_args = snapshot.boxes + snapshot_args = snapshot.boxes new_snapshot_args = [] for a in snapshot_args: a = self.getvalue(a).get_key_box() new_snapshot_args.append(a) prev = self.fix_snapshot(jump_args, snapshot.prev) return Snapshot(prev, new_snapshot_args) - + def propagate_all_forward(self): loop = self.optimizer.loop self.optimizer.clear_newoperations() @@ -78,7 +79,7 @@ # will clear heap caches self.optimizer.send_extra_operation(start_label) else: - start_label = None + start_label = None jumpop = loop.operations[-1] if jumpop.getopnum() == rop.JUMP or jumpop.getopnum() == rop.LABEL: @@ -96,7 +97,6 @@ assert isinstance(cell_token, JitCellToken) stop_label = ResOperation(rop.LABEL, jumpop.getarglist(), None, TargetToken(cell_token)) - if jumpop.getopnum() == rop.JUMP: if self.jump_to_already_compiled_trace(jumpop): # Found a compiled trace to jump to @@ -130,7 +130,7 @@ return # Found nothing to jump to, emit a label instead - + if self.short: # Construct our short preamble assert start_label @@ -146,7 +146,7 @@ def jump_to_start_label(self, start_label, stop_label): if not start_label or not stop_label: return False - + stop_target = stop_label.getdescr() start_target = start_label.getdescr() assert isinstance(stop_target, TargetToken) @@ -161,7 +161,6 @@ #virtual_state = modifier.get_virtual_state(args) #if self.initial_virtual_state.generalization_of(virtual_state): # return True - def export_state(self, targetop): original_jump_args = targetop.getarglist() @@ -174,12 +173,11 @@ modifier = VirtualStateAdder(self.optimizer) virtual_state = modifier.get_virtual_state(jump_args) - + values = [self.getvalue(arg) for arg in jump_args] inputargs = virtual_state.make_inputargs(values, self.optimizer) short_inputargs = virtual_state.make_inputargs(values, self.optimizer, keyboxes=True) - if self.boxes_created_this_iteration is not None: for box in self.inputargs: self.boxes_created_this_iteration[box] = True @@ -210,7 +208,7 @@ if op and op.result: box = op.result exported_values[box] = self.optimizer.getvalue(box) - + target_token.exported_state = ExportedState(short_boxes, inputarg_setup_ops, exported_values) @@ -232,7 +230,7 @@ virtual_state = modifier.get_virtual_state(self.inputargs) self.initial_virtual_state = virtual_state return - + self.short = target_token.short_preamble[:] self.short_seen = {} self.short_boxes = exported_state.short_boxes @@ -276,7 +274,7 @@ #if self.optimizer.loop.logops: # debug_print(' Falling back to add extra: ' + # self.optimizer.loop.logops.repr_of_resop(op)) - + self.optimizer.flush() self.optimizer.emitting_dissabled = False @@ -287,7 +285,7 @@ # We dont need to inline the short preamble we are creating as we are conneting # the bridge to a different trace with a different short preamble self.short_inliner = None - + newoperations = self.optimizer.get_newoperations() self.boxes_created_this_iteration = {} i = 0 @@ -333,7 +331,7 @@ 'same box passed to multiple of its ' + 'inputargs, but the jump at the ' + 'end of this bridge does not do that.') - + args[short_inputargs[i]] = jmp_to_short_args[i] self.short_inliner = Inliner(short_inputargs, jmp_to_short_args) i = 1 @@ -400,7 +398,7 @@ 'loop is not compatible with the virtual ' + 'state at the start of the loop which makes ' + 'it impossible to close the loop') - + #debug_stop('jit-log-virtualstate') maxguards = self.optimizer.metainterp_sd.warmrunnerdesc.memory_manager.max_retrace_guards @@ -408,9 +406,9 @@ target_token = jumpop.getdescr() assert isinstance(target_token, TargetToken) target_token.targeting_jitcell_token.retraced_count = sys.maxint - + self.finilize_short_preamble(start_label) - + def finilize_short_preamble(self, start_label): short = self.short assert short[-1].getopnum() == rop.JUMP @@ -482,11 +480,11 @@ if op is None: return None if op.result is not None and op.result in self.short_seen: - if emit and self.short_inliner: + if emit and self.short_inliner: return self.short_inliner.inline_arg(op.result) else: return None - + for a in op.getarglist(): if not isinstance(a, Const) and a not in self.short_seen: self.add_op_to_short(self.short_boxes.producer(a), emit, guards_needed) @@ -497,7 +495,7 @@ if guards_needed and self.short_boxes.has_producer(op.result): value_guards = self.getvalue(op.result).make_guards(op.result) else: - value_guards = [] + value_guards = [] self.short.append(op) self.short_seen[op.result] = True @@ -517,7 +515,7 @@ if newop: return newop.result return None - + def import_box(self, box, inputargs, short_jumpargs, jumpargs): if isinstance(box, Const) or box in inputargs: return @@ -603,7 +601,7 @@ classbox = self.getvalue(newop.result).get_constant_class(self.optimizer.cpu) if not classbox or not classbox.same_constant(target.assumed_classes[shop.result]): raise InvalidLoop('The class of an opaque pointer at the end ' + - 'of the bridge does not mach the class ' + + 'of the bridge does not mach the class ' + 'it has at the start of the target loop') except InvalidLoop: #debug_print("Inlining failed unexpectedly", @@ -615,6 +613,7 @@ debug_stop('jit-log-virtualstate') return False + class ValueImporter(object): def __init__(self, unroll, value, op): self.unroll = unroll @@ -623,7 +622,8 @@ def import_value(self, value): value.import_from(self.preamble_value, self.unroll.optimizer) - self.unroll.add_op_to_short(self.op, False, True) + self.unroll.add_op_to_short(self.op, False, True) + class ExportedState(object): def __init__(self, short_boxes, inputarg_setup_ops, exported_values): From noreply at buildbot.pypy.org Wed Jan 30 08:04:28 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 30 Jan 2013 08:04:28 +0100 (CET) Subject: [pypy-commit] pypy stdlib-2.7.4: merge default Message-ID: <20130130070428.28A811C039A@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: stdlib-2.7.4 Changeset: r60707:ac5e46dfb1d8 Date: 2013-01-29 23:29 -0500 http://bitbucket.org/pypy/pypy/changeset/ac5e46dfb1d8/ Log: merge default diff --git a/lib-python/2.7/inspect.py b/lib-python/2.7/inspect.py --- a/lib-python/2.7/inspect.py +++ b/lib-python/2.7/inspect.py @@ -960,7 +960,7 @@ raise TypeError('%s() takes exactly 0 arguments ' '(%d given)' % (f_name, num_total)) else: - raise TypeError('%s() takes no argument (%d given)' % + raise TypeError('%s() takes no arguments (%d given)' % (f_name, num_total)) for arg in args: if isinstance(arg, str) and arg in named: diff --git a/lib-python/2.7/site.py b/lib-python/2.7/site.py --- a/lib-python/2.7/site.py +++ b/lib-python/2.7/site.py @@ -75,6 +75,7 @@ USER_SITE = None USER_BASE = None + def makepath(*paths): dir = os.path.join(*paths) try: diff --git a/lib-python/2.7/test/test_capi.py b/lib-python/2.7/test/test_capi.py --- a/lib-python/2.7/test/test_capi.py +++ b/lib-python/2.7/test/test_capi.py @@ -13,6 +13,20 @@ threading = None import _testcapi +skips = [] +if test_support.check_impl_detail(pypy=True): + skips += [ + 'test_broken_memoryview', + 'test_capsule', + 'test_lazy_hash_inheritance', + 'test_long_api', + 'test_longlong_api', + 'test_null_strings', + 'test_widechar', + 'TestThreadState', + 'TestPendingCalls', + ] + @unittest.skipUnless(threading, 'Threading required for this test.') class TestPendingCalls(unittest.TestCase): @@ -99,7 +113,7 @@ def test_main(): for name in dir(_testcapi): - if name.startswith('test_'): + if name.startswith('test_') and name not in skips: test = getattr(_testcapi, name) if test_support.verbose: print "internal", name @@ -126,7 +140,7 @@ raise test_support.TestFailed, \ "Couldn't find main thread correctly in the list" - if threading: + if threading and 'TestThreadState' not in skips: import thread import time TestThreadState() @@ -134,7 +148,8 @@ t.start() t.join() - test_support.run_unittest(TestPendingCalls) + if 'TestPendingCalls' not in skips: + test_support.run_unittest(TestPendingCalls) if __name__ == "__main__": test_main() diff --git a/lib-python/2.7/test/test_itertools.py b/lib-python/2.7/test/test_itertools.py --- a/lib-python/2.7/test/test_itertools.py +++ b/lib-python/2.7/test/test_itertools.py @@ -533,11 +533,11 @@ self.assertEqual(list(izip()), zip()) self.assertRaises(TypeError, izip, 3) self.assertRaises(TypeError, izip, range(3), 3) - self.assertEqual([tuple(list(pair)) for pair in izip('abc', 'def')], zip('abc', 'def')) self.assertEqual([pair for pair in izip('abc', 'def')], zip('abc', 'def')) + @test_support.impl_detail("tuple reuse is specific to CPython") def test_izip_tuple_reuse(self): ids = map(id, izip('abc', 'def')) @@ -588,6 +588,7 @@ zip('abc', 'def')) self.assertEqual([pair for pair in izip_longest('abc', 'def')], zip('abc', 'def')) + @test_support.impl_detail("tuple reuse is specific to CPython") def test_izip_longest_tuple_reuse(self): ids = map(id, izip_longest('abc', 'def')) diff --git a/lib-python/2.7/test/test_support.py b/lib-python/2.7/test/test_support.py --- a/lib-python/2.7/test/test_support.py +++ b/lib-python/2.7/test/test_support.py @@ -1181,7 +1181,6 @@ else: runner = BasicTestRunner() - result = runner.run(suite) if not result.wasSuccessful(): if len(result.errors) == 1 and not result.failures: diff --git a/lib-python/conftest.py b/lib-python/conftest.py --- a/lib-python/conftest.py +++ b/lib-python/conftest.py @@ -132,7 +132,7 @@ RegrTest('test_bz2.py', usemodules='bz2'), RegrTest('test_calendar.py'), RegrTest('test_call.py', core=True), - RegrTest('test_capi.py', skip="not applicable"), + RegrTest('test_capi.py'), RegrTest('test_cd.py'), RegrTest('test_cfgparser.py'), RegrTest('test_cgi.py'), diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -922,8 +922,8 @@ kwds["link_extra"] = ["msvcrt.lib"] elif sys.platform.startswith('linux'): compile_extra.append("-Werror=implicit-function-declaration") + compile_extra.append('-g') export_symbols_eci.append('pypyAPI') - compile_extra.append('-g') else: kwds["includes"] = ['Python.h'] # this is our Python.h diff --git a/pypy/module/thread/test/support.py b/pypy/module/thread/test/support.py --- a/pypy/module/thread/test/support.py +++ b/pypy/module/thread/test/support.py @@ -2,6 +2,7 @@ import time import thread import os +import errno from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.module.thread import gil @@ -28,7 +29,12 @@ def kill(): for x in range(delay * 10): time.sleep(0.1) - os.kill(pid, 0) + try: + os.kill(pid, 0) + except OSError, e: + if e.errno == errno.ESRCH: # no such process + return + raise os.kill(pid, 9) print "process %s killed!" % (pid,) thread.start_new_thread(kill, ()) diff --git a/pypy/module/thread/test/test_fork.py b/pypy/module/thread/test/test_fork.py --- a/pypy/module/thread/test/test_fork.py +++ b/pypy/module/thread/test/test_fork.py @@ -1,7 +1,7 @@ from pypy.module.thread.test.support import GenericTestThread class AppTestFork(GenericTestThread): - def test_fork(self): + def test_fork_with_thread(self): # XXX This test depends on a multicore machine, as busy_thread must # aquire the GIL the instant that the main thread releases it. # It will incorrectly pass if the GIL is not grabbed in time. @@ -12,45 +12,48 @@ if not hasattr(os, 'fork'): skip("No fork on this platform") - run = True - done = [] def busy_thread(): while run: time.sleep(0) done.append(None) - try: - thread.start_new(busy_thread, ()) + for i in range(1): + run = True + done = [] + try: + thread.start_new(busy_thread, ()) + print 'sleep' - pid = os.fork() - - if pid == 0: - os._exit(0) - - else: - time.sleep(1) - spid, status = os.waitpid(pid, os.WNOHANG) - assert spid == pid - finally: - run = False - self.waitfor(lambda: done) + pid = os.fork() + if pid == 0: + os._exit(0) + else: + self.timeout_killer(pid, 5) + exitcode = os.waitpid(pid, 0)[1] + assert exitcode == 0 # if 9, process was killed by timer! + finally: + run = False + self.waitfor(lambda: done) + assert done def test_forked_can_thread(self): "Checks that a forked interpreter can start a thread" - import os, thread, time + import thread + import os if not hasattr(os, 'fork'): skip("No fork on this platform") - # pre-allocate some locks - thread.start_new_thread(lambda: None, ()) + for i in range(10): + # pre-allocate some locks + thread.start_new_thread(lambda: None, ()) + print 'sleep' - pid = os.fork() - if pid == 0: - print 'in child' - thread.start_new_thread(lambda: None, ()) - os._exit(0) - else: - self.timeout_killer(pid, 5) - exitcode = os.waitpid(pid, 0)[1] - assert exitcode == 0 # if 9, process was killed by timer! + pid = os.fork() + if pid == 0: + thread.start_new_thread(lambda: None, ()) + os._exit(0) + else: + self.timeout_killer(pid, 5) + exitcode = os.waitpid(pid, 0)[1] + assert exitcode == 0 # if 9, process was killed by timer! diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -239,6 +239,8 @@ def optimize_GUARD_VALUE(self, op): value = self.getvalue(op.getarg(0)) + if value.is_virtual(): + raise InvalidLoop('A promote of a virtual (a recently allocated object) never makes sense!') if value.last_guard: # there already has been a guard_nonnull or guard_class or # guard_nonnull_class on this value, which is rather silly. diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -2623,7 +2623,7 @@ jump(p2) """ self.raises(InvalidLoop, self.optimize_loop, - ops, ops) + ops, "crash!") def test_invalid_loop_2(self): ops = """ @@ -2635,7 +2635,7 @@ jump(p2) """ self.raises(InvalidLoop, self.optimize_loop, - ops, ops) + ops, "crash!") def test_invalid_loop_3(self): ops = """ @@ -2648,8 +2648,17 @@ setfield_gc(p3, p4, descr=nextdescr) jump(p3) """ - self.raises(InvalidLoop, self.optimize_loop, ops, ops) - + self.raises(InvalidLoop, self.optimize_loop, ops, "crash!") + + def test_invalid_loop_guard_value_of_virtual(self): + ops = """ + [p1] + p2 = new_with_vtable(ConstClass(node_vtable)) + guard_value(p2, ConstPtr(myptr)) [] + jump(p2) + """ + self.raises(InvalidLoop, self.optimize_loop, + ops, "crash!") def test_merge_guard_class_guard_value(self): ops = """ diff --git a/rpython/translator/c/src/thread_nt.c b/rpython/translator/c/src/thread_nt.c --- a/rpython/translator/c/src/thread_nt.c +++ b/rpython/translator/c/src/thread_nt.c @@ -13,6 +13,7 @@ /* * Thread support. */ +/* In rpython, this file is pulled in by thread.c */ typedef struct RPyOpaque_ThreadLock NRMUTEX, *PNRMUTEX; diff --git a/rpython/translator/c/src/thread_nt.h b/rpython/translator/c/src/thread_nt.h --- a/rpython/translator/c/src/thread_nt.h +++ b/rpython/translator/c/src/thread_nt.h @@ -9,6 +9,7 @@ } NRMUTEX, *PNRMUTEX; /* prototypes */ +long RPyThreadGetIdent(void); long RPyThreadStart(void (*func)(void)); int RPyThreadLockInit(struct RPyOpaque_ThreadLock *lock); void RPyOpaqueDealloc_ThreadLock(struct RPyOpaque_ThreadLock *lock); From noreply at buildbot.pypy.org Wed Jan 30 08:04:29 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 30 Jan 2013 08:04:29 +0100 (CET) Subject: [pypy-commit] pypy default: avoid overflow in _bisect (cpython issue13496) Message-ID: <20130130070429.63A871C039A@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60708:1ac9a04e31c4 Date: 2013-01-30 02:02 -0500 http://bitbucket.org/pypy/pypy/changeset/1ac9a04e31c4/ Log: avoid overflow in _bisect (cpython issue13496) diff --git a/pypy/module/_bisect/interp_bisect.py b/pypy/module/_bisect/interp_bisect.py --- a/pypy/module/_bisect/interp_bisect.py +++ b/pypy/module/_bisect/interp_bisect.py @@ -1,5 +1,6 @@ from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import unwrap_spec +from rpython.rlib.rarithmetic import intmask, r_uint @unwrap_spec(lo=int, hi=int) @@ -18,7 +19,7 @@ if hi == -1: hi = space.len_w(w_a) while lo < hi: - mid = (lo + hi) >> 1 + mid = intmask((r_uint(lo) + r_uint(hi)) >> 1) w_litem = space.getitem(w_a, space.wrap(mid)) if space.is_true(space.lt(w_litem, w_x)): lo = mid + 1 @@ -43,7 +44,7 @@ if hi == -1: hi = space.len_w(w_a) while lo < hi: - mid = (lo + hi) >> 1 + mid = intmask((r_uint(lo) + r_uint(hi)) >> 1) w_litem = space.getitem(w_a, space.wrap(mid)) if space.is_true(space.lt(w_x, w_litem)): hi = mid diff --git a/pypy/module/_bisect/test/test_bisect.py b/pypy/module/_bisect/test/test_bisect.py --- a/pypy/module/_bisect/test/test_bisect.py +++ b/pypy/module/_bisect/test/test_bisect.py @@ -89,3 +89,12 @@ insort_right(a, 6.0) assert a == [0, 5, 6, 6, 6, 6.0, 7] assert map(type, a) == [int, int, int, int, int, float, int] + + def test_bisect_overflow(self): + from _bisect import bisect_left, bisect_right + import sys + + size = sys.maxsize + data = xrange(size - 1) + assert bisect_left(data, size - 3) == size - 3 + assert bisect_right(data, size - 3) == size - 2 From noreply at buildbot.pypy.org Wed Jan 30 09:24:00 2013 From: noreply at buildbot.pypy.org (arigo) Date: Wed, 30 Jan 2013 09:24:00 +0100 (CET) Subject: [pypy-commit] pypy default: Expand this section Message-ID: <20130130082400.A79F51C0041@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r60709:8f7466dbf151 Date: 2013-01-30 09:23 +0100 http://bitbucket.org/pypy/pypy/changeset/8f7466dbf151/ Log: Expand this section diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst --- a/pypy/doc/faq.rst +++ b/pypy/doc/faq.rst @@ -300,12 +300,26 @@ Do I have to rewrite my programs in RPython? -------------------------------------------- -No. And you shouldn't try. PyPy always runs your code in its own interpreter, which is a -full and compliant Python 2.7 interpreter. RPython is only the -language in which parts of PyPy itself are written and extension -modules for it. Not only is it not necessary for you to rewrite your -code in RPython, it probably won't give you any speed improvements if you -try. +No. And you shouldn't try. First and foremost, RPython is a language +that is designed to write interpreters in. It is a restricted subset of +Python. If you program is not an interpreter but tries to do "real +things", like use *any* part of the standard Python library or *any* +3rd-party library, then it is not RPython to start with. You should +only look at RPython if you try to `write your own interpreter`__. + +.. __: `how do I compile my own interpreters`_ + +If your goal is to speed up Python code, then look at the regular PyPy, +which is a full and compliant Python 2.7 interpreter (which happens to +be written in RPython). Not only is it not necessary for you to rewrite +your code in RPython, it might not give you any speed improvements even +if you manage to. + +Yes, it is possible with enough effort to compile small self-contained +pieces of RPython code doing a few performance-sensitive things. But +this case is not interesting for us. If you needed to rewrite the code +in RPython, you could as well have rewritten it in C for example. The +latter is a much more supported, much more documented language `:-)` --------------------------------------------------- Which backends are there for the RPython toolchain? From noreply at buildbot.pypy.org Wed Jan 30 10:02:31 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 30 Jan 2013 10:02:31 +0100 (CET) Subject: [pypy-commit] pypy stdlib-2.7.4: file descriptors that don't fit in a c_int now raise OverflowError Message-ID: <20130130090231.4334F1C0DC1@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: stdlib-2.7.4 Changeset: r60711:590e925436af Date: 2013-01-30 03:58 -0500 http://bitbucket.org/pypy/pypy/changeset/590e925436af/ Log: file descriptors that don't fit in a c_int now raise OverflowError diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -1486,7 +1486,7 @@ raise OperationError(self.w_TypeError, self.wrap("fileno() must return an integer") ) - fd = self.int_w(w_fd) + fd = self.c_int_w(w_fd) if fd < 0: raise operationerrfmt(self.w_ValueError, "file descriptor cannot be a negative integer (%d)", fd From noreply at buildbot.pypy.org Wed Jan 30 10:02:30 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 30 Jan 2013 10:02:30 +0100 (CET) Subject: [pypy-commit] pypy stdlib-2.7.4: merge default Message-ID: <20130130090230.04E551C03E3@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: stdlib-2.7.4 Changeset: r60710:354042e98f1a Date: 2013-01-30 02:06 -0500 http://bitbucket.org/pypy/pypy/changeset/354042e98f1a/ Log: merge default diff --git a/pypy/module/_bisect/interp_bisect.py b/pypy/module/_bisect/interp_bisect.py --- a/pypy/module/_bisect/interp_bisect.py +++ b/pypy/module/_bisect/interp_bisect.py @@ -1,5 +1,6 @@ from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import unwrap_spec +from rpython.rlib.rarithmetic import intmask, r_uint @unwrap_spec(lo=int, hi=int) @@ -18,7 +19,7 @@ if hi == -1: hi = space.len_w(w_a) while lo < hi: - mid = (lo + hi) >> 1 + mid = intmask((r_uint(lo) + r_uint(hi)) >> 1) w_litem = space.getitem(w_a, space.wrap(mid)) if space.is_true(space.lt(w_litem, w_x)): lo = mid + 1 @@ -43,7 +44,7 @@ if hi == -1: hi = space.len_w(w_a) while lo < hi: - mid = (lo + hi) >> 1 + mid = intmask((r_uint(lo) + r_uint(hi)) >> 1) w_litem = space.getitem(w_a, space.wrap(mid)) if space.is_true(space.lt(w_x, w_litem)): hi = mid diff --git a/pypy/module/_bisect/test/test_bisect.py b/pypy/module/_bisect/test/test_bisect.py --- a/pypy/module/_bisect/test/test_bisect.py +++ b/pypy/module/_bisect/test/test_bisect.py @@ -89,3 +89,12 @@ insort_right(a, 6.0) assert a == [0, 5, 6, 6, 6, 6.0, 7] assert map(type, a) == [int, int, int, int, int, float, int] + + def test_bisect_overflow(self): + from _bisect import bisect_left, bisect_right + import sys + + size = sys.maxsize + data = xrange(size - 1) + assert bisect_left(data, size - 3) == size - 3 + assert bisect_right(data, size - 3) == size - 2 diff --git a/rpython/flowspace/framestate.py b/rpython/flowspace/framestate.py --- a/rpython/flowspace/framestate.py +++ b/rpython/flowspace/framestate.py @@ -1,5 +1,6 @@ +from rpython.flowspace.model import Variable, Constant from rpython.rlib.unroll import SpecTag -from rpython.flowspace.model import * + class FrameState: def __init__(self, mergeable, blocklist, next_instr): @@ -86,6 +87,7 @@ raise TypeError('union of %r and %r' % (w1.__class__.__name__, w2.__class__.__name__)) + # ____________________________________________________________ # # We have to flatten out the state of the frame into a list of @@ -102,6 +104,7 @@ PICKLE_TAGS = {} UNPICKLE_TAGS = {} + def recursively_flatten(space, lst): from rpython.flowspace.flowcontext import SuspendedUnroller i = 0 @@ -117,14 +120,15 @@ except: tag = PICKLE_TAGS[key] = Constant(PickleTag()) UNPICKLE_TAGS[tag] = key - lst[i:i+1] = [tag] + vars + lst[i:i + 1] = [tag] + vars + def recursively_unflatten(space, lst): - for i in xrange(len(lst)-1, -1, -1): + for i in xrange(len(lst) - 1, -1, -1): item = lst[i] if item in UNPICKLE_TAGS: unrollerclass, argcount = UNPICKLE_TAGS[item] - arguments = lst[i+1: i+1+argcount] - del lst[i+1: i+1+argcount] + arguments = lst[i + 1:i + 1 + argcount] + del lst[i + 1:i + 1 + argcount] unroller = unrollerclass.state_pack_variables(space, *arguments) lst[i] = unroller diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py --- a/rpython/jit/metainterp/optimizeopt/unroll.py +++ b/rpython/jit/metainterp/optimizeopt/unroll.py @@ -1,16 +1,16 @@ -from rpython.jit.codewriter.effectinfo import EffectInfo +import sys + +from rpython.jit.metainterp.compile import ResumeGuardDescr +from rpython.jit.metainterp.history import TargetToken, JitCellToken, Const +from rpython.jit.metainterp.inliner import Inliner +from rpython.jit.metainterp.optimize import InvalidLoop +from rpython.jit.metainterp.optimizeopt.generalize import KillHugeIntBounds +from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer, Optimization from rpython.jit.metainterp.optimizeopt.virtualstate import VirtualStateAdder, ShortBoxes, BadVirtualState -from rpython.jit.metainterp.compile import ResumeGuardDescr -from rpython.jit.metainterp.history import TreeLoop, TargetToken, JitCellToken -from rpython.jit.metainterp.jitexc import JitException -from rpython.jit.metainterp.optimize import InvalidLoop -from rpython.rlib.debug import debug_print, debug_start, debug_stop -from rpython.jit.metainterp.optimizeopt.optimizer import * -from rpython.jit.metainterp.optimizeopt.generalize import KillHugeIntBounds -from rpython.jit.metainterp.inliner import Inliner from rpython.jit.metainterp.resoperation import rop, ResOperation from rpython.jit.metainterp.resume import Snapshot -import sys, os +from rpython.rlib.debug import debug_print, debug_start, debug_stop + # FIXME: Introduce some VirtualOptimizer super class instead @@ -19,6 +19,7 @@ opt.inline_short_preamble = inline_short_preamble opt.propagate_all_forward() + class UnrollableOptimizer(Optimizer): def setup(self): self.importable_values = {} @@ -51,7 +52,7 @@ distinction anymore)""" inline_short_preamble = True - + def __init__(self, metainterp_sd, loop, optimizations): self.optimizer = UnrollableOptimizer(metainterp_sd, loop, optimizations) self.boxes_created_this_iteration = None @@ -59,14 +60,14 @@ def fix_snapshot(self, jump_args, snapshot): if snapshot is None: return None - snapshot_args = snapshot.boxes + snapshot_args = snapshot.boxes new_snapshot_args = [] for a in snapshot_args: a = self.getvalue(a).get_key_box() new_snapshot_args.append(a) prev = self.fix_snapshot(jump_args, snapshot.prev) return Snapshot(prev, new_snapshot_args) - + def propagate_all_forward(self): loop = self.optimizer.loop self.optimizer.clear_newoperations() @@ -78,7 +79,7 @@ # will clear heap caches self.optimizer.send_extra_operation(start_label) else: - start_label = None + start_label = None jumpop = loop.operations[-1] if jumpop.getopnum() == rop.JUMP or jumpop.getopnum() == rop.LABEL: @@ -96,7 +97,6 @@ assert isinstance(cell_token, JitCellToken) stop_label = ResOperation(rop.LABEL, jumpop.getarglist(), None, TargetToken(cell_token)) - if jumpop.getopnum() == rop.JUMP: if self.jump_to_already_compiled_trace(jumpop): # Found a compiled trace to jump to @@ -130,7 +130,7 @@ return # Found nothing to jump to, emit a label instead - + if self.short: # Construct our short preamble assert start_label @@ -146,7 +146,7 @@ def jump_to_start_label(self, start_label, stop_label): if not start_label or not stop_label: return False - + stop_target = stop_label.getdescr() start_target = start_label.getdescr() assert isinstance(stop_target, TargetToken) @@ -161,7 +161,6 @@ #virtual_state = modifier.get_virtual_state(args) #if self.initial_virtual_state.generalization_of(virtual_state): # return True - def export_state(self, targetop): original_jump_args = targetop.getarglist() @@ -174,12 +173,11 @@ modifier = VirtualStateAdder(self.optimizer) virtual_state = modifier.get_virtual_state(jump_args) - + values = [self.getvalue(arg) for arg in jump_args] inputargs = virtual_state.make_inputargs(values, self.optimizer) short_inputargs = virtual_state.make_inputargs(values, self.optimizer, keyboxes=True) - if self.boxes_created_this_iteration is not None: for box in self.inputargs: self.boxes_created_this_iteration[box] = True @@ -210,7 +208,7 @@ if op and op.result: box = op.result exported_values[box] = self.optimizer.getvalue(box) - + target_token.exported_state = ExportedState(short_boxes, inputarg_setup_ops, exported_values) @@ -232,7 +230,7 @@ virtual_state = modifier.get_virtual_state(self.inputargs) self.initial_virtual_state = virtual_state return - + self.short = target_token.short_preamble[:] self.short_seen = {} self.short_boxes = exported_state.short_boxes @@ -276,7 +274,7 @@ #if self.optimizer.loop.logops: # debug_print(' Falling back to add extra: ' + # self.optimizer.loop.logops.repr_of_resop(op)) - + self.optimizer.flush() self.optimizer.emitting_dissabled = False @@ -287,7 +285,7 @@ # We dont need to inline the short preamble we are creating as we are conneting # the bridge to a different trace with a different short preamble self.short_inliner = None - + newoperations = self.optimizer.get_newoperations() self.boxes_created_this_iteration = {} i = 0 @@ -333,7 +331,7 @@ 'same box passed to multiple of its ' + 'inputargs, but the jump at the ' + 'end of this bridge does not do that.') - + args[short_inputargs[i]] = jmp_to_short_args[i] self.short_inliner = Inliner(short_inputargs, jmp_to_short_args) i = 1 @@ -400,7 +398,7 @@ 'loop is not compatible with the virtual ' + 'state at the start of the loop which makes ' + 'it impossible to close the loop') - + #debug_stop('jit-log-virtualstate') maxguards = self.optimizer.metainterp_sd.warmrunnerdesc.memory_manager.max_retrace_guards @@ -408,9 +406,9 @@ target_token = jumpop.getdescr() assert isinstance(target_token, TargetToken) target_token.targeting_jitcell_token.retraced_count = sys.maxint - + self.finilize_short_preamble(start_label) - + def finilize_short_preamble(self, start_label): short = self.short assert short[-1].getopnum() == rop.JUMP @@ -482,11 +480,11 @@ if op is None: return None if op.result is not None and op.result in self.short_seen: - if emit and self.short_inliner: + if emit and self.short_inliner: return self.short_inliner.inline_arg(op.result) else: return None - + for a in op.getarglist(): if not isinstance(a, Const) and a not in self.short_seen: self.add_op_to_short(self.short_boxes.producer(a), emit, guards_needed) @@ -497,7 +495,7 @@ if guards_needed and self.short_boxes.has_producer(op.result): value_guards = self.getvalue(op.result).make_guards(op.result) else: - value_guards = [] + value_guards = [] self.short.append(op) self.short_seen[op.result] = True @@ -517,7 +515,7 @@ if newop: return newop.result return None - + def import_box(self, box, inputargs, short_jumpargs, jumpargs): if isinstance(box, Const) or box in inputargs: return @@ -603,7 +601,7 @@ classbox = self.getvalue(newop.result).get_constant_class(self.optimizer.cpu) if not classbox or not classbox.same_constant(target.assumed_classes[shop.result]): raise InvalidLoop('The class of an opaque pointer at the end ' + - 'of the bridge does not mach the class ' + + 'of the bridge does not mach the class ' + 'it has at the start of the target loop') except InvalidLoop: #debug_print("Inlining failed unexpectedly", @@ -615,6 +613,7 @@ debug_stop('jit-log-virtualstate') return False + class ValueImporter(object): def __init__(self, unroll, value, op): self.unroll = unroll @@ -623,7 +622,8 @@ def import_value(self, value): value.import_from(self.preamble_value, self.unroll.optimizer) - self.unroll.add_op_to_short(self.op, False, True) + self.unroll.add_op_to_short(self.op, False, True) + class ExportedState(object): def __init__(self, short_boxes, inputarg_setup_ops, exported_values): diff --git a/rpython/jit/metainterp/virtualizable.py b/rpython/jit/metainterp/virtualizable.py --- a/rpython/jit/metainterp/virtualizable.py +++ b/rpython/jit/metainterp/virtualizable.py @@ -1,14 +1,12 @@ +from rpython.jit.metainterp import history +from rpython.jit.metainterp.typesystem import deref, fieldType, arrayItem +from rpython.jit.metainterp.warmstate import wrap, unwrap +from rpython.rlib.unroll import unrolling_iterable +from rpython.rtyper import rvirtualizable2 from rpython.rtyper.lltypesystem import lltype, llmemory from rpython.rtyper.ootypesystem import ootype -from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance from rpython.rtyper.rclass import IR_IMMUTABLE_ARRAY, IR_IMMUTABLE -from rpython.rtyper import rvirtualizable2 -from rpython.rlib.unroll import unrolling_iterable -from rpython.rlib.nonconst import NonConstant -from rpython.jit.metainterp.typesystem import deref, fieldType, arrayItem -from rpython.jit.metainterp import history -from rpython.jit.metainterp.warmstate import wrap, unwrap -from rpython.rlib.objectmodel import specialize + class VirtualizableInfo(object): TOKEN_NONE = 0 # must be 0 -- see also x86.call_assembler @@ -76,7 +74,7 @@ getlength = cpu.ts.getlength getarrayitem = cpu.ts.getarrayitem setarrayitem = cpu.ts.setarrayitem - # + def read_boxes(cpu, virtualizable): assert lltype.typeOf(virtualizable) == llmemory.GCREF virtualizable = cast_gcref_to_vtype(virtualizable) @@ -89,7 +87,7 @@ for i in range(getlength(lst)): boxes.append(wrap(cpu, getarrayitem(lst, i))) return boxes - # + def write_boxes(virtualizable, boxes): virtualizable = cast_gcref_to_vtype(virtualizable) i = 0 @@ -104,7 +102,7 @@ setarrayitem(lst, j, x) i = i + 1 assert len(boxes) == i + 1 - # + def write_from_resume_data_partial(virtualizable, reader, numb): virtualizable = cast_gcref_to_vtype(virtualizable) # Load values from the reader (see resume.py) described by @@ -117,7 +115,7 @@ assert i >= 0 for ARRAYITEMTYPE, fieldname in unroll_array_fields_rev: lst = getattr(virtualizable, fieldname) - for j in range(getlength(lst)-1, -1, -1): + for j in range(getlength(lst) - 1, -1, -1): i -= 1 assert i >= 0 x = reader.load_value_of_type(ARRAYITEMTYPE, numb.nums[i]) @@ -128,7 +126,7 @@ x = reader.load_value_of_type(FIELDTYPE, numb.nums[i]) setattr(virtualizable, fieldname, x) return i - # + def load_list_of_boxes(virtualizable, reader, numb): virtualizable = cast_gcref_to_vtype(virtualizable) # Uses 'virtualizable' only to know the length of the arrays; @@ -140,10 +138,10 @@ boxes = [reader.decode_box_of_type(self.VTYPEPTR, numb.nums[i])] for ARRAYITEMTYPE, fieldname in unroll_array_fields_rev: lst = getattr(virtualizable, fieldname) - for j in range(getlength(lst)-1, -1, -1): + for j in range(getlength(lst) - 1, -1, -1): i -= 1 assert i >= 0 - box = reader.decode_box_of_type(ARRAYITEMTYPE,numb.nums[i]) + box = reader.decode_box_of_type(ARRAYITEMTYPE, numb.nums[i]) boxes.append(box) for FIELDTYPE, fieldname in unroll_static_fields_rev: i -= 1 @@ -152,7 +150,7 @@ boxes.append(box) boxes.reverse() return boxes - # + def check_boxes(virtualizable, boxes): virtualizable = cast_gcref_to_vtype(virtualizable) # for debugging @@ -168,7 +166,7 @@ assert getarrayitem(lst, j) == x i = i + 1 assert len(boxes) == i + 1 - # + def get_index_in_array(virtualizable, arrayindex, index): virtualizable = cast_gcref_to_vtype(virtualizable) index += self.num_static_extra_boxes @@ -180,7 +178,7 @@ index += getlength(lst) j = j + 1 assert False, "invalid arrayindex" - # + def get_array_length(virtualizable, arrayindex): virtualizable = cast_gcref_to_vtype(virtualizable) j = 0 @@ -188,16 +186,16 @@ if arrayindex == j: lst = getattr(virtualizable, fieldname) return getlength(lst) - j = j + 1 + j += 1 assert False, "invalid arrayindex" - # + unroll_static_fields = unrolling_iterable(zip(FIELDTYPES, static_fields)) unroll_array_fields = unrolling_iterable(zip(ARRAYITEMTYPES, array_fields)) unroll_static_fields_rev = unrolling_iterable( reversed(list(unroll_static_fields))) - unroll_array_fields_rev = unrolling_iterable( + unroll_array_fields_rev = unrolling_iterable( reversed(list(unroll_array_fields))) self.read_boxes = read_boxes self.write_boxes = write_boxes @@ -291,7 +289,7 @@ def unwrap_virtualizable_box(self, virtualizable_box): return virtualizable_box.getref(llmemory.GCREF) - + def is_vtypeptr(self, TYPE): return rvirtualizable2.match_virtualizable_type(TYPE, self.VTYPEPTR) From noreply at buildbot.pypy.org Wed Jan 30 10:25:52 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 30 Jan 2013 10:25:52 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: some ugly checks Message-ID: <20130130092552.3CD061C039A@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60712:b7bd1559a188 Date: 2013-01-30 11:24 +0200 http://bitbucket.org/pypy/pypy/changeset/b7bd1559a188/ Log: some ugly checks diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py --- a/rpython/jit/backend/llsupport/llmodel.py +++ b/rpython/jit/backend/llsupport/llmodel.py @@ -72,6 +72,16 @@ llop.gc_assume_young_pointers(lltype.Void, new_frame) return lltype.cast_opaque_ptr(llmemory.GCREF, new_frame) + def realloc_frame_check(frame, size): + frame = lltype.cast_opaque_ptr(jitframe.JITFRAMEPTR, frame) + if size > len(frame.jf_frame) or size > frame.jf_frame_info.jfi_frame_depth: + print "CHECK FAILED" + import pdb + pdb.set_trace() + assert False + print "SOMETHING INCREDIBLY FISHY" + return lltype.cast_opaque_ptr(llmemory.GCREF, frame) + if not translate_support_code: fptr = llhelper(FUNC_TP, realloc_frame) else: @@ -84,6 +94,18 @@ mixlevelann.finish() self.realloc_frame = heaptracker.adr2int(llmemory.cast_ptr_to_adr(fptr)) + if not translate_support_code: + fptr = llhelper(FUNC_TP, realloc_frame_check) + else: + FUNC = FUNC_TP.TO + args_s = [annmodel.lltype_to_annotation(ARG) for ARG in FUNC.ARGS] + s_result = annmodel.lltype_to_annotation(FUNC.RESULT) + mixlevelann = MixLevelHelperAnnotator(self.rtyper) + graph = mixlevelann.getgraph(realloc_frame, args_s, s_result) + fptr = mixlevelann.graph2delayed(graph, FUNC) + mixlevelann.finish() + self.realloc_frame_check = heaptracker.adr2int(llmemory.cast_ptr_to_adr(fptr)) + def _setup_exception_handling_untranslated(self): # for running un-translated only, all exceptions occurring in the # llinterpreter are stored in '_exception_emulator', which is then diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -220,6 +220,27 @@ mc.RET() self._stack_check_failure = mc.materialize(self.cpu.asmmemmgr, []) + mc = codebuf.MachineCodeBlockWrapper() + self._push_all_regs_to_frame(mc, [], self.cpu.supports_floats) + assert not IS_X86_32 + # this is the gcmap stored by push_gcmap(mov=True) in _check_stack_frame + mc.MOV_rs(ecx.value, WORD) + gcmap_ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') + mc.MOV_br(gcmap_ofs, ecx.value) + # this is size that we're after, sanity checking only + mc.MOV_rs(esi.value, WORD*2) + # push first arg + mc.LEA_rb(edi.value, -base_ofs) + # align + mc.SUB_ri(esp.value, WORD) + mc.CALL(imm(self.cpu.realloc_frame_check)) + mc.ADD_ri(esp.value, WORD) + mc.LEA_rm(ebp.value, (eax.value, base_ofs)) + mc.MOV_bi(gcmap_ofs, 0) + self._pop_all_regs_from_frame(mc, [], self.cpu.supports_floats) + mc.RET() + self._stack_check_failure_2 = mc.materialize(self.cpu.asmmemmgr, []) + def _build_malloc_slowpath(self): """ While arriving on slowpath, we have a gcpattern on stack, nursery_head in eax and the size in edi - eax @@ -553,6 +574,9 @@ looptoken._x86_fullsize = full_size looptoken._x86_ops_offset = ops_offset looptoken._x86_function_addr = rawstart + for label in self.labels_to_patch: + self._patch_stackadjust(label + rawstart, frame_depth + JITFRAME_FIXED_SIZE) + self.labels_to_patch = None self.fixup_target_tokens(rawstart) self.teardown() @@ -603,6 +627,9 @@ frame_depth + JITFRAME_FIXED_SIZE) self._patch_stackadjust(stack_check_patch_ofs + rawstart, frame_depth) self._patch_stackadjust(ofs2 + rawstart, frame_depth) + for label in self.labels_to_patch: + self._patch_stackadjust(label + rawstart, frame_depth) + self.labels_to_patch = None self.fixup_target_tokens(rawstart) self.update_frame_depth(frame_depth) self.teardown() @@ -667,7 +694,7 @@ baseofs = self.cpu.get_baseofs_of_frame_field() self.current_clt.frame_info.set_frame_depth(baseofs, frame_depth) - def _check_frame_depth(self, mc, gcmap, expected_size=-1): + def _check_frame_depth(self, mc, gcmap, expected_size=-1, check_only=False): """ check if the frame is of enough depth to follow this bridge. Otherwise reallocate the frame in a helper. There are other potential solutions @@ -690,7 +717,10 @@ mc.MOV_si(WORD, expected_size) ofs2 = mc.get_relative_pos() - 4 self.push_gcmap(mc, gcmap, mov=True) - mc.CALL(imm(self._stack_check_failure)) + if check_only: + mc.CALL(imm(self._stack_check_failure_2)) + else: + mc.CALL(imm(self._stack_check_failure)) # patch the JG above offset = mc.get_relative_pos() - jg_location assert 0 < offset <= 127 @@ -792,6 +822,7 @@ def _assemble(self, regalloc, inputargs, operations): self._regalloc = regalloc + self.labels_to_patch = [] regalloc.compute_hint_frame_locations(operations) regalloc.walk_operations(inputargs, operations) if we_are_translated() or self.cpu.dont_keepalive_stuff: @@ -2486,6 +2517,12 @@ expected_size=expected_size) self.mc.JMP(imm(target)) + def label(self): + ofs, ofs2 = self._check_frame_depth(self.mc, self._regalloc.get_gcmap(), + check_only=True) + self.labels_to_patch.append(ofs) + self.labels_to_patch.append(ofs2) + def malloc_cond(self, nursery_free_adr, nursery_top_adr, size, gcmap): assert size & (WORD-1) == 0 # must be correctly aligned self.mc.MOV(eax, heap(nursery_free_adr)) diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -1322,6 +1322,7 @@ descr._x86_clt = self.assembler.current_clt self.assembler.target_tokens_currently_compiling[descr] = None self.possibly_free_vars_for_op(op) + self.assembler.label() # # if the LABEL's descr is precisely the target of the JUMP at the # end of the same loop, i.e. if what we are compiling is a single From noreply at buildbot.pypy.org Wed Jan 30 10:25:53 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 30 Jan 2013 10:25:53 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: merge Message-ID: <20130130092553.8DD4C1C039A@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60713:39bf1dd52892 Date: 2013-01-30 11:25 +0200 http://bitbucket.org/pypy/pypy/changeset/39bf1dd52892/ Log: merge diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py --- a/rpython/jit/backend/arm/assembler.py +++ b/rpython/jit/backend/arm/assembler.py @@ -115,7 +115,8 @@ self.set_debug(have_debug_prints()) debug_stop('jit-backend-counts') # when finishing, we only have one value at [0], the rest dies - self.gcmap_for_finish = lltype.malloc(jitframe.GCMAP, 1, zero=True) + self.gcmap_for_finish = lltype.malloc(jitframe.GCMAP, 1, + flavor='raw', immortal=True) self.gcmap_for_finish[0] = r_uint(1) def setup_failure_recovery(self): @@ -502,6 +503,8 @@ positions[i] = v * WORD # write down the positions of locs guardtok.faildescr.rd_locs = positions + # we want the descr to keep alive + guardtok.faildescr.rd_loop_token = self.current_clt self.regalloc_push(imm(fail_descr)) self.push_gcmap(self.mc, gcmap=guardtok.gcmap, push=True) self.mc.BL(target) @@ -603,9 +606,6 @@ # cpu interface def assemble_loop(self, loopname, inputargs, operations, looptoken, log): clt = CompiledLoopToken(self.cpu, looptoken.number) - clt.frame_info = lltype.malloc(jitframe.JITFRAMEINFO) - clt.allgcrefs = [] - clt.frame_info.jfi_frame_depth = 0 # for now looptoken.compiled_loop_token = clt clt._debug_nbargs = len(inputargs) @@ -614,6 +614,13 @@ assert len(set(inputargs)) == len(inputargs) self.setup(looptoken) + + frame_info = self.datablockwrapper.malloc_aligned( + jitframe.JITFRAMEINFO_SIZE, alignment=WORD) + clt.frame_info = rffi.cast(jitframe.JITFRAMEINFOPTR, frame_info) + clt.allgcrefs = [] + clt.frame_info.set_frame_depth(0, 0) # for now + if False and log: operations = self._inject_debugging_code(looptoken, operations, 'e', looptoken.number) @@ -624,8 +631,6 @@ regalloc = Regalloc(assembler=self) operations = regalloc.prepare_loop(inputargs, operations, looptoken, clt.allgcrefs) - rgc._make_sure_does_not_move(lltype.cast_opaque_ptr(llmemory.GCREF, - clt.frame_info)) loop_head = self.mc.get_relative_pos() looptoken._arm_loop_code = loop_head @@ -755,7 +760,8 @@ self.cpu.gc_ll_descr.gcrootmap) def update_frame_depth(self, frame_depth): - self.current_clt.frame_info.jfi_frame_depth = frame_depth + baseofs = self.cpu.get_baseofs_of_frame_field() + self.current_clt.frame_info.set_frame_depth(baseofs, frame_depth) def write_pending_failure_recoveries(self): for tok in self.pending_guards: @@ -975,13 +981,13 @@ offset = loc.value if not check_imm_arg(offset, size=0xFFF): self.mc.PUSH([temp.value], cond=cond) - self.mc.gen_load_int(temp.value, -offset, cond=cond) + self.mc.gen_load_int(temp.value, offset, cond=cond) self.mc.STR_rr(prev_loc.value, r.fp.value, temp.value, cond=cond) self.mc.POP([temp.value], cond=cond) else: self.mc.STR_ri(prev_loc.value, r.fp.value, - imm=-offset, cond=cond) + imm=offset, cond=cond) else: assert 0, 'unsupported case' @@ -1089,15 +1095,15 @@ offset = vfp_loc.value if not check_imm_arg(offset, size=0xFFF): self.mc.PUSH([r.ip.value], cond=cond) - self.mc.gen_load_int(r.ip.value, -offset, cond=cond) + self.mc.gen_load_int(r.ip.value, offset, cond=cond) self.mc.LDR_rr(reg1.value, r.fp.value, r.ip.value, cond=cond) self.mc.ADD_ri(r.ip.value, r.ip.value, imm=WORD, cond=cond) self.mc.LDR_rr(reg2.value, r.fp.value, r.ip.value, cond=cond) self.mc.POP([r.ip.value], cond=cond) else: - self.mc.LDR_ri(reg1.value, r.fp.value, imm=-offset, cond=cond) + self.mc.LDR_ri(reg1.value, r.fp.value, imm=offset, cond=cond) self.mc.LDR_ri(reg2.value, r.fp.value, - imm=-offset + WORD, cond=cond) + imm=offset + WORD, cond=cond) else: assert 0, 'unsupported case' @@ -1112,15 +1118,16 @@ offset = vfp_loc.value if not check_imm_arg(offset, size=0xFFF): self.mc.PUSH([r.ip.value], cond=cond) - self.mc.gen_load_int(r.ip.value, -offset, cond=cond) + self.mc.gen_load_int(r.ip.value, offset, cond=cond) self.mc.STR_rr(reg1.value, r.fp.value, r.ip.value, cond=cond) self.mc.ADD_ri(r.ip.value, r.ip.value, imm=WORD, cond=cond) self.mc.STR_rr(reg2.value, r.fp.value, r.ip.value, cond=cond) self.mc.POP([r.ip.value], cond=cond) else: - self.mc.STR_ri(reg1.value, r.fp.value, imm=-offset, cond=cond) + assert 0, 'verify this code' + self.mc.STR_ri(reg1.value, r.fp.value, imm=offset, cond=cond) self.mc.STR_ri(reg2.value, r.fp.value, - imm=-offset + WORD, cond=cond) + imm=offset + WORD, cond=cond) else: assert 0, 'unsupported case' @@ -1221,21 +1228,17 @@ return 0 def push_gcmap(self, mc, gcmap, push=False, mov=False, store=False): - gcmapref = lltype.cast_opaque_ptr(llmemory.GCREF, gcmap) - # keep the ref alive - self.current_clt.allgcrefs.append(gcmapref) - rgc._make_sure_does_not_move(gcmapref) + ptr = rffi.cast(lltype.Signed, gcmap) if push: - mc.gen_load_int(r.ip.value, rffi.cast(lltype.Signed, gcmapref)) + mc.gen_load_int(r.ip.value, ptr) mc.PUSH([r.ip.value]) elif mov: assert 0 - mc.MOV(RawEspLoc(0, REF), - imm(rffi.cast(lltype.Signed, gcmapref))) + mc.MOV(RawEspLoc(0, REF), ptr) else: assert store ofs = self.cpu.get_ofs_of_frame_field('jf_gcmap') - mc.gen_load_int(r.ip.value, rffi.cast(lltype.Signed, gcmapref)) + mc.gen_load_int(r.ip.value, ptr) mc.STR_ri(r.ip.value, r.fp.value, imm=ofs) def pop_gcmap(self, mc): diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -30,31 +30,31 @@ from rpython.rlib.objectmodel import we_are_translated from rpython.rlib import rgc from rpython.rtyper.lltypesystem import rstr, rffi, lltype, llmemory +from rpython.rlib.rarithmetic import r_uint NO_FORCE_INDEX = -1 class GuardToken(object): - def __init__(self, faildescr, failargs, fail_locs, offset, exc, fcond=c.AL, - is_guard_not_invalidated=False, is_guard_not_forced=False): + def __init__(self, gcmap, faildescr, failargs, fail_locs, offset, exc, + frame_depth, fcond=c.AL, is_guard_not_invalidated=False, + is_guard_not_forced=False): assert isinstance(exc, bool) self.faildescr = faildescr self.failargs = failargs - self.fail_locs = fail_locs[1:] + self.fail_locs = fail_locs self.offset = offset - self.gcmap = self.compute_gcmap(failargs, fail_locs, fail_locs[0].value) + self.gcmap = self.compute_gcmap(gcmap, failargs, + fail_locs, frame_depth) self.exc = exc self.is_guard_not_invalidated = is_guard_not_invalidated self.is_guard_not_forced = is_guard_not_forced self.fcond = fcond - def compute_gcmap(self, failargs, fail_locs, frame_depth): + def compute_gcmap(self, gcmap, failargs, fail_locs, frame_depth): # note that regalloc has a very similar compute, but # one that does iteration over all bindings, so slightly different, # eh - size = frame_depth + JITFRAME_FIXED_SIZE - gcmap = lltype.malloc(jitframe.GCMAP, size // WORD // 8 + 1, - zero=True) input_i = 0 for i in range(len(failargs)): arg = failargs[i] @@ -234,16 +234,32 @@ self.mc.NOP() else: self.mc.BKPT() - self.pending_guards.append(GuardToken(descr, + gcmap = self.allocate_gcmap(arglocs[0].value) + self.pending_guards.append(GuardToken(gcmap, + descr, failargs=op.getfailargs(), - fail_locs=arglocs, + fail_locs=arglocs[1:], offset=pos, exc=save_exc, + frame_depth=arglocs[0].value, is_guard_not_invalidated=is_guard_not_invalidated, is_guard_not_forced=is_guard_not_forced, fcond=fcond)) return c.AL + def allocate_gcmap(self, frame_depth): + size = frame_depth + JITFRAME_FIXED_SIZE + malloc_size = (size // WORD // 8 + 1) + 1 + rawgcmap = self.datablockwrapper.malloc_aligned(WORD * malloc_size, + WORD) + # set the length field + rffi.cast(rffi.CArrayPtr(lltype.Signed), rawgcmap)[0] = malloc_size - 1 + gcmap = rffi.cast(lltype.Ptr(jitframe.GCMAP), rawgcmap) + # zero the area + for i in range(malloc_size - 1): + gcmap[i] = r_uint(0) + return gcmap + def _emit_guard_overflow(self, guard, failargs, fcond): if guard.getopnum() == rop.GUARD_OVERFLOW: fcond = self._emit_guard(guard, failargs, c.VS, save_exc=False) @@ -327,18 +343,18 @@ # so that the locations [ebp+8..] of the input arguments are valid # stack locations both before and after the jump. # - descr = op.getdescr() - assert isinstance(descr, TargetToken) + target_token = op.getdescr() + assert isinstance(target_token, TargetToken) assert fcond == c.AL my_nbargs = self.current_clt._debug_nbargs - target_nbargs = descr._arm_clt._debug_nbargs + target_nbargs = target_token._arm_clt._debug_nbargs assert my_nbargs == target_nbargs self._insert_checks() - if descr in self.target_tokens_currently_compiling: - self.mc.B_offs(descr._arm_loop_code, fcond) + if target_token in self.target_tokens_currently_compiling: + self.mc.B_offs(target_token._arm_loop_code, fcond) else: - self.mc.B(descr._arm_loop_code, fcond) + self.mc.B(target_token._arm_loop_code, fcond) return fcond def emit_op_finish(self, op, arglocs, regalloc, fcond): diff --git a/rpython/jit/backend/arm/runner.py b/rpython/jit/backend/arm/runner.py --- a/rpython/jit/backend/arm/runner.py +++ b/rpython/jit/backend/arm/runner.py @@ -9,6 +9,7 @@ from rpython.rlib.unroll import unrolling_iterable from rpython.rtyper.llinterp import LLInterpreter from rpython.rtyper.lltypesystem import lltype, rffi, llmemory +from rpython.rtyper.lltypesystem.lloperation import llop jitframe.STATICSIZE = JITFRAME_FIXED_SIZE @@ -99,6 +100,9 @@ assert kind == history.REF self.set_ref_value(ll_frame, num, arg) num += WORD + # no GC operation between gc_assume_young_pointers and + # the actual call to assembler! + llop.gc_assume_young_pointers(lltype.Void, frame) ll_frame = func(ll_frame) finally: if not self.translate_support_code: diff --git a/rpython/jit/backend/arm/test/test_arch.py b/rpython/jit/backend/arm/test/test_arch.py --- a/rpython/jit/backend/arm/test/test_arch.py +++ b/rpython/jit/backend/arm/test/test_arch.py @@ -1,23 +1,23 @@ -from rpython.jit.backend.arm import arch +from rpython.jit.backend.arm import support def test_mod(): - assert arch.arm_int_mod(10, 2) == 0 - assert arch.arm_int_mod(11, 2) == 1 - assert arch.arm_int_mod(11, 3) == 2 + assert support.arm_int_mod(10, 2) == 0 + assert support.arm_int_mod(11, 2) == 1 + assert support.arm_int_mod(11, 3) == 2 def test_mod2(): - assert arch.arm_int_mod(-10, 2) == 0 - assert arch.arm_int_mod(-11, 2) == -1 - assert arch.arm_int_mod(-11, 3) == -2 + assert support.arm_int_mod(-10, 2) == 0 + assert support.arm_int_mod(-11, 2) == -1 + assert support.arm_int_mod(-11, 3) == -2 def test_mod3(): - assert arch.arm_int_mod(10, -2) == 0 - assert arch.arm_int_mod(11, -2) == 1 - assert arch.arm_int_mod(11, -3) == 2 + assert support.arm_int_mod(10, -2) == 0 + assert support.arm_int_mod(11, -2) == 1 + assert support.arm_int_mod(11, -3) == 2 def test_div(): - assert arch.arm_int_div(-7, 2) == -3 - assert arch.arm_int_div(9, 2) == 4 - assert arch.arm_int_div(10, 5) == 2 + assert support.arm_int_div(-7, 2) == -3 + assert support.arm_int_div(9, 2) == 4 + assert support.arm_int_div(10, 5) == 2 diff --git a/rpython/jit/backend/arm/test/test_jump.py b/rpython/jit/backend/arm/test/test_jump.py --- a/rpython/jit/backend/arm/test/test_jump.py +++ b/rpython/jit/backend/arm/test/test_jump.py @@ -1,6 +1,5 @@ import random import py -from rpython.jit.backend.x86.test.test_jump import MockAssembler from rpython.jit.backend.arm.registers import * from rpython.jit.backend.arm.locations import * from rpython.jit.backend.arm.regalloc import ARMFrameManager @@ -9,6 +8,35 @@ frame_pos = ARMFrameManager.frame_pos +class MockAssembler: + def __init__(self): + self.ops = [] + + def regalloc_mov(self, from_loc, to_loc): + self.ops.append(('mov', from_loc, to_loc)) + + def regalloc_push(self, loc): + self.ops.append(('push', loc)) + + def regalloc_pop(self, loc): + self.ops.append(('pop', loc)) + + def got(self, expected): + print '------------------------ comparing ---------------------------' + for op1, op2 in zip(self.ops, expected): + print '%-38s| %-38s' % (op1, op2) + if op1 == op2: + continue + assert len(op1) == len(op2) + for x, y in zip(op1, op2): + if isinstance(x, StackLoc) and isinstance(y, MODRM): + assert x.byte == y.byte + assert x.extradata == y.extradata + else: + assert x == y + assert len(self.ops) == len(expected) + return True + class TestJump(object): def setup_method(self, m): self.assembler = MockAssembler() From noreply at buildbot.pypy.org Wed Jan 30 10:29:36 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 30 Jan 2013 10:29:36 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: oops not a great idea Message-ID: <20130130092936.72E351C039A@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60714:0520de970ee0 Date: 2013-01-30 11:29 +0200 http://bitbucket.org/pypy/pypy/changeset/0520de970ee0/ Log: oops not a great idea diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -2502,19 +2502,24 @@ not_implemented("not implemented operation (guard): %s" % op.getopname()) + def check_frame_before_jump(self, target_token): + if target_token in self.target_tokens_currently_compiling: + return + if target_token._x86_clt is self.current_clt: + return + # We can have a frame coming from god knows where that's + # passed to a jump to another loop. Make sure it has the + # correct depth + expected_size = target_token._x86_clt.frame_info.jfi_frame_depth + self._check_frame_depth(self.mc, self._regalloc.get_gcmap(), + expected_size=expected_size) + def closing_jump(self, target_token): target = target_token._x86_loop_code if target_token in self.target_tokens_currently_compiling: curpos = self.mc.get_relative_pos() + 5 self.mc.JMP_l(target - curpos) else: - if target_token._x86_clt is not self.current_clt: - # We can have a frame coming from god knows where that's - # passed to a jump to another loop. Make sure it has the - # correct depth - expected_size = target_token._x86_clt.frame_info.jfi_frame_depth - self._check_frame_depth(self.mc, self._regalloc.get_gcmap(), - expected_size=expected_size) self.mc.JMP(imm(target)) def label(self): diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -1260,6 +1260,7 @@ tmpreg = None xmmtmp = None # Do the remapping + assembler.check_frame_before_jump(self.jump_target_descr) remap_frame_layout_mixed(assembler, src_locations1, dst_locations1, tmpreg, src_locations2, dst_locations2, xmmtmp) From noreply at buildbot.pypy.org Wed Jan 30 10:45:32 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 30 Jan 2013 10:45:32 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: work on stack check Message-ID: <20130130094532.B04C51C03E3@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60715:58fd72df1444 Date: 2013-01-30 11:44 +0200 http://bitbucket.org/pypy/pypy/changeset/58fd72df1444/ Log: work on stack check diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -313,9 +313,8 @@ def _build_stack_check_slowpath(self): _, _, slowpathaddr = self.cpu.insert_stack_check() - #if slowpathaddr == 0 or not self.cpu.propagate_exception_descr: - return # no stack check (for tests, or non-translated) - xxx + if slowpathaddr == 0 or not self.cpu.propagate_exception_descr: + return # no stack check (for tests, or non-translated) # # make a "function" that is called immediately at the start of # an assembler function. In particular, the stack looks like: @@ -327,18 +326,14 @@ # mc = codebuf.MachineCodeBlockWrapper() # - stack_size = WORD if IS_X86_64: # on the x86_64, we have to save all the registers that may - # have been used to pass arguments - stack_size += 6*WORD + 8*8 - for reg in [edi, esi, edx, ecx, r8, r9]: - mc.PUSH_r(reg.value) - mc.SUB_ri(esp.value, 8*8) - for i in range(8): - mc.MOVSD_sx(8*i, i) # xmm0 to xmm7 + # have been used to pass arguments. Note that we pass only + # one argument, that is the frame + mc.PUSH_r(edi.value) # if IS_X86_32: + xxx stack_size += 2*WORD mc.PUSH_r(eax.value) # alignment mc.PUSH_r(esp.value) @@ -354,14 +349,11 @@ jnz_location = mc.get_relative_pos() # if IS_X86_32: + xxxx mc.ADD_ri(esp.value, 2*WORD) # cancel the two PUSHes above elif IS_X86_64: - # restore the registers - for i in range(7, -1, -1): - mc.MOVSD_xs(i, 8*i) - mc.ADD_ri(esp.value, 8*8) - for reg in [r9, r8, ecx, edx, esi, edi]: - mc.POP_r(reg.value) + # restore the edi + mc.POP_r(edi.value) # mc.RET() # @@ -379,7 +371,7 @@ # function, and will instead return to the caller's caller. Note # also that we completely ignore the saved arguments, because we # are interrupting the function. - mc.ADD_ri(esp.value, stack_size) + mc.ADD_ri(esp.value, 2*WORD) mc.RET() # rawstart = mc.materialize(self.cpu.asmmemmgr, []) From noreply at buildbot.pypy.org Wed Jan 30 11:15:53 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 30 Jan 2013 11:15:53 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: fix single floats Message-ID: <20130130101553.9EA571C0041@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60716:586e0157f4d5 Date: 2013-01-30 12:15 +0200 http://bitbucket.org/pypy/pypy/changeset/586e0157f4d5/ Log: fix single floats diff --git a/rpython/jit/backend/test/calling_convention_test.py b/rpython/jit/backend/test/calling_convention_test.py --- a/rpython/jit/backend/test/calling_convention_test.py +++ b/rpython/jit/backend/test/calling_convention_test.py @@ -315,7 +315,6 @@ def test_call_with_singlefloats(self): - py.test.skip("skip for now") cpu = self.cpu if not cpu.supports_floats or not cpu.supports_singlefloats: py.test.skip('requires floats and singlefloats') diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -1241,10 +1241,13 @@ # Load the singlefloat arguments from main regs or stack to xmm regs if singlefloats is not None: for src, dst in singlefloats: - if isinstance(dst, RawEspLoc) and isinstance(src, RawStackLoc): + if isinstance(dst, RawEspLoc): # XXX too much special logic - self.mc.MOV32(X86_64_SCRATCH_REG, src) - self.mc.MOV32(dst, X86_64_SCRATCH_REG) + if isinstance(src, RawStackLoc): + self.mc.MOV32(X86_64_SCRATCH_REG, src) + self.mc.MOV32(dst, X86_64_SCRATCH_REG) + else: + self.mc.MOV32(dst, src) continue if isinstance(src, ImmedLoc): self.mc.MOV(X86_64_SCRATCH_REG, src) From noreply at buildbot.pypy.org Wed Jan 30 11:16:14 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 30 Jan 2013 11:16:14 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: done Message-ID: <20130130101614.B0D3D1C039A@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60717:884b8fe27749 Date: 2013-01-30 12:15 +0200 http://bitbucket.org/pypy/pypy/changeset/884b8fe27749/ Log: done diff --git a/pypy/TODO b/pypy/TODO --- a/pypy/TODO +++ b/pypy/TODO @@ -2,6 +2,4 @@ * 32bit x86 * ARM * asmgcc -* shadowstack details - slowpath of stack check * kill jit2gc on translator -* fix test_singlefloats in test_calling_conventions From noreply at buildbot.pypy.org Wed Jan 30 11:28:22 2013 From: noreply at buildbot.pypy.org (cfbolz) Date: Wed, 30 Jan 2013 11:28:22 +0100 (CET) Subject: [pypy-commit] pypy default: change this test to no longer use a virtual. Message-ID: <20130130102822.6C6B91C039A@cobra.cs.uni-duesseldorf.de> Author: Carl Friedrich Bolz Branch: Changeset: r60718:792dd359c9ac Date: 2013-01-30 11:10 +0100 http://bitbucket.org/pypy/pypy/changeset/792dd359c9ac/ Log: change this test to no longer use a virtual. I checked that it's still testing what it is supposed to, by disabling the logic in make_a_counter_per_value and seeing it fail. diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py --- a/rpython/jit/metainterp/test/test_ajit.py +++ b/rpython/jit/metainterp/test/test_ajit.py @@ -1694,14 +1694,15 @@ assert res == -2 def test_guard_always_changing_value(self): - myjitdriver = JitDriver(greens = [], reds = ['x']) + myjitdriver = JitDriver(greens = [], reds = ['x', 'a']) class A: pass def f(x): + a = 0 while x > 0: - myjitdriver.can_enter_jit(x=x) - myjitdriver.jit_merge_point(x=x) - a = A() + myjitdriver.can_enter_jit(x=x, a=a) + myjitdriver.jit_merge_point(x=x, a=a) + a += 1 promote(a) x -= 1 self.meta_interp(f, [50]) From noreply at buildbot.pypy.org Wed Jan 30 11:32:00 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 30 Jan 2013 11:32:00 +0100 (CET) Subject: [pypy-commit] pypy default: now unused Message-ID: <20130130103200.07B591C03E3@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60719:dd68409a9954 Date: 2013-01-30 05:31 -0500 http://bitbucket.org/pypy/pypy/changeset/dd68409a9954/ Log: now unused diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py --- a/rpython/jit/metainterp/test/test_ajit.py +++ b/rpython/jit/metainterp/test/test_ajit.py @@ -1695,8 +1695,6 @@ def test_guard_always_changing_value(self): myjitdriver = JitDriver(greens = [], reds = ['x', 'a']) - class A: - pass def f(x): a = 0 while x > 0: From noreply at buildbot.pypy.org Wed Jan 30 11:58:23 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 30 Jan 2013 11:58:23 +0100 (CET) Subject: [pypy-commit] pypy stdlib-2.7.4: some new OverflowErrors for socket functions Message-ID: <20130130105823.86FD51C12D8@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: stdlib-2.7.4 Changeset: r60720:b016dc93ead2 Date: 2013-01-30 05:47 -0500 http://bitbucket.org/pypy/pypy/changeset/b016dc93ead2/ Log: some new OverflowErrors for socket functions diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -272,7 +272,7 @@ return space.w_None return space.wrap(timeout) - @unwrap_spec(backlog=int) + @unwrap_spec(backlog="c_int") def listen_w(self, space, backlog): """listen(backlog) @@ -492,7 +492,7 @@ finally: lltype.free(recv_ptr, flavor='raw') - @unwrap_spec(how=int) + @unwrap_spec(how="c_int") def shutdown_w(self, space, how): """shutdown(flag) From noreply at buildbot.pypy.org Wed Jan 30 12:38:42 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 30 Jan 2013 12:38:42 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: shuffle those tests around Message-ID: <20130130113842.6A1021C13D1@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60721:87e655bb59b1 Date: 2013-01-30 12:21 +0200 http://bitbucket.org/pypy/pypy/changeset/87e655bb59b1/ Log: shuffle those tests around diff --git a/rpython/jit/backend/x86/test/test_zrpy_gc.py b/rpython/jit/backend/x86/test/test_zrpy_gc.py --- a/rpython/jit/backend/x86/test/test_zrpy_gc.py +++ b/rpython/jit/backend/x86/test/test_zrpy_gc.py @@ -7,15 +7,13 @@ import weakref import os from rpython.rlib import rgc -from rpython.rtyper.lltypesystem import lltype, llmemory, rffi +from rpython.rtyper.lltypesystem import lltype from rpython.rlib.jit import JitDriver, dont_look_inside from rpython.rlib.jit import elidable, unroll_safe from rpython.jit.backend.llsupport.gc import GcLLDescr_framework from rpython.tool.udir import udir from rpython.config.translationoption import DEFL_GC -from rpython.rlib.libffi import CDLL, types, ArgChain, clibffi -from rpython.rtyper.annlowlevel import llhelper -from rpython.rtyper.lltypesystem.ll2ctypes import libc_name + class X(object): def __init__(self, x=0): @@ -30,18 +28,6 @@ if not flag: raise CheckError -def get_g(main): - main._dont_inline_ = True - def g(name, n): - x = X() - x.foo = 2 - main(n, x) - x.foo = 5 - return weakref.ref(x) - g._dont_inline_ = True - return g - - def get_entry(g): def entrypoint(args): @@ -67,7 +53,6 @@ return entrypoint - def get_functions_to_patch(): from rpython.jit.backend.llsupport import gc # @@ -124,31 +109,6 @@ data = cbuilder.cmdexec(args, env={'PYPYLOG': ':%s' % pypylog}) return data.strip() -def compile_and_run(f, gc, **kwds): - cbuilder = compile(f, gc, **kwds) - return run(cbuilder) - - - -def test_compile_boehm(): - myjitdriver = JitDriver(greens = [], reds = ['n', 'x']) - @dont_look_inside - def see(lst, n): - assert len(lst) == 3 - assert lst[0] == n+10 - assert lst[1] == n+20 - assert lst[2] == n+30 - def main(n, x): - while n > 0: - myjitdriver.can_enter_jit(n=n, x=x) - myjitdriver.jit_merge_point(n=n, x=x) - y = X() - y.foo = x.foo - n -= y.foo - see([n+10, n+20, n+30], n) - res = compile_and_run(get_entry(get_g(main)), "boehm", jit=True) - assert int(res) >= 16 - # ______________________________________________________________________ diff --git a/rpython/jit/backend/x86/test/test_zrpy_gc_boehm.py b/rpython/jit/backend/x86/test/test_zrpy_gc_boehm.py new file mode 100644 --- /dev/null +++ b/rpython/jit/backend/x86/test/test_zrpy_gc_boehm.py @@ -0,0 +1,51 @@ + +import weakref +from rpython.rlib.jit import JitDriver, dont_look_inside +from rpython.jit.backend.x86.test.test_zrpy_gc import run, get_entry + +class X(object): + def __init__(self, x=0): + self.x = x + + next = None + +class CheckError(Exception): + pass + +def check(flag): + if not flag: + raise CheckError + +def compile_and_run(f, gc, **kwds): + cbuilder = compile(f, gc, **kwds) + return run(cbuilder) + +def get_g(main): + main._dont_inline_ = True + def g(name, n): + x = X() + x.foo = 2 + main(n, x) + x.foo = 5 + return weakref.ref(x) + g._dont_inline_ = True + return g + +def test_compile_boehm(): + myjitdriver = JitDriver(greens = [], reds = ['n', 'x']) + @dont_look_inside + def see(lst, n): + assert len(lst) == 3 + assert lst[0] == n+10 + assert lst[1] == n+20 + assert lst[2] == n+30 + def main(n, x): + while n > 0: + myjitdriver.can_enter_jit(n=n, x=x) + myjitdriver.jit_merge_point(n=n, x=x) + y = X() + y.foo = x.foo + n -= y.foo + see([n+10, n+20, n+30], n) + res = compile_and_run(get_entry(get_g(main)), "boehm", jit=True) + assert int(res) >= 16 From noreply at buildbot.pypy.org Wed Jan 30 12:38:43 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 30 Jan 2013 12:38:43 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: oops. a fresh frame *may* contain young pointers. a few debug checks extra Message-ID: <20130130113843.BAAAF1C13D1@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60722:bca909dfddfb Date: 2013-01-30 13:38 +0200 http://bitbucket.org/pypy/pypy/changeset/bca909dfddfb/ Log: oops. a fresh frame *may* contain young pointers. a few debug checks extra diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py --- a/rpython/jit/backend/llsupport/gc.py +++ b/rpython/jit/backend/llsupport/gc.py @@ -139,6 +139,7 @@ """ Allocate a new frame, overwritten by tests """ frame = jitframe.JITFRAME.allocate(frame_info) + llop.gc_assume_young_pointers(lltype.Void, frame) return frame class JitFrameDescrs: diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -1261,7 +1261,8 @@ dst_locs.append(r10) x = r10 remap_frame_layout(self, src_locs, dst_locs, X86_64_SCRATCH_REG) - self.push_gcmap(self.mc, self._regalloc.get_gcmap([eax]), store=True) + self.push_gcmap(self.mc, self._regalloc.get_gcmap([eax], noregs=True), + store=True) self.mc.CALL(x) self._reload_frame_if_necessary(self.mc) if align: diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -749,8 +749,7 @@ # - at least the non-callee-saved registers # # - for shadowstack, we assume that any call can collect, and we - # save also the callee-saved registers that contain GC pointers, - # so that they can be found by follow_stack_frame_of_assembler() + # save also the callee-saved registers that contain GC pointers. # # - for CALL_MAY_FORCE or CALL_ASSEMBLER, we have to save all regs # anyway, in case we need to do cpu.force(). The issue is that @@ -894,13 +893,14 @@ gc_ll_descr.get_nursery_top_addr(), sizeloc, gcmap) - def get_gcmap(self, forbidden_regs=[]): + def get_gcmap(self, forbidden_regs=[], noregs=False): frame_depth = self.fm.get_frame_depth() gcmap = self.assembler.allocate_gcmap(frame_depth) for box, loc in self.rm.reg_bindings.iteritems(): if loc in forbidden_regs: continue if box.type == REF: + assert not noregs assert isinstance(loc, RegLoc) val = gpr_reg_mgr_cls.all_reg_indexes[loc.value] gcmap[val // WORD // 8] |= r_uint(1) << (val % (WORD * 8)) From noreply at buildbot.pypy.org Wed Jan 30 13:04:22 2013 From: noreply at buildbot.pypy.org (bivab) Date: Wed, 30 Jan 2013 13:04:22 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: test_float_operations passes Message-ID: <20130130120422.8A4A41C12DE@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: jitframe-on-heap Changeset: r60723:f410b2ca03ba Date: 2013-01-30 12:59 +0100 http://bitbucket.org/pypy/pypy/changeset/f410b2ca03ba/ Log: test_float_operations passes diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py --- a/rpython/jit/backend/arm/assembler.py +++ b/rpython/jit/backend/arm/assembler.py @@ -1018,9 +1018,9 @@ pushed = True if not check_imm_arg(offset): self.mc.gen_load_int(r.ip.value, offset, cond=cond) - self.mc.SUB_rr(r.ip.value, r.fp.value, r.ip.value, cond=cond) + self.mc.ADD_rr(r.ip.value, r.fp.value, r.ip.value, cond=cond) else: - self.mc.SUB_ri(r.ip.value, r.fp.value, offset, cond=cond) + self.mc.ADD_ri(r.ip.value, r.fp.value, offset, cond=cond) self.mc.VLDR(loc.value, r.ip.value, cond=cond) if pushed: self.mc.POP([r.ip.value], cond=cond) diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -361,12 +361,10 @@ base_ofs = self.cpu.get_baseofs_of_frame_field() - WORD if len(arglocs) == 2: [return_val, fail_descr_loc] = arglocs - if op.getarg(0).type == FLOAT and not IS_X86_64: - XXX - size = WORD * 2 + if op.getarg(0).type == FLOAT: + self.mc.VSTR(return_val.value, r.fp.value)#, imm=-base_ofs) else: - size = WORD - self.mc.STR_ri(return_val.value, r.fp.value)#, imm=-base_ofs) + self.mc.STR_ri(return_val.value, r.fp.value)#, imm=-base_ofs) #self.save_into_mem(raw_stack(0), return_val, imm(size)) else: [fail_descr_loc] = arglocs diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py --- a/rpython/jit/backend/arm/regalloc.py +++ b/rpython/jit/backend/arm/regalloc.py @@ -66,10 +66,7 @@ @staticmethod def frame_pos(i, box_type): - if box_type == FLOAT: - return locations.StackLocation(i, get_fp_offset(i + 1), box_type) - else: - return locations.StackLocation(i, get_fp_offset(i), box_type) + return locations.StackLocation(i, get_fp_offset(i), box_type) @staticmethod def frame_size(type): @@ -317,7 +314,6 @@ # the input args are passed in the jitframe for box in inputargs: assert isinstance(box, Box) - assert box.type != FLOAT self.fm.get_new_loc(box) def _update_bindings(self, locs, inputargs): From noreply at buildbot.pypy.org Wed Jan 30 13:04:23 2013 From: noreply at buildbot.pypy.org (bivab) Date: Wed, 30 Jan 2013 13:04:23 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: merge upstream Message-ID: <20130130120423.C52751C12DE@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: jitframe-on-heap Changeset: r60724:bdaf37b11afb Date: 2013-01-30 13:01 +0100 http://bitbucket.org/pypy/pypy/changeset/bdaf37b11afb/ Log: merge upstream diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py --- a/rpython/jit/backend/llsupport/gc.py +++ b/rpython/jit/backend/llsupport/gc.py @@ -139,6 +139,7 @@ """ Allocate a new frame, overwritten by tests """ frame = jitframe.JITFRAME.allocate(frame_info) + llop.gc_assume_young_pointers(lltype.Void, frame) return frame class JitFrameDescrs: diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -1261,7 +1261,8 @@ dst_locs.append(r10) x = r10 remap_frame_layout(self, src_locs, dst_locs, X86_64_SCRATCH_REG) - self.push_gcmap(self.mc, self._regalloc.get_gcmap([eax]), store=True) + self.push_gcmap(self.mc, self._regalloc.get_gcmap([eax], noregs=True), + store=True) self.mc.CALL(x) self._reload_frame_if_necessary(self.mc) if align: diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -749,8 +749,7 @@ # - at least the non-callee-saved registers # # - for shadowstack, we assume that any call can collect, and we - # save also the callee-saved registers that contain GC pointers, - # so that they can be found by follow_stack_frame_of_assembler() + # save also the callee-saved registers that contain GC pointers. # # - for CALL_MAY_FORCE or CALL_ASSEMBLER, we have to save all regs # anyway, in case we need to do cpu.force(). The issue is that @@ -894,13 +893,14 @@ gc_ll_descr.get_nursery_top_addr(), sizeloc, gcmap) - def get_gcmap(self, forbidden_regs=[]): + def get_gcmap(self, forbidden_regs=[], noregs=False): frame_depth = self.fm.get_frame_depth() gcmap = self.assembler.allocate_gcmap(frame_depth) for box, loc in self.rm.reg_bindings.iteritems(): if loc in forbidden_regs: continue if box.type == REF: + assert not noregs assert isinstance(loc, RegLoc) val = gpr_reg_mgr_cls.all_reg_indexes[loc.value] gcmap[val // WORD // 8] |= r_uint(1) << (val % (WORD * 8)) diff --git a/rpython/jit/backend/x86/test/test_zrpy_gc.py b/rpython/jit/backend/x86/test/test_zrpy_gc.py --- a/rpython/jit/backend/x86/test/test_zrpy_gc.py +++ b/rpython/jit/backend/x86/test/test_zrpy_gc.py @@ -7,15 +7,13 @@ import weakref import os from rpython.rlib import rgc -from rpython.rtyper.lltypesystem import lltype, llmemory, rffi +from rpython.rtyper.lltypesystem import lltype from rpython.rlib.jit import JitDriver, dont_look_inside from rpython.rlib.jit import elidable, unroll_safe from rpython.jit.backend.llsupport.gc import GcLLDescr_framework from rpython.tool.udir import udir from rpython.config.translationoption import DEFL_GC -from rpython.rlib.libffi import CDLL, types, ArgChain, clibffi -from rpython.rtyper.annlowlevel import llhelper -from rpython.rtyper.lltypesystem.ll2ctypes import libc_name + class X(object): def __init__(self, x=0): @@ -30,18 +28,6 @@ if not flag: raise CheckError -def get_g(main): - main._dont_inline_ = True - def g(name, n): - x = X() - x.foo = 2 - main(n, x) - x.foo = 5 - return weakref.ref(x) - g._dont_inline_ = True - return g - - def get_entry(g): def entrypoint(args): @@ -67,7 +53,6 @@ return entrypoint - def get_functions_to_patch(): from rpython.jit.backend.llsupport import gc # @@ -124,31 +109,6 @@ data = cbuilder.cmdexec(args, env={'PYPYLOG': ':%s' % pypylog}) return data.strip() -def compile_and_run(f, gc, **kwds): - cbuilder = compile(f, gc, **kwds) - return run(cbuilder) - - - -def test_compile_boehm(): - myjitdriver = JitDriver(greens = [], reds = ['n', 'x']) - @dont_look_inside - def see(lst, n): - assert len(lst) == 3 - assert lst[0] == n+10 - assert lst[1] == n+20 - assert lst[2] == n+30 - def main(n, x): - while n > 0: - myjitdriver.can_enter_jit(n=n, x=x) - myjitdriver.jit_merge_point(n=n, x=x) - y = X() - y.foo = x.foo - n -= y.foo - see([n+10, n+20, n+30], n) - res = compile_and_run(get_entry(get_g(main)), "boehm", jit=True) - assert int(res) >= 16 - # ______________________________________________________________________ diff --git a/rpython/jit/backend/x86/test/test_zrpy_gc_boehm.py b/rpython/jit/backend/x86/test/test_zrpy_gc_boehm.py new file mode 100644 --- /dev/null +++ b/rpython/jit/backend/x86/test/test_zrpy_gc_boehm.py @@ -0,0 +1,51 @@ + +import weakref +from rpython.rlib.jit import JitDriver, dont_look_inside +from rpython.jit.backend.x86.test.test_zrpy_gc import run, get_entry + +class X(object): + def __init__(self, x=0): + self.x = x + + next = None + +class CheckError(Exception): + pass + +def check(flag): + if not flag: + raise CheckError + +def compile_and_run(f, gc, **kwds): + cbuilder = compile(f, gc, **kwds) + return run(cbuilder) + +def get_g(main): + main._dont_inline_ = True + def g(name, n): + x = X() + x.foo = 2 + main(n, x) + x.foo = 5 + return weakref.ref(x) + g._dont_inline_ = True + return g + +def test_compile_boehm(): + myjitdriver = JitDriver(greens = [], reds = ['n', 'x']) + @dont_look_inside + def see(lst, n): + assert len(lst) == 3 + assert lst[0] == n+10 + assert lst[1] == n+20 + assert lst[2] == n+30 + def main(n, x): + while n > 0: + myjitdriver.can_enter_jit(n=n, x=x) + myjitdriver.jit_merge_point(n=n, x=x) + y = X() + y.foo = x.foo + n -= y.foo + see([n+10, n+20, n+30], n) + res = compile_and_run(get_entry(get_g(main)), "boehm", jit=True) + assert int(res) >= 16 From noreply at buildbot.pypy.org Wed Jan 30 13:29:12 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 30 Jan 2013 13:29:12 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: this is nonsense Message-ID: <20130130122912.2735E1C03E3@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60725:a6da35e6d791 Date: 2013-01-30 14:21 +0200 http://bitbucket.org/pypy/pypy/changeset/a6da35e6d791/ Log: this is nonsense diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py --- a/rpython/jit/backend/llsupport/gc.py +++ b/rpython/jit/backend/llsupport/gc.py @@ -139,7 +139,6 @@ """ Allocate a new frame, overwritten by tests """ frame = jitframe.JITFRAME.allocate(frame_info) - llop.gc_assume_young_pointers(lltype.Void, frame) return frame class JitFrameDescrs: From noreply at buildbot.pypy.org Wed Jan 30 13:29:13 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 30 Jan 2013 13:29:13 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: some basic sanity check that seem to fail Message-ID: <20130130122913.5D4151C03E3@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60726:76fcfafc2c24 Date: 2013-01-30 14:24 +0200 http://bitbucket.org/pypy/pypy/changeset/76fcfafc2c24/ Log: some basic sanity check that seem to fail diff --git a/rpython/jit/backend/llsupport/jitframe.py b/rpython/jit/backend/llsupport/jitframe.py --- a/rpython/jit/backend/llsupport/jitframe.py +++ b/rpython/jit/backend/llsupport/jitframe.py @@ -81,6 +81,7 @@ GCMAPLENGTHOFS = llmemory.arraylengthoffset(GCMAP) GCMAPBASEOFS = llmemory.itemoffsetof(GCMAP, 0) BASEITEMOFS = llmemory.itemoffsetof(JITFRAME.jf_frame, 0) +LENGTHOFS = llmemory.arraylengthoffset(JITFRAME.jf_frame) SIGN_SIZE = llmemory.sizeof(lltype.Signed) UNSIGN_SIZE = llmemory.sizeof(lltype.Unsigned) @@ -128,8 +129,12 @@ else: new_state = 3 | ((state + 1) << 3) | (no << 9) (obj_addr + getofs('jf_gc_trace_state')).signed[0] = new_state + index = no * SIZEOFSIGNED * 8 + state + # sanity check + frame_lgt = (obj_addr + getofs('jf_frame') + LENGTHOFS).signed[0] + ll_assert(index < frame_lgt, "bogus frame field get") return (obj_addr + getofs('jf_frame') + BASEITEMOFS + SIGN_SIZE * - (no * SIZEOFSIGNED * 8 + state)) + (index)) no += 1 state = 0 return llmemory.NULL diff --git a/rpython/jit/backend/llsupport/test/test_gc.py b/rpython/jit/backend/llsupport/test/test_gc.py --- a/rpython/jit/backend/llsupport/test/test_gc.py +++ b/rpython/jit/backend/llsupport/test/test_gc.py @@ -272,7 +272,7 @@ jitframe.BASEITEMOFS + jitframe.SIGN_SIZE * no) frame_info = lltype.malloc(jitframe.JITFRAMEINFO, zero=True, flavor='raw') - frame = lltype.malloc(jitframe.JITFRAME, 15, zero=True) + frame = lltype.malloc(jitframe.JITFRAME, 100, zero=True) frame.jf_frame_info = frame_info frame.jf_gcmap = lltype.malloc(jitframe.GCMAP, 2, flavor='raw') if sys.maxint == 2**31 - 1: From noreply at buildbot.pypy.org Wed Jan 30 13:29:14 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 30 Jan 2013 13:29:14 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: merge Message-ID: <20130130122914.8D2C01C03E3@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60727:9d5504affe3e Date: 2013-01-30 14:28 +0200 http://bitbucket.org/pypy/pypy/changeset/9d5504affe3e/ Log: merge diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py --- a/rpython/jit/backend/arm/assembler.py +++ b/rpython/jit/backend/arm/assembler.py @@ -1018,9 +1018,9 @@ pushed = True if not check_imm_arg(offset): self.mc.gen_load_int(r.ip.value, offset, cond=cond) - self.mc.SUB_rr(r.ip.value, r.fp.value, r.ip.value, cond=cond) + self.mc.ADD_rr(r.ip.value, r.fp.value, r.ip.value, cond=cond) else: - self.mc.SUB_ri(r.ip.value, r.fp.value, offset, cond=cond) + self.mc.ADD_ri(r.ip.value, r.fp.value, offset, cond=cond) self.mc.VLDR(loc.value, r.ip.value, cond=cond) if pushed: self.mc.POP([r.ip.value], cond=cond) diff --git a/rpython/jit/backend/arm/opassembler.py b/rpython/jit/backend/arm/opassembler.py --- a/rpython/jit/backend/arm/opassembler.py +++ b/rpython/jit/backend/arm/opassembler.py @@ -361,12 +361,10 @@ base_ofs = self.cpu.get_baseofs_of_frame_field() - WORD if len(arglocs) == 2: [return_val, fail_descr_loc] = arglocs - if op.getarg(0).type == FLOAT and not IS_X86_64: - XXX - size = WORD * 2 + if op.getarg(0).type == FLOAT: + self.mc.VSTR(return_val.value, r.fp.value)#, imm=-base_ofs) else: - size = WORD - self.mc.STR_ri(return_val.value, r.fp.value)#, imm=-base_ofs) + self.mc.STR_ri(return_val.value, r.fp.value)#, imm=-base_ofs) #self.save_into_mem(raw_stack(0), return_val, imm(size)) else: [fail_descr_loc] = arglocs diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py --- a/rpython/jit/backend/arm/regalloc.py +++ b/rpython/jit/backend/arm/regalloc.py @@ -66,10 +66,7 @@ @staticmethod def frame_pos(i, box_type): - if box_type == FLOAT: - return locations.StackLocation(i, get_fp_offset(i + 1), box_type) - else: - return locations.StackLocation(i, get_fp_offset(i), box_type) + return locations.StackLocation(i, get_fp_offset(i), box_type) @staticmethod def frame_size(type): @@ -317,7 +314,6 @@ # the input args are passed in the jitframe for box in inputargs: assert isinstance(box, Box) - assert box.type != FLOAT self.fm.get_new_loc(box) def _update_bindings(self, locs, inputargs): From noreply at buildbot.pypy.org Wed Jan 30 14:00:08 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 30 Jan 2013 14:00:08 +0100 (CET) Subject: [pypy-commit] pypy default: more closely imitate cpython handling of file descriptors Message-ID: <20130130130008.ED2621C12DE@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60728:63ac31bd95de Date: 2013-01-30 07:59 -0500 http://bitbucket.org/pypy/pypy/changeset/63ac31bd95de/ Log: more closely imitate cpython handling of file descriptors diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -1482,9 +1482,10 @@ ) raise w_fd = self.call_function(w_fileno) - if not self.isinstance_w(w_fd, self.w_int): + if (not self.isinstance_w(w_fd, self.w_int) and + not self.isinstance_w(w_fd, self.w_long)): raise OperationError(self.w_TypeError, - self.wrap("fileno() must return an integer") + self.wrap("fileno() returned a non-integer") ) fd = self.int_w(w_fd) if fd < 0: diff --git a/pypy/module/fcntl/test/test_fcntl.py b/pypy/module/fcntl/test/test_fcntl.py --- a/pypy/module/fcntl/test/test_fcntl.py +++ b/pypy/module/fcntl/test/test_fcntl.py @@ -22,13 +22,23 @@ import sys import struct + class F: + def __init__(self, fn): + self.fn = fn + def fileno(self): + return self.fn + f = open(self.tmp + "b", "w+") fcntl.fcntl(f, 1, 0) fcntl.fcntl(f, 1) + fcntl.fcntl(F(long(f.fileno())), 1) raises(TypeError, fcntl.fcntl, "foo") raises(TypeError, fcntl.fcntl, f, "foo") - raises((IOError, ValueError), fcntl.fcntl, -1, 1, 0) + raises(TypeError, fcntl.fcntl, F("foo"), 1) + raises(ValueError, fcntl.fcntl, -1, 1, 0) + raises(ValueError, fcntl.fcntl, F(-1), 1, 0) + raises(ValueError, fcntl.fcntl, F(long(-1)), 1, 0) assert fcntl.fcntl(f, 1, 0) == 0 assert fcntl.fcntl(f, 2, "foo") == "foo" assert fcntl.fcntl(f, 2, buffer("foo")) == "foo" From noreply at buildbot.pypy.org Wed Jan 30 14:08:52 2013 From: noreply at buildbot.pypy.org (mattip) Date: Wed, 30 Jan 2013 14:08:52 +0100 (CET) Subject: [pypy-commit] pypy missing-ndarray-attributes: detour around strange translation problem Message-ID: <20130130130852.924F11C12DE@cobra.cs.uni-duesseldorf.de> Author: mattip Branch: missing-ndarray-attributes Changeset: r60729:2bdcf6e1287e Date: 2013-01-30 15:09 +0200 http://bitbucket.org/pypy/pypy/changeset/2bdcf6e1287e/ Log: detour around strange translation problem diff --git a/pypy/module/micronumpy/arrayimpl/sort.py b/pypy/module/micronumpy/arrayimpl/sort.py --- a/pypy/module/micronumpy/arrayimpl/sort.py +++ b/pypy/module/micronumpy/arrayimpl/sort.py @@ -82,12 +82,12 @@ def argsort_array(arr, space, w_axis): itemtype = arr.dtype.itemtype - if isinstance(itemtype, types.Float) or \ - isinstance(itemtype, types.Integer) or \ - isinstance(itemtype, types.ComplexFloating): - pass - else: - raise OperationError(space.w_NotImplementedError, + ok_to_continue = isinstance(itemtype, types.Float) + if isinstance(itemtype, types.Integer): + ok_to_continue = True + if isinstance(itemtype, types.ComplexFloating): + ok_to_continue = True + if not ok_to_continue: space.wrap("sorting of non-numeric types " + \ "'%s' is not implemented" % arr.dtype.get_name() )) if w_axis is space.w_None: From noreply at buildbot.pypy.org Wed Jan 30 15:57:13 2013 From: noreply at buildbot.pypy.org (bivab) Date: Wed, 30 Jan 2013 15:57:13 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: support for floats in failargs Message-ID: <20130130145713.985D71C0041@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: jitframe-on-heap Changeset: r60730:e71902fae352 Date: 2013-01-30 15:49 +0100 http://bitbucket.org/pypy/pypy/changeset/e71902fae352/ Log: support for floats in failargs diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py --- a/rpython/jit/backend/arm/assembler.py +++ b/rpython/jit/backend/arm/assembler.py @@ -8,9 +8,10 @@ N_REGISTERS_SAVED_BY_MALLOC, \ JITFRAME_FIXED_SIZE, FRAME_FIXED_SIZE from rpython.jit.backend.arm.codebuilder import ARMv7Builder, OverwritingBuilder -from rpython.jit.backend.arm.locations import get_fp_offset, imm +from rpython.jit.backend.arm.locations import get_fp_offset, imm, StackLocation from rpython.jit.backend.arm.regalloc import (Regalloc, ARMFrameManager, CoreRegisterManager, check_imm_arg, + VFPRegisterManager, operations as regalloc_operations, operations_with_guard as regalloc_operations_with_guard) from rpython.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper @@ -373,7 +374,7 @@ # see ../x86/assembler.py:propagate_memoryerror_if_eax_is_null self.mc.CMP_ri(r.r0.value, 0) self.mc.B(self.propagate_exception_path, c=c.EQ) - + def _push_all_regs_to_jitframe(self, mc, ignored_regs, withfloats, callee_only=False): if callee_only: @@ -386,7 +387,15 @@ continue mc.STR_ri(gpr.value, r.fp.value, i * WORD) if withfloats: - assert 0, 'implement me' + if callee_only: + regs = VFPRegisterManager.save_around_call_regs + else: + regs = VFPRegisterManager.all_regs + for i, vfpr in enumerate(regs): + if vfpr in ignored_regs: + continue + # add the offset of the gpr_regs + mc.VSTR(vfpr.value, r.fp.value, imm=i * DOUBLE_WORD) def _build_failure_recovery(self, exc, withfloats=False): mc = ARMv7Builder() @@ -497,9 +506,8 @@ assert loc is not r.fp # for now v = loc.value else: - assert 0, 'fix for floats' assert loc.is_vfp_reg() - #v = len(VFPRegisterManager.all_regs) + loc.value + v = len(CoreRegisterManager.all_regs) + loc.value * 2 positions[i] = v * WORD # write down the positions of locs guardtok.faildescr.rd_locs = positions From noreply at buildbot.pypy.org Wed Jan 30 15:57:14 2013 From: noreply at buildbot.pypy.org (bivab) Date: Wed, 30 Jan 2013 15:57:14 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: bridges start to work again Message-ID: <20130130145714.C496C1C0041@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: jitframe-on-heap Changeset: r60731:281e082e0a08 Date: 2013-01-30 15:50 +0100 http://bitbucket.org/pypy/pypy/changeset/281e082e0a08/ Log: bridges start to work again diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py --- a/rpython/jit/backend/arm/assembler.py +++ b/rpython/jit/backend/arm/assembler.py @@ -688,31 +688,35 @@ def assemble_bridge(self, faildescr, inputargs, operations, original_loop_token, log): - assert 0 - operations = self.setup(original_loop_token, operations) - descr_number = self.cpu.get_fail_descr_number(faildescr) + if not we_are_translated(): + # Arguments should be unique + assert len(set(inputargs)) == len(inputargs) + + self.setup(original_loop_token) + descr_number = compute_unique_id(faildescr) if log: operations = self._inject_debugging_code(faildescr, operations, 'b', descr_number) + assert isinstance(faildescr, AbstractFailDescr) - code = self._find_failure_recovery_bytecode(faildescr) - frame_depth = faildescr._arm_current_frame_depth - arglocs = self.decode_inputargs(code) - if not we_are_translated(): - assert len(inputargs) == len(arglocs) - regalloc = Regalloc(assembler=self, frame_manager=ARMFrameManager()) - regalloc.prepare_bridge(inputargs, arglocs, operations) + arglocs = self.rebuild_faillocs_from_descr(faildescr, inputargs) - sp_patch_location = self._prepare_sp_patch_position() + regalloc = Regalloc(assembler=self) + operations = regalloc.prepare_bridge(inputargs, arglocs, + operations, + self.current_clt.allgcrefs, + self.current_clt.frame_info) + + #sp_patch_location = self._prepare_sp_patch_position() startpos = self.mc.get_relative_pos() - frame_depth = self._assemble(operations, regalloc) + frame_depth = self._assemble(regalloc, inputargs, operations) codeendpos = self.mc.get_relative_pos() - self._patch_sp_offset(sp_patch_location, frame_depth) + #self._patch_sp_offset(sp_patch_location, frame_depth) self.write_pending_failure_recoveries() @@ -721,6 +725,7 @@ self.process_pending_guards(rawstart) self.fixup_target_tokens(rawstart) + # patch the jump from original guard self.patch_trace(faildescr, original_loop_token, rawstart, regalloc) @@ -728,10 +733,9 @@ # for the benefit of tests faildescr._arm_bridge_frame_depth = frame_depth if log: - self.mc._dump_trace(rawstart, 'bridge_%d.asm' % - self.cpu.total_compiled_bridges) - self.current_clt.frame_depth = max(self.current_clt.frame_depth, - frame_depth) + self.mc._dump_trace(rawstart, 'bridge.asm') + frame_depth = max(self.current_clt.frame_info.jfi_frame_depth, + frame_depth + JITFRAME_FIXED_SIZE) ops_offset = self.mc.ops_offset self.teardown() @@ -742,15 +746,26 @@ return AsmInfo(ops_offset, startpos + rawstart, codeendpos - startpos) - def _find_failure_recovery_bytecode(self, faildescr): - guard_stub_addr = faildescr._arm_failure_recovery_block - if guard_stub_addr == 0: - # This case should be prevented by the logic in compile.py: - # look for CNT_BUSY_FLAG, which disables tracing from a guard - # when another tracing from the same guard is already in progress. - raise BridgeAlreadyCompiled - # a guard requires 3 words to encode the jump to the exit code. - return guard_stub_addr + 3 * WORD + def rebuild_faillocs_from_descr(self, descr, inputargs): + locs = [] + GPR_REGS = len(CoreRegisterManager.all_regs) + VFP_REGS = len(VFPRegisterManager.all_regs) + input_i = 0 + for pos in descr.rd_locs: + if pos == -1: + continue + elif pos < GPR_REGS * WORD: + locs.append(CoreRegisterManager.all_regs[pos // WORD]) + elif pos < (GPR_REGS * WORD + VFP_REGS * DOUBLE_WORD): + pos = pos // DOUBLE_WORD - GPR_REGS * WORD // DOUBLE_WORD + locs.append(VFPRegisterManager.all_regs[pos]) + else: + i = pos // WORD - JITFRAME_FIXED_SIZE + assert i >= 0 + tp = inputargs[input_i].type + locs.append(StackLocation(i, pos, tp)) + input_i += 1 + return locs def fixup_target_tokens(self, rawstart): for targettoken in self.target_tokens_currently_compiling: diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py --- a/rpython/jit/backend/arm/regalloc.py +++ b/rpython/jit/backend/arm/regalloc.py @@ -303,9 +303,12 @@ self.possibly_free_vars(list(inputargs)) return operations - def prepare_bridge(self, inputargs, arglocs, ops): - self._prepare(inputargs, ops) + def prepare_bridge(self, inputargs, arglocs, operations, allgcrefs, + frame_info): + operations = self._prepare(inputargs, operations, allgcrefs) self._update_bindings(arglocs, inputargs) + return operations + def get_final_frame_depth(self): return self.frame_manager.get_frame_depth() @@ -320,16 +323,19 @@ used = {} i = 0 for loc in locs: + if loc is None: + loc = r.fp arg = inputargs[i] i += 1 if loc.is_reg(): self.rm.reg_bindings[arg] = loc + used[loc] = None elif loc.is_vfp_reg(): self.vfprm.reg_bindings[arg] = loc + used[loc] = None else: assert loc.is_stack() - self.frame_manager.set_binding(arg, loc) - used[loc] = None + self.frame_manager.bind(arg, loc) # XXX combine with x86 code and move to llsupport self.rm.free_regs = [] From noreply at buildbot.pypy.org Wed Jan 30 15:57:16 2013 From: noreply at buildbot.pypy.org (bivab) Date: Wed, 30 Jan 2013 15:57:16 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: merge Message-ID: <20130130145716.073061C0041@cobra.cs.uni-duesseldorf.de> Author: David Schneider Branch: jitframe-on-heap Changeset: r60732:8b3593ba24a1 Date: 2013-01-30 15:55 +0100 http://bitbucket.org/pypy/pypy/changeset/8b3593ba24a1/ Log: merge diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py --- a/rpython/jit/backend/llsupport/gc.py +++ b/rpython/jit/backend/llsupport/gc.py @@ -139,7 +139,6 @@ """ Allocate a new frame, overwritten by tests """ frame = jitframe.JITFRAME.allocate(frame_info) - llop.gc_assume_young_pointers(lltype.Void, frame) return frame class JitFrameDescrs: diff --git a/rpython/jit/backend/llsupport/jitframe.py b/rpython/jit/backend/llsupport/jitframe.py --- a/rpython/jit/backend/llsupport/jitframe.py +++ b/rpython/jit/backend/llsupport/jitframe.py @@ -81,6 +81,7 @@ GCMAPLENGTHOFS = llmemory.arraylengthoffset(GCMAP) GCMAPBASEOFS = llmemory.itemoffsetof(GCMAP, 0) BASEITEMOFS = llmemory.itemoffsetof(JITFRAME.jf_frame, 0) +LENGTHOFS = llmemory.arraylengthoffset(JITFRAME.jf_frame) SIGN_SIZE = llmemory.sizeof(lltype.Signed) UNSIGN_SIZE = llmemory.sizeof(lltype.Unsigned) @@ -128,8 +129,12 @@ else: new_state = 3 | ((state + 1) << 3) | (no << 9) (obj_addr + getofs('jf_gc_trace_state')).signed[0] = new_state + index = no * SIZEOFSIGNED * 8 + state + # sanity check + frame_lgt = (obj_addr + getofs('jf_frame') + LENGTHOFS).signed[0] + ll_assert(index < frame_lgt, "bogus frame field get") return (obj_addr + getofs('jf_frame') + BASEITEMOFS + SIGN_SIZE * - (no * SIZEOFSIGNED * 8 + state)) + (index)) no += 1 state = 0 return llmemory.NULL diff --git a/rpython/jit/backend/llsupport/test/test_gc.py b/rpython/jit/backend/llsupport/test/test_gc.py --- a/rpython/jit/backend/llsupport/test/test_gc.py +++ b/rpython/jit/backend/llsupport/test/test_gc.py @@ -272,7 +272,7 @@ jitframe.BASEITEMOFS + jitframe.SIGN_SIZE * no) frame_info = lltype.malloc(jitframe.JITFRAMEINFO, zero=True, flavor='raw') - frame = lltype.malloc(jitframe.JITFRAME, 15, zero=True) + frame = lltype.malloc(jitframe.JITFRAME, 100, zero=True) frame.jf_frame_info = frame_info frame.jf_gcmap = lltype.malloc(jitframe.GCMAP, 2, flavor='raw') if sys.maxint == 2**31 - 1: From noreply at buildbot.pypy.org Wed Jan 30 17:15:13 2013 From: noreply at buildbot.pypy.org (rlamy) Date: Wed, 30 Jan 2013 17:15:13 +0100 (CET) Subject: [pypy-commit] pypy kill-flowobjspace: move ArgumentsForTranslation and friends to rpython/annotator/ Message-ID: <20130130161513.DBE2B1C0041@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: kill-flowobjspace Changeset: r60733:c0328610075e Date: 2013-01-28 04:17 +0000 http://bitbucket.org/pypy/pypy/changeset/c0328610075e/ Log: move ArgumentsForTranslation and friends to rpython/annotator/ diff --git a/rpython/flowspace/argument.py b/rpython/annotator/argument.py copy from rpython/flowspace/argument.py copy to rpython/annotator/argument.py --- a/rpython/flowspace/argument.py +++ b/rpython/annotator/argument.py @@ -1,77 +1,6 @@ """ Arguments objects. """ - -class Signature(object): - _immutable_ = True - _immutable_fields_ = ["argnames[*]"] - __slots__ = ("argnames", "varargname", "kwargname") - - def __init__(self, argnames, varargname=None, kwargname=None): - self.argnames = argnames - self.varargname = varargname - self.kwargname = kwargname - - def find_argname(self, name): - try: - return self.argnames.index(name) - except ValueError: - return -1 - - def num_argnames(self): - return len(self.argnames) - - def has_vararg(self): - return self.varargname is not None - - def has_kwarg(self): - return self.kwargname is not None - - def scope_length(self): - scopelen = len(self.argnames) - scopelen += self.has_vararg() - scopelen += self.has_kwarg() - return scopelen - - def getallvarnames(self): - argnames = self.argnames - if self.varargname is not None: - argnames = argnames + [self.varargname] - if self.kwargname is not None: - argnames = argnames + [self.kwargname] - return argnames - - def __repr__(self): - return "Signature(%r, %r, %r)" % ( - self.argnames, self.varargname, self.kwargname) - - def __eq__(self, other): - if not isinstance(other, Signature): - return NotImplemented - return (self.argnames == other.argnames and - self.varargname == other.varargname and - self.kwargname == other.kwargname) - - def __ne__(self, other): - if not isinstance(other, Signature): - return NotImplemented - return not self == other - - - # make it look tuply for its use in the annotator - - def __len__(self): - return 3 - - def __getitem__(self, i): - if i == 0: - return self.argnames - if i == 1: - return self.varargname - if i == 2: - return self.kwargname - raise IndexError - class ArgumentsForTranslation(object): def __init__(self, space, args_w, keywords=None, keywords_w=None, w_stararg=None, w_starstararg=None): @@ -374,41 +303,6 @@ return args._rawshape(nextra) -class CallSpec(object): - """Represents the arguments passed into a function call, i.e. the - `a, b, *c, **d` part in `return func(a, b, *c, **d)`. - """ - def __init__(self, space, args_w, keywords=None, w_stararg=None, - w_starstararg=None): - self.w_stararg = w_stararg - assert w_starstararg is None, "No **-unpacking in RPython" - self.combine_has_happened = False - self.space = space - assert isinstance(args_w, list) - self.arguments_w = args_w - self.keywords = keywords or {} - - def unpack(self): - "Return a ([w1,w2...], {'kw':w3...}) pair." - if self.w_stararg is not None: - stargs_w = self.space.unpackiterable(self.w_stararg) - args_w = self.arguments_w + stargs_w - else: - args_w = self.arguments_w - return args_w, self.keywords - - def flatten(self): - """ Argument <-> list of w_objects together with "shape" information """ - shape_cnt = len(self.arguments_w) # Number of positional args - shape_keys = tuple(sorted(self.keywords)) - shape_star = self.w_stararg is not None # Flag: presence of *arg - shape_stst = False # Flag: presence of **kwds - data_w = self.arguments_w + [self.keywords[key] for key in shape_keys] - if shape_star: - data_w.append(self.w_stararg) - return (shape_cnt, shape_keys, shape_star, shape_stst), data_w - - # # ArgErr family of exceptions raised in case of argument mismatch. # We try to give error messages following CPython's, which are very informative. diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py --- a/rpython/annotator/bookkeeper.py +++ b/rpython/annotator/bookkeeper.py @@ -19,7 +19,7 @@ from rpython.annotator.dictdef import DictDef from rpython.annotator import description from rpython.annotator.signature import annotationoftype -from rpython.flowspace.argument import ArgumentsForTranslation +from rpython.annotator.argument import ArgumentsForTranslation from rpython.rlib.objectmodel import r_dict, Symbolic from rpython.tool.algo.unionfind import UnionFind from rpython.rtyper.lltypesystem import lltype, llmemory diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py --- a/rpython/annotator/description.py +++ b/rpython/annotator/description.py @@ -3,7 +3,7 @@ from rpython.annotator.signature import enforce_signature_args, enforce_signature_return from rpython.flowspace.model import Constant, FunctionGraph from rpython.flowspace.bytecode import cpython_code_signature -from rpython.flowspace.argument import rawshape, ArgErr +from rpython.annotator.argument import rawshape, ArgErr from rpython.tool.sourcetools import valid_identifier from rpython.tool.pairtype import extendabletype diff --git a/rpython/flowspace/test/test_argument.py b/rpython/annotator/test/test_argument.py copy from rpython/flowspace/test/test_argument.py copy to rpython/annotator/test/test_argument.py --- a/rpython/flowspace/test/test_argument.py +++ b/rpython/annotator/test/test_argument.py @@ -1,55 +1,7 @@ # -*- coding: utf-8 -*- import py -from rpython.flowspace.argument import (ArgumentsForTranslation, rawshape, - Signature) - - -class TestSignature(object): - def test_helpers(self): - sig = Signature(["a", "b", "c"], None, None) - assert sig.num_argnames() == 3 - assert not sig.has_vararg() - assert not sig.has_kwarg() - assert sig.scope_length() == 3 - assert sig.getallvarnames() == ["a", "b", "c"] - sig = Signature(["a", "b", "c"], "c", None) - assert sig.num_argnames() == 3 - assert sig.has_vararg() - assert not sig.has_kwarg() - assert sig.scope_length() == 4 - assert sig.getallvarnames() == ["a", "b", "c", "c"] - sig = Signature(["a", "b", "c"], None, "c") - assert sig.num_argnames() == 3 - assert not sig.has_vararg() - assert sig.has_kwarg() - assert sig.scope_length() == 4 - assert sig.getallvarnames() == ["a", "b", "c", "c"] - sig = Signature(["a", "b", "c"], "d", "c") - assert sig.num_argnames() == 3 - assert sig.has_vararg() - assert sig.has_kwarg() - assert sig.scope_length() == 5 - assert sig.getallvarnames() == ["a", "b", "c", "d", "c"] - - def test_eq(self): - sig1 = Signature(["a", "b", "c"], "d", "c") - sig2 = Signature(["a", "b", "c"], "d", "c") - assert sig1 == sig2 - - - def test_find_argname(self): - sig = Signature(["a", "b", "c"], None, None) - assert sig.find_argname("a") == 0 - assert sig.find_argname("b") == 1 - assert sig.find_argname("c") == 2 - assert sig.find_argname("d") == -1 - - def test_tuply(self): - sig = Signature(["a", "b", "c"], "d", "e") - x, y, z = sig - assert x == ["a", "b", "c"] - assert y == "d" - assert z == "e" +from rpython.annotator.argument import ArgumentsForTranslation, rawshape +from rpython.flowspace.argument import Signature class dummy_wrapped_dict(dict): def __nonzero__(self): diff --git a/rpython/flowspace/argument.py b/rpython/flowspace/argument.py --- a/rpython/flowspace/argument.py +++ b/rpython/flowspace/argument.py @@ -72,307 +72,6 @@ return self.kwargname raise IndexError -class ArgumentsForTranslation(object): - def __init__(self, space, args_w, keywords=None, keywords_w=None, - w_stararg=None, w_starstararg=None): - self.w_stararg = w_stararg - self.w_starstararg = w_starstararg - self.combine_has_happened = False - self.space = space - assert isinstance(args_w, list) - self.arguments_w = args_w - self.keywords = keywords - self.keywords_w = keywords_w - self.keyword_names_w = None - - def __repr__(self): - """ NOT_RPYTHON """ - name = self.__class__.__name__ - if not self.keywords: - return '%s(%s)' % (name, self.arguments_w,) - else: - return '%s(%s, %s, %s)' % (name, self.arguments_w, - self.keywords, self.keywords_w) - - def _combine_wrapped(self, w_stararg, w_starstararg): - "unpack the *arg and **kwd into arguments_w and keywords_w" - if w_stararg is not None: - self._combine_starargs_wrapped(w_stararg) - if w_starstararg is not None: - self._combine_starstarargs_wrapped(w_starstararg) - - def _combine_starargs_wrapped(self, w_stararg): - # unpack the * arguments - space = self.space - args_w = space.unpackiterable(w_stararg) - self.arguments_w = self.arguments_w + args_w - - def _combine_starstarargs_wrapped(self, w_starstararg): - # unpack the ** arguments - space = self.space - keywords, values_w = space.view_as_kwargs(w_starstararg) - if keywords is not None: # this path also taken for empty dicts - if self.keywords is None: - self.keywords = keywords - self.keywords_w = values_w - else: - if set(keywords) & set(self.keywords): - raise TypeError("got multiple values for keyword arguments '%s'", set(keywords) & set(self.keywords)) - self.keywords = self.keywords + keywords - self.keywords_w = self.keywords_w + values_w - return - if space.isinstance_w(w_starstararg, space.w_dict): - keys_w = space.unpackiterable(w_starstararg) - else: - w_keys = space.call_method(w_starstararg, "keys") - keys_w = space.unpackiterable(w_keys) - keywords_w = [None] * len(keys_w) - keywords = [None] * len(keys_w) - for i, w_key in enumerate(keys_w): - key = space.str_w(w_key) - if key in self.keywords: - raise TypeError("got multiple values for keyword argument '%s'" % key) - keywords[i] = key - keywords_w[i] = space.getitem(w_starstararg, w_key) - self.keyword_names_w = keys_w - if self.keywords is None: - self.keywords = keywords - self.keywords_w = keywords_w - else: - self.keywords = self.keywords + keywords - self.keywords_w = self.keywords_w + keywords_w - - - def fixedunpack(self, argcount): - """The simplest argument parsing: get the 'argcount' arguments, - or raise a real ValueError if the length is wrong.""" - if self.keywords: - raise ValueError, "no keyword arguments expected" - if len(self.arguments_w) > argcount: - raise ValueError, "too many arguments (%d expected)" % argcount - elif len(self.arguments_w) < argcount: - raise ValueError, "not enough arguments (%d expected)" % argcount - return self.arguments_w - - def combine_if_necessary(self): - if self.combine_has_happened: - return - self._combine_wrapped(self.w_stararg, self.w_starstararg) - self.combine_has_happened = True - - def prepend(self, w_firstarg): # used often - "Return a new Arguments with a new argument inserted first." - return ArgumentsForTranslation(self.space, [w_firstarg] + self.arguments_w, - self.keywords, self.keywords_w, self.w_stararg, - self.w_starstararg) - - def copy(self): - return ArgumentsForTranslation(self.space, self.arguments_w, - self.keywords, self.keywords_w, self.w_stararg, - self.w_starstararg) - - def _match_signature(self, scope_w, signature, defaults_w=None): - """Parse args and kwargs according to the signature of a code object, - or raise an ArgErr in case of failure. - """ - # args_w = list of the normal actual parameters, wrapped - # scope_w = resulting list of wrapped values - # - self.combine_if_necessary() - co_argcount = signature.num_argnames() # expected formal arguments, without */** - - args_w = self.arguments_w - num_args = len(args_w) - keywords = self.keywords or [] - num_kwds = len(keywords) - - # put as many positional input arguments into place as available - take = min(num_args, co_argcount) - scope_w[:take] = args_w[:take] - input_argcount = take - - # collect extra positional arguments into the *vararg - if signature.has_vararg(): - if num_args > co_argcount: - starargs_w = args_w[co_argcount:] - else: - starargs_w = [] - scope_w[co_argcount] = self.space.newtuple(starargs_w) - elif num_args > co_argcount: - raise ArgErrCount(num_args, num_kwds, signature, defaults_w, 0) - - # if a **kwargs argument is needed, explode - if signature.has_kwarg(): - raise TypeError("Keyword arguments as **kwargs is not supported by RPython") - - # handle keyword arguments - num_remainingkwds = 0 - keywords_w = self.keywords_w - kwds_mapping = None - if num_kwds: - # kwds_mapping maps target indexes in the scope (minus input_argcount) - # to positions in the keywords_w list - kwds_mapping = [-1] * (co_argcount - input_argcount) - # match the keywords given at the call site to the argument names - # the called function takes - # this function must not take a scope_w, to make the scope not - # escape - num_remainingkwds = len(keywords) - for i, name in enumerate(keywords): - # If name was not encoded as a string, it could be None. In that - # case, it's definitely not going to be in the signature. - if name is None: - continue - j = signature.find_argname(name) - # if j == -1 nothing happens - if j < input_argcount: - # check that no keyword argument conflicts with these. - if j >= 0: - raise ArgErrMultipleValues(name) - else: - kwds_mapping[j - input_argcount] = i # map to the right index - num_remainingkwds -= 1 - - if num_remainingkwds: - if co_argcount == 0: - raise ArgErrCount(num_args, num_kwds, signature, defaults_w, 0) - raise ArgErrUnknownKwds(self.space, num_remainingkwds, keywords, - kwds_mapping, self.keyword_names_w) - - # check for missing arguments and fill them from the kwds, - # or with defaults, if available - missing = 0 - if input_argcount < co_argcount: - def_first = co_argcount - (0 if defaults_w is None else len(defaults_w)) - j = 0 - kwds_index = -1 - for i in range(input_argcount, co_argcount): - if kwds_mapping is not None: - kwds_index = kwds_mapping[j] - j += 1 - if kwds_index >= 0: - scope_w[i] = keywords_w[kwds_index] - continue - defnum = i - def_first - if defnum >= 0: - scope_w[i] = defaults_w[defnum] - else: - missing += 1 - if missing: - raise ArgErrCount(num_args, num_kwds, signature, defaults_w, missing) - - def unpack(self): - "Return a ([w1,w2...], {'kw':w3...}) pair." - self.combine_if_necessary() - kwds_w = {} - if self.keywords: - for i in range(len(self.keywords)): - kwds_w[self.keywords[i]] = self.keywords_w[i] - return self.arguments_w, kwds_w - - - def match_signature(self, signature, defaults_w): - """Parse args and kwargs according to the signature of a code object, - or raise an ArgErr in case of failure. - """ - scopelen = signature.scope_length() - scope_w = [None] * scopelen - self._match_signature(scope_w, signature, defaults_w) - return scope_w - - def unmatch_signature(self, signature, data_w): - """kind of inverse of match_signature""" - args_w, kwds_w = self.unpack() - need_cnt = len(args_w) - need_kwds = kwds_w.keys() - space = self.space - argnames, varargname, kwargname = signature - cnt = len(argnames) - data_args_w = data_w[:cnt] - if varargname: - data_w_stararg = data_w[cnt] - cnt += 1 - else: - data_w_stararg = space.newtuple([]) - - unfiltered_kwds_w = {} - if kwargname: - data_w_starargarg = data_w[cnt] - for w_key in space.unpackiterable(data_w_starargarg): - key = space.str_w(w_key) - w_value = space.getitem(data_w_starargarg, w_key) - unfiltered_kwds_w[key] = w_value - cnt += 1 - assert len(data_w) == cnt - - ndata_args_w = len(data_args_w) - if ndata_args_w >= need_cnt: - args_w = data_args_w[:need_cnt] - for argname, w_arg in zip(argnames[need_cnt:], data_args_w[need_cnt:]): - unfiltered_kwds_w[argname] = w_arg - assert not space.is_true(data_w_stararg) - else: - stararg_w = space.unpackiterable(data_w_stararg) - datalen = len(data_args_w) - args_w = [None] * (datalen + len(stararg_w)) - for i in range(0, datalen): - args_w[i] = data_args_w[i] - for i in range(0, len(stararg_w)): - args_w[i + datalen] = stararg_w[i] - assert len(args_w) == need_cnt - - keywords = [] - keywords_w = [] - for key in need_kwds: - keywords.append(key) - keywords_w.append(unfiltered_kwds_w[key]) - - return ArgumentsForTranslation(self.space, args_w, keywords, keywords_w) - - @staticmethod - def fromshape(space, (shape_cnt,shape_keys,shape_star,shape_stst), data_w): - args_w = data_w[:shape_cnt] - p = end_keys = shape_cnt + len(shape_keys) - if shape_star: - w_star = data_w[p] - p += 1 - else: - w_star = None - if shape_stst: - w_starstar = data_w[p] - p += 1 - else: - w_starstar = None - return ArgumentsForTranslation(space, args_w, list(shape_keys), - data_w[shape_cnt:end_keys], w_star, - w_starstar) - - def flatten(self): - """ Argument <-> list of w_objects together with "shape" information """ - shape_cnt, shape_keys, shape_star, shape_stst = self._rawshape() - data_w = self.arguments_w + [self.keywords_w[self.keywords.index(key)] - for key in shape_keys] - if shape_star: - data_w.append(self.w_stararg) - if shape_stst: - data_w.append(self.w_starstararg) - return (shape_cnt, shape_keys, shape_star, shape_stst), data_w - - def _rawshape(self, nextra=0): - assert not self.combine_has_happened - shape_cnt = len(self.arguments_w)+nextra # Number of positional args - if self.keywords: - shape_keys = self.keywords[:] # List of keywords (strings) - shape_keys.sort() - else: - shape_keys = [] - shape_star = self.w_stararg is not None # Flag: presence of *arg - shape_stst = self.w_starstararg is not None # Flag: presence of **kwds - return shape_cnt, tuple(shape_keys), shape_star, shape_stst # shape_keys are sorted - -def rawshape(args, nextra=0): - return args._rawshape(nextra) - class CallSpec(object): """Represents the arguments passed into a function call, i.e. the @@ -407,111 +106,3 @@ if shape_star: data_w.append(self.w_stararg) return (shape_cnt, shape_keys, shape_star, shape_stst), data_w - - -# -# ArgErr family of exceptions raised in case of argument mismatch. -# We try to give error messages following CPython's, which are very informative. -# - -class ArgErr(Exception): - - def getmsg(self): - raise NotImplementedError - -class ArgErrCount(ArgErr): - - def __init__(self, got_nargs, nkwds, signature, - defaults_w, missing_args): - self.signature = signature - - self.num_defaults = 0 if defaults_w is None else len(defaults_w) - self.missing_args = missing_args - self.num_args = got_nargs - self.num_kwds = nkwds - - def getmsg(self): - n = self.signature.num_argnames() - if n == 0: - msg = "takes no arguments (%d given)" % ( - self.num_args + self.num_kwds) - else: - defcount = self.num_defaults - has_kwarg = self.signature.has_kwarg() - num_args = self.num_args - num_kwds = self.num_kwds - if defcount == 0 and not self.signature.has_vararg(): - msg1 = "exactly" - if not has_kwarg: - num_args += num_kwds - num_kwds = 0 - elif not self.missing_args: - msg1 = "at most" - else: - msg1 = "at least" - has_kwarg = False - n -= defcount - if n == 1: - plural = "" - else: - plural = "s" - if has_kwarg or num_kwds > 0: - msg2 = " non-keyword" - else: - msg2 = "" - msg = "takes %s %d%s argument%s (%d given)" % ( - msg1, - n, - msg2, - plural, - num_args) - return msg - -class ArgErrMultipleValues(ArgErr): - - def __init__(self, argname): - self.argname = argname - - def getmsg(self): - msg = "got multiple values for keyword argument '%s'" % ( - self.argname) - return msg - -class ArgErrUnknownKwds(ArgErr): - - def __init__(self, space, num_remainingkwds, keywords, kwds_mapping, - keyword_names_w): - name = '' - self.num_kwds = num_remainingkwds - if num_remainingkwds == 1: - for i in range(len(keywords)): - if i not in kwds_mapping: - name = keywords[i] - if name is None: - # We'll assume it's unicode. Encode it. - # Careful, I *think* it should not be possible to - # get an IndexError here but you never know. - try: - if keyword_names_w is None: - raise IndexError - # note: negative-based indexing from the end - w_name = keyword_names_w[i - len(keywords)] - except IndexError: - name = '?' - else: - w_enc = space.wrap(space.sys.defaultencoding) - w_err = space.wrap("replace") - w_name = space.call_method(w_name, "encode", w_enc, - w_err) - name = space.str_w(w_name) - break - self.kwd_name = name - - def getmsg(self): - if self.num_kwds == 1: - msg = "got an unexpected keyword argument '%s'" % ( - self.kwd_name) - else: - msg = "got %d unexpected keyword arguments" % ( - self.num_kwds) - return msg diff --git a/rpython/flowspace/test/test_argument.py b/rpython/flowspace/test/test_argument.py --- a/rpython/flowspace/test/test_argument.py +++ b/rpython/flowspace/test/test_argument.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- import py -from rpython.flowspace.argument import (ArgumentsForTranslation, rawshape, - Signature) +from rpython.flowspace.argument import Signature class TestSignature(object): @@ -50,277 +49,3 @@ assert x == ["a", "b", "c"] assert y == "d" assert z == "e" - -class dummy_wrapped_dict(dict): - def __nonzero__(self): - raise NotImplementedError - -class kwargsdict(dict): - pass - -class DummySpace(object): - def newtuple(self, items): - return tuple(items) - - def is_true(self, obj): - if isinstance(obj, dummy_wrapped_dict): - return bool(dict(obj)) - return bool(obj) - - def fixedview(self, it): - return list(it) - - def listview(self, it): - return list(it) - - def unpackiterable(self, it): - return list(it) - - def view_as_kwargs(self, x): - if len(x) == 0: - return [], [] - return None, None - - def newdict(self): - return {} - - def newlist(self, l=[]): - return l - - def setitem(self, obj, key, value): - obj[key] = value - - def getitem(self, obj, key): - return obj[key] - - def wrap(self, obj): - return obj - - def str_w(self, s): - return str(s) - - def len(self, x): - return len(x) - - def int_w(self, x): - return x - - def eq_w(self, x, y): - return x == y - - def isinstance(self, obj, cls): - return isinstance(obj, cls) - isinstance_w = isinstance - - def exception_match(self, w_type1, w_type2): - return issubclass(w_type1, w_type2) - - def call_method(self, obj, name, *args): - method = getattr(obj, name) - return method(*args) - - def type(self, obj): - class Type: - def getname(self, space, default='?'): - return type(obj).__name__ - return Type() - - - w_TypeError = TypeError - w_AttributeError = AttributeError - w_UnicodeEncodeError = UnicodeEncodeError - w_dict = dict - w_str = str - -def make_arguments_for_translation(space, args_w, keywords_w={}, - w_stararg=None, w_starstararg=None): - return ArgumentsForTranslation(space, args_w, keywords_w.keys(), - keywords_w.values(), w_stararg, - w_starstararg) - -class TestArgumentsForTranslation(object): - - def test_prepend(self): - space = DummySpace() - args = ArgumentsForTranslation(space, ["0"]) - args1 = args.prepend("thingy") - assert args1 is not args - assert args1.arguments_w == ["thingy", "0"] - assert args1.keywords is args.keywords - assert args1.keywords_w is args.keywords_w - - def test_fixedunpacked(self): - space = DummySpace() - - args = ArgumentsForTranslation(space, [], ["k"], [1]) - py.test.raises(ValueError, args.fixedunpack, 1) - - args = ArgumentsForTranslation(space, ["a", "b"]) - py.test.raises(ValueError, args.fixedunpack, 0) - py.test.raises(ValueError, args.fixedunpack, 1) - py.test.raises(ValueError, args.fixedunpack, 3) - py.test.raises(ValueError, args.fixedunpack, 4) - - assert args.fixedunpack(2) == ['a', 'b'] - - def test_unmatch_signature(self): - space = DummySpace() - args = make_arguments_for_translation(space, [1,2,3]) - sig = Signature(['a', 'b', 'c'], None, None) - data = args.match_signature(sig, []) - new_args = args.unmatch_signature(sig, data) - assert args.unpack() == new_args.unpack() - - args = make_arguments_for_translation(space, [1]) - sig = Signature(['a', 'b', 'c'], None, None) - data = args.match_signature(sig, [2, 3]) - new_args = args.unmatch_signature(sig, data) - assert args.unpack() == new_args.unpack() - - args = make_arguments_for_translation(space, [1,2,3,4,5]) - sig = Signature(['a', 'b', 'c'], 'r', None) - data = args.match_signature(sig, []) - new_args = args.unmatch_signature(sig, data) - assert args.unpack() == new_args.unpack() - - args = make_arguments_for_translation(space, [1], {'c': 3, 'b': 2}) - sig = Signature(['a', 'b', 'c'], None, None) - data = args.match_signature(sig, []) - new_args = args.unmatch_signature(sig, data) - assert args.unpack() == new_args.unpack() - - args = make_arguments_for_translation(space, [1], {'c': 5}) - sig = Signature(['a', 'b', 'c'], None, None) - data = args.match_signature(sig, [2, 3]) - new_args = args.unmatch_signature(sig, data) - assert args.unpack() == new_args.unpack() - - args = make_arguments_for_translation(space, [1], {'c': 5, 'd': 7}) - sig = Signature(['a', 'b', 'c'], None, 'kw') - py.test.raises(TypeError, args.match_signature, sig, [2, 3]) - - def test_rawshape(self): - space = DummySpace() - args = make_arguments_for_translation(space, [1,2,3]) - assert rawshape(args) == (3, (), False, False) - - args = make_arguments_for_translation(space, [1]) - assert rawshape(args, 2) == (3, (), False, False) - - args = make_arguments_for_translation(space, [1,2,3,4,5]) - assert rawshape(args) == (5, (), False, False) - - args = make_arguments_for_translation(space, [1], {'c': 3, 'b': 2}) - assert rawshape(args) == (1, ('b', 'c'), False, False) - - args = make_arguments_for_translation(space, [1], {'c': 5}) - assert rawshape(args) == (1, ('c', ), False, False) - - args = make_arguments_for_translation(space, [1], {'c': 5, 'd': 7}) - assert rawshape(args) == (1, ('c', 'd'), False, False) - - args = make_arguments_for_translation(space, [1,2,3,4,5], {'e': 5, 'd': 7}) - assert rawshape(args) == (5, ('d', 'e'), False, False) - - args = make_arguments_for_translation(space, [], {}, - w_stararg=[1], - w_starstararg={'c': 5, 'd': 7}) - assert rawshape(args) == (0, (), True, True) - - args = make_arguments_for_translation(space, [1,2], {'g': 9}, - w_stararg=[3,4,5], - w_starstararg={'e': 5, 'd': 7}) - assert rawshape(args) == (2, ('g', ), True, True) - - def test_copy_and_shape(self): - space = DummySpace() - args = ArgumentsForTranslation(space, ['a'], ['x'], [1], - ['w1'], {'y': 'w2'}) - args1 = args.copy() - args.combine_if_necessary() - assert rawshape(args1) == (1, ('x',), True, True) - - - def test_flatten(self): - space = DummySpace() - args = make_arguments_for_translation(space, [1,2,3]) - assert args.flatten() == ((3, (), False, False), [1, 2, 3]) - - args = make_arguments_for_translation(space, [1]) - assert args.flatten() == ((1, (), False, False), [1]) - - args = make_arguments_for_translation(space, [1,2,3,4,5]) - assert args.flatten() == ((5, (), False, False), [1,2,3,4,5]) - - args = make_arguments_for_translation(space, [1], {'c': 3, 'b': 2}) - assert args.flatten() == ((1, ('b', 'c'), False, False), [1, 2, 3]) - - args = make_arguments_for_translation(space, [1], {'c': 5}) - assert args.flatten() == ((1, ('c', ), False, False), [1, 5]) - - args = make_arguments_for_translation(space, [1], {'c': 5, 'd': 7}) - assert args.flatten() == ((1, ('c', 'd'), False, False), [1, 5, 7]) - - args = make_arguments_for_translation(space, [1,2,3,4,5], {'e': 5, 'd': 7}) - assert args.flatten() == ((5, ('d', 'e'), False, False), [1, 2, 3, 4, 5, 7, 5]) - - args = make_arguments_for_translation(space, [], {}, - w_stararg=[1], - w_starstararg={'c': 5, 'd': 7}) - assert args.flatten() == ((0, (), True, True), [[1], {'c': 5, 'd': 7}]) - - args = make_arguments_for_translation(space, [1,2], {'g': 9}, - w_stararg=[3,4,5], - w_starstararg={'e': 5, 'd': 7}) - assert args.flatten() == ((2, ('g', ), True, True), [1, 2, 9, [3, 4, 5], {'e': 5, 'd': 7}]) - - def test_stararg_flowspace_variable(self): - space = DummySpace() - var = object() - shape = ((2, ('g', ), True, False), [1, 2, 9, var]) - args = make_arguments_for_translation(space, [1,2], {'g': 9}, - w_stararg=var) - assert args.flatten() == shape - - args = ArgumentsForTranslation.fromshape(space, *shape) - assert args.flatten() == shape - - - def test_fromshape(self): - space = DummySpace() - shape = ((3, (), False, False), [1, 2, 3]) - args = ArgumentsForTranslation.fromshape(space, *shape) - assert args.flatten() == shape - - shape = ((1, (), False, False), [1]) - args = ArgumentsForTranslation.fromshape(space, *shape) - assert args.flatten() == shape - - shape = ((5, (), False, False), [1,2,3,4,5]) - args = ArgumentsForTranslation.fromshape(space, *shape) - assert args.flatten() == shape - - shape = ((1, ('b', 'c'), False, False), [1, 2, 3]) - args = ArgumentsForTranslation.fromshape(space, *shape) - assert args.flatten() == shape - - shape = ((1, ('c', ), False, False), [1, 5]) - args = ArgumentsForTranslation.fromshape(space, *shape) - assert args.flatten() == shape - - shape = ((1, ('c', 'd'), False, False), [1, 5, 7]) - args = ArgumentsForTranslation.fromshape(space, *shape) - assert args.flatten() == shape - - shape = ((5, ('d', 'e'), False, False), [1, 2, 3, 4, 5, 7, 5]) - args = ArgumentsForTranslation.fromshape(space, *shape) - assert args.flatten() == shape - - shape = ((0, (), True, True), [[1], {'c': 5, 'd': 7}]) - args = ArgumentsForTranslation.fromshape(space, *shape) - assert args.flatten() == shape - - shape = ((2, ('g', ), True, True), [1, 2, 9, [3, 4, 5], {'e': 5, 'd': 7}]) - args = ArgumentsForTranslation.fromshape(space, *shape) - assert args.flatten() == shape - diff --git a/rpython/rtyper/callparse.py b/rpython/rtyper/callparse.py --- a/rpython/rtyper/callparse.py +++ b/rpython/rtyper/callparse.py @@ -1,4 +1,4 @@ -from rpython.flowspace.argument import ArgumentsForTranslation, ArgErr +from rpython.annotator.argument import ArgumentsForTranslation, ArgErr from rpython.annotator import model as annmodel from rpython.rtyper import rtuple from rpython.rtyper.error import TyperError diff --git a/rpython/rtyper/rbuiltin.py b/rpython/rtyper/rbuiltin.py --- a/rpython/rtyper/rbuiltin.py +++ b/rpython/rtyper/rbuiltin.py @@ -44,7 +44,7 @@ def call_args_expand(hop, takes_kwds = True): hop = hop.copy() - from rpython.flowspace.argument import ArgumentsForTranslation + from rpython.annotator.argument import ArgumentsForTranslation arguments = ArgumentsForTranslation.fromshape( None, hop.args_s[1].const, # shape range(hop.nb_args-2)) From noreply at buildbot.pypy.org Wed Jan 30 17:15:15 2013 From: noreply at buildbot.pypy.org (rlamy) Date: Wed, 30 Jan 2013 17:15:15 +0100 (CET) Subject: [pypy-commit] pypy kill-flowobjspace: move RPythonCallsSpace to rpython/annotator/argument.py Message-ID: <20130130161515.169501C0041@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: kill-flowobjspace Changeset: r60734:702f5433fe4d Date: 2013-01-30 04:33 +0000 http://bitbucket.org/pypy/pypy/changeset/702f5433fe4d/ Log: move RPythonCallsSpace to rpython/annotator/argument.py diff --git a/rpython/annotator/argument.py b/rpython/annotator/argument.py --- a/rpython/annotator/argument.py +++ b/rpython/annotator/argument.py @@ -1,6 +1,52 @@ """ Arguments objects. """ +from rpython.annotator.model import SomeTuple, SomeObject + +# for parsing call arguments +class RPythonCallsSpace(object): + """Pseudo Object Space providing almost no real operation. + For the Arguments class: if it really needs other operations, it means + that the call pattern is too complex for R-Python. + """ + w_tuple = SomeTuple + def newtuple(self, items_s): + if len(items_s) == 1 and items_s[0] is Ellipsis: + res = SomeObject() # hack to get a SomeObject as the *arg + res.from_ellipsis = True + return res + else: + return SomeTuple(items_s) + + def newdict(self): + raise CallPatternTooComplex, "'**' argument" + + def unpackiterable(self, s_obj, expected_length=None): + if isinstance(s_obj, SomeTuple): + if (expected_length is not None and + expected_length != len(s_obj.items)): + raise ValueError + return list(s_obj.items) + if (s_obj.__class__ is SomeObject and + getattr(s_obj, 'from_ellipsis', False)): # see newtuple() + return [Ellipsis] + raise CallPatternTooComplex, "'*' argument must be SomeTuple" + fixedview = unpackiterable + listview = unpackiterable + + def is_w(self, one, other): + return one is other + + def type(self, item): + return type(item) + + def is_true(self, s_tup): + assert isinstance(s_tup, SomeTuple) + return bool(s_tup.items) + +class CallPatternTooComplex(Exception): + pass + class ArgumentsForTranslation(object): def __init__(self, space, args_w, keywords=None, keywords_w=None, w_stararg=None, w_starstararg=None): diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py --- a/rpython/annotator/bookkeeper.py +++ b/rpython/annotator/bookkeeper.py @@ -19,7 +19,7 @@ from rpython.annotator.dictdef import DictDef from rpython.annotator import description from rpython.annotator.signature import annotationoftype -from rpython.annotator.argument import ArgumentsForTranslation +from rpython.annotator.argument import ArgumentsForTranslation, RPythonCallsSpace from rpython.rlib.objectmodel import r_dict, Symbolic from rpython.tool.algo.unionfind import UnionFind from rpython.rtyper.lltypesystem import lltype, llmemory @@ -734,51 +734,6 @@ return False else: return True - -# for parsing call arguments -class RPythonCallsSpace(object): - """Pseudo Object Space providing almost no real operation. - For the Arguments class: if it really needs other operations, it means - that the call pattern is too complex for R-Python. - """ - w_tuple = SomeTuple - def newtuple(self, items_s): - if len(items_s) == 1 and items_s[0] is Ellipsis: - res = SomeObject() # hack to get a SomeObject as the *arg - res.from_ellipsis = True - return res - else: - return SomeTuple(items_s) - - def newdict(self): - raise CallPatternTooComplex, "'**' argument" - - def unpackiterable(self, s_obj, expected_length=None): - if isinstance(s_obj, SomeTuple): - if (expected_length is not None and - expected_length != len(s_obj.items)): - raise ValueError - return list(s_obj.items) - if (s_obj.__class__ is SomeObject and - getattr(s_obj, 'from_ellipsis', False)): # see newtuple() - return [Ellipsis] - raise CallPatternTooComplex, "'*' argument must be SomeTuple" - fixedview = unpackiterable - listview = unpackiterable - - def is_w(self, one, other): - return one is other - - def type(self, item): - return type(item) - - def is_true(self, s_tup): - assert isinstance(s_tup, SomeTuple) - return bool(s_tup.items) - -class CallPatternTooComplex(Exception): - pass - # get current bookkeeper def getbookkeeper(): From noreply at buildbot.pypy.org Wed Jan 30 17:15:16 2013 From: noreply at buildbot.pypy.org (rlamy) Date: Wed, 30 Jan 2013 17:15:16 +0100 (CET) Subject: [pypy-commit] pypy kill-flowobjspace: Kill dead code in RPythonCallsSpace Message-ID: <20130130161516.402B71C0041@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: kill-flowobjspace Changeset: r60735:5aecf9579c66 Date: 2013-01-30 04:49 +0000 http://bitbucket.org/pypy/pypy/changeset/5aecf9579c66/ Log: Kill dead code in RPythonCallsSpace diff --git a/rpython/annotator/argument.py b/rpython/annotator/argument.py --- a/rpython/annotator/argument.py +++ b/rpython/annotator/argument.py @@ -9,7 +9,6 @@ For the Arguments class: if it really needs other operations, it means that the call pattern is too complex for R-Python. """ - w_tuple = SomeTuple def newtuple(self, items_s): if len(items_s) == 1 and items_s[0] is Ellipsis: res = SomeObject() # hack to get a SomeObject as the *arg @@ -18,27 +17,13 @@ else: return SomeTuple(items_s) - def newdict(self): - raise CallPatternTooComplex, "'**' argument" - def unpackiterable(self, s_obj, expected_length=None): if isinstance(s_obj, SomeTuple): - if (expected_length is not None and - expected_length != len(s_obj.items)): - raise ValueError return list(s_obj.items) if (s_obj.__class__ is SomeObject and getattr(s_obj, 'from_ellipsis', False)): # see newtuple() return [Ellipsis] - raise CallPatternTooComplex, "'*' argument must be SomeTuple" - fixedview = unpackiterable - listview = unpackiterable - - def is_w(self, one, other): - return one is other - - def type(self, item): - return type(item) + raise CallPatternTooComplex("'*' argument must be SomeTuple") def is_true(self, s_tup): assert isinstance(s_tup, SomeTuple) From noreply at buildbot.pypy.org Wed Jan 30 17:15:17 2013 From: noreply at buildbot.pypy.org (rlamy) Date: Wed, 30 Jan 2013 17:15:17 +0100 (CET) Subject: [pypy-commit] pypy kill-flowobjspace: Kill dead code and tests in ArgumentsForTranslation Message-ID: <20130130161517.640851C0041@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: kill-flowobjspace Changeset: r60736:c3132de8201b Date: 2013-01-30 05:11 +0000 http://bitbucket.org/pypy/pypy/changeset/c3132de8201b/ Log: Kill dead code and tests in ArgumentsForTranslation diff --git a/rpython/annotator/argument.py b/rpython/annotator/argument.py --- a/rpython/annotator/argument.py +++ b/rpython/annotator/argument.py @@ -58,8 +58,7 @@ "unpack the *arg and **kwd into arguments_w and keywords_w" if w_stararg is not None: self._combine_starargs_wrapped(w_stararg) - if w_starstararg is not None: - self._combine_starstarargs_wrapped(w_starstararg) + assert w_starstararg is None def _combine_starargs_wrapped(self, w_stararg): # unpack the * arguments @@ -67,42 +66,6 @@ args_w = space.unpackiterable(w_stararg) self.arguments_w = self.arguments_w + args_w - def _combine_starstarargs_wrapped(self, w_starstararg): - # unpack the ** arguments - space = self.space - keywords, values_w = space.view_as_kwargs(w_starstararg) - if keywords is not None: # this path also taken for empty dicts - if self.keywords is None: - self.keywords = keywords - self.keywords_w = values_w - else: - if set(keywords) & set(self.keywords): - raise TypeError("got multiple values for keyword arguments '%s'", set(keywords) & set(self.keywords)) - self.keywords = self.keywords + keywords - self.keywords_w = self.keywords_w + values_w - return - if space.isinstance_w(w_starstararg, space.w_dict): - keys_w = space.unpackiterable(w_starstararg) - else: - w_keys = space.call_method(w_starstararg, "keys") - keys_w = space.unpackiterable(w_keys) - keywords_w = [None] * len(keys_w) - keywords = [None] * len(keys_w) - for i, w_key in enumerate(keys_w): - key = space.str_w(w_key) - if key in self.keywords: - raise TypeError("got multiple values for keyword argument '%s'" % key) - keywords[i] = key - keywords_w[i] = space.getitem(w_starstararg, w_key) - self.keyword_names_w = keys_w - if self.keywords is None: - self.keywords = keywords - self.keywords_w = keywords_w - else: - self.keywords = self.keywords + keywords - self.keywords_w = self.keywords_w + keywords_w - - def fixedunpack(self, argcount): """The simplest argument parsing: get the 'argcount' arguments, or raise a real ValueError if the length is wrong.""" diff --git a/rpython/annotator/test/test_argument.py b/rpython/annotator/test/test_argument.py --- a/rpython/annotator/test/test_argument.py +++ b/rpython/annotator/test/test_argument.py @@ -3,86 +3,16 @@ from rpython.annotator.argument import ArgumentsForTranslation, rawshape from rpython.flowspace.argument import Signature -class dummy_wrapped_dict(dict): - def __nonzero__(self): - raise NotImplementedError - -class kwargsdict(dict): - pass - class DummySpace(object): def newtuple(self, items): return tuple(items) def is_true(self, obj): - if isinstance(obj, dummy_wrapped_dict): - return bool(dict(obj)) return bool(obj) - def fixedview(self, it): - return list(it) - - def listview(self, it): - return list(it) - def unpackiterable(self, it): return list(it) - def view_as_kwargs(self, x): - if len(x) == 0: - return [], [] - return None, None - - def newdict(self): - return {} - - def newlist(self, l=[]): - return l - - def setitem(self, obj, key, value): - obj[key] = value - - def getitem(self, obj, key): - return obj[key] - - def wrap(self, obj): - return obj - - def str_w(self, s): - return str(s) - - def len(self, x): - return len(x) - - def int_w(self, x): - return x - - def eq_w(self, x, y): - return x == y - - def isinstance(self, obj, cls): - return isinstance(obj, cls) - isinstance_w = isinstance - - def exception_match(self, w_type1, w_type2): - return issubclass(w_type1, w_type2) - - def call_method(self, obj, name, *args): - method = getattr(obj, name) - return method(*args) - - def type(self, obj): - class Type: - def getname(self, space, default='?'): - return type(obj).__name__ - return Type() - - - w_TypeError = TypeError - w_AttributeError = AttributeError - w_UnicodeEncodeError = UnicodeEncodeError - w_dict = dict - w_str = str def make_arguments_for_translation(space, args_w, keywords_w={}, w_stararg=None, w_starstararg=None): @@ -184,14 +114,6 @@ w_starstararg={'e': 5, 'd': 7}) assert rawshape(args) == (2, ('g', ), True, True) - def test_copy_and_shape(self): - space = DummySpace() - args = ArgumentsForTranslation(space, ['a'], ['x'], [1], - ['w1'], {'y': 'w2'}) - args1 = args.copy() - args.combine_if_necessary() - assert rawshape(args1) == (1, ('x',), True, True) - def test_flatten(self): space = DummySpace() From noreply at buildbot.pypy.org Wed Jan 30 17:15:33 2013 From: noreply at buildbot.pypy.org (rlamy) Date: Wed, 30 Jan 2013 17:15:33 +0100 (CET) Subject: [pypy-commit] pypy kill-flowobjspace: merge default Message-ID: <20130130161533.EE4F21C0041@cobra.cs.uni-duesseldorf.de> Author: Ronan Lamy Branch: kill-flowobjspace Changeset: r60737:a2e872a6466b Date: 2013-01-30 16:13 +0000 http://bitbucket.org/pypy/pypy/changeset/a2e872a6466b/ Log: merge default diff too long, truncating to 2000 out of 145850 lines diff --git a/lib-python/2.7/inspect.py b/lib-python/2.7/inspect.py --- a/lib-python/2.7/inspect.py +++ b/lib-python/2.7/inspect.py @@ -960,7 +960,7 @@ raise TypeError('%s() takes exactly 0 arguments ' '(%d given)' % (f_name, num_total)) else: - raise TypeError('%s() takes no argument (%d given)' % + raise TypeError('%s() takes no arguments (%d given)' % (f_name, num_total)) for arg in args: if isinstance(arg, str) and arg in named: diff --git a/lib-python/2.7/site.py b/lib-python/2.7/site.py --- a/lib-python/2.7/site.py +++ b/lib-python/2.7/site.py @@ -75,6 +75,7 @@ USER_SITE = None USER_BASE = None + def makepath(*paths): dir = os.path.join(*paths) try: diff --git a/lib-python/2.7/test/test_capi.py b/lib-python/2.7/test/test_capi.py --- a/lib-python/2.7/test/test_capi.py +++ b/lib-python/2.7/test/test_capi.py @@ -13,6 +13,20 @@ threading = None import _testcapi +skips = [] +if test_support.check_impl_detail(pypy=True): + skips += [ + 'test_broken_memoryview', + 'test_capsule', + 'test_lazy_hash_inheritance', + 'test_long_api', + 'test_longlong_api', + 'test_null_strings', + 'test_widechar', + 'TestThreadState', + 'TestPendingCalls', + ] + @unittest.skipUnless(threading, 'Threading required for this test.') class TestPendingCalls(unittest.TestCase): @@ -99,7 +113,7 @@ def test_main(): for name in dir(_testcapi): - if name.startswith('test_'): + if name.startswith('test_') and name not in skips: test = getattr(_testcapi, name) if test_support.verbose: print "internal", name @@ -126,7 +140,7 @@ raise test_support.TestFailed, \ "Couldn't find main thread correctly in the list" - if threading: + if threading and 'TestThreadState' not in skips: import thread import time TestThreadState() @@ -134,7 +148,8 @@ t.start() t.join() - test_support.run_unittest(TestPendingCalls) + if 'TestPendingCalls' not in skips: + test_support.run_unittest(TestPendingCalls) if __name__ == "__main__": test_main() diff --git a/lib-python/2.7/test/test_itertools.py b/lib-python/2.7/test/test_itertools.py --- a/lib-python/2.7/test/test_itertools.py +++ b/lib-python/2.7/test/test_itertools.py @@ -533,11 +533,11 @@ self.assertEqual(list(izip()), zip()) self.assertRaises(TypeError, izip, 3) self.assertRaises(TypeError, izip, range(3), 3) - self.assertEqual([tuple(list(pair)) for pair in izip('abc', 'def')], zip('abc', 'def')) self.assertEqual([pair for pair in izip('abc', 'def')], zip('abc', 'def')) + @test_support.impl_detail("tuple reuse is specific to CPython") def test_izip_tuple_reuse(self): ids = map(id, izip('abc', 'def')) @@ -588,6 +588,7 @@ zip('abc', 'def')) self.assertEqual([pair for pair in izip_longest('abc', 'def')], zip('abc', 'def')) + @test_support.impl_detail("tuple reuse is specific to CPython") def test_izip_longest_tuple_reuse(self): ids = map(id, izip_longest('abc', 'def')) diff --git a/lib-python/2.7/test/test_support.py b/lib-python/2.7/test/test_support.py --- a/lib-python/2.7/test/test_support.py +++ b/lib-python/2.7/test/test_support.py @@ -1085,7 +1085,6 @@ else: runner = BasicTestRunner() - result = runner.run(suite) if not result.wasSuccessful(): if len(result.errors) == 1 and not result.failures: diff --git a/lib-python/conftest.py b/lib-python/conftest.py --- a/lib-python/conftest.py +++ b/lib-python/conftest.py @@ -61,7 +61,7 @@ usemodules = '', skip=None): self.basename = basename - self._usemodules = usemodules.split() + ['signal'] + self._usemodules = usemodules.split() + ['signal', 'rctime', 'itertools', '_socket'] self._compiler = compiler self.core = core self.skip = skip @@ -93,63 +93,57 @@ m.test_main() ''' % locals()) -if sys.platform == 'win32': - skip_win32 = "Not supported on Windows" - only_win32 = False -else: - skip_win32 = False - only_win32 = "Only on Windows" - testmap = [ RegrTest('test___all__.py', core=True), RegrTest('test___future__.py', core=True), - RegrTest('test__locale.py', skip=skip_win32), + RegrTest('test__locale.py', usemodules='_locale'), RegrTest('test_abc.py'), RegrTest('test_abstract_numbers.py'), - RegrTest('test_aepack.py', skip=True), + RegrTest('test_aepack.py'), RegrTest('test_aifc.py'), - RegrTest('test_argparse.py'), - RegrTest('test_al.py', skip=True), - RegrTest('test_ast.py', core=True), - RegrTest('test_anydbm.py'), - RegrTest('test_applesingle.py', skip=True), - RegrTest('test_array.py', core=True, usemodules='struct array'), + RegrTest('test_al.py'), + RegrTest('test_anydbm.py', usemodules='struct'), + RegrTest('test_applesingle.py'), + RegrTest('test_argparse.py', usemodules='binascii'), + RegrTest('test_array.py', core=True, usemodules='struct array binascii'), RegrTest('test_ascii_formatd.py'), - RegrTest('test_asynchat.py', usemodules='thread'), - RegrTest('test_asyncore.py'), + RegrTest('test_ast.py', core=True, usemodules='struct'), + RegrTest('test_asynchat.py', usemodules='select fcntl'), + RegrTest('test_asyncore.py', usemodules='select fcntl'), RegrTest('test_atexit.py', core=True), - RegrTest('test_audioop.py', skip=True), + RegrTest('test_audioop.py', skip="unsupported extension module"), RegrTest('test_augassign.py', core=True), - RegrTest('test_base64.py'), + RegrTest('test_base64.py', usemodules='struct'), RegrTest('test_bastion.py'), + RegrTest('test_bigaddrspace.py'), + RegrTest('test_bigmem.py'), RegrTest('test_binascii.py', usemodules='binascii'), - RegrTest('test_binhex.py'), - RegrTest('test_binop.py', core=True), RegrTest('test_bisect.py', core=True, usemodules='_bisect'), RegrTest('test_bool.py', core=True), - RegrTest('test_bsddb.py', skip="unsupported extension module"), - RegrTest('test_bsddb185.py', skip="unsupported extension module"), - RegrTest('test_bsddb3.py', skip="unsupported extension module"), + RegrTest('test_bsddb.py'), + RegrTest('test_bsddb185.py'), + RegrTest('test_bsddb3.py'), RegrTest('test_buffer.py'), RegrTest('test_bufio.py', core=True), - RegrTest('test_builtin.py', core=True), - RegrTest('test_bytes.py'), + RegrTest('test_builtin.py', core=True, usemodules='binascii'), + RegrTest('test_bytes.py', usemodules='struct binascii'), RegrTest('test_bz2.py', usemodules='bz2'), RegrTest('test_calendar.py'), RegrTest('test_call.py', core=True), - RegrTest('test_capi.py', skip="not applicable"), - RegrTest('test_cd.py', skip=True), + RegrTest('test_capi.py'), + RegrTest('test_cd.py'), RegrTest('test_cfgparser.py'), - RegrTest('test_cgi.py'), RegrTest('test_charmapcodec.py', core=True), - RegrTest('test_cl.py', skip=True), + RegrTest('test_cl.py'), RegrTest('test_class.py', core=True), RegrTest('test_cmath.py', core=True), RegrTest('test_cmd.py'), + RegrTest('test_cmd_line.py'), RegrTest('test_cmd_line_script.py'), + RegrTest('test_code.py', core=True), RegrTest('test_codeccallbacks.py', core=True), RegrTest('test_codecencodings_cn.py', usemodules='_multibytecodec'), RegrTest('test_codecencodings_hk.py', usemodules='_multibytecodec'), @@ -157,7 +151,6 @@ RegrTest('test_codecencodings_jp.py', usemodules='_multibytecodec'), RegrTest('test_codecencodings_kr.py', usemodules='_multibytecodec'), RegrTest('test_codecencodings_tw.py', usemodules='_multibytecodec'), - RegrTest('test_codecmaps_cn.py', usemodules='_multibytecodec'), RegrTest('test_codecmaps_hk.py', usemodules='_multibytecodec'), RegrTest('test_codecmaps_jp.py', usemodules='_multibytecodec'), @@ -165,6 +158,7 @@ RegrTest('test_codecmaps_tw.py', usemodules='_multibytecodec'), RegrTest('test_codecs.py', core=True, usemodules='_multibytecodec'), RegrTest('test_codeop.py', core=True), + RegrTest('test_coding.py', core=True), RegrTest('test_coercion.py', core=True), RegrTest('test_collections.py'), RegrTest('test_colorsys.py'), @@ -174,22 +168,24 @@ RegrTest('test_compileall.py'), RegrTest('test_compiler.py', core=False, skip="slowly deprecating compiler"), RegrTest('test_complex.py', core=True), - + RegrTest('test_complex_args.py'), RegrTest('test_contains.py', core=True), + RegrTest('test_contextlib.py', usemodules="thread"), RegrTest('test_cookie.py'), RegrTest('test_cookielib.py'), RegrTest('test_copy.py', core=True), RegrTest('test_copy_reg.py', core=True), RegrTest('test_cpickle.py', core=True), - RegrTest('test_cprofile.py'), - RegrTest('test_crypt.py', usemodules='crypt', skip=skip_win32), + RegrTest('test_cprofile.py'), + RegrTest('test_crypt.py', usemodules='crypt'), RegrTest('test_csv.py', usemodules='_csv'), - - RegrTest('test_curses.py', skip="unsupported extension module"), + RegrTest('test_ctypes.py', usemodules="_rawffi thread"), + RegrTest('test_curses.py'), RegrTest('test_datetime.py'), RegrTest('test_dbm.py'), RegrTest('test_decimal.py'), RegrTest('test_decorators.py', core=True), + RegrTest('test_defaultdict.py', usemodules='_collections'), RegrTest('test_deque.py', core=True, usemodules='_collections'), RegrTest('test_descr.py', core=True, usemodules='_weakref'), RegrTest('test_descrtut.py', core=True), @@ -200,7 +196,7 @@ RegrTest('test_dircache.py', core=True), RegrTest('test_dis.py'), RegrTest('test_distutils.py', skip=True), - RegrTest('test_dl.py', skip=True), + RegrTest('test_dl.py'), RegrTest('test_doctest.py', usemodules="thread"), RegrTest('test_doctest2.py'), RegrTest('test_docxmlrpc.py'), @@ -208,20 +204,22 @@ RegrTest('test_dummy_thread.py', core=True), RegrTest('test_dummy_threading.py', core=True), RegrTest('test_email.py'), - RegrTest('test_email_codecs.py'), + RegrTest('test_email_renamed.py'), RegrTest('test_enumerate.py', core=True), RegrTest('test_eof.py', core=True), RegrTest('test_epoll.py'), RegrTest('test_errno.py', usemodules="errno"), + RegrTest('test_exception_variations.py'), RegrTest('test_exceptions.py', core=True), RegrTest('test_extcall.py', core=True), - RegrTest('test_fcntl.py', usemodules='fcntl', skip=skip_win32), + RegrTest('test_fcntl.py', usemodules='fcntl'), RegrTest('test_file.py', usemodules="posix", core=True), RegrTest('test_file2k.py', usemodules="posix", core=True), RegrTest('test_filecmp.py', core=True), RegrTest('test_fileinput.py', core=True), RegrTest('test_fileio.py'), + RegrTest('test_float.py', core=True), RegrTest('test_fnmatch.py', core=True), RegrTest('test_fork1.py', usemodules="thread"), RegrTest('test_format.py', core=True), @@ -230,6 +228,7 @@ RegrTest('test_frozen.py', skip="unsupported extension module"), RegrTest('test_ftplib.py'), RegrTest('test_funcattrs.py', core=True), + RegrTest('test_functools.py'), RegrTest('test_future.py', core=True), RegrTest('test_future1.py', core=True), RegrTest('test_future2.py', core=True), @@ -239,41 +238,37 @@ RegrTest('test_future_builtins.py'), RegrTest('test_gc.py', usemodules='_weakref', skip="implementation detail"), RegrTest('test_gdb.py', skip="not applicable"), - RegrTest('test_gdbm.py', skip="unsupported extension module"), + RegrTest('test_gdbm.py'), RegrTest('test_generators.py', core=True, usemodules='thread _weakref'), RegrTest('test_genericpath.py'), RegrTest('test_genexps.py', core=True, usemodules='_weakref'), - RegrTest('test_getargs.py', skip="unsupported extension module"), - RegrTest('test_getargs2.py', skip="unsupported extension module"), - + RegrTest('test_getargs.py'), + RegrTest('test_getargs2.py', usemodules='binascii', skip=True), RegrTest('test_getopt.py', core=True), RegrTest('test_gettext.py'), - - RegrTest('test_gl.py', skip=True), + RegrTest('test_gl.py'), RegrTest('test_glob.py', core=True), RegrTest('test_global.py', core=True), RegrTest('test_grammar.py', core=True), - RegrTest('test_grp.py', skip=skip_win32), - - RegrTest('test_gzip.py'), + RegrTest('test_grp.py'), + RegrTest('test_gzip.py', usemodules='zlib'), RegrTest('test_hash.py', core=True), RegrTest('test_hashlib.py', core=True), - RegrTest('test_heapq.py', core=True), RegrTest('test_hmac.py'), RegrTest('test_hotshot.py', skip="unsupported extension module"), - RegrTest('test_htmllib.py'), RegrTest('test_htmlparser.py'), RegrTest('test_httplib.py'), RegrTest('test_httpservers.py'), - RegrTest('test_imageop.py', skip="unsupported extension module"), + RegrTest('test_imageop.py'), RegrTest('test_imaplib.py'), - RegrTest('test_imgfile.py', skip="unsupported extension module"), + RegrTest('test_imgfile.py'), RegrTest('test_imp.py', core=True, usemodules='thread'), RegrTest('test_import.py', core=True), RegrTest('test_importhooks.py', core=True), RegrTest('test_importlib.py'), + RegrTest('test_index.py'), RegrTest('test_inspect.py'), RegrTest('test_int.py', core=True), RegrTest('test_int_literal.py', core=True), @@ -296,7 +291,7 @@ RegrTest('test_long_future.py', core=True), RegrTest('test_longexp.py', core=True), RegrTest('test_macos.py'), - RegrTest('test_macostools.py', skip=True), + RegrTest('test_macostools.py'), RegrTest('test_macpath.py'), RegrTest('test_mailbox.py'), RegrTest('test_marshal.py', core=True), @@ -307,30 +302,29 @@ RegrTest('test_mhlib.py'), RegrTest('test_mimetools.py'), RegrTest('test_mimetypes.py'), - RegrTest('test_MimeWriter.py', core=False), + RegrTest('test_MimeWriter.py', core=False, usemodules='binascii'), RegrTest('test_minidom.py'), RegrTest('test_mmap.py', usemodules="mmap"), RegrTest('test_module.py', core=True), RegrTest('test_modulefinder.py'), - RegrTest('test_msilib.py', skip=only_win32), + RegrTest('test_msilib.py'), RegrTest('test_multibytecodec.py', usemodules='_multibytecodec'), - RegrTest('test_multibytecodec_support.py', skip="not a test"), RegrTest('test_multifile.py'), - RegrTest('test_multiprocessing.py', skip="FIXME leaves subprocesses"), + RegrTest('test_multiprocessing.py', skip=True), RegrTest('test_mutants.py', core="possibly"), RegrTest('test_mutex.py'), RegrTest('test_netrc.py'), RegrTest('test_new.py', core=True), - RegrTest('test_nis.py', skip="unsupported extension module"), + RegrTest('test_nis.py'), RegrTest('test_normalization.py'), RegrTest('test_ntpath.py'), + RegrTest('test_old_mailbox.py'), RegrTest('test_opcodes.py', core=True), RegrTest('test_openpty.py'), RegrTest('test_operator.py', core=True), RegrTest('test_optparse.py'), - RegrTest('test_os.py', core=True), - RegrTest('test_ossaudiodev.py', skip="unsupported extension module"), + RegrTest('test_ossaudiodev.py'), RegrTest('test_parser.py', skip="slowly deprecating compiler"), RegrTest('test_pdb.py'), RegrTest('test_peepholer.py'), @@ -338,14 +332,16 @@ RegrTest('test_pep263.py'), RegrTest('test_pep277.py'), RegrTest('test_pep292.py'), + RegrTest('test_pep352.py'), RegrTest('test_pickle.py', core=True), RegrTest('test_pickletools.py', core=False), RegrTest('test_pipes.py'), RegrTest('test_pkg.py', core=True), RegrTest('test_pkgimport.py', core=True), RegrTest('test_pkgutil.py'), - RegrTest('test_plistlib.py', skip="unsupported module"), - RegrTest('test_poll.py', skip=skip_win32), + RegrTest('test_platform.py'), + RegrTest('test_plistlib.py'), + RegrTest('test_poll.py'), RegrTest('test_popen.py'), RegrTest('test_popen2.py'), RegrTest('test_poplib.py'), @@ -357,8 +353,8 @@ RegrTest('test_profile.py'), RegrTest('test_property.py', core=True), RegrTest('test_pstats.py'), - RegrTest('test_pty.py', skip="unsupported extension module"), - RegrTest('test_pwd.py', usemodules="pwd", skip=skip_win32), + RegrTest('test_pty.py', usemodules='fcntl termios select'), + RegrTest('test_pwd.py', usemodules="pwd"), RegrTest('test_py3kwarn.py'), RegrTest('test_py_compile.py'), RegrTest('test_pyclbr.py'), @@ -370,15 +366,15 @@ RegrTest('test_re.py', core=True), RegrTest('test_readline.py'), RegrTest('test_repr.py', core=True), - RegrTest('test_resource.py', skip=skip_win32), + RegrTest('test_resource.py'), RegrTest('test_rfc822.py'), RegrTest('test_richcmp.py', core=True), RegrTest('test_rlcompleter.py'), - RegrTest('test_robotparser.py'), + RegrTest('test_runpy.py'), RegrTest('test_sax.py'), RegrTest('test_scope.py', core=True), - RegrTest('test_scriptpackages.py', skip="unsupported extension module"), + RegrTest('test_scriptpackages.py'), RegrTest('test_select.py'), RegrTest('test_set.py', core=True), RegrTest('test_sets.py'), @@ -389,64 +385,59 @@ RegrTest('test_shlex.py'), RegrTest('test_shutil.py'), RegrTest('test_signal.py'), - RegrTest('test_SimpleHTTPServer.py'), + RegrTest('test_SimpleHTTPServer.py', usemodules='binascii'), RegrTest('test_site.py', core=False), RegrTest('test_slice.py', core=True), RegrTest('test_smtplib.py'), RegrTest('test_smtpnet.py'), RegrTest('test_socket.py', usemodules='thread _weakref'), - RegrTest('test_socketserver.py', usemodules='thread'), - RegrTest('test_softspace.py', core=True), RegrTest('test_sort.py', core=True), + RegrTest('test_sqlite.py', usemodules="thread _rawffi zlib"), RegrTest('test_ssl.py', usemodules='_ssl _socket select'), + RegrTest('test_startfile.py'), RegrTest('test_str.py', core=True), - RegrTest('test_strftime.py'), RegrTest('test_string.py', core=True), - RegrTest('test_StringIO.py', core=True, usemodules='cStringIO'), + RegrTest('test_StringIO.py', core=True, usemodules='cStringIO array'), RegrTest('test_stringprep.py'), RegrTest('test_strop.py', skip="deprecated"), - RegrTest('test_strptime.py'), RegrTest('test_strtod.py'), RegrTest('test_struct.py', usemodules='struct'), RegrTest('test_structmembers.py', skip="CPython specific"), RegrTest('test_structseq.py'), RegrTest('test_subprocess.py', usemodules='signal'), - RegrTest('test_sunaudiodev.py', skip=True), + RegrTest('test_sunaudiodev.py'), RegrTest('test_sundry.py'), RegrTest('test_symtable.py', skip="implementation detail"), RegrTest('test_syntax.py', core=True), RegrTest('test_sys.py', core=True, usemodules='struct'), + RegrTest('test_sys_setprofile.py', core=True), RegrTest('test_sys_settrace.py', core=True), - RegrTest('test_sys_setprofile.py', core=True), RegrTest('test_sysconfig.py'), - RegrTest('test_tcl.py', skip="unsupported extension module"), RegrTest('test_tarfile.py'), + RegrTest('test_tcl.py'), RegrTest('test_telnetlib.py'), RegrTest('test_tempfile.py'), - RegrTest('test_textwrap.py'), RegrTest('test_thread.py', usemodules="thread", core=True), RegrTest('test_threaded_import.py', usemodules="thread", core=True), RegrTest('test_threadedtempfile.py', usemodules="thread", core=False), - RegrTest('test_threading.py', usemodules="thread", core=True), RegrTest('test_threading_local.py', usemodules="thread", core=True), RegrTest('test_threadsignals.py', usemodules="thread"), - RegrTest('test_time.py', core=True), RegrTest('test_timeout.py'), RegrTest('test_tk.py'), - RegrTest('test_ttk_guionly.py'), - RegrTest('test_ttk_textonly.py'), RegrTest('test_tokenize.py'), RegrTest('test_trace.py'), RegrTest('test_traceback.py', core=True), RegrTest('test_transformer.py', core=True), + RegrTest('test_ttk_guionly.py'), + RegrTest('test_ttk_textonly.py'), RegrTest('test_tuple.py', core=True), RegrTest('test_typechecks.py'), RegrTest('test_types.py', core=True), @@ -462,6 +453,7 @@ RegrTest('test_unpack.py', core=True), RegrTest('test_urllib.py'), RegrTest('test_urllib2.py'), + RegrTest('test_urllib2_localnet.py', usemodules="thread"), RegrTest('test_urllib2net.py'), RegrTest('test_urllibnet.py'), RegrTest('test_urlparse.py'), @@ -469,61 +461,38 @@ RegrTest('test_userlist.py', core=True), RegrTest('test_userstring.py', core=True), RegrTest('test_uu.py'), - - RegrTest('test_warnings.py', core=True), - RegrTest('test_wave.py', skip="unsupported extension module"), - RegrTest('test_weakref.py', core=True, usemodules='_weakref'), - RegrTest('test_weakset.py'), - - RegrTest('test_whichdb.py'), - RegrTest('test_winreg.py', skip=only_win32), - RegrTest('test_winsound.py', skip="unsupported extension module"), - RegrTest('test_xmllib.py'), - RegrTest('test_xmlrpc.py'), - - RegrTest('test_xpickle.py'), - RegrTest('test_xrange.py', core=True), - RegrTest('test_zipfile.py'), - RegrTest('test_zipimport.py', usemodules='zlib zipimport'), - RegrTest('test_zipimport_support.py', usemodules='zlib zipimport'), - RegrTest('test_zlib.py', usemodules='zlib'), - - RegrTest('test_bigaddrspace.py'), - RegrTest('test_bigmem.py'), - RegrTest('test_cmd_line.py'), - RegrTest('test_code.py'), - RegrTest('test_coding.py'), - RegrTest('test_complex_args.py'), - RegrTest('test_contextlib.py', usemodules="thread"), - RegrTest('test_ctypes.py', usemodules="_rawffi thread"), - RegrTest('test_defaultdict.py', usemodules='_collections'), - RegrTest('test_email_renamed.py'), - RegrTest('test_exception_variations.py'), - RegrTest('test_float.py'), - RegrTest('test_functools.py'), - RegrTest('test_index.py'), - RegrTest('test_old_mailbox.py'), - RegrTest('test_pep352.py'), - RegrTest('test_platform.py'), - RegrTest('test_runpy.py'), - RegrTest('test_sqlite.py', usemodules="thread _rawffi zlib"), - RegrTest('test_startfile.py', skip="bogus test"), - RegrTest('test_structmembers.py', skip="depends on _testcapi"), - RegrTest('test_urllib2_localnet.py', usemodules="thread"), RegrTest('test_uuid.py'), RegrTest('test_wait3.py', usemodules="thread"), RegrTest('test_wait4.py', usemodules="thread"), + RegrTest('test_warnings.py', core=True), + RegrTest('test_wave.py'), + RegrTest('test_weakref.py', core=True, usemodules='_weakref'), + RegrTest('test_weakset.py'), + RegrTest('test_whichdb.py'), + RegrTest('test_winreg.py'), + RegrTest('test_winsound.py'), RegrTest('test_with.py'), RegrTest('test_wsgiref.py'), RegrTest('test_xdrlib.py'), RegrTest('test_xml_etree.py'), RegrTest('test_xml_etree_c.py'), + RegrTest('test_xmllib.py'), + RegrTest('test_xmlrpc.py'), + RegrTest('test_xpickle.py'), + RegrTest('test_xrange.py', core=True), + RegrTest('test_zipfile.py'), RegrTest('test_zipfile64.py'), + RegrTest('test_zipimport.py', usemodules='zlib zipimport'), + RegrTest('test_zipimport_support.py', usemodules='zlib zipimport'), + RegrTest('test_zlib.py', usemodules='zlib'), ] def check_testmap_complete(): listed_names = dict.fromkeys([regrtest.basename for regrtest in testmap]) - listed_names['test_support.py'] = True # ignore this + assert len(listed_names) == len(testmap) + # names to ignore + listed_names['test_support.py'] = True + listed_names['test_multibytecodec_support.py'] = True missing = [] for path in testdir.listdir(fil='test_*.py'): name = path.basename @@ -578,7 +547,7 @@ def getinvocation(self, regrtest): fspath = regrtest.getfspath() python = sys.executable - pypy_script = pypydir.join('bin', 'py.py') + pypy_script = pypydir.join('bin', 'pyinteractive.py') alarm_script = pypydir.join('tool', 'alarm.py') if sys.platform == 'win32': watchdog_name = 'watchdog_nt.py' diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst --- a/pypy/doc/faq.rst +++ b/pypy/doc/faq.rst @@ -300,12 +300,26 @@ Do I have to rewrite my programs in RPython? -------------------------------------------- -No. And you shouldn't try. PyPy always runs your code in its own interpreter, which is a -full and compliant Python 2.7 interpreter. RPython is only the -language in which parts of PyPy itself are written and extension -modules for it. Not only is it not necessary for you to rewrite your -code in RPython, it probably won't give you any speed improvements if you -try. +No. And you shouldn't try. First and foremost, RPython is a language +that is designed to write interpreters in. It is a restricted subset of +Python. If you program is not an interpreter but tries to do "real +things", like use *any* part of the standard Python library or *any* +3rd-party library, then it is not RPython to start with. You should +only look at RPython if you try to `write your own interpreter`__. + +.. __: `how do I compile my own interpreters`_ + +If your goal is to speed up Python code, then look at the regular PyPy, +which is a full and compliant Python 2.7 interpreter (which happens to +be written in RPython). Not only is it not necessary for you to rewrite +your code in RPython, it might not give you any speed improvements even +if you manage to. + +Yes, it is possible with enough effort to compile small self-contained +pieces of RPython code doing a few performance-sensitive things. But +this case is not interesting for us. If you needed to rewrite the code +in RPython, you could as well have rewritten it in C for example. The +latter is a much more supported, much more documented language `:-)` --------------------------------------------------- Which backends are there for the RPython toolchain? diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -34,6 +34,9 @@ .. branch: kill-faking .. branch: improved_ebnfparse_error .. branch: task-decorator +.. branch: fix-e4fa0b2 +.. branch: win32-fixes +.. branch: fix-version-tool .. branch: release-2.0-beta1 diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -713,13 +713,13 @@ def pypy_find_stdlib(s): from os.path import abspath, join, dirname as dn thisfile = abspath(__file__) - root = dn(dn(dn(dn(thisfile)))) + root = dn(dn(dn(thisfile))) return [join(root, 'lib-python', '2.7'), join(root, 'lib_pypy')] def pypy_resolvedirof(s): # we ignore the issue of symlinks; for tests, the executable is always - # translator/goal/app_main.py anyway + # interpreter/app_main.py anyway import os return os.path.abspath(os.path.join(s, '..')) diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -1482,9 +1482,10 @@ ) raise w_fd = self.call_function(w_fileno) - if not self.isinstance_w(w_fd, self.w_int): + if (not self.isinstance_w(w_fd, self.w_int) and + not self.isinstance_w(w_fd, self.w_long)): raise OperationError(self.w_TypeError, - self.wrap("fileno() must return an integer") + self.wrap("fileno() returned a non-integer") ) fd = self.int_w(w_fd) if fd < 0: diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -343,9 +343,13 @@ signal, the tick counter is set to -1 by C code in signals.h. """ assert isinstance(action, PeriodicAsyncAction) - self._periodic_actions.append(action) + # hack to put the release-the-GIL one at the end of the list, + # and the report-the-signals one at the start of the list. if use_bytecode_counter: + self._periodic_actions.append(action) self.has_bytecode_counter = True + else: + self._periodic_actions.insert(0, action) self._rebuild_action_dispatcher() def getcheckinterval(self): @@ -419,15 +423,6 @@ The action must have been registered at space initalization time.""" self.space.actionflag.fire(self) - def fire_after_thread_switch(self): - """Bit of a hack: fire() the action but only the next time the GIL - is released and re-acquired (i.e. after a potential thread switch). - Don't call this if threads are not enabled. Currently limited to - one action (i.e. reserved for CheckSignalAction from module/signal). - """ - from pypy.module.thread.gil import spacestate - spacestate.action_after_thread_switch = self - def perform(self, executioncontext, frame): """To be overridden.""" diff --git a/pypy/interpreter/test2/test_app_main.py b/pypy/interpreter/test2/test_app_main.py --- a/pypy/interpreter/test2/test_app_main.py +++ b/pypy/interpreter/test2/test_app_main.py @@ -547,7 +547,7 @@ assert line.rstrip() == 'Not at all. They could be carried.' print 'A five ounce bird could not carry a one pound coconut.' """) - py_py = os.path.join(pypydir, 'bin', 'py.py') + py_py = os.path.join(pypydir, 'bin', 'pyinteractive.py') child = self._spawn(sys.executable, [py_py, '-S', path]) child.expect('Are you suggesting coconuts migrate?', timeout=120) child.sendline('Not at all. They could be carried.') @@ -774,7 +774,7 @@ assert data == p + os.sep + '\n' def test_getfilesystemencoding(self): - py.test.skip("this has been failing since forever, but it's not tested nightly because buildbot uses python2.6 :-(") + py.test.skip("encoding is only set if stdout.isatty(), test is flawed") if sys.version_info < (2, 7): skip("test requires Python >= 2.7") p = getscript_in_dir(""" @@ -881,7 +881,6 @@ def test_setup_bootstrap_path(self): import sys - import os old_sys_path = sys.path[:] sys.path.append(self.goal_dir) try: @@ -907,7 +906,7 @@ sys.path.append(self.goal_dir) try: import app_main - pypy_c = os.path.join(self.trunkdir, 'pypy', 'translator', 'goal', 'pypy-c') + pypy_c = os.path.join(self.trunkdir, 'pypy', 'goal', 'pypy-c') app_main.setup_bootstrap_path(pypy_c) newpath = sys.path[:] # we get at least lib_pypy @@ -925,7 +924,7 @@ sys.path.append(self.goal_dir) try: import app_main - pypy_c = os.path.join(self.trunkdir, 'pypy', 'translator', 'goal', 'pypy-c') + pypy_c = os.path.join(self.trunkdir, 'pypy', 'goal', 'pypy-c') app_main.entry_point(pypy_c, [self.foo_py]) # assert it did not crash finally: diff --git a/pypy/interpreter/test2/test_targetpypy.py b/pypy/interpreter/test2/test_targetpypy.py --- a/pypy/interpreter/test2/test_targetpypy.py +++ b/pypy/interpreter/test2/test_targetpypy.py @@ -1,5 +1,3 @@ - -import py from pypy.goal.targetpypystandalone import get_entry_point from pypy.config.pypyoption import get_pypy_config @@ -7,6 +5,4 @@ def test_run(self): config = get_pypy_config(translating=False) entry_point = get_entry_point(config)[0] - space = self.space - py.test.skip("not working so far") entry_point(['pypy-c' , '-S', '-c', 'print 3']) diff --git a/pypy/module/_bisect/interp_bisect.py b/pypy/module/_bisect/interp_bisect.py --- a/pypy/module/_bisect/interp_bisect.py +++ b/pypy/module/_bisect/interp_bisect.py @@ -1,5 +1,6 @@ from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import unwrap_spec +from rpython.rlib.rarithmetic import intmask, r_uint @unwrap_spec(lo=int, hi=int) @@ -18,7 +19,7 @@ if hi == -1: hi = space.len_w(w_a) while lo < hi: - mid = (lo + hi) >> 1 + mid = intmask((r_uint(lo) + r_uint(hi)) >> 1) w_litem = space.getitem(w_a, space.wrap(mid)) if space.is_true(space.lt(w_litem, w_x)): lo = mid + 1 @@ -43,7 +44,7 @@ if hi == -1: hi = space.len_w(w_a) while lo < hi: - mid = (lo + hi) >> 1 + mid = intmask((r_uint(lo) + r_uint(hi)) >> 1) w_litem = space.getitem(w_a, space.wrap(mid)) if space.is_true(space.lt(w_x, w_litem)): hi = mid diff --git a/pypy/module/_bisect/test/test_bisect.py b/pypy/module/_bisect/test/test_bisect.py --- a/pypy/module/_bisect/test/test_bisect.py +++ b/pypy/module/_bisect/test/test_bisect.py @@ -89,3 +89,12 @@ insort_right(a, 6.0) assert a == [0, 5, 6, 6, 6, 6.0, 7] assert map(type, a) == [int, int, int, int, int, float, int] + + def test_bisect_overflow(self): + from _bisect import bisect_left, bisect_right + import sys + + size = sys.maxsize + data = xrange(size - 1) + assert bisect_left(data, size - 3) == size - 3 + assert bisect_right(data, size - 3) == size - 2 diff --git a/pypy/module/_ffi/test/test_funcptr.py b/pypy/module/_ffi/test/test_funcptr.py --- a/pypy/module/_ffi/test/test_funcptr.py +++ b/pypy/module/_ffi/test/test_funcptr.py @@ -96,7 +96,7 @@ libm = CDLL(self.libm_name) pow_addr = libm.getaddressindll('pow') fff = sys.maxint*2-1 - if sys.platform == 'win32': + if sys.platform == 'win32' or sys.platform == 'darwin': fff = sys.maxint*2+1 assert pow_addr == self.pow_addr & fff diff --git a/pypy/module/_file/test/test_file.py b/pypy/module/_file/test/test_file.py --- a/pypy/module/_file/test/test_file.py +++ b/pypy/module/_file/test/test_file.py @@ -430,15 +430,15 @@ assert f.subclass_closed def test_readline_unbuffered_should_read_one_line_only(self): - import posix + import os - with self.file(self.temppath, 'w') as f: + with self.file(self.temppath, 'wb') as f: f.write('foo\nbar\n') - with self.file(self.temppath, 'r', 0) as f: + with self.file(self.temppath, 'rb', 0) as f: s = f.readline() assert s == 'foo\n' - s = posix.read(f.fileno(), 10) + s = os.read(f.fileno(), 10) assert s == 'bar\n' def test_flush_at_exit(): diff --git a/pypy/module/_file/test/test_file_extra.py b/pypy/module/_file/test/test_file_extra.py --- a/pypy/module/_file/test/test_file_extra.py +++ b/pypy/module/_file/test/test_file_extra.py @@ -612,6 +612,7 @@ repr(unicode(self.temptestfile))) f.close() + @py.test.mark.skipif("os.name != 'posix'") def test_EAGAIN(self): import _socket, posix s1, s2 = _socket.socketpair() diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -2,7 +2,7 @@ import os class AppTestFileIO: - spaceconfig = dict(usemodules=['_io']) + spaceconfig = dict(usemodules=['_io'] + (['fcntl'] if os.name != 'nt' else [])) def setup_class(cls): tmpfile = udir.join('tmpfile') diff --git a/pypy/module/_multiprocessing/test/test_connection.py b/pypy/module/_multiprocessing/test/test_connection.py --- a/pypy/module/_multiprocessing/test/test_connection.py +++ b/pypy/module/_multiprocessing/test/test_connection.py @@ -68,7 +68,7 @@ assert rhandle.readable class AppTestWinpipeConnection(BaseConnectionTest): - spaceconfig = dict(usemodules=('_multiprocessing', 'thread')) + spaceconfig = dict(usemodules=('_multiprocessing', 'thread', 'signal')) def setup_class(cls): if sys.platform != "win32": diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py --- a/pypy/module/_socket/interp_socket.py +++ b/pypy/module/_socket/interp_socket.py @@ -181,7 +181,8 @@ try: self.close() except SocketError, e: - raise converted_error(space, e) + # cpython doesn't return any errors on close + pass def connect_w(self, space, w_addr): """connect(address) @@ -448,7 +449,7 @@ w_addr = space.w_None return space.newtuple([space.wrap(readlgt), w_addr]) except SocketError, e: - raise converted_error(space, e) + raise converted_error(space, e) @unwrap_spec(cmd=int) def ioctl_w(self, space, cmd, w_option): diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py --- a/pypy/module/_socket/test/test_sock_app.py +++ b/pypy/module/_socket/test/test_sock_app.py @@ -362,13 +362,15 @@ assert isinstance(s.fileno(), int) def test_socket_close(self): - import _socket + import _socket, os s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM, 0) fileno = s.fileno() assert s.fileno() >= 0 s.close() assert s.fileno() < 0 s.close() + if os.name != 'nt': + raises(OSError, os.close, fileno) def test_socket_close_error(self): import _socket, os @@ -376,7 +378,7 @@ skip("Windows sockets are not files") s = _socket.socket(_socket.AF_INET, _socket.SOCK_STREAM, 0) os.close(s.fileno()) - raises(_socket.error, s.close) + s.close() def test_socket_connect(self): import _socket, os diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -922,8 +922,8 @@ kwds["link_extra"] = ["msvcrt.lib"] elif sys.platform.startswith('linux'): compile_extra.append("-Werror=implicit-function-declaration") + compile_extra.append('-g') export_symbols_eci.append('pypyAPI') - compile_extra.append('-g') else: kwds["includes"] = ['Python.h'] # this is our Python.h diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -49,6 +49,8 @@ kwds["libraries"] = [api_library] # '%s' undefined; assuming extern returning int kwds["compile_extra"] = ["/we4013"] + elif sys.platform == 'darwin': + kwds["link_files"] = [str(api_library + '.dylib')] else: kwds["link_files"] = [str(api_library + '.so')] if sys.platform.startswith('linux'): diff --git a/pypy/module/crypt/test/test_crypt.py b/pypy/module/crypt/test/test_crypt.py --- a/pypy/module/crypt/test/test_crypt.py +++ b/pypy/module/crypt/test/test_crypt.py @@ -1,6 +1,12 @@ +import os +import py + +if os.name != 'posix': + py.test.skip('crypt module only available on unix') + class AppTestCrypt: spaceconfig = dict(usemodules=['crypt']) - + def test_crypt(self): import crypt res = crypt.crypt("pass", "ab") diff --git a/pypy/module/fcntl/test/test_fcntl.py b/pypy/module/fcntl/test/test_fcntl.py --- a/pypy/module/fcntl/test/test_fcntl.py +++ b/pypy/module/fcntl/test/test_fcntl.py @@ -1,9 +1,9 @@ import os +import py from rpython.tool.udir import udir -if os.name == "nt": - from py.test import skip - skip("fcntl module is not available on Windows") +if os.name != 'posix': + py.test.skip("fcntl module only available on unix") def teardown_module(mod): for i in "abcde": @@ -22,13 +22,23 @@ import sys import struct + class F: + def __init__(self, fn): + self.fn = fn + def fileno(self): + return self.fn + f = open(self.tmp + "b", "w+") fcntl.fcntl(f, 1, 0) fcntl.fcntl(f, 1) + fcntl.fcntl(F(long(f.fileno())), 1) raises(TypeError, fcntl.fcntl, "foo") raises(TypeError, fcntl.fcntl, f, "foo") - raises((IOError, ValueError), fcntl.fcntl, -1, 1, 0) + raises(TypeError, fcntl.fcntl, F("foo"), 1) + raises(ValueError, fcntl.fcntl, -1, 1, 0) + raises(ValueError, fcntl.fcntl, F(-1), 1, 0) + raises(ValueError, fcntl.fcntl, F(long(-1)), 1, 0) assert fcntl.fcntl(f, 1, 0) == 0 assert fcntl.fcntl(f, 2, "foo") == "foo" assert fcntl.fcntl(f, 2, buffer("foo")) == "foo" diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -823,14 +823,19 @@ if hasattr(os, 'chmod'): def test_chmod(self): + import sys os = self.posix os.unlink(self.path) raises(OSError, os.chmod, self.path, 0600) f = open(self.path, "w") f.write("this is a test") f.close() - os.chmod(self.path, 0200) - assert (os.stat(self.path).st_mode & 0777) == 0200 + if sys.platform == 'win32': + os.chmod(self.path, 0400) + assert (os.stat(self.path).st_mode & 0600) == 0400 + else: + os.chmod(self.path, 0200) + assert (os.stat(self.path).st_mode & 0777) == 0200 if hasattr(os, 'fchmod'): def test_fchmod(self): diff --git a/pypy/module/pwd/test/test_pwd.py b/pypy/module/pwd/test/test_pwd.py --- a/pypy/module/pwd/test/test_pwd.py +++ b/pypy/module/pwd/test/test_pwd.py @@ -1,3 +1,9 @@ +import os +import py + +if os.name != 'posix': + py.test.skip('pwd module only available on unix') + class AppTestPwd: spaceconfig = dict(usemodules=['pwd']) diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py --- a/pypy/module/rctime/test/test_rctime.py +++ b/pypy/module/rctime/test/test_rctime.py @@ -114,7 +114,10 @@ assert rctime.mktime(rctime.localtime(-1)) == -1 res = rctime.mktime((2000, 1, 1, 0, 0, 0, -1, -1, -1)) - assert rctime.ctime(res) == 'Sat Jan 1 00:00:00 2000' + if os.name == 'nt': + assert rctime.ctime(res) == 'Sat Jan 01 00:00:00 2000' + else: + assert rctime.ctime(res) == 'Sat Jan 1 00:00:00 2000' def test_asctime(self): import time as rctime @@ -213,7 +216,7 @@ def test_strftime(self): import time as rctime - import os + import os, sys t = rctime.time() tt = rctime.gmtime(t) @@ -234,6 +237,10 @@ # input to [w]strftime is not kosher. if os.name == 'nt': raises(ValueError, rctime.strftime, '%f') + elif sys.platform == 'darwin': + # darwin strips % of unknown format codes + # http://bugs.python.org/issue9811 + assert rctime.strftime('%f') == 'f' else: assert rctime.strftime('%f') == '%f' diff --git a/pypy/module/signal/__init__.py b/pypy/module/signal/__init__.py --- a/pypy/module/signal/__init__.py +++ b/pypy/module/signal/__init__.py @@ -30,9 +30,10 @@ } def buildloaders(cls): - from pypy.module.signal import interp_signal - for name in interp_signal.signal_names: - signum = getattr(interp_signal, name) + from rpython.rlib import rsignal + + for name in rsignal.signal_names: + signum = getattr(rsignal, name) if signum is not None: Module.interpleveldefs[name] = 'space.wrap(%d)' % (signum,) super(Module, cls).buildloaders() diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py --- a/pypy/module/signal/interp_signal.py +++ b/pypy/module/signal/interp_signal.py @@ -1,25 +1,28 @@ from __future__ import with_statement + +import signal as cpy_signal +import sys + from pypy.interpreter.error import OperationError, exception_from_errno -from pypy.interpreter.executioncontext import AsyncAction, AbstractActionFlag -from pypy.interpreter.executioncontext import PeriodicAsyncAction +from pypy.interpreter.executioncontext import (AsyncAction, AbstractActionFlag, + PeriodicAsyncAction) from pypy.interpreter.gateway import unwrap_spec -import signal as cpy_signal -from rpython.rtyper.lltypesystem import lltype, rffi -from rpython.rtyper.tool import rffi_platform -from rpython.translator.tool.cbuild import ExternalCompilationInfo -from rpython.conftest import cdir -import py -import sys -from rpython.rlib import jit, rposix + +from rpython.rlib import jit, rposix, rgc +from rpython.rlib.objectmodel import we_are_translated from rpython.rlib.rarithmetic import intmask from rpython.rlib.rsignal import * +from rpython.rtyper.lltypesystem import lltype, rffi + WIN32 = sys.platform == 'win32' + class SignalActionFlag(AbstractActionFlag): # This class uses the C-level pypysig_counter variable as the tick # counter. The C-level signal handler will reset it to -1 whenever - # a signal is received. + # a signal is received. This causes CheckSignalAction.perform() to + # be called. def get_ticker(self): p = pypysig_getaddr_occurred() @@ -29,6 +32,11 @@ p = pypysig_getaddr_occurred() p.c_value = value + @staticmethod + def rearm_ticker(): + p = pypysig_getaddr_occurred() + p.c_value = -1 + def decrement_ticker(self, by): p = pypysig_getaddr_occurred() value = p.c_value @@ -44,48 +52,65 @@ class CheckSignalAction(PeriodicAsyncAction): """An action that is automatically invoked when a signal is received.""" + # Note that this is a PeriodicAsyncAction: it means more precisely + # that it is called whenever the C-level ticker becomes < 0. + # Without threads, it is only ever set to -1 when we receive a + # signal. With threads, it also decrements steadily (but slowly). + def __init__(self, space): + "NOT_RPYTHON" AsyncAction.__init__(self, space) self.handlers_w = {} - if space.config.objspace.usemodules.thread: - # need a helper action in case signals arrive in a non-main thread - self.pending_signals = {} - self.reissue_signal_action = ReissueSignalAction(space) - else: - self.reissue_signal_action = None + self.pending_signal = -1 + self.fire_in_main_thread = False + if self.space.config.objspace.usemodules.thread: + from pypy.module.thread import gil + gil.after_thread_switch = self._after_thread_switch + + @rgc.no_collect + def _after_thread_switch(self): + if self.fire_in_main_thread: + if self.space.threadlocals.ismainthread(): + self.fire_in_main_thread = False + SignalActionFlag.rearm_ticker() + # this occurs when we just switched to the main thread + # and there is a signal pending: we force the ticker to + # -1, which should ensure perform() is called quickly. @jit.dont_look_inside def perform(self, executioncontext, frame): - while True: - n = pypysig_poll() - if n < 0: + # Poll for the next signal, if any + n = self.pending_signal + if n < 0: n = pypysig_poll() + while n >= 0: + if self.space.config.objspace.usemodules.thread: + in_main = self.space.threadlocals.ismainthread() + else: + in_main = True + if in_main: + # If we are in the main thread, report the signal now, + # and poll more + self.pending_signal = -1 + self._report_signal(n) + n = self.pending_signal + if n < 0: n = pypysig_poll() + else: + # Otherwise, arrange for perform() to be called again + # after we switch to the main thread. + self.pending_signal = n + self.fire_in_main_thread = True break - self.perform_signal(executioncontext, n) - @jit.dont_look_inside - def perform_signal(self, executioncontext, n): - if self.reissue_signal_action is None: - # no threads: we can report the signal immediately - self.report_signal(n) - else: - main_ec = self.space.threadlocals.getmainthreadvalue() - if executioncontext is main_ec: - # running in the main thread: we can report the - # signal immediately - self.report_signal(n) - else: - # running in another thread: we need to hack a bit - self.pending_signals[n] = None - self.reissue_signal_action.fire_after_thread_switch() - - @jit.dont_look_inside def set_interrupt(self): "Simulates the effect of a SIGINT signal arriving" - ec = self.space.getexecutioncontext() - self.perform_signal(ec, cpy_signal.SIGINT) + if not we_are_translated(): + self.pending_signal = cpy_signal.SIGINT + # ^^^ may override another signal, but it's just for testing + else: + pypysig_pushback(cpy_signal.SIGINT) + self.fire_in_main_thread = True - @jit.dont_look_inside - def report_signal(self, n): + def _report_signal(self, n): try: w_handler = self.handlers_w[n] except KeyError: @@ -100,39 +125,6 @@ w_frame = space.wrap(ec.gettopframe_nohidden()) space.call_function(w_handler, space.wrap(n), w_frame) - @jit.dont_look_inside - def report_pending_signals(self): - # XXX this logic isn't so complicated but I have no clue how - # to test it :-( - pending_signals = self.pending_signals.keys() - self.pending_signals.clear() - try: - while pending_signals: - self.report_signal(pending_signals.pop()) - finally: - # in case of exception, put the undelivered signals back - # into the dict instead of silently swallowing them - if pending_signals: - for n in pending_signals: - self.pending_signals[n] = None - self.reissue_signal_action.fire() - - -class ReissueSignalAction(AsyncAction): - """A special action to help deliver signals to the main thread. If - a non-main thread caught a signal, this action fires after every - thread switch until we land in the main thread. - """ - - def perform(self, executioncontext, frame): - main_ec = self.space.threadlocals.getmainthreadvalue() - if executioncontext is main_ec: - # now running in the main thread: we can really report the signals - self.space.check_signal_action.report_pending_signals() - else: - # still running in some other thread: try again later - self.fire_after_thread_switch() - @unwrap_spec(signum=int) def getsignal(space, signum): @@ -154,6 +146,7 @@ return action.handlers_w[signum] return space.wrap(SIG_DFL) + def default_int_handler(space, w_signum, w_frame): """ default_int_handler(...) @@ -164,22 +157,26 @@ raise OperationError(space.w_KeyboardInterrupt, space.w_None) + @jit.dont_look_inside @unwrap_spec(timeout=int) def alarm(space, timeout): return space.wrap(c_alarm(timeout)) + @jit.dont_look_inside def pause(space): c_pause() return space.w_None + def check_signum_exists(space, signum): if signum in signal_values: return raise OperationError(space.w_ValueError, space.wrap("invalid signal value")) + def check_signum_in_range(space, signum): if 1 <= signum < NSIG: return @@ -201,7 +198,7 @@ A signal handler function is called with two arguments: the first is the signal number, the second is the interrupted stack frame. """ - ec = space.getexecutioncontext() + ec = space.getexecutioncontext() main_ec = space.threadlocals.getmainthreadvalue() old_handler = getsignal(space, signum) @@ -224,13 +221,14 @@ action.handlers_w[signum] = w_handler return old_handler + @jit.dont_look_inside @unwrap_spec(fd=int) def set_wakeup_fd(space, fd): """Sets the fd to be written to (with '\0') when a signal comes in. Returns the old fd. A library can use this to wakeup select or poll. The previous fd is returned. - + The fd must be non-blocking. """ if space.config.objspace.usemodules.thread: @@ -243,6 +241,7 @@ old_fd = pypysig_set_wakeup_fd(fd) return space.wrap(intmask(old_fd)) + @jit.dont_look_inside @unwrap_spec(signum=int, flag=int) def siginterrupt(space, signum, flag): @@ -258,33 +257,38 @@ rffi.setintfield(timeval, 'c_tv_sec', int(d)) rffi.setintfield(timeval, 'c_tv_usec', int((d - int(d)) * 1000000)) + def double_from_timeval(tv): return rffi.getintfield(tv, 'c_tv_sec') + ( rffi.getintfield(tv, 'c_tv_usec') / 1000000.0) + def itimer_retval(space, val): w_value = space.wrap(double_from_timeval(val.c_it_value)) w_interval = space.wrap(double_from_timeval(val.c_it_interval)) return space.newtuple([w_value, w_interval]) + class Cache: def __init__(self, space): self.w_itimererror = space.new_exception_class("signal.ItimerError", space.w_IOError) + def get_itimer_error(space): return space.fromcache(Cache).w_itimererror + @jit.dont_look_inside @unwrap_spec(which=int, first=float, interval=float) def setitimer(space, which, first, interval=0): """setitimer(which, seconds[, interval]) - Sets given itimer (one of ITIMER_REAL, ITIMER_VIRTUAL + or ITIMER_PROF) to fire after value seconds and after that every interval seconds. The itimer can be cleared by setting seconds to zero. - + Returns old values as a tuple: (delay, interval). """ with lltype.scoped_alloc(itimervalP.TO, 1) as new: @@ -298,14 +302,14 @@ if ret != 0: raise exception_from_errno(space, get_itimer_error(space)) + return itimer_retval(space, old[0]) - return itimer_retval(space, old[0]) @jit.dont_look_inside @unwrap_spec(which=int) def getitimer(space, which): """getitimer(which) - + Returns current value of given itimer. """ with lltype.scoped_alloc(itimervalP.TO, 1) as old: diff --git a/pypy/module/signal/test/test_signal.py b/pypy/module/signal/test/test_signal.py --- a/pypy/module/signal/test/test_signal.py +++ b/pypy/module/signal/test/test_signal.py @@ -36,7 +36,7 @@ class AppTestSignal: spaceconfig = { - "usemodules": ['signal', 'rctime'], + "usemodules": ['signal', 'rctime'] + (['fcntl'] if os.name != 'nt' else []), } def setup_class(cls): @@ -157,6 +157,8 @@ if sys.platform == 'win32': raises(ValueError, signal, 42, lambda *args: None) raises(ValueError, signal, 7, lambda *args: None) + elif sys.platform == 'darwin': + raises(ValueError, signal, 42, lambda *args: None) else: signal(42, lambda *args: None) signal(42, SIG_DFL) diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -28,6 +28,7 @@ import pypy pypydir = os.path.dirname(os.path.abspath(pypy.__file__)) +pypyroot = os.path.dirname(pypydir) del pypy from rpython.tool.version import get_repo_version_info @@ -68,7 +69,7 @@ CPYTHON_VERSION[0], CPYTHON_VERSION[1], CPYTHON_VERSION[2], - get_repo_version_info()[2], + get_repo_version_info(root=pypyroot)[1], date, time, ver, @@ -91,10 +92,10 @@ return space.wrap(('PyPy', '', '')) def get_repo_info(space): - info = get_repo_version_info() + info = get_repo_version_info(root=pypyroot) if info: - project, repo_tag, repo_version = info - return space.newtuple([space.wrap(project), + repo_tag, repo_version = info + return space.newtuple([space.wrap('PyPy'), space.wrap(repo_tag), space.wrap(repo_version)]) else: diff --git a/pypy/module/termios/test/test_termios.py b/pypy/module/termios/test/test_termios.py --- a/pypy/module/termios/test/test_termios.py +++ b/pypy/module/termios/test/test_termios.py @@ -1,9 +1,12 @@ - +import os +import sys import py -import sys from pypy.conftest import pypydir from rpython.tool.udir import udir +if os.name != 'posix': + py.test.skip('termios module only available on unix') + class TestTermios(object): def setup_class(cls): try: diff --git a/pypy/module/thread/gil.py b/pypy/module/thread/gil.py --- a/pypy/module/thread/gil.py +++ b/pypy/module/thread/gil.py @@ -62,22 +62,7 @@ do_yield_thread() -class SpaceState: - - def _cleanup_(self): - self.action_after_thread_switch = None - # ^^^ set by AsyncAction.fire_after_thread_switch() - - def after_thread_switch(self): - # this is support logic for the signal module, to help it deliver - # signals to the main thread. - action = self.action_after_thread_switch - if action is not None: - self.action_after_thread_switch = None - action.fire() - -spacestate = SpaceState() -spacestate._cleanup_() +after_thread_switch = lambda: None # hook for signal.py # Fragile code below. We have to preserve the C-level errno manually... @@ -94,7 +79,7 @@ e = get_errno() thread.gil_acquire() thread.gc_thread_run() - spacestate.after_thread_switch() + after_thread_switch() set_errno(e) after_external_call._gctransformer_hint_cannot_collect_ = True after_external_call._dont_reach_me_in_del_ = True @@ -112,7 +97,7 @@ # the same thread). if thread.gil_yield_thread(): thread.gc_thread_run() - spacestate.after_thread_switch() + after_thread_switch() do_yield_thread._gctransformer_hint_close_stack_ = True do_yield_thread._dont_reach_me_in_del_ = True do_yield_thread._dont_inline_ = True diff --git a/pypy/module/thread/os_local.py b/pypy/module/thread/os_local.py --- a/pypy/module/thread/os_local.py +++ b/pypy/module/thread/os_local.py @@ -98,5 +98,5 @@ local = wref() if local is not None: del local.dicts[ec] - local.last_dict = None - local.last_ec = None + local.last_dict = None + local.last_ec = None diff --git a/pypy/module/thread/test/support.py b/pypy/module/thread/test/support.py --- a/pypy/module/thread/test/support.py +++ b/pypy/module/thread/test/support.py @@ -2,6 +2,7 @@ import time import thread import os +import errno from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.module.thread import gil @@ -28,7 +29,12 @@ def kill(): for x in range(delay * 10): time.sleep(0.1) - os.kill(pid, 0) + try: + os.kill(pid, 0) + except OSError, e: + if e.errno == errno.ESRCH: # no such process + return + raise os.kill(pid, 9) print "process %s killed!" % (pid,) thread.start_new_thread(kill, ()) diff --git a/pypy/module/thread/test/test_fork.py b/pypy/module/thread/test/test_fork.py --- a/pypy/module/thread/test/test_fork.py +++ b/pypy/module/thread/test/test_fork.py @@ -1,7 +1,7 @@ from pypy.module.thread.test.support import GenericTestThread class AppTestFork(GenericTestThread): - def test_fork(self): + def test_fork_with_thread(self): # XXX This test depends on a multicore machine, as busy_thread must # aquire the GIL the instant that the main thread releases it. # It will incorrectly pass if the GIL is not grabbed in time. @@ -12,45 +12,48 @@ if not hasattr(os, 'fork'): skip("No fork on this platform") - run = True - done = [] def busy_thread(): while run: time.sleep(0) done.append(None) - try: - thread.start_new(busy_thread, ()) + for i in range(1): + run = True + done = [] + try: + thread.start_new(busy_thread, ()) + print 'sleep' - pid = os.fork() - - if pid == 0: - os._exit(0) - - else: - time.sleep(1) - spid, status = os.waitpid(pid, os.WNOHANG) - assert spid == pid - finally: - run = False - self.waitfor(lambda: done) + pid = os.fork() + if pid == 0: + os._exit(0) + else: + self.timeout_killer(pid, 5) + exitcode = os.waitpid(pid, 0)[1] + assert exitcode == 0 # if 9, process was killed by timer! + finally: + run = False + self.waitfor(lambda: done) + assert done def test_forked_can_thread(self): "Checks that a forked interpreter can start a thread" - import os, thread, time + import thread + import os if not hasattr(os, 'fork'): skip("No fork on this platform") - # pre-allocate some locks - thread.start_new_thread(lambda: None, ()) + for i in range(10): + # pre-allocate some locks + thread.start_new_thread(lambda: None, ()) + print 'sleep' - pid = os.fork() - if pid == 0: - print 'in child' - thread.start_new_thread(lambda: None, ()) - os._exit(0) - else: - self.timeout_killer(pid, 5) - exitcode = os.waitpid(pid, 0)[1] - assert exitcode == 0 # if 9, process was killed by timer! + pid = os.fork() + if pid == 0: + thread.start_new_thread(lambda: None, ()) + os._exit(0) + else: + self.timeout_killer(pid, 5) + exitcode = os.waitpid(pid, 0)[1] + assert exitcode == 0 # if 9, process was killed by timer! diff --git a/pypy/module/thread/test/test_thread.py b/pypy/module/thread/test/test_thread.py --- a/pypy/module/thread/test/test_thread.py +++ b/pypy/module/thread/test/test_thread.py @@ -220,16 +220,24 @@ import signal def f(): - time.sleep(0.5) - thread.interrupt_main() + for x in range(50): + if waiting: + thread.interrupt_main() + return + print 'tock...', x # <-force the GIL to be released, as + time.sleep(0.01) # time.sleep doesn't do non-translated def busy_wait(): - for x in range(1000): + waiting.append(None) + for x in range(100): print 'tick...', x # <-force the GIL to be released, as time.sleep(0.01) # time.sleep doesn't do non-translated + waiting.pop() # This is normally called by app_main.py signal.signal(signal.SIGINT, signal.default_int_handler) - thread.start_new_thread(f, ()) - raises(KeyboardInterrupt, busy_wait) + for i in range(100): + waiting = [] + thread.start_new_thread(f, ()) + raises(KeyboardInterrupt, busy_wait) diff --git a/pypy/module/thread/threadlocals.py b/pypy/module/thread/threadlocals.py --- a/pypy/module/thread/threadlocals.py +++ b/pypy/module/thread/threadlocals.py @@ -48,6 +48,9 @@ ident = self._mainthreadident return self._valuedict.get(ident, None) + def ismainthread(self): + return thread.get_ident() == self._mainthreadident + def getallvalues(self): return self._valuedict diff --git a/pypy/module/unicodedata/interp_ucd.py b/pypy/module/unicodedata/interp_ucd.py --- a/pypy/module/unicodedata/interp_ucd.py +++ b/pypy/module/unicodedata/interp_ucd.py @@ -9,7 +9,7 @@ from rpython.rlib.objectmodel import we_are_translated from rpython.rlib.runicode import MAXUNICODE from rpython.rlib.unicodedata import unicodedb_5_2_0, unicodedb_3_2_0 -from rpython.rlib.runicode import code_to_unichr, ORD +from rpython.rlib.runicode import code_to_unichr, ord_accepts_surrogate import sys @@ -28,8 +28,6 @@ # handling: on narrow unicode builds, a surrogate pair is considered as one # unicode code point. -# The functions below are subtly different from the ones in runicode.py. -# When PyPy implements Python 3 they should be merged. if MAXUNICODE > 0xFFFF: # Target is wide build @@ -41,7 +39,7 @@ if not we_are_translated() and sys.maxunicode == 0xFFFF: # Host CPython is narrow build, accept surrogates try: - return ORD(space.unicode_w(w_unichr)) + return ord_accepts_surrogate(space.unicode_w(w_unichr)) except ValueError: raise OperationError(space.w_TypeError, space.wrap( 'need a single Unicode character as parameter')) @@ -68,7 +66,7 @@ else: # Accept surrogates try: - return ORD(space.unicode_w(w_unichr)) + return ord_accepts_surrogate(space.unicode_w(w_unichr)) except ValueError: raise OperationError(space.w_TypeError, space.wrap( 'need a single Unicode character as parameter')) diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py --- a/pypy/tool/release/package.py +++ b/pypy/tool/release/package.py @@ -1,6 +1,6 @@ #!/usr/bin/env python """ A sample script that packages PyPy, provided that it's already built. -It uses 'pypy/translator/goal/pypy-c' and parts of the rest of the working +It uses 'pypy/goal/pypy-c' and parts of the rest of the working copy. Usage: package.py root-pypy-dir [name-of-archive] [name-of-pypy-c] [destination-for-tarball] [pypy-c-path] diff --git a/pypy/tool/release/test/test_package.py b/pypy/tool/release/test/test_package.py --- a/pypy/tool/release/test/test_package.py +++ b/pypy/tool/release/test/test_package.py @@ -15,7 +15,7 @@ basename = 'pypy-c' rename_pypy_c = 'pypy' exe_name_in_archive = 'bin/pypy' - pypy_c = py.path.local(pypydir).join('..', basename) + pypy_c = py.path.local(pypydir).join('goal', basename) if not pypy_c.check(): os.system("echo faked_pypy_c> %s" % (pypy_c,)) pypy_c.chmod(0755) diff --git a/pypy/tool/rundictbenchmarks.py b/pypy/tool/rundictbenchmarks.py --- a/pypy/tool/rundictbenchmarks.py +++ b/pypy/tool/rundictbenchmarks.py @@ -3,7 +3,7 @@ # this file runs some benchmarks with a pypy-c that is assumed to be # built using the MeasuringDictImplementation. -# it should be run with pypy/translator/goal as the cwd, and you'll +# it should be run with pypy/goal as the cwd, and you'll # need to hack a copy of rst2html for yourself (svn docutils # required). diff --git a/rpython/annotator/annrpython.py b/rpython/annotator/annrpython.py --- a/rpython/annotator/annrpython.py +++ b/rpython/annotator/annrpython.py @@ -113,6 +113,9 @@ if policy is None: from rpython.annotator.policy import AnnotatorPolicy policy = AnnotatorPolicy() + # XXX hack + annmodel.TLS.check_str_without_nul = ( + self.translator.config.translation.check_str_without_nul) graph, inputcells = self.get_call_parameters(function, args_s, policy) self.build_graph_types(graph, inputcells, complete_now=False) self.complete_helpers(policy) diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py --- a/rpython/annotator/binaryop.py +++ b/rpython/annotator/binaryop.py @@ -446,7 +446,8 @@ class __extend__(pairtype(SomeChar, SomeChar)): def union((chr1, chr2)): - return SomeChar() + no_nul = chr1.no_nul and chr2.no_nul + return SomeChar(no_nul=no_nul) class __extend__(pairtype(SomeChar, SomeUnicodeCodePoint), @@ -664,14 +665,14 @@ def getitem((str1, int2)): getbookkeeper().count("str_getitem", int2) - return SomeChar() + return SomeChar(no_nul=str1.no_nul) getitem.can_only_throw = [] getitem_key = getitem def getitem_idx((str1, int2)): getbookkeeper().count("str_getitem", int2) - return SomeChar() + return SomeChar(no_nul=str1.no_nul) getitem_idx.can_only_throw = [IndexError] getitem_idx_key = getitem_idx diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py --- a/rpython/annotator/test/test_annrpython.py +++ b/rpython/annotator/test/test_annrpython.py @@ -2066,6 +2066,22 @@ assert isinstance(s, annmodel.SomeString) assert s.no_nul + def test_getitem_str0(self): + def f(s, n): + if n == 1: + return s[0] + elif n == 2: + return s[1] + elif n == 3: + return s[1:] + return s + a = self.RPythonAnnotator() + a.translator.config.translation.check_str_without_nul = True + + s = a.build_types(f, [annmodel.SomeString(no_nul=True), + annmodel.SomeInteger()]) + assert isinstance(s, annmodel.SomeString) + assert s.no_nul def test_non_none_and_none_with_isinstance(self): class A(object): diff --git a/rpython/flowspace/framestate.py b/rpython/flowspace/framestate.py --- a/rpython/flowspace/framestate.py +++ b/rpython/flowspace/framestate.py @@ -1,5 +1,6 @@ +from rpython.flowspace.model import Variable, Constant from rpython.rlib.unroll import SpecTag -from rpython.flowspace.model import * + class FrameState: def __init__(self, mergeable, blocklist, next_instr): @@ -86,6 +87,7 @@ raise TypeError('union of %r and %r' % (w1.__class__.__name__, w2.__class__.__name__)) + # ____________________________________________________________ # # We have to flatten out the state of the frame into a list of @@ -102,6 +104,7 @@ PICKLE_TAGS = {} UNPICKLE_TAGS = {} + def recursively_flatten(space, lst): from rpython.flowspace.flowcontext import SuspendedUnroller i = 0 @@ -117,14 +120,15 @@ except: tag = PICKLE_TAGS[key] = Constant(PickleTag()) UNPICKLE_TAGS[tag] = key - lst[i:i+1] = [tag] + vars + lst[i:i + 1] = [tag] + vars + def recursively_unflatten(space, lst): - for i in xrange(len(lst)-1, -1, -1): + for i in xrange(len(lst) - 1, -1, -1): item = lst[i] if item in UNPICKLE_TAGS: unrollerclass, argcount = UNPICKLE_TAGS[item] - arguments = lst[i+1: i+1+argcount] - del lst[i+1: i+1+argcount] + arguments = lst[i + 1:i + 1 + argcount] + del lst[i + 1:i + 1 + argcount] unroller = unrollerclass.state_pack_variables(space, *arguments) lst[i] = unroller diff --git a/rpython/jit/backend/llgraph/llimpl.py b/rpython/jit/backend/llgraph/llimpl.py deleted file mode 100644 --- a/rpython/jit/backend/llgraph/llimpl.py +++ /dev/null @@ -1,1968 +0,0 @@ -""" -The non-RPythonic part of the llgraph backend. -This contains all the code that is directly run -when executing on top of the llinterpreter. -""" - -import weakref -from rpython.flowspace.model import Variable, Constant -from rpython.annotator import model as annmodel -from rpython.jit.metainterp.history import REF, INT, FLOAT -from rpython.jit.metainterp import history -from rpython.jit.codewriter import heaptracker -from rpython.rtyper.lltypesystem import lltype, llmemory, rclass, rstr, rffi -from rpython.rtyper.ootypesystem import ootype -from rpython.rtyper.module.support import LLSupport, OOSupport -from rpython.rtyper.llinterp import LLException -from rpython.rtyper.extregistry import ExtRegistryEntry - -from rpython.jit.metainterp import resoperation -from rpython.jit.metainterp.resoperation import rop -from rpython.jit.backend.llgraph import symbolic -from rpython.jit.codewriter import longlong -from rpython.jit.codewriter.effectinfo import EffectInfo - -from rpython.rlib.objectmodel import ComputedIntSymbolic, we_are_translated -from rpython.rlib.rarithmetic import ovfcheck -from rpython.rlib.rarithmetic import r_longlong, r_ulonglong, r_uint -from rpython.rlib.rtimer import read_timestamp - -import py -from rpython.tool.ansi_print import ansi_log -log = py.log.Producer('runner') -py.log.setconsumer('runner', ansi_log) - -IS_32_BIT = r_ulonglong is not r_uint - - -def _from_opaque(opq): - return opq._obj.externalobj - -_TO_OPAQUE = {} - -def _to_opaque(value): - try: - return value._the_opaque_pointer - except AttributeError: - op = lltype.opaqueptr(_TO_OPAQUE[value.__class__], 'opaque', - externalobj=value) - value._the_opaque_pointer = op - return op - -def _normalize(value): - if isinstance(value, lltype._ptr): - value = lltype.top_container(value._obj) - return value - -def from_opaque_string(s): - if isinstance(s, str): - return s - elif isinstance(s, ootype._string): - return OOSupport.from_rstr(s) - else: - return LLSupport.from_rstr(s) From noreply at buildbot.pypy.org Wed Jan 30 17:22:44 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 30 Jan 2013 17:22:44 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: a test and some debug prints Message-ID: <20130130162244.8E9701C00A8@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60738:d564608f9ea1 Date: 2013-01-30 17:33 +0200 http://bitbucket.org/pypy/pypy/changeset/d564608f9ea1/ Log: a test and some debug prints diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -1883,6 +1883,8 @@ malloc_size = (size // WORD // 8 + 1) + 1 rawgcmap = self.datablockwrapper.malloc_aligned(WORD * malloc_size, WORD) + debug_print("gcmap: %x, len %d for %d" % (rawgcmap, malloc_size - 1, + self.mc.get_relative_pos())) # set the length field rffi.cast(rffi.CArrayPtr(lltype.Signed), rawgcmap)[0] = malloc_size - 1 gcmap = rffi.cast(lltype.Ptr(jitframe.GCMAP), rawgcmap) diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -29,6 +29,8 @@ from rpython.jit.backend.x86.arch import IS_X86_32, IS_X86_64 from rpython.jit.backend.x86 import rx86 from rpython.rlib.rarithmetic import r_longlong, r_uint +from rpython.rlib.debug import (debug_print, debug_start, debug_stop, + have_debug_prints) class X86RegisterManager(RegisterManager): @@ -896,6 +898,7 @@ def get_gcmap(self, forbidden_regs=[], noregs=False): frame_depth = self.fm.get_frame_depth() gcmap = self.assembler.allocate_gcmap(frame_depth) + debug_start("jit-backend-gcmap") for box, loc in self.rm.reg_bindings.iteritems(): if loc in forbidden_regs: continue @@ -909,6 +912,9 @@ assert isinstance(loc, StackLoc) val = loc.value // WORD gcmap[val // WORD // 8] |= r_uint(1) << (val % (WORD * 8)) + for i in range(len(gcmap)): + debug_print(str(gcmap[i])) + debug_stop('jit-backend-gcmap') return gcmap diff --git a/rpython/translator/c/test/test_symbolic.py b/rpython/translator/c/test/test_symbolic.py --- a/rpython/translator/c/test/test_symbolic.py +++ b/rpython/translator/c/test/test_symbolic.py @@ -19,6 +19,29 @@ res = fn() assert res == 12 +def test_offsetof_nogc(): + ARR = lltype.Array(lltype.Signed) + offsetx = llmemory.arraylengthoffset(ARR) + offsety = llmemory.itemoffsetof(ARR, 0) + ARR2 = lltype.GcArray(lltype.Signed) + offsetx2 = llmemory.arraylengthoffset(ARR2) + offsety2 = llmemory.itemoffsetof(ARR2, 0) + + def f(): + a = lltype.malloc(ARR, 5, flavor='raw') + a2 = lltype.malloc(ARR2, 6, flavor='raw') + a2[0] = 1 + a[0] = 3 + adr = llmemory.cast_ptr_to_adr(a) + adr2 = llmemory.cast_ptr_to_adr(a2) + return ((adr + offsetx).signed[0] * 1000 + + (adr + offsety).signed[0] * 100 + + (adr2 + offsetx2).signed[0] * 10 + (adr2 + offsety2).signed[0]) + + fn = compile(f, []) + res = fn() + assert res == 5361 + def test_sizeof_array_with_no_length(): A = lltype.Array(lltype.Signed, hints={'nolength': True}) arraysize = llmemory.sizeof(A, 10) From noreply at buildbot.pypy.org Wed Jan 30 17:22:45 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 30 Jan 2013 17:22:45 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: merge Message-ID: <20130130162245.CBFD91C00A8@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60739:260450a5706c Date: 2013-01-30 18:22 +0200 http://bitbucket.org/pypy/pypy/changeset/260450a5706c/ Log: merge diff --git a/rpython/jit/backend/arm/assembler.py b/rpython/jit/backend/arm/assembler.py --- a/rpython/jit/backend/arm/assembler.py +++ b/rpython/jit/backend/arm/assembler.py @@ -8,9 +8,10 @@ N_REGISTERS_SAVED_BY_MALLOC, \ JITFRAME_FIXED_SIZE, FRAME_FIXED_SIZE from rpython.jit.backend.arm.codebuilder import ARMv7Builder, OverwritingBuilder -from rpython.jit.backend.arm.locations import get_fp_offset, imm +from rpython.jit.backend.arm.locations import get_fp_offset, imm, StackLocation from rpython.jit.backend.arm.regalloc import (Regalloc, ARMFrameManager, CoreRegisterManager, check_imm_arg, + VFPRegisterManager, operations as regalloc_operations, operations_with_guard as regalloc_operations_with_guard) from rpython.jit.backend.llsupport.asmmemmgr import MachineDataBlockWrapper @@ -373,7 +374,7 @@ # see ../x86/assembler.py:propagate_memoryerror_if_eax_is_null self.mc.CMP_ri(r.r0.value, 0) self.mc.B(self.propagate_exception_path, c=c.EQ) - + def _push_all_regs_to_jitframe(self, mc, ignored_regs, withfloats, callee_only=False): if callee_only: @@ -386,7 +387,15 @@ continue mc.STR_ri(gpr.value, r.fp.value, i * WORD) if withfloats: - assert 0, 'implement me' + if callee_only: + regs = VFPRegisterManager.save_around_call_regs + else: + regs = VFPRegisterManager.all_regs + for i, vfpr in enumerate(regs): + if vfpr in ignored_regs: + continue + # add the offset of the gpr_regs + mc.VSTR(vfpr.value, r.fp.value, imm=i * DOUBLE_WORD) def _build_failure_recovery(self, exc, withfloats=False): mc = ARMv7Builder() @@ -497,9 +506,8 @@ assert loc is not r.fp # for now v = loc.value else: - assert 0, 'fix for floats' assert loc.is_vfp_reg() - #v = len(VFPRegisterManager.all_regs) + loc.value + v = len(CoreRegisterManager.all_regs) + loc.value * 2 positions[i] = v * WORD # write down the positions of locs guardtok.faildescr.rd_locs = positions @@ -680,31 +688,35 @@ def assemble_bridge(self, faildescr, inputargs, operations, original_loop_token, log): - assert 0 - operations = self.setup(original_loop_token, operations) - descr_number = self.cpu.get_fail_descr_number(faildescr) + if not we_are_translated(): + # Arguments should be unique + assert len(set(inputargs)) == len(inputargs) + + self.setup(original_loop_token) + descr_number = compute_unique_id(faildescr) if log: operations = self._inject_debugging_code(faildescr, operations, 'b', descr_number) + assert isinstance(faildescr, AbstractFailDescr) - code = self._find_failure_recovery_bytecode(faildescr) - frame_depth = faildescr._arm_current_frame_depth - arglocs = self.decode_inputargs(code) - if not we_are_translated(): - assert len(inputargs) == len(arglocs) - regalloc = Regalloc(assembler=self, frame_manager=ARMFrameManager()) - regalloc.prepare_bridge(inputargs, arglocs, operations) + arglocs = self.rebuild_faillocs_from_descr(faildescr, inputargs) - sp_patch_location = self._prepare_sp_patch_position() + regalloc = Regalloc(assembler=self) + operations = regalloc.prepare_bridge(inputargs, arglocs, + operations, + self.current_clt.allgcrefs, + self.current_clt.frame_info) + + #sp_patch_location = self._prepare_sp_patch_position() startpos = self.mc.get_relative_pos() - frame_depth = self._assemble(operations, regalloc) + frame_depth = self._assemble(regalloc, inputargs, operations) codeendpos = self.mc.get_relative_pos() - self._patch_sp_offset(sp_patch_location, frame_depth) + #self._patch_sp_offset(sp_patch_location, frame_depth) self.write_pending_failure_recoveries() @@ -713,6 +725,7 @@ self.process_pending_guards(rawstart) self.fixup_target_tokens(rawstart) + # patch the jump from original guard self.patch_trace(faildescr, original_loop_token, rawstart, regalloc) @@ -720,10 +733,9 @@ # for the benefit of tests faildescr._arm_bridge_frame_depth = frame_depth if log: - self.mc._dump_trace(rawstart, 'bridge_%d.asm' % - self.cpu.total_compiled_bridges) - self.current_clt.frame_depth = max(self.current_clt.frame_depth, - frame_depth) + self.mc._dump_trace(rawstart, 'bridge.asm') + frame_depth = max(self.current_clt.frame_info.jfi_frame_depth, + frame_depth + JITFRAME_FIXED_SIZE) ops_offset = self.mc.ops_offset self.teardown() @@ -734,15 +746,26 @@ return AsmInfo(ops_offset, startpos + rawstart, codeendpos - startpos) - def _find_failure_recovery_bytecode(self, faildescr): - guard_stub_addr = faildescr._arm_failure_recovery_block - if guard_stub_addr == 0: - # This case should be prevented by the logic in compile.py: - # look for CNT_BUSY_FLAG, which disables tracing from a guard - # when another tracing from the same guard is already in progress. - raise BridgeAlreadyCompiled - # a guard requires 3 words to encode the jump to the exit code. - return guard_stub_addr + 3 * WORD + def rebuild_faillocs_from_descr(self, descr, inputargs): + locs = [] + GPR_REGS = len(CoreRegisterManager.all_regs) + VFP_REGS = len(VFPRegisterManager.all_regs) + input_i = 0 + for pos in descr.rd_locs: + if pos == -1: + continue + elif pos < GPR_REGS * WORD: + locs.append(CoreRegisterManager.all_regs[pos // WORD]) + elif pos < (GPR_REGS * WORD + VFP_REGS * DOUBLE_WORD): + pos = pos // DOUBLE_WORD - GPR_REGS * WORD // DOUBLE_WORD + locs.append(VFPRegisterManager.all_regs[pos]) + else: + i = pos // WORD - JITFRAME_FIXED_SIZE + assert i >= 0 + tp = inputargs[input_i].type + locs.append(StackLocation(i, pos, tp)) + input_i += 1 + return locs def fixup_target_tokens(self, rawstart): for targettoken in self.target_tokens_currently_compiling: diff --git a/rpython/jit/backend/arm/regalloc.py b/rpython/jit/backend/arm/regalloc.py --- a/rpython/jit/backend/arm/regalloc.py +++ b/rpython/jit/backend/arm/regalloc.py @@ -303,9 +303,12 @@ self.possibly_free_vars(list(inputargs)) return operations - def prepare_bridge(self, inputargs, arglocs, ops): - self._prepare(inputargs, ops) + def prepare_bridge(self, inputargs, arglocs, operations, allgcrefs, + frame_info): + operations = self._prepare(inputargs, operations, allgcrefs) self._update_bindings(arglocs, inputargs) + return operations + def get_final_frame_depth(self): return self.frame_manager.get_frame_depth() @@ -320,16 +323,19 @@ used = {} i = 0 for loc in locs: + if loc is None: + loc = r.fp arg = inputargs[i] i += 1 if loc.is_reg(): self.rm.reg_bindings[arg] = loc + used[loc] = None elif loc.is_vfp_reg(): self.vfprm.reg_bindings[arg] = loc + used[loc] = None else: assert loc.is_stack() - self.frame_manager.set_binding(arg, loc) - used[loc] = None + self.frame_manager.bind(arg, loc) # XXX combine with x86 code and move to llsupport self.rm.free_regs = [] From noreply at buildbot.pypy.org Wed Jan 30 17:30:15 2013 From: noreply at buildbot.pypy.org (fijal) Date: Wed, 30 Jan 2013 17:30:15 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: memcpy called like that cannot collect (and does not free the GIL either) Message-ID: <20130130163015.7A8871C0246@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60740:c8a77fc582c4 Date: 2013-01-30 18:29 +0200 http://bitbucket.org/pypy/pypy/changeset/c8a77fc582c4/ Log: memcpy called like that cannot collect (and does not free the GIL either) diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -1143,9 +1143,10 @@ return genop_cmp_guard_float def _emit_call(self, x, arglocs, start=0, tmp=eax, - argtypes=None, callconv=FFI_DEFAULT_ABI): + argtypes=None, callconv=FFI_DEFAULT_ABI, can_collect=True): if IS_X86_64: - return self._emit_call_64(x, arglocs, start, argtypes) + return self._emit_call_64(x, arglocs, start, argtypes, + can_collect=can_collect) XXX p = 0 n = len(arglocs) @@ -1183,7 +1184,7 @@ # the called function just added 'p' to ESP, by subtracting it again. self.mc.SUB_ri(esp.value, p) - def _emit_call_64(self, x, arglocs, start, argtypes): + def _emit_call_64(self, x, arglocs, start, argtypes, can_collect=True): src_locs = [] dst_locs = [] xmm_src_locs = [] @@ -1261,13 +1262,16 @@ dst_locs.append(r10) x = r10 remap_frame_layout(self, src_locs, dst_locs, X86_64_SCRATCH_REG) - self.push_gcmap(self.mc, self._regalloc.get_gcmap([eax], noregs=True), - store=True) + if can_collect: + self.push_gcmap(self.mc, self._regalloc.get_gcmap([eax], noregs=True), + store=True) self.mc.CALL(x) - self._reload_frame_if_necessary(self.mc) + if can_collect: + self._reload_frame_if_necessary(self.mc) if align: self.mc.ADD_ri(esp.value, align * WORD) - self.pop_gcmap(self.mc) + if can_collect: + self.pop_gcmap(self.mc) def _reload_frame_if_necessary(self, mc): gcrootmap = self.cpu.gc_ll_descr.gcrootmap diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py --- a/rpython/jit/backend/x86/regalloc.py +++ b/rpython/jit/backend/x86/regalloc.py @@ -1151,7 +1151,8 @@ self.rm.before_call() self.xrm.before_call() self.assembler._emit_call(imm(self.assembler.memcpy_addr), - [dstaddr_loc, srcaddr_loc, length_loc]) + [dstaddr_loc, srcaddr_loc, length_loc], + can_collect=False) self.rm.possibly_free_var(length_box) self.rm.possibly_free_var(dstaddr_box) self.rm.possibly_free_var(srcaddr_box) From noreply at buildbot.pypy.org Wed Jan 30 19:28:40 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Wed, 30 Jan 2013 19:28:40 +0100 (CET) Subject: [pypy-commit] pypy default: we now use __length_hint__ Message-ID: <20130130182840.D71521C1028@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: Changeset: r60741:da784fe683c0 Date: 2013-01-30 10:28 -0800 http://bitbucket.org/pypy/pypy/changeset/da784fe683c0/ Log: we now use __length_hint__ diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -318,10 +318,5 @@ type and vice versa. For builtin types, a dictionary will be returned that cannot be changed (but still looks and behaves like a normal dictionary). -* the ``__len__`` or ``__length_hint__`` special methods are sometimes - called by CPython to get a length estimate to preallocate internal arrays. - So far, PyPy never calls ``__len__`` for this purpose, and never calls - ``__length_hint__`` at all. - .. include:: _ref.txt From noreply at buildbot.pypy.org Wed Jan 30 20:23:46 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 30 Jan 2013 20:23:46 +0100 (CET) Subject: [pypy-commit] pypy stdlib-2.7.4: merge default Message-ID: <20130130192346.6F0951C0246@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: stdlib-2.7.4 Changeset: r60742:6b92769a66d3 Date: 2013-01-30 08:01 -0500 http://bitbucket.org/pypy/pypy/changeset/6b92769a66d3/ Log: merge default diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst --- a/pypy/doc/faq.rst +++ b/pypy/doc/faq.rst @@ -300,12 +300,26 @@ Do I have to rewrite my programs in RPython? -------------------------------------------- -No. And you shouldn't try. PyPy always runs your code in its own interpreter, which is a -full and compliant Python 2.7 interpreter. RPython is only the -language in which parts of PyPy itself are written and extension -modules for it. Not only is it not necessary for you to rewrite your -code in RPython, it probably won't give you any speed improvements if you -try. +No. And you shouldn't try. First and foremost, RPython is a language +that is designed to write interpreters in. It is a restricted subset of +Python. If you program is not an interpreter but tries to do "real +things", like use *any* part of the standard Python library or *any* +3rd-party library, then it is not RPython to start with. You should +only look at RPython if you try to `write your own interpreter`__. + +.. __: `how do I compile my own interpreters`_ + +If your goal is to speed up Python code, then look at the regular PyPy, +which is a full and compliant Python 2.7 interpreter (which happens to +be written in RPython). Not only is it not necessary for you to rewrite +your code in RPython, it might not give you any speed improvements even +if you manage to. + +Yes, it is possible with enough effort to compile small self-contained +pieces of RPython code doing a few performance-sensitive things. But +this case is not interesting for us. If you needed to rewrite the code +in RPython, you could as well have rewritten it in C for example. The +latter is a much more supported, much more documented language `:-)` --------------------------------------------------- Which backends are there for the RPython toolchain? diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -1482,9 +1482,10 @@ ) raise w_fd = self.call_function(w_fileno) - if not self.isinstance_w(w_fd, self.w_int): + if (not self.isinstance_w(w_fd, self.w_int) and + not self.isinstance_w(w_fd, self.w_long)): raise OperationError(self.w_TypeError, - self.wrap("fileno() must return an integer") + self.wrap("fileno() returned a non-integer") ) fd = self.c_int_w(w_fd) if fd < 0: diff --git a/pypy/module/fcntl/test/test_fcntl.py b/pypy/module/fcntl/test/test_fcntl.py --- a/pypy/module/fcntl/test/test_fcntl.py +++ b/pypy/module/fcntl/test/test_fcntl.py @@ -22,13 +22,23 @@ import sys import struct + class F: + def __init__(self, fn): + self.fn = fn + def fileno(self): + return self.fn + f = open(self.tmp + "b", "w+") fcntl.fcntl(f, 1, 0) fcntl.fcntl(f, 1) + fcntl.fcntl(F(long(f.fileno())), 1) raises(TypeError, fcntl.fcntl, "foo") raises(TypeError, fcntl.fcntl, f, "foo") - raises((IOError, ValueError), fcntl.fcntl, -1, 1, 0) + raises(TypeError, fcntl.fcntl, F("foo"), 1) + raises(ValueError, fcntl.fcntl, -1, 1, 0) + raises(ValueError, fcntl.fcntl, F(-1), 1, 0) + raises(ValueError, fcntl.fcntl, F(long(-1)), 1, 0) assert fcntl.fcntl(f, 1, 0) == 0 assert fcntl.fcntl(f, 2, "foo") == "foo" assert fcntl.fcntl(f, 2, buffer("foo")) == "foo" diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py --- a/rpython/jit/metainterp/test/test_ajit.py +++ b/rpython/jit/metainterp/test/test_ajit.py @@ -1694,14 +1694,13 @@ assert res == -2 def test_guard_always_changing_value(self): - myjitdriver = JitDriver(greens = [], reds = ['x']) - class A: - pass + myjitdriver = JitDriver(greens = [], reds = ['x', 'a']) def f(x): + a = 0 while x > 0: - myjitdriver.can_enter_jit(x=x) - myjitdriver.jit_merge_point(x=x) - a = A() + myjitdriver.can_enter_jit(x=x, a=a) + myjitdriver.jit_merge_point(x=x, a=a) + a += 1 promote(a) x -= 1 self.meta_interp(f, [50]) From noreply at buildbot.pypy.org Wed Jan 30 20:23:47 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Wed, 30 Jan 2013 20:23:47 +0100 (CET) Subject: [pypy-commit] pypy stdlib-2.7.4: fix/test fcntl ValueError on invalid fd Message-ID: <20130130192347.AD20A1C0246@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: stdlib-2.7.4 Changeset: r60743:b7e7e4b395c7 Date: 2013-01-30 08:05 -0500 http://bitbucket.org/pypy/pypy/changeset/b7e7e4b395c7/ Log: fix/test fcntl ValueError on invalid fd diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -1487,8 +1487,8 @@ raise OperationError(self.w_TypeError, self.wrap("fileno() returned a non-integer") ) - fd = self.c_int_w(w_fd) - if fd < 0: + fd = self.int_w(w_fd) + if fd < 0 or fd > 2147483647: raise operationerrfmt(self.w_ValueError, "file descriptor cannot be a negative integer (%d)", fd ) diff --git a/pypy/module/fcntl/test/test_fcntl.py b/pypy/module/fcntl/test/test_fcntl.py --- a/pypy/module/fcntl/test/test_fcntl.py +++ b/pypy/module/fcntl/test/test_fcntl.py @@ -36,6 +36,10 @@ raises(TypeError, fcntl.fcntl, "foo") raises(TypeError, fcntl.fcntl, f, "foo") raises(TypeError, fcntl.fcntl, F("foo"), 1) + raises(ValueError, fcntl.fcntl, 2147483647 + 1, 1, 0) + raises(ValueError, fcntl.fcntl, F(2147483647 + 1), 1, 0) + raises(ValueError, fcntl.fcntl, -2147483648 - 1, 1, 0) + raises(ValueError, fcntl.fcntl, F(-2147483648 - 1), 1, 0) raises(ValueError, fcntl.fcntl, -1, 1, 0) raises(ValueError, fcntl.fcntl, F(-1), 1, 0) raises(ValueError, fcntl.fcntl, F(long(-1)), 1, 0) From noreply at buildbot.pypy.org Wed Jan 30 22:44:04 2013 From: noreply at buildbot.pypy.org (pjenvey) Date: Wed, 30 Jan 2013 22:44:04 +0100 (CET) Subject: [pypy-commit] pypy py3k: really fix translation this time Message-ID: <20130130214404.246101C0DC1@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r60744:3334c8f5aea5 Date: 2013-01-30 13:28 -0800 http://bitbucket.org/pypy/pypy/changeset/3334c8f5aea5/ Log: really fix translation this time diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -649,29 +649,25 @@ self.space.delitem(self.w_globals, w_varname) def LOAD_NAME(self, nameindex, next_instr): + w_varname = self.getname_w(nameindex) if self.w_locals is not self.w_globals: - w_varname = self.getname_w(nameindex) w_value = self.space.finditem(self.w_locals, w_varname) if w_value is not None: self.pushvalue(w_value) return # fall-back - self.LOAD_GLOBAL(nameindex, next_instr, self._load_name_failed) + varname = self.space.str_w(w_varname) + w_value = self._load_global(varname) + if w_value is None: + message = "name '%s' is not defined" + raise operationerrfmt(self.space.w_NameError, message, varname) + self.pushvalue(w_value) - def _load_name_failed(self, varname): - message = "name '%s' is not defined" - raise operationerrfmt(self.space.w_NameError, message, varname) - _load_name_failed._dont_inline_ = True - - def _load_global(self, varname, error_handler): + def _load_global(self, varname): w_value = self.space.finditem_str(self.w_globals, varname) if w_value is None: # not in the globals, now look in the built-ins w_value = self.get_builtin().getdictvalue(self.space, varname) - if w_value is None: - if error_handler is None: - error_handler = self._load_global_failed - error_handler(varname) return w_value _load_global._always_inline_ = True @@ -680,9 +676,12 @@ raise operationerrfmt(self.space.w_NameError, message, varname) _load_global_failed._dont_inline_ = True - def LOAD_GLOBAL(self, nameindex, next_instr, error_handler=None): - self.pushvalue(self._load_global(self.getname_u(nameindex), - error_handler)) + def LOAD_GLOBAL(self, nameindex, next_instr): + varname = self.getname_u(nameindex) + w_value = self._load_global(varname) + if w_value is None: + self._load_global_failed(varname) + self.pushvalue(w_value) LOAD_GLOBAL._always_inline_ = True def DELETE_FAST(self, varindex, next_instr): From noreply at buildbot.pypy.org Thu Jan 31 04:30:53 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 31 Jan 2013 04:30:53 +0100 (CET) Subject: [pypy-commit] pypy default: fileio raise exception when given fd pointing to dir (cpython issue15247) Message-ID: <20130131033053.19B6A1C00A8@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60745:0bc690239b1a Date: 2013-01-30 18:36 -0500 http://bitbucket.org/pypy/pypy/changeset/0bc690239b1a/ Log: fileio raise exception when given fd pointing to dir (cpython issue15247) diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py --- a/pypy/module/_io/interp_fileio.py +++ b/pypy/module/_io/interp_fileio.py @@ -153,41 +153,49 @@ self.readable, self.writable, append, flags = decode_mode(space, mode) - if fd >= 0: - verify_fd(fd) - try: - os.fstat(fd) - except OSError, e: - if e.errno == errno.EBADF: - raise wrap_oserror(space, e) - # else: pass - self.fd = fd - self.closefd = bool(closefd) - else: - if not closefd: - raise OperationError(space.w_ValueError, space.wrap( - "Cannot use closefd=False with file name")) - self.closefd = True + fd_is_own = False + try: + if fd >= 0: + verify_fd(fd) + try: + os.fstat(fd) + except OSError, e: + if e.errno == errno.EBADF: + raise wrap_oserror(space, e) + # else: pass + self.fd = fd + self.closefd = bool(closefd) + else: + self.closefd = True + if not closefd: + raise OperationError(space.w_ValueError, space.wrap( + "Cannot use closefd=False with file name")) - from pypy.module.posix.interp_posix import ( - dispatch_filename, rposix) - try: - self.fd = dispatch_filename(rposix.open)( - space, w_name, flags, 0666) - except OSError, e: - raise wrap_oserror2(space, e, w_name, - exception_name='w_IOError') + from pypy.module.posix.interp_posix import ( + dispatch_filename, rposix) + try: + self.fd = dispatch_filename(rposix.open)( + space, w_name, flags, 0666) + except OSError, e: + raise wrap_oserror2(space, e, w_name, + exception_name='w_IOError') + finally: + fd_is_own = True self._dircheck(space, w_name) - self.w_name = w_name + self.w_name = w_name - if append: - # For consistent behaviour, we explicitly seek to the end of file - # (otherwise, it might be done only on the first write()). - try: - os.lseek(self.fd, 0, os.SEEK_END) - except OSError, e: - raise wrap_oserror(space, e, exception_name='w_IOError') + if append: + # For consistent behaviour, we explicitly seek to the end of file + # (otherwise, it might be done only on the first write()). + try: + os.lseek(self.fd, 0, os.SEEK_END) + except OSError, e: + raise wrap_oserror(space, e, exception_name='w_IOError') + except: + if not fd_is_own: + self.fd = -1 + raise def _mode(self): if self.readable: @@ -253,7 +261,6 @@ except OSError: return if stat.S_ISDIR(st.st_mode): - self._close(space) raise wrap_oserror2(space, OSError(errno.EISDIR, "fstat"), w_filename, exception_name='w_IOError') diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -41,7 +41,11 @@ def test_open_directory(self): import _io + import os raises(IOError, _io.FileIO, self.tmpdir, "rb") + fd = os.open(self.tmpdir, os.O_RDONLY) + raises(IOError, _io.FileIO, fd, "rb") + os.close(fd) def test_readline(self): import _io From noreply at buildbot.pypy.org Thu Jan 31 04:30:54 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 31 Jan 2013 04:30:54 +0100 (CET) Subject: [pypy-commit] pypy default: fix PyObject_Str/Repr/Unicode for NULL argument, test Message-ID: <20130131033054.764271C0246@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60746:4dc55ac8a8fa Date: 2013-01-30 22:07 -0500 http://bitbucket.org/pypy/pypy/changeset/4dc55ac8a8fa/ Log: fix PyObject_Str/Repr/Unicode for NULL argument, test diff --git a/lib-python/2.7/test/test_capi.py b/lib-python/2.7/test/test_capi.py --- a/lib-python/2.7/test/test_capi.py +++ b/lib-python/2.7/test/test_capi.py @@ -21,7 +21,6 @@ 'test_lazy_hash_inheritance', 'test_long_api', 'test_longlong_api', - 'test_null_strings', 'test_widechar', 'TestThreadState', 'TestPendingCalls', diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py --- a/pypy/module/cpyext/object.py +++ b/pypy/module/cpyext/object.py @@ -218,6 +218,8 @@ @cpython_api([PyObject], PyObject) def PyObject_Str(space, w_obj): + if w_obj is None: + return space.wrap("") return space.str(w_obj) @cpython_api([PyObject], PyObject) @@ -226,6 +228,8 @@ representation on success, NULL on failure. This is the equivalent of the Python expression repr(o). Called by the repr() built-in function and by reverse quotes.""" + if w_obj is None: + return space.wrap("") return space.repr(w_obj) @cpython_api([PyObject], PyObject) @@ -234,6 +238,8 @@ string representation on success, NULL on failure. This is the equivalent of the Python expression unicode(o). Called by the unicode() built-in function.""" + if w_obj is None: + return space.wrap(u"") return space.call_function(space.w_unicode, w_obj) @cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py --- a/pypy/module/cpyext/test/test_object.py +++ b/pypy/module/cpyext/test/test_object.py @@ -94,27 +94,30 @@ def test_size(self, space, api): assert api.PyObject_Size(space.newlist([space.w_None])) == 1 - + + def test_str(self, space, api): + w_list = space.newlist([space.w_None, space.wrap(42)]) + assert space.str_w(api.PyObject_Str(None)) == "" + assert space.str_w(api.PyObject_Str(w_list)) == "[None, 42]" + assert space.str_w(api.PyObject_Str(space.wrap("a"))) == "a" + def test_repr(self, space, api): w_list = space.newlist([space.w_None, space.wrap(42)]) + assert space.str_w(api.PyObject_Repr(None)) == "" assert space.str_w(api.PyObject_Repr(w_list)) == "[None, 42]" assert space.str_w(api.PyObject_Repr(space.wrap("a"))) == "'a'" - - w_list = space.newlist([space.w_None, space.wrap(42)]) - assert space.str_w(api.PyObject_Str(w_list)) == "[None, 42]" - assert space.str_w(api.PyObject_Str(space.wrap("a"))) == "a" - + def test_RichCompare(self, space, api): def compare(w_o1, w_o2, opid): res = api.PyObject_RichCompareBool(w_o1, w_o2, opid) w_res = api.PyObject_RichCompare(w_o1, w_o2, opid) assert space.is_true(w_res) == res return res - + def test_compare(o1, o2): w_o1 = space.wrap(o1) w_o2 = space.wrap(o2) - + for opid, expected in [ (Py_LT, o1 < o2), (Py_LE, o1 <= o2), (Py_NE, o1 != o2), (Py_EQ, o1 == o2), @@ -190,6 +193,7 @@ api.PyErr_Clear() def test_unicode(self, space, api): + assert space.unwrap(api.PyObject_Unicode(None)) == u"" assert space.unwrap(api.PyObject_Unicode(space.wrap([]))) == u"[]" assert space.unwrap(api.PyObject_Unicode(space.wrap("e"))) == u"e" assert api.PyObject_Unicode(space.wrap("\xe9")) is None From noreply at buildbot.pypy.org Thu Jan 31 05:41:09 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 31 Jan 2013 05:41:09 +0100 (CET) Subject: [pypy-commit] pypy default: missing sys.path line Message-ID: <20130131044109.AB7231C0041@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60747:1e66ac6b5318 Date: 2013-01-30 23:40 -0500 http://bitbucket.org/pypy/pypy/changeset/1e66ac6b5318/ Log: missing sys.path line diff --git a/pypy/bin/checkmodule.py b/pypy/bin/checkmodule.py --- a/pypy/bin/checkmodule.py +++ b/pypy/bin/checkmodule.py @@ -8,6 +8,8 @@ """ import sys, os +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) + from pypy.objspace.fake.checkmodule import checkmodule def main(argv): From noreply at buildbot.pypy.org Thu Jan 31 07:41:04 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 31 Jan 2013 07:41:04 +0100 (CET) Subject: [pypy-commit] pypy default: fix test_capi's test_long/test_longlong Message-ID: <20130131064104.1418A1C009B@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60748:c8febcb60f46 Date: 2013-01-31 00:59 -0500 http://bitbucket.org/pypy/pypy/changeset/c8febcb60f46/ Log: fix test_capi's test_long/test_longlong diff --git a/lib-python/2.7/test/test_capi.py b/lib-python/2.7/test/test_capi.py --- a/lib-python/2.7/test/test_capi.py +++ b/lib-python/2.7/test/test_capi.py @@ -19,8 +19,6 @@ 'test_broken_memoryview', 'test_capsule', 'test_lazy_hash_inheritance', - 'test_long_api', - 'test_longlong_api', 'test_widechar', 'TestThreadState', 'TestPendingCalls', diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py --- a/pypy/module/cpyext/longobject.py +++ b/pypy/module/cpyext/longobject.py @@ -47,7 +47,12 @@ Return a C unsigned long representation of the contents of pylong. If pylong is greater than ULONG_MAX, an OverflowError is raised.""" - return rffi.cast(rffi.ULONG, space.uint_w(w_long)) + try: + return rffi.cast(rffi.ULONG, space.uint_w(w_long)) + except OperationError, e: + if e.match(space, space.w_ValueError): + e.w_type = space.w_OverflowError + raise @cpython_api([PyObject], rffi.ULONG, error=-1) def PyLong_AsUnsignedLongMask(space, w_long): @@ -86,7 +91,12 @@ Return a C unsigned long representation of the contents of pylong. If pylong is greater than ULONG_MAX, an OverflowError is raised.""" - return rffi.cast(rffi.ULONGLONG, space.r_ulonglong_w(w_long)) + try: + return rffi.cast(rffi.ULONGLONG, space.r_ulonglong_w(w_long)) + except OperationError, e: + if e.match(space, space.w_ValueError): + e.w_type = space.w_OverflowError + raise @cpython_api([PyObject], rffi.ULONGLONG, error=-1) def PyLong_AsUnsignedLongLongMask(space, w_long): diff --git a/pypy/module/cpyext/test/test_longobject.py b/pypy/module/cpyext/test/test_longobject.py --- a/pypy/module/cpyext/test/test_longobject.py +++ b/pypy/module/cpyext/test/test_longobject.py @@ -31,6 +31,8 @@ value = api.PyLong_AsUnsignedLong(w_value) assert value == (sys.maxint - 1) * 2 + self.raises(space, api, OverflowError, api.PyLong_AsUnsignedLong, space.wrap(-1)) + def test_as_ssize_t(self, space, api): w_value = space.newlong(2) value = api.PyLong_AsSsize_t(w_value) @@ -46,11 +48,11 @@ w_l = space.wrap(sys.maxint + 1) assert api.PyLong_Check(w_l) assert api.PyLong_CheckExact(w_l) - + w_i = space.wrap(sys.maxint) assert not api.PyLong_Check(w_i) assert not api.PyLong_CheckExact(w_i) - + L = space.appexec([], """(): class L(long): pass @@ -73,6 +75,8 @@ assert api.PyLong_AsUnsignedLongLongMask( space.wrap(1<<64)) == 0 + self.raises(space, api, OverflowError, api.PyLong_AsUnsignedLongLong, space.wrap(-1)) + def test_as_long_and_overflow(self, space, api): overflow = lltype.malloc(rffi.CArrayPtr(rffi.INT_real).TO, 1, flavor='raw') assert api.PyLong_AsLongAndOverflow( From noreply at buildbot.pypy.org Thu Jan 31 07:41:05 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 31 Jan 2013 07:41:05 +0100 (CET) Subject: [pypy-commit] pypy default: fix sysconfig.py paths Message-ID: <20130131064105.5EDB11C009B@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60749:f4706c9d56b1 Date: 2013-01-31 01:38 -0500 http://bitbucket.org/pypy/pypy/changeset/f4706c9d56b1/ Log: fix sysconfig.py paths diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py --- a/lib-python/2.7/sysconfig.py +++ b/lib-python/2.7/sysconfig.py @@ -27,10 +27,10 @@ 'data' : '{base}', }, 'pypy': { - 'stdlib': '{base}/lib-python', - 'platstdlib': '{base}/lib-python', - 'purelib': '{base}/lib-python', - 'platlib': '{base}/lib-python', + 'stdlib': '{base}/lib-python/{py_version_short}', + 'platstdlib': '{base}/lib-python/{py_version_short}', + 'purelib': '{base}/lib-python/{py_version_short}', + 'platlib': '{base}/lib-python/{py_version_short}', 'include': '{base}/include', 'platinclude': '{base}/include', 'scripts': '{base}/bin', From noreply at buildbot.pypy.org Thu Jan 31 07:43:07 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 31 Jan 2013 07:43:07 +0100 (CET) Subject: [pypy-commit] pypy stdlib-2.7.4: merge default Message-ID: <20130131064307.8F5441C05D7@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: stdlib-2.7.4 Changeset: r60750:5b8631b71974 Date: 2013-01-31 01:42 -0500 http://bitbucket.org/pypy/pypy/changeset/5b8631b71974/ Log: merge default diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py --- a/lib-python/2.7/sysconfig.py +++ b/lib-python/2.7/sysconfig.py @@ -27,10 +27,10 @@ 'data' : '{base}', }, 'pypy': { - 'stdlib': '{base}/lib-python', - 'platstdlib': '{base}/lib-python', - 'purelib': '{base}/lib-python', - 'platlib': '{base}/lib-python', + 'stdlib': '{base}/lib-python/{py_version_short}', + 'platstdlib': '{base}/lib-python/{py_version_short}', + 'purelib': '{base}/lib-python/{py_version_short}', + 'platlib': '{base}/lib-python/{py_version_short}', 'include': '{base}/include', 'platinclude': '{base}/include', 'scripts': '{base}/bin', diff --git a/lib-python/2.7/test/test_capi.py b/lib-python/2.7/test/test_capi.py --- a/lib-python/2.7/test/test_capi.py +++ b/lib-python/2.7/test/test_capi.py @@ -19,9 +19,6 @@ 'test_broken_memoryview', 'test_capsule', 'test_lazy_hash_inheritance', - 'test_long_api', - 'test_longlong_api', - 'test_null_strings', 'test_widechar', 'TestThreadState', 'TestPendingCalls', diff --git a/pypy/bin/checkmodule.py b/pypy/bin/checkmodule.py --- a/pypy/bin/checkmodule.py +++ b/pypy/bin/checkmodule.py @@ -8,6 +8,8 @@ """ import sys, os +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) + from pypy.objspace.fake.checkmodule import checkmodule def main(argv): diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -318,10 +318,5 @@ type and vice versa. For builtin types, a dictionary will be returned that cannot be changed (but still looks and behaves like a normal dictionary). -* the ``__len__`` or ``__length_hint__`` special methods are sometimes - called by CPython to get a length estimate to preallocate internal arrays. - So far, PyPy never calls ``__len__`` for this purpose, and never calls - ``__length_hint__`` at all. - .. include:: _ref.txt diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py --- a/pypy/module/_io/interp_fileio.py +++ b/pypy/module/_io/interp_fileio.py @@ -153,41 +153,49 @@ self.readable, self.writable, append, flags = decode_mode(space, mode) - if fd >= 0: - verify_fd(fd) - try: - os.fstat(fd) - except OSError, e: - if e.errno == errno.EBADF: - raise wrap_oserror(space, e) - # else: pass - self.fd = fd - self.closefd = bool(closefd) - else: - if not closefd: - raise OperationError(space.w_ValueError, space.wrap( - "Cannot use closefd=False with file name")) - self.closefd = True + fd_is_own = False + try: + if fd >= 0: + verify_fd(fd) + try: + os.fstat(fd) + except OSError, e: + if e.errno == errno.EBADF: + raise wrap_oserror(space, e) + # else: pass + self.fd = fd + self.closefd = bool(closefd) + else: + self.closefd = True + if not closefd: + raise OperationError(space.w_ValueError, space.wrap( + "Cannot use closefd=False with file name")) - from pypy.module.posix.interp_posix import ( - dispatch_filename, rposix) - try: - self.fd = dispatch_filename(rposix.open)( - space, w_name, flags, 0666) - except OSError, e: - raise wrap_oserror2(space, e, w_name, - exception_name='w_IOError') + from pypy.module.posix.interp_posix import ( + dispatch_filename, rposix) + try: + self.fd = dispatch_filename(rposix.open)( + space, w_name, flags, 0666) + except OSError, e: + raise wrap_oserror2(space, e, w_name, + exception_name='w_IOError') + finally: + fd_is_own = True self._dircheck(space, w_name) - self.w_name = w_name + self.w_name = w_name - if append: - # For consistent behaviour, we explicitly seek to the end of file - # (otherwise, it might be done only on the first write()). - try: - os.lseek(self.fd, 0, os.SEEK_END) - except OSError, e: - raise wrap_oserror(space, e, exception_name='w_IOError') + if append: + # For consistent behaviour, we explicitly seek to the end of file + # (otherwise, it might be done only on the first write()). + try: + os.lseek(self.fd, 0, os.SEEK_END) + except OSError, e: + raise wrap_oserror(space, e, exception_name='w_IOError') + except: + if not fd_is_own: + self.fd = -1 + raise def _mode(self): if self.readable: @@ -253,7 +261,6 @@ except OSError: return if stat.S_ISDIR(st.st_mode): - self._close(space) raise wrap_oserror2(space, OSError(errno.EISDIR, "fstat"), w_filename, exception_name='w_IOError') diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -41,7 +41,11 @@ def test_open_directory(self): import _io + import os raises(IOError, _io.FileIO, self.tmpdir, "rb") + fd = os.open(self.tmpdir, os.O_RDONLY) + raises(IOError, _io.FileIO, fd, "rb") + os.close(fd) def test_readline(self): import _io diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py --- a/pypy/module/cpyext/longobject.py +++ b/pypy/module/cpyext/longobject.py @@ -47,7 +47,12 @@ Return a C unsigned long representation of the contents of pylong. If pylong is greater than ULONG_MAX, an OverflowError is raised.""" - return rffi.cast(rffi.ULONG, space.uint_w(w_long)) + try: + return rffi.cast(rffi.ULONG, space.uint_w(w_long)) + except OperationError, e: + if e.match(space, space.w_ValueError): + e.w_type = space.w_OverflowError + raise @cpython_api([PyObject], rffi.ULONG, error=-1) def PyLong_AsUnsignedLongMask(space, w_long): @@ -86,7 +91,12 @@ Return a C unsigned long representation of the contents of pylong. If pylong is greater than ULONG_MAX, an OverflowError is raised.""" - return rffi.cast(rffi.ULONGLONG, space.r_ulonglong_w(w_long)) + try: + return rffi.cast(rffi.ULONGLONG, space.r_ulonglong_w(w_long)) + except OperationError, e: + if e.match(space, space.w_ValueError): + e.w_type = space.w_OverflowError + raise @cpython_api([PyObject], rffi.ULONGLONG, error=-1) def PyLong_AsUnsignedLongLongMask(space, w_long): diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py --- a/pypy/module/cpyext/object.py +++ b/pypy/module/cpyext/object.py @@ -218,6 +218,8 @@ @cpython_api([PyObject], PyObject) def PyObject_Str(space, w_obj): + if w_obj is None: + return space.wrap("") return space.str(w_obj) @cpython_api([PyObject], PyObject) @@ -226,6 +228,8 @@ representation on success, NULL on failure. This is the equivalent of the Python expression repr(o). Called by the repr() built-in function and by reverse quotes.""" + if w_obj is None: + return space.wrap("") return space.repr(w_obj) @cpython_api([PyObject], PyObject) @@ -234,6 +238,8 @@ string representation on success, NULL on failure. This is the equivalent of the Python expression unicode(o). Called by the unicode() built-in function.""" + if w_obj is None: + return space.wrap(u"") return space.call_function(space.w_unicode, w_obj) @cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) diff --git a/pypy/module/cpyext/test/test_longobject.py b/pypy/module/cpyext/test/test_longobject.py --- a/pypy/module/cpyext/test/test_longobject.py +++ b/pypy/module/cpyext/test/test_longobject.py @@ -31,6 +31,8 @@ value = api.PyLong_AsUnsignedLong(w_value) assert value == (sys.maxint - 1) * 2 + self.raises(space, api, OverflowError, api.PyLong_AsUnsignedLong, space.wrap(-1)) + def test_as_ssize_t(self, space, api): w_value = space.newlong(2) value = api.PyLong_AsSsize_t(w_value) @@ -46,11 +48,11 @@ w_l = space.wrap(sys.maxint + 1) assert api.PyLong_Check(w_l) assert api.PyLong_CheckExact(w_l) - + w_i = space.wrap(sys.maxint) assert not api.PyLong_Check(w_i) assert not api.PyLong_CheckExact(w_i) - + L = space.appexec([], """(): class L(long): pass @@ -73,6 +75,8 @@ assert api.PyLong_AsUnsignedLongLongMask( space.wrap(1<<64)) == 0 + self.raises(space, api, OverflowError, api.PyLong_AsUnsignedLongLong, space.wrap(-1)) + def test_as_long_and_overflow(self, space, api): overflow = lltype.malloc(rffi.CArrayPtr(rffi.INT_real).TO, 1, flavor='raw') assert api.PyLong_AsLongAndOverflow( diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py --- a/pypy/module/cpyext/test/test_object.py +++ b/pypy/module/cpyext/test/test_object.py @@ -94,27 +94,30 @@ def test_size(self, space, api): assert api.PyObject_Size(space.newlist([space.w_None])) == 1 - + + def test_str(self, space, api): + w_list = space.newlist([space.w_None, space.wrap(42)]) + assert space.str_w(api.PyObject_Str(None)) == "" + assert space.str_w(api.PyObject_Str(w_list)) == "[None, 42]" + assert space.str_w(api.PyObject_Str(space.wrap("a"))) == "a" + def test_repr(self, space, api): w_list = space.newlist([space.w_None, space.wrap(42)]) + assert space.str_w(api.PyObject_Repr(None)) == "" assert space.str_w(api.PyObject_Repr(w_list)) == "[None, 42]" assert space.str_w(api.PyObject_Repr(space.wrap("a"))) == "'a'" - - w_list = space.newlist([space.w_None, space.wrap(42)]) - assert space.str_w(api.PyObject_Str(w_list)) == "[None, 42]" - assert space.str_w(api.PyObject_Str(space.wrap("a"))) == "a" - + def test_RichCompare(self, space, api): def compare(w_o1, w_o2, opid): res = api.PyObject_RichCompareBool(w_o1, w_o2, opid) w_res = api.PyObject_RichCompare(w_o1, w_o2, opid) assert space.is_true(w_res) == res return res - + def test_compare(o1, o2): w_o1 = space.wrap(o1) w_o2 = space.wrap(o2) - + for opid, expected in [ (Py_LT, o1 < o2), (Py_LE, o1 <= o2), (Py_NE, o1 != o2), (Py_EQ, o1 == o2), @@ -190,6 +193,7 @@ api.PyErr_Clear() def test_unicode(self, space, api): + assert space.unwrap(api.PyObject_Unicode(None)) == u"" assert space.unwrap(api.PyObject_Unicode(space.wrap([]))) == u"[]" assert space.unwrap(api.PyObject_Unicode(space.wrap("e"))) == u"e" assert api.PyObject_Unicode(space.wrap("\xe9")) is None From alex.gaynor at gmail.com Thu Jan 31 08:01:18 2013 From: alex.gaynor at gmail.com (Alex Gaynor) Date: Wed, 30 Jan 2013 23:01:18 -0800 Subject: [pypy-commit] pypy default: fix test_capi's test_long/test_longlong In-Reply-To: <20130131064104.1418A1C009B@cobra.cs.uni-duesseldorf.de> References: <20130131064104.1418A1C009B@cobra.cs.uni-duesseldorf.de> Message-ID: I don't think we want to be mutating w_type on an OperationError, I'm not even sure what it does, since w_value will have a different type from w_type. Alex On Wed, Jan 30, 2013 at 10:41 PM, bdkearns wrote: > Author: Brian Kearns > Branch: > Changeset: r60748:c8febcb60f46 > Date: 2013-01-31 00:59 -0500 > http://bitbucket.org/pypy/pypy/changeset/c8febcb60f46/ > > Log: fix test_capi's test_long/test_longlong > > diff --git a/lib-python/2.7/test/test_capi.py > b/lib-python/2.7/test/test_capi.py > --- a/lib-python/2.7/test/test_capi.py > +++ b/lib-python/2.7/test/test_capi.py > @@ -19,8 +19,6 @@ > 'test_broken_memoryview', > 'test_capsule', > 'test_lazy_hash_inheritance', > - 'test_long_api', > - 'test_longlong_api', > 'test_widechar', > 'TestThreadState', > 'TestPendingCalls', > diff --git a/pypy/module/cpyext/longobject.py > b/pypy/module/cpyext/longobject.py > --- a/pypy/module/cpyext/longobject.py > +++ b/pypy/module/cpyext/longobject.py > @@ -47,7 +47,12 @@ > Return a C unsigned long representation of the contents of pylong. > If pylong is greater than ULONG_MAX, an OverflowError is > raised.""" > - return rffi.cast(rffi.ULONG, space.uint_w(w_long)) > + try: > + return rffi.cast(rffi.ULONG, space.uint_w(w_long)) > + except OperationError, e: > + if e.match(space, space.w_ValueError): > + e.w_type = space.w_OverflowError > + raise > > @cpython_api([PyObject], rffi.ULONG, error=-1) > def PyLong_AsUnsignedLongMask(space, w_long): > @@ -86,7 +91,12 @@ > Return a C unsigned long representation of the contents of pylong. > If pylong is greater than ULONG_MAX, an OverflowError is > raised.""" > - return rffi.cast(rffi.ULONGLONG, space.r_ulonglong_w(w_long)) > + try: > + return rffi.cast(rffi.ULONGLONG, space.r_ulonglong_w(w_long)) > + except OperationError, e: > + if e.match(space, space.w_ValueError): > + e.w_type = space.w_OverflowError > + raise > > @cpython_api([PyObject], rffi.ULONGLONG, error=-1) > def PyLong_AsUnsignedLongLongMask(space, w_long): > diff --git a/pypy/module/cpyext/test/test_longobject.py > b/pypy/module/cpyext/test/test_longobject.py > --- a/pypy/module/cpyext/test/test_longobject.py > +++ b/pypy/module/cpyext/test/test_longobject.py > @@ -31,6 +31,8 @@ > value = api.PyLong_AsUnsignedLong(w_value) > assert value == (sys.maxint - 1) * 2 > > + self.raises(space, api, OverflowError, api.PyLong_AsUnsignedLong, > space.wrap(-1)) > + > def test_as_ssize_t(self, space, api): > w_value = space.newlong(2) > value = api.PyLong_AsSsize_t(w_value) > @@ -46,11 +48,11 @@ > w_l = space.wrap(sys.maxint + 1) > assert api.PyLong_Check(w_l) > assert api.PyLong_CheckExact(w_l) > - > + > w_i = space.wrap(sys.maxint) > assert not api.PyLong_Check(w_i) > assert not api.PyLong_CheckExact(w_i) > - > + > L = space.appexec([], """(): > class L(long): > pass > @@ -73,6 +75,8 @@ > assert api.PyLong_AsUnsignedLongLongMask( > space.wrap(1<<64)) == 0 > > + self.raises(space, api, OverflowError, > api.PyLong_AsUnsignedLongLong, space.wrap(-1)) > + > def test_as_long_and_overflow(self, space, api): > overflow = lltype.malloc(rffi.CArrayPtr(rffi.INT_real).TO, 1, > flavor='raw') > assert api.PyLong_AsLongAndOverflow( > _______________________________________________ > pypy-commit mailing list > pypy-commit at python.org > http://mail.python.org/mailman/listinfo/pypy-commit > -- "I disapprove of what you say, but I will defend to the death your right to say it." -- Evelyn Beatrice Hall (summarizing Voltaire) "The people's good is the highest law." -- Cicero -------------- next part -------------- An HTML attachment was scrubbed... URL: From noreply at buildbot.pypy.org Thu Jan 31 10:47:55 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 31 Jan 2013 10:47:55 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: Trying out using signals and signal.pause()... Message-ID: <20130131094755.2BABB1C009B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r60751:009b40655c5f Date: 2013-01-31 10:47 +0100 http://bitbucket.org/pypy/pypy/changeset/009b40655c5f/ Log: Trying out using signals and signal.pause()... diff --git a/lib_pypy/transaction.py b/lib_pypy/transaction.py --- a/lib_pypy/transaction.py +++ b/lib_pypy/transaction.py @@ -15,6 +15,12 @@ import sys, thread, collections try: + import signal, posix + from signal import SIGUSR2 as SIG_DONE +except ImportError: + signal = None + +try: from thread import atomic except ImportError: # Not a STM-enabled PyPy. We can still provide a version of 'atomic' @@ -109,12 +115,29 @@ self.in_transaction = True def run(self): + self.use_signals = hasattr(signal, 'pause') + if self.use_signals: + try: + self.signalled = 0 + def signal_handler(*args): + self.signalled = 1 + prev_handler = signal.signal(SIG_DONE, signal_handler) + except ValueError: + self.use_signals = False # start the N threads for i in range(self.num_threads): thread.start_new_thread(self._run_thread, ()) # now wait. When we manage to acquire the following lock, then # we are finished. - self.lock_if_released_then_finished.acquire() + if self.use_signals: + try: + while not self.signalled: + signal.pause() + finally: + self.use_signals = False + signal.signal(SIG_DONE, prev_handler) + else: + self.lock_if_released_then_finished.acquire() def teardown(self): self.in_transaction = False @@ -193,6 +216,9 @@ self.lock_mutex.release() if last_one_to_leave: self.lock_if_released_then_finished.release() + with atomic: + if self.use_signals: + posix.kill(posix.getpid(), SIG_DONE) raise _Done @staticmethod From noreply at buildbot.pypy.org Thu Jan 31 10:56:01 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 31 Jan 2013 10:56:01 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: we only ever allocate the jitframe before assembler, move Message-ID: <20130131095601.81DFD1C009B@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60752:5e9244c2920d Date: 2013-01-31 11:55 +0200 http://bitbucket.org/pypy/pypy/changeset/5e9244c2920d/ Log: we only ever allocate the jitframe before assembler, move gc_assume_young_pointers to malloc itself. diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py --- a/rpython/jit/backend/llsupport/gc.py +++ b/rpython/jit/backend/llsupport/gc.py @@ -139,6 +139,7 @@ """ Allocate a new frame, overwritten by tests """ frame = jitframe.JITFRAME.allocate(frame_info) + llop.gc_assume_young_pointers(lltype.Void, frame) return frame class JitFrameDescrs: diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py --- a/rpython/jit/backend/x86/runner.py +++ b/rpython/jit/backend/x86/runner.py @@ -137,9 +137,6 @@ assert kind == history.REF self.set_ref_value(ll_frame, num, arg) num += WORD - # no GC operation between gc_assume_young_pointers and - # the actual call to assembler! - llop.gc_assume_young_pointers(lltype.Void, frame) ll_frame = func(ll_frame) finally: if not self.translate_support_code: From noreply at buildbot.pypy.org Thu Jan 31 11:13:25 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 31 Jan 2013 11:13:25 +0100 (CET) Subject: [pypy-commit] pypy default: get test_multiprocessing.py working Message-ID: <20130131101325.88B3E1C040D@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60753:635fed2dc9c4 Date: 2013-01-31 03:23 -0500 http://bitbucket.org/pypy/pypy/changeset/635fed2dc9c4/ Log: get test_multiprocessing.py working diff --git a/lib-python/2.7/test/test_multiprocessing.py b/lib-python/2.7/test/test_multiprocessing.py --- a/lib-python/2.7/test/test_multiprocessing.py +++ b/lib-python/2.7/test/test_multiprocessing.py @@ -1343,7 +1343,7 @@ # Because we are using xmlrpclib for serialization instead of # pickle this will cause a serialization error. - self.assertRaises(Exception, queue.put, time.sleep) + self.assertRaises(Exception, queue.put, object) # Make queue finalizer run before the server is stopped del queue @@ -1800,9 +1800,9 @@ if not gc.isenabled(): gc.enable() self.addCleanup(gc.disable) - thresholds = gc.get_threshold() - self.addCleanup(gc.set_threshold, *thresholds) - gc.set_threshold(10) + #thresholds = gc.get_threshold() + #self.addCleanup(gc.set_threshold, *thresholds) + #gc.set_threshold(10) # perform numerous block allocations, with cyclic references to make # sure objects are collected asynchronously by the gc @@ -1865,6 +1865,7 @@ def test_synchronize(self): self.test_sharedctypes(lock=True) + @unittest.skipUnless(test_support.check_impl_detail(pypy=False), "pypy ctypes differences") def test_copy(self): foo = _Foo(2, 5.0) bar = copy(foo) diff --git a/lib-python/conftest.py b/lib-python/conftest.py --- a/lib-python/conftest.py +++ b/lib-python/conftest.py @@ -310,7 +310,7 @@ RegrTest('test_msilib.py'), RegrTest('test_multibytecodec.py', usemodules='_multibytecodec'), RegrTest('test_multifile.py'), - RegrTest('test_multiprocessing.py', skip=True), + RegrTest('test_multiprocessing.py'), RegrTest('test_mutants.py', core="possibly"), RegrTest('test_mutex.py'), RegrTest('test_netrc.py'), From noreply at buildbot.pypy.org Thu Jan 31 11:13:27 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 31 Jan 2013 11:13:27 +0100 (CET) Subject: [pypy-commit] pypy stdlib-2.7.4: merge default Message-ID: <20130131101327.00C351C040D@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: stdlib-2.7.4 Changeset: r60754:37072a4c2abf Date: 2013-01-31 03:40 -0500 http://bitbucket.org/pypy/pypy/changeset/37072a4c2abf/ Log: merge default diff --git a/lib-python/2.7/test/test_multiprocessing.py b/lib-python/2.7/test/test_multiprocessing.py --- a/lib-python/2.7/test/test_multiprocessing.py +++ b/lib-python/2.7/test/test_multiprocessing.py @@ -1404,7 +1404,7 @@ # Because we are using xmlrpclib for serialization instead of # pickle this will cause a serialization error. - self.assertRaises(Exception, queue.put, time.sleep) + self.assertRaises(Exception, queue.put, object) # Make queue finalizer run before the server is stopped del queue @@ -1879,9 +1879,9 @@ if not gc.isenabled(): gc.enable() self.addCleanup(gc.disable) - thresholds = gc.get_threshold() - self.addCleanup(gc.set_threshold, *thresholds) - gc.set_threshold(10) + #thresholds = gc.get_threshold() + #self.addCleanup(gc.set_threshold, *thresholds) + #gc.set_threshold(10) # perform numerous block allocations, with cyclic references to make # sure objects are collected asynchronously by the gc @@ -1944,6 +1944,7 @@ def test_synchronize(self): self.test_sharedctypes(lock=True) + @unittest.skipUnless(test_support.check_impl_detail(pypy=False), "pypy ctypes differences") def test_copy(self): foo = _Foo(2, 5.0) bar = copy(foo) diff --git a/lib-python/conftest.py b/lib-python/conftest.py --- a/lib-python/conftest.py +++ b/lib-python/conftest.py @@ -311,7 +311,7 @@ RegrTest('test_msilib.py'), RegrTest('test_multibytecodec.py', usemodules='_multibytecodec'), RegrTest('test_multifile.py'), - RegrTest('test_multiprocessing.py', skip=True), + RegrTest('test_multiprocessing.py'), RegrTest('test_mutants.py', core="possibly"), RegrTest('test_mutex.py'), RegrTest('test_netrc.py'), From noreply at buildbot.pypy.org Thu Jan 31 11:13:28 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 31 Jan 2013 11:13:28 +0100 (CET) Subject: [pypy-commit] pypy stdlib-2.7.4: fix for a new test in test_multiprocessing.py Message-ID: <20130131101328.253C31C040D@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: stdlib-2.7.4 Changeset: r60755:6b3d7106d645 Date: 2013-01-31 03:41 -0500 http://bitbucket.org/pypy/pypy/changeset/6b3d7106d645/ Log: fix for a new test in test_multiprocessing.py diff --git a/lib-python/2.7/test/test_multiprocessing.py b/lib-python/2.7/test/test_multiprocessing.py --- a/lib-python/2.7/test/test_multiprocessing.py +++ b/lib-python/2.7/test/test_multiprocessing.py @@ -1196,7 +1196,9 @@ p.join() def unpickleable_result(): - return lambda: 42 + class C: + pass + return C class _TestPoolWorkerErrors(BaseTestCase): ALLOWED_TYPES = ('processes', ) From noreply at buildbot.pypy.org Thu Jan 31 11:13:29 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 31 Jan 2013 11:13:29 +0100 (CET) Subject: [pypy-commit] pypy stdlib-2.7.4: fixes for test_weakset.py Message-ID: <20130131101329.403691C040D@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: stdlib-2.7.4 Changeset: r60756:6b5838c00f9b Date: 2013-01-31 03:49 -0500 http://bitbucket.org/pypy/pypy/changeset/6b5838c00f9b/ Log: fixes for test_weakset.py diff --git a/lib-python/2.7/test/test_weakset.py b/lib-python/2.7/test/test_weakset.py --- a/lib-python/2.7/test/test_weakset.py +++ b/lib-python/2.7/test/test_weakset.py @@ -416,7 +416,7 @@ gc.collect() n1 = len(s) del it - gc.collect() + test_support.gc_collect() n2 = len(s) # one item may be kept alive inside the iterator self.assertIn(n1, (0, 1)) @@ -424,11 +424,11 @@ def test_len_race(self): # Extended sanity checks for len() in the face of cyclic collection - self.addCleanup(gc.set_threshold, *gc.get_threshold()) + #self.addCleanup(gc.set_threshold, *gc.get_threshold()) for th in range(1, 100): N = 20 - gc.collect(0) - gc.set_threshold(th, th, th) + test_support.gc_collect() + #gc.set_threshold(th, th, th) items = [RefCycle() for i in range(N)] s = WeakSet(items) del items From noreply at buildbot.pypy.org Thu Jan 31 11:13:30 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 31 Jan 2013 11:13:30 +0100 (CET) Subject: [pypy-commit] pypy stdlib-2.7.4: fixes for test_weakref.py Message-ID: <20130131101330.704551C040D@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: stdlib-2.7.4 Changeset: r60757:6bbcd70786dd Date: 2013-01-31 05:03 -0500 http://bitbucket.org/pypy/pypy/changeset/6bbcd70786dd/ Log: fixes for test_weakref.py diff --git a/lib-python/2.7/test/test_weakref.py b/lib-python/2.7/test/test_weakref.py --- a/lib-python/2.7/test/test_weakref.py +++ b/lib-python/2.7/test/test_weakref.py @@ -1,3 +1,4 @@ +import gc import sys import unittest import UserList @@ -919,7 +920,7 @@ gc.collect() n1 = len(dct) del it - gc.collect() + test_support.gc_collect() n2 = len(dct) # one item may be kept alive inside the iterator self.assertIn(n1, (0, 1)) @@ -933,11 +934,11 @@ def check_len_race(self, dict_type, cons): # Extended sanity checks for len() in the face of cyclic collection - self.addCleanup(gc.set_threshold, *gc.get_threshold()) + #self.addCleanup(gc.set_threshold, *gc.get_threshold()) for th in range(1, 100): N = 20 - gc.collect(0) - gc.set_threshold(th, th, th) + test_support.gc_collect() + #gc.set_threshold(th, th, th) items = [RefCycle() for i in range(N)] dct = dict_type(cons(o) for o in items) del items From noreply at buildbot.pypy.org Thu Jan 31 11:14:44 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 31 Jan 2013 11:14:44 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: rename the variable to confuse armin less Message-ID: <20130131101444.1E00F1C040D@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60758:0543b2c5413c Date: 2013-01-31 12:13 +0200 http://bitbucket.org/pypy/pypy/changeset/0543b2c5413c/ Log: rename the variable to confuse armin less diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -541,8 +541,9 @@ operations = regalloc.prepare_loop(inputargs, operations, looptoken, clt.allgcrefs) looppos = self.mc.get_relative_pos() - frame_depth = self._assemble(regalloc, inputargs, operations) - self.update_frame_depth(frame_depth + JITFRAME_FIXED_SIZE) + frame_depth_no_fixed_size = self._assemble(regalloc, inputargs, + operations) + self.update_frame_depth(frame_depth_no_fixed_size + JITFRAME_FIXED_SIZE) # size_excluding_failure_stuff = self.mc.get_relative_pos() self.write_pending_failure_recoveries() @@ -567,7 +568,8 @@ looptoken._x86_ops_offset = ops_offset looptoken._x86_function_addr = rawstart for label in self.labels_to_patch: - self._patch_stackadjust(label + rawstart, frame_depth + JITFRAME_FIXED_SIZE) + self._patch_stackadjust(label + rawstart, frame_depth_no_fixed_size + + JITFRAME_FIXED_SIZE) self.labels_to_patch = None self.fixup_target_tokens(rawstart) From noreply at buildbot.pypy.org Thu Jan 31 11:19:17 2013 From: noreply at buildbot.pypy.org (Aquana) Date: Thu, 31 Jan 2013 11:19:17 +0100 (CET) Subject: [pypy-commit] pypy quiet-rpython: hg merge default Message-ID: <20130131101917.9EC441C040D@cobra.cs.uni-duesseldorf.de> Author: Alexander Hesse Branch: quiet-rpython Changeset: r60759:a493f5ab195e Date: 2013-01-31 11:18 +0100 http://bitbucket.org/pypy/pypy/changeset/a493f5ab195e/ Log: hg merge default diff --git a/lib-python/2.7/inspect.py b/lib-python/2.7/inspect.py --- a/lib-python/2.7/inspect.py +++ b/lib-python/2.7/inspect.py @@ -960,7 +960,7 @@ raise TypeError('%s() takes exactly 0 arguments ' '(%d given)' % (f_name, num_total)) else: - raise TypeError('%s() takes no argument (%d given)' % + raise TypeError('%s() takes no arguments (%d given)' % (f_name, num_total)) for arg in args: if isinstance(arg, str) and arg in named: diff --git a/lib-python/2.7/site.py b/lib-python/2.7/site.py --- a/lib-python/2.7/site.py +++ b/lib-python/2.7/site.py @@ -75,6 +75,7 @@ USER_SITE = None USER_BASE = None + def makepath(*paths): dir = os.path.join(*paths) try: diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py --- a/lib-python/2.7/sysconfig.py +++ b/lib-python/2.7/sysconfig.py @@ -27,10 +27,10 @@ 'data' : '{base}', }, 'pypy': { - 'stdlib': '{base}/lib-python', - 'platstdlib': '{base}/lib-python', - 'purelib': '{base}/lib-python', - 'platlib': '{base}/lib-python', + 'stdlib': '{base}/lib-python/{py_version_short}', + 'platstdlib': '{base}/lib-python/{py_version_short}', + 'purelib': '{base}/lib-python/{py_version_short}', + 'platlib': '{base}/lib-python/{py_version_short}', 'include': '{base}/include', 'platinclude': '{base}/include', 'scripts': '{base}/bin', diff --git a/lib-python/2.7/test/test_capi.py b/lib-python/2.7/test/test_capi.py --- a/lib-python/2.7/test/test_capi.py +++ b/lib-python/2.7/test/test_capi.py @@ -13,6 +13,17 @@ threading = None import _testcapi +skips = [] +if test_support.check_impl_detail(pypy=True): + skips += [ + 'test_broken_memoryview', + 'test_capsule', + 'test_lazy_hash_inheritance', + 'test_widechar', + 'TestThreadState', + 'TestPendingCalls', + ] + @unittest.skipUnless(threading, 'Threading required for this test.') class TestPendingCalls(unittest.TestCase): @@ -99,7 +110,7 @@ def test_main(): for name in dir(_testcapi): - if name.startswith('test_'): + if name.startswith('test_') and name not in skips: test = getattr(_testcapi, name) if test_support.verbose: print "internal", name @@ -126,7 +137,7 @@ raise test_support.TestFailed, \ "Couldn't find main thread correctly in the list" - if threading: + if threading and 'TestThreadState' not in skips: import thread import time TestThreadState() @@ -134,7 +145,8 @@ t.start() t.join() - test_support.run_unittest(TestPendingCalls) + if 'TestPendingCalls' not in skips: + test_support.run_unittest(TestPendingCalls) if __name__ == "__main__": test_main() diff --git a/lib-python/2.7/test/test_itertools.py b/lib-python/2.7/test/test_itertools.py --- a/lib-python/2.7/test/test_itertools.py +++ b/lib-python/2.7/test/test_itertools.py @@ -533,11 +533,11 @@ self.assertEqual(list(izip()), zip()) self.assertRaises(TypeError, izip, 3) self.assertRaises(TypeError, izip, range(3), 3) - self.assertEqual([tuple(list(pair)) for pair in izip('abc', 'def')], zip('abc', 'def')) self.assertEqual([pair for pair in izip('abc', 'def')], zip('abc', 'def')) + @test_support.impl_detail("tuple reuse is specific to CPython") def test_izip_tuple_reuse(self): ids = map(id, izip('abc', 'def')) @@ -588,6 +588,7 @@ zip('abc', 'def')) self.assertEqual([pair for pair in izip_longest('abc', 'def')], zip('abc', 'def')) + @test_support.impl_detail("tuple reuse is specific to CPython") def test_izip_longest_tuple_reuse(self): ids = map(id, izip_longest('abc', 'def')) diff --git a/lib-python/2.7/test/test_multiprocessing.py b/lib-python/2.7/test/test_multiprocessing.py --- a/lib-python/2.7/test/test_multiprocessing.py +++ b/lib-python/2.7/test/test_multiprocessing.py @@ -1343,7 +1343,7 @@ # Because we are using xmlrpclib for serialization instead of # pickle this will cause a serialization error. - self.assertRaises(Exception, queue.put, time.sleep) + self.assertRaises(Exception, queue.put, object) # Make queue finalizer run before the server is stopped del queue @@ -1800,9 +1800,9 @@ if not gc.isenabled(): gc.enable() self.addCleanup(gc.disable) - thresholds = gc.get_threshold() - self.addCleanup(gc.set_threshold, *thresholds) - gc.set_threshold(10) + #thresholds = gc.get_threshold() + #self.addCleanup(gc.set_threshold, *thresholds) + #gc.set_threshold(10) # perform numerous block allocations, with cyclic references to make # sure objects are collected asynchronously by the gc @@ -1865,6 +1865,7 @@ def test_synchronize(self): self.test_sharedctypes(lock=True) + @unittest.skipUnless(test_support.check_impl_detail(pypy=False), "pypy ctypes differences") def test_copy(self): foo = _Foo(2, 5.0) bar = copy(foo) diff --git a/lib-python/2.7/test/test_support.py b/lib-python/2.7/test/test_support.py --- a/lib-python/2.7/test/test_support.py +++ b/lib-python/2.7/test/test_support.py @@ -1085,7 +1085,6 @@ else: runner = BasicTestRunner() - result = runner.run(suite) if not result.wasSuccessful(): if len(result.errors) == 1 and not result.failures: diff --git a/lib-python/conftest.py b/lib-python/conftest.py --- a/lib-python/conftest.py +++ b/lib-python/conftest.py @@ -132,7 +132,7 @@ RegrTest('test_bz2.py', usemodules='bz2'), RegrTest('test_calendar.py'), RegrTest('test_call.py', core=True), - RegrTest('test_capi.py', skip="not applicable"), + RegrTest('test_capi.py'), RegrTest('test_cd.py'), RegrTest('test_cfgparser.py'), RegrTest('test_cgi.py'), @@ -310,7 +310,7 @@ RegrTest('test_msilib.py'), RegrTest('test_multibytecodec.py', usemodules='_multibytecodec'), RegrTest('test_multifile.py'), - RegrTest('test_multiprocessing.py', skip=True), + RegrTest('test_multiprocessing.py'), RegrTest('test_mutants.py', core="possibly"), RegrTest('test_mutex.py'), RegrTest('test_netrc.py'), diff --git a/pypy/bin/checkmodule.py b/pypy/bin/checkmodule.py --- a/pypy/bin/checkmodule.py +++ b/pypy/bin/checkmodule.py @@ -8,6 +8,8 @@ """ import sys, os +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..')) + from pypy.objspace.fake.checkmodule import checkmodule def main(argv): diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -318,10 +318,5 @@ type and vice versa. For builtin types, a dictionary will be returned that cannot be changed (but still looks and behaves like a normal dictionary). -* the ``__len__`` or ``__length_hint__`` special methods are sometimes - called by CPython to get a length estimate to preallocate internal arrays. - So far, PyPy never calls ``__len__`` for this purpose, and never calls - ``__length_hint__`` at all. - .. include:: _ref.txt diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst --- a/pypy/doc/faq.rst +++ b/pypy/doc/faq.rst @@ -300,12 +300,26 @@ Do I have to rewrite my programs in RPython? -------------------------------------------- -No. And you shouldn't try. PyPy always runs your code in its own interpreter, which is a -full and compliant Python 2.7 interpreter. RPython is only the -language in which parts of PyPy itself are written and extension -modules for it. Not only is it not necessary for you to rewrite your -code in RPython, it probably won't give you any speed improvements if you -try. +No. And you shouldn't try. First and foremost, RPython is a language +that is designed to write interpreters in. It is a restricted subset of +Python. If you program is not an interpreter but tries to do "real +things", like use *any* part of the standard Python library or *any* +3rd-party library, then it is not RPython to start with. You should +only look at RPython if you try to `write your own interpreter`__. + +.. __: `how do I compile my own interpreters`_ + +If your goal is to speed up Python code, then look at the regular PyPy, +which is a full and compliant Python 2.7 interpreter (which happens to +be written in RPython). Not only is it not necessary for you to rewrite +your code in RPython, it might not give you any speed improvements even +if you manage to. + +Yes, it is possible with enough effort to compile small self-contained +pieces of RPython code doing a few performance-sensitive things. But +this case is not interesting for us. If you needed to rewrite the code +in RPython, you could as well have rewritten it in C for example. The +latter is a much more supported, much more documented language `:-)` --------------------------------------------------- Which backends are there for the RPython toolchain? diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -1482,9 +1482,10 @@ ) raise w_fd = self.call_function(w_fileno) - if not self.isinstance_w(w_fd, self.w_int): + if (not self.isinstance_w(w_fd, self.w_int) and + not self.isinstance_w(w_fd, self.w_long)): raise OperationError(self.w_TypeError, - self.wrap("fileno() must return an integer") + self.wrap("fileno() returned a non-integer") ) fd = self.int_w(w_fd) if fd < 0: diff --git a/pypy/module/_bisect/interp_bisect.py b/pypy/module/_bisect/interp_bisect.py --- a/pypy/module/_bisect/interp_bisect.py +++ b/pypy/module/_bisect/interp_bisect.py @@ -1,5 +1,6 @@ from pypy.interpreter.error import OperationError from pypy.interpreter.gateway import unwrap_spec +from rpython.rlib.rarithmetic import intmask, r_uint @unwrap_spec(lo=int, hi=int) @@ -18,7 +19,7 @@ if hi == -1: hi = space.len_w(w_a) while lo < hi: - mid = (lo + hi) >> 1 + mid = intmask((r_uint(lo) + r_uint(hi)) >> 1) w_litem = space.getitem(w_a, space.wrap(mid)) if space.is_true(space.lt(w_litem, w_x)): lo = mid + 1 @@ -43,7 +44,7 @@ if hi == -1: hi = space.len_w(w_a) while lo < hi: - mid = (lo + hi) >> 1 + mid = intmask((r_uint(lo) + r_uint(hi)) >> 1) w_litem = space.getitem(w_a, space.wrap(mid)) if space.is_true(space.lt(w_x, w_litem)): hi = mid diff --git a/pypy/module/_bisect/test/test_bisect.py b/pypy/module/_bisect/test/test_bisect.py --- a/pypy/module/_bisect/test/test_bisect.py +++ b/pypy/module/_bisect/test/test_bisect.py @@ -89,3 +89,12 @@ insort_right(a, 6.0) assert a == [0, 5, 6, 6, 6, 6.0, 7] assert map(type, a) == [int, int, int, int, int, float, int] + + def test_bisect_overflow(self): + from _bisect import bisect_left, bisect_right + import sys + + size = sys.maxsize + data = xrange(size - 1) + assert bisect_left(data, size - 3) == size - 3 + assert bisect_right(data, size - 3) == size - 2 diff --git a/pypy/module/_io/interp_fileio.py b/pypy/module/_io/interp_fileio.py --- a/pypy/module/_io/interp_fileio.py +++ b/pypy/module/_io/interp_fileio.py @@ -153,41 +153,49 @@ self.readable, self.writable, append, flags = decode_mode(space, mode) - if fd >= 0: - verify_fd(fd) - try: - os.fstat(fd) - except OSError, e: - if e.errno == errno.EBADF: - raise wrap_oserror(space, e) - # else: pass - self.fd = fd - self.closefd = bool(closefd) - else: - if not closefd: - raise OperationError(space.w_ValueError, space.wrap( - "Cannot use closefd=False with file name")) - self.closefd = True + fd_is_own = False + try: + if fd >= 0: + verify_fd(fd) + try: + os.fstat(fd) + except OSError, e: + if e.errno == errno.EBADF: + raise wrap_oserror(space, e) + # else: pass + self.fd = fd + self.closefd = bool(closefd) + else: + self.closefd = True + if not closefd: + raise OperationError(space.w_ValueError, space.wrap( + "Cannot use closefd=False with file name")) - from pypy.module.posix.interp_posix import ( - dispatch_filename, rposix) - try: - self.fd = dispatch_filename(rposix.open)( - space, w_name, flags, 0666) - except OSError, e: - raise wrap_oserror2(space, e, w_name, - exception_name='w_IOError') + from pypy.module.posix.interp_posix import ( + dispatch_filename, rposix) + try: + self.fd = dispatch_filename(rposix.open)( + space, w_name, flags, 0666) + except OSError, e: + raise wrap_oserror2(space, e, w_name, + exception_name='w_IOError') + finally: + fd_is_own = True self._dircheck(space, w_name) - self.w_name = w_name + self.w_name = w_name - if append: - # For consistent behaviour, we explicitly seek to the end of file - # (otherwise, it might be done only on the first write()). - try: - os.lseek(self.fd, 0, os.SEEK_END) - except OSError, e: - raise wrap_oserror(space, e, exception_name='w_IOError') + if append: + # For consistent behaviour, we explicitly seek to the end of file + # (otherwise, it might be done only on the first write()). + try: + os.lseek(self.fd, 0, os.SEEK_END) + except OSError, e: + raise wrap_oserror(space, e, exception_name='w_IOError') + except: + if not fd_is_own: + self.fd = -1 + raise def _mode(self): if self.readable: @@ -253,7 +261,6 @@ except OSError: return if stat.S_ISDIR(st.st_mode): - self._close(space) raise wrap_oserror2(space, OSError(errno.EISDIR, "fstat"), w_filename, exception_name='w_IOError') diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -41,7 +41,11 @@ def test_open_directory(self): import _io + import os raises(IOError, _io.FileIO, self.tmpdir, "rb") + fd = os.open(self.tmpdir, os.O_RDONLY) + raises(IOError, _io.FileIO, fd, "rb") + os.close(fd) def test_readline(self): import _io diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -922,8 +922,8 @@ kwds["link_extra"] = ["msvcrt.lib"] elif sys.platform.startswith('linux'): compile_extra.append("-Werror=implicit-function-declaration") + compile_extra.append('-g') export_symbols_eci.append('pypyAPI') - compile_extra.append('-g') else: kwds["includes"] = ['Python.h'] # this is our Python.h diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py --- a/pypy/module/cpyext/longobject.py +++ b/pypy/module/cpyext/longobject.py @@ -47,7 +47,12 @@ Return a C unsigned long representation of the contents of pylong. If pylong is greater than ULONG_MAX, an OverflowError is raised.""" - return rffi.cast(rffi.ULONG, space.uint_w(w_long)) + try: + return rffi.cast(rffi.ULONG, space.uint_w(w_long)) + except OperationError, e: + if e.match(space, space.w_ValueError): + e.w_type = space.w_OverflowError + raise @cpython_api([PyObject], rffi.ULONG, error=-1) def PyLong_AsUnsignedLongMask(space, w_long): @@ -86,7 +91,12 @@ Return a C unsigned long representation of the contents of pylong. If pylong is greater than ULONG_MAX, an OverflowError is raised.""" - return rffi.cast(rffi.ULONGLONG, space.r_ulonglong_w(w_long)) + try: + return rffi.cast(rffi.ULONGLONG, space.r_ulonglong_w(w_long)) + except OperationError, e: + if e.match(space, space.w_ValueError): + e.w_type = space.w_OverflowError + raise @cpython_api([PyObject], rffi.ULONGLONG, error=-1) def PyLong_AsUnsignedLongLongMask(space, w_long): diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py --- a/pypy/module/cpyext/object.py +++ b/pypy/module/cpyext/object.py @@ -218,6 +218,8 @@ @cpython_api([PyObject], PyObject) def PyObject_Str(space, w_obj): + if w_obj is None: + return space.wrap("") return space.str(w_obj) @cpython_api([PyObject], PyObject) @@ -226,6 +228,8 @@ representation on success, NULL on failure. This is the equivalent of the Python expression repr(o). Called by the repr() built-in function and by reverse quotes.""" + if w_obj is None: + return space.wrap("") return space.repr(w_obj) @cpython_api([PyObject], PyObject) @@ -234,6 +238,8 @@ string representation on success, NULL on failure. This is the equivalent of the Python expression unicode(o). Called by the unicode() built-in function.""" + if w_obj is None: + return space.wrap(u"") return space.call_function(space.w_unicode, w_obj) @cpython_api([PyObject, PyObject], rffi.INT_real, error=-1) diff --git a/pypy/module/cpyext/test/test_longobject.py b/pypy/module/cpyext/test/test_longobject.py --- a/pypy/module/cpyext/test/test_longobject.py +++ b/pypy/module/cpyext/test/test_longobject.py @@ -31,6 +31,8 @@ value = api.PyLong_AsUnsignedLong(w_value) assert value == (sys.maxint - 1) * 2 + self.raises(space, api, OverflowError, api.PyLong_AsUnsignedLong, space.wrap(-1)) + def test_as_ssize_t(self, space, api): w_value = space.newlong(2) value = api.PyLong_AsSsize_t(w_value) @@ -46,11 +48,11 @@ w_l = space.wrap(sys.maxint + 1) assert api.PyLong_Check(w_l) assert api.PyLong_CheckExact(w_l) - + w_i = space.wrap(sys.maxint) assert not api.PyLong_Check(w_i) assert not api.PyLong_CheckExact(w_i) - + L = space.appexec([], """(): class L(long): pass @@ -73,6 +75,8 @@ assert api.PyLong_AsUnsignedLongLongMask( space.wrap(1<<64)) == 0 + self.raises(space, api, OverflowError, api.PyLong_AsUnsignedLongLong, space.wrap(-1)) + def test_as_long_and_overflow(self, space, api): overflow = lltype.malloc(rffi.CArrayPtr(rffi.INT_real).TO, 1, flavor='raw') assert api.PyLong_AsLongAndOverflow( diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py --- a/pypy/module/cpyext/test/test_object.py +++ b/pypy/module/cpyext/test/test_object.py @@ -94,27 +94,30 @@ def test_size(self, space, api): assert api.PyObject_Size(space.newlist([space.w_None])) == 1 - + + def test_str(self, space, api): + w_list = space.newlist([space.w_None, space.wrap(42)]) + assert space.str_w(api.PyObject_Str(None)) == "" + assert space.str_w(api.PyObject_Str(w_list)) == "[None, 42]" + assert space.str_w(api.PyObject_Str(space.wrap("a"))) == "a" + def test_repr(self, space, api): w_list = space.newlist([space.w_None, space.wrap(42)]) + assert space.str_w(api.PyObject_Repr(None)) == "" assert space.str_w(api.PyObject_Repr(w_list)) == "[None, 42]" assert space.str_w(api.PyObject_Repr(space.wrap("a"))) == "'a'" - - w_list = space.newlist([space.w_None, space.wrap(42)]) - assert space.str_w(api.PyObject_Str(w_list)) == "[None, 42]" - assert space.str_w(api.PyObject_Str(space.wrap("a"))) == "a" - + def test_RichCompare(self, space, api): def compare(w_o1, w_o2, opid): res = api.PyObject_RichCompareBool(w_o1, w_o2, opid) w_res = api.PyObject_RichCompare(w_o1, w_o2, opid) assert space.is_true(w_res) == res return res - + def test_compare(o1, o2): w_o1 = space.wrap(o1) w_o2 = space.wrap(o2) - + for opid, expected in [ (Py_LT, o1 < o2), (Py_LE, o1 <= o2), (Py_NE, o1 != o2), (Py_EQ, o1 == o2), @@ -190,6 +193,7 @@ api.PyErr_Clear() def test_unicode(self, space, api): + assert space.unwrap(api.PyObject_Unicode(None)) == u"" assert space.unwrap(api.PyObject_Unicode(space.wrap([]))) == u"[]" assert space.unwrap(api.PyObject_Unicode(space.wrap("e"))) == u"e" assert api.PyObject_Unicode(space.wrap("\xe9")) is None diff --git a/pypy/module/fcntl/test/test_fcntl.py b/pypy/module/fcntl/test/test_fcntl.py --- a/pypy/module/fcntl/test/test_fcntl.py +++ b/pypy/module/fcntl/test/test_fcntl.py @@ -22,13 +22,23 @@ import sys import struct + class F: + def __init__(self, fn): + self.fn = fn + def fileno(self): + return self.fn + f = open(self.tmp + "b", "w+") fcntl.fcntl(f, 1, 0) fcntl.fcntl(f, 1) + fcntl.fcntl(F(long(f.fileno())), 1) raises(TypeError, fcntl.fcntl, "foo") raises(TypeError, fcntl.fcntl, f, "foo") - raises((IOError, ValueError), fcntl.fcntl, -1, 1, 0) + raises(TypeError, fcntl.fcntl, F("foo"), 1) + raises(ValueError, fcntl.fcntl, -1, 1, 0) + raises(ValueError, fcntl.fcntl, F(-1), 1, 0) + raises(ValueError, fcntl.fcntl, F(long(-1)), 1, 0) assert fcntl.fcntl(f, 1, 0) == 0 assert fcntl.fcntl(f, 2, "foo") == "foo" assert fcntl.fcntl(f, 2, buffer("foo")) == "foo" diff --git a/pypy/module/thread/test/support.py b/pypy/module/thread/test/support.py --- a/pypy/module/thread/test/support.py +++ b/pypy/module/thread/test/support.py @@ -2,6 +2,7 @@ import time import thread import os +import errno from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.module.thread import gil @@ -28,7 +29,12 @@ def kill(): for x in range(delay * 10): time.sleep(0.1) - os.kill(pid, 0) + try: + os.kill(pid, 0) + except OSError, e: + if e.errno == errno.ESRCH: # no such process + return + raise os.kill(pid, 9) print "process %s killed!" % (pid,) thread.start_new_thread(kill, ()) diff --git a/pypy/module/thread/test/test_fork.py b/pypy/module/thread/test/test_fork.py --- a/pypy/module/thread/test/test_fork.py +++ b/pypy/module/thread/test/test_fork.py @@ -1,7 +1,7 @@ from pypy.module.thread.test.support import GenericTestThread class AppTestFork(GenericTestThread): - def test_fork(self): + def test_fork_with_thread(self): # XXX This test depends on a multicore machine, as busy_thread must # aquire the GIL the instant that the main thread releases it. # It will incorrectly pass if the GIL is not grabbed in time. @@ -12,45 +12,48 @@ if not hasattr(os, 'fork'): skip("No fork on this platform") - run = True - done = [] def busy_thread(): while run: time.sleep(0) done.append(None) - try: - thread.start_new(busy_thread, ()) + for i in range(1): + run = True + done = [] + try: + thread.start_new(busy_thread, ()) + print 'sleep' - pid = os.fork() - - if pid == 0: - os._exit(0) - - else: - time.sleep(1) - spid, status = os.waitpid(pid, os.WNOHANG) - assert spid == pid - finally: - run = False - self.waitfor(lambda: done) + pid = os.fork() + if pid == 0: + os._exit(0) + else: + self.timeout_killer(pid, 5) + exitcode = os.waitpid(pid, 0)[1] + assert exitcode == 0 # if 9, process was killed by timer! + finally: + run = False + self.waitfor(lambda: done) + assert done def test_forked_can_thread(self): "Checks that a forked interpreter can start a thread" - import os, thread, time + import thread + import os if not hasattr(os, 'fork'): skip("No fork on this platform") - # pre-allocate some locks - thread.start_new_thread(lambda: None, ()) + for i in range(10): + # pre-allocate some locks + thread.start_new_thread(lambda: None, ()) + print 'sleep' - pid = os.fork() - if pid == 0: - print 'in child' - thread.start_new_thread(lambda: None, ()) - os._exit(0) - else: - self.timeout_killer(pid, 5) - exitcode = os.waitpid(pid, 0)[1] - assert exitcode == 0 # if 9, process was killed by timer! + pid = os.fork() + if pid == 0: + thread.start_new_thread(lambda: None, ()) + os._exit(0) + else: + self.timeout_killer(pid, 5) + exitcode = os.waitpid(pid, 0)[1] + assert exitcode == 0 # if 9, process was killed by timer! diff --git a/rpython/flowspace/framestate.py b/rpython/flowspace/framestate.py --- a/rpython/flowspace/framestate.py +++ b/rpython/flowspace/framestate.py @@ -1,5 +1,6 @@ +from rpython.flowspace.model import Variable, Constant from rpython.rlib.unroll import SpecTag -from rpython.flowspace.model import * + class FrameState: def __init__(self, mergeable, blocklist, next_instr): @@ -86,6 +87,7 @@ raise TypeError('union of %r and %r' % (w1.__class__.__name__, w2.__class__.__name__)) + # ____________________________________________________________ # # We have to flatten out the state of the frame into a list of @@ -102,6 +104,7 @@ PICKLE_TAGS = {} UNPICKLE_TAGS = {} + def recursively_flatten(space, lst): from rpython.flowspace.flowcontext import SuspendedUnroller i = 0 @@ -117,14 +120,15 @@ except: tag = PICKLE_TAGS[key] = Constant(PickleTag()) UNPICKLE_TAGS[tag] = key - lst[i:i+1] = [tag] + vars + lst[i:i + 1] = [tag] + vars + def recursively_unflatten(space, lst): - for i in xrange(len(lst)-1, -1, -1): + for i in xrange(len(lst) - 1, -1, -1): item = lst[i] if item in UNPICKLE_TAGS: unrollerclass, argcount = UNPICKLE_TAGS[item] - arguments = lst[i+1: i+1+argcount] - del lst[i+1: i+1+argcount] + arguments = lst[i + 1:i + 1 + argcount] + del lst[i + 1:i + 1 + argcount] unroller = unrollerclass.state_pack_variables(space, *arguments) lst[i] = unroller diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py --- a/rpython/jit/metainterp/optimizeopt/rewrite.py +++ b/rpython/jit/metainterp/optimizeopt/rewrite.py @@ -239,6 +239,8 @@ def optimize_GUARD_VALUE(self, op): value = self.getvalue(op.getarg(0)) + if value.is_virtual(): + raise InvalidLoop('A promote of a virtual (a recently allocated object) never makes sense!') if value.last_guard: # there already has been a guard_nonnull or guard_class or # guard_nonnull_class on this value, which is rather silly. diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py --- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py +++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py @@ -2623,7 +2623,7 @@ jump(p2) """ self.raises(InvalidLoop, self.optimize_loop, - ops, ops) + ops, "crash!") def test_invalid_loop_2(self): ops = """ @@ -2635,7 +2635,7 @@ jump(p2) """ self.raises(InvalidLoop, self.optimize_loop, - ops, ops) + ops, "crash!") def test_invalid_loop_3(self): ops = """ @@ -2648,8 +2648,17 @@ setfield_gc(p3, p4, descr=nextdescr) jump(p3) """ - self.raises(InvalidLoop, self.optimize_loop, ops, ops) - + self.raises(InvalidLoop, self.optimize_loop, ops, "crash!") + + def test_invalid_loop_guard_value_of_virtual(self): + ops = """ + [p1] + p2 = new_with_vtable(ConstClass(node_vtable)) + guard_value(p2, ConstPtr(myptr)) [] + jump(p2) + """ + self.raises(InvalidLoop, self.optimize_loop, + ops, "crash!") def test_merge_guard_class_guard_value(self): ops = """ diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py --- a/rpython/jit/metainterp/optimizeopt/unroll.py +++ b/rpython/jit/metainterp/optimizeopt/unroll.py @@ -1,16 +1,16 @@ -from rpython.jit.codewriter.effectinfo import EffectInfo +import sys + +from rpython.jit.metainterp.compile import ResumeGuardDescr +from rpython.jit.metainterp.history import TargetToken, JitCellToken, Const +from rpython.jit.metainterp.inliner import Inliner +from rpython.jit.metainterp.optimize import InvalidLoop +from rpython.jit.metainterp.optimizeopt.generalize import KillHugeIntBounds +from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer, Optimization from rpython.jit.metainterp.optimizeopt.virtualstate import VirtualStateAdder, ShortBoxes, BadVirtualState -from rpython.jit.metainterp.compile import ResumeGuardDescr -from rpython.jit.metainterp.history import TreeLoop, TargetToken, JitCellToken -from rpython.jit.metainterp.jitexc import JitException -from rpython.jit.metainterp.optimize import InvalidLoop -from rpython.rlib.debug import debug_print, debug_start, debug_stop -from rpython.jit.metainterp.optimizeopt.optimizer import * -from rpython.jit.metainterp.optimizeopt.generalize import KillHugeIntBounds -from rpython.jit.metainterp.inliner import Inliner from rpython.jit.metainterp.resoperation import rop, ResOperation from rpython.jit.metainterp.resume import Snapshot -import sys, os +from rpython.rlib.debug import debug_print, debug_start, debug_stop + # FIXME: Introduce some VirtualOptimizer super class instead @@ -19,6 +19,7 @@ opt.inline_short_preamble = inline_short_preamble opt.propagate_all_forward() + class UnrollableOptimizer(Optimizer): def setup(self): self.importable_values = {} @@ -51,7 +52,7 @@ distinction anymore)""" inline_short_preamble = True - + def __init__(self, metainterp_sd, loop, optimizations): self.optimizer = UnrollableOptimizer(metainterp_sd, loop, optimizations) self.boxes_created_this_iteration = None @@ -59,14 +60,14 @@ def fix_snapshot(self, jump_args, snapshot): if snapshot is None: return None - snapshot_args = snapshot.boxes + snapshot_args = snapshot.boxes new_snapshot_args = [] for a in snapshot_args: a = self.getvalue(a).get_key_box() new_snapshot_args.append(a) prev = self.fix_snapshot(jump_args, snapshot.prev) return Snapshot(prev, new_snapshot_args) - + def propagate_all_forward(self): loop = self.optimizer.loop self.optimizer.clear_newoperations() @@ -78,7 +79,7 @@ # will clear heap caches self.optimizer.send_extra_operation(start_label) else: - start_label = None + start_label = None jumpop = loop.operations[-1] if jumpop.getopnum() == rop.JUMP or jumpop.getopnum() == rop.LABEL: @@ -96,7 +97,6 @@ assert isinstance(cell_token, JitCellToken) stop_label = ResOperation(rop.LABEL, jumpop.getarglist(), None, TargetToken(cell_token)) - if jumpop.getopnum() == rop.JUMP: if self.jump_to_already_compiled_trace(jumpop): # Found a compiled trace to jump to @@ -130,7 +130,7 @@ return # Found nothing to jump to, emit a label instead - + if self.short: # Construct our short preamble assert start_label @@ -146,7 +146,7 @@ def jump_to_start_label(self, start_label, stop_label): if not start_label or not stop_label: return False - + stop_target = stop_label.getdescr() start_target = start_label.getdescr() assert isinstance(stop_target, TargetToken) @@ -161,7 +161,6 @@ #virtual_state = modifier.get_virtual_state(args) #if self.initial_virtual_state.generalization_of(virtual_state): # return True - def export_state(self, targetop): original_jump_args = targetop.getarglist() @@ -174,12 +173,11 @@ modifier = VirtualStateAdder(self.optimizer) virtual_state = modifier.get_virtual_state(jump_args) - + values = [self.getvalue(arg) for arg in jump_args] inputargs = virtual_state.make_inputargs(values, self.optimizer) short_inputargs = virtual_state.make_inputargs(values, self.optimizer, keyboxes=True) - if self.boxes_created_this_iteration is not None: for box in self.inputargs: self.boxes_created_this_iteration[box] = True @@ -210,7 +208,7 @@ if op and op.result: box = op.result exported_values[box] = self.optimizer.getvalue(box) - + target_token.exported_state = ExportedState(short_boxes, inputarg_setup_ops, exported_values) @@ -232,7 +230,7 @@ virtual_state = modifier.get_virtual_state(self.inputargs) self.initial_virtual_state = virtual_state return - + self.short = target_token.short_preamble[:] self.short_seen = {} self.short_boxes = exported_state.short_boxes @@ -276,7 +274,7 @@ #if self.optimizer.loop.logops: # debug_print(' Falling back to add extra: ' + # self.optimizer.loop.logops.repr_of_resop(op)) - + self.optimizer.flush() self.optimizer.emitting_dissabled = False @@ -287,7 +285,7 @@ # We dont need to inline the short preamble we are creating as we are conneting # the bridge to a different trace with a different short preamble self.short_inliner = None - + newoperations = self.optimizer.get_newoperations() self.boxes_created_this_iteration = {} i = 0 @@ -333,7 +331,7 @@ 'same box passed to multiple of its ' + 'inputargs, but the jump at the ' + 'end of this bridge does not do that.') - + args[short_inputargs[i]] = jmp_to_short_args[i] self.short_inliner = Inliner(short_inputargs, jmp_to_short_args) i = 1 @@ -400,7 +398,7 @@ 'loop is not compatible with the virtual ' + 'state at the start of the loop which makes ' + 'it impossible to close the loop') - + #debug_stop('jit-log-virtualstate') maxguards = self.optimizer.metainterp_sd.warmrunnerdesc.memory_manager.max_retrace_guards @@ -408,9 +406,9 @@ target_token = jumpop.getdescr() assert isinstance(target_token, TargetToken) target_token.targeting_jitcell_token.retraced_count = sys.maxint - + self.finilize_short_preamble(start_label) - + def finilize_short_preamble(self, start_label): short = self.short assert short[-1].getopnum() == rop.JUMP @@ -482,11 +480,11 @@ if op is None: return None if op.result is not None and op.result in self.short_seen: - if emit and self.short_inliner: + if emit and self.short_inliner: return self.short_inliner.inline_arg(op.result) else: return None - + for a in op.getarglist(): if not isinstance(a, Const) and a not in self.short_seen: self.add_op_to_short(self.short_boxes.producer(a), emit, guards_needed) @@ -497,7 +495,7 @@ if guards_needed and self.short_boxes.has_producer(op.result): value_guards = self.getvalue(op.result).make_guards(op.result) else: - value_guards = [] + value_guards = [] self.short.append(op) self.short_seen[op.result] = True @@ -517,7 +515,7 @@ if newop: return newop.result return None - + def import_box(self, box, inputargs, short_jumpargs, jumpargs): if isinstance(box, Const) or box in inputargs: return @@ -603,7 +601,7 @@ classbox = self.getvalue(newop.result).get_constant_class(self.optimizer.cpu) if not classbox or not classbox.same_constant(target.assumed_classes[shop.result]): raise InvalidLoop('The class of an opaque pointer at the end ' + - 'of the bridge does not mach the class ' + + 'of the bridge does not mach the class ' + 'it has at the start of the target loop') except InvalidLoop: #debug_print("Inlining failed unexpectedly", @@ -615,6 +613,7 @@ debug_stop('jit-log-virtualstate') return False + class ValueImporter(object): def __init__(self, unroll, value, op): self.unroll = unroll @@ -623,7 +622,8 @@ def import_value(self, value): value.import_from(self.preamble_value, self.unroll.optimizer) - self.unroll.add_op_to_short(self.op, False, True) + self.unroll.add_op_to_short(self.op, False, True) + class ExportedState(object): def __init__(self, short_boxes, inputarg_setup_ops, exported_values): diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py --- a/rpython/jit/metainterp/test/test_ajit.py +++ b/rpython/jit/metainterp/test/test_ajit.py @@ -1694,14 +1694,13 @@ assert res == -2 def test_guard_always_changing_value(self): - myjitdriver = JitDriver(greens = [], reds = ['x']) - class A: - pass + myjitdriver = JitDriver(greens = [], reds = ['x', 'a']) def f(x): + a = 0 while x > 0: - myjitdriver.can_enter_jit(x=x) - myjitdriver.jit_merge_point(x=x) - a = A() + myjitdriver.can_enter_jit(x=x, a=a) + myjitdriver.jit_merge_point(x=x, a=a) + a += 1 promote(a) x -= 1 self.meta_interp(f, [50]) diff --git a/rpython/jit/metainterp/virtualizable.py b/rpython/jit/metainterp/virtualizable.py --- a/rpython/jit/metainterp/virtualizable.py +++ b/rpython/jit/metainterp/virtualizable.py @@ -1,14 +1,12 @@ +from rpython.jit.metainterp import history +from rpython.jit.metainterp.typesystem import deref, fieldType, arrayItem +from rpython.jit.metainterp.warmstate import wrap, unwrap +from rpython.rlib.unroll import unrolling_iterable +from rpython.rtyper import rvirtualizable2 from rpython.rtyper.lltypesystem import lltype, llmemory from rpython.rtyper.ootypesystem import ootype -from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance from rpython.rtyper.rclass import IR_IMMUTABLE_ARRAY, IR_IMMUTABLE -from rpython.rtyper import rvirtualizable2 -from rpython.rlib.unroll import unrolling_iterable -from rpython.rlib.nonconst import NonConstant -from rpython.jit.metainterp.typesystem import deref, fieldType, arrayItem -from rpython.jit.metainterp import history -from rpython.jit.metainterp.warmstate import wrap, unwrap -from rpython.rlib.objectmodel import specialize + class VirtualizableInfo(object): TOKEN_NONE = 0 # must be 0 -- see also x86.call_assembler @@ -76,7 +74,7 @@ getlength = cpu.ts.getlength getarrayitem = cpu.ts.getarrayitem setarrayitem = cpu.ts.setarrayitem - # + def read_boxes(cpu, virtualizable): assert lltype.typeOf(virtualizable) == llmemory.GCREF virtualizable = cast_gcref_to_vtype(virtualizable) @@ -89,7 +87,7 @@ for i in range(getlength(lst)): boxes.append(wrap(cpu, getarrayitem(lst, i))) return boxes - # + def write_boxes(virtualizable, boxes): virtualizable = cast_gcref_to_vtype(virtualizable) i = 0 @@ -104,7 +102,7 @@ setarrayitem(lst, j, x) i = i + 1 assert len(boxes) == i + 1 - # + def write_from_resume_data_partial(virtualizable, reader, numb): virtualizable = cast_gcref_to_vtype(virtualizable) # Load values from the reader (see resume.py) described by @@ -117,7 +115,7 @@ assert i >= 0 for ARRAYITEMTYPE, fieldname in unroll_array_fields_rev: lst = getattr(virtualizable, fieldname) - for j in range(getlength(lst)-1, -1, -1): + for j in range(getlength(lst) - 1, -1, -1): i -= 1 assert i >= 0 x = reader.load_value_of_type(ARRAYITEMTYPE, numb.nums[i]) @@ -128,7 +126,7 @@ x = reader.load_value_of_type(FIELDTYPE, numb.nums[i]) setattr(virtualizable, fieldname, x) return i - # + def load_list_of_boxes(virtualizable, reader, numb): virtualizable = cast_gcref_to_vtype(virtualizable) # Uses 'virtualizable' only to know the length of the arrays; @@ -140,10 +138,10 @@ boxes = [reader.decode_box_of_type(self.VTYPEPTR, numb.nums[i])] for ARRAYITEMTYPE, fieldname in unroll_array_fields_rev: lst = getattr(virtualizable, fieldname) - for j in range(getlength(lst)-1, -1, -1): + for j in range(getlength(lst) - 1, -1, -1): i -= 1 assert i >= 0 - box = reader.decode_box_of_type(ARRAYITEMTYPE,numb.nums[i]) + box = reader.decode_box_of_type(ARRAYITEMTYPE, numb.nums[i]) boxes.append(box) for FIELDTYPE, fieldname in unroll_static_fields_rev: i -= 1 @@ -152,7 +150,7 @@ boxes.append(box) boxes.reverse() return boxes - # + def check_boxes(virtualizable, boxes): virtualizable = cast_gcref_to_vtype(virtualizable) # for debugging @@ -168,7 +166,7 @@ assert getarrayitem(lst, j) == x i = i + 1 assert len(boxes) == i + 1 - # + def get_index_in_array(virtualizable, arrayindex, index): virtualizable = cast_gcref_to_vtype(virtualizable) index += self.num_static_extra_boxes @@ -180,7 +178,7 @@ index += getlength(lst) j = j + 1 assert False, "invalid arrayindex" - # + def get_array_length(virtualizable, arrayindex): virtualizable = cast_gcref_to_vtype(virtualizable) j = 0 @@ -188,16 +186,16 @@ if arrayindex == j: lst = getattr(virtualizable, fieldname) return getlength(lst) - j = j + 1 + j += 1 assert False, "invalid arrayindex" - # + unroll_static_fields = unrolling_iterable(zip(FIELDTYPES, static_fields)) unroll_array_fields = unrolling_iterable(zip(ARRAYITEMTYPES, array_fields)) unroll_static_fields_rev = unrolling_iterable( reversed(list(unroll_static_fields))) - unroll_array_fields_rev = unrolling_iterable( + unroll_array_fields_rev = unrolling_iterable( reversed(list(unroll_array_fields))) self.read_boxes = read_boxes self.write_boxes = write_boxes @@ -291,7 +289,7 @@ def unwrap_virtualizable_box(self, virtualizable_box): return virtualizable_box.getref(llmemory.GCREF) - + def is_vtypeptr(self, TYPE): return rvirtualizable2.match_virtualizable_type(TYPE, self.VTYPEPTR) diff --git a/rpython/translator/c/src/thread_nt.c b/rpython/translator/c/src/thread_nt.c --- a/rpython/translator/c/src/thread_nt.c +++ b/rpython/translator/c/src/thread_nt.c @@ -13,6 +13,7 @@ /* * Thread support. */ +/* In rpython, this file is pulled in by thread.c */ typedef struct RPyOpaque_ThreadLock NRMUTEX, *PNRMUTEX; diff --git a/rpython/translator/c/src/thread_nt.h b/rpython/translator/c/src/thread_nt.h --- a/rpython/translator/c/src/thread_nt.h +++ b/rpython/translator/c/src/thread_nt.h @@ -9,6 +9,7 @@ } NRMUTEX, *PNRMUTEX; /* prototypes */ +long RPyThreadGetIdent(void); long RPyThreadStart(void (*func)(void)); int RPyThreadLockInit(struct RPyOpaque_ThreadLock *lock); void RPyOpaqueDealloc_ThreadLock(struct RPyOpaque_ThreadLock *lock); From noreply at buildbot.pypy.org Thu Jan 31 11:21:02 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 31 Jan 2013 11:21:02 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: Unconfuse myself properly. The previous checkin failed at doing so. :-) Message-ID: <20130131102102.B1F071C040D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: jitframe-on-heap Changeset: r60760:76165f7a6113 Date: 2013-01-31 11:20 +0100 http://bitbucket.org/pypy/pypy/changeset/76165f7a6113/ Log: Unconfuse myself properly. The previous checkin failed at doing so. :-) diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -603,7 +603,7 @@ self.current_clt.frame_info) stack_check_patch_ofs, ofs2 = self._check_frame_depth(self.mc, regalloc.get_gcmap()) - frame_depth = self._assemble(regalloc, inputargs, operations) + frame_depth_no_fixed_size = self._assemble(regalloc, inputargs, operations) codeendpos = self.mc.get_relative_pos() self.write_pending_failure_recoveries() fullsize = self.mc.get_relative_pos() @@ -618,7 +618,7 @@ self.patch_jump_for_descr(faildescr, rawstart) ops_offset = self.mc.ops_offset frame_depth = max(self.current_clt.frame_info.jfi_frame_depth, - frame_depth + JITFRAME_FIXED_SIZE) + frame_depth_no_fixed_size + JITFRAME_FIXED_SIZE) self._patch_stackadjust(stack_check_patch_ofs + rawstart, frame_depth) self._patch_stackadjust(ofs2 + rawstart, frame_depth) for label in self.labels_to_patch: From noreply at buildbot.pypy.org Thu Jan 31 11:37:08 2013 From: noreply at buildbot.pypy.org (fijal) Date: Thu, 31 Jan 2013 11:37:08 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: (arigo, fijal) Fight a bit with realloc_frame. We should put stuff back Message-ID: <20130131103708.65D251C1305@cobra.cs.uni-duesseldorf.de> Author: Maciej Fijalkowski Branch: jitframe-on-heap Changeset: r60761:95cbf1e7e19e Date: 2013-01-31 12:36 +0200 http://bitbucket.org/pypy/pypy/changeset/95cbf1e7e19e/ Log: (arigo, fijal) Fight a bit with realloc_frame. We should put stuff back on top of shadowstack if we choose to reallocate frame diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -215,11 +215,19 @@ mc.CALL(imm(self.cpu.realloc_frame)) mc.ADD_ri(esp.value, WORD) mc.LEA_rm(ebp.value, (eax.value, base_ofs)) + + gcrootmap = self.cpu.gc_ll_descr.gcrootmap + if gcrootmap and gcrootmap.is_shadow_stack: + self._load_shadowstack_top_in_ebx(mc, gcrootmap) + mc.MOV_mr((ebx.value, -WORD), eax.value) + mc.MOV_bi(gcmap_ofs, 0) self._pop_all_regs_from_frame(mc, [], self.cpu.supports_floats) mc.RET() self._stack_check_failure = mc.materialize(self.cpu.asmmemmgr, []) + # XXX an almost identical copy for debugging, remove + mc = codebuf.MachineCodeBlockWrapper() self._push_all_regs_to_frame(mc, [], self.cpu.supports_floats) assert not IS_X86_32 @@ -236,6 +244,11 @@ mc.CALL(imm(self.cpu.realloc_frame_check)) mc.ADD_ri(esp.value, WORD) mc.LEA_rm(ebp.value, (eax.value, base_ofs)) + + gcrootmap = self.cpu.gc_ll_descr.gcrootmap + if gcrootmap and gcrootmap.is_shadow_stack: + self._load_shadowstack_top_in_ebx(mc, gcrootmap) + mc.MOV_mr((ebx.value, -WORD), eax.value) mc.MOV_bi(gcmap_ofs, 0) self._pop_all_regs_from_frame(mc, [], self.cpu.supports_floats) mc.RET() @@ -877,27 +890,33 @@ self.mc.ADD_ri(esp.value, FRAME_FIXED_SIZE * WORD) self.mc.RET() + def _load_shadowstack_top_in_ebx(self, mc, gcrootmap): + rst = gcrootmap.get_root_stack_top_addr() + if rx86.fits_in_32bits(rst): + mc.MOV_rj(ebx.value, rst) # MOV ebx, [rootstacktop] + else: + mc.MOV_ri(X86_64_SCRATCH_REG.value, rst) # MOV r11, rootstacktop + mc.MOV_rm(ebx.value, (X86_64_SCRATCH_REG.value, 0)) + # MOV ebx, [r11] + # + return rst + def _call_header_shadowstack(self, gcrootmap): # we don't *really* have to do it, since we have the frame # being referenced by the caller. However, we still do it # to provide a place where we can read the frame from, in case # we need to reload it after a collection - rst = gcrootmap.get_root_stack_top_addr() - if rx86.fits_in_32bits(rst): - self.mc.MOV_rj(eax.value, rst) # MOV eax, [rootstacktop] - else: - self.mc.MOV_ri(r13.value, rst) # MOV r13, rootstacktop - self.mc.MOV_rm(eax.value, (r13.value, 0)) # MOV eax, [r13] - # + rst = self._load_shadowstack_top_in_ebx(self.mc, gcrootmap) if IS_X86_64: - self.mc.MOV_mr((eax.value, 0), edi.value) # MOV [eax], edi + self.mc.MOV_mr((ebx.value, 0), edi.value) # MOV [ebx], edi else: xxx - self.mc.ADD_ri(eax.value, WORD) + self.mc.ADD_ri(ebx.value, WORD) if rx86.fits_in_32bits(rst): - self.mc.MOV_jr(rst, eax.value) # MOV [rootstacktop], eax + self.mc.MOV_jr(rst, ebx.value) # MOV [rootstacktop], edx else: - self.mc.MOV_mr((r13.value, 0), eax.value) # MOV [r13], eax + self.mc.MOV_mr((X86_64_SCRATCH_REG.value, 0), + ebx.value) # MOV [r11], edx def _call_footer_shadowstack(self, gcrootmap): rst = gcrootmap.get_root_stack_top_addr() From noreply at buildbot.pypy.org Thu Jan 31 11:38:18 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 31 Jan 2013 11:38:18 +0100 (CET) Subject: [pypy-commit] pypy jitframe-on-heap: Fix comments Message-ID: <20130131103818.18B651C1305@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: jitframe-on-heap Changeset: r60762:7e35d73fb7ab Date: 2013-01-31 11:38 +0100 http://bitbucket.org/pypy/pypy/changeset/7e35d73fb7ab/ Log: Fix comments diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -913,10 +913,10 @@ xxx self.mc.ADD_ri(ebx.value, WORD) if rx86.fits_in_32bits(rst): - self.mc.MOV_jr(rst, ebx.value) # MOV [rootstacktop], edx + self.mc.MOV_jr(rst, ebx.value) # MOV [rootstacktop], ebx else: self.mc.MOV_mr((X86_64_SCRATCH_REG.value, 0), - ebx.value) # MOV [r11], edx + ebx.value) # MOV [r11], ebx def _call_footer_shadowstack(self, gcrootmap): rst = gcrootmap.get_root_stack_top_addr() From noreply at buildbot.pypy.org Thu Jan 31 12:46:12 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 31 Jan 2013 12:46:12 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: Preparing for the merge of default Message-ID: <20130131114612.634751C009B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r60763:131909e488a9 Date: 2013-01-31 12:20 +0100 http://bitbucket.org/pypy/pypy/changeset/131909e488a9/ Log: Preparing for the merge of default diff --git a/pypy/module/signal/__init__.py b/pypy/module/signal/__init__.py --- a/pypy/module/signal/__init__.py +++ b/pypy/module/signal/__init__.py @@ -30,7 +30,8 @@ } def buildloaders(cls): - from pypy.rlib import rsignal + from rpython.rlib import rsignal + for name in rsignal.signal_names: signum = getattr(rsignal, name) if signum is not None: diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py --- a/pypy/module/signal/interp_signal.py +++ b/pypy/module/signal/interp_signal.py @@ -1,20 +1,21 @@ from __future__ import with_statement import signal as cpy_signal +import sys from pypy.interpreter.error import OperationError, exception_from_errno from pypy.interpreter.executioncontext import (AsyncAction, AbstractActionFlag, PeriodicAsyncAction) from pypy.interpreter.gateway import unwrap_spec -from pypy.rlib import jit, rposix -from pypy.rlib.rarithmetic import intmask -from pypy.rlib.rsignal import (pypysig_getaddr_occurred, pypysig_setflag, - pypysig_poll, pypysig_reinstall, pypysig_ignore, pypysig_default, - pypysig_set_wakeup_fd, c_alarm, c_pause, c_getitimer, c_setitimer, - c_siginterrupt, itimervalP, NSIG, SIG_DFL, SIG_IGN, ITIMER_REAL, - ITIMER_PROF, ITIMER_VIRTUAL, signal_values) -from pypy.rpython.lltypesystem import lltype, rffi +from rpython.rlib import jit, rposix, rgc +from rpython.rlib.objectmodel import we_are_translated +from rpython.rlib.rarithmetic import intmask +from rpython.rlib.rsignal import * +from rpython.rtyper.lltypesystem import lltype, rffi + + +WIN32 = sys.platform == 'win32' class SignalActionFlag(AbstractActionFlag): @@ -31,6 +32,11 @@ p = pypysig_getaddr_occurred() p.c_value = value + @staticmethod + def rearm_ticker(): + p = pypysig_getaddr_occurred() + p.c_value = -1 + def decrement_ticker(self, by): p = pypysig_getaddr_occurred() value = p.c_value @@ -46,41 +52,63 @@ class CheckSignalAction(PeriodicAsyncAction): """An action that is automatically invoked when a signal is received.""" + # Note that this is a PeriodicAsyncAction: it means more precisely + # that it is called whenever the C-level ticker becomes < 0. + # Without threads, it is only ever set to -1 when we receive a + # signal. With threads, it also decrements steadily (but slowly). + def __init__(self, space): + "NOT_RPYTHON" AsyncAction.__init__(self, space) self.handlers_w = {} - self.emulated_sigint = False + self.pending_signal = -1 + self.fire_in_main_thread = False + if self.space.config.objspace.usemodules.thread: + from pypy.module.thread import gil + gil.after_thread_switch = self._after_thread_switch + + @rgc.no_collect + def _after_thread_switch(self): + if self.fire_in_main_thread: + if self.space.threadlocals.ismainthread(): + self.fire_in_main_thread = False + SignalActionFlag.rearm_ticker() + # this occurs when we just switched to the main thread + # and there is a signal pending: we force the ticker to + # -1, which should ensure perform() is called quickly. @jit.dont_look_inside def perform(self, executioncontext, frame): - if self.space.config.objspace.usemodules.thread: - main_ec = self.space.threadlocals.getmainthreadvalue() - in_main = executioncontext is main_ec - else: - in_main = True - # If we are in the main thread, poll and report the signals now. - if in_main: - if self.emulated_sigint: - self.emulated_sigint = False - self._report_signal(cpy_signal.SIGINT) - while True: - n = pypysig_poll() - if n < 0: - break + # Poll for the next signal, if any + n = self.pending_signal + if n < 0: n = pypysig_poll() + while n >= 0: + if self.space.config.objspace.usemodules.thread: + in_main = self.space.threadlocals.ismainthread() + else: + in_main = True + if in_main: + # If we are in the main thread, report the signal now, + # and poll more + self.pending_signal = -1 self._report_signal(n) - else: - # Otherwise, don't call pypysig_poll() at all. Instead, - # arrange for perform() to be called again after a thread - # switch. It might be called again and again, until we - # land in the main thread. - self.fire_after_thread_switch() + n = self.pending_signal + if n < 0: n = pypysig_poll() + else: + # Otherwise, arrange for perform() to be called again + # after we switch to the main thread. + self.pending_signal = n + self.fire_in_main_thread = True + break - @jit.dont_look_inside def set_interrupt(self): "Simulates the effect of a SIGINT signal arriving" - ec = self.space.getexecutioncontext() - self.emulated_sigint = True - self.perform(ec, None) + if not we_are_translated(): + self.pending_signal = cpy_signal.SIGINT + # ^^^ may override another signal, but it's just for testing + else: + pypysig_pushback(cpy_signal.SIGINT) + self.fire_in_main_thread = True def _report_signal(self, n): try: @@ -109,12 +137,16 @@ None -- if an unknown handler is in effect (XXX UNIMPLEMENTED) anything else -- the callable Python object used as a handler """ - check_signum_in_range(space, signum) + if WIN32: + check_signum_exists(space, signum) + else: + check_signum_in_range(space, signum) action = space.check_signal_action if signum in action.handlers_w: return action.handlers_w[signum] return space.wrap(SIG_DFL) + def default_int_handler(space, w_signum, w_frame): """ default_int_handler(...) @@ -125,22 +157,26 @@ raise OperationError(space.w_KeyboardInterrupt, space.w_None) + @jit.dont_look_inside @unwrap_spec(timeout=int) def alarm(space, timeout): return space.wrap(c_alarm(timeout)) + @jit.dont_look_inside def pause(space): c_pause() return space.w_None + def check_signum_exists(space, signum): if signum in signal_values: return raise OperationError(space.w_ValueError, space.wrap("invalid signal value")) + def check_signum_in_range(space, signum): if 1 <= signum < NSIG: return @@ -162,7 +198,7 @@ A signal handler function is called with two arguments: the first is the signal number, the second is the interrupted stack frame. """ - ec = space.getexecutioncontext() + ec = space.getexecutioncontext() main_ec = space.threadlocals.getmainthreadvalue() old_handler = getsignal(space, signum) @@ -185,13 +221,14 @@ action.handlers_w[signum] = w_handler return old_handler + @jit.dont_look_inside @unwrap_spec(fd=int) def set_wakeup_fd(space, fd): """Sets the fd to be written to (with '\0') when a signal comes in. Returns the old fd. A library can use this to wakeup select or poll. The previous fd is returned. - + The fd must be non-blocking. """ if space.config.objspace.usemodules.thread: @@ -204,6 +241,7 @@ old_fd = pypysig_set_wakeup_fd(fd) return space.wrap(intmask(old_fd)) + @jit.dont_look_inside @unwrap_spec(signum=int, flag=int) def siginterrupt(space, signum, flag): @@ -219,33 +257,38 @@ rffi.setintfield(timeval, 'c_tv_sec', int(d)) rffi.setintfield(timeval, 'c_tv_usec', int((d - int(d)) * 1000000)) + def double_from_timeval(tv): return rffi.getintfield(tv, 'c_tv_sec') + ( rffi.getintfield(tv, 'c_tv_usec') / 1000000.0) + def itimer_retval(space, val): w_value = space.wrap(double_from_timeval(val.c_it_value)) w_interval = space.wrap(double_from_timeval(val.c_it_interval)) return space.newtuple([w_value, w_interval]) + class Cache: def __init__(self, space): self.w_itimererror = space.new_exception_class("signal.ItimerError", space.w_IOError) + def get_itimer_error(space): return space.fromcache(Cache).w_itimererror + @jit.dont_look_inside @unwrap_spec(which=int, first=float, interval=float) def setitimer(space, which, first, interval=0): """setitimer(which, seconds[, interval]) - Sets given itimer (one of ITIMER_REAL, ITIMER_VIRTUAL + or ITIMER_PROF) to fire after value seconds and after that every interval seconds. The itimer can be cleared by setting seconds to zero. - + Returns old values as a tuple: (delay, interval). """ with lltype.scoped_alloc(itimervalP.TO, 1) as new: @@ -259,14 +302,14 @@ if ret != 0: raise exception_from_errno(space, get_itimer_error(space)) + return itimer_retval(space, old[0]) - return itimer_retval(space, old[0]) @jit.dont_look_inside @unwrap_spec(which=int) def getitimer(space, which): """getitimer(which) - + Returns current value of given itimer. """ with lltype.scoped_alloc(itimervalP.TO, 1) as old: diff --git a/pypy/module/signal/test/test_interp_signal.py b/pypy/module/signal/test/test_interp_signal.py deleted file mode 100644 --- a/pypy/module/signal/test/test_interp_signal.py +++ /dev/null @@ -1,39 +0,0 @@ -import os, py -from pypy.translator.c.test.test_genc import compile -from pypy.module.signal import interp_signal - -def setup_module(mod): - if not hasattr(os, 'kill') or not hasattr(os, 'getpid'): - py.test.skip("requires os.kill() and os.getpid()") - if not hasattr(interp_signal, 'SIGUSR1'): - py.test.skip("requires SIGUSR1 in signal") - - -def check(expected): - res = interp_signal.pypysig_poll() - os.write(1, "poll() => %d, expected %d\n" % (res, expected)) - assert res == expected - -def test_simple(): - import os - check(-1) - check(-1) - for i in range(3): - interp_signal.pypysig_setflag(interp_signal.SIGUSR1) - os.kill(os.getpid(), interp_signal.SIGUSR1) - check(interp_signal.SIGUSR1) - check(-1) - check(-1) - - interp_signal.pypysig_ignore(interp_signal.SIGUSR1) - os.kill(os.getpid(), interp_signal.SIGUSR1) - check(-1) - check(-1) - - interp_signal.pypysig_default(interp_signal.SIGUSR1) - check(-1) - - -def test_compile(): - fn = compile(test_simple, []) - fn() diff --git a/pypy/module/signal/test/test_signal.py b/pypy/module/signal/test/test_signal.py --- a/pypy/module/signal/test/test_signal.py +++ b/pypy/module/signal/test/test_signal.py @@ -36,7 +36,7 @@ class AppTestSignal: spaceconfig = { - "usemodules": ['signal', 'rctime'], + "usemodules": ['signal', 'rctime'] + (['fcntl'] if os.name != 'nt' else []), } def setup_class(cls): @@ -55,7 +55,7 @@ skip("requires os.kill() and os.getpid()") signal = self.signal # the signal module to test if not hasattr(signal, 'SIGUSR1'): - py.test.skip("requires SIGUSR1 in signal") + skip("requires SIGUSR1 in signal") signum = signal.SIGUSR1 received = [] @@ -156,6 +156,9 @@ import sys if sys.platform == 'win32': raises(ValueError, signal, 42, lambda *args: None) + raises(ValueError, signal, 7, lambda *args: None) + elif sys.platform == 'darwin': + raises(ValueError, signal, 42, lambda *args: None) else: signal(42, lambda *args: None) signal(42, SIG_DFL) From noreply at buildbot.pypy.org Thu Jan 31 12:46:14 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 31 Jan 2013 12:46:14 +0100 (CET) Subject: [pypy-commit] pypy default: Factor out this class in a file used only during tests. Message-ID: <20130131114614.14B4A1C0246@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r60764:2ac58df6d115 Date: 2013-01-31 12:36 +0100 http://bitbucket.org/pypy/pypy/changeset/2ac58df6d115/ Log: Factor out this class in a file used only during tests. diff --git a/rpython/rtyper/memory/gc/minimark.py b/rpython/rtyper/memory/gc/minimark.py --- a/rpython/rtyper/memory/gc/minimark.py +++ b/rpython/rtyper/memory/gc/minimark.py @@ -47,7 +47,7 @@ from rpython.rtyper.lltypesystem.lloperation import llop from rpython.rtyper.lltypesystem.llmemory import raw_malloc_usage from rpython.rtyper.memory.gc.base import GCBase, MovingGCBase -from rpython.rtyper.memory.gc import minimarkpage, env +from rpython.rtyper.memory.gc import env from rpython.rtyper.memory.support import mangle_hash from rpython.rlib.rarithmetic import ovfcheck, LONG_BIT, intmask, r_uint from rpython.rlib.rarithmetic import LONG_BIT_SHIFT @@ -254,6 +254,7 @@ # # The ArenaCollection() handles the nonmovable objects allocation. if ArenaCollectionClass is None: + from rpython.rtyper.memory.gc import minimarkpage ArenaCollectionClass = minimarkpage.ArenaCollection self.ac = ArenaCollectionClass(arena_size, page_size, small_request_threshold) @@ -2033,43 +2034,3 @@ (obj + offset).address[0] = llmemory.NULL self.old_objects_with_weakrefs.delete() self.old_objects_with_weakrefs = new_with_weakref - - -# ____________________________________________________________ - -# For testing, a simple implementation of ArenaCollection. -# This version could be used together with obmalloc.c, but -# it requires an extra word per object in the 'all_objects' -# list. - -class SimpleArenaCollection(object): - - def __init__(self, arena_size, page_size, small_request_threshold): - self.arena_size = arena_size # ignored - self.page_size = page_size - self.small_request_threshold = small_request_threshold - self.all_objects = [] - self.total_memory_used = 0 - - def malloc(self, size): - nsize = raw_malloc_usage(size) - ll_assert(nsize > 0, "malloc: size is null or negative") - ll_assert(nsize <= self.small_request_threshold,"malloc: size too big") - ll_assert((nsize & (WORD-1)) == 0, "malloc: size is not aligned") - # - result = llarena.arena_malloc(nsize, False) - llarena.arena_reserve(result, size) - self.all_objects.append((result, nsize)) - self.total_memory_used += nsize - return result - - def mass_free(self, ok_to_free_func): - objs = self.all_objects - self.all_objects = [] - self.total_memory_used = 0 - for rawobj, nsize in objs: - if ok_to_free_func(rawobj): - llarena.arena_free(rawobj) - else: - self.all_objects.append((rawobj, nsize)) - self.total_memory_used += nsize diff --git a/rpython/rtyper/memory/gc/minimarktest.py b/rpython/rtyper/memory/gc/minimarktest.py new file mode 100644 --- /dev/null +++ b/rpython/rtyper/memory/gc/minimarktest.py @@ -0,0 +1,44 @@ +from rpython.rtyper.lltypesystem import llarena +from rpython.rtyper.lltypesystem.llmemory import raw_malloc_usage +from rpython.rlib.debug import ll_assert +from rpython.rlib.rarithmetic import LONG_BIT + +# For testing, a simple implementation of ArenaCollection. +# This version could be used together with obmalloc.c, but +# it requires an extra word per object in the 'all_objects' +# list. + +WORD = LONG_BIT // 8 + + +class SimpleArenaCollection(object): + + def __init__(self, arena_size, page_size, small_request_threshold): + self.arena_size = arena_size # ignored + self.page_size = page_size + self.small_request_threshold = small_request_threshold + self.all_objects = [] + self.total_memory_used = 0 + + def malloc(self, size): + nsize = raw_malloc_usage(size) + ll_assert(nsize > 0, "malloc: size is null or negative") + ll_assert(nsize <= self.small_request_threshold,"malloc: size too big") + ll_assert((nsize & (WORD-1)) == 0, "malloc: size is not aligned") + # + result = llarena.arena_malloc(nsize, False) + llarena.arena_reserve(result, size) + self.all_objects.append((result, nsize)) + self.total_memory_used += nsize + return result + + def mass_free(self, ok_to_free_func): + objs = self.all_objects + self.all_objects = [] + self.total_memory_used = 0 + for rawobj, nsize in objs: + if ok_to_free_func(rawobj): + llarena.arena_free(rawobj) + else: + self.all_objects.append((rawobj, nsize)) + self.total_memory_used += nsize diff --git a/rpython/rtyper/memory/gc/test/test_direct.py b/rpython/rtyper/memory/gc/test/test_direct.py --- a/rpython/rtyper/memory/gc/test/test_direct.py +++ b/rpython/rtyper/memory/gc/test/test_direct.py @@ -481,7 +481,7 @@ class TestMiniMarkGCSimple(DirectGCTest): from rpython.rtyper.memory.gc.minimark import MiniMarkGC as GCClass - from rpython.rtyper.memory.gc.minimark import SimpleArenaCollection + from rpython.rtyper.memory.gc.minimarktest import SimpleArenaCollection # test the GC itself, providing a simple class for ArenaCollection GC_PARAMS = {'ArenaCollectionClass': SimpleArenaCollection} diff --git a/rpython/rtyper/memory/gc/test/test_inspector.py b/rpython/rtyper/memory/gc/test/test_inspector.py --- a/rpython/rtyper/memory/gc/test/test_inspector.py +++ b/rpython/rtyper/memory/gc/test/test_inspector.py @@ -43,6 +43,6 @@ class TestMiniMarkGCSimple(InspectorTest): from rpython.rtyper.memory.gc.minimark import MiniMarkGC as GCClass - from rpython.rtyper.memory.gc.minimark import SimpleArenaCollection + from rpython.rtyper.memory.gc.minimarktest import SimpleArenaCollection GC_PARAMS = {'ArenaCollectionClass': SimpleArenaCollection, "card_page_indices": 4} From noreply at buildbot.pypy.org Thu Jan 31 12:46:46 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 31 Jan 2013 12:46:46 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: hg merge default (untested so far) Message-ID: <20130131114646.DB2051C009B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r60765:5e612b71875b Date: 2013-01-31 12:44 +0100 http://bitbucket.org/pypy/pypy/changeset/5e612b71875b/ Log: hg merge default (untested so far) diff too long, truncating to 2000 out of 812147 lines diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -85,3 +85,4 @@ ^compiled ^.git/ ^release/ +^rpython/_cache$ diff --git a/dotviewer/test/test_translator.py b/dotviewer/test/test_translator.py --- a/dotviewer/test/test_translator.py +++ b/dotviewer/test/test_translator.py @@ -21,7 +21,7 @@ def test_annotated(): - from pypy.translator.interactive import Translation + from rpython.translator.interactive import Translation t = Translation(is_prime) t.annotate([int]) t.viewcg() diff --git a/lib-python/2.7/inspect.py b/lib-python/2.7/inspect.py --- a/lib-python/2.7/inspect.py +++ b/lib-python/2.7/inspect.py @@ -960,7 +960,7 @@ raise TypeError('%s() takes exactly 0 arguments ' '(%d given)' % (f_name, num_total)) else: - raise TypeError('%s() takes no argument (%d given)' % + raise TypeError('%s() takes no arguments (%d given)' % (f_name, num_total)) for arg in args: if isinstance(arg, str) and arg in named: diff --git a/lib-python/2.7/site.py b/lib-python/2.7/site.py --- a/lib-python/2.7/site.py +++ b/lib-python/2.7/site.py @@ -75,6 +75,7 @@ USER_SITE = None USER_BASE = None + def makepath(*paths): dir = os.path.join(*paths) try: diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py --- a/lib-python/2.7/sysconfig.py +++ b/lib-python/2.7/sysconfig.py @@ -27,10 +27,10 @@ 'data' : '{base}', }, 'pypy': { - 'stdlib': '{base}/lib-python', - 'platstdlib': '{base}/lib-python', - 'purelib': '{base}/lib-python', - 'platlib': '{base}/lib-python', + 'stdlib': '{base}/lib-python/{py_version_short}', + 'platstdlib': '{base}/lib-python/{py_version_short}', + 'purelib': '{base}/lib-python/{py_version_short}', + 'platlib': '{base}/lib-python/{py_version_short}', 'include': '{base}/include', 'platinclude': '{base}/include', 'scripts': '{base}/bin', diff --git a/lib-python/2.7/test/test_capi.py b/lib-python/2.7/test/test_capi.py --- a/lib-python/2.7/test/test_capi.py +++ b/lib-python/2.7/test/test_capi.py @@ -13,6 +13,17 @@ threading = None import _testcapi +skips = [] +if test_support.check_impl_detail(pypy=True): + skips += [ + 'test_broken_memoryview', + 'test_capsule', + 'test_lazy_hash_inheritance', + 'test_widechar', + 'TestThreadState', + 'TestPendingCalls', + ] + @unittest.skipUnless(threading, 'Threading required for this test.') class TestPendingCalls(unittest.TestCase): @@ -99,7 +110,7 @@ def test_main(): for name in dir(_testcapi): - if name.startswith('test_'): + if name.startswith('test_') and name not in skips: test = getattr(_testcapi, name) if test_support.verbose: print "internal", name @@ -126,7 +137,7 @@ raise test_support.TestFailed, \ "Couldn't find main thread correctly in the list" - if threading: + if threading and 'TestThreadState' not in skips: import thread import time TestThreadState() @@ -134,7 +145,8 @@ t.start() t.join() - test_support.run_unittest(TestPendingCalls) + if 'TestPendingCalls' not in skips: + test_support.run_unittest(TestPendingCalls) if __name__ == "__main__": test_main() diff --git a/lib-python/2.7/test/test_iterlen.py b/lib-python/2.7/test/test_iterlen.py --- a/lib-python/2.7/test/test_iterlen.py +++ b/lib-python/2.7/test/test_iterlen.py @@ -94,7 +94,11 @@ def test_no_len_for_infinite_repeat(self): # The repeat() object can also be infinite - self.assertRaises(TypeError, len, repeat(None)) + if test_support.check_impl_detail(pypy=True): + # 3.4 (PEP 424) behavior + self.assertEqual(len(repeat(None)), NotImplemented) + else: + self.assertRaises(TypeError, len, repeat(None)) class TestXrange(TestInvariantWithoutMutations): @@ -230,6 +234,7 @@ self.assertRaises(RuntimeError, b.extend, BadLen()) self.assertRaises(RuntimeError, b.extend, BadLengthHint()) + @test_support.impl_detail("PEP 424 disallows None results", pypy=False) def test_invalid_hint(self): # Make sure an invalid result doesn't muck-up the works self.assertEqual(list(NoneLengthHint()), list(range(10))) diff --git a/lib-python/2.7/test/test_itertools.py b/lib-python/2.7/test/test_itertools.py --- a/lib-python/2.7/test/test_itertools.py +++ b/lib-python/2.7/test/test_itertools.py @@ -533,11 +533,11 @@ self.assertEqual(list(izip()), zip()) self.assertRaises(TypeError, izip, 3) self.assertRaises(TypeError, izip, range(3), 3) - self.assertEqual([tuple(list(pair)) for pair in izip('abc', 'def')], zip('abc', 'def')) self.assertEqual([pair for pair in izip('abc', 'def')], zip('abc', 'def')) + @test_support.impl_detail("tuple reuse is specific to CPython") def test_izip_tuple_reuse(self): ids = map(id, izip('abc', 'def')) @@ -588,6 +588,7 @@ zip('abc', 'def')) self.assertEqual([pair for pair in izip_longest('abc', 'def')], zip('abc', 'def')) + @test_support.impl_detail("tuple reuse is specific to CPython") def test_izip_longest_tuple_reuse(self): ids = map(id, izip_longest('abc', 'def')) diff --git a/lib-python/2.7/test/test_multiprocessing.py b/lib-python/2.7/test/test_multiprocessing.py --- a/lib-python/2.7/test/test_multiprocessing.py +++ b/lib-python/2.7/test/test_multiprocessing.py @@ -1343,7 +1343,7 @@ # Because we are using xmlrpclib for serialization instead of # pickle this will cause a serialization error. - self.assertRaises(Exception, queue.put, time.sleep) + self.assertRaises(Exception, queue.put, object) # Make queue finalizer run before the server is stopped del queue @@ -1800,9 +1800,9 @@ if not gc.isenabled(): gc.enable() self.addCleanup(gc.disable) - thresholds = gc.get_threshold() - self.addCleanup(gc.set_threshold, *thresholds) - gc.set_threshold(10) + #thresholds = gc.get_threshold() + #self.addCleanup(gc.set_threshold, *thresholds) + #gc.set_threshold(10) # perform numerous block allocations, with cyclic references to make # sure objects are collected asynchronously by the gc @@ -1865,6 +1865,7 @@ def test_synchronize(self): self.test_sharedctypes(lock=True) + @unittest.skipUnless(test_support.check_impl_detail(pypy=False), "pypy ctypes differences") def test_copy(self): foo = _Foo(2, 5.0) bar = copy(foo) diff --git a/lib-python/2.7/test/test_support.py b/lib-python/2.7/test/test_support.py --- a/lib-python/2.7/test/test_support.py +++ b/lib-python/2.7/test/test_support.py @@ -1085,7 +1085,6 @@ else: runner = BasicTestRunner() - result = runner.run(suite) if not result.wasSuccessful(): if len(result.errors) == 1 and not result.failures: diff --git a/lib-python/conftest.py b/lib-python/conftest.py --- a/lib-python/conftest.py +++ b/lib-python/conftest.py @@ -17,8 +17,8 @@ from pypy.conftest import option as pypy_option from pypy.tool.pytest import appsupport -from pypy.tool.pytest.confpath import pypydir, testdir, testresultdir -from pypy.config.parse import parse_info +from pypy.tool.pytest.confpath import pypydir, rpythondir, testdir, testresultdir +from rpython.config.parse import parse_info pytest_plugins = "resultlog", rsyncdirs = ['.', '../pypy/'] @@ -40,7 +40,7 @@ dest="unittest_filter", help="Similar to -k, XXX") def gettimeout(timeout): - from test import pystone + from rpython.translator.test import rpystone if timeout.endswith('mp'): megapystone = float(timeout[:-2]) t, stone = pystone.Proc0(10000) @@ -61,7 +61,7 @@ usemodules = '', skip=None): self.basename = basename - self._usemodules = usemodules.split() + ['signal'] + self._usemodules = usemodules.split() + ['signal', 'rctime', 'itertools', '_socket'] self._compiler = compiler self.core = core self.skip = skip @@ -93,63 +93,57 @@ m.test_main() ''' % locals()) -if sys.platform == 'win32': - skip_win32 = "Not supported on Windows" - only_win32 = False -else: - skip_win32 = False - only_win32 = "Only on Windows" - testmap = [ RegrTest('test___all__.py', core=True), RegrTest('test___future__.py', core=True), - RegrTest('test__locale.py', skip=skip_win32), + RegrTest('test__locale.py', usemodules='_locale'), RegrTest('test_abc.py'), RegrTest('test_abstract_numbers.py'), - RegrTest('test_aepack.py', skip=True), + RegrTest('test_aepack.py'), RegrTest('test_aifc.py'), - RegrTest('test_argparse.py'), - RegrTest('test_al.py', skip=True), - RegrTest('test_ast.py', core=True), - RegrTest('test_anydbm.py'), - RegrTest('test_applesingle.py', skip=True), - RegrTest('test_array.py', core=True, usemodules='struct array'), + RegrTest('test_al.py'), + RegrTest('test_anydbm.py', usemodules='struct'), + RegrTest('test_applesingle.py'), + RegrTest('test_argparse.py', usemodules='binascii'), + RegrTest('test_array.py', core=True, usemodules='struct array binascii'), RegrTest('test_ascii_formatd.py'), - RegrTest('test_asynchat.py', usemodules='thread'), - RegrTest('test_asyncore.py'), + RegrTest('test_ast.py', core=True, usemodules='struct'), + RegrTest('test_asynchat.py', usemodules='select fcntl'), + RegrTest('test_asyncore.py', usemodules='select fcntl'), RegrTest('test_atexit.py', core=True), - RegrTest('test_audioop.py', skip=True), + RegrTest('test_audioop.py', skip="unsupported extension module"), RegrTest('test_augassign.py', core=True), - RegrTest('test_base64.py'), + RegrTest('test_base64.py', usemodules='struct'), RegrTest('test_bastion.py'), + RegrTest('test_bigaddrspace.py'), + RegrTest('test_bigmem.py'), RegrTest('test_binascii.py', usemodules='binascii'), - RegrTest('test_binhex.py'), - RegrTest('test_binop.py', core=True), RegrTest('test_bisect.py', core=True, usemodules='_bisect'), RegrTest('test_bool.py', core=True), - RegrTest('test_bsddb.py', skip="unsupported extension module"), - RegrTest('test_bsddb185.py', skip="unsupported extension module"), - RegrTest('test_bsddb3.py', skip="unsupported extension module"), + RegrTest('test_bsddb.py'), + RegrTest('test_bsddb185.py'), + RegrTest('test_bsddb3.py'), RegrTest('test_buffer.py'), RegrTest('test_bufio.py', core=True), - RegrTest('test_builtin.py', core=True), - RegrTest('test_bytes.py'), + RegrTest('test_builtin.py', core=True, usemodules='binascii'), + RegrTest('test_bytes.py', usemodules='struct binascii'), RegrTest('test_bz2.py', usemodules='bz2'), RegrTest('test_calendar.py'), RegrTest('test_call.py', core=True), - RegrTest('test_capi.py', skip="not applicable"), - RegrTest('test_cd.py', skip=True), + RegrTest('test_capi.py'), + RegrTest('test_cd.py'), RegrTest('test_cfgparser.py'), - RegrTest('test_cgi.py'), RegrTest('test_charmapcodec.py', core=True), - RegrTest('test_cl.py', skip=True), + RegrTest('test_cl.py'), RegrTest('test_class.py', core=True), RegrTest('test_cmath.py', core=True), RegrTest('test_cmd.py'), + RegrTest('test_cmd_line.py'), RegrTest('test_cmd_line_script.py'), + RegrTest('test_code.py', core=True), RegrTest('test_codeccallbacks.py', core=True), RegrTest('test_codecencodings_cn.py', usemodules='_multibytecodec'), RegrTest('test_codecencodings_hk.py', usemodules='_multibytecodec'), @@ -157,7 +151,6 @@ RegrTest('test_codecencodings_jp.py', usemodules='_multibytecodec'), RegrTest('test_codecencodings_kr.py', usemodules='_multibytecodec'), RegrTest('test_codecencodings_tw.py', usemodules='_multibytecodec'), - RegrTest('test_codecmaps_cn.py', usemodules='_multibytecodec'), RegrTest('test_codecmaps_hk.py', usemodules='_multibytecodec'), RegrTest('test_codecmaps_jp.py', usemodules='_multibytecodec'), @@ -165,6 +158,7 @@ RegrTest('test_codecmaps_tw.py', usemodules='_multibytecodec'), RegrTest('test_codecs.py', core=True, usemodules='_multibytecodec'), RegrTest('test_codeop.py', core=True), + RegrTest('test_coding.py', core=True), RegrTest('test_coercion.py', core=True), RegrTest('test_collections.py'), RegrTest('test_colorsys.py'), @@ -174,22 +168,24 @@ RegrTest('test_compileall.py'), RegrTest('test_compiler.py', core=False, skip="slowly deprecating compiler"), RegrTest('test_complex.py', core=True), - + RegrTest('test_complex_args.py'), RegrTest('test_contains.py', core=True), + RegrTest('test_contextlib.py', usemodules="thread"), RegrTest('test_cookie.py'), RegrTest('test_cookielib.py'), RegrTest('test_copy.py', core=True), RegrTest('test_copy_reg.py', core=True), RegrTest('test_cpickle.py', core=True), - RegrTest('test_cprofile.py'), - RegrTest('test_crypt.py', usemodules='crypt', skip=skip_win32), + RegrTest('test_cprofile.py'), + RegrTest('test_crypt.py', usemodules='crypt'), RegrTest('test_csv.py', usemodules='_csv'), - - RegrTest('test_curses.py', skip="unsupported extension module"), + RegrTest('test_ctypes.py', usemodules="_rawffi thread"), + RegrTest('test_curses.py'), RegrTest('test_datetime.py'), RegrTest('test_dbm.py'), RegrTest('test_decimal.py'), RegrTest('test_decorators.py', core=True), + RegrTest('test_defaultdict.py', usemodules='_collections'), RegrTest('test_deque.py', core=True, usemodules='_collections'), RegrTest('test_descr.py', core=True, usemodules='_weakref'), RegrTest('test_descrtut.py', core=True), @@ -200,7 +196,7 @@ RegrTest('test_dircache.py', core=True), RegrTest('test_dis.py'), RegrTest('test_distutils.py', skip=True), - RegrTest('test_dl.py', skip=True), + RegrTest('test_dl.py'), RegrTest('test_doctest.py', usemodules="thread"), RegrTest('test_doctest2.py'), RegrTest('test_docxmlrpc.py'), @@ -208,20 +204,22 @@ RegrTest('test_dummy_thread.py', core=True), RegrTest('test_dummy_threading.py', core=True), RegrTest('test_email.py'), - RegrTest('test_email_codecs.py'), + RegrTest('test_email_renamed.py'), RegrTest('test_enumerate.py', core=True), RegrTest('test_eof.py', core=True), RegrTest('test_epoll.py'), RegrTest('test_errno.py', usemodules="errno"), + RegrTest('test_exception_variations.py'), RegrTest('test_exceptions.py', core=True), RegrTest('test_extcall.py', core=True), - RegrTest('test_fcntl.py', usemodules='fcntl', skip=skip_win32), + RegrTest('test_fcntl.py', usemodules='fcntl'), RegrTest('test_file.py', usemodules="posix", core=True), RegrTest('test_file2k.py', usemodules="posix", core=True), RegrTest('test_filecmp.py', core=True), RegrTest('test_fileinput.py', core=True), RegrTest('test_fileio.py'), + RegrTest('test_float.py', core=True), RegrTest('test_fnmatch.py', core=True), RegrTest('test_fork1.py', usemodules="thread"), RegrTest('test_format.py', core=True), @@ -230,6 +228,7 @@ RegrTest('test_frozen.py', skip="unsupported extension module"), RegrTest('test_ftplib.py'), RegrTest('test_funcattrs.py', core=True), + RegrTest('test_functools.py'), RegrTest('test_future.py', core=True), RegrTest('test_future1.py', core=True), RegrTest('test_future2.py', core=True), @@ -239,41 +238,37 @@ RegrTest('test_future_builtins.py'), RegrTest('test_gc.py', usemodules='_weakref', skip="implementation detail"), RegrTest('test_gdb.py', skip="not applicable"), - RegrTest('test_gdbm.py', skip="unsupported extension module"), + RegrTest('test_gdbm.py'), RegrTest('test_generators.py', core=True, usemodules='thread _weakref'), RegrTest('test_genericpath.py'), RegrTest('test_genexps.py', core=True, usemodules='_weakref'), - RegrTest('test_getargs.py', skip="unsupported extension module"), - RegrTest('test_getargs2.py', skip="unsupported extension module"), - + RegrTest('test_getargs.py'), + RegrTest('test_getargs2.py', usemodules='binascii', skip=True), RegrTest('test_getopt.py', core=True), RegrTest('test_gettext.py'), - - RegrTest('test_gl.py', skip=True), + RegrTest('test_gl.py'), RegrTest('test_glob.py', core=True), RegrTest('test_global.py', core=True), RegrTest('test_grammar.py', core=True), - RegrTest('test_grp.py', skip=skip_win32), - - RegrTest('test_gzip.py'), + RegrTest('test_grp.py'), + RegrTest('test_gzip.py', usemodules='zlib'), RegrTest('test_hash.py', core=True), RegrTest('test_hashlib.py', core=True), - RegrTest('test_heapq.py', core=True), RegrTest('test_hmac.py'), RegrTest('test_hotshot.py', skip="unsupported extension module"), - RegrTest('test_htmllib.py'), RegrTest('test_htmlparser.py'), RegrTest('test_httplib.py'), RegrTest('test_httpservers.py'), - RegrTest('test_imageop.py', skip="unsupported extension module"), + RegrTest('test_imageop.py'), RegrTest('test_imaplib.py'), - RegrTest('test_imgfile.py', skip="unsupported extension module"), + RegrTest('test_imgfile.py'), RegrTest('test_imp.py', core=True, usemodules='thread'), RegrTest('test_import.py', core=True), RegrTest('test_importhooks.py', core=True), RegrTest('test_importlib.py'), + RegrTest('test_index.py'), RegrTest('test_inspect.py'), RegrTest('test_int.py', core=True), RegrTest('test_int_literal.py', core=True), @@ -281,7 +276,7 @@ RegrTest('test_ioctl.py'), RegrTest('test_isinstance.py', core=True), RegrTest('test_iter.py', core=True), - RegrTest('test_iterlen.py', skip="undocumented internal API behavior __length_hint__"), + RegrTest('test_iterlen.py', core=True, usemodules="_collections itertools"), RegrTest('test_itertools.py', core=True, usemodules="itertools struct"), RegrTest('test_json.py'), RegrTest('test_kqueue.py'), @@ -296,7 +291,7 @@ RegrTest('test_long_future.py', core=True), RegrTest('test_longexp.py', core=True), RegrTest('test_macos.py'), - RegrTest('test_macostools.py', skip=True), + RegrTest('test_macostools.py'), RegrTest('test_macpath.py'), RegrTest('test_mailbox.py'), RegrTest('test_marshal.py', core=True), @@ -307,30 +302,29 @@ RegrTest('test_mhlib.py'), RegrTest('test_mimetools.py'), RegrTest('test_mimetypes.py'), - RegrTest('test_MimeWriter.py', core=False), + RegrTest('test_MimeWriter.py', core=False, usemodules='binascii'), RegrTest('test_minidom.py'), RegrTest('test_mmap.py', usemodules="mmap"), RegrTest('test_module.py', core=True), RegrTest('test_modulefinder.py'), - RegrTest('test_msilib.py', skip=only_win32), + RegrTest('test_msilib.py'), RegrTest('test_multibytecodec.py', usemodules='_multibytecodec'), - RegrTest('test_multibytecodec_support.py', skip="not a test"), RegrTest('test_multifile.py'), - RegrTest('test_multiprocessing.py', skip="FIXME leaves subprocesses"), + RegrTest('test_multiprocessing.py'), RegrTest('test_mutants.py', core="possibly"), RegrTest('test_mutex.py'), RegrTest('test_netrc.py'), RegrTest('test_new.py', core=True), - RegrTest('test_nis.py', skip="unsupported extension module"), + RegrTest('test_nis.py'), RegrTest('test_normalization.py'), RegrTest('test_ntpath.py'), + RegrTest('test_old_mailbox.py'), RegrTest('test_opcodes.py', core=True), RegrTest('test_openpty.py'), RegrTest('test_operator.py', core=True), RegrTest('test_optparse.py'), - RegrTest('test_os.py', core=True), - RegrTest('test_ossaudiodev.py', skip="unsupported extension module"), + RegrTest('test_ossaudiodev.py'), RegrTest('test_parser.py', skip="slowly deprecating compiler"), RegrTest('test_pdb.py'), RegrTest('test_peepholer.py'), @@ -338,14 +332,16 @@ RegrTest('test_pep263.py'), RegrTest('test_pep277.py'), RegrTest('test_pep292.py'), + RegrTest('test_pep352.py'), RegrTest('test_pickle.py', core=True), RegrTest('test_pickletools.py', core=False), RegrTest('test_pipes.py'), RegrTest('test_pkg.py', core=True), RegrTest('test_pkgimport.py', core=True), RegrTest('test_pkgutil.py'), - RegrTest('test_plistlib.py', skip="unsupported module"), - RegrTest('test_poll.py', skip=skip_win32), + RegrTest('test_platform.py'), + RegrTest('test_plistlib.py'), + RegrTest('test_poll.py'), RegrTest('test_popen.py'), RegrTest('test_popen2.py'), RegrTest('test_poplib.py'), @@ -357,8 +353,8 @@ RegrTest('test_profile.py'), RegrTest('test_property.py', core=True), RegrTest('test_pstats.py'), - RegrTest('test_pty.py', skip="unsupported extension module"), - RegrTest('test_pwd.py', usemodules="pwd", skip=skip_win32), + RegrTest('test_pty.py', usemodules='fcntl termios select'), + RegrTest('test_pwd.py', usemodules="pwd"), RegrTest('test_py3kwarn.py'), RegrTest('test_py_compile.py'), RegrTest('test_pyclbr.py'), @@ -370,15 +366,15 @@ RegrTest('test_re.py', core=True), RegrTest('test_readline.py'), RegrTest('test_repr.py', core=True), - RegrTest('test_resource.py', skip=skip_win32), + RegrTest('test_resource.py'), RegrTest('test_rfc822.py'), RegrTest('test_richcmp.py', core=True), RegrTest('test_rlcompleter.py'), - RegrTest('test_robotparser.py'), + RegrTest('test_runpy.py'), RegrTest('test_sax.py'), RegrTest('test_scope.py', core=True), - RegrTest('test_scriptpackages.py', skip="unsupported extension module"), + RegrTest('test_scriptpackages.py'), RegrTest('test_select.py'), RegrTest('test_set.py', core=True), RegrTest('test_sets.py'), @@ -389,64 +385,59 @@ RegrTest('test_shlex.py'), RegrTest('test_shutil.py'), RegrTest('test_signal.py'), - RegrTest('test_SimpleHTTPServer.py'), + RegrTest('test_SimpleHTTPServer.py', usemodules='binascii'), RegrTest('test_site.py', core=False), RegrTest('test_slice.py', core=True), RegrTest('test_smtplib.py'), RegrTest('test_smtpnet.py'), RegrTest('test_socket.py', usemodules='thread _weakref'), - RegrTest('test_socketserver.py', usemodules='thread'), - RegrTest('test_softspace.py', core=True), RegrTest('test_sort.py', core=True), + RegrTest('test_sqlite.py', usemodules="thread _rawffi zlib"), RegrTest('test_ssl.py', usemodules='_ssl _socket select'), + RegrTest('test_startfile.py'), RegrTest('test_str.py', core=True), - RegrTest('test_strftime.py'), RegrTest('test_string.py', core=True), - RegrTest('test_StringIO.py', core=True, usemodules='cStringIO'), + RegrTest('test_StringIO.py', core=True, usemodules='cStringIO array'), RegrTest('test_stringprep.py'), RegrTest('test_strop.py', skip="deprecated"), - RegrTest('test_strptime.py'), RegrTest('test_strtod.py'), RegrTest('test_struct.py', usemodules='struct'), RegrTest('test_structmembers.py', skip="CPython specific"), RegrTest('test_structseq.py'), RegrTest('test_subprocess.py', usemodules='signal'), - RegrTest('test_sunaudiodev.py', skip=True), + RegrTest('test_sunaudiodev.py'), RegrTest('test_sundry.py'), RegrTest('test_symtable.py', skip="implementation detail"), RegrTest('test_syntax.py', core=True), RegrTest('test_sys.py', core=True, usemodules='struct'), + RegrTest('test_sys_setprofile.py', core=True), RegrTest('test_sys_settrace.py', core=True), - RegrTest('test_sys_setprofile.py', core=True), RegrTest('test_sysconfig.py'), - RegrTest('test_tcl.py', skip="unsupported extension module"), RegrTest('test_tarfile.py'), + RegrTest('test_tcl.py'), RegrTest('test_telnetlib.py'), RegrTest('test_tempfile.py'), - RegrTest('test_textwrap.py'), RegrTest('test_thread.py', usemodules="thread", core=True), RegrTest('test_threaded_import.py', usemodules="thread", core=True), RegrTest('test_threadedtempfile.py', usemodules="thread", core=False), - RegrTest('test_threading.py', usemodules="thread", core=True), RegrTest('test_threading_local.py', usemodules="thread", core=True), RegrTest('test_threadsignals.py', usemodules="thread"), - RegrTest('test_time.py', core=True), RegrTest('test_timeout.py'), RegrTest('test_tk.py'), - RegrTest('test_ttk_guionly.py'), - RegrTest('test_ttk_textonly.py'), RegrTest('test_tokenize.py'), RegrTest('test_trace.py'), RegrTest('test_traceback.py', core=True), RegrTest('test_transformer.py', core=True), + RegrTest('test_ttk_guionly.py'), + RegrTest('test_ttk_textonly.py'), RegrTest('test_tuple.py', core=True), RegrTest('test_typechecks.py'), RegrTest('test_types.py', core=True), @@ -462,6 +453,7 @@ RegrTest('test_unpack.py', core=True), RegrTest('test_urllib.py'), RegrTest('test_urllib2.py'), + RegrTest('test_urllib2_localnet.py', usemodules="thread"), RegrTest('test_urllib2net.py'), RegrTest('test_urllibnet.py'), RegrTest('test_urlparse.py'), @@ -469,61 +461,38 @@ RegrTest('test_userlist.py', core=True), RegrTest('test_userstring.py', core=True), RegrTest('test_uu.py'), - - RegrTest('test_warnings.py', core=True), - RegrTest('test_wave.py', skip="unsupported extension module"), - RegrTest('test_weakref.py', core=True, usemodules='_weakref'), - RegrTest('test_weakset.py'), - - RegrTest('test_whichdb.py'), - RegrTest('test_winreg.py', skip=only_win32), - RegrTest('test_winsound.py', skip="unsupported extension module"), - RegrTest('test_xmllib.py'), - RegrTest('test_xmlrpc.py'), - - RegrTest('test_xpickle.py'), - RegrTest('test_xrange.py', core=True), - RegrTest('test_zipfile.py'), - RegrTest('test_zipimport.py', usemodules='zlib zipimport'), - RegrTest('test_zipimport_support.py', usemodules='zlib zipimport'), - RegrTest('test_zlib.py', usemodules='zlib'), - - RegrTest('test_bigaddrspace.py'), - RegrTest('test_bigmem.py'), - RegrTest('test_cmd_line.py'), - RegrTest('test_code.py'), - RegrTest('test_coding.py'), - RegrTest('test_complex_args.py'), - RegrTest('test_contextlib.py', usemodules="thread"), - RegrTest('test_ctypes.py', usemodules="_rawffi thread"), - RegrTest('test_defaultdict.py', usemodules='_collections'), - RegrTest('test_email_renamed.py'), - RegrTest('test_exception_variations.py'), - RegrTest('test_float.py'), - RegrTest('test_functools.py'), - RegrTest('test_index.py'), - RegrTest('test_old_mailbox.py'), - RegrTest('test_pep352.py'), - RegrTest('test_platform.py'), - RegrTest('test_runpy.py'), - RegrTest('test_sqlite.py', usemodules="thread _rawffi zlib"), - RegrTest('test_startfile.py', skip="bogus test"), - RegrTest('test_structmembers.py', skip="depends on _testcapi"), - RegrTest('test_urllib2_localnet.py', usemodules="thread"), RegrTest('test_uuid.py'), RegrTest('test_wait3.py', usemodules="thread"), RegrTest('test_wait4.py', usemodules="thread"), + RegrTest('test_warnings.py', core=True), + RegrTest('test_wave.py'), + RegrTest('test_weakref.py', core=True, usemodules='_weakref'), + RegrTest('test_weakset.py'), + RegrTest('test_whichdb.py'), + RegrTest('test_winreg.py'), + RegrTest('test_winsound.py'), RegrTest('test_with.py'), RegrTest('test_wsgiref.py'), RegrTest('test_xdrlib.py'), RegrTest('test_xml_etree.py'), RegrTest('test_xml_etree_c.py'), + RegrTest('test_xmllib.py'), + RegrTest('test_xmlrpc.py'), + RegrTest('test_xpickle.py'), + RegrTest('test_xrange.py', core=True), + RegrTest('test_zipfile.py'), RegrTest('test_zipfile64.py'), + RegrTest('test_zipimport.py', usemodules='zlib zipimport'), + RegrTest('test_zipimport_support.py', usemodules='zlib zipimport'), + RegrTest('test_zlib.py', usemodules='zlib'), ] def check_testmap_complete(): listed_names = dict.fromkeys([regrtest.basename for regrtest in testmap]) - listed_names['test_support.py'] = True # ignore this + assert len(listed_names) == len(testmap) + # names to ignore + listed_names['test_support.py'] = True + listed_names['test_multibytecodec_support.py'] = True missing = [] for path in testdir.listdir(fil='test_*.py'): name = path.basename @@ -578,13 +547,13 @@ def getinvocation(self, regrtest): fspath = regrtest.getfspath() python = sys.executable - pypy_script = pypydir.join('bin', 'py.py') + pypy_script = pypydir.join('bin', 'pyinteractive.py') alarm_script = pypydir.join('tool', 'alarm.py') if sys.platform == 'win32': watchdog_name = 'watchdog_nt.py' else: watchdog_name = 'watchdog.py' - watchdog_script = pypydir.join('tool', watchdog_name) + watchdog_script = rpythondir.join('tool', watchdog_name) regr_script = pypydir.join('tool', 'pytest', 'run-script', 'regrverbose.py') diff --git a/lib_pypy/ctypes_config_cache/autopath.py b/lib_pypy/ctypes_config_cache/autopath.py deleted file mode 100644 --- a/lib_pypy/ctypes_config_cache/autopath.py +++ /dev/null @@ -1,131 +0,0 @@ -""" -self cloning, automatic path configuration - -copy this into any subdirectory of pypy from which scripts need -to be run, typically all of the test subdirs. -The idea is that any such script simply issues - - import autopath - -and this will make sure that the parent directory containing "pypy" -is in sys.path. - -If you modify the master "autopath.py" version (in pypy/tool/autopath.py) -you can directly run it which will copy itself on all autopath.py files -it finds under the pypy root directory. - -This module always provides these attributes: - - pypydir pypy root directory path - this_dir directory where this autopath.py resides - -""" - -def __dirinfo(part): - """ return (partdir, this_dir) and insert parent of partdir - into sys.path. If the parent directories don't have the part - an EnvironmentError is raised.""" - - import sys, os - try: - head = this_dir = os.path.realpath(os.path.dirname(__file__)) - except NameError: - head = this_dir = os.path.realpath(os.path.dirname(sys.argv[0])) - - error = None - while head: - partdir = head - head, tail = os.path.split(head) - if tail == part: - checkfile = os.path.join(partdir, os.pardir, 'pypy', '__init__.py') - if not os.path.exists(checkfile): - error = "Cannot find %r" % (os.path.normpath(checkfile),) - break - else: - error = "Cannot find the parent directory %r of the path %r" % ( - partdir, this_dir) - if not error: - # check for bogus end-of-line style (e.g. files checked out on - # Windows and moved to Unix) - f = open(__file__.replace('.pyc', '.py'), 'r') - data = f.read() - f.close() - if data.endswith('\r\n') or data.endswith('\r'): - error = ("Bad end-of-line style in the .py files. Typically " - "caused by a zip file or a checkout done on Windows and " - "moved to Unix or vice-versa.") - if error: - raise EnvironmentError("Invalid source tree - bogus checkout! " + - error) - - pypy_root = os.path.join(head, '') - try: - sys.path.remove(head) - except ValueError: - pass - sys.path.insert(0, head) - - munged = {} - for name, mod in sys.modules.items(): - if '.' in name: - continue - fn = getattr(mod, '__file__', None) - if not isinstance(fn, str): - continue - newname = os.path.splitext(os.path.basename(fn))[0] - if not newname.startswith(part + '.'): - continue - path = os.path.join(os.path.dirname(os.path.realpath(fn)), '') - if path.startswith(pypy_root) and newname != part: - modpaths = os.path.normpath(path[len(pypy_root):]).split(os.sep) - if newname != '__init__': - modpaths.append(newname) - modpath = '.'.join(modpaths) - if modpath not in sys.modules: - munged[modpath] = mod - - for name, mod in munged.iteritems(): - if name not in sys.modules: - sys.modules[name] = mod - if '.' in name: - prename = name[:name.rfind('.')] - postname = name[len(prename)+1:] - if prename not in sys.modules: - __import__(prename) - if not hasattr(sys.modules[prename], postname): - setattr(sys.modules[prename], postname, mod) - - return partdir, this_dir - -def __clone(): - """ clone master version of autopath.py into all subdirs """ - from os.path import join, walk - if not this_dir.endswith(join('pypy','tool')): - raise EnvironmentError("can only clone master version " - "'%s'" % join(pypydir, 'tool',_myname)) - - - def sync_walker(arg, dirname, fnames): - if _myname in fnames: - fn = join(dirname, _myname) - f = open(fn, 'rwb+') - try: - if f.read() == arg: - print "checkok", fn - else: - print "syncing", fn - f = open(fn, 'w') - f.write(arg) - finally: - f.close() - s = open(join(pypydir, 'tool', _myname), 'rb').read() - walk(pypydir, sync_walker, s) - -_myname = 'autopath.py' - -# set guaranteed attributes - -pypydir, this_dir = __dirinfo('pypy') - -if __name__ == '__main__': - __clone() diff --git a/lib_pypy/ctypes_config_cache/dumpcache.py b/lib_pypy/ctypes_config_cache/dumpcache.py --- a/lib_pypy/ctypes_config_cache/dumpcache.py +++ b/lib_pypy/ctypes_config_cache/dumpcache.py @@ -1,6 +1,6 @@ import os from ctypes_configure import dumpcache -from pypy.jit.backend import detect_cpu +from rpython.jit.backend import detect_cpu def dumpcache2(basename, config): model = detect_cpu.autodetect_main_model_and_size() @@ -14,7 +14,7 @@ try: from __pypy__ import cpumodel except ImportError: - from pypy.jit.backend import detect_cpu + from rpython.jit.backend import detect_cpu cpumodel = detect_cpu.autodetect_main_model_and_size() # XXX relative import, should be removed together with # XXX the relative imports done e.g. by lib_pypy/pypy_test/test_hashlib diff --git a/lib_pypy/ctypes_config_cache/rebuild.py b/lib_pypy/ctypes_config_cache/rebuild.py --- a/lib_pypy/ctypes_config_cache/rebuild.py +++ b/lib_pypy/ctypes_config_cache/rebuild.py @@ -1,21 +1,15 @@ #! /usr/bin/env python # Run this script to rebuild all caches from the *.ctc.py files. -# hack: we cannot directly import autopath, as we are outside the pypy -# package. However, we pretend to be inside pypy/tool and manually run it, to -# get the correct path -import os.path -this_dir = os.path.dirname(__file__) -autopath_py = os.path.join(this_dir, '../../pypy/tool/autopath.py') -autopath_py = os.path.abspath(autopath_py) -execfile(autopath_py, dict(__name__='autopath', __file__=autopath_py)) +import os, sys -import os, sys +sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..'))) + import py _dirpath = os.path.dirname(__file__) or os.curdir -from pypy.tool.ansi_print import ansi_log +from rpython.tool.ansi_print import ansi_log log = py.log.Producer("ctypes_config_cache") py.log.setconsumer("ctypes_config_cache", ansi_log) @@ -31,7 +25,7 @@ sys.path[:] = path def try_rebuild(): - from pypy.jit.backend import detect_cpu + from rpython.jit.backend import detect_cpu model = detect_cpu.autodetect_main_model_and_size() # remove the files '_*_model_.py' left = {} diff --git a/lib_pypy/ctypes_config_cache/test/test_cache.py b/lib_pypy/ctypes_config_cache/test/test_cache.py --- a/lib_pypy/ctypes_config_cache/test/test_cache.py +++ b/lib_pypy/ctypes_config_cache/test/test_cache.py @@ -1,6 +1,6 @@ import py import sys, os -from pypy.tool.udir import udir +from rpython.tool.udir import udir dirpath = py.path.local(__file__).dirpath().dirpath() diff --git a/lib_pypy/pypy_test/test_dbm_extra.py b/lib_pypy/pypy_test/test_dbm_extra.py --- a/lib_pypy/pypy_test/test_dbm_extra.py +++ b/lib_pypy/pypy_test/test_dbm_extra.py @@ -1,6 +1,6 @@ from __future__ import absolute_import import py -from pypy.tool.udir import udir +from rpython.tool.udir import udir try: from lib_pypy import dbm except ImportError, e: diff --git a/lib_pypy/pypy_test/test_marshal_extra.py b/lib_pypy/pypy_test/test_marshal_extra.py --- a/lib_pypy/pypy_test/test_marshal_extra.py +++ b/lib_pypy/pypy_test/test_marshal_extra.py @@ -4,7 +4,7 @@ import marshal as cpy_marshal from lib_pypy import _marshal as marshal -from pypy.tool.udir import udir +from rpython.tool.udir import udir hello = "he" hello += "llo" diff --git a/py/_code/source.py b/py/_code/source.py --- a/py/_code/source.py +++ b/py/_code/source.py @@ -141,6 +141,8 @@ trysource = self[start:end] if trysource.isparseable(): return start, end + if end == start + 100: # XXX otherwise, it takes forever + break # XXX if end is None: raise IndexError("no valid source range around line %d " % (lineno,)) return start, end diff --git a/pypy/annotation/annrpython.py b/pypy/annotation/annrpython.py deleted file mode 100644 --- a/pypy/annotation/annrpython.py +++ /dev/null @@ -1,676 +0,0 @@ -from __future__ import absolute_import - -import types -from pypy.tool.ansi_print import ansi_log -from pypy.tool.pairtype import pair -from pypy.tool.error import (format_blocked_annotation_error, - AnnotatorError, gather_error, ErrorWrapper) -from pypy.objspace.flow.model import (Variable, Constant, FunctionGraph, - c_last_exception, checkgraph) -from pypy.translator import simplify, transform -from pypy.annotation import model as annmodel, signature, unaryop, binaryop -from pypy.annotation.bookkeeper import Bookkeeper -import py -log = py.log.Producer("annrpython") -py.log.setconsumer("annrpython", ansi_log) - - -FAIL = object() - -class RPythonAnnotator(object): - """Block annotator for RPython. - See description in doc/translation.txt.""" - - def __init__(self, translator=None, policy=None, bookkeeper=None): - import pypy.rpython.ootypesystem.ooregistry # has side effects - import pypy.rpython.extfuncregistry # has side effects - import pypy.rlib.nonconst # has side effects - - if translator is None: - # interface for tests - from pypy.translator.translator import TranslationContext - translator = TranslationContext() - translator.annotator = self - self.translator = translator - self.pendingblocks = {} # map {block: graph-containing-it} - self.bindings = {} # map Variables to SomeValues - self.annotated = {} # set of blocks already seen - self.added_blocks = None # see processblock() below - self.links_followed = {} # set of links that have ever been followed - self.notify = {} # {block: {positions-to-reflow-from-when-done}} - self.fixed_graphs = {} # set of graphs not to annotate again - self.blocked_blocks = {} # set of {blocked_block: (graph, index)} - # --- the following information is recorded for debugging --- - self.blocked_graphs = {} # set of graphs that have blocked blocks - # --- end of debugging information --- - self.frozen = False - if policy is None: - from pypy.annotation.policy import AnnotatorPolicy - self.policy = AnnotatorPolicy() - else: - self.policy = policy - if bookkeeper is None: - bookkeeper = Bookkeeper(self) - self.bookkeeper = bookkeeper - - def __getstate__(self): - attrs = """translator pendingblocks bindings annotated links_followed - notify bookkeeper frozen policy added_blocks""".split() - ret = self.__dict__.copy() - for key, value in ret.items(): - if key not in attrs: - assert type(value) is dict, ( - "%r is not dict. please update %s.__getstate__" % - (key, self.__class__.__name__)) - ret[key] = {} - return ret - - #___ convenience high-level interface __________________ - - def build_types(self, function, input_arg_types, complete_now=True, - main_entry_point=False): - """Recursively build annotations about the specific entry point.""" - assert isinstance(function, types.FunctionType), "fix that!" - - from pypy.annotation.policy import AnnotatorPolicy - policy = AnnotatorPolicy() - # make input arguments and set their type - args_s = [self.typeannotation(t) for t in input_arg_types] - - # XXX hack - annmodel.TLS.check_str_without_nul = ( - self.translator.config.translation.check_str_without_nul) - - flowgraph, inputcells = self.get_call_parameters(function, args_s, policy) - if not isinstance(flowgraph, FunctionGraph): - assert isinstance(flowgraph, annmodel.SomeObject) - return flowgraph - - if main_entry_point: - self.translator.entry_point_graph = flowgraph - return self.build_graph_types(flowgraph, inputcells, complete_now=complete_now) - - def get_call_parameters(self, function, args_s, policy): - desc = self.bookkeeper.getdesc(function) - args = self.bookkeeper.build_args("simple_call", args_s[:]) - result = [] - def schedule(graph, inputcells): - result.append((graph, inputcells)) - return annmodel.s_ImpossibleValue - - prevpolicy = self.policy - self.policy = policy - self.bookkeeper.enter(None) - try: - desc.pycall(schedule, args, annmodel.s_ImpossibleValue) - finally: - self.bookkeeper.leave() - self.policy = prevpolicy - [(graph, inputcells)] = result - return graph, inputcells - - def annotate_helper(self, function, args_s, policy=None): - if policy is None: - from pypy.annotation.policy import AnnotatorPolicy - policy = AnnotatorPolicy() - graph, inputcells = self.get_call_parameters(function, args_s, policy) - self.build_graph_types(graph, inputcells, complete_now=False) - self.complete_helpers(policy) - return graph - - def complete_helpers(self, policy): - saved = self.policy, self.added_blocks - self.policy = policy - try: - self.added_blocks = {} - self.complete() - # invoke annotation simplifications for the new blocks - self.simplify(block_subset=self.added_blocks) - finally: - self.policy, self.added_blocks = saved - - def build_graph_types(self, flowgraph, inputcells, complete_now=True): - checkgraph(flowgraph) - - nbarg = len(flowgraph.getargs()) - if len(inputcells) != nbarg: - raise TypeError("%s expects %d args, got %d" %( - flowgraph, nbarg, len(inputcells))) - - # register the entry point - self.addpendinggraph(flowgraph, inputcells) - # recursively proceed until no more pending block is left - if complete_now: - self.complete() - return self.binding(flowgraph.getreturnvar(), None) - - def gettype(self, variable): - """Return the known type of a control flow graph variable, - defaulting to 'object'.""" - if isinstance(variable, Constant): - return type(variable.value) - elif isinstance(variable, Variable): - cell = self.bindings.get(variable) - if cell: - return cell.knowntype - else: - return object - else: - raise TypeError, ("Variable or Constant instance expected, " - "got %r" % (variable,)) - - def getuserclassdefinitions(self): - """Return a list of ClassDefs.""" - return self.bookkeeper.classdefs - - #___ medium-level interface ____________________________ - - def addpendinggraph(self, flowgraph, inputcells): - self.addpendingblock(flowgraph, flowgraph.startblock, inputcells) - - def addpendingblock(self, graph, block, cells): - """Register an entry point into block with the given input cells.""" - if graph in self.fixed_graphs: - # special case for annotating/rtyping in several phases: calling - # a graph that has already been rtyped. Safety-check the new - # annotations that are passed in, and don't annotate the old - # graph -- it's already low-level operations! - for a, s_newarg in zip(graph.getargs(), cells): - s_oldarg = self.binding(a) - assert annmodel.unionof(s_oldarg, s_newarg) == s_oldarg - else: - assert not self.frozen - for a in cells: - assert isinstance(a, annmodel.SomeObject) - if block not in self.annotated: - self.bindinputargs(graph, block, cells) - else: - self.mergeinputargs(graph, block, cells) - if not self.annotated[block]: - self.pendingblocks[block] = graph - - def complete(self): - """Process pending blocks until none is left.""" - while True: - while self.pendingblocks: - block, graph = self.pendingblocks.popitem() - self.processblock(graph, block) - self.policy.no_more_blocks_to_annotate(self) - if not self.pendingblocks: - break # finished - # make sure that the return variables of all graphs is annotated - if self.added_blocks is not None: - newgraphs = [self.annotated[block] for block in self.added_blocks] - newgraphs = dict.fromkeys(newgraphs) - got_blocked_blocks = False in newgraphs - else: - newgraphs = self.translator.graphs #all of them - got_blocked_blocks = False in self.annotated.values() - if got_blocked_blocks: - for graph in self.blocked_graphs.values(): - self.blocked_graphs[graph] = True - - blocked_blocks = [block for block, done in self.annotated.items() - if done is False] - assert len(blocked_blocks) == len(self.blocked_blocks) - - text = format_blocked_annotation_error(self, self.blocked_blocks) - #raise SystemExit() - raise AnnotatorError(text) - for graph in newgraphs: - v = graph.getreturnvar() - if v not in self.bindings: - self.setbinding(v, annmodel.s_ImpossibleValue) - # policy-dependent computation - self.bookkeeper.compute_at_fixpoint() - - def binding(self, arg, default=FAIL): - "Gives the SomeValue corresponding to the given Variable or Constant." - if isinstance(arg, Variable): - try: - return self.bindings[arg] - except KeyError: - if default is not FAIL: - return default - else: - raise - elif isinstance(arg, Constant): - #if arg.value is undefined_value: # undefined local variables - # return annmodel.s_ImpossibleValue - return self.bookkeeper.immutableconstant(arg) - else: - raise TypeError, 'Variable or Constant expected, got %r' % (arg,) - - def typeannotation(self, t): - return signature.annotation(t, self.bookkeeper) - - def setbinding(self, arg, s_value): - if arg in self.bindings: - assert s_value.contains(self.bindings[arg]) - self.bindings[arg] = s_value - - def transfer_binding(self, v_target, v_source): - assert v_source in self.bindings - self.bindings[v_target] = self.bindings[v_source] - - def warning(self, msg, pos=None): - if pos is None: - try: - pos = self.bookkeeper.position_key - except AttributeError: - pos = '?' - if pos != '?': - pos = self.whereami(pos) - - log.WARNING("%s/ %s" % (pos, msg)) - - - #___ interface for annotator.bookkeeper _______ - - def recursivecall(self, graph, whence, inputcells): - if isinstance(whence, tuple): - parent_graph, parent_block, parent_index = whence - tag = parent_block, parent_index - self.translator.update_call_graph(parent_graph, graph, tag) - # self.notify[graph.returnblock] is a dictionary of call - # points to this func which triggers a reflow whenever the - # return block of this graph has been analysed. - callpositions = self.notify.setdefault(graph.returnblock, {}) - if whence is not None: - if callable(whence): - def callback(): - whence(self, graph) - else: - callback = whence - callpositions[callback] = True - - # generalize the function's input arguments - self.addpendingblock(graph, graph.startblock, inputcells) - - # get the (current) return value - v = graph.getreturnvar() - try: - return self.bindings[v] - except KeyError: - # the function didn't reach any return statement so far. - # (some functions actually never do, they always raise exceptions) - return annmodel.s_ImpossibleValue - - def reflowfromposition(self, position_key): - graph, block, index = position_key - self.reflowpendingblock(graph, block) - - - #___ simplification (should be moved elsewhere?) _______ - - def simplify(self, block_subset=None, extra_passes=None): - # Generic simplifications - transform.transform_graph(self, block_subset=block_subset, - extra_passes=extra_passes) - if block_subset is None: - graphs = self.translator.graphs - else: - graphs = {} - for block in block_subset: - graph = self.annotated.get(block) - if graph: - graphs[graph] = True - for graph in graphs: - simplify.eliminate_empty_blocks(graph) - - - #___ flowing annotations in blocks _____________________ - - def processblock(self, graph, block): - # Important: this is not called recursively. - # self.flowin() can only issue calls to self.addpendingblock(). - # The analysis of a block can be in three states: - # * block not in self.annotated: - # never seen the block. - # * self.annotated[block] == False: - # the input variables of the block are in self.bindings but we - # still have to consider all the operations in the block. - # * self.annotated[block] == graph-containing-block: - # analysis done (at least until we find we must generalize the - # input variables). - - #print '* processblock', block, cells - self.annotated[block] = graph - if block in self.blocked_blocks: - del self.blocked_blocks[block] - try: - self.flowin(graph, block) - except BlockedInference, e: - self.annotated[block] = False # failed, hopefully temporarily - self.blocked_blocks[block] = (graph, e.opindex) - except Exception, e: - # hack for debug tools only - if not hasattr(e, '__annotator_block'): - setattr(e, '__annotator_block', block) - raise - - # The dict 'added_blocks' is used by rpython.annlowlevel to - # detect which are the new blocks that annotating an additional - # small helper creates. - if self.added_blocks is not None: - self.added_blocks[block] = True - - def reflowpendingblock(self, graph, block): - assert not self.frozen - assert graph not in self.fixed_graphs - self.pendingblocks[block] = graph - assert block in self.annotated - self.annotated[block] = False # must re-flow - self.blocked_blocks[block] = (graph, None) - - def bindinputargs(self, graph, block, inputcells): - # Create the initial bindings for the input args of a block. - assert len(block.inputargs) == len(inputcells) - for a, cell in zip(block.inputargs, inputcells): - self.setbinding(a, cell) - self.annotated[block] = False # must flowin. - self.blocked_blocks[block] = (graph, None) - - def mergeinputargs(self, graph, block, inputcells): - # Merge the new 'cells' with each of the block's existing input - # variables. - oldcells = [self.binding(a) for a in block.inputargs] - try: - unions = [annmodel.unionof(c1,c2) for c1, c2 in zip(oldcells,inputcells)] - except annmodel.UnionError, e: - e.args = e.args + ( - ErrorWrapper(gather_error(self, graph, block, None)),) - raise - # if the merged cells changed, we must redo the analysis - if unions != oldcells: - self.bindinputargs(graph, block, unions) - - def whereami(self, position_key): - graph, block, i = position_key - blk = "" - if block: - at = block.at() - if at: - blk = " block"+at - opid="" - if i is not None: - opid = " op=%d" % i - return repr(graph) + blk + opid - - def flowin(self, graph, block): - #print 'Flowing', block, [self.binding(a) for a in block.inputargs] - try: - for i in range(len(block.operations)): - try: - self.bookkeeper.enter((graph, block, i)) - self.consider_op(block, i) - finally: - self.bookkeeper.leave() - - except BlockedInference, e: - if (e.op is block.operations[-1] and - block.exitswitch == c_last_exception): - # this is the case where the last operation of the block will - # always raise an exception which is immediately caught by - # an exception handler. We then only follow the exceptional - # branches. - exits = [link for link in block.exits - if link.exitcase is not None] - - elif e.op.opname in ('simple_call', 'call_args', 'next'): - # XXX warning, keep the name of the call operations in sync - # with the flow object space. These are the operations for - # which it is fine to always raise an exception. We then - # swallow the BlockedInference and that's it. - # About 'next': see test_annotate_iter_empty_container(). - return - - else: - # other cases are problematic (but will hopefully be solved - # later by reflowing). Throw the BlockedInference up to - # processblock(). - raise - - except annmodel.HarmlesslyBlocked: - return - - else: - # dead code removal: don't follow all exits if the exitswitch - # is known - exits = block.exits - if isinstance(block.exitswitch, Variable): - s_exitswitch = self.bindings[block.exitswitch] - if s_exitswitch.is_constant(): - exits = [link for link in exits - if link.exitcase == s_exitswitch.const] - - # mapping (exitcase, variable) -> s_annotation - # that can be attached to booleans, exitswitches - knowntypedata = getattr(self.bindings.get(block.exitswitch), - "knowntypedata", {}) - - # filter out those exceptions which cannot - # occour for this specific, typed operation. - if block.exitswitch == c_last_exception: - op = block.operations[-1] - if op.opname in binaryop.BINARY_OPERATIONS: - arg1 = self.binding(op.args[0]) - arg2 = self.binding(op.args[1]) - binop = getattr(pair(arg1, arg2), op.opname, None) - can_only_throw = annmodel.read_can_only_throw(binop, arg1, arg2) - elif op.opname in unaryop.UNARY_OPERATIONS: - arg1 = self.binding(op.args[0]) - opname = op.opname - if opname == 'contains': opname = 'op_contains' - unop = getattr(arg1, opname, None) - can_only_throw = annmodel.read_can_only_throw(unop, arg1) - else: - can_only_throw = None - - if can_only_throw is not None: - candidates = can_only_throw - candidate_exits = exits - exits = [] - for link in candidate_exits: - case = link.exitcase - if case is None: - exits.append(link) - continue - covered = [c for c in candidates if issubclass(c, case)] - if covered: - exits.append(link) - candidates = [c for c in candidates if c not in covered] - - for link in exits: - in_except_block = False - - last_exception_var = link.last_exception # may be None for non-exception link - last_exc_value_var = link.last_exc_value # may be None for non-exception link - - if isinstance(link.exitcase, (types.ClassType, type)) \ - and issubclass(link.exitcase, py.builtin.BaseException): - assert last_exception_var and last_exc_value_var - last_exc_value_object = self.bookkeeper.valueoftype(link.exitcase) - last_exception_object = annmodel.SomeType() - if isinstance(last_exception_var, Constant): - last_exception_object.const = last_exception_var.value - last_exception_object.is_type_of = [last_exc_value_var] - - if isinstance(last_exception_var, Variable): - self.setbinding(last_exception_var, last_exception_object) - if isinstance(last_exc_value_var, Variable): - self.setbinding(last_exc_value_var, last_exc_value_object) - - last_exception_object = annmodel.SomeType() - if isinstance(last_exception_var, Constant): - last_exception_object.const = last_exception_var.value - #if link.exitcase is Exception: - # last_exc_value_object = annmodel.SomeObject() - #else: - last_exc_value_vars = [] - in_except_block = True - - ignore_link = False - cells = [] - renaming = {} - for a,v in zip(link.args,link.target.inputargs): - renaming.setdefault(a, []).append(v) - for a,v in zip(link.args,link.target.inputargs): - if a == last_exception_var: - assert in_except_block - cells.append(last_exception_object) - elif a == last_exc_value_var: - assert in_except_block - cells.append(last_exc_value_object) - last_exc_value_vars.append(v) - else: - cell = self.binding(a) - if (link.exitcase, a) in knowntypedata: - knownvarvalue = knowntypedata[(link.exitcase, a)] - cell = pair(cell, knownvarvalue).improve() - # ignore links that try to pass impossible values - if cell == annmodel.s_ImpossibleValue: - ignore_link = True - - if hasattr(cell,'is_type_of'): - renamed_is_type_of = [] - for v in cell.is_type_of: - new_vs = renaming.get(v,[]) - renamed_is_type_of += new_vs - assert cell.knowntype is type - newcell = annmodel.SomeType() - if cell.is_constant(): - newcell.const = cell.const - cell = newcell - cell.is_type_of = renamed_is_type_of - - if hasattr(cell, 'knowntypedata'): - renamed_knowntypedata = {} - for (value, v), s in cell.knowntypedata.items(): - new_vs = renaming.get(v, []) - for new_v in new_vs: - renamed_knowntypedata[value, new_v] = s - assert isinstance(cell, annmodel.SomeBool) - newcell = annmodel.SomeBool() - if cell.is_constant(): - newcell.const = cell.const - cell = newcell - cell.set_knowntypedata(renamed_knowntypedata) - - cells.append(cell) - - if ignore_link: - continue - - if in_except_block: - last_exception_object.is_type_of = last_exc_value_vars - - self.links_followed[link] = True - self.addpendingblock(graph, link.target, cells) - - if block in self.notify: - # reflow from certain positions when this block is done - for callback in self.notify[block]: - if isinstance(callback, tuple): - self.reflowfromposition(callback) # callback is a position - else: - callback() - - - #___ creating the annotations based on operations ______ - - def consider_op(self, block, opindex): - op = block.operations[opindex] - argcells = [self.binding(a) for a in op.args] - consider_meth = getattr(self,'consider_op_'+op.opname, - None) - if not consider_meth: - raise Exception,"unknown op: %r" % op - - # let's be careful about avoiding propagated SomeImpossibleValues - # to enter an op; the latter can result in violations of the - # more general results invariant: e.g. if SomeImpossibleValue enters is_ - # is_(SomeImpossibleValue, None) -> SomeBool - # is_(SomeInstance(not None), None) -> SomeBool(const=False) ... - # boom -- in the assert of setbinding() - for arg in argcells: - if isinstance(arg, annmodel.SomeImpossibleValue): - raise BlockedInference(self, op, opindex) - try: - resultcell = consider_meth(*argcells) - except Exception, e: - graph = self.bookkeeper.position_key[0] - e.args = e.args + ( - ErrorWrapper(gather_error(self, graph, block, opindex)),) - raise - if resultcell is None: - resultcell = self.noreturnvalue(op) - elif resultcell == annmodel.s_ImpossibleValue: - raise BlockedInference(self, op, opindex) # the operation cannot succeed - assert isinstance(resultcell, annmodel.SomeObject) - assert isinstance(op.result, Variable) - self.setbinding(op.result, resultcell) # bind resultcell to op.result - - def noreturnvalue(self, op): - return annmodel.s_ImpossibleValue # no return value (hook method) - - # XXX "contains" clash with SomeObject method - def consider_op_contains(self, seq, elem): - self.bookkeeper.count("contains", seq) - return seq.op_contains(elem) - - def consider_op_newtuple(self, *args): - return annmodel.SomeTuple(items = args) - - def consider_op_newlist(self, *args): - return self.bookkeeper.newlist(*args) - - def consider_op_newdict(self): - return self.bookkeeper.newdict() - - - def _registeroperations(cls, unary_ops, binary_ops): - # All unary operations - d = {} - for opname in unary_ops: - fnname = 'consider_op_' + opname - exec py.code.Source(""" -def consider_op_%s(self, arg, *args): - return arg.%s(*args) -""" % (opname, opname)).compile() in globals(), d - setattr(cls, fnname, d[fnname]) - # All binary operations - for opname in binary_ops: - fnname = 'consider_op_' + opname - exec py.code.Source(""" -def consider_op_%s(self, arg1, arg2, *args): - return pair(arg1,arg2).%s(*args) -""" % (opname, opname)).compile() in globals(), d - setattr(cls, fnname, d[fnname]) - _registeroperations = classmethod(_registeroperations) - -# register simple operations handling -RPythonAnnotator._registeroperations(unaryop.UNARY_OPERATIONS, binaryop.BINARY_OPERATIONS) - - -class BlockedInference(Exception): - """This exception signals the type inference engine that the situation - is currently blocked, and that it should try to progress elsewhere.""" - - def __init__(self, annotator, op, opindex): - self.annotator = annotator - try: - self.break_at = annotator.bookkeeper.position_key - except AttributeError: - self.break_at = None - self.op = op - self.opindex = opindex - - def __repr__(self): - if not self.break_at: - break_at = "?" - else: - break_at = self.annotator.whereami(self.break_at) - return "" %(break_at, self.op) - - __str__ = __repr__ diff --git a/pypy/annotation/binaryop.py b/pypy/annotation/binaryop.py deleted file mode 100644 --- a/pypy/annotation/binaryop.py +++ /dev/null @@ -1,1052 +0,0 @@ -""" -Binary operations between SomeValues. -""" - -import py -import operator -from pypy.tool.pairtype import pair, pairtype -from pypy.annotation.model import SomeObject, SomeInteger, SomeBool, s_Bool -from pypy.annotation.model import SomeString, SomeChar, SomeList, SomeDict -from pypy.annotation.model import SomeUnicodeCodePoint, SomeUnicodeString -from pypy.annotation.model import SomeTuple, SomeImpossibleValue, s_ImpossibleValue -from pypy.annotation.model import SomeInstance, SomeBuiltin, SomeIterator -from pypy.annotation.model import SomePBC, SomeFloat, s_None, SomeByteArray -from pypy.annotation.model import SomeExternalObject, SomeWeakRef -from pypy.annotation.model import SomeAddress, SomeTypedAddressAccess -from pypy.annotation.model import SomeSingleFloat, SomeLongFloat, SomeType -from pypy.annotation.model import unionof, UnionError, missing_operation -from pypy.annotation.model import TLS -from pypy.annotation.model import read_can_only_throw -from pypy.annotation.model import add_knowntypedata, merge_knowntypedata -from pypy.annotation.model import SomeGenericCallable -from pypy.annotation.bookkeeper import getbookkeeper -from pypy.objspace.flow.model import Variable, Constant -from pypy.rlib import rarithmetic -from pypy.tool.error import AnnotatorError - -# convenience only! -def immutablevalue(x): - return getbookkeeper().immutablevalue(x) - -# XXX unify this with ObjSpace.MethodTable -BINARY_OPERATIONS = set(['add', 'sub', 'mul', 'div', 'mod', - 'truediv', 'floordiv', 'divmod', - 'and_', 'or_', 'xor', - 'lshift', 'rshift', - 'getitem', 'setitem', 'delitem', - 'getitem_idx', 'getitem_key', 'getitem_idx_key', - 'inplace_add', 'inplace_sub', 'inplace_mul', - 'inplace_truediv', 'inplace_floordiv', 'inplace_div', - 'inplace_mod', - 'inplace_lshift', 'inplace_rshift', - 'inplace_and', 'inplace_or', 'inplace_xor', - 'lt', 'le', 'eq', 'ne', 'gt', 'ge', 'is_', 'cmp', - 'coerce', - ] - +[opname+'_ovf' for opname in - """add sub mul floordiv div mod lshift - """.split() - ]) - -for opname in BINARY_OPERATIONS: - missing_operation(pairtype(SomeObject, SomeObject), opname) - -class __extend__(pairtype(SomeObject, SomeObject)): - - def union((obj1, obj2)): - raise UnionError(obj1, obj2) - - # inplace_xxx ---> xxx by default - def inplace_add((obj1, obj2)): return pair(obj1, obj2).add() - def inplace_sub((obj1, obj2)): return pair(obj1, obj2).sub() - def inplace_mul((obj1, obj2)): return pair(obj1, obj2).mul() - def inplace_truediv((obj1, obj2)): return pair(obj1, obj2).truediv() - def inplace_floordiv((obj1, obj2)): return pair(obj1, obj2).floordiv() - def inplace_div((obj1, obj2)): return pair(obj1, obj2).div() - def inplace_mod((obj1, obj2)): return pair(obj1, obj2).mod() - def inplace_lshift((obj1, obj2)): return pair(obj1, obj2).lshift() - def inplace_rshift((obj1, obj2)): return pair(obj1, obj2).rshift() - def inplace_and((obj1, obj2)): return pair(obj1, obj2).and_() - def inplace_or((obj1, obj2)): return pair(obj1, obj2).or_() - def inplace_xor((obj1, obj2)): return pair(obj1, obj2).xor() - - for name, func in locals().items(): - if name.startswith('inplace_'): - func.can_only_throw = [] - - inplace_div.can_only_throw = [ZeroDivisionError] - inplace_truediv.can_only_throw = [ZeroDivisionError] - inplace_floordiv.can_only_throw = [ZeroDivisionError] - inplace_mod.can_only_throw = [ZeroDivisionError] - - def lt((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const < obj2.const) - else: - getbookkeeper().count("non_int_comp", obj1, obj2) - return s_Bool - - def le((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const <= obj2.const) - else: - getbookkeeper().count("non_int_comp", obj1, obj2) - return s_Bool - - def eq((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const == obj2.const) - else: - getbookkeeper().count("non_int_eq", obj1, obj2) - return s_Bool - - def ne((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const != obj2.const) - else: - getbookkeeper().count("non_int_eq", obj1, obj2) - return s_Bool - - def gt((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const > obj2.const) - else: - getbookkeeper().count("non_int_comp", obj1, obj2) - return s_Bool - - def ge((obj1, obj2)): - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(obj1.const >= obj2.const) - else: - getbookkeeper().count("non_int_comp", obj1, obj2) - return s_Bool - - def cmp((obj1, obj2)): - getbookkeeper().count("cmp", obj1, obj2) - if obj1.is_immutable_constant() and obj2.is_immutable_constant(): - return immutablevalue(cmp(obj1.const, obj2.const)) - else: - return SomeInteger() - - def is_((obj1, obj2)): - r = SomeBool() - if obj2.is_constant(): - if obj1.is_constant(): - r.const = obj1.const is obj2.const - if obj2.const is None and not obj1.can_be_none(): - r.const = False - elif obj1.is_constant(): - if obj1.const is None and not obj2.can_be_none(): - r.const = False - # XXX HACK HACK HACK - # XXX HACK HACK HACK - # XXX HACK HACK HACK - bk = getbookkeeper() - if bk is not None: # for testing - knowntypedata = {} - fn, block, i = bk.position_key - - annotator = bk.annotator - op = block.operations[i] - assert op.opname == "is_" - assert len(op.args) == 2 - - def bind(src_obj, tgt_obj, tgt_arg): - if hasattr(tgt_obj, 'is_type_of') and src_obj.is_constant(): - add_knowntypedata(knowntypedata, True, tgt_obj.is_type_of, - bk.valueoftype(src_obj.const)) - - assert annotator.binding(op.args[tgt_arg]) == tgt_obj - add_knowntypedata(knowntypedata, True, [op.args[tgt_arg]], src_obj) - - nonnone_obj = tgt_obj - if src_obj.is_constant() and src_obj.const is None and tgt_obj.can_be_none(): - nonnone_obj = tgt_obj.nonnoneify() - - add_knowntypedata(knowntypedata, False, [op.args[tgt_arg]], nonnone_obj) - - bind(obj2, obj1, 0) - bind(obj1, obj2, 1) - r.set_knowntypedata(knowntypedata) - - return r - - def divmod((obj1, obj2)): - getbookkeeper().count("divmod", obj1, obj2) - return SomeTuple([pair(obj1, obj2).div(), pair(obj1, obj2).mod()]) - - def coerce((obj1, obj2)): - getbookkeeper().count("coerce", obj1, obj2) - return pair(obj1, obj2).union() # reasonable enough - - # approximation of an annotation intersection, the result should be the annotation obj or - # the intersection of obj and improvement - def improve((obj, improvement)): - if not improvement.contains(obj) and obj.contains(improvement): - return improvement - else: - return obj - - # checked getitems - - def _getitem_can_only_throw(s_c1, s_o2): - impl = pair(s_c1, s_o2).getitem - return read_can_only_throw(impl, s_c1, s_o2) - - def getitem_idx_key((s_c1, s_o2)): - impl = pair(s_c1, s_o2).getitem - return impl() - getitem_idx_key.can_only_throw = _getitem_can_only_throw - - getitem_idx = getitem_idx_key - getitem_key = getitem_idx_key - - -class __extend__(pairtype(SomeType, SomeType)): - - def union((obj1, obj2)): - result = SomeType() - is_type_of1 = getattr(obj1, 'is_type_of', None) - is_type_of2 = getattr(obj2, 'is_type_of', None) - if obj1.is_immutable_constant() and obj2.is_immutable_constant() and obj1.const == obj2.const: - result.const = obj1.const - is_type_of = {} - if is_type_of1: - for v in is_type_of1: - is_type_of[v] = True - if is_type_of2: - for v in is_type_of2: - is_type_of[v] = True - if is_type_of: - result.is_type_of = is_type_of.keys() - else: - if is_type_of1 and is_type_of1 == is_type_of2: - result.is_type_of = is_type_of1 - return result - - -# cloning a function with identical code, for the can_only_throw attribute -def _clone(f, can_only_throw = None): - newfunc = type(f)(f.func_code, f.func_globals, f.func_name, - f.func_defaults, f.func_closure) - if can_only_throw is not None: - newfunc.can_only_throw = can_only_throw - return newfunc - -class __extend__(pairtype(SomeInteger, SomeInteger)): - # unsignedness is considered a rare and contagious disease - - def union((int1, int2)): - if int1.unsigned == int2.unsigned: - knowntype = rarithmetic.compute_restype(int1.knowntype, int2.knowntype) - else: - t1 = int1.knowntype - if t1 is bool: - t1 = int - t2 = int2.knowntype - if t2 is bool: - t2 = int - - if t2 is int: - if int2.nonneg == False: - raise UnionError, "Merging %s and a possibly negative int is not allowed" % t1 - knowntype = t1 - elif t1 is int: - if int1.nonneg == False: - raise UnionError, "Merging %s and a possibly negative int is not allowed" % t2 - knowntype = t2 - else: - raise UnionError, "Merging these types (%s, %s) is not supported" % (t1, t2) - return SomeInteger(nonneg=int1.nonneg and int2.nonneg, - knowntype=knowntype) - - or_ = xor = add = mul = _clone(union, []) - add_ovf = mul_ovf = _clone(union, [OverflowError]) - div = floordiv = mod = _clone(union, [ZeroDivisionError]) - div_ovf= floordiv_ovf = mod_ovf = _clone(union, [ZeroDivisionError, OverflowError]) - - def truediv((int1, int2)): - return SomeFloat() - truediv.can_only_throw = [ZeroDivisionError] - truediv_ovf = _clone(truediv, [ZeroDivisionError, OverflowError]) - - inplace_div = div - inplace_truediv = truediv - - def sub((int1, int2)): - knowntype = rarithmetic.compute_restype(int1.knowntype, int2.knowntype) - return SomeInteger(knowntype=knowntype) - sub.can_only_throw = [] - sub_ovf = _clone(sub, [OverflowError]) - - def and_((int1, int2)): - knowntype = rarithmetic.compute_restype(int1.knowntype, int2.knowntype) - return SomeInteger(nonneg=int1.nonneg or int2.nonneg, - knowntype=knowntype) - and_.can_only_throw = [] - - def lshift((int1, int2)): - if isinstance(int1, SomeBool): - return SomeInteger() - else: - return SomeInteger(knowntype=int1.knowntype) - lshift.can_only_throw = [] - lshift_ovf = _clone(lshift, [OverflowError]) - - def rshift((int1, int2)): - if isinstance(int1, SomeBool): - return SomeInteger(nonneg=True) - else: - return SomeInteger(nonneg=int1.nonneg, knowntype=int1.knowntype) - rshift.can_only_throw = [] - - def _compare_helper((int1, int2), opname, operation): - r = SomeBool() - if int1.is_immutable_constant() and int2.is_immutable_constant(): - r.const = operation(int1.const, int2.const) - # - # The rest of the code propagates nonneg information between - # the two arguments. - # - # Doing the right thing when int1 or int2 change from signed - # to unsigned (r_uint) is almost impossible. See test_intcmp_bug. - # Instead, we only deduce constrains on the operands in the - # case where they are both signed. In other words, if y is - # nonneg then "assert x>=y" will let the annotator know that - # x is nonneg too, but it will not work if y is unsigned. - # - if not (rarithmetic.signedtype(int1.knowntype) and - rarithmetic.signedtype(int2.knowntype)): - return r - knowntypedata = {} - # XXX HACK HACK HACK - fn, block, i = getbookkeeper().position_key - op = block.operations[i] - assert op.opname == opname - assert len(op.args) == 2 - def tointtype(int0): - if int0.knowntype is bool: - return int - return int0.knowntype From noreply at buildbot.pypy.org Thu Jan 31 12:47:33 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 31 Jan 2013 12:47:33 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: hg merge default (again) Message-ID: <20130131114733.EBF8A1C009B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r60766:519029980f3c Date: 2013-01-31 12:47 +0100 http://bitbucket.org/pypy/pypy/changeset/519029980f3c/ Log: hg merge default (again) diff --git a/rpython/rtyper/memory/gc/minimark.py b/rpython/rtyper/memory/gc/minimark.py --- a/rpython/rtyper/memory/gc/minimark.py +++ b/rpython/rtyper/memory/gc/minimark.py @@ -254,7 +254,7 @@ # # The ArenaCollection() handles the nonmovable objects allocation. if ArenaCollectionClass is None: - from pypy.rpython.memory.gc import minimarkpage + from rpython.rtyper.memory.gc import minimarkpage ArenaCollectionClass = minimarkpage.ArenaCollection self.ac = ArenaCollectionClass(arena_size, page_size, small_request_threshold) diff --git a/rpython/rtyper/memory/gc/minimarktest.py b/rpython/rtyper/memory/gc/minimarktest.py --- a/rpython/rtyper/memory/gc/minimarktest.py +++ b/rpython/rtyper/memory/gc/minimarktest.py @@ -1,7 +1,7 @@ -from pypy.rpython.lltypesystem import llarena -from pypy.rpython.lltypesystem.llmemory import raw_malloc_usage -from pypy.rlib.debug import ll_assert -from pypy.rlib.rarithmetic import LONG_BIT +from rpython.rtyper.lltypesystem import llarena +from rpython.rtyper.lltypesystem.llmemory import raw_malloc_usage +from rpython.rlib.debug import ll_assert +from rpython.rlib.rarithmetic import LONG_BIT # For testing, a simple implementation of ArenaCollection. # This version could be used together with obmalloc.c, but From noreply at buildbot.pypy.org Thu Jan 31 14:32:32 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 31 Jan 2013 14:32:32 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: Move this file into its proper location. Message-ID: <20130131133232.A3AD21C040D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r60767:1349c5778fb0 Date: 2013-01-31 14:30 +0100 http://bitbucket.org/pypy/pypy/changeset/1349c5778fb0/ Log: Move this file into its proper location. diff --git a/pypy/rlib/rsignal.py b/rpython/rlib/rsignal.py rename from pypy/rlib/rsignal.py rename to rpython/rlib/rsignal.py From noreply at buildbot.pypy.org Thu Jan 31 14:32:34 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 31 Jan 2013 14:32:34 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: Move this file as well. Message-ID: <20130131133234.10ECA1C040D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r60768:f19480ac0255 Date: 2013-01-31 14:31 +0100 http://bitbucket.org/pypy/pypy/changeset/f19480ac0255/ Log: Move this file as well. diff --git a/pypy/rpython/memory/gctransform/nogcstm.py b/rpython/rtyper/memory/gctransform/nogcstm.py rename from pypy/rpython/memory/gctransform/nogcstm.py rename to rpython/rtyper/memory/gctransform/nogcstm.py From noreply at buildbot.pypy.org Thu Jan 31 14:48:10 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 31 Jan 2013 14:48:10 +0100 (CET) Subject: [pypy-commit] pypy default: Clean up, and make signals-but-no-thread working again. Message-ID: <20130131134810.BD5271C040D@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r60769:a9622905c38c Date: 2013-01-31 14:47 +0100 http://bitbucket.org/pypy/pypy/changeset/a9622905c38c/ Log: Clean up, and make signals-but-no-thread working again. diff --git a/pypy/interpreter/miscutils.py b/pypy/interpreter/miscutils.py --- a/pypy/interpreter/miscutils.py +++ b/pypy/interpreter/miscutils.py @@ -17,9 +17,8 @@ def setvalue(self, value): self._value = value - def getmainthreadvalue(self): - return self._value + def ismainthread(self): + return True def getallvalues(self): return {0: self._value} - diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py --- a/pypy/module/signal/interp_signal.py +++ b/pypy/module/signal/interp_signal.py @@ -198,15 +198,12 @@ A signal handler function is called with two arguments: the first is the signal number, the second is the interrupted stack frame. """ - ec = space.getexecutioncontext() - main_ec = space.threadlocals.getmainthreadvalue() - - old_handler = getsignal(space, signum) - - if ec is not main_ec: + if not space.threadlocals.ismainthread(): raise OperationError(space.w_ValueError, space.wrap("signal() must be called from the " "main thread")) + old_handler = getsignal(space, signum) + action = space.check_signal_action if space.eq_w(w_handler, space.wrap(SIG_DFL)): pypysig_default(signum) @@ -231,13 +228,10 @@ The fd must be non-blocking. """ - if space.config.objspace.usemodules.thread: - main_ec = space.threadlocals.getmainthreadvalue() - ec = space.getexecutioncontext() - if ec is not main_ec: - raise OperationError( - space.w_ValueError, - space.wrap("set_wakeup_fd only works in main thread")) + if not space.threadlocals.ismainthread(): + raise OperationError( + space.w_ValueError, + space.wrap("set_wakeup_fd only works in main thread")) old_fd = pypysig_set_wakeup_fd(fd) return space.wrap(intmask(old_fd)) diff --git a/pypy/module/thread/threadlocals.py b/pypy/module/thread/threadlocals.py --- a/pypy/module/thread/threadlocals.py +++ b/pypy/module/thread/threadlocals.py @@ -44,10 +44,6 @@ self._mostrecentkey = ident self._mostrecentvalue = value - def getmainthreadvalue(self): - ident = self._mainthreadident - return self._valuedict.get(ident, None) - def ismainthread(self): return thread.get_ident() == self._mainthreadident From noreply at buildbot.pypy.org Thu Jan 31 15:19:46 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 31 Jan 2013 15:19:46 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: hg merge default Message-ID: <20130131141946.BBF231C009B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r60770:393048c68eb3 Date: 2013-01-31 14:49 +0100 http://bitbucket.org/pypy/pypy/changeset/393048c68eb3/ Log: hg merge default diff --git a/pypy/interpreter/miscutils.py b/pypy/interpreter/miscutils.py --- a/pypy/interpreter/miscutils.py +++ b/pypy/interpreter/miscutils.py @@ -17,9 +17,8 @@ def setvalue(self, value): self._value = value - def getmainthreadvalue(self): - return self._value + def ismainthread(self): + return True def getallvalues(self): return {0: self._value} - diff --git a/pypy/module/signal/interp_signal.py b/pypy/module/signal/interp_signal.py --- a/pypy/module/signal/interp_signal.py +++ b/pypy/module/signal/interp_signal.py @@ -198,15 +198,12 @@ A signal handler function is called with two arguments: the first is the signal number, the second is the interrupted stack frame. """ - ec = space.getexecutioncontext() - main_ec = space.threadlocals.getmainthreadvalue() - - old_handler = getsignal(space, signum) - - if ec is not main_ec: + if not space.threadlocals.ismainthread(): raise OperationError(space.w_ValueError, space.wrap("signal() must be called from the " "main thread")) + old_handler = getsignal(space, signum) + action = space.check_signal_action if space.eq_w(w_handler, space.wrap(SIG_DFL)): pypysig_default(signum) @@ -231,13 +228,10 @@ The fd must be non-blocking. """ - if space.config.objspace.usemodules.thread: - main_ec = space.threadlocals.getmainthreadvalue() - ec = space.getexecutioncontext() - if ec is not main_ec: - raise OperationError( - space.w_ValueError, - space.wrap("set_wakeup_fd only works in main thread")) + if not space.threadlocals.ismainthread(): + raise OperationError( + space.w_ValueError, + space.wrap("set_wakeup_fd only works in main thread")) old_fd = pypysig_set_wakeup_fd(fd) return space.wrap(intmask(old_fd)) diff --git a/pypy/module/thread/threadlocals.py b/pypy/module/thread/threadlocals.py --- a/pypy/module/thread/threadlocals.py +++ b/pypy/module/thread/threadlocals.py @@ -49,10 +49,6 @@ self._mostrecentkey = ident self._mostrecentvalue = value - def getmainthreadvalue(self): - ident = self._mainthreadident - return self._valuedict.get(ident, None) - def ismainthread(self): return thread.get_ident() == self._mainthreadident From noreply at buildbot.pypy.org Thu Jan 31 15:19:48 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 31 Jan 2013 15:19:48 +0100 (CET) Subject: [pypy-commit] pypy default: Fix some imports from the doc Message-ID: <20130131141948.41BCD1C009B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r60771:c954b5469840 Date: 2013-01-31 15:04 +0100 http://bitbucket.org/pypy/pypy/changeset/c954b5469840/ Log: Fix some imports from the doc diff --git a/pypy/doc/discussion/improve-rpython.rst b/pypy/doc/discussion/improve-rpython.rst --- a/pypy/doc/discussion/improve-rpython.rst +++ b/pypy/doc/discussion/improve-rpython.rst @@ -15,7 +15,7 @@ from pypy.interpreter.typedef import interp_attrproperty, interp_attrproperty_w from pypy.interpreter.gateway import interp2app from pypy.interpreter.error import OperationError - from pypy.rpython.lltypesystem import rffi, lltype + from rpython.rtyper.lltypesystem import rffi, lltype - A more direct declarative way to write Typedef:: diff --git a/pypy/doc/rffi.rst b/pypy/doc/rffi.rst --- a/pypy/doc/rffi.rst +++ b/pypy/doc/rffi.rst @@ -17,7 +17,7 @@ lltype.Signed or lltype.Array) and memory management must be done by hand. To declare a function, we write:: - from pypy.rpython.lltypesystem import rffi + from rpython.rtyper.lltypesystem import rffi external_function = rffi.llexternal(name, args, result) @@ -31,7 +31,7 @@ libraries and sources by passing in the optional ``compilation_info`` parameter:: - from pypy.rpython.lltypesystem import rffi + from rpython.rtyper.lltypesystem import rffi from rpython.translator.tool.cbuild import ExternalCompilationInfo info = ExternalCompilationInfo(includes=[], libraries=[]) From noreply at buildbot.pypy.org Thu Jan 31 15:19:49 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 31 Jan 2013 15:19:49 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: hg merge default Message-ID: <20130131141949.6EF5B1C009B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r60772:08fd9ba5355b Date: 2013-01-31 15:04 +0100 http://bitbucket.org/pypy/pypy/changeset/08fd9ba5355b/ Log: hg merge default diff --git a/pypy/doc/discussion/improve-rpython.rst b/pypy/doc/discussion/improve-rpython.rst --- a/pypy/doc/discussion/improve-rpython.rst +++ b/pypy/doc/discussion/improve-rpython.rst @@ -15,7 +15,7 @@ from pypy.interpreter.typedef import interp_attrproperty, interp_attrproperty_w from pypy.interpreter.gateway import interp2app from pypy.interpreter.error import OperationError - from pypy.rpython.lltypesystem import rffi, lltype + from rpython.rtyper.lltypesystem import rffi, lltype - A more direct declarative way to write Typedef:: diff --git a/pypy/doc/rffi.rst b/pypy/doc/rffi.rst --- a/pypy/doc/rffi.rst +++ b/pypy/doc/rffi.rst @@ -17,7 +17,7 @@ lltype.Signed or lltype.Array) and memory management must be done by hand. To declare a function, we write:: - from pypy.rpython.lltypesystem import rffi + from rpython.rtyper.lltypesystem import rffi external_function = rffi.llexternal(name, args, result) @@ -31,7 +31,7 @@ libraries and sources by passing in the optional ``compilation_info`` parameter:: - from pypy.rpython.lltypesystem import rffi + from rpython.rtyper.lltypesystem import rffi from rpython.translator.tool.cbuild import ExternalCompilationInfo info = ExternalCompilationInfo(includes=[], libraries=[]) From noreply at buildbot.pypy.org Thu Jan 31 15:19:50 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 31 Jan 2013 15:19:50 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: Move these in place too. Message-ID: <20130131141950.A52E11C009B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r60773:b289e16ed1f5 Date: 2013-01-31 15:09 +0100 http://bitbucket.org/pypy/pypy/changeset/b289e16ed1f5/ Log: Move these in place too. diff --git a/pypy/translator/stm/src_stm/et.c b/rpython/translator/stm/src_stm/et.c rename from pypy/translator/stm/src_stm/et.c rename to rpython/translator/stm/src_stm/et.c diff --git a/pypy/translator/stm/test/richards.py b/rpython/translator/stm/test/richards.py rename from pypy/translator/stm/test/richards.py rename to rpython/translator/stm/test/richards.py diff --git a/pypy/translator/stm/test/test_writebarrier.py b/rpython/translator/stm/test/test_writebarrier.py rename from pypy/translator/stm/test/test_writebarrier.py rename to rpython/translator/stm/test/test_writebarrier.py diff --git a/pypy/translator/stm/test/transform2_support.py b/rpython/translator/stm/test/transform2_support.py rename from pypy/translator/stm/test/transform2_support.py rename to rpython/translator/stm/test/transform2_support.py diff --git a/pypy/translator/stm/writebarrier.py b/rpython/translator/stm/writebarrier.py rename from pypy/translator/stm/writebarrier.py rename to rpython/translator/stm/writebarrier.py From noreply at buildbot.pypy.org Thu Jan 31 15:19:51 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 31 Jan 2013 15:19:51 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: Run a mechanical translation of imports. Message-ID: <20130131141951.DD9151C009B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r60774:2607c5096b83 Date: 2013-01-31 15:11 +0100 http://bitbucket.org/pypy/pypy/changeset/2607c5096b83/ Log: Run a mechanical translation of imports. diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -289,7 +289,7 @@ # one of the opcodes in the one of the sequences # * POP_TOP/LOAD_CONST/RETURN_VALUE # * POP_TOP/LOAD_FAST/RETURN_VALUE - from pypy.rlib import rstm + from rpython.rlib import rstm if rstm.should_break_transaction(): opcode = ord(co_code[next_instr]) if opcode not in (self.opcodedesc.RETURN_VALUE.index, diff --git a/pypy/module/thread/atomic.py b/pypy/module/thread/atomic.py --- a/pypy/module/thread/atomic.py +++ b/pypy/module/thread/atomic.py @@ -5,7 +5,7 @@ def exclusive_atomic_enter(space): if space.config.translation.stm: - from pypy.rlib.rstm import is_atomic + from rpython.rlib.rstm import is_atomic count = is_atomic() else: giltl = space.threadlocals @@ -18,7 +18,7 @@ def atomic_enter(space): if space.config.translation.stm: - from pypy.rlib.rstm import increment_atomic + from rpython.rlib.rstm import increment_atomic increment_atomic() else: giltl = space.threadlocals @@ -27,7 +27,7 @@ def atomic_exit(space, w_ignored1=None, w_ignored2=None, w_ignored3=None): if space.config.translation.stm: - from pypy.rlib.rstm import decrement_atomic, is_atomic + from rpython.rlib.rstm import decrement_atomic, is_atomic if is_atomic(): decrement_atomic() return diff --git a/pypy/module/thread/stm.py b/pypy/module/thread/stm.py --- a/pypy/module/thread/stm.py +++ b/pypy/module/thread/stm.py @@ -5,8 +5,8 @@ from pypy.module.thread.threadlocals import OSThreadLocals from pypy.module.thread.error import wrap_thread_error from pypy.module.thread import ll_thread -from pypy.rlib import rstm -from pypy.rlib.objectmodel import invoke_around_extcall +from rpython.rlib import rstm +from rpython.rlib.objectmodel import invoke_around_extcall class STMThreadLocals(OSThreadLocals): diff --git a/rpython/jit/backend/llsupport/gc.py b/rpython/jit/backend/llsupport/gc.py --- a/rpython/jit/backend/llsupport/gc.py +++ b/rpython/jit/backend/llsupport/gc.py @@ -106,9 +106,9 @@ def rewrite_assembler(self, cpu, operations, gcrefs_output_list): if not self.stm: - from pypy.jit.backend.llsupport.rewrite import GcRewriterAssembler + from rpython.jit.backend.llsupport.rewrite import GcRewriterAssembler else: - from pypy.jit.backend.llsupport import stmrewrite + from rpython.jit.backend.llsupport import stmrewrite GcRewriterAssembler = stmrewrite.GcStmRewriterAssembler rewriter = GcRewriterAssembler(self, cpu) newops = rewriter.rewrite(operations) @@ -824,7 +824,7 @@ self.do_write_barrier = do_write_barrier def _setup_barriers_for_stm(self): - from pypy.rpython.memory.gc import stmgc + from rpython.rtyper.memory.gc import stmgc WBDescr = WriteBarrierDescr self.P2Rdescr = WBDescr(self, (stmgc.GCFLAG_GLOBAL, 'P2R', 'stm_DirectReadBarrier')) @@ -933,7 +933,7 @@ if self.stm: # XXX remove the indirections in the following calls - from pypy.rlib import rstm + from rpython.rlib import rstm self.generate_function('stm_try_inevitable', rstm.become_inevitable, [], RESULT=lltype.Void) diff --git a/rpython/jit/backend/llsupport/stmrewrite.py b/rpython/jit/backend/llsupport/stmrewrite.py --- a/rpython/jit/backend/llsupport/stmrewrite.py +++ b/rpython/jit/backend/llsupport/stmrewrite.py @@ -1,7 +1,7 @@ -from pypy.jit.backend.llsupport.rewrite import GcRewriterAssembler -from pypy.jit.metainterp.resoperation import ResOperation, rop -from pypy.jit.metainterp.history import BoxPtr, ConstPtr, ConstInt -from pypy.rlib.objectmodel import specialize +from rpython.jit.backend.llsupport.rewrite import GcRewriterAssembler +from rpython.jit.metainterp.resoperation import ResOperation, rop +from rpython.jit.metainterp.history import BoxPtr, ConstPtr, ConstInt +from rpython.rlib.objectmodel import specialize # # STM Support diff --git a/rpython/jit/backend/llsupport/test/test_gc.py b/rpython/jit/backend/llsupport/test/test_gc.py --- a/rpython/jit/backend/llsupport/test/test_gc.py +++ b/rpython/jit/backend/llsupport/test/test_gc.py @@ -406,7 +406,7 @@ assert self.llop1.record == [('barrier', s_adr)] def test_gen_write_barrier(self): - from pypy.jit.backend.llsupport.rewrite import GcRewriterAssembler + from rpython.jit.backend.llsupport.rewrite import GcRewriterAssembler gc_ll_descr = self.gc_ll_descr llop1 = self.llop1 # diff --git a/rpython/jit/backend/llsupport/test/test_stmrewrite.py b/rpython/jit/backend/llsupport/test/test_stmrewrite.py --- a/rpython/jit/backend/llsupport/test/test_stmrewrite.py +++ b/rpython/jit/backend/llsupport/test/test_stmrewrite.py @@ -1,7 +1,7 @@ -from pypy.jit.backend.llsupport.descr import * -from pypy.jit.backend.llsupport.gc import * -from pypy.jit.metainterp.gc import get_description -from pypy.jit.backend.llsupport.test.test_rewrite import RewriteTests +from rpython.jit.backend.llsupport.descr import * +from rpython.jit.backend.llsupport.gc import * +from rpython.jit.metainterp.gc import get_description +from rpython.jit.backend.llsupport.test.test_rewrite import RewriteTests class TestStm(RewriteTests): diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py --- a/rpython/jit/backend/x86/assembler.py +++ b/rpython/jit/backend/x86/assembler.py @@ -430,7 +430,7 @@ def stm_threadlocal(self, globaladdr): if self.with_stm(): - from pypy.jit.backend.x86 import stmtlocal + from rpython.jit.backend.x86 import stmtlocal globaladdr -= stmtlocal.threadlocal_base() assert -0x10000 <= globaladdr < 0x10000 #estimate: "should be small" # --- in particular, fits_in_32bits() must be true @@ -438,7 +438,7 @@ def stm_SEGPREFIX(self, mc): if self.with_stm(): - from pypy.jit.backend.x86 import stmtlocal + from rpython.jit.backend.x86 import stmtlocal stmtlocal.tl_segment_prefix(mc) @staticmethod diff --git a/rpython/jit/backend/x86/stmtlocal.py b/rpython/jit/backend/x86/stmtlocal.py --- a/rpython/jit/backend/x86/stmtlocal.py +++ b/rpython/jit/backend/x86/stmtlocal.py @@ -1,6 +1,6 @@ -from pypy.rpython.lltypesystem import lltype, rffi -from pypy.translator.tool.cbuild import ExternalCompilationInfo -from pypy.jit.backend.x86.arch import WORD +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.translator.tool.cbuild import ExternalCompilationInfo +from rpython.jit.backend.x86.arch import WORD if WORD == 4: diff --git a/rpython/rlib/atomic_ops.py b/rpython/rlib/atomic_ops.py --- a/rpython/rlib/atomic_ops.py +++ b/rpython/rlib/atomic_ops.py @@ -1,7 +1,7 @@ import py from pypy.tool.autopath import pypydir -from pypy.rpython.lltypesystem import lltype, llmemory, rffi -from pypy.translator.tool.cbuild import ExternalCompilationInfo +from rpython.rtyper.lltypesystem import lltype, llmemory, rffi +from rpython.translator.tool.cbuild import ExternalCompilationInfo cdir = py.path.local(pypydir) / 'translator' / 'stm' diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py --- a/rpython/rlib/objectmodel.py +++ b/rpython/rlib/objectmodel.py @@ -608,7 +608,7 @@ return rffi.stackcounter.stacks_counter > 1 def has_around_extcall(): - from pypy.rpython.lltypesystem import rffi + from rpython.rtyper.lltypesystem import rffi return rffi.aroundstate.before is not None diff --git a/rpython/rlib/rstm.py b/rpython/rlib/rstm.py --- a/rpython/rlib/rstm.py +++ b/rpython/rlib/rstm.py @@ -1,12 +1,12 @@ import threading -from pypy.translator.stm import stmgcintf -from pypy.rlib.debug import ll_assert, fatalerror -from pypy.rlib.objectmodel import keepalive_until_here, specialize -from pypy.rlib.objectmodel import we_are_translated -from pypy.rlib.rposix import get_errno, set_errno -from pypy.rpython.lltypesystem import lltype, llmemory, rffi, rclass -from pypy.rpython.lltypesystem.lloperation import llop -from pypy.rpython.annlowlevel import (cast_instance_to_base_ptr, +from rpython.translator.stm import stmgcintf +from rpython.rlib.debug import ll_assert, fatalerror +from rpython.rlib.objectmodel import keepalive_until_here, specialize +from rpython.rlib.objectmodel import we_are_translated +from rpython.rlib.rposix import get_errno, set_errno +from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rclass +from rpython.rtyper.lltypesystem.lloperation import llop +from rpython.rtyper.annlowlevel import (cast_instance_to_base_ptr, llhelper) def is_inevitable(): diff --git a/rpython/rlib/test/test_atomic_ops.py b/rpython/rlib/test/test_atomic_ops.py --- a/rpython/rlib/test/test_atomic_ops.py +++ b/rpython/rlib/test/test_atomic_ops.py @@ -1,5 +1,5 @@ -from pypy.rlib.atomic_ops import bool_cas -from pypy.rpython.lltypesystem import lltype, llmemory, rffi +from rpython.rlib.atomic_ops import bool_cas +from rpython.rtyper.lltypesystem import lltype, llmemory, rffi ARRAY = rffi.CArray(llmemory.Address) @@ -24,7 +24,7 @@ return 0 def test_translate_bool_cas(): - from pypy.translator.c.test.test_genc import compile + from rpython.translator.c.test.test_genc import compile f = compile(test_bool_cas, []) res = f() diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py --- a/rpython/rtyper/llinterp.py +++ b/rpython/rtyper/llinterp.py @@ -61,7 +61,7 @@ self.tracer = Tracer() def eval_entry_point(self, graph, args=()): - from pypy.rpython.lltypesystem import lltype, rffi + from rpython.rtyper.lltypesystem import lltype, rffi ARGV = graph.startblock.inputargs[1].concretetype args = [''] + list(args) ll_args = lltype.malloc(ARGV.TO, len(args), flavor='raw') diff --git a/rpython/rtyper/lltypesystem/llmemory.py b/rpython/rtyper/lltypesystem/llmemory.py --- a/rpython/rtyper/lltypesystem/llmemory.py +++ b/rpython/rtyper/lltypesystem/llmemory.py @@ -564,7 +564,7 @@ class AddressAsUInt(AddressAsInt): def annotation(self): - from pypy.annotation import model + from rpython.annotator import model return model.SomeInteger(unsigned=True) def lltype(self): return lltype.Unsigned diff --git a/rpython/rtyper/memory/gc/stmgc.py b/rpython/rtyper/memory/gc/stmgc.py --- a/rpython/rtyper/memory/gc/stmgc.py +++ b/rpython/rtyper/memory/gc/stmgc.py @@ -1,12 +1,12 @@ -from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup -from pypy.rpython.lltypesystem.lloperation import llop -from pypy.rpython.lltypesystem.llmemory import raw_malloc_usage, raw_memcopy -from pypy.rpython.memory.gc.base import GCBase, MovingGCBase -from pypy.rpython.memory.support import mangle_hash -from pypy.rpython.annlowlevel import llhelper -from pypy.rlib.rarithmetic import LONG_BIT, r_uint -from pypy.rlib.debug import ll_assert, debug_start, debug_stop, fatalerror -from pypy.rlib.debug import debug_print +from rpython.rtyper.lltypesystem import lltype, llmemory, llarena, llgroup +from rpython.rtyper.lltypesystem.lloperation import llop +from rpython.rtyper.lltypesystem.llmemory import raw_malloc_usage, raw_memcopy +from rpython.rtyper.memory.gc.base import GCBase, MovingGCBase +from rpython.rtyper.memory.support import mangle_hash +from rpython.rtyper.annlowlevel import llhelper +from rpython.rlib.rarithmetic import LONG_BIT, r_uint +from rpython.rlib.debug import ll_assert, debug_start, debug_stop, fatalerror +from rpython.rlib.debug import debug_print from pypy.module.thread import ll_thread @@ -165,10 +165,10 @@ assert stm_operations == 'use_real_one', ( "XXX not provided so far: stm_operations == %r" % ( stm_operations,)) - from pypy.translator.stm.stmgcintf import StmOperations + from rpython.translator.stm.stmgcintf import StmOperations stm_operations = StmOperations() # - from pypy.rpython.memory.gc import stmshared + from rpython.rtyper.memory.gc import stmshared self.stm_operations = stm_operations self.nursery_size = nursery_size self.maximum_extra_threshold = 0 @@ -202,7 +202,7 @@ of the GC. The C-level transaction should already be started.""" ll_assert(self.stm_operations.in_transaction(), "setup_thread: not in a transaction") - from pypy.rpython.memory.gc.stmtls import StmGCTLS + from rpython.rtyper.memory.gc.stmtls import StmGCTLS stmtls = StmGCTLS(self) stmtls.start_transaction() @@ -221,7 +221,7 @@ @always_inline def get_tls(self): - from pypy.rpython.memory.gc.stmtls import StmGCTLS + from rpython.rtyper.memory.gc.stmtls import StmGCTLS tls = self.stm_operations.get_tls() return StmGCTLS.cast_address_to_tls_object(tls) diff --git a/rpython/rtyper/memory/gc/stmshared.py b/rpython/rtyper/memory/gc/stmshared.py --- a/rpython/rtyper/memory/gc/stmshared.py +++ b/rpython/rtyper/memory/gc/stmshared.py @@ -1,5 +1,5 @@ -from pypy.rpython.lltypesystem import lltype, llmemory, llarena -from pypy.rlib.objectmodel import free_non_gc_object +from rpython.rtyper.lltypesystem import lltype, llmemory, llarena +from rpython.rlib.objectmodel import free_non_gc_object NULL = llmemory.NULL diff --git a/rpython/rtyper/memory/gc/stmtls.py b/rpython/rtyper/memory/gc/stmtls.py --- a/rpython/rtyper/memory/gc/stmtls.py +++ b/rpython/rtyper/memory/gc/stmtls.py @@ -1,21 +1,21 @@ -from pypy.rpython.lltypesystem import lltype, llmemory, llarena, rffi -from pypy.rpython.lltypesystem.lloperation import llop -from pypy.rpython.annlowlevel import cast_instance_to_base_ptr, llhelper -from pypy.rpython.annlowlevel import cast_base_ptr_to_instance, base_ptr_lltype -from pypy.rlib.objectmodel import we_are_translated, free_non_gc_object -from pypy.rlib.objectmodel import specialize -from pypy.rlib.rarithmetic import r_uint -from pypy.rlib.debug import ll_assert, fatalerror -from pypy.rlib.debug import debug_start, debug_stop, debug_print +from rpython.rtyper.lltypesystem import lltype, llmemory, llarena, rffi +from rpython.rtyper.lltypesystem.lloperation import llop +from rpython.rtyper.annlowlevel import cast_instance_to_base_ptr, llhelper +from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance, base_ptr_lltype +from rpython.rlib.objectmodel import we_are_translated, free_non_gc_object +from rpython.rlib.objectmodel import specialize +from rpython.rlib.rarithmetic import r_uint +from rpython.rlib.debug import ll_assert, fatalerror +from rpython.rlib.debug import debug_start, debug_stop, debug_print -from pypy.rpython.memory.gc.stmgc import WORD, NULL -from pypy.rpython.memory.gc.stmgc import always_inline, dont_inline -from pypy.rpython.memory.gc.stmgc import GCFLAG_GLOBAL, GCFLAG_VISITED -from pypy.rpython.memory.gc.stmgc import GCFLAG_LOCAL_COPY -from pypy.rpython.memory.gc.stmgc import GCFLAG_POSSIBLY_OUTDATED -from pypy.rpython.memory.gc.stmgc import GCFLAG_NOT_WRITTEN -from pypy.rpython.memory.gc.stmgc import GCFLAG_HASH_FIELD, GCFLAG_NEW_HASH -from pypy.rpython.memory.gc.stmgc import hdr_revision, set_hdr_revision +from rpython.rtyper.memory.gc.stmgc import WORD, NULL +from rpython.rtyper.memory.gc.stmgc import always_inline, dont_inline +from rpython.rtyper.memory.gc.stmgc import GCFLAG_GLOBAL, GCFLAG_VISITED +from rpython.rtyper.memory.gc.stmgc import GCFLAG_LOCAL_COPY +from rpython.rtyper.memory.gc.stmgc import GCFLAG_POSSIBLY_OUTDATED +from rpython.rtyper.memory.gc.stmgc import GCFLAG_NOT_WRITTEN +from rpython.rtyper.memory.gc.stmgc import GCFLAG_HASH_FIELD, GCFLAG_NEW_HASH +from rpython.rtyper.memory.gc.stmgc import hdr_revision, set_hdr_revision SIZE_OF_SIGNED = llmemory.sizeof(lltype.Signed) @@ -49,7 +49,7 @@ self.set_extra_threshold(self.gc.maximum_extra_threshold) # # --- a thread-local allocator for the shared area - from pypy.rpython.memory.gc.stmshared import StmGCThreadLocalAllocator + from rpython.rtyper.memory.gc.stmshared import StmGCThreadLocalAllocator self.sharedarea_tls = StmGCThreadLocalAllocator(self.gc.sharedarea) self.copied_local_objects = self.AddressStack() # XXX KILL # --- the LOCAL objects which are weakrefs. They are also listed @@ -185,7 +185,7 @@ self.detect_flag_combination = -1 # # Move away the previous sharedarea_tls and start a new one. - from pypy.rpython.memory.gc.stmshared import StmGCThreadLocalAllocator + from rpython.rtyper.memory.gc.stmshared import StmGCThreadLocalAllocator previous_sharedarea_tls = self.sharedarea_tls self.sharedarea_tls = StmGCThreadLocalAllocator(self.gc.sharedarea) # diff --git a/rpython/rtyper/memory/gc/test/test_stmgc.py b/rpython/rtyper/memory/gc/test/test_stmgc.py --- a/rpython/rtyper/memory/gc/test/test_stmgc.py +++ b/rpython/rtyper/memory/gc/test/test_stmgc.py @@ -1,13 +1,13 @@ import py -from pypy.rlib.rarithmetic import r_uint -from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup, rffi -from pypy.rpython.memory.gc.stmgc import StmGC, WORD, REV_INITIAL -from pypy.rpython.memory.gc.stmgc import GCFLAG_GLOBAL, GCFLAG_NOT_WRITTEN -from pypy.rpython.memory.gc.stmgc import GCFLAG_POSSIBLY_OUTDATED -from pypy.rpython.memory.gc.stmgc import GCFLAG_LOCAL_COPY, GCFLAG_VISITED -from pypy.rpython.memory.gc.stmgc import GCFLAG_HASH_FIELD -from pypy.rpython.memory.gc.stmgc import hdr_revision, set_hdr_revision -from pypy.rpython.memory.support import mangle_hash +from rpython.rlib.rarithmetic import r_uint +from rpython.rtyper.lltypesystem import lltype, llmemory, llarena, llgroup, rffi +from rpython.rtyper.memory.gc.stmgc import StmGC, WORD, REV_INITIAL +from rpython.rtyper.memory.gc.stmgc import GCFLAG_GLOBAL, GCFLAG_NOT_WRITTEN +from rpython.rtyper.memory.gc.stmgc import GCFLAG_POSSIBLY_OUTDATED +from rpython.rtyper.memory.gc.stmgc import GCFLAG_LOCAL_COPY, GCFLAG_VISITED +from rpython.rtyper.memory.gc.stmgc import GCFLAG_HASH_FIELD +from rpython.rtyper.memory.gc.stmgc import hdr_revision, set_hdr_revision +from rpython.rtyper.memory.support import mangle_hash S = lltype.GcStruct('S', ('a', lltype.Signed), ('b', lltype.Signed), @@ -84,7 +84,7 @@ tldict[obj] = localobj def tldict_enum(self): - from pypy.rpython.memory.gc.stmtls import StmGCTLS + from rpython.rtyper.memory.gc.stmtls import StmGCTLS callback = StmGCTLS._stm_enum_callback tls = self.get_tls() for key, value in self._tldicts[self.threadnum].iteritems(): @@ -347,7 +347,7 @@ def test_random_gc_usage(self): import random - from pypy.rpython.memory.gc.test import test_stmtls + from rpython.rtyper.memory.gc.test import test_stmtls self.gc.root_walker = test_stmtls.FakeRootWalker() # sr2 = {} # {obj._obj: obj._obj} following the 'sr2' attribute @@ -452,7 +452,7 @@ print 'Iteration %d finished' % iteration def test_relocalize_objects_after_transaction_break(self): - from pypy.rpython.memory.gc.test import test_stmtls + from rpython.rtyper.memory.gc.test import test_stmtls self.gc.root_walker = test_stmtls.FakeRootWalker() # tr1 = self.malloc(SR, globl=True) # three prebuilt objects @@ -477,7 +477,7 @@ self.checkflags(tr3_adr, True, True) # tr3 has become global again def test_obj_with_invalid_offset_after_transaction_stop(self): - from pypy.rpython.memory.gc.test import test_stmtls + from rpython.rtyper.memory.gc.test import test_stmtls self.gc.root_walker = test_stmtls.FakeRootWalker() # tr1, tr1_adr = self.malloc(SR, globl=False) # local @@ -489,7 +489,7 @@ py.test.raises(llarena.ArenaError, self.gc.root_walker.pop) def test_non_prebuilt_relocalize_after_transaction_break(self): - from pypy.rpython.memory.gc.test import test_stmtls + from rpython.rtyper.memory.gc.test import test_stmtls self.gc.root_walker = test_stmtls.FakeRootWalker() # tr1, tr1_adr = self.malloc(SR, globl=False) # local @@ -707,7 +707,7 @@ assert s2 == tr1.s1 # tr1 is a root, so not copied yet def test_weakref_to_local_in_main_thread(self): - from pypy.rpython.memory.gc.test import test_stmtls + from rpython.rtyper.memory.gc.test import test_stmtls self.gc.root_walker = test_stmtls.FakeRootWalker() # sr1, sr1_adr = self.malloc(SR, globl=False) @@ -750,7 +750,7 @@ assert a == sr1_adr def test_prebuilt_nongc(self): - from pypy.rpython.memory.gc.test import test_stmtls + from rpython.rtyper.memory.gc.test import test_stmtls self.gc.root_walker = test_stmtls.FakeRootWalker() NONGC = lltype.Struct('NONGC', ('s', lltype.Ptr(S))) nongc = lltype.malloc(NONGC, immortal=True, flavor='raw') diff --git a/rpython/rtyper/memory/gc/test/test_stmtls.py b/rpython/rtyper/memory/gc/test/test_stmtls.py --- a/rpython/rtyper/memory/gc/test/test_stmtls.py +++ b/rpython/rtyper/memory/gc/test/test_stmtls.py @@ -1,9 +1,9 @@ import py -from pypy.rlib.rarithmetic import r_uint -from pypy.rpython.lltypesystem import lltype, llmemory, llarena, llgroup -from pypy.rpython.memory.gc.stmtls import StmGCTLS, WORD -from pypy.rpython.memory.support import get_address_stack, get_address_deque -from pypy.rpython.memory.gcheader import GCHeaderBuilder +from rpython.rlib.rarithmetic import r_uint +from rpython.rtyper.lltypesystem import lltype, llmemory, llarena, llgroup +from rpython.rtyper.memory.gc.stmtls import StmGCTLS, WORD +from rpython.rtyper.memory.support import get_address_stack, get_address_deque +from rpython.rtyper.memory.gcheader import GCHeaderBuilder NULL = llmemory.NULL @@ -66,7 +66,7 @@ callback(arg, root) class FakeGC: - from pypy.rpython.memory.support import AddressDict, null_address_dict + from rpython.rtyper.memory.support import AddressDict, null_address_dict DEBUG = True AddressStack = get_address_stack() AddressDeque = get_address_deque() diff --git a/rpython/rtyper/memory/gctransform/framework.py b/rpython/rtyper/memory/gctransform/framework.py --- a/rpython/rtyper/memory/gctransform/framework.py +++ b/rpython/rtyper/memory/gctransform/framework.py @@ -270,8 +270,8 @@ def _declare_functions(self, GCClass, getfn, s_gc, s_typeid16): - from pypy.rpython.memory.gc.base import ARRAY_TYPEID_MAP - from pypy.rpython.memory.gc import inspector + from rpython.rtyper.memory.gc.base import ARRAY_TYPEID_MAP + from rpython.rtyper.memory.gc import inspector s_gcref = annmodel.SomePtr(llmemory.GCREF) gcdata = self.gcdata diff --git a/rpython/rtyper/memory/gctransform/nogcstm.py b/rpython/rtyper/memory/gctransform/nogcstm.py --- a/rpython/rtyper/memory/gctransform/nogcstm.py +++ b/rpython/rtyper/memory/gctransform/nogcstm.py @@ -1,6 +1,6 @@ -from pypy.rpython.memory.gctransform.boehm import BoehmGCTransformer -from pypy.rpython.lltypesystem import lltype, llmemory -from pypy.rlib.rarithmetic import r_uint, LONG_BIT +from rpython.rtyper.memory.gctransform.boehm import BoehmGCTransformer +from rpython.rtyper.lltypesystem import lltype, llmemory +from rpython.rlib.rarithmetic import r_uint, LONG_BIT _first_gcflag = 1 << (LONG_BIT//2) diff --git a/rpython/rtyper/memory/gctransform/stmframework.py b/rpython/rtyper/memory/gctransform/stmframework.py --- a/rpython/rtyper/memory/gctransform/stmframework.py +++ b/rpython/rtyper/memory/gctransform/stmframework.py @@ -1,12 +1,12 @@ -from pypy.rpython.memory.gctransform import shadowstack -from pypy.rpython.memory.gctransform.framework import BaseRootWalker -from pypy.rpython.memory.gctransform.framework import sizeofaddr -from pypy.rpython.lltypesystem import lltype, llmemory -from pypy.rpython import rmodel -from pypy.annotation import model as annmodel -from pypy.rlib.debug import fatalerror_notb -from pypy.rlib.nonconst import NonConstant -from pypy.rlib.objectmodel import specialize +from rpython.rtyper.memory.gctransform import shadowstack +from rpython.rtyper.memory.gctransform.framework import BaseRootWalker +from rpython.rtyper.memory.gctransform.framework import sizeofaddr +from rpython.rtyper.lltypesystem import lltype, llmemory +from rpython.rtyper import rmodel +from rpython.annotator import model as annmodel +from rpython.rlib.debug import fatalerror_notb +from rpython.rlib.nonconst import NonConstant +from rpython.rlib.objectmodel import specialize END_MARKER = -8 # keep in sync with src_stm/rpyintf.c @@ -84,7 +84,7 @@ root_stack_depth = 163840 def __init__(self, gctransformer): - from pypy.rpython.memory.gctransform import shadowstack + from rpython.rtyper.memory.gctransform import shadowstack # BaseRootWalker.__init__(self, gctransformer) # we use the thread-local self.stackgcdata to store state; diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py --- a/rpython/translator/c/funcgen.py +++ b/rpython/translator/c/funcgen.py @@ -582,7 +582,7 @@ if not self._is_stm(): raise AssertionError("STM transformation not applied. " "You need '--stm'") - from pypy.translator.stm.funcgen import op_stm + from rpython.translator.stm.funcgen import op_stm self.__class__.op_stm = op_stm return self.op_stm(op) OP_STM_START_TRANSACTION = _OP_STM diff --git a/rpython/translator/c/gc.py b/rpython/translator/c/gc.py --- a/rpython/translator/c/gc.py +++ b/rpython/translator/c/gc.py @@ -300,7 +300,7 @@ def gettransformer(self): if self.db.translator.config.translation.stm: - from pypy.rpython.memory.gctransform import nogcstm + from rpython.rtyper.memory.gctransform import nogcstm return nogcstm.NoneSTMGCTransformer(self.db.translator) return BoehmGcPolicy.gettransformer(self) @@ -460,7 +460,7 @@ class StmFrameworkGcPolicy(BasicFrameworkGcPolicy): def gettransformer(self): - from pypy.rpython.memory.gctransform import stmframework + from rpython.rtyper.memory.gctransform import stmframework return stmframework.StmFrameworkGCTransformer(self.db.translator) diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py --- a/rpython/translator/c/genc.py +++ b/rpython/translator/c/genc.py @@ -139,7 +139,7 @@ translator = self.translator if self.config.translation.stm: - from pypy.translator.stm import transform2 + from rpython.translator.stm import transform2 self.getentrypointptr() # build the wrapper first # ^^ this is needed to make sure we see the no-GC wrapper function # calling the GC entrypoint function. @@ -195,7 +195,7 @@ self.merge_eci(db.gcpolicy.compilation_info()) if self.config.translation.stm: - from pypy.translator.stm.stmgcintf import eci + from rpython.translator.stm.stmgcintf import eci self.merge_eci(eci) all = [] @@ -322,16 +322,16 @@ if self._entrypoint_wrapper is not None: return self._entrypoint_wrapper # - from pypy.annotation import model as annmodel - from pypy.rpython.lltypesystem import rffi - from pypy.rpython.annlowlevel import MixLevelHelperAnnotator + from rpython.annotator import model as annmodel + from rpython.rtyper.lltypesystem import rffi + from rpython.rtyper.annlowlevel import MixLevelHelperAnnotator entrypoint = self.entrypoint stm_nogc = (self.config.translation.stm and self.config.translation.gc == "none") # def entrypoint_wrapper(argc, argv): if stm_nogc: - from pypy.translator.stm.funcgen import _stm_nogc_init_function + from rpython.translator.stm.funcgen import _stm_nogc_init_function _stm_nogc_init_function() list = [""] * argc i = 0 diff --git a/rpython/translator/stm/funcgen.py b/rpython/translator/stm/funcgen.py --- a/rpython/translator/stm/funcgen.py +++ b/rpython/translator/stm/funcgen.py @@ -1,5 +1,5 @@ -from pypy.translator.c.support import c_string_constant -from pypy.translator.stm.stmgcintf import StmOperations +from rpython.translator.c.support import c_string_constant +from rpython.translator.stm.stmgcintf import StmOperations def stm_start_transaction(funcgen, op): diff --git a/rpython/translator/stm/inevitable.py b/rpython/translator/stm/inevitable.py --- a/rpython/translator/stm/inevitable.py +++ b/rpython/translator/stm/inevitable.py @@ -1,8 +1,8 @@ -from pypy.rpython.lltypesystem import lltype, lloperation, rclass -from pypy.translator.stm.writebarrier import is_immutable -from pypy.objspace.flow.model import SpaceOperation, Constant -from pypy.translator.unsimplify import varoftype -from pypy.translator.simplify import get_funcobj +from rpython.rtyper.lltypesystem import lltype, lloperation, rclass +from rpython.translator.stm.writebarrier import is_immutable +from rpython.flowspace.model import SpaceOperation, Constant +from rpython.translator.unsimplify import varoftype +from rpython.translator.simplify import get_funcobj ALWAYS_ALLOW_OPERATIONS = set([ @@ -112,7 +112,7 @@ varoftype(lltype.Void)) def insert_turn_inevitable(graph): - from pypy.translator.backendopt.writeanalyze import FreshMallocs + from rpython.translator.backendopt.writeanalyze import FreshMallocs fresh_mallocs = FreshMallocs(graph) for block in graph.iterblocks(): for i in range(len(block.operations)-1, -1, -1): diff --git a/rpython/translator/stm/jitdriver.py b/rpython/translator/stm/jitdriver.py --- a/rpython/translator/stm/jitdriver.py +++ b/rpython/translator/stm/jitdriver.py @@ -1,12 +1,12 @@ -from pypy.rpython.lltypesystem import lltype, rclass -from pypy.objspace.flow.model import checkgraph, copygraph -from pypy.objspace.flow.model import Block, Link, SpaceOperation, Constant -from pypy.translator.unsimplify import split_block, varoftype -from pypy.translator.stm.stmgcintf import StmOperations -from pypy.annotation.model import lltype_to_annotation, s_Int -from pypy.rpython.annlowlevel import (MixLevelHelperAnnotator, +from rpython.rtyper.lltypesystem import lltype, rclass +from rpython.flowspace.model import checkgraph, copygraph +from rpython.flowspace.model import Block, Link, SpaceOperation, Constant +from rpython.translator.unsimplify import split_block, varoftype +from rpython.translator.stm.stmgcintf import StmOperations +from rpython.annotator.model import lltype_to_annotation, s_Int +from rpython.rtyper.annlowlevel import (MixLevelHelperAnnotator, cast_base_ptr_to_instance) -from pypy.rlib import rstm +from rpython.rlib import rstm from pypy.tool.sourcetools import compile2 diff --git a/rpython/translator/stm/stmgcintf.py b/rpython/translator/stm/stmgcintf.py --- a/rpython/translator/stm/stmgcintf.py +++ b/rpython/translator/stm/stmgcintf.py @@ -1,8 +1,8 @@ import py from pypy.tool.autopath import pypydir -from pypy.rpython.lltypesystem import lltype, llmemory, rffi -from pypy.translator.tool.cbuild import ExternalCompilationInfo -from pypy.rlib.rarithmetic import LONG_BIT +from rpython.rtyper.lltypesystem import lltype, llmemory, rffi +from rpython.translator.tool.cbuild import ExternalCompilationInfo +from rpython.rlib.rarithmetic import LONG_BIT cdir = py.path.local(pypydir) / 'translator' / 'stm' diff --git a/rpython/translator/stm/test/support.py b/rpython/translator/stm/test/support.py --- a/rpython/translator/stm/test/support.py +++ b/rpython/translator/stm/test/support.py @@ -1,6 +1,6 @@ """CompiledSTMTests, a support class for translated tests with STM""" -from pypy.translator.c.test.test_standalone import StandaloneTests +from rpython.translator.c.test.test_standalone import StandaloneTests class CompiledSTMTests(StandaloneTests): @@ -14,7 +14,7 @@ # # Prevent the RaiseAnalyzer from just emitting "WARNING: Unknown # operation". We want instead it to crash. - from pypy.translator.backendopt.canraise import RaiseAnalyzer + from rpython.translator.backendopt.canraise import RaiseAnalyzer RaiseAnalyzer.fail_on_unknown_operation = True try: res = StandaloneTests.compile(self, entry_point, debug=True, diff --git a/rpython/translator/stm/test/targetdemo2.py b/rpython/translator/stm/test/targetdemo2.py --- a/rpython/translator/stm/test/targetdemo2.py +++ b/rpython/translator/stm/test/targetdemo2.py @@ -1,10 +1,10 @@ import time from pypy.module.thread import ll_thread -from pypy.rlib import rstm, jit -from pypy.rlib.objectmodel import invoke_around_extcall, we_are_translated -from pypy.rlib.objectmodel import compute_identity_hash -from pypy.rlib.debug import ll_assert -from pypy.rpython.lltypesystem import lltype, rffi, rclass +from rpython.rlib import rstm, jit +from rpython.rlib.objectmodel import invoke_around_extcall, we_are_translated +from rpython.rlib.objectmodel import compute_identity_hash +from rpython.rlib.debug import ll_assert +from rpython.rtyper.lltypesystem import lltype, rffi, rclass class Node: diff --git a/rpython/translator/stm/test/targetjit1.py b/rpython/translator/stm/test/targetjit1.py --- a/rpython/translator/stm/test/targetjit1.py +++ b/rpython/translator/stm/test/targetjit1.py @@ -1,6 +1,6 @@ from pypy.module.thread import ll_thread -from pypy.rlib import jit, rstm -from pypy.rlib.objectmodel import invoke_around_extcall +from rpython.rlib import jit, rstm +from rpython.rlib.objectmodel import invoke_around_extcall class Global(object): @@ -162,7 +162,7 @@ # _____ Define and setup target ___ -from pypy.jit.codewriter.policy import JitPolicy +from rpython.jit.codewriter.policy import JitPolicy def jitpolicy(driver): return JitPolicy() diff --git a/rpython/translator/stm/test/test_inevitable.py b/rpython/translator/stm/test/test_inevitable.py --- a/rpython/translator/stm/test/test_inevitable.py +++ b/rpython/translator/stm/test/test_inevitable.py @@ -1,8 +1,8 @@ -from pypy.rpython.lltypesystem import lltype, llmemory, rffi -from pypy.rpython.llinterp import LLFrame -from pypy.rpython.test import test_llinterp -from pypy.rpython.test.test_llinterp import get_interpreter, clear_tcache -from pypy.translator.stm.inevitable import insert_turn_inevitable +from rpython.rtyper.lltypesystem import lltype, llmemory, rffi +from rpython.rtyper.llinterp import LLFrame +from rpython.rtyper.test import test_llinterp +from rpython.rtyper.test.test_llinterp import get_interpreter, clear_tcache +from rpython.translator.stm.inevitable import insert_turn_inevitable from pypy.conftest import option diff --git a/rpython/translator/stm/test/test_jitdriver.py b/rpython/translator/stm/test/test_jitdriver.py --- a/rpython/translator/stm/test/test_jitdriver.py +++ b/rpython/translator/stm/test/test_jitdriver.py @@ -1,6 +1,6 @@ -from pypy.rpython.lltypesystem import lltype, rffi -from pypy.translator.stm.test.transform2_support import BaseTestTransform -from pypy.rlib.jit import JitDriver +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.translator.stm.test.transform2_support import BaseTestTransform +from rpython.rlib.jit import JitDriver class TestJitDriver(BaseTestTransform): diff --git a/rpython/translator/stm/test/test_writebarrier.py b/rpython/translator/stm/test/test_writebarrier.py --- a/rpython/translator/stm/test/test_writebarrier.py +++ b/rpython/translator/stm/test/test_writebarrier.py @@ -1,5 +1,5 @@ -from pypy.rpython.lltypesystem import lltype, rffi -from pypy.translator.stm.test.transform2_support import BaseTestTransform +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.translator.stm.test.transform2_support import BaseTestTransform class TestTransform(BaseTestTransform): diff --git a/rpython/translator/stm/test/test_ztranslated.py b/rpython/translator/stm/test/test_ztranslated.py --- a/rpython/translator/stm/test/test_ztranslated.py +++ b/rpython/translator/stm/test/test_ztranslated.py @@ -1,8 +1,8 @@ import py -from pypy.rlib import rstm, rgc -from pypy.translator.stm.test.support import NoGcCompiledSTMTests -from pypy.translator.stm.test.support import CompiledSTMTests -from pypy.translator.stm.test import targetdemo2 +from rpython.rlib import rstm, rgc +from rpython.translator.stm.test.support import NoGcCompiledSTMTests +from rpython.translator.stm.test.support import CompiledSTMTests +from rpython.translator.stm.test import targetdemo2 class TestNoGcSTMTranslated(NoGcCompiledSTMTests): @@ -81,7 +81,7 @@ pass def check(foobar, retry_counter): return 0 # do nothing - from pypy.rpython.lltypesystem import lltype + from rpython.rtyper.lltypesystem import lltype R = lltype.GcStruct('R', ('x', lltype.Signed)) S1 = lltype.Struct('S1', ('r', lltype.Ptr(R))) s1 = lltype.malloc(S1, immortal=True, flavor='raw') diff --git a/rpython/translator/stm/test/transform2_support.py b/rpython/translator/stm/test/transform2_support.py --- a/rpython/translator/stm/test/transform2_support.py +++ b/rpython/translator/stm/test/transform2_support.py @@ -1,8 +1,8 @@ -from pypy.rpython.lltypesystem import lltype, opimpl -from pypy.rpython.llinterp import LLFrame -from pypy.rpython.test.test_llinterp import get_interpreter, clear_tcache -from pypy.translator.stm.transform2 import STMTransformer -from pypy.translator.stm.writebarrier import MORE_PRECISE_CATEGORIES +from rpython.rtyper.lltypesystem import lltype, opimpl +from rpython.rtyper.llinterp import LLFrame +from rpython.rtyper.test.test_llinterp import get_interpreter, clear_tcache +from rpython.translator.stm.transform2 import STMTransformer +from rpython.translator.stm.writebarrier import MORE_PRECISE_CATEGORIES from pypy.conftest import option diff --git a/rpython/translator/stm/transform2.py b/rpython/translator/stm/transform2.py --- a/rpython/translator/stm/transform2.py +++ b/rpython/translator/stm/transform2.py @@ -14,8 +14,8 @@ self.translator.stm_transformation_applied = True def transform_write_barrier(self): - from pypy.translator.backendopt.writeanalyze import WriteAnalyzer - from pypy.translator.stm.writebarrier import insert_stm_barrier + from rpython.translator.backendopt.writeanalyze import WriteAnalyzer + from rpython.translator.stm.writebarrier import insert_stm_barrier # self.write_analyzer = WriteAnalyzer(self.translator) for graph in self.translator.graphs: @@ -23,21 +23,21 @@ del self.write_analyzer def transform_turn_inevitable(self): - from pypy.translator.stm.inevitable import insert_turn_inevitable + from rpython.translator.stm.inevitable import insert_turn_inevitable # for graph in self.translator.graphs: insert_turn_inevitable(graph) def transform_jit_driver(self): - from pypy.translator.stm.jitdriver import reorganize_around_jit_driver + from rpython.translator.stm.jitdriver import reorganize_around_jit_driver # for graph in self.translator.graphs: reorganize_around_jit_driver(self, graph) def start_log(self): - from pypy.translator.c.support import log + from rpython.translator.c.support import log log.info("Software Transactional Memory transformation") def print_logs(self): - from pypy.translator.c.support import log + from rpython.translator.c.support import log log.info("Software Transactional Memory transformation applied") diff --git a/rpython/translator/stm/writebarrier.py b/rpython/translator/stm/writebarrier.py --- a/rpython/translator/stm/writebarrier.py +++ b/rpython/translator/stm/writebarrier.py @@ -1,7 +1,7 @@ -from pypy.objspace.flow.model import SpaceOperation, Constant, Variable -from pypy.translator.unsimplify import varoftype, insert_empty_block -from pypy.rpython.lltypesystem import lltype -from pypy.translator.backendopt.writeanalyze import top_set +from rpython.flowspace.model import SpaceOperation, Constant, Variable +from rpython.translator.unsimplify import varoftype, insert_empty_block +from rpython.rtyper.lltypesystem import lltype +from rpython.translator.backendopt.writeanalyze import top_set MALLOCS = set([ From noreply at buildbot.pypy.org Thu Jan 31 15:19:53 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 31 Jan 2013 15:19:53 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: More import fixes Message-ID: <20130131141953.1703E1C009B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r60775:7036ba94afef Date: 2013-01-31 15:18 +0100 http://bitbucket.org/pypy/pypy/changeset/7036ba94afef/ Log: More import fixes diff --git a/rpython/translator/stm/jitdriver.py b/rpython/translator/stm/jitdriver.py --- a/rpython/translator/stm/jitdriver.py +++ b/rpython/translator/stm/jitdriver.py @@ -7,7 +7,7 @@ from rpython.rtyper.annlowlevel import (MixLevelHelperAnnotator, cast_base_ptr_to_instance) from rpython.rlib import rstm -from pypy.tool.sourcetools import compile2 +from rpython.tool.sourcetools import compile2 def find_jit_merge_point(graph): diff --git a/rpython/translator/stm/stmgcintf.py b/rpython/translator/stm/stmgcintf.py --- a/rpython/translator/stm/stmgcintf.py +++ b/rpython/translator/stm/stmgcintf.py @@ -1,12 +1,11 @@ -import py -from pypy.tool.autopath import pypydir +import py, os from rpython.rtyper.lltypesystem import lltype, llmemory, rffi from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.rlib.rarithmetic import LONG_BIT +from rpython.conftest import cdir as cdir2 -cdir = py.path.local(pypydir) / 'translator' / 'stm' -cdir2 = py.path.local(pypydir) / 'translator' / 'c' +cdir = os.path.abspath(os.path.join(cdir2, '..', 'stm')) eci = ExternalCompilationInfo( include_dirs = [cdir, cdir2], diff --git a/rpython/translator/stm/test/targetdemo2.py b/rpython/translator/stm/test/targetdemo2.py --- a/rpython/translator/stm/test/targetdemo2.py +++ b/rpython/translator/stm/test/targetdemo2.py @@ -1,5 +1,5 @@ import time -from pypy.module.thread import ll_thread +from rpython.rlib import rthread from rpython.rlib import rstm, jit from rpython.rlib.objectmodel import invoke_around_extcall, we_are_translated from rpython.rlib.objectmodel import compute_identity_hash @@ -76,7 +76,7 @@ def __init__(self, i): self.index = i - self.finished_lock = ll_thread.allocate_lock() + self.finished_lock = rthread.allocate_lock() self.finished_lock.acquire(True) def run(self): @@ -198,7 +198,7 @@ @staticmethod def setup(): if bootstrapper.lock is None: - bootstrapper.lock = ll_thread.allocate_lock() + bootstrapper.lock = rthread.allocate_lock() @staticmethod def reinit(): @@ -213,14 +213,14 @@ # Note that when this runs, we already hold the GIL. This is ensured # by rffi's callback mecanism: we are a callback for the # c_thread_start() external function. - ll_thread.gc_thread_start() + rthread.gc_thread_start() args = bootstrapper.args bootstrapper.release() # run! try: args.run() finally: - ll_thread.gc_thread_die() + rthread.gc_thread_die() @staticmethod def acquire(args): @@ -249,8 +249,8 @@ def start_thread(args): bootstrapper.acquire(args) try: - ll_thread.gc_thread_prepare() # (this has no effect any more) - ident = ll_thread.start_new_thread(bootstrapper.bootstrap, ()) + rthread.gc_thread_prepare() # (this has no effect any more) + ident = rthread.start_new_thread(bootstrapper.bootstrap, ()) except Exception, e: bootstrapper.release() # normally called by the new thread raise diff --git a/rpython/translator/stm/test/test_inevitable.py b/rpython/translator/stm/test/test_inevitable.py --- a/rpython/translator/stm/test/test_inevitable.py +++ b/rpython/translator/stm/test/test_inevitable.py @@ -3,7 +3,7 @@ from rpython.rtyper.test import test_llinterp from rpython.rtyper.test.test_llinterp import get_interpreter, clear_tcache from rpython.translator.stm.inevitable import insert_turn_inevitable -from pypy.conftest import option +from rpython.conftest import option class LLSTMInevFrame(LLFrame): diff --git a/rpython/translator/stm/test/test_stmgcintf.py b/rpython/translator/stm/test/test_stmgcintf.py --- a/rpython/translator/stm/test/test_stmgcintf.py +++ b/rpython/translator/stm/test/test_stmgcintf.py @@ -1,12 +1,11 @@ import os -from pypy.tool import autopath -from pypy.tool.udir import udir +from rpython.tool.udir import udir def test_all(): executable = str(udir.join('test_stmgcintf')) prevdir = os.getcwd() - thisdir = os.path.join(autopath.pypydir, 'translator', 'stm', 'test') + thisdir = os.path.dirname(__file__) try: os.chdir(thisdir) exitcode = os.system( diff --git a/rpython/translator/stm/test/transform2_support.py b/rpython/translator/stm/test/transform2_support.py --- a/rpython/translator/stm/test/transform2_support.py +++ b/rpython/translator/stm/test/transform2_support.py @@ -3,7 +3,7 @@ from rpython.rtyper.test.test_llinterp import get_interpreter, clear_tcache from rpython.translator.stm.transform2 import STMTransformer from rpython.translator.stm.writebarrier import MORE_PRECISE_CATEGORIES -from pypy.conftest import option +from rpython.conftest import option class _stmptr(lltype._ptr): From noreply at buildbot.pypy.org Thu Jan 31 17:38:36 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 31 Jan 2013 17:38:36 +0100 (CET) Subject: [pypy-commit] pypy default: Test for acquire_timed() returning RPY_LOCK_INTR. Message-ID: <20130131163836.C9E3F1C05CA@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r60776:e80549fefb75 Date: 2013-01-31 17:38 +0100 http://bitbucket.org/pypy/pypy/changeset/e80549fefb75/ Log: Test for acquire_timed() returning RPY_LOCK_INTR. diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py --- a/rpython/rlib/rthread.py +++ b/rpython/rlib/rthread.py @@ -126,10 +126,11 @@ return bool(res) def acquire_timed(self, timeout): - "timeout is in microseconds." + """Timeout is in microseconds. Returns 0 in case of failure, + 1 in case it works, 2 if interrupted by a signal.""" res = c_thread_acquirelock_timed(self._lock, timeout, 1) res = rffi.cast(lltype.Signed, res) - return bool(res) + return res def release(self): # Sanity check: the lock must be locked diff --git a/rpython/rlib/test/test_rthread.py b/rpython/rlib/test/test_rthread.py --- a/rpython/rlib/test/test_rthread.py +++ b/rpython/rlib/test/test_rthread.py @@ -159,17 +159,46 @@ l = allocate_lock() l.acquire(True) t1 = time.time() - ok = l.acquire_timed(1000000) + ok = l.acquire_timed(1000001) t2 = time.time() delay = t2 - t1 - if ok: + if ok == 0: # RPY_LOCK_FAILURE + return -delay + elif ok == 2: # RPY_LOCK_INTR return delay - else: - return -delay + else: # RPY_LOCK_ACQUIRED + return 0.0 fn = self.getcompiled(f, []) res = fn() assert res < -1.0 + def test_acquire_timed_alarm(self): + import sys + if not sys.platform.startswith('linux'): + py.test.skip("skipped on non-linux") + import time + from rpython.rlib import rsignal + def f(): + l = allocate_lock() + l.acquire(True) + # + rsignal.pypysig_setflag(rsignal.SIGALRM) + rsignal.c_alarm(1) + # + t1 = time.time() + ok = l.acquire_timed(2500000) + t2 = time.time() + delay = t2 - t1 + if ok == 0: # RPY_LOCK_FAILURE + return -delay + elif ok == 2: # RPY_LOCK_INTR + return delay + else: # RPY_LOCK_ACQUIRED + return 0.0 + fn = self.getcompiled(f, []) + res = fn() + assert res >= 0.95 + #class TestRunDirectly(AbstractThreadTests): # def getcompiled(self, f, argtypes): # return f diff --git a/rpython/translator/c/src/thread.h b/rpython/translator/c/src/thread.h --- a/rpython/translator/c/src/thread.h +++ b/rpython/translator/c/src/thread.h @@ -7,7 +7,7 @@ typedef enum RPyLockStatus { RPY_LOCK_FAILURE = 0, RPY_LOCK_ACQUIRED = 1, - RPY_LOCK_INTR + RPY_LOCK_INTR = 2 } RPyLockStatus; #ifdef _WIN32 From noreply at buildbot.pypy.org Thu Jan 31 17:40:08 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 31 Jan 2013 17:40:08 +0100 (CET) Subject: [pypy-commit] pypy py3k: Add an XXX about acquire_timed(), which was fixed (in 'default') Message-ID: <20130131164008.E58931C009B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: py3k Changeset: r60777:05832d0ee006 Date: 2013-01-31 17:39 +0100 http://bitbucket.org/pypy/pypy/changeset/05832d0ee006/ Log: Add an XXX about acquire_timed(), which was fixed (in 'default') to return three possible results instead of two. diff --git a/pypy/module/thread/os_lock.py b/pypy/module/thread/os_lock.py --- a/pypy/module/thread/os_lock.py +++ b/pypy/module/thread/os_lock.py @@ -72,6 +72,7 @@ microseconds = parse_acquire_args(space, blocking, timeout) mylock = self.lock result = mylock.acquire_timed(microseconds) + result = (result == 1) # XXX handle RPY_LOCK_INTR (see e80549fefb75) return space.newbool(result) def descr_lock_release(self, space): @@ -184,6 +185,7 @@ if not blocking: return space.w_False r = self.lock.acquire_timed(microseconds) + r = (r == 1) # XXX handle RPY_LOCK_INTR (see e80549fefb75) if r: assert self.rlock_count == 0 self.rlock_owner = tid From noreply at buildbot.pypy.org Thu Jan 31 17:52:42 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 31 Jan 2013 17:52:42 +0100 (CET) Subject: [pypy-commit] pypy default: try to fix _testcapi/_ctypes_test for win32 Message-ID: <20130131165242.CCDE71C0246@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60778:5c18387c65e8 Date: 2013-01-31 11:49 -0500 http://bitbucket.org/pypy/pypy/changeset/5c18387c65e8/ Log: try to fix _testcapi/_ctypes_test for win32 diff --git a/lib_pypy/_ctypes_test.py b/lib_pypy/_ctypes_test.py --- a/lib_pypy/_ctypes_test.py +++ b/lib_pypy/_ctypes_test.py @@ -1,6 +1,5 @@ import os, sys import tempfile -import gc def compile_shared(): """Compile '_ctypes_test.c' into an extension module, and import it @@ -19,7 +18,7 @@ if sys.platform == 'win32': ccflags = ['-D_CRT_SECURE_NO_WARNINGS'] else: - ccflags = ['-fPIC'] + ccflags = ['-fPIC', '-Wimplicit-function-declaration'] res = compiler.compile([os.path.join(thisdir, '_ctypes_test.c')], include_dirs=[include_dir], extra_preargs=ccflags) @@ -35,10 +34,10 @@ library = os.path.join(thisdir, '..', 'include', 'python27') if not os.path.exists(library + '.lib'): # For a local translation - library = os.path.join(thisdir, '..', 'pypy', 'translator', - 'goal', 'libpypy-c') + library = os.path.join(thisdir, '..', 'pypy', 'goal', 'libpypy-c') libraries = [library, 'oleaut32'] - extra_ldargs = ['/MANIFEST'] # needed for VC10 + extra_ldargs = ['/MANIFEST', # needed for VC10 + '/EXPORT:init_ctypes_test'] else: libraries = [] extra_ldargs = [] diff --git a/lib_pypy/_testcapi.py b/lib_pypy/_testcapi.py --- a/lib_pypy/_testcapi.py +++ b/lib_pypy/_testcapi.py @@ -34,8 +34,7 @@ library = os.path.join(thisdir, '..', 'include', 'python27') if not os.path.exists(library + '.lib'): # For a local translation - library = os.path.join(thisdir, '..', 'pypy', 'translator', - 'goal', 'libpypy-c') + library = os.path.join(thisdir, '..', 'pypy', 'goal', 'libpypy-c') libraries = [library, 'oleaut32'] extra_ldargs = ['/MANIFEST', # needed for VC10 '/EXPORT:init_testcapi'] From noreply at buildbot.pypy.org Thu Jan 31 18:04:57 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 31 Jan 2013 18:04:57 +0100 (CET) Subject: [pypy-commit] pypy default: fix test_fileio.test_open_directory for win32 Message-ID: <20130131170457.566E91C05CA@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60779:80bb4b831b9f Date: 2013-01-31 12:04 -0500 http://bitbucket.org/pypy/pypy/changeset/80bb4b831b9f/ Log: fix test_fileio.test_open_directory for win32 diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -43,9 +43,10 @@ import _io import os raises(IOError, _io.FileIO, self.tmpdir, "rb") - fd = os.open(self.tmpdir, os.O_RDONLY) - raises(IOError, _io.FileIO, fd, "rb") - os.close(fd) + if os.name != 'nt': + fd = os.open(self.tmpdir, os.O_RDONLY) + raises(IOError, _io.FileIO, fd, "rb") + os.close(fd) def test_readline(self): import _io From noreply at buildbot.pypy.org Thu Jan 31 18:10:26 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 31 Jan 2013 18:10:26 +0100 (CET) Subject: [pypy-commit] pypy default: fix test_rabspath_relative for darwin Message-ID: <20130131171026.916691C05CA@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60780:dfdc0e0251d6 Date: 2013-01-31 12:10 -0500 http://bitbucket.org/pypy/pypy/changeset/dfdc0e0251d6/ Log: fix test_rabspath_relative for darwin diff --git a/rpython/rlib/test/test_rpath.py b/rpython/rlib/test/test_rpath.py --- a/rpython/rlib/test/test_rpath.py +++ b/rpython/rlib/test/test_rpath.py @@ -6,7 +6,7 @@ def test_rabspath_relative(tmpdir): tmpdir.chdir() - assert rpath.rabspath('foo') == tmpdir.join('foo') + assert rpath.rabspath('foo') == os.path.realpath(tmpdir.join('foo')) @py.test.mark.skipif("IS_WINDOWS") def test_rabspath_absolute_posix(): From noreply at buildbot.pypy.org Thu Jan 31 18:14:54 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 31 Jan 2013 18:14:54 +0100 (CET) Subject: [pypy-commit] pypy default: fix again Message-ID: <20130131171454.730441C1306@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: Changeset: r60781:8ae312cf9330 Date: 2013-01-31 12:14 -0500 http://bitbucket.org/pypy/pypy/changeset/8ae312cf9330/ Log: fix again diff --git a/rpython/rlib/test/test_rpath.py b/rpython/rlib/test/test_rpath.py --- a/rpython/rlib/test/test_rpath.py +++ b/rpython/rlib/test/test_rpath.py @@ -6,7 +6,7 @@ def test_rabspath_relative(tmpdir): tmpdir.chdir() - assert rpath.rabspath('foo') == os.path.realpath(tmpdir.join('foo')) + assert rpath.rabspath('foo') == os.path.realpath(str(tmpdir.join('foo'))) @py.test.mark.skipif("IS_WINDOWS") def test_rabspath_absolute_posix(): From noreply at buildbot.pypy.org Thu Jan 31 18:19:47 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 31 Jan 2013 18:19:47 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: hg merge default Message-ID: <20130131171947.6AC631C1306@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r60782:49a1afc2949f Date: 2013-01-31 17:40 +0100 http://bitbucket.org/pypy/pypy/changeset/49a1afc2949f/ Log: hg merge default diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py --- a/rpython/rlib/rthread.py +++ b/rpython/rlib/rthread.py @@ -130,10 +130,11 @@ return bool(res) def acquire_timed(self, timeout): - "timeout is in microseconds." + """Timeout is in microseconds. Returns 0 in case of failure, + 1 in case it works, 2 if interrupted by a signal.""" res = c_thread_acquirelock_timed(self._lock, timeout, 1) res = rffi.cast(lltype.Signed, res) - return bool(res) + return res def release(self): # Sanity check: the lock must be locked diff --git a/rpython/rlib/test/test_rthread.py b/rpython/rlib/test/test_rthread.py --- a/rpython/rlib/test/test_rthread.py +++ b/rpython/rlib/test/test_rthread.py @@ -159,17 +159,46 @@ l = allocate_lock() l.acquire(True) t1 = time.time() - ok = l.acquire_timed(1000000) + ok = l.acquire_timed(1000001) t2 = time.time() delay = t2 - t1 - if ok: + if ok == 0: # RPY_LOCK_FAILURE + return -delay + elif ok == 2: # RPY_LOCK_INTR return delay - else: - return -delay + else: # RPY_LOCK_ACQUIRED + return 0.0 fn = self.getcompiled(f, []) res = fn() assert res < -1.0 + def test_acquire_timed_alarm(self): + import sys + if not sys.platform.startswith('linux'): + py.test.skip("skipped on non-linux") + import time + from rpython.rlib import rsignal + def f(): + l = allocate_lock() + l.acquire(True) + # + rsignal.pypysig_setflag(rsignal.SIGALRM) + rsignal.c_alarm(1) + # + t1 = time.time() + ok = l.acquire_timed(2500000) + t2 = time.time() + delay = t2 - t1 + if ok == 0: # RPY_LOCK_FAILURE + return -delay + elif ok == 2: # RPY_LOCK_INTR + return delay + else: # RPY_LOCK_ACQUIRED + return 0.0 + fn = self.getcompiled(f, []) + res = fn() + assert res >= 0.95 + #class TestRunDirectly(AbstractThreadTests): # def getcompiled(self, f, argtypes): # return f diff --git a/rpython/translator/c/src/thread.h b/rpython/translator/c/src/thread.h --- a/rpython/translator/c/src/thread.h +++ b/rpython/translator/c/src/thread.h @@ -7,7 +7,7 @@ typedef enum RPyLockStatus { RPY_LOCK_FAILURE = 0, RPY_LOCK_ACQUIRED = 1, - RPY_LOCK_INTR + RPY_LOCK_INTR = 2 } RPyLockStatus; #ifdef _WIN32 From noreply at buildbot.pypy.org Thu Jan 31 18:19:48 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 31 Jan 2013 18:19:48 +0100 (CET) Subject: [pypy-commit] pypy default: Fix the docstring (bogus on CPython too, reported). Test the real Message-ID: <20130131171948.95AFA1C1306@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r60783:95de7278a36c Date: 2013-01-31 17:58 +0100 http://bitbucket.org/pypy/pypy/changeset/95de7278a36c/ Log: Fix the docstring (bogus on CPython too, reported). Test the real behavior. diff --git a/pypy/module/thread/os_lock.py b/pypy/module/thread/os_lock.py --- a/pypy/module/thread/os_lock.py +++ b/pypy/module/thread/os_lock.py @@ -38,10 +38,10 @@ @unwrap_spec(waitflag=int) def descr_lock_acquire(self, space, waitflag=1): - """Lock the lock. Without argument, this blocks if the lock is already -locked (even by the same thread), waiting for another thread to release -the lock, and return None once the lock is acquired. -With an argument, this will only block if the argument is true, + """Lock the lock. With the default argument of True, this blocks +if the lock is already locked (even by the same thread), waiting for +another thread to release the lock, and returns True once the lock is +acquired. With an argument of False, this will always return immediately and the return value reflects whether the lock is acquired. The blocking operation is not interruptible.""" mylock = self.lock @@ -113,4 +113,4 @@ def allocate_lock(space): """Create a new lock object. (allocate() is an obsolete synonym.) See LockType.__doc__ for information about locks.""" - return space.wrap(Lock(space)) \ No newline at end of file + return space.wrap(Lock(space)) diff --git a/pypy/module/thread/test/test_lock.py b/pypy/module/thread/test/test_lock.py --- a/pypy/module/thread/test/test_lock.py +++ b/pypy/module/thread/test/test_lock.py @@ -12,7 +12,10 @@ assert lock.locked() is False raises(thread.error, lock.release) assert lock.locked() is False - lock.acquire() + r = lock.acquire() + assert r is True + r = lock.acquire(False) + assert r is False assert lock.locked() is True lock.release() assert lock.locked() is False From noreply at buildbot.pypy.org Thu Jan 31 18:19:49 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 31 Jan 2013 18:19:49 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: A PyPy extension: add to thread locks a method acquire_interruptible(). Message-ID: <20130131171949.C84F21C1306@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r60784:b2f107c5c3c5 Date: 2013-01-31 18:14 +0100 http://bitbucket.org/pypy/pypy/changeset/b2f107c5c3c5/ Log: A PyPy extension: add to thread locks a method acquire_interruptible(). This is like acquire() but handles signals, similarly to Python 3's acquire() method. diff --git a/pypy/module/thread/os_lock.py b/pypy/module/thread/os_lock.py --- a/pypy/module/thread/os_lock.py +++ b/pypy/module/thread/os_lock.py @@ -2,6 +2,7 @@ Python locks, based on true threading locks provided by the OS. """ +import sys from rpython.rlib import rthread as thread from pypy.module.thread.error import wrap_thread_error from pypy.interpreter.baseobjspace import Wrappable @@ -51,6 +52,23 @@ result = mylock.acquire(bool(waitflag)) return space.newbool(result) + def descr_lock_acquire_interruptible(self, space): + """Lock the lock. Unlike acquire(), this is always blocking +but may be interrupted: signal handlers are still called, and may +raise (e.g. a Ctrl-C will correctly raise KeyboardInterrupt). + +This is an extension only available on PyPy.""" + mylock = self.lock + while True: + result = mylock.acquire_timed(-1) + if result == 1: # RPY_LOCK_ACQUIRED + return + assert result == 2 # RPY_LOCK_INTR + space.getexecutioncontext().checksignals() + # then retry, if the signal handler did not raise + assert sys.platform != 'win32', ( + "acquire_interruptible: fix acquire_timed() on Windows") + def descr_lock_release(self, space): """Release the lock, allowing another thread that is blocked waiting for the lock to acquire the lock. The lock must be in the locked state, @@ -83,6 +101,7 @@ self.descr_lock_release(self.space) descr_acquire = interp2app(Lock.descr_lock_acquire) +descr_acquire_interruptible = interp2app(Lock.descr_lock_acquire_interruptible) descr_release = interp2app(Lock.descr_lock_release) descr_locked = interp2app(Lock.descr_lock_locked) descr__enter__ = interp2app(Lock.descr__enter__) @@ -102,6 +121,7 @@ unlock it. A thread attempting to lock a lock that it has already locked will block until another thread unlocks it. Deadlocks may ensue.""", acquire = descr_acquire, + acquire_interruptible = descr_acquire_interruptible, release = descr_release, locked = descr_locked, __enter__ = descr__enter__, diff --git a/pypy/module/thread/test/test_lock.py b/pypy/module/thread/test/test_lock.py --- a/pypy/module/thread/test/test_lock.py +++ b/pypy/module/thread/test/test_lock.py @@ -46,6 +46,34 @@ assert feedback == [42] assert lock.locked() is False + def test_acquire_interruptible(self): + import thread, signal, posix + ticks = [] + def tick(*args): + ticks.append(1) + if len(ticks) == 3: + raise OverflowError + prev_handler = signal.signal(signal.SIGUSR1, tick) + # + lock = thread.allocate_lock() + lock.acquire() + def f(): + self.busywait(0.25) + posix.kill(posix.getpid(), signal.SIGUSR1) + self.busywait(0.25) + posix.kill(posix.getpid(), signal.SIGUSR1) + self.busywait(0.25) + posix.kill(posix.getpid(), signal.SIGUSR1) + thread.start_new_thread(f, ()) + try: + lock.acquire_interruptible() + raise AssertionError("should not reach here") + except OverflowError: + pass + assert ticks == [1, 1, 1] + signal.signal(signal.SIGUSR1, prev_handler) + + def test_compile_lock(): from rpython.rlib import rgc from rpython.rlib.rthread import allocate_lock From noreply at buildbot.pypy.org Thu Jan 31 18:19:50 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 31 Jan 2013 18:19:50 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: hg backout 009b40655c5f: remove the hack based on signals. Message-ID: <20130131171950.EB22D1C1306@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r60785:1a3d8664bdde Date: 2013-01-31 18:16 +0100 http://bitbucket.org/pypy/pypy/changeset/1a3d8664bdde/ Log: hg backout 009b40655c5f: remove the hack based on signals. diff --git a/lib_pypy/transaction.py b/lib_pypy/transaction.py --- a/lib_pypy/transaction.py +++ b/lib_pypy/transaction.py @@ -15,12 +15,6 @@ import sys, thread, collections try: - import signal, posix - from signal import SIGUSR2 as SIG_DONE -except ImportError: - signal = None - -try: from thread import atomic except ImportError: # Not a STM-enabled PyPy. We can still provide a version of 'atomic' @@ -115,29 +109,12 @@ self.in_transaction = True def run(self): - self.use_signals = hasattr(signal, 'pause') - if self.use_signals: - try: - self.signalled = 0 - def signal_handler(*args): - self.signalled = 1 - prev_handler = signal.signal(SIG_DONE, signal_handler) - except ValueError: - self.use_signals = False # start the N threads for i in range(self.num_threads): thread.start_new_thread(self._run_thread, ()) # now wait. When we manage to acquire the following lock, then # we are finished. - if self.use_signals: - try: - while not self.signalled: - signal.pause() - finally: - self.use_signals = False - signal.signal(SIG_DONE, prev_handler) - else: - self.lock_if_released_then_finished.acquire() + self.lock_if_released_then_finished.acquire() def teardown(self): self.in_transaction = False @@ -216,9 +193,6 @@ self.lock_mutex.release() if last_one_to_leave: self.lock_if_released_then_finished.release() - with atomic: - if self.use_signals: - posix.kill(posix.getpid(), SIG_DONE) raise _Done @staticmethod From noreply at buildbot.pypy.org Thu Jan 31 18:19:52 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 31 Jan 2013 18:19:52 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: A clean replacement for 009b40655c5f. Message-ID: <20130131171952.1BAEB1C1306@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r60786:0c978d2c5fce Date: 2013-01-31 18:18 +0100 http://bitbucket.org/pypy/pypy/changeset/0c978d2c5fce/ Log: A clean replacement for 009b40655c5f. diff --git a/lib_pypy/transaction.py b/lib_pypy/transaction.py --- a/lib_pypy/transaction.py +++ b/lib_pypy/transaction.py @@ -114,7 +114,11 @@ thread.start_new_thread(self._run_thread, ()) # now wait. When we manage to acquire the following lock, then # we are finished. - self.lock_if_released_then_finished.acquire() + try: + acquire = self.lock_if_released_then_finished.acquire_interruptible + except AttributeError: # not on pypy-stm + acquire = self.lock_if_released_then_finished.acquire + acquire() def teardown(self): self.in_transaction = False From noreply at buildbot.pypy.org Thu Jan 31 18:19:53 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 31 Jan 2013 18:19:53 +0100 (CET) Subject: [pypy-commit] pypy default: merge heads Message-ID: <20130131171953.3F3D41C1306@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r60787:d65584f8c928 Date: 2013-01-31 18:19 +0100 http://bitbucket.org/pypy/pypy/changeset/d65584f8c928/ Log: merge heads diff --git a/lib_pypy/_ctypes_test.py b/lib_pypy/_ctypes_test.py --- a/lib_pypy/_ctypes_test.py +++ b/lib_pypy/_ctypes_test.py @@ -1,6 +1,5 @@ import os, sys import tempfile -import gc def compile_shared(): """Compile '_ctypes_test.c' into an extension module, and import it @@ -19,7 +18,7 @@ if sys.platform == 'win32': ccflags = ['-D_CRT_SECURE_NO_WARNINGS'] else: - ccflags = ['-fPIC'] + ccflags = ['-fPIC', '-Wimplicit-function-declaration'] res = compiler.compile([os.path.join(thisdir, '_ctypes_test.c')], include_dirs=[include_dir], extra_preargs=ccflags) @@ -35,10 +34,10 @@ library = os.path.join(thisdir, '..', 'include', 'python27') if not os.path.exists(library + '.lib'): # For a local translation - library = os.path.join(thisdir, '..', 'pypy', 'translator', - 'goal', 'libpypy-c') + library = os.path.join(thisdir, '..', 'pypy', 'goal', 'libpypy-c') libraries = [library, 'oleaut32'] - extra_ldargs = ['/MANIFEST'] # needed for VC10 + extra_ldargs = ['/MANIFEST', # needed for VC10 + '/EXPORT:init_ctypes_test'] else: libraries = [] extra_ldargs = [] diff --git a/lib_pypy/_testcapi.py b/lib_pypy/_testcapi.py --- a/lib_pypy/_testcapi.py +++ b/lib_pypy/_testcapi.py @@ -34,8 +34,7 @@ library = os.path.join(thisdir, '..', 'include', 'python27') if not os.path.exists(library + '.lib'): # For a local translation - library = os.path.join(thisdir, '..', 'pypy', 'translator', - 'goal', 'libpypy-c') + library = os.path.join(thisdir, '..', 'pypy', 'goal', 'libpypy-c') libraries = [library, 'oleaut32'] extra_ldargs = ['/MANIFEST', # needed for VC10 '/EXPORT:init_testcapi'] diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -43,9 +43,10 @@ import _io import os raises(IOError, _io.FileIO, self.tmpdir, "rb") - fd = os.open(self.tmpdir, os.O_RDONLY) - raises(IOError, _io.FileIO, fd, "rb") - os.close(fd) + if os.name != 'nt': + fd = os.open(self.tmpdir, os.O_RDONLY) + raises(IOError, _io.FileIO, fd, "rb") + os.close(fd) def test_readline(self): import _io diff --git a/rpython/rlib/test/test_rpath.py b/rpython/rlib/test/test_rpath.py --- a/rpython/rlib/test/test_rpath.py +++ b/rpython/rlib/test/test_rpath.py @@ -6,7 +6,7 @@ def test_rabspath_relative(tmpdir): tmpdir.chdir() - assert rpath.rabspath('foo') == tmpdir.join('foo') + assert rpath.rabspath('foo') == os.path.realpath(str(tmpdir.join('foo'))) @py.test.mark.skipif("IS_WINDOWS") def test_rabspath_absolute_posix(): From noreply at buildbot.pypy.org Thu Jan 31 18:20:36 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 31 Jan 2013 18:20:36 +0100 (CET) Subject: [pypy-commit] pypy stm-thread-2: hg merge default Message-ID: <20130131172036.AFC9F1C1306@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: stm-thread-2 Changeset: r60788:308102001cb3 Date: 2013-01-31 18:20 +0100 http://bitbucket.org/pypy/pypy/changeset/308102001cb3/ Log: hg merge default diff --git a/lib_pypy/_ctypes_test.py b/lib_pypy/_ctypes_test.py --- a/lib_pypy/_ctypes_test.py +++ b/lib_pypy/_ctypes_test.py @@ -1,6 +1,5 @@ import os, sys import tempfile -import gc def compile_shared(): """Compile '_ctypes_test.c' into an extension module, and import it @@ -19,7 +18,7 @@ if sys.platform == 'win32': ccflags = ['-D_CRT_SECURE_NO_WARNINGS'] else: - ccflags = ['-fPIC'] + ccflags = ['-fPIC', '-Wimplicit-function-declaration'] res = compiler.compile([os.path.join(thisdir, '_ctypes_test.c')], include_dirs=[include_dir], extra_preargs=ccflags) @@ -35,10 +34,10 @@ library = os.path.join(thisdir, '..', 'include', 'python27') if not os.path.exists(library + '.lib'): # For a local translation - library = os.path.join(thisdir, '..', 'pypy', 'translator', - 'goal', 'libpypy-c') + library = os.path.join(thisdir, '..', 'pypy', 'goal', 'libpypy-c') libraries = [library, 'oleaut32'] - extra_ldargs = ['/MANIFEST'] # needed for VC10 + extra_ldargs = ['/MANIFEST', # needed for VC10 + '/EXPORT:init_ctypes_test'] else: libraries = [] extra_ldargs = [] diff --git a/lib_pypy/_testcapi.py b/lib_pypy/_testcapi.py --- a/lib_pypy/_testcapi.py +++ b/lib_pypy/_testcapi.py @@ -34,8 +34,7 @@ library = os.path.join(thisdir, '..', 'include', 'python27') if not os.path.exists(library + '.lib'): # For a local translation - library = os.path.join(thisdir, '..', 'pypy', 'translator', - 'goal', 'libpypy-c') + library = os.path.join(thisdir, '..', 'pypy', 'goal', 'libpypy-c') libraries = [library, 'oleaut32'] extra_ldargs = ['/MANIFEST', # needed for VC10 '/EXPORT:init_testcapi'] diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py --- a/pypy/module/_io/test/test_fileio.py +++ b/pypy/module/_io/test/test_fileio.py @@ -43,9 +43,10 @@ import _io import os raises(IOError, _io.FileIO, self.tmpdir, "rb") - fd = os.open(self.tmpdir, os.O_RDONLY) - raises(IOError, _io.FileIO, fd, "rb") - os.close(fd) + if os.name != 'nt': + fd = os.open(self.tmpdir, os.O_RDONLY) + raises(IOError, _io.FileIO, fd, "rb") + os.close(fd) def test_readline(self): import _io diff --git a/pypy/module/thread/os_lock.py b/pypy/module/thread/os_lock.py --- a/pypy/module/thread/os_lock.py +++ b/pypy/module/thread/os_lock.py @@ -42,10 +42,10 @@ @unwrap_spec(waitflag=int) def descr_lock_acquire(self, space, waitflag=1): - """Lock the lock. Without argument, this blocks if the lock is already -locked (even by the same thread), waiting for another thread to release -the lock, and return None once the lock is acquired. -With an argument, this will only block if the argument is true, + """Lock the lock. With the default argument of True, this blocks +if the lock is already locked (even by the same thread), waiting for +another thread to release the lock, and returns True once the lock is +acquired. With an argument of False, this will always return immediately and the return value reflects whether the lock is acquired. The blocking operation is not interruptible.""" mylock = self.lock diff --git a/pypy/module/thread/test/test_lock.py b/pypy/module/thread/test/test_lock.py --- a/pypy/module/thread/test/test_lock.py +++ b/pypy/module/thread/test/test_lock.py @@ -12,7 +12,10 @@ assert lock.locked() is False raises(thread.error, lock.release) assert lock.locked() is False - lock.acquire() + r = lock.acquire() + assert r is True + r = lock.acquire(False) + assert r is False assert lock.locked() is True lock.release() assert lock.locked() is False diff --git a/rpython/rlib/test/test_rpath.py b/rpython/rlib/test/test_rpath.py --- a/rpython/rlib/test/test_rpath.py +++ b/rpython/rlib/test/test_rpath.py @@ -6,7 +6,7 @@ def test_rabspath_relative(tmpdir): tmpdir.chdir() - assert rpath.rabspath('foo') == tmpdir.join('foo') + assert rpath.rabspath('foo') == os.path.realpath(str(tmpdir.join('foo'))) @py.test.mark.skipif("IS_WINDOWS") def test_rabspath_absolute_posix(): From noreply at buildbot.pypy.org Thu Jan 31 21:29:48 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 31 Jan 2013 21:29:48 +0100 (CET) Subject: [pypy-commit] pypy cleanup-tests: move lib_pypy/pypy_test tests to test_lib_pypy Message-ID: <20130131202948.A1F211C1306@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: cleanup-tests Changeset: r60789:4dc59684ba7b Date: 2013-01-31 15:29 -0500 http://bitbucket.org/pypy/pypy/changeset/4dc59684ba7b/ Log: move lib_pypy/pypy_test tests to test_lib_pypy diff --git a/lib_pypy/pypy_test/__init__.py b/lib_pypy/pypy_test/__init__.py deleted file mode 100644 --- a/lib_pypy/pypy_test/__init__.py +++ /dev/null @@ -1,1 +0,0 @@ -# diff --git a/lib_pypy/pypy_test/test_collections.py b/lib_pypy/pypy_test/test_collections.py deleted file mode 100644 --- a/lib_pypy/pypy_test/test_collections.py +++ /dev/null @@ -1,53 +0,0 @@ -from __future__ import absolute_import -from lib_pypy import _collections as collections -import py - -class TestDeque: - def test_remove_empty(self): - d = collections.deque([]) - py.test.raises(ValueError, d.remove, 1) - - def test_remove_mutating(self): - class MutatingCmp(object): - def __eq__(self, other): - d.clear() - return True - - d = collections.deque([MutatingCmp()]) - py.test.raises(IndexError, d.remove, 1) - - def test_maxlen(self): - d = collections.deque([], 3) - d.append(1); d.append(2); d.append(3); d.append(4) - assert list(d) == [2, 3, 4] - assert repr(d) == "deque([2, 3, 4], maxlen=3)" - - import pickle - d2 = pickle.loads(pickle.dumps(d)) - assert repr(d2) == "deque([2, 3, 4], maxlen=3)" - - import copy - d3 = copy.copy(d) - assert repr(d3) == "deque([2, 3, 4], maxlen=3)" - - def test_count(self): - d = collections.deque([1, 2, 2, 3, 2]) - assert d.count(2) == 3 - assert d.count(4) == 0 - - def test_reverse(self): - d = collections.deque([1, 2, 2, 3, 2]) - d.reverse() - assert list(d) == [2, 3, 2, 2, 1] - - d = collections.deque(range(100)) - d.reverse() - assert list(d) == range(99, -1, -1) - - def test_subclass_with_kwargs(self): - class SubclassWithKwargs(collections.deque): - def __init__(self, newarg=1): - collections.deque.__init__(self) - - # SF bug #1486663 -- this used to erroneously raise a TypeError - SubclassWithKwargs(newarg=1) diff --git a/lib_pypy/pypy_test/test_datetime.py b/lib_pypy/pypy_test/test_datetime.py deleted file mode 100644 --- a/lib_pypy/pypy_test/test_datetime.py +++ /dev/null @@ -1,59 +0,0 @@ -from __future__ import absolute_import -import py - -from lib_pypy import datetime - -def test_repr(): - print datetime - expected = "datetime.datetime(1, 2, 3, 0, 0)" - assert repr(datetime.datetime(1,2,3)) == expected - -def test_strptime(): - import time, sys - if sys.version_info < (2, 6): - py.test.skip("needs the _strptime module") - - string = '2004-12-01 13:02:47' - format = '%Y-%m-%d %H:%M:%S' - expected = datetime.datetime(*(time.strptime(string, format)[0:6])) - got = datetime.datetime.strptime(string, format) - assert expected == got - -def test_datetime_rounding(): - b = 0.0000001 - a = 0.9999994 - - assert datetime.datetime.utcfromtimestamp(a).microsecond == 999999 - assert datetime.datetime.utcfromtimestamp(a).second == 0 - a += b - assert datetime.datetime.utcfromtimestamp(a).microsecond == 999999 - assert datetime.datetime.utcfromtimestamp(a).second == 0 - a += b - assert datetime.datetime.utcfromtimestamp(a).microsecond == 0 - assert datetime.datetime.utcfromtimestamp(a).second == 1 - -def test_more_datetime_rounding(): - # this test verified on top of CPython 2.7 (using a plain - # "import datetime" above) - expected_results = { - -1000.0: 'datetime.datetime(1970, 1, 1, 0, 43, 20)', - -999.9999996: 'datetime.datetime(1970, 1, 1, 0, 43, 20)', - -999.4: 'datetime.datetime(1970, 1, 1, 0, 43, 20, 600000)', - -999.0000004: 'datetime.datetime(1970, 1, 1, 0, 43, 21)', - -1.0: 'datetime.datetime(1970, 1, 1, 0, 59, 59)', - -0.9999996: 'datetime.datetime(1970, 1, 1, 0, 59, 59)', - -0.4: 'datetime.datetime(1970, 1, 1, 0, 59, 59, 600000)', - -0.0000004: 'datetime.datetime(1970, 1, 1, 1, 0)', - 0.0: 'datetime.datetime(1970, 1, 1, 1, 0)', - 0.0000004: 'datetime.datetime(1970, 1, 1, 1, 0)', - 0.4: 'datetime.datetime(1970, 1, 1, 1, 0, 0, 400000)', - 0.9999996: 'datetime.datetime(1970, 1, 1, 1, 0, 1)', - 1000.0: 'datetime.datetime(1970, 1, 1, 1, 16, 40)', - 1000.0000004: 'datetime.datetime(1970, 1, 1, 1, 16, 40)', - 1000.4: 'datetime.datetime(1970, 1, 1, 1, 16, 40, 400000)', - 1000.9999996: 'datetime.datetime(1970, 1, 1, 1, 16, 41)', - 1293843661.191: 'datetime.datetime(2011, 1, 1, 2, 1, 1, 191000)', - } - for t in sorted(expected_results): - dt = datetime.datetime.fromtimestamp(t) - assert repr(dt) == expected_results[t] diff --git a/lib_pypy/pypy_test/hack___pypy__.py b/pypy/module/test_lib_pypy/hack___pypy__.py rename from lib_pypy/pypy_test/hack___pypy__.py rename to pypy/module/test_lib_pypy/hack___pypy__.py diff --git a/lib_pypy/pypy_test/inprogress_test_binascii_extra.py b/pypy/module/test_lib_pypy/inprogress_test_binascii_extra.py rename from lib_pypy/pypy_test/inprogress_test_binascii_extra.py rename to pypy/module/test_lib_pypy/inprogress_test_binascii_extra.py diff --git a/lib_pypy/pypy_test/no_test_pickle_extra.py b/pypy/module/test_lib_pypy/no_test_pickle_extra.py rename from lib_pypy/pypy_test/no_test_pickle_extra.py rename to pypy/module/test_lib_pypy/no_test_pickle_extra.py diff --git a/pypy/module/test_lib_pypy/test_collections.py b/pypy/module/test_lib_pypy/test_collections.py --- a/pypy/module/test_lib_pypy/test_collections.py +++ b/pypy/module/test_lib_pypy/test_collections.py @@ -1,25 +1,53 @@ +from __future__ import absolute_import +from lib_pypy import _collections as collections +import py -""" -Extra tests for the pure Python PyPy _collections module -(not used in normal PyPy's) -""" +class TestDeque: + def test_remove_empty(self): + d = collections.deque([]) + py.test.raises(ValueError, d.remove, 1) -class AppTestCollections: - def test_copy(self): - import _collections - def f(): - return 42 - d = _collections.defaultdict(f, {2: 3}) - # - d1 = d.copy() - assert type(d1) is _collections.defaultdict - assert len(d1) == 1 - assert d1[2] == 3 - assert d1[3] == 42 - # + def test_remove_mutating(self): + class MutatingCmp(object): + def __eq__(self, other): + d.clear() + return True + + d = collections.deque([MutatingCmp()]) + py.test.raises(IndexError, d.remove, 1) + + def test_maxlen(self): + d = collections.deque([], 3) + d.append(1); d.append(2); d.append(3); d.append(4) + assert list(d) == [2, 3, 4] + assert repr(d) == "deque([2, 3, 4], maxlen=3)" + + import pickle + d2 = pickle.loads(pickle.dumps(d)) + assert repr(d2) == "deque([2, 3, 4], maxlen=3)" + import copy - d2 = copy.deepcopy(d) - assert type(d2) is _collections.defaultdict - assert len(d2) == 1 - assert d2[2] == 3 - assert d2[3] == 42 + d3 = copy.copy(d) + assert repr(d3) == "deque([2, 3, 4], maxlen=3)" + + def test_count(self): + d = collections.deque([1, 2, 2, 3, 2]) + assert d.count(2) == 3 + assert d.count(4) == 0 + + def test_reverse(self): + d = collections.deque([1, 2, 2, 3, 2]) + d.reverse() + assert list(d) == [2, 3, 2, 2, 1] + + d = collections.deque(range(100)) + d.reverse() + assert list(d) == range(99, -1, -1) + + def test_subclass_with_kwargs(self): + class SubclassWithKwargs(collections.deque): + def __init__(self, newarg=1): + collections.deque.__init__(self) + + # SF bug #1486663 -- this used to erroneously raise a TypeError + SubclassWithKwargs(newarg=1) diff --git a/pypy/module/test_lib_pypy/test_collections.py b/pypy/module/test_lib_pypy/test_collections_extra.py copy from pypy/module/test_lib_pypy/test_collections.py copy to pypy/module/test_lib_pypy/test_collections_extra.py diff --git a/lib_pypy/pypy_test/test_coroutine.py b/pypy/module/test_lib_pypy/test_coroutine.py rename from lib_pypy/pypy_test/test_coroutine.py rename to pypy/module/test_lib_pypy/test_coroutine.py diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime.py --- a/pypy/module/test_lib_pypy/test_datetime.py +++ b/pypy/module/test_lib_pypy/test_datetime.py @@ -1,52 +1,59 @@ -"""Additional tests for datetime.""" - +from __future__ import absolute_import import py -import time from lib_pypy import datetime -import copy -import os -def test_utcfromtimestamp(): - """Confirm that utcfromtimestamp and fromtimestamp give consistent results. +def test_repr(): + print datetime + expected = "datetime.datetime(1, 2, 3, 0, 0)" + assert repr(datetime.datetime(1,2,3)) == expected - Based on danchr's test script in https://bugs.pypy.org/issue986 - """ - try: - prev_tz = os.environ.get("TZ") - os.environ["TZ"] = "GMT" - for unused in xrange(100): - now = time.time() - delta = (datetime.datetime.utcfromtimestamp(now) - - datetime.datetime.fromtimestamp(now)) - assert delta.days * 86400 + delta.seconds == 0 - finally: - if prev_tz is None: - del os.environ["TZ"] - else: - os.environ["TZ"] = prev_tz +def test_strptime(): + import time, sys + if sys.version_info < (2, 6): + py.test.skip("needs the _strptime module") -def test_utcfromtimestamp_microsecond(): - dt = datetime.datetime.utcfromtimestamp(0) - assert isinstance(dt.microsecond, int) + string = '2004-12-01 13:02:47' + format = '%Y-%m-%d %H:%M:%S' + expected = datetime.datetime(*(time.strptime(string, format)[0:6])) + got = datetime.datetime.strptime(string, format) + assert expected == got +def test_datetime_rounding(): + b = 0.0000001 + a = 0.9999994 -def test_integer_args(): - with py.test.raises(TypeError): - datetime.datetime(10, 10, 10.) - with py.test.raises(TypeError): - datetime.datetime(10, 10, 10, 10, 10.) - with py.test.raises(TypeError): - datetime.datetime(10, 10, 10, 10, 10, 10.) + assert datetime.datetime.utcfromtimestamp(a).microsecond == 999999 + assert datetime.datetime.utcfromtimestamp(a).second == 0 + a += b + assert datetime.datetime.utcfromtimestamp(a).microsecond == 999999 + assert datetime.datetime.utcfromtimestamp(a).second == 0 + a += b + assert datetime.datetime.utcfromtimestamp(a).microsecond == 0 + assert datetime.datetime.utcfromtimestamp(a).second == 1 -def test_utcnow_microsecond(): - dt = datetime.datetime.utcnow() - assert type(dt.microsecond) is int - - copy.copy(dt) - -def test_radd(): - class X(object): - def __radd__(self, other): - return "radd" - assert datetime.date(10, 10, 10) + X() == "radd" +def test_more_datetime_rounding(): + # this test verified on top of CPython 2.7 (using a plain + # "import datetime" above) + expected_results = { + -1000.0: 'datetime.datetime(1970, 1, 1, 0, 43, 20)', + -999.9999996: 'datetime.datetime(1970, 1, 1, 0, 43, 20)', + -999.4: 'datetime.datetime(1970, 1, 1, 0, 43, 20, 600000)', + -999.0000004: 'datetime.datetime(1970, 1, 1, 0, 43, 21)', + -1.0: 'datetime.datetime(1970, 1, 1, 0, 59, 59)', + -0.9999996: 'datetime.datetime(1970, 1, 1, 0, 59, 59)', + -0.4: 'datetime.datetime(1970, 1, 1, 0, 59, 59, 600000)', + -0.0000004: 'datetime.datetime(1970, 1, 1, 1, 0)', + 0.0: 'datetime.datetime(1970, 1, 1, 1, 0)', + 0.0000004: 'datetime.datetime(1970, 1, 1, 1, 0)', + 0.4: 'datetime.datetime(1970, 1, 1, 1, 0, 0, 400000)', + 0.9999996: 'datetime.datetime(1970, 1, 1, 1, 0, 1)', + 1000.0: 'datetime.datetime(1970, 1, 1, 1, 16, 40)', + 1000.0000004: 'datetime.datetime(1970, 1, 1, 1, 16, 40)', + 1000.4: 'datetime.datetime(1970, 1, 1, 1, 16, 40, 400000)', + 1000.9999996: 'datetime.datetime(1970, 1, 1, 1, 16, 41)', + 1293843661.191: 'datetime.datetime(2011, 1, 1, 2, 1, 1, 191000)', + } + for t in sorted(expected_results): + dt = datetime.datetime.fromtimestamp(t) + assert repr(dt) == expected_results[t] diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime_extra.py copy from pypy/module/test_lib_pypy/test_datetime.py copy to pypy/module/test_lib_pypy/test_datetime_extra.py diff --git a/lib_pypy/pypy_test/test_dbm_extra.py b/pypy/module/test_lib_pypy/test_dbm_extra.py rename from lib_pypy/pypy_test/test_dbm_extra.py rename to pypy/module/test_lib_pypy/test_dbm_extra.py diff --git a/lib_pypy/pypy_test/test_defaultdict.py b/pypy/module/test_lib_pypy/test_defaultdict.py rename from lib_pypy/pypy_test/test_defaultdict.py rename to pypy/module/test_lib_pypy/test_defaultdict.py diff --git a/lib_pypy/pypy_test/test_deque_extra.py b/pypy/module/test_lib_pypy/test_deque_extra.py rename from lib_pypy/pypy_test/test_deque_extra.py rename to pypy/module/test_lib_pypy/test_deque_extra.py diff --git a/lib_pypy/pypy_test/test_exception_extra.py b/pypy/module/test_lib_pypy/test_exception_extra.py rename from lib_pypy/pypy_test/test_exception_extra.py rename to pypy/module/test_lib_pypy/test_exception_extra.py diff --git a/lib_pypy/pypy_test/test_grp_extra.py b/pypy/module/test_lib_pypy/test_grp_extra.py rename from lib_pypy/pypy_test/test_grp_extra.py rename to pypy/module/test_lib_pypy/test_grp_extra.py diff --git a/lib_pypy/pypy_test/test_marshal_extra.py b/pypy/module/test_lib_pypy/test_marshal_extra.py rename from lib_pypy/pypy_test/test_marshal_extra.py rename to pypy/module/test_lib_pypy/test_marshal_extra.py diff --git a/lib_pypy/pypy_test/test_md5_extra.py b/pypy/module/test_lib_pypy/test_md5_extra.py rename from lib_pypy/pypy_test/test_md5_extra.py rename to pypy/module/test_lib_pypy/test_md5_extra.py diff --git a/lib_pypy/pypy_test/test_os_wait.py b/pypy/module/test_lib_pypy/test_os_wait.py rename from lib_pypy/pypy_test/test_os_wait.py rename to pypy/module/test_lib_pypy/test_os_wait.py diff --git a/lib_pypy/pypy_test/test_pickle_extra.py b/pypy/module/test_lib_pypy/test_pickle_extra.py rename from lib_pypy/pypy_test/test_pickle_extra.py rename to pypy/module/test_lib_pypy/test_pickle_extra.py diff --git a/lib_pypy/pypy_test/test_resource.py b/pypy/module/test_lib_pypy/test_resource.py rename from lib_pypy/pypy_test/test_resource.py rename to pypy/module/test_lib_pypy/test_resource.py diff --git a/lib_pypy/pypy_test/test_sha_extra.py b/pypy/module/test_lib_pypy/test_sha_extra.py rename from lib_pypy/pypy_test/test_sha_extra.py rename to pypy/module/test_lib_pypy/test_sha_extra.py diff --git a/lib_pypy/pypy_test/test_site_extra.py b/pypy/module/test_lib_pypy/test_site_extra.py rename from lib_pypy/pypy_test/test_site_extra.py rename to pypy/module/test_lib_pypy/test_site_extra.py diff --git a/lib_pypy/pypy_test/test_stackless.py b/pypy/module/test_lib_pypy/test_stackless.py rename from lib_pypy/pypy_test/test_stackless.py rename to pypy/module/test_lib_pypy/test_stackless.py diff --git a/lib_pypy/pypy_test/test_stackless_pickling.py b/pypy/module/test_lib_pypy/test_stackless_pickling.py rename from lib_pypy/pypy_test/test_stackless_pickling.py rename to pypy/module/test_lib_pypy/test_stackless_pickling.py diff --git a/lib_pypy/pypy_test/test_structseq.py b/pypy/module/test_lib_pypy/test_structseq.py rename from lib_pypy/pypy_test/test_structseq.py rename to pypy/module/test_lib_pypy/test_structseq.py diff --git a/lib_pypy/pypy_test/test_syslog.py b/pypy/module/test_lib_pypy/test_syslog.py rename from lib_pypy/pypy_test/test_syslog.py rename to pypy/module/test_lib_pypy/test_syslog.py From noreply at buildbot.pypy.org Thu Jan 31 21:46:06 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 31 Jan 2013 21:46:06 +0100 (CET) Subject: [pypy-commit] pypy cleanup-tests: trash this file Message-ID: <20130131204606.5F2551C009B@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: cleanup-tests Changeset: r60790:5e9979fc7e67 Date: 2013-01-31 15:33 -0500 http://bitbucket.org/pypy/pypy/changeset/5e9979fc7e67/ Log: trash this file diff --git a/pypy/module/test_lib_pypy/hack___pypy__.py b/pypy/module/test_lib_pypy/hack___pypy__.py deleted file mode 100644 --- a/pypy/module/test_lib_pypy/hack___pypy__.py +++ /dev/null @@ -1,9 +0,0 @@ -# here only to make test runs work even if not running on top of PyPy -import sys, new - -def builtinify(f): - return f - -pypy = new.module('__pypy__') -pypy.builtinify = builtinify -sys.modules.setdefault('__pypy__', pypy) From noreply at buildbot.pypy.org Thu Jan 31 21:46:07 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 31 Jan 2013 21:46:07 +0100 (CET) Subject: [pypy-commit] pypy cleanup-tests: fix this bogus test in test_datetime Message-ID: <20130131204607.A5C011C009B@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: cleanup-tests Changeset: r60791:ab61429dffb6 Date: 2013-01-31 15:42 -0500 http://bitbucket.org/pypy/pypy/changeset/ab61429dffb6/ Log: fix this bogus test in test_datetime diff --git a/pypy/module/test_lib_pypy/test_datetime.py b/pypy/module/test_lib_pypy/test_datetime.py --- a/pypy/module/test_lib_pypy/test_datetime.py +++ b/pypy/module/test_lib_pypy/test_datetime.py @@ -36,24 +36,24 @@ # this test verified on top of CPython 2.7 (using a plain # "import datetime" above) expected_results = { - -1000.0: 'datetime.datetime(1970, 1, 1, 0, 43, 20)', - -999.9999996: 'datetime.datetime(1970, 1, 1, 0, 43, 20)', - -999.4: 'datetime.datetime(1970, 1, 1, 0, 43, 20, 600000)', - -999.0000004: 'datetime.datetime(1970, 1, 1, 0, 43, 21)', - -1.0: 'datetime.datetime(1970, 1, 1, 0, 59, 59)', - -0.9999996: 'datetime.datetime(1970, 1, 1, 0, 59, 59)', - -0.4: 'datetime.datetime(1970, 1, 1, 0, 59, 59, 600000)', - -0.0000004: 'datetime.datetime(1970, 1, 1, 1, 0)', - 0.0: 'datetime.datetime(1970, 1, 1, 1, 0)', - 0.0000004: 'datetime.datetime(1970, 1, 1, 1, 0)', - 0.4: 'datetime.datetime(1970, 1, 1, 1, 0, 0, 400000)', - 0.9999996: 'datetime.datetime(1970, 1, 1, 1, 0, 1)', - 1000.0: 'datetime.datetime(1970, 1, 1, 1, 16, 40)', - 1000.0000004: 'datetime.datetime(1970, 1, 1, 1, 16, 40)', - 1000.4: 'datetime.datetime(1970, 1, 1, 1, 16, 40, 400000)', - 1000.9999996: 'datetime.datetime(1970, 1, 1, 1, 16, 41)', - 1293843661.191: 'datetime.datetime(2011, 1, 1, 2, 1, 1, 191000)', + -1000.0: 'datetime.datetime(1969, 12, 31, 23, 43, 20)', + -999.9999996: 'datetime.datetime(1969, 12, 31, 23, 43, 20)', + -999.4: 'datetime.datetime(1969, 12, 31, 23, 43, 20, 600000)', + -999.0000004: 'datetime.datetime(1969, 12, 31, 23, 43, 21)', + -1.0: 'datetime.datetime(1969, 12, 31, 23, 59, 59)', + -0.9999996: 'datetime.datetime(1969, 12, 31, 23, 59, 59)', + -0.4: 'datetime.datetime(1969, 12, 31, 23, 59, 59, 600000)', + -0.0000004: 'datetime.datetime(1970, 1, 1, 0, 0)', + 0.0: 'datetime.datetime(1970, 1, 1, 0, 0)', + 0.0000004: 'datetime.datetime(1970, 1, 1, 0, 0)', + 0.4: 'datetime.datetime(1970, 1, 1, 0, 0, 0, 400000)', + 0.9999996: 'datetime.datetime(1970, 1, 1, 0, 0, 1)', + 1000.0: 'datetime.datetime(1970, 1, 1, 0, 16, 40)', + 1000.0000004: 'datetime.datetime(1970, 1, 1, 0, 16, 40)', + 1000.4: 'datetime.datetime(1970, 1, 1, 0, 16, 40, 400000)', + 1000.9999996: 'datetime.datetime(1970, 1, 1, 0, 16, 41)', + 1293843661.191: 'datetime.datetime(2011, 1, 1, 1, 1, 1, 191000)', } for t in sorted(expected_results): - dt = datetime.datetime.fromtimestamp(t) + dt = datetime.datetime.utcfromtimestamp(t) assert repr(dt) == expected_results[t] From noreply at buildbot.pypy.org Thu Jan 31 22:07:40 2013 From: noreply at buildbot.pypy.org (amauryfa) Date: Thu, 31 Jan 2013 22:07:40 +0100 (CET) Subject: [pypy-commit] pypy default: Issue912 resolved: backport py3k changes to zipfile.py, Message-ID: <20130131210740.5FA611C009B@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: Changeset: r60792:a50a5905f738 Date: 2013-01-31 22:07 +0100 http://bitbucket.org/pypy/pypy/changeset/a50a5905f738/ Log: Issue912 resolved: backport py3k changes to zipfile.py, and don't rely on the garbage collector to close file descriptors. diff --git a/lib-python/2.7/test/test_zipfile.py b/lib-python/2.7/test/test_zipfile.py --- a/lib-python/2.7/test/test_zipfile.py +++ b/lib-python/2.7/test/test_zipfile.py @@ -1213,6 +1213,17 @@ self.assertEqual(data1, '1'*FIXEDTEST_SIZE) self.assertEqual(data2, '2'*FIXEDTEST_SIZE) + def test_many_opens(self): + # Verify that read() and open() promptly close the file descriptor, + # and don't rely on the garbage collector to free resources. + with zipfile.ZipFile(TESTFN2, mode="r") as zipf: + for x in range(100): + zipf.read('ones') + with zipf.open('ones') as zopen1: + pass + for x in range(10): + self.assertLess(open('/dev/null').fileno(), 100) + def tearDown(self): unlink(TESTFN2) diff --git a/lib-python/2.7/zipfile.py b/lib-python/2.7/zipfile.py --- a/lib-python/2.7/zipfile.py +++ b/lib-python/2.7/zipfile.py @@ -475,9 +475,11 @@ # Search for universal newlines or line chunks. PATTERN = re.compile(r'^(?P[^\r\n]+)|(?P\n|\r\n?)') - def __init__(self, fileobj, mode, zipinfo, decrypter=None): + def __init__(self, fileobj, mode, zipinfo, decrypter=None, + close_fileobj=False): self._fileobj = fileobj self._decrypter = decrypter + self._close_fileobj = close_fileobj self._compress_type = zipinfo.compress_type self._compress_size = zipinfo.compress_size @@ -649,6 +651,12 @@ self._offset += len(data) return data + def close(self): + try: + if self._close_fileobj: + self._fileobj.close() + finally: + super(ZipExtFile, self).close() class ZipFile: @@ -866,7 +874,8 @@ def read(self, name, pwd=None): """Return file bytes (as a string) for name.""" - return self.open(name, "r", pwd).read() + with self.open(name, "r", pwd) as fp: + return fp.read() def open(self, name, mode="r", pwd=None): """Return file-like object for 'name'.""" @@ -889,8 +898,12 @@ zinfo = name else: # Get info object for name - zinfo = self.getinfo(name) - + try: + zinfo = self.getinfo(name) + except KeyError: + if not self._filePassed: + zef_file.close() + raise zef_file.seek(zinfo.header_offset, 0) # Skip the file header: @@ -904,6 +917,8 @@ zef_file.read(fheader[_FH_EXTRA_FIELD_LENGTH]) if fname != zinfo.orig_filename: + if not self._filePassed: + zef_file.close() raise BadZipfile, \ 'File name in directory "%s" and header "%s" differ.' % ( zinfo.orig_filename, fname) @@ -915,6 +930,8 @@ if not pwd: pwd = self.pwd if not pwd: + if not self._filePassed: + zef_file.close() raise RuntimeError, "File %s is encrypted, " \ "password required for extraction" % name @@ -933,9 +950,12 @@ # compare against the CRC otherwise check_byte = (zinfo.CRC >> 24) & 0xff if ord(h[11]) != check_byte: + if not self._filePassed: + zef_file.close() raise RuntimeError("Bad password for file", name) - return ZipExtFile(zef_file, mode, zinfo, zd) + return ZipExtFile(zef_file, mode, zinfo, zd, + close_fileobj=not self._filePassed) def extract(self, member, path=None, pwd=None): """Extract a member from the archive to the current working directory, @@ -993,7 +1013,7 @@ return targetpath source = self.open(member, pwd=pwd) - target = file(targetpath, "wb") + target = open(targetpath, "wb") shutil.copyfileobj(source, target) source.close() target.close() From noreply at buildbot.pypy.org Thu Jan 31 22:29:08 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 31 Jan 2013 22:29:08 +0100 (CET) Subject: [pypy-commit] pypy default: Don't include profiling.c by default. It should be included by Message-ID: <20130131212908.B55B61C009B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r60793:b366d0104c42 Date: 2013-01-31 22:25 +0100 http://bitbucket.org/pypy/pypy/changeset/b366d0104c42/ Log: Don't include profiling.c by default. It should be included by interp_lsprof already. It would end with the file listed twice in the Makefile... diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py --- a/rpython/translator/c/genc.py +++ b/rpython/translator/c/genc.py @@ -761,7 +761,6 @@ srcdir / 'exception.c', srcdir / 'rtyper.c', # ifdef HAVE_RTYPER srcdir / 'support.c', - srcdir / 'profiling.c', srcdir / 'debug_print.c', srcdir / 'debug_traceback.c', # ifdef HAVE_RTYPER srcdir / 'asm.c', From noreply at buildbot.pypy.org Thu Jan 31 22:29:10 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 31 Jan 2013 22:29:10 +0100 (CET) Subject: [pypy-commit] pypy default: Turn some variables static. Message-ID: <20130131212910.074461C009B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r60794:0a9fb5374698 Date: 2013-01-31 22:26 +0100 http://bitbucket.org/pypy/pypy/changeset/0a9fb5374698/ Log: Turn some variables static. diff --git a/rpython/translator/c/src/profiling.c b/rpython/translator/c/src/profiling.c --- a/rpython/translator/c/src/profiling.c +++ b/rpython/translator/c/src/profiling.c @@ -8,8 +8,8 @@ #include #endif -cpu_set_t base_cpu_set; -int profiling_setup = 0; +static cpu_set_t base_cpu_set; +static int profiling_setup = 0; void pypy_setup_profiling() { @@ -37,8 +37,8 @@ #include -DWORD_PTR base_affinity_mask; -int profiling_setup = 0; +static DWORD_PTR base_affinity_mask; +static int profiling_setup = 0; void pypy_setup_profiling() { if (!profiling_setup) { From noreply at buildbot.pypy.org Thu Jan 31 22:29:11 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 31 Jan 2013 22:29:11 +0100 (CET) Subject: [pypy-commit] pypy default: merge heads Message-ID: <20130131212911.5C7301C009B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r60795:620c44fd4b9f Date: 2013-01-31 22:28 +0100 http://bitbucket.org/pypy/pypy/changeset/620c44fd4b9f/ Log: merge heads diff --git a/lib-python/2.7/test/test_zipfile.py b/lib-python/2.7/test/test_zipfile.py --- a/lib-python/2.7/test/test_zipfile.py +++ b/lib-python/2.7/test/test_zipfile.py @@ -1213,6 +1213,17 @@ self.assertEqual(data1, '1'*FIXEDTEST_SIZE) self.assertEqual(data2, '2'*FIXEDTEST_SIZE) + def test_many_opens(self): + # Verify that read() and open() promptly close the file descriptor, + # and don't rely on the garbage collector to free resources. + with zipfile.ZipFile(TESTFN2, mode="r") as zipf: + for x in range(100): + zipf.read('ones') + with zipf.open('ones') as zopen1: + pass + for x in range(10): + self.assertLess(open('/dev/null').fileno(), 100) + def tearDown(self): unlink(TESTFN2) diff --git a/lib-python/2.7/zipfile.py b/lib-python/2.7/zipfile.py --- a/lib-python/2.7/zipfile.py +++ b/lib-python/2.7/zipfile.py @@ -475,9 +475,11 @@ # Search for universal newlines or line chunks. PATTERN = re.compile(r'^(?P[^\r\n]+)|(?P\n|\r\n?)') - def __init__(self, fileobj, mode, zipinfo, decrypter=None): + def __init__(self, fileobj, mode, zipinfo, decrypter=None, + close_fileobj=False): self._fileobj = fileobj self._decrypter = decrypter + self._close_fileobj = close_fileobj self._compress_type = zipinfo.compress_type self._compress_size = zipinfo.compress_size @@ -649,6 +651,12 @@ self._offset += len(data) return data + def close(self): + try: + if self._close_fileobj: + self._fileobj.close() + finally: + super(ZipExtFile, self).close() class ZipFile: @@ -866,7 +874,8 @@ def read(self, name, pwd=None): """Return file bytes (as a string) for name.""" - return self.open(name, "r", pwd).read() + with self.open(name, "r", pwd) as fp: + return fp.read() def open(self, name, mode="r", pwd=None): """Return file-like object for 'name'.""" @@ -889,8 +898,12 @@ zinfo = name else: # Get info object for name - zinfo = self.getinfo(name) - + try: + zinfo = self.getinfo(name) + except KeyError: + if not self._filePassed: + zef_file.close() + raise zef_file.seek(zinfo.header_offset, 0) # Skip the file header: @@ -904,6 +917,8 @@ zef_file.read(fheader[_FH_EXTRA_FIELD_LENGTH]) if fname != zinfo.orig_filename: + if not self._filePassed: + zef_file.close() raise BadZipfile, \ 'File name in directory "%s" and header "%s" differ.' % ( zinfo.orig_filename, fname) @@ -915,6 +930,8 @@ if not pwd: pwd = self.pwd if not pwd: + if not self._filePassed: + zef_file.close() raise RuntimeError, "File %s is encrypted, " \ "password required for extraction" % name @@ -933,9 +950,12 @@ # compare against the CRC otherwise check_byte = (zinfo.CRC >> 24) & 0xff if ord(h[11]) != check_byte: + if not self._filePassed: + zef_file.close() raise RuntimeError("Bad password for file", name) - return ZipExtFile(zef_file, mode, zinfo, zd) + return ZipExtFile(zef_file, mode, zinfo, zd, + close_fileobj=not self._filePassed) def extract(self, member, path=None, pwd=None): """Extract a member from the archive to the current working directory, @@ -993,7 +1013,7 @@ return targetpath source = self.open(member, pwd=pwd) - target = file(targetpath, "wb") + target = open(targetpath, "wb") shutil.copyfileobj(source, target) source.close() target.close() From noreply at buildbot.pypy.org Thu Jan 31 22:44:25 2013 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 31 Jan 2013 22:44:25 +0100 (CET) Subject: [pypy-commit] pypy ppc-jit-backend: Add a comment: looks wrong here Message-ID: <20130131214425.932271C009B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: ppc-jit-backend Changeset: r60796:0ad5a737469c Date: 2013-01-31 22:44 +0100 http://bitbucket.org/pypy/pypy/changeset/0ad5a737469c/ Log: Add a comment: looks wrong here diff --git a/pypy/translator/c/src/asm_ppc.h b/pypy/translator/c/src/asm_ppc.h --- a/pypy/translator/c/src/asm_ppc.h +++ b/pypy/translator/c/src/asm_ppc.h @@ -16,6 +16,7 @@ { long i; + fprintf(stderr, "please review c/src/asm_ppc.c: looks wrong\n"); for (i = 0; i < size; i += 32){ __dcbst(base, i); } From noreply at buildbot.pypy.org Thu Jan 31 23:02:23 2013 From: noreply at buildbot.pypy.org (bdkearns) Date: Thu, 31 Jan 2013 23:02:23 +0100 (CET) Subject: [pypy-commit] pypy cleanup-tests: remove outdated test Message-ID: <20130131220223.C010B1C009B@cobra.cs.uni-duesseldorf.de> Author: Brian Kearns Branch: cleanup-tests Changeset: r60797:6237eb0262c2 Date: 2013-01-31 15:55 -0500 http://bitbucket.org/pypy/pypy/changeset/6237eb0262c2/ Log: remove outdated test diff --git a/pypy/module/test_lib_pypy/inprogress_test_binascii_extra.py b/pypy/module/test_lib_pypy/inprogress_test_binascii_extra.py deleted file mode 100644 --- a/pypy/module/test_lib_pypy/inprogress_test_binascii_extra.py +++ /dev/null @@ -1,19 +0,0 @@ -from __future__ import absolute_import -from lib_pypy import binascii - -def test_uu(): - assert binascii.b2a_uu('1234567') == "',3(S-#4V-P \n" - assert binascii.b2a_uu('123456789012345678901234567890123456789012345') == 'M,3(S-#4V-S at Y,#$R,S0U-C Author: Brian Kearns Branch: cleanup-tests Changeset: r60798:5acfa3319a45 Date: 2013-01-31 16:53 -0500 http://bitbucket.org/pypy/pypy/changeset/5acfa3319a45/ Log: fix collection of test_lib_pypy tests when runappdirect is set diff --git a/pypy/conftest.py b/pypy/conftest.py --- a/pypy/conftest.py +++ b/pypy/conftest.py @@ -84,10 +84,11 @@ if self.config.option.runappdirect: # only collect regular tests if we are in an 'app_test' directory, # or in test_lib_pypy - names = self.listnames() - return "app_test" in names or "test_lib_pypy" in names - else: - return True + for name in self.listnames(): + if "app_test" in name or "test_lib_pypy" in name: + return True + return False + return True def funcnamefilter(self, name): if name.startswith('test_'):